ftrace.c 159.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
13
 *  Copyright (C) 2004 Nadia Yvette Chambers
14 15
 */

16 17
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
18
#include <linux/sched/task.h>
19
#include <linux/kallsyms.h>
20
#include <linux/seq_file.h>
21
#include <linux/suspend.h>
22
#include <linux/tracefs.h>
23
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
24
#include <linux/kthread.h>
25
#include <linux/uaccess.h>
26
#include <linux/bsearch.h>
27
#include <linux/module.h>
I
Ingo Molnar 已提交
28
#include <linux/ftrace.h>
29
#include <linux/sysctl.h>
30
#include <linux/slab.h>
31
#include <linux/ctype.h>
32
#include <linux/sort.h>
33
#include <linux/list.h>
34
#include <linux/hash.h>
35
#include <linux/rcupdate.h>
36

37
#include <trace/events/sched.h>
38

39
#include <asm/sections.h>
40
#include <asm/setup.h>
41

42
#include "trace_output.h"
S
Steven Rostedt 已提交
43
#include "trace_stat.h"
44

45
#define FTRACE_WARN_ON(cond)			\
46 47 48
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
49
			ftrace_kill();		\
50 51
		___r;				\
	})
52 53

#define FTRACE_WARN_ON_ONCE(cond)		\
54 55 56
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
57
			ftrace_kill();		\
58 59
		___r;				\
	})
60

61 62 63
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
64 65
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
66

67
#ifdef CONFIG_DYNAMIC_FTRACE
68 69 70
#define INIT_OPS_HASH(opsname)	\
	.func_hash		= &opsname.local_hash,			\
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
71 72 73
#define ASSIGN_OPS_HASH(opsname, val) \
	.func_hash		= val, \
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
74
#else
75
#define INIT_OPS_HASH(opsname)
76
#define ASSIGN_OPS_HASH(opsname, val)
77 78
#endif

79 80
static struct ftrace_ops ftrace_list_end __read_mostly = {
	.func		= ftrace_stub,
81
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
82
	INIT_OPS_HASH(ftrace_list_end)
83 84
};

85 86
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
87
static int last_ftrace_enabled;
88

89 90
/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
91 92
/* What to set function_trace_op to */
static struct ftrace_ops *set_function_trace_op;
93

94
static bool ftrace_pids_enabled(struct ftrace_ops *ops)
95
{
96 97 98 99 100 101 102 103
	struct trace_array *tr;

	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
		return false;

	tr = ops->private;

	return tr->function_pids != NULL;
104 105 106 107
}

static void ftrace_update_trampoline(struct ftrace_ops *ops);

108 109 110 111 112 113
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

114
static DEFINE_MUTEX(ftrace_lock);
115

116
static struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
117
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
118
static struct ftrace_ops global_ops;
119

120 121
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
122
				 struct ftrace_ops *op, struct pt_regs *regs);
123 124 125 126 127
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
128

129 130
/*
 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
131
 * can use rcu_dereference_raw_notrace() is that elements removed from this list
132
 * are simply leaked, so there is no need to interact with a grace-period
133
 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
134 135 136 137 138
 * concurrent insertions into the ftrace_global_list.
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
#define do_for_each_ftrace_op(op, list)			\
139
	op = rcu_dereference_raw_notrace(list);			\
140 141 142 143 144 145
	do

/*
 * Optimized for just a single item in the list (as that is the normal case).
 */
#define while_for_each_ftrace_op(op)				\
146
	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
147 148
	       unlikely((op) != &ftrace_list_end))

149 150 151 152
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
153 154
		mutex_init(&ops->local_hash.regex_lock);
		ops->func_hash = &ops->local_hash;
155 156 157 158 159
		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
	}
#endif
}

160 161 162 163 164 165 166 167 168 169 170 171
/**
 * ftrace_nr_registered_ops - return number of ops registered
 *
 * Returns the number of ftrace_ops registered and tracing functions
 */
int ftrace_nr_registered_ops(void)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	mutex_lock(&ftrace_lock);

172 173 174 175 176
	for (ops = rcu_dereference_protected(ftrace_ops_list,
					     lockdep_is_held(&ftrace_lock));
	     ops != &ftrace_list_end;
	     ops = rcu_dereference_protected(ops->next,
					     lockdep_is_held(&ftrace_lock)))
177 178 179 180 181 182 183
		cnt++;

	mutex_unlock(&ftrace_lock);

	return cnt;
}

184
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
185
			    struct ftrace_ops *op, struct pt_regs *regs)
186
{
187 188 189
	struct trace_array *tr = op->private;

	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
190 191
		return;

192
	op->saved_func(ip, parent_ip, op, regs);
193 194
}

195
/**
196
 * clear_ftrace_function - reset the ftrace function
197
 *
198 199
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
200
 */
201
void clear_ftrace_function(void)
202
{
203 204 205
	ftrace_trace_function = ftrace_stub;
}

206
static void per_cpu_ops_disable_all(struct ftrace_ops *ops)
207 208 209 210 211 212 213
{
	int cpu;

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(ops->disabled, cpu) = 1;
}

214
static int per_cpu_ops_alloc(struct ftrace_ops *ops)
215 216 217
{
	int __percpu *disabled;

218 219 220
	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU)))
		return -EINVAL;

221 222 223 224 225
	disabled = alloc_percpu(int);
	if (!disabled)
		return -ENOMEM;

	ops->disabled = disabled;
226
	per_cpu_ops_disable_all(ops);
227 228 229
	return 0;
}

230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
static void ftrace_sync(struct work_struct *work)
{
	/*
	 * This function is just a stub to implement a hard force
	 * of synchronize_sched(). This requires synchronizing
	 * tasks even in userspace and idle.
	 *
	 * Yes, function tracing is rude.
	 */
}

static void ftrace_sync_ipi(void *data)
{
	/* Probably not needed, but do it anyway */
	smp_rmb();
}

247 248
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static void update_function_graph_func(void);
249 250 251 252 253

/* Both enabled by default (can be cleared by function_graph tracer flags */
static bool fgraph_sleep_time = true;
static bool fgraph_graph_time = true;

254 255 256 257
#else
static inline void update_function_graph_func(void) { }
#endif

258 259 260 261

static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
	/*
262
	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
263 264
	 * then it needs to call the list anyway.
	 */
265 266
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU |
			  FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC)
267 268 269 270 271
		return ftrace_ops_list_func;

	return ftrace_ops_get_func(ops);
}

272 273 274 275
static void update_ftrace_function(void)
{
	ftrace_func_t func;

276 277 278 279 280
	/*
	 * Prepare the ftrace_ops that the arch callback will use.
	 * If there's only one ftrace_ops registered, the ftrace_ops_list
	 * will point to the ops we want.
	 */
281 282
	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
						lockdep_is_held(&ftrace_lock));
283 284

	/* If there's no ftrace_ops registered, just call the stub function */
285
	if (set_function_trace_op == &ftrace_list_end) {
286 287
		func = ftrace_stub;

288 289
	/*
	 * If we are at the end of the list and this ops is
290 291
	 * recursion safe and not dynamic and the arch supports passing ops,
	 * then have the mcount trampoline call the function directly.
292
	 */
293 294
	} else if (rcu_dereference_protected(ftrace_ops_list->next,
			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
295
		func = ftrace_ops_get_list_func(ftrace_ops_list);
296

297 298
	} else {
		/* Just use the default ftrace_ops */
299
		set_function_trace_op = &ftrace_list_end;
300
		func = ftrace_ops_list_func;
301
	}
302

303 304
	update_function_graph_func();

305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
	/* If there's no change, then do nothing more here */
	if (ftrace_trace_function == func)
		return;

	/*
	 * If we are using the list function, it doesn't care
	 * about the function_trace_ops.
	 */
	if (func == ftrace_ops_list_func) {
		ftrace_trace_function = func;
		/*
		 * Don't even bother setting function_trace_ops,
		 * it would be racy to do so anyway.
		 */
		return;
	}

#ifndef CONFIG_DYNAMIC_FTRACE
	/*
	 * For static tracing, we need to be a bit more careful.
	 * The function change takes affect immediately. Thus,
	 * we need to coorditate the setting of the function_trace_ops
	 * with the setting of the ftrace_trace_function.
	 *
	 * Set the function to the list ops, which will call the
	 * function we want, albeit indirectly, but it handles the
	 * ftrace_ops and doesn't depend on function_trace_op.
	 */
	ftrace_trace_function = ftrace_ops_list_func;
	/*
	 * Make sure all CPUs see this. Yes this is slow, but static
	 * tracing is slow and nasty to have enabled.
	 */
	schedule_on_each_cpu(ftrace_sync);
	/* Now all cpus are using the list ops. */
	function_trace_op = set_function_trace_op;
	/* Make sure the function_trace_op is visible on all CPUs */
	smp_wmb();
	/* Nasty way to force a rmb on all cpus */
	smp_call_function(ftrace_sync_ipi, NULL, 1);
	/* OK, we are all set to update the ftrace_trace_function now! */
#endif /* !CONFIG_DYNAMIC_FTRACE */

348 349 350
	ftrace_trace_function = func;
}

351 352 353 354 355
int using_ftrace_ops_list_func(void)
{
	return ftrace_trace_function == ftrace_ops_list_func;
}

356 357
static void add_ftrace_ops(struct ftrace_ops __rcu **list,
			   struct ftrace_ops *ops)
358
{
359 360
	rcu_assign_pointer(ops->next, *list);

361
	/*
362
	 * We are entering ops into the list but another
363 364
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
365
	 * the ops pointer included into the list.
366
	 */
367
	rcu_assign_pointer(*list, ops);
368 369
}

370 371
static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
			     struct ftrace_ops *ops)
372 373 374 375
{
	struct ftrace_ops **p;

	/*
376 377
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
378
	 */
379 380 381 382
	if (rcu_dereference_protected(*list,
			lockdep_is_held(&ftrace_lock)) == ops &&
	    rcu_dereference_protected(ops->next,
			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
383
		*list = &ftrace_list_end;
S
Steven Rostedt 已提交
384
		return 0;
385 386
	}

387
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
388 389 390
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
391 392
	if (*p != ops)
		return -1;
393 394

	*p = (*p)->next;
395 396
	return 0;
}
397

398 399
static void ftrace_update_trampoline(struct ftrace_ops *ops);

400 401
static int __register_ftrace_function(struct ftrace_ops *ops)
{
402 403 404
	if (ops->flags & FTRACE_OPS_FL_DELETED)
		return -EINVAL;

405 406 407
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

408
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
409 410 411 412 413 414 415 416 417 418 419 420 421
	/*
	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
	 */
	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
		return -EINVAL;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif

422 423 424
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

425 426
	if (ops->flags & FTRACE_OPS_FL_PER_CPU) {
		if (per_cpu_ops_alloc(ops))
427
			return -ENOMEM;
428 429 430
	}

	add_ftrace_ops(&ftrace_ops_list, ops);
431

432 433 434
	/* Always save the function, and reset at unregistering */
	ops->saved_func = ops->func;

435
	if (ftrace_pids_enabled(ops))
436 437
		ops->func = ftrace_pid_func;

438 439
	ftrace_update_trampoline(ops);

440 441 442 443 444 445 446 447 448 449
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

450 451 452
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

453
	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
454

455 456
	if (ret < 0)
		return ret;
457

458 459
	if (ftrace_enabled)
		update_ftrace_function();
460

461 462
	ops->func = ops->saved_func;

S
Steven Rostedt 已提交
463
	return 0;
464 465
}

466 467
static void ftrace_update_pid_func(void)
{
468 469
	struct ftrace_ops *op;

470
	/* Only do something if we are tracing something */
471
	if (ftrace_trace_function == ftrace_stub)
472
		return;
473

474 475
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->flags & FTRACE_OPS_FL_PID) {
476 477
			op->func = ftrace_pids_enabled(op) ?
				ftrace_pid_func : op->saved_func;
478 479 480 481
			ftrace_update_trampoline(op);
		}
	} while_for_each_ftrace_op(op);

482
	update_ftrace_function();
483 484
}

485 486 487 488 489
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
490 491
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
492
	unsigned long long		time_squared;
493
#endif
494 495
};

496 497 498 499
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
500 501
};

502 503 504 505 506 507 508 509
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

510 511
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
512

513 514
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
515

516 517 518
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
519 520
static DEFINE_MUTEX(ftrace_profile_lock);

521
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
522

523 524
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
525

S
Steven Rostedt 已提交
526 527 528
static void *
function_stat_next(void *v, int idx)
{
529 530
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
531

532
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
533 534

 again:
L
Li Zefan 已提交
535 536 537
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
538 539 540 541 542
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
543 544
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
545 546 547 548 549 550 551
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
552 553 554 555 556 557 558
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
559 560
}

561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
577 578
static int function_stat_cmp(void *p1, void *p2)
{
579 580
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
581 582 583 584 585 586 587 588

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
589
#endif
S
Steven Rostedt 已提交
590 591 592

static int function_stat_headers(struct seq_file *m)
{
593
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
594 595 596 597
	seq_puts(m, "  Function                               "
		 "Hit    Time            Avg             s^2\n"
		    "  --------                               "
		 "---    ----            ---             ---\n");
598
#else
599 600
	seq_puts(m, "  Function                               Hit\n"
		    "  --------                               ---\n");
601
#endif
S
Steven Rostedt 已提交
602 603 604 605 606
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
607
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
608
	char str[KSYM_SYMBOL_LEN];
609
	int ret = 0;
610
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
611 612
	static struct trace_seq s;
	unsigned long long avg;
613
	unsigned long long stddev;
614
#endif
615 616 617 618 619 620 621
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
S
Steven Rostedt 已提交
622

623 624 625 626 627 628 629
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	avg = rec->time;
	do_div(avg, rec->counter);
	if (tracing_thresh && (avg < tracing_thresh))
		goto out;
#endif

S
Steven Rostedt 已提交
630
	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
631 632 633
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
634
	seq_puts(m, "    ");
635

636 637 638 639
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
640 641 642 643 644 645 646
		/*
		 * Apply Welford's method:
		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
		 */
		stddev = rec->counter * rec->time_squared -
			 rec->time * rec->time;

647 648 649 650
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
651
		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
652 653
	}

654 655 656 657
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
658 659
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
660 661 662
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
663 664
out:
	mutex_unlock(&ftrace_profile_lock);
S
Steven Rostedt 已提交
665

666
	return ret;
S
Steven Rostedt 已提交
667 668
}

669
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
670
{
671
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
672

673
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
674

675 676 677 678
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
679 680
	}

681
	memset(stat->hash, 0,
682 683
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
684

685
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
686 687
{
	struct ftrace_profile_page *pg;
688 689
	int functions;
	int pages;
690
	int i;
S
Steven Rostedt 已提交
691

692
	/* If we already allocated, do nothing */
693
	if (stat->pages)
694
		return 0;
S
Steven Rostedt 已提交
695

696 697
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
698
		return -ENOMEM;
S
Steven Rostedt 已提交
699

700 701 702 703 704 705 706 707 708 709 710 711 712
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

713
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
714

715 716
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

717
	for (i = 1; i < pages; i++) {
718 719
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
720
			goto out_free;
721 722 723 724
		pg = pg->next;
	}

	return 0;
725 726 727 728 729 730 731 732 733 734 735 736 737 738

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
739 740
}

741
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
742
{
743
	struct ftrace_profile_stat *stat;
744
	int size;
S
Steven Rostedt 已提交
745

746 747 748
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
749
		/* If the profile is already created, simply reset it */
750
		ftrace_profile_reset(stat);
751 752
		return 0;
	}
S
Steven Rostedt 已提交
753

754 755 756 757 758
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
759

760
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
761

762
	if (!stat->hash)
763 764
		return -ENOMEM;

765
	/* Preallocate the function profiling pages */
766 767 768
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
769 770 771 772
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
773 774
}

775 776 777 778 779
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

780
	for_each_possible_cpu(cpu) {
781 782 783 784 785 786 787 788
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

789
/* interrupts must be disabled */
790 791
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
792
{
793
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
794 795 796
	struct hlist_head *hhd;
	unsigned long key;

797
	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
798
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
799 800 801 802

	if (hlist_empty(hhd))
		return NULL;

803
	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
S
Steven Rostedt 已提交
804
		if (rec->ip == ip)
805 806 807 808 809 810
			return rec;
	}

	return NULL;
}

811 812
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
813 814 815
{
	unsigned long key;

816
	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
817
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
818 819
}

820 821 822
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
823
static struct ftrace_profile *
824
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
825 826 827
{
	struct ftrace_profile *rec = NULL;

828
	/* prevent recursion (from NMIs) */
829
	if (atomic_inc_return(&stat->disabled) != 1)
830 831 832
		goto out;

	/*
833 834
	 * Try to find the function again since an NMI
	 * could have added it
835
	 */
836
	rec = ftrace_find_profiled_func(stat, ip);
837
	if (rec)
838
		goto out;
839

840 841 842 843
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
844
	}
845

846
	rec = &stat->pages->records[stat->pages->index++];
847
	rec->ip = ip;
848
	ftrace_add_profile(stat, rec);
849

S
Steven Rostedt 已提交
850
 out:
851
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
852 853 854 855 856

	return rec;
}

static void
857
function_profile_call(unsigned long ip, unsigned long parent_ip,
858
		      struct ftrace_ops *ops, struct pt_regs *regs)
S
Steven Rostedt 已提交
859
{
860
	struct ftrace_profile_stat *stat;
861
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
862 863 864 865 866 867
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
868

869
	stat = this_cpu_ptr(&ftrace_profile_stats);
870
	if (!stat->hash || !ftrace_profile_enabled)
871 872 873
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
874
	if (!rec) {
875
		rec = ftrace_profile_alloc(stat, ip);
876 877 878
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
879 880 881 882 883 884

	rec->counter++;
 out:
	local_irq_restore(flags);
}

885 886 887
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
888 889
	int index = trace->depth;

890
	function_profile_call(trace->func, 0, NULL, NULL);
891

892 893 894 895
	/* If function graph is shutting down, ret_stack can be NULL */
	if (!current->ret_stack)
		return 0;

896 897 898
	if (index >= 0 && index < FTRACE_RETFUNC_DEPTH)
		current->ret_stack[index].subtime = 0;

899 900 901 902 903
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
904
	struct ftrace_profile_stat *stat;
905
	unsigned long long calltime;
906
	struct ftrace_profile *rec;
907
	unsigned long flags;
908 909

	local_irq_save(flags);
910
	stat = this_cpu_ptr(&ftrace_profile_stats);
911
	if (!stat->hash || !ftrace_profile_enabled)
912 913
		goto out;

914 915 916 917
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

918 919
	calltime = trace->rettime - trace->calltime;

920
	if (!fgraph_graph_time) {
921 922 923 924 925 926 927 928 929 930 931 932 933 934
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

935
	rec = ftrace_find_profiled_func(stat, trace->func);
936
	if (rec) {
937
		rec->time += calltime;
938 939
		rec->time_squared += calltime * calltime;
	}
940

941
 out:
942 943 944 945 946 947 948 949 950 951 952 953 954 955
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
956
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
957
	.func		= function_profile_call,
958
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
959
	INIT_OPS_HASH(ftrace_profile_ops)
S
Steven Rostedt 已提交
960 961
};

962 963 964 965 966 967 968 969 970 971 972
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
973 974 975 976 977 978 979
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

980 981
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
S
Steven Rostedt 已提交
982 983 984 985 986 987 988
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
989 990 991 992 993 994
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

995 996 997 998 999
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
1000 1001 1002
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
1003 1004 1005 1006
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
1007
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
1008 1009
		}
	}
1010
 out:
S
Steven Rostedt 已提交
1011 1012
	mutex_unlock(&ftrace_profile_lock);

1013
	*ppos += cnt;
S
Steven Rostedt 已提交
1014 1015 1016 1017

	return cnt;
}

1018 1019 1020 1021
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
1022
	char buf[64];		/* big enough to hold a number */
1023 1024 1025 1026 1027 1028
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
1029 1030 1031 1032
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
1033
	.llseek		= default_llseek,
S
Steven Rostedt 已提交
1034 1035
};

1036 1037
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
1038 1039 1040 1041 1042 1043
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
1044 1045
};

1046
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1047
{
1048
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
1049
	struct dentry *entry;
1050
	char *name;
S
Steven Rostedt 已提交
1051
	int ret;
1052 1053 1054 1055 1056
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

1057
		name = kasprintf(GFP_KERNEL, "function%d", cpu);
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
1078 1079
	}

1080
	entry = tracefs_create_file("function_profile_enabled", 0644,
S
Steven Rostedt 已提交
1081 1082
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
1083
		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
S
Steven Rostedt 已提交
1084 1085 1086
}

#else /* CONFIG_FUNCTION_PROFILER */
1087
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1088 1089 1090 1091
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

1092 1093
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

1094 1095 1096 1097 1098 1099
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int ftrace_graph_active;
#else
# define ftrace_graph_active 0
#endif

1100 1101
#ifdef CONFIG_DYNAMIC_FTRACE

1102 1103
static struct ftrace_ops *removed_ops;

1104 1105 1106 1107 1108 1109
/*
 * Set when doing a global update, like enabling all recs or disabling them.
 * It is not set when just updating a single ftrace_ops.
 */
static bool update_all_ops;

1110 1111 1112 1113
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

1114 1115 1116 1117 1118
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

1119 1120 1121 1122 1123
struct ftrace_func_probe {
	struct ftrace_probe_ops	*probe_ops;
	struct ftrace_ops	ops;
	struct trace_array	*tr;
	struct list_head	list;
1124
	void			*data;
1125 1126 1127
	int			ref;
};

1128 1129 1130 1131 1132 1133 1134 1135 1136
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
1137
};
1138
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1139

1140
static struct ftrace_ops global_ops = {
1141 1142 1143 1144 1145
	.func				= ftrace_stub,
	.local_hash.notrace_hash	= EMPTY_HASH,
	.local_hash.filter_hash		= EMPTY_HASH,
	INIT_OPS_HASH(global_ops)
	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1146 1147
					  FTRACE_OPS_FL_INITIALIZED |
					  FTRACE_OPS_FL_PID,
1148 1149
};

1150 1151
/*
 * This is used by __kernel_text_address() to return true if the
1152
 * address is on a dynamically allocated trampoline that would
1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186
 * not return true for either core_kernel_text() or
 * is_module_text_address().
 */
bool is_ftrace_trampoline(unsigned long addr)
{
	struct ftrace_ops *op;
	bool ret = false;

	/*
	 * Some of the ops may be dynamically allocated,
	 * they are freed after a synchronize_sched().
	 */
	preempt_disable_notrace();

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/*
		 * This is to check for dynamically allocated trampolines.
		 * Trampolines that are in kernel text will have
		 * core_kernel_text() return true.
		 */
		if (op->trampoline && op->trampoline_size)
			if (addr >= op->trampoline &&
			    addr < op->trampoline + op->trampoline_size) {
				ret = true;
				goto out;
			}
	} while_for_each_ftrace_op(op);

 out:
	preempt_enable_notrace();

	return ret;
}

1187 1188
struct ftrace_page {
	struct ftrace_page	*next;
1189
	struct dyn_ftrace	*records;
1190
	int			index;
1191
	int			size;
1192 1193
};

1194 1195
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1196 1197 1198 1199 1200 1201 1202

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

1203 1204 1205 1206 1207 1208 1209 1210 1211
static __always_inline unsigned long
ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
{
	if (hash->size_bits > 0)
		return hash_long(ip, hash->size_bits);

	return 0;
}

1212 1213 1214
/* Only use this function if ftrace_hash_empty() has already been tested */
static __always_inline struct ftrace_func_entry *
__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1215 1216 1217 1218 1219
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;

1220
	key = ftrace_hash_key(hash, ip);
1221 1222
	hhd = &hash->buckets[key];

1223
	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1224 1225 1226 1227 1228 1229
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
/**
 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
 * @hash: The hash to look at
 * @ip: The instruction pointer to test
 *
 * Search a given @hash to see if a given instruction pointer (@ip)
 * exists in it.
 *
 * Returns the entry that holds the @ip if found. NULL otherwise.
 */
struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	if (ftrace_hash_empty(hash))
		return NULL;

	return __ftrace_lookup_ip(hash, ip);
}

1249 1250
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1251 1252 1253 1254
{
	struct hlist_head *hhd;
	unsigned long key;

1255
	key = ftrace_hash_key(hash, entry->ip);
1256 1257 1258
	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1271 1272 1273 1274 1275

	return 0;
}

static void
1276
free_hash_entry(struct ftrace_hash *hash,
1277 1278 1279 1280 1281 1282 1283
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1284 1285 1286 1287
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
1288
	hlist_del_rcu(&entry->hlist);
1289 1290 1291
	hash->count--;
}

1292 1293 1294
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
1295
	struct hlist_node *tn;
1296 1297 1298 1299
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1300 1301 1302
	if (!hash->count)
		return;

1303 1304
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
1305
		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1306
			free_hash_entry(hash, entry);
1307 1308 1309 1310
	}
	FTRACE_WARN_ON(hash->count);
}

1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
{
	list_del(&ftrace_mod->list);
	kfree(ftrace_mod->module);
	kfree(ftrace_mod->func);
	kfree(ftrace_mod);
}

static void clear_ftrace_mod_list(struct list_head *head)
{
	struct ftrace_mod_load *p, *n;

	/* stack tracer isn't supported yet */
	if (!head)
		return;

	mutex_lock(&ftrace_lock);
	list_for_each_entry_safe(p, n, head, list)
		free_ftrace_mod(p);
	mutex_unlock(&ftrace_lock);
}

1333 1334 1335 1336 1337 1338 1339 1340 1341
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}

1357 1358
void ftrace_free_filter(struct ftrace_ops *ops)
{
1359
	ftrace_ops_init(ops);
1360 1361
	free_ftrace_hash(ops->func_hash->filter_hash);
	free_ftrace_hash(ops->func_hash->notrace_hash);
1362 1363
}

1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
1374
	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414

static int ftrace_add_mod(struct trace_array *tr,
			  const char *func, const char *module,
			  int enable)
{
	struct ftrace_mod_load *ftrace_mod;
	struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;

	ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
	if (!ftrace_mod)
		return -ENOMEM;

	ftrace_mod->func = kstrdup(func, GFP_KERNEL);
	ftrace_mod->module = kstrdup(module, GFP_KERNEL);
	ftrace_mod->enable = enable;

	if (!ftrace_mod->func || !ftrace_mod->module)
		goto out_free;

	list_add(&ftrace_mod->list, mod_head);

	return 0;

 out_free:
	free_ftrace_mod(ftrace_mod);

	return -ENOMEM;
}

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

1428 1429 1430
	if (hash)
		new_hash->flags = hash->flags;

1431
	/* Empty hash? */
1432
	if (ftrace_hash_empty(hash))
1433 1434 1435 1436
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
1437
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

1453
static void
1454
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1455
static void
1456
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1457

1458 1459 1460
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash);

1461 1462
static struct ftrace_hash *
__ftrace_hash_move(struct ftrace_hash *src)
1463 1464
{
	struct ftrace_func_entry *entry;
1465
	struct hlist_node *tn;
1466
	struct hlist_head *hhd;
1467
	struct ftrace_hash *new_hash;
1468 1469 1470 1471 1472
	int size = src->count;
	int bits = 0;
	int i;

	/*
1473
	 * If the new source is empty, just return the empty_hash.
1474
	 */
1475
	if (ftrace_hash_empty(src))
1476
		return EMPTY_HASH;
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1488 1489
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
1490
		return NULL;
1491

1492 1493
	new_hash->flags = src->flags;

1494 1495 1496
	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
1497
		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1498
			remove_hash_entry(src, entry);
1499
			__add_hash_entry(new_hash, entry);
1500 1501 1502
		}
	}

1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
	return new_hash;
}

static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
		 struct ftrace_hash **dst, struct ftrace_hash *src)
{
	struct ftrace_hash *new_hash;
	int ret;

	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
		return -EINVAL;

	new_hash = __ftrace_hash_move(src);
	if (!new_hash)
		return -ENOMEM;

1521 1522 1523 1524 1525 1526 1527 1528 1529 1530
	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
	if (enable) {
		/* IPMODIFY should be updated only when filter_hash updating */
		ret = ftrace_hash_ipmodify_update(ops, new_hash);
		if (ret < 0) {
			free_ftrace_hash(new_hash);
			return ret;
		}
	}

1531 1532 1533 1534
	/*
	 * Remove the current set, update the hash and add
	 * them back.
	 */
1535
	ftrace_hash_rec_disable_modify(ops, enable);
1536

1537 1538
	rcu_assign_pointer(*dst, new_hash);

1539
	ftrace_hash_rec_enable_modify(ops, enable);
1540

1541
	return 0;
1542 1543
}

1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
static bool hash_contains_ip(unsigned long ip,
			     struct ftrace_ops_hash *hash)
{
	/*
	 * The function record is a match if it exists in the filter
	 * hash and not in the notrace hash. Note, an emty hash is
	 * considered a match for the filter hash, but an empty
	 * notrace hash is considered not in the notrace hash.
	 */
	return (ftrace_hash_empty(hash->filter_hash) ||
1554
		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1555
		(ftrace_hash_empty(hash->notrace_hash) ||
1556
		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1557 1558
}

1559 1560 1561 1562 1563 1564 1565 1566
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1567 1568 1569
 *
 * This needs to be called with preemption disabled as
 * the hashes are freed with call_rcu_sched().
1570 1571
 */
static int
1572
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1573
{
1574
	struct ftrace_ops_hash hash;
1575 1576
	int ret;

1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	/*
	 * There's a small race when adding ops that the ftrace handler
	 * that wants regs, may be called without them. We can not
	 * allow that handler to be called if regs is NULL.
	 */
	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
		return 0;
#endif

1587 1588
	rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
	rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
1589

1590
	if (hash_contains_ip(ip, &hash))
1591 1592 1593 1594 1595 1596 1597
		ret = 1;
	else
		ret = 0;

	return ret;
}

1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1612 1613 1614

static int ftrace_cmp_recs(const void *a, const void *b)
{
1615 1616
	const struct dyn_ftrace *key = a;
	const struct dyn_ftrace *rec = b;
1617

1618
	if (key->flags < rec->ip)
1619
		return -1;
1620 1621
	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
		return 1;
1622 1623 1624
	return 0;
}

1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
/**
 * ftrace_location_range - return the first address of a traced location
 *	if it touches the given ip range
 * @start: start of range to search.
 * @end: end of range to search (inclusive). @end points to the last byte
 *	to check.
 *
 * Returns rec->ip if the related ftrace location is a least partly within
 * the given address range. That is, the first address of the instruction
 * that is either a NOP or call to the function tracer. It checks the ftrace
 * internal tables to determine if the address belongs or not.
 */
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1638 1639 1640
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1641
	struct dyn_ftrace key;
1642

1643 1644
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */
1645 1646

	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1647 1648
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1649
			continue;
1650 1651 1652 1653
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (rec)
1654
			return rec->ip;
1655
	}
1656 1657 1658 1659

	return 0;
}

1660 1661 1662 1663
/**
 * ftrace_location - return true if the ip giving is a traced location
 * @ip: the instruction pointer to check
 *
1664
 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1665 1666 1667 1668
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1669
unsigned long ftrace_location(unsigned long ip)
1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
{
	return ftrace_location_range(ip, ip);
}

/**
 * ftrace_text_reserved - return true if range contains an ftrace location
 * @start: start of range to search
 * @end: end of range to search (inclusive). @end points to the last byte to check.
 *
 * Returns 1 if @start and @end contains a ftrace location.
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1684
int ftrace_text_reserved(const void *start, const void *end)
1685
{
1686 1687 1688 1689 1690 1691
	unsigned long ret;

	ret = ftrace_location_range((unsigned long)start,
				    (unsigned long)end);

	return (int)!!ret;
1692 1693
}

1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
/* Test if ops registered to this rec needs regs */
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	bool keep_regs = false;

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next) {
		/* pass rec in as regs to have non-NULL val */
		if (ftrace_ops_test(ops, rec->ip, rec)) {
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				keep_regs = true;
				break;
			}
		}
	}

	return  keep_regs;
}

1714
static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1715 1716 1717 1718 1719 1720 1721
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1722
	bool update = false;
1723
	int count = 0;
1724
	int all = false;
1725 1726 1727

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1728
		return false;
1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
1742 1743
		hash = ops->func_hash->filter_hash;
		other_hash = ops->func_hash->notrace_hash;
1744
		if (ftrace_hash_empty(hash))
1745
			all = true;
1746 1747
	} else {
		inc = !inc;
1748 1749
		hash = ops->func_hash->notrace_hash;
		other_hash = ops->func_hash->filter_hash;
1750 1751 1752 1753
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1754
		if (ftrace_hash_empty(hash))
1755
			return false;
1756 1757 1758 1759 1760 1761 1762
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

1763 1764 1765
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1766 1767 1768 1769 1770
		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1771
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1772 1773
				match = 1;
		} else {
1774 1775
			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1776 1777

			/*
1778 1779
			 * If filter_hash is set, we want to match all functions
			 * that are in the hash but not in the other hash.
1780
			 *
1781 1782 1783 1784 1785
			 * If filter_hash is not set, then we are decrementing.
			 * That means we match anything that is in the hash
			 * and also in the other_hash. That is, we need to turn
			 * off functions in the other hash because they are disabled
			 * by this hash.
1786 1787 1788 1789
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
1790
				 (in_other_hash || ftrace_hash_empty(other_hash)))
1791 1792 1793 1794 1795 1796 1797
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
1798
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1799
				return false;
1800 1801 1802 1803 1804 1805

			/*
			 * If there's only a single callback registered to a
			 * function, and the ops has a trampoline registered
			 * for it, then we can call it directly.
			 */
1806
			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1807
				rec->flags |= FTRACE_FL_TRAMP;
1808
			else
1809 1810 1811
				/*
				 * If we are adding another function callback
				 * to this function, and the previous had a
1812 1813
				 * custom trampoline in use, then we need to go
				 * back to the default trampoline.
1814
				 */
1815
				rec->flags &= ~FTRACE_FL_TRAMP;
1816

1817 1818 1819 1820 1821 1822
			/*
			 * If any ops wants regs saved for this function
			 * then all ops will get saved regs.
			 */
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
				rec->flags |= FTRACE_FL_REGS;
1823
		} else {
1824
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1825
				return false;
1826
			rec->flags--;
1827

1828 1829 1830 1831 1832 1833
			/*
			 * If the rec had REGS enabled and the ops that is
			 * being removed had REGS set, then see if there is
			 * still any ops for this record that wants regs.
			 * If not, we can stop recording them.
			 */
1834
			if (ftrace_rec_count(rec) > 0 &&
1835 1836 1837 1838 1839
			    rec->flags & FTRACE_FL_REGS &&
			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				if (!test_rec_ops_needs_regs(rec))
					rec->flags &= ~FTRACE_FL_REGS;
			}
1840

1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851
			/*
			 * If the rec had TRAMP enabled, then it needs to
			 * be cleared. As TRAMP can only be enabled iff
			 * there is only a single ops attached to it.
			 * In otherwords, always disable it on decrementing.
			 * In the future, we may set it if rec count is
			 * decremented to one, and the ops that is left
			 * has a trampoline.
			 */
			rec->flags &= ~FTRACE_FL_TRAMP;

1852 1853 1854 1855
			/*
			 * flags will be cleared in ftrace_check_record()
			 * if rec count is zero.
			 */
1856 1857
		}
		count++;
1858 1859 1860 1861

		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;

1862 1863
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
1864
			return update;
1865
	} while_for_each_ftrace_rec();
1866 1867

	return update;
1868 1869
}

1870
static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1871 1872
				    int filter_hash)
{
1873
	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1874 1875
}

1876
static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1877 1878
				   int filter_hash)
{
1879
	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1880 1881
}

1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
					  int filter_hash, int inc)
{
	struct ftrace_ops *op;

	__ftrace_hash_rec_update(ops, filter_hash, inc);

	if (ops->func_hash != &global_ops.local_hash)
		return;

	/*
	 * If the ops shares the global_ops hash, then we need to update
	 * all ops that are enabled and use this hash.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* Already done */
		if (op == ops)
			continue;
		if (op->func_hash == &global_ops.local_hash)
			__ftrace_hash_rec_update(op, filter_hash, inc);
	} while_for_each_ftrace_op(op);
}

static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
					   int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
					  int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}

1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949
/*
 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
 * or no-needed to update, -EBUSY if it detects a conflict of the flag
 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
 * Note that old_hash and new_hash has below meanings
 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
 *  - If the hash is EMPTY_HASH, it hits nothing
 *  - Anything else hits the recs which match the hash entries.
 */
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
					 struct ftrace_hash *old_hash,
					 struct ftrace_hash *new_hash)
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec, *end = NULL;
	int in_old, in_new;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
		return 0;

	/*
	 * Since the IPMODIFY is a very address sensitive action, we do not
	 * allow ftrace_ops to set all functions to new hash.
	 */
	if (!new_hash || !old_hash)
		return -EINVAL;

	/* Update rec->flags */
	do_for_each_ftrace_rec(pg, rec) {
1950 1951 1952 1953

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975
		/* We need to update only differences of filter_hash */
		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new) {
			/* New entries must ensure no others are using it */
			if (rec->flags & FTRACE_FL_IPMODIFY)
				goto rollback;
			rec->flags |= FTRACE_FL_IPMODIFY;
		} else /* Removed entry */
			rec->flags &= ~FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

	return 0;

rollback:
	end = rec;

	/* Roll back what we did above */
	do_for_each_ftrace_rec(pg, rec) {
1976 1977 1978 1979

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032
		if (rec == end)
			goto err_out;

		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new)
			rec->flags &= ~FTRACE_FL_IPMODIFY;
		else
			rec->flags |= FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

err_out:
	return -EBUSY;
}

static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}

/* Disabling always succeeds */
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}

static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash)
{
	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(old_hash))
		old_hash = NULL;

	if (ftrace_hash_empty(new_hash))
		new_hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}

2033
static void print_ip_ins(const char *fmt, const unsigned char *p)
2034 2035 2036 2037 2038 2039 2040 2041 2042
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

2043 2044
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
2045 2046
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
2047

2048
enum ftrace_bug_type ftrace_bug_type;
2049
const void *ftrace_expected;
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070

static void print_bug_type(void)
{
	switch (ftrace_bug_type) {
	case FTRACE_BUG_UNKNOWN:
		break;
	case FTRACE_BUG_INIT:
		pr_info("Initializing ftrace call sites\n");
		break;
	case FTRACE_BUG_NOP:
		pr_info("Setting ftrace call site to NOP\n");
		break;
	case FTRACE_BUG_CALL:
		pr_info("Setting ftrace call site to call ftrace function\n");
		break;
	case FTRACE_BUG_UPDATE:
		pr_info("Updating ftrace call site to call a different ftrace function\n");
		break;
	}
}

2071 2072 2073
/**
 * ftrace_bug - report and shutdown function tracer
 * @failed: The failed type (EFAULT, EINVAL, EPERM)
2074
 * @rec: The record that failed
2075 2076 2077 2078 2079 2080 2081 2082
 *
 * The arch code that enables or disables the function tracing
 * can call ftrace_bug() when it has detected a problem in
 * modifying the code. @failed should be one of either:
 * EFAULT - if the problem happens on reading the @ip address
 * EINVAL - if what is read at @ip is not what was expected
 * EPERM - if the problem happens on writting to the @ip address
 */
2083
void ftrace_bug(int failed, struct dyn_ftrace *rec)
2084
{
2085 2086
	unsigned long ip = rec ? rec->ip : 0;

2087 2088 2089 2090 2091 2092 2093 2094 2095 2096
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
2097
		print_ip_ins(" actual:   ", (unsigned char *)ip);
2098
		pr_cont("\n");
2099 2100 2101 2102
		if (ftrace_expected) {
			print_ip_ins(" expected: ", ftrace_expected);
			pr_cont("\n");
		}
2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
2114
	print_bug_type();
2115 2116 2117 2118 2119 2120 2121 2122
	if (rec) {
		struct ftrace_ops *ops = NULL;

		pr_info("ftrace record flags: %lx\n", rec->flags);
		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
			ops = ftrace_find_tramp_ops_any(rec);
2123 2124 2125 2126 2127 2128 2129 2130
			if (ops) {
				do {
					pr_cont("\ttramp: %pS (%pS)",
						(void *)ops->trampoline,
						(void *)ops->func);
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
2131 2132 2133 2134
				pr_cont("\ttramp: ERROR!");

		}
		ip = ftrace_get_addr_curr(rec);
2135
		pr_cont("\n expected tramp: %lx\n", ip);
2136
	}
2137 2138
}

2139
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2140
{
2141
	unsigned long flag = 0UL;
2142

2143 2144
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2145 2146 2147
	if (rec->flags & FTRACE_FL_DISABLED)
		return FTRACE_UPDATE_IGNORE;

S
Steven Rostedt 已提交
2148
	/*
2149
	 * If we are updating calls:
S
Steven Rostedt 已提交
2150
	 *
2151 2152
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
S
Steven Rostedt 已提交
2153
	 *
2154 2155
	 *   Otherwise we make sure its disabled.
	 *
2156
	 * If we are disabling calls, then disable all records that
2157
	 * are enabled.
S
Steven Rostedt 已提交
2158
	 */
2159
	if (enable && ftrace_rec_count(rec))
2160
		flag = FTRACE_FL_ENABLED;
S
Steven Rostedt 已提交
2161

2162
	/*
2163 2164 2165
	 * If enabling and the REGS flag does not match the REGS_EN, or
	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
	 * this record. Set flags to fail the compare against ENABLED.
2166
	 */
2167 2168 2169 2170 2171 2172 2173 2174 2175
	if (flag) {
		if (!(rec->flags & FTRACE_FL_REGS) != 
		    !(rec->flags & FTRACE_FL_REGS_EN))
			flag |= FTRACE_FL_REGS;

		if (!(rec->flags & FTRACE_FL_TRAMP) != 
		    !(rec->flags & FTRACE_FL_TRAMP_EN))
			flag |= FTRACE_FL_TRAMP;
	}
2176

2177 2178
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2179
		return FTRACE_UPDATE_IGNORE;
S
Steven Rostedt 已提交
2180

2181
	if (flag) {
2182 2183 2184 2185
		/* Save off if rec is being enabled (for return value) */
		flag ^= rec->flags & FTRACE_FL_ENABLED;

		if (update) {
2186
			rec->flags |= FTRACE_FL_ENABLED;
2187 2188 2189 2190 2191 2192
			if (flag & FTRACE_FL_REGS) {
				if (rec->flags & FTRACE_FL_REGS)
					rec->flags |= FTRACE_FL_REGS_EN;
				else
					rec->flags &= ~FTRACE_FL_REGS_EN;
			}
2193 2194 2195 2196 2197 2198
			if (flag & FTRACE_FL_TRAMP) {
				if (rec->flags & FTRACE_FL_TRAMP)
					rec->flags |= FTRACE_FL_TRAMP_EN;
				else
					rec->flags &= ~FTRACE_FL_TRAMP_EN;
			}
2199 2200 2201 2202 2203 2204 2205
		}

		/*
		 * If this record is being updated from a nop, then
		 *   return UPDATE_MAKE_CALL.
		 * Otherwise,
		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2206
		 *   from the save regs, to a non-save regs function or
2207
		 *   vice versa, or from a trampoline call.
2208
		 */
2209 2210
		if (flag & FTRACE_FL_ENABLED) {
			ftrace_bug_type = FTRACE_BUG_CALL;
2211
			return FTRACE_UPDATE_MAKE_CALL;
2212
		}
2213

2214
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2215
		return FTRACE_UPDATE_MODIFY_CALL;
2216 2217
	}

2218 2219
	if (update) {
		/* If there's no more users, clear all flags */
2220
		if (!ftrace_rec_count(rec))
2221 2222
			rec->flags = 0;
		else
2223 2224 2225 2226 2227 2228
			/*
			 * Just disable the record, but keep the ops TRAMP
			 * and REGS states. The _EN flags must be disabled though.
			 */
			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
					FTRACE_FL_REGS_EN);
2229
	}
2230

2231
	ftrace_bug_type = FTRACE_BUG_NOP;
2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261
	return FTRACE_UPDATE_MAKE_NOP;
}

/**
 * ftrace_update_record, set a record that now is tracing or not
 * @rec: the record to update
 * @enable: set to 1 if the record is tracing, zero to force disable
 *
 * The records that represent all functions that can be traced need
 * to be updated when tracing has been enabled.
 */
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 1);
}

/**
 * ftrace_test_record, check if the record has been enabled or not
 * @rec: the record to test
 * @enable: set to 1 to check if enabled, 0 if it is disabled
 *
 * The arch code may need to test if a record is already set to
 * tracing to determine how to modify the function code that it
 * represents.
 */
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 0);
}

2262 2263 2264 2265
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2266
	unsigned long ip = rec->ip;
2267 2268 2269 2270 2271 2272

	do_for_each_ftrace_op(op, ftrace_ops_list) {

		if (!op->trampoline)
			continue;

2273
		if (hash_contains_ip(ip, op->func_hash))
2274 2275 2276 2277 2278 2279
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
			   struct ftrace_ops *op)
{
	unsigned long ip = rec->ip;

	while_for_each_ftrace_op(op) {

		if (!op->trampoline)
			continue;

		if (hash_contains_ip(ip, op->func_hash))
			return op;
	} 

	return NULL;
}

2298 2299 2300 2301
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2302
	unsigned long ip = rec->ip;
2303

2304 2305 2306 2307 2308 2309 2310 2311
	/*
	 * Need to check removed ops first.
	 * If they are being removed, and this rec has a tramp,
	 * and this rec is in the ops list, then it would be the
	 * one with the tramp.
	 */
	if (removed_ops) {
		if (hash_contains_ip(ip, &removed_ops->old_hash))
2312 2313 2314
			return removed_ops;
	}

2315 2316 2317 2318 2319 2320 2321
	/*
	 * Need to find the current trampoline for a rec.
	 * Now, a trampoline is only attached to a rec if there
	 * was a single 'ops' attached to it. But this can be called
	 * when we are adding another op to the rec or removing the
	 * current one. Thus, if the op is being added, we can
	 * ignore it because it hasn't attached itself to the rec
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331
	 * yet.
	 *
	 * If an ops is being modified (hooking to different functions)
	 * then we don't care about the new functions that are being
	 * added, just the old ones (that are probably being removed).
	 *
	 * If we are adding an ops to a function that already is using
	 * a trampoline, it needs to be removed (trampolines are only
	 * for single ops connected), then an ops that is not being
	 * modified also needs to be checked.
2332
	 */
2333
	do_for_each_ftrace_op(op, ftrace_ops_list) {
2334 2335 2336 2337 2338 2339 2340 2341 2342

		if (!op->trampoline)
			continue;

		/*
		 * If the ops is being added, it hasn't gotten to
		 * the point to be removed from this tree yet.
		 */
		if (op->flags & FTRACE_OPS_FL_ADDING)
2343 2344
			continue;

2345

2346
		/*
2347 2348 2349
		 * If the ops is being modified and is in the old
		 * hash, then it is probably being removed from this
		 * function.
2350 2351 2352
		 */
		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, &op->old_hash))
2353
			return op;
2354 2355 2356 2357 2358 2359 2360 2361
		/*
		 * If the ops is not being added or modified, and it's
		 * in its normal filter hash, then this must be the one
		 * we want!
		 */
		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, op->func_hash))
			return op;
2362 2363 2364 2365 2366 2367 2368 2369 2370 2371

	} while_for_each_ftrace_op(op);

	return NULL;
}

static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2372
	unsigned long ip = rec->ip;
2373 2374 2375

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* pass rec in as regs to have non-NULL val */
2376
		if (hash_contains_ip(ip, op->func_hash))
2377 2378 2379 2380 2381 2382
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394
/**
 * ftrace_get_addr_new - Get the call address to set to
 * @rec:  The ftrace record descriptor
 *
 * If the record has the FTRACE_FL_REGS set, that means that it
 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
 * is not not set, then it wants to convert to the normal callback.
 *
 * Returns the address of the trampoline to set to
 */
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
2395 2396 2397 2398 2399 2400
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP) {
		ops = ftrace_find_tramp_ops_new(rec);
		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2401 2402
			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
				(void *)rec->ip, (void *)rec->ip, rec->flags);
2403 2404 2405 2406 2407 2408
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426
	if (rec->flags & FTRACE_FL_REGS)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

/**
 * ftrace_get_addr_curr - Get the call address that is already there
 * @rec:  The ftrace record descriptor
 *
 * The FTRACE_FL_REGS_EN is set when the record already points to
 * a function that saves all the regs. Basically the '_EN' version
 * represents the current state of the function.
 *
 * Returns the address of the trampoline that is currently being called
 */
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
2427 2428 2429 2430 2431 2432
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP_EN) {
		ops = ftrace_find_tramp_ops_curr(rec);
		if (FTRACE_WARN_ON(!ops)) {
2433 2434
			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
				(void *)rec->ip, (void *)rec->ip);
2435 2436 2437 2438 2439 2440
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2441 2442 2443 2444 2445 2446
	if (rec->flags & FTRACE_FL_REGS_EN)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

2447 2448 2449
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
2450
	unsigned long ftrace_old_addr;
2451 2452 2453
	unsigned long ftrace_addr;
	int ret;

2454
	ftrace_addr = ftrace_get_addr_new(rec);
2455

2456 2457 2458 2459
	/* This needs to be done before we call ftrace_update_record */
	ftrace_old_addr = ftrace_get_addr_curr(rec);

	ret = ftrace_update_record(rec, enable);
2460

2461 2462
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2463 2464 2465 2466 2467
	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
2468
		ftrace_bug_type = FTRACE_BUG_CALL;
2469
		return ftrace_make_call(rec, ftrace_addr);
2470 2471

	case FTRACE_UPDATE_MAKE_NOP:
2472
		ftrace_bug_type = FTRACE_BUG_NOP;
2473
		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2474 2475

	case FTRACE_UPDATE_MODIFY_CALL:
2476
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2477
		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2478 2479
	}

2480
	return -1; /* unknow ftrace bug */
2481 2482
}

2483
void __weak ftrace_replace_code(int enable)
2484 2485 2486
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
2487
	int failed;
2488

2489 2490 2491
	if (unlikely(ftrace_disabled))
		return;

2492
	do_for_each_ftrace_rec(pg, rec) {
2493 2494 2495 2496

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

2497
		failed = __ftrace_replace_code(rec, enable);
2498
		if (failed) {
2499
			ftrace_bug(failed, rec);
2500 2501
			/* Stop processing */
			return;
2502
		}
2503
	} while_for_each_ftrace_rec();
2504 2505
}

2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
struct ftrace_rec_iter {
	struct ftrace_page	*pg;
	int			index;
};

/**
 * ftrace_rec_iter_start, start up iterating over traced functions
 *
 * Returns an iterator handle that is used to iterate over all
 * the records that represent address locations where functions
 * are traced.
 *
 * May return NULL if no records are available.
 */
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
	/*
	 * We only use a single iterator.
	 * Protected by the ftrace_lock mutex.
	 */
	static struct ftrace_rec_iter ftrace_rec_iter;
	struct ftrace_rec_iter *iter = &ftrace_rec_iter;

	iter->pg = ftrace_pages_start;
	iter->index = 0;

	/* Could have empty pages */
	while (iter->pg && !iter->pg->index)
		iter->pg = iter->pg->next;

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_next, get the next record to process.
 * @iter: The handle to the iterator.
 *
 * Returns the next iterator after the given iterator @iter.
 */
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
	iter->index++;

	if (iter->index >= iter->pg->index) {
		iter->pg = iter->pg->next;
		iter->index = 0;

		/* Could have empty pages */
		while (iter->pg && !iter->pg->index)
			iter->pg = iter->pg->next;
	}

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_record, get the record at the iterator location
 * @iter: The current iterator location
 *
 * Returns the record that the current @iter is at.
 */
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
	return &iter->pg->records[iter->index];
}

2578
static int
2579
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2580
{
2581
	int ret;
2582

2583 2584 2585
	if (unlikely(ftrace_disabled))
		return 0;

2586
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2587
	if (ret) {
2588
		ftrace_bug_type = FTRACE_BUG_INIT;
2589
		ftrace_bug(ret, rec);
2590
		return 0;
2591
	}
2592
	return 1;
2593 2594
}

2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

2613
void ftrace_modify_all_code(int command)
2614
{
2615
	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2616
	int err = 0;
2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627

	/*
	 * If the ftrace_caller calls a ftrace_ops func directly,
	 * we need to make sure that it only traces functions it
	 * expects to trace. When doing the switch of functions,
	 * we need to update to the ftrace_ops_list_func first
	 * before the transition between old and new calls are set,
	 * as the ftrace_ops_list_func will check the ops hashes
	 * to make sure the ops are having the right functions
	 * traced.
	 */
2628 2629 2630 2631 2632
	if (update) {
		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
		if (FTRACE_WARN_ON(err))
			return;
	}
2633

2634
	if (command & FTRACE_UPDATE_CALLS)
2635
		ftrace_replace_code(1);
2636
	else if (command & FTRACE_DISABLE_CALLS)
2637 2638
		ftrace_replace_code(0);

2639 2640 2641 2642 2643 2644
	if (update && ftrace_trace_function != ftrace_ops_list_func) {
		function_trace_op = set_function_trace_op;
		smp_wmb();
		/* If irqs are disabled, we are in stop machine */
		if (!irqs_disabled())
			smp_call_function(ftrace_sync_ipi, NULL, 1);
2645 2646 2647
		err = ftrace_update_ftrace_func(ftrace_trace_function);
		if (FTRACE_WARN_ON(err))
			return;
2648
	}
2649

2650
	if (command & FTRACE_START_FUNC_RET)
2651
		err = ftrace_enable_ftrace_graph_caller();
2652
	else if (command & FTRACE_STOP_FUNC_RET)
2653 2654
		err = ftrace_disable_ftrace_graph_caller();
	FTRACE_WARN_ON(err);
2655 2656 2657 2658 2659 2660 2661
}

static int __ftrace_modify_code(void *data)
{
	int *command = data;

	ftrace_modify_all_code(*command);
2662

2663
	return 0;
2664 2665
}

2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689
/**
 * ftrace_run_stop_machine, go back to the stop machine method
 * @command: The command to tell ftrace what to do
 *
 * If an arch needs to fall back to the stop machine method, the
 * it can call this function.
 */
void ftrace_run_stop_machine(int command)
{
	stop_machine(__ftrace_modify_code, &command, NULL);
}

/**
 * arch_ftrace_update_code, modify the code to trace or not trace
 * @command: The command that needs to be done
 *
 * Archs can override this function if it does not need to
 * run stop_machine() to modify code.
 */
void __weak arch_ftrace_update_code(int command)
{
	ftrace_run_stop_machine(command);
}

I
Ingo Molnar 已提交
2690
static void ftrace_run_update_code(int command)
2691
{
2692 2693 2694 2695 2696 2697 2698
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;

2699 2700 2701 2702 2703 2704 2705 2706
	/*
	 * By default we use stop_machine() to modify the code.
	 * But archs can do what ever they want as long as it
	 * is safe. The stop_machine() is the safest, but also
	 * produces the most overhead.
	 */
	arch_ftrace_update_code(command);

2707 2708
	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
2709 2710
}

2711
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2712
				   struct ftrace_ops_hash *old_hash)
2713 2714
{
	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2715 2716
	ops->old_hash.filter_hash = old_hash->filter_hash;
	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2717
	ftrace_run_update_code(command);
2718
	ops->old_hash.filter_hash = NULL;
2719
	ops->old_hash.notrace_hash = NULL;
2720 2721 2722
	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}

2723
static ftrace_func_t saved_ftrace_func;
2724
static int ftrace_start_up;
2725

2726 2727 2728 2729
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}

2730
static void per_cpu_ops_free(struct ftrace_ops *ops)
2731 2732 2733 2734
{
	free_percpu(ops->disabled);
}

2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746
static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
2747

2748 2749 2750 2751 2752 2753 2754
static void ftrace_startup_all(int command)
{
	update_all_ops = true;
	ftrace_startup_enable(command);
	update_all_ops = false;
}

2755
static int ftrace_startup(struct ftrace_ops *ops, int command)
2756
{
2757
	int ret;
2758

2759
	if (unlikely(ftrace_disabled))
2760
		return -ENODEV;
2761

2762 2763 2764 2765
	ret = __register_ftrace_function(ops);
	if (ret)
		return ret;

2766
	ftrace_start_up++;
2767

2768 2769 2770 2771 2772 2773 2774 2775 2776
	/*
	 * Note that ftrace probes uses this to start up
	 * and modify functions it will probe. But we still
	 * set the ADDING flag for modification, as probes
	 * do not have trampolines. If they add them in the
	 * future, then the probes will need to distinguish
	 * between adding and updating probes.
	 */
	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2777

2778 2779 2780 2781 2782 2783 2784 2785 2786
	ret = ftrace_hash_ipmodify_enable(ops);
	if (ret < 0) {
		/* Rollback registration process */
		__unregister_ftrace_function(ops);
		ftrace_start_up--;
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
		return ret;
	}

2787 2788
	if (ftrace_hash_rec_enable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2789

2790
	ftrace_startup_enable(command);
2791

2792 2793
	ops->flags &= ~FTRACE_OPS_FL_ADDING;

2794
	return 0;
2795 2796
}

2797
static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2798
{
2799
	int ret;
2800

2801
	if (unlikely(ftrace_disabled))
2802 2803 2804 2805 2806
		return -ENODEV;

	ret = __unregister_ftrace_function(ops);
	if (ret)
		return ret;
2807

2808
	ftrace_start_up--;
2809 2810 2811 2812 2813 2814 2815
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

2816 2817
	/* Disabling ipmodify never fails */
	ftrace_hash_ipmodify_disable(ops);
2818

2819 2820
	if (ftrace_hash_rec_disable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2821

2822
	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2823

2824 2825 2826 2827
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
2828

2829 2830
	if (!command || !ftrace_enabled) {
		/*
2831
		 * If these are per_cpu ops, they still need their
2832 2833 2834 2835
		 * per_cpu field freed. Since, function tracing is
		 * not currently active, we can just free them
		 * without synchronizing all CPUs.
		 */
2836 2837
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2838
		return 0;
2839
	}
2840

2841 2842 2843 2844
	/*
	 * If the ops uses a trampoline, then it needs to be
	 * tested first on update.
	 */
2845
	ops->flags |= FTRACE_OPS_FL_REMOVING;
2846 2847
	removed_ops = ops;

2848 2849 2850 2851
	/* The trampoline logic checks the old hashes */
	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;

2852
	ftrace_run_update_code(command);
2853

2854 2855 2856 2857
	/*
	 * If there's no more ops registered with ftrace, run a
	 * sanity check to make sure all rec flags are cleared.
	 */
2858 2859
	if (rcu_dereference_protected(ftrace_ops_list,
			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
2860 2861 2862 2863
		struct ftrace_page *pg;
		struct dyn_ftrace *rec;

		do_for_each_ftrace_rec(pg, rec) {
2864
			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2865 2866 2867 2868 2869
				pr_warn("  %pS flags:%lx\n",
					(void *)rec->ip, rec->flags);
		} while_for_each_ftrace_rec();
	}

2870 2871 2872 2873
	ops->old_hash.filter_hash = NULL;
	ops->old_hash.notrace_hash = NULL;

	removed_ops = NULL;
2874
	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2875

2876 2877 2878
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
2879
	 * The same goes for freeing the per_cpu data of the per_cpu
2880 2881
	 * ops.
	 */
2882
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) {
2883 2884 2885 2886 2887 2888 2889 2890
		/*
		 * We need to do a hard force of sched synchronization.
		 * This is because we use preempt_disable() to do RCU, but
		 * the function tracers can be called where RCU is not watching
		 * (like before user_exit()). We can not rely on the RCU
		 * infrastructure to do the synchronization, thus we must do it
		 * ourselves.
		 */
2891 2892
		schedule_on_each_cpu(ftrace_sync);

2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
		/*
		 * When the kernel is preeptive, tasks can be preempted
		 * while on a ftrace trampoline. Just scheduling a task on
		 * a CPU is not good enough to flush them. Calling
		 * synchornize_rcu_tasks() will wait for those tasks to
		 * execute and either schedule voluntarily or enter user space.
		 */
		if (IS_ENABLED(CONFIG_PREEMPT))
			synchronize_rcu_tasks();

2903 2904
		arch_ftrace_trampoline_free(ops);

2905 2906
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2907 2908
	}

2909
	return 0;
2910 2911
}

I
Ingo Molnar 已提交
2912
static void ftrace_startup_sysctl(void)
2913
{
2914 2915
	int command;

2916 2917 2918
	if (unlikely(ftrace_disabled))
		return;

2919 2920
	/* Force update next time */
	saved_ftrace_func = NULL;
2921
	/* ftrace_start_up is true if we want ftrace running */
2922 2923 2924 2925
	if (ftrace_start_up) {
		command = FTRACE_UPDATE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_START_FUNC_RET;
2926
		ftrace_startup_enable(command);
2927
	}
2928 2929
}

I
Ingo Molnar 已提交
2930
static void ftrace_shutdown_sysctl(void)
2931
{
2932 2933
	int command;

2934 2935 2936
	if (unlikely(ftrace_disabled))
		return;

2937
	/* ftrace_start_up is true if ftrace is running */
2938 2939 2940 2941 2942 2943
	if (ftrace_start_up) {
		command = FTRACE_DISABLE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_STOP_FUNC_RET;
		ftrace_run_update_code(command);
	}
2944 2945
}

2946
static u64		ftrace_update_time;
2947 2948
unsigned long		ftrace_update_tot_cnt;

2949
static inline int ops_traces_mod(struct ftrace_ops *ops)
2950
{
2951 2952 2953 2954
	/*
	 * Filter_hash being empty will default to trace module.
	 * But notrace hash requires a test of individual module functions.
	 */
2955 2956
	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
		ftrace_hash_empty(ops->func_hash->notrace_hash);
2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972
}

/*
 * Check if the current ops references the record.
 *
 * If the ops traces all functions, then it was already accounted for.
 * If the ops does not trace the current record function, skip it.
 * If the ops ignores the function via notrace filter, skip it.
 */
static inline bool
ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	/* If ops isn't enabled, ignore it */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

2973
	/* If ops traces all then it includes this function */
2974
	if (ops_traces_mod(ops))
2975
		return 1;
2976 2977

	/* The function must be in the filter */
2978
	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2979
	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2980
		return 0;
2981

2982
	/* If in notrace hash, we ignore it too */
2983
	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2984 2985 2986 2987 2988
		return 0;

	return 1;
}

2989
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2990
{
2991
	struct ftrace_page *pg;
2992
	struct dyn_ftrace *p;
2993
	u64 start, stop;
2994
	unsigned long update_cnt = 0;
2995
	unsigned long rec_flags = 0;
2996
	int i;
2997

2998 2999
	start = ftrace_now(raw_smp_processor_id());

3000
	/*
3001 3002 3003 3004 3005 3006 3007 3008 3009
	 * When a module is loaded, this function is called to convert
	 * the calls to mcount in its text to nops, and also to create
	 * an entry in the ftrace data. Now, if ftrace is activated
	 * after this call, but before the module sets its text to
	 * read-only, the modification of enabling ftrace can fail if
	 * the read-only is done while ftrace is converting the calls.
	 * To prevent this, the module's records are set as disabled
	 * and will be enabled after the call to set the module's text
	 * to read-only.
3010
	 */
3011 3012
	if (mod)
		rec_flags |= FTRACE_FL_DISABLED;
3013

3014
	for (pg = new_pgs; pg; pg = pg->next) {
3015

3016
		for (i = 0; i < pg->index; i++) {
3017

3018 3019 3020
			/* If something went wrong, bail without enabling anything */
			if (unlikely(ftrace_disabled))
				return -1;
3021

3022
			p = &pg->records[i];
3023
			p->flags = rec_flags;
3024

3025 3026 3027 3028 3029 3030
			/*
			 * Do the initial record conversion from mcount jump
			 * to the NOP instructions.
			 */
			if (!ftrace_code_disable(mod, p))
				break;
3031

3032
			update_cnt++;
3033
		}
3034 3035
	}

I
Ingo Molnar 已提交
3036
	stop = ftrace_now(raw_smp_processor_id());
3037
	ftrace_update_time = stop - start;
3038
	ftrace_update_tot_cnt += update_cnt;
3039

3040 3041 3042
	return 0;
}

3043
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
3044
{
3045
	int order;
3046 3047
	int cnt;

3048 3049 3050 3051
	if (WARN_ON(!count))
		return -EINVAL;

	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
3052 3053

	/*
3054 3055
	 * We want to fill as much as possible. No more than a page
	 * may be empty.
3056
	 */
3057 3058
	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
		order--;
3059

3060 3061
 again:
	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
3062

3063 3064 3065 3066 3067 3068 3069
	if (!pg->records) {
		/* if we can't allocate this size, try something smaller */
		if (!order)
			return -ENOMEM;
		order >>= 1;
		goto again;
	}
3070

3071 3072
	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
	pg->size = cnt;
3073

3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106
	if (cnt > count)
		cnt = count;

	return cnt;
}

static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
	struct ftrace_page *start_pg;
	struct ftrace_page *pg;
	int order;
	int cnt;

	if (!num_to_init)
		return 0;

	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
	if (!pg)
		return NULL;

	/*
	 * Try to allocate as much as possible in one continues
	 * location that fills in all of the space. We want to
	 * waste as little space as possible.
	 */
	for (;;) {
		cnt = ftrace_allocate_records(pg, num_to_init);
		if (cnt < 0)
			goto free_pages;

		num_to_init -= cnt;
		if (!num_to_init)
3107 3108
			break;

3109 3110 3111 3112
		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
		if (!pg->next)
			goto free_pages;

3113 3114 3115
		pg = pg->next;
	}

3116 3117 3118
	return start_pg;

 free_pages:
3119 3120
	pg = start_pg;
	while (pg) {
3121 3122 3123 3124 3125 3126 3127 3128 3129 3130
		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		start_pg = pg->next;
		kfree(pg);
		pg = start_pg;
	}
	pr_info("ftrace: FAILED to allocate memory for functions\n");
	return NULL;
}

3131 3132 3133
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
3134
	loff_t				pos;
3135
	loff_t				func_pos;
3136
	loff_t				mod_pos;
3137 3138 3139
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
3140
	struct ftrace_func_entry	*probe_entry;
3141
	struct trace_parser		parser;
3142
	struct ftrace_hash		*hash;
3143
	struct ftrace_ops		*ops;
3144 3145
	struct trace_array		*tr;
	struct list_head		*mod_list;
3146
	int				pidx;
3147 3148
	int				idx;
	unsigned			flags;
3149 3150
};

3151
static void *
3152
t_probe_next(struct seq_file *m, loff_t *pos)
3153 3154
{
	struct ftrace_iterator *iter = m->private;
3155
	struct trace_array *tr = iter->ops->private;
3156
	struct list_head *func_probes;
3157 3158
	struct ftrace_hash *hash;
	struct list_head *next;
3159
	struct hlist_node *hnd = NULL;
3160
	struct hlist_head *hhd;
3161
	int size;
3162 3163

	(*pos)++;
3164
	iter->pos = *pos;
3165

3166
	if (!tr)
3167 3168
		return NULL;

3169 3170
	func_probes = &tr->func_probes;
	if (list_empty(func_probes))
3171 3172
		return NULL;

3173
	if (!iter->probe) {
3174
		next = func_probes->next;
3175
		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3176 3177 3178 3179 3180 3181 3182 3183 3184 3185
	}

	if (iter->probe_entry)
		hnd = &iter->probe_entry->hlist;

	hash = iter->probe->ops.func_hash->filter_hash;
	size = 1 << hash->size_bits;

 retry:
	if (iter->pidx >= size) {
3186
		if (iter->probe->list.next == func_probes)
3187 3188
			return NULL;
		next = iter->probe->list.next;
3189
		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3190 3191 3192 3193 3194 3195
		hash = iter->probe->ops.func_hash->filter_hash;
		size = 1 << hash->size_bits;
		iter->pidx = 0;
	}

	hhd = &hash->buckets[iter->pidx];
3196 3197

	if (hlist_empty(hhd)) {
3198
		iter->pidx++;
3199 3200 3201 3202 3203 3204 3205 3206 3207
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
3208
			iter->pidx++;
3209 3210 3211 3212
			goto retry;
		}
	}

3213 3214 3215
	if (WARN_ON_ONCE(!hnd))
		return NULL;

3216
	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
3217 3218

	return iter;
3219 3220
}

3221
static void *t_probe_start(struct seq_file *m, loff_t *pos)
3222 3223 3224
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
3225 3226
	loff_t l;

3227
	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
3228 3229
		return NULL;

3230
	if (iter->mod_pos > *pos)
3231
		return NULL;
3232

3233 3234 3235
	iter->probe = NULL;
	iter->probe_entry = NULL;
	iter->pidx = 0;
3236
	for (l = 0; l <= (*pos - iter->mod_pos); ) {
3237
		p = t_probe_next(m, &l);
L
Li Zefan 已提交
3238 3239 3240
		if (!p)
			break;
	}
3241 3242 3243
	if (!p)
		return NULL;

3244
	/* Only set this if we have an item */
3245
	iter->flags |= FTRACE_ITER_PROBE;
3246

3247
	return iter;
3248 3249
}

3250
static int
3251
t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
3252
{
3253
	struct ftrace_func_entry *probe_entry;
3254 3255
	struct ftrace_probe_ops *probe_ops;
	struct ftrace_func_probe *probe;
3256

3257 3258
	probe = iter->probe;
	probe_entry = iter->probe_entry;
3259

3260
	if (WARN_ON_ONCE(!probe || !probe_entry))
3261
		return -EIO;
3262

3263
	probe_ops = probe->probe_ops;
3264

3265
	if (probe_ops->print)
3266
		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
3267

3268 3269
	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
		   (void *)probe_ops->func);
3270 3271 3272 3273

	return 0;
}

3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349
static void *
t_mod_next(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	struct trace_array *tr = iter->tr;

	(*pos)++;
	iter->pos = *pos;

	iter->mod_list = iter->mod_list->next;

	if (iter->mod_list == &tr->mod_trace ||
	    iter->mod_list == &tr->mod_notrace) {
		iter->flags &= ~FTRACE_ITER_MOD;
		return NULL;
	}

	iter->mod_pos = *pos;

	return iter;
}

static void *t_mod_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
	loff_t l;

	if (iter->func_pos > *pos)
		return NULL;

	iter->mod_pos = iter->func_pos;

	/* probes are only available if tr is set */
	if (!iter->tr)
		return NULL;

	for (l = 0; l <= (*pos - iter->func_pos); ) {
		p = t_mod_next(m, &l);
		if (!p)
			break;
	}
	if (!p) {
		iter->flags &= ~FTRACE_ITER_MOD;
		return t_probe_start(m, pos);
	}

	/* Only set this if we have an item */
	iter->flags |= FTRACE_ITER_MOD;

	return iter;
}

static int
t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
{
	struct ftrace_mod_load *ftrace_mod;
	struct trace_array *tr = iter->tr;

	if (WARN_ON_ONCE(!iter->mod_list) ||
			 iter->mod_list == &tr->mod_trace ||
			 iter->mod_list == &tr->mod_notrace)
		return -EIO;

	ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);

	if (ftrace_mod->func)
		seq_printf(m, "%s", ftrace_mod->func);
	else
		seq_putc(m, '*');

	seq_printf(m, ":mod:%s\n", ftrace_mod->module);

	return 0;
}

I
Ingo Molnar 已提交
3350
static void *
3351
t_func_next(struct seq_file *m, loff_t *pos)
3352 3353 3354 3355 3356
{
	struct ftrace_iterator *iter = m->private;
	struct dyn_ftrace *rec = NULL;

	(*pos)++;
3357

3358 3359 3360 3361 3362 3363 3364 3365 3366
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
3367 3368
		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3369 3370

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3371
		     !(rec->flags & FTRACE_FL_ENABLED))) {
3372

3373 3374 3375 3376 3377
			rec = NULL;
			goto retry;
		}
	}

3378
	if (!rec)
3379
		return NULL;
3380

3381
	iter->pos = iter->func_pos = *pos;
3382 3383 3384
	iter->func = rec;

	return iter;
3385 3386
}

3387 3388 3389 3390
static void *
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3391
	loff_t l = *pos; /* t_probe_start() must use original pos */
3392 3393 3394 3395 3396
	void *ret;

	if (unlikely(ftrace_disabled))
		return NULL;

3397 3398
	if (iter->flags & FTRACE_ITER_PROBE)
		return t_probe_next(m, pos);
3399

3400 3401 3402
	if (iter->flags & FTRACE_ITER_MOD)
		return t_mod_next(m, pos);

3403
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3404
		/* next must increment pos, and t_probe_start does not */
3405
		(*pos)++;
3406
		return t_mod_start(m, &l);
3407 3408 3409 3410 3411
	}

	ret = t_func_next(m, pos);

	if (!ret)
3412
		return t_mod_start(m, &l);
3413 3414 3415 3416

	return ret;
}

3417 3418 3419 3420
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
3421
	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
3422 3423 3424 3425 3426 3427
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
3428
	loff_t l;
3429

3430
	mutex_lock(&ftrace_lock);
3431 3432 3433 3434

	if (unlikely(ftrace_disabled))
		return NULL;

3435 3436 3437 3438 3439 3440
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

3441 3442 3443 3444 3445
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
3446 3447
	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
	    ftrace_hash_empty(iter->hash)) {
3448
		iter->func_pos = 1; /* Account for the message */
3449
		if (*pos > 0)
3450
			return t_mod_start(m, pos);
3451
		iter->flags |= FTRACE_ITER_PRINTALL;
3452
		/* reset in case of seek/pread */
3453
		iter->flags &= ~FTRACE_ITER_PROBE;
3454 3455 3456
		return iter;
	}

3457 3458
	if (iter->flags & FTRACE_ITER_MOD)
		return t_mod_start(m, pos);
3459

3460 3461 3462 3463 3464
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
3465 3466 3467
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
3468
		p = t_func_next(m, &l);
3469 3470
		if (!p)
			break;
3471
	}
3472

3473
	if (!p)
3474
		return t_mod_start(m, pos);
3475 3476

	return iter;
3477 3478 3479 3480
}

static void t_stop(struct seq_file *m, void *p)
{
3481
	mutex_unlock(&ftrace_lock);
3482 3483
}

3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	return NULL;
}

static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
				struct dyn_ftrace *rec)
{
	void *ptr;

	ptr = arch_ftrace_trampoline_func(ops, rec);
	if (ptr)
		seq_printf(m, " ->%pS", ptr);
}

3500 3501
static int t_show(struct seq_file *m, void *v)
{
3502
	struct ftrace_iterator *iter = m->private;
3503
	struct dyn_ftrace *rec;
3504

3505 3506
	if (iter->flags & FTRACE_ITER_PROBE)
		return t_probe_show(m, iter);
3507

3508 3509 3510
	if (iter->flags & FTRACE_ITER_MOD)
		return t_mod_show(m, iter);

3511
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3512
		if (iter->flags & FTRACE_ITER_NOTRACE)
3513
			seq_puts(m, "#### no functions disabled ####\n");
3514
		else
3515
			seq_puts(m, "#### all functions enabled ####\n");
3516 3517 3518
		return 0;
	}

3519 3520
	rec = iter->func;

3521 3522 3523
	if (!rec)
		return 0;

3524
	seq_printf(m, "%ps", (void *)rec->ip);
3525
	if (iter->flags & FTRACE_ITER_ENABLED) {
3526
		struct ftrace_ops *ops;
3527

3528
		seq_printf(m, " (%ld)%s%s",
3529
			   ftrace_rec_count(rec),
3530 3531
			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
3532
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3533
			ops = ftrace_find_tramp_ops_any(rec);
3534 3535 3536 3537 3538
			if (ops) {
				do {
					seq_printf(m, "\ttramp: %pS (%pS)",
						   (void *)ops->trampoline,
						   (void *)ops->func);
3539
					add_trampoline_func(m, ops, rec);
3540 3541 3542
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
3543
				seq_puts(m, "\ttramp: ERROR!");
3544 3545
		} else {
			add_trampoline_func(m, NULL, rec);
3546 3547 3548
		}
	}	

3549
	seq_putc(m, '\n');
3550 3551 3552 3553

	return 0;
}

J
James Morris 已提交
3554
static const struct seq_operations show_ftrace_seq_ops = {
3555 3556 3557 3558 3559 3560
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
3561
static int
3562 3563 3564 3565
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3566 3567 3568
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3569
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3570 3571
	if (!iter)
		return -ENOMEM;
3572

3573 3574 3575 3576
	iter->pg = ftrace_pages_start;
	iter->ops = &global_ops;

	return 0;
3577 3578
}

3579 3580 3581 3582 3583
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3584
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3585 3586
	if (!iter)
		return -ENOMEM;
3587

3588 3589 3590 3591 3592
	iter->pg = ftrace_pages_start;
	iter->flags = FTRACE_ITER_ENABLED;
	iter->ops = &global_ops;

	return 0;
3593 3594
}

3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607
/**
 * ftrace_regex_open - initialize function tracer filter files
 * @ops: The ftrace_ops that hold the hash filters
 * @flag: The type of filter to process
 * @inode: The inode, usually passed in to your open routine
 * @file: The file, usually passed in to your open routine
 *
 * ftrace_regex_open() initializes the filter files for the
 * @ops. Depending on @flag it may process the filter hash or
 * the notrace hash of @ops. With this called from the open
 * routine, you can use ftrace_filter_write() for the write
 * routine if @flag has FTRACE_ITER_FILTER set, or
 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3608
 * tracing_lseek() should be used as the lseek routine, and
3609 3610 3611
 * release must call ftrace_regex_release().
 */
int
3612
ftrace_regex_open(struct ftrace_ops *ops, int flag,
3613
		  struct inode *inode, struct file *file)
3614 3615
{
	struct ftrace_iterator *iter;
3616
	struct ftrace_hash *hash;
3617 3618
	struct list_head *mod_head;
	struct trace_array *tr = ops->private;
3619 3620
	int ret = 0;

3621 3622
	ftrace_ops_init(ops);

3623 3624 3625
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3626 3627 3628 3629
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

3630 3631 3632 3633 3634
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

3635 3636
	iter->ops = ops;
	iter->flags = flag;
3637
	iter->tr = tr;
3638

3639
	mutex_lock(&ops->func_hash->regex_lock);
3640

3641
	if (flag & FTRACE_ITER_NOTRACE) {
3642
		hash = ops->func_hash->notrace_hash;
3643
		mod_head = tr ? &tr->mod_notrace : NULL;
3644
	} else {
3645
		hash = ops->func_hash->filter_hash;
3646
		mod_head = tr ? &tr->mod_trace : NULL;
3647
	}
3648

3649 3650
	iter->mod_list = mod_head;

3651
	if (file->f_mode & FMODE_WRITE) {
3652 3653
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

3654
		if (file->f_flags & O_TRUNC) {
3655
			iter->hash = alloc_ftrace_hash(size_bits);
3656 3657
			clear_ftrace_mod_list(mod_head);
	        } else {
3658
			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
3659
		}
3660

3661 3662 3663
		if (!iter->hash) {
			trace_parser_put(&iter->parser);
			kfree(iter);
3664 3665
			ret = -ENOMEM;
			goto out_unlock;
3666
		}
3667 3668
	} else
		iter->hash = hash;
3669

3670 3671 3672 3673 3674 3675 3676
	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
3677
		} else {
3678 3679
			/* Failed */
			free_ftrace_hash(iter->hash);
3680
			trace_parser_put(&iter->parser);
3681
			kfree(iter);
3682
		}
3683 3684
	} else
		file->private_data = iter;
3685 3686

 out_unlock:
3687
	mutex_unlock(&ops->func_hash->regex_lock);
3688 3689 3690 3691

	return ret;
}

3692 3693 3694
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
3695 3696 3697
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops,
3698
			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
3699
			inode, file);
3700 3701 3702 3703 3704
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
3705 3706 3707
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3708
				 inode, file);
3709 3710
}

3711 3712 3713 3714 3715 3716 3717
/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
struct ftrace_glob {
	char *search;
	unsigned len;
	int type;
};

3718 3719 3720 3721 3722 3723 3724 3725 3726 3727
/*
 * If symbols in an architecture don't correspond exactly to the user-visible
 * name of what they represent, it is possible to define this function to
 * perform the necessary adjustments.
*/
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
	return str;
}

3728
static int ftrace_match(char *str, struct ftrace_glob *g)
3729 3730
{
	int matched = 0;
3731
	int slen;
3732

3733 3734
	str = arch_ftrace_match_adjust(str, g->search);

3735
	switch (g->type) {
3736
	case MATCH_FULL:
3737
		if (strcmp(str, g->search) == 0)
3738 3739 3740
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
3741
		if (strncmp(str, g->search, g->len) == 0)
3742 3743 3744
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
3745
		if (strstr(str, g->search))
3746 3747 3748
			matched = 1;
		break;
	case MATCH_END_ONLY:
3749
		slen = strlen(str);
3750 3751
		if (slen >= g->len &&
		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3752 3753
			matched = 1;
		break;
3754 3755 3756 3757
	case MATCH_GLOB:
		if (glob_match(g->search, str))
			matched = 1;
		break;
3758 3759 3760 3761 3762
	}

	return matched;
}

3763
static int
3764
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3765
{
3766 3767 3768
	struct ftrace_func_entry *entry;
	int ret = 0;

3769
	entry = ftrace_lookup_ip(hash, rec->ip);
3770
	if (clear_filter) {
3771 3772 3773
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
3774

3775
		free_hash_entry(hash, entry);
3776 3777 3778 3779
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
3780

3781
		ret = add_hash_entry(hash, rec->ip);
3782 3783
	}
	return ret;
3784 3785
}

3786
static int
D
Dmitry Safonov 已提交
3787 3788
ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
		struct ftrace_glob *mod_g, int exclude_mod)
3789 3790
{
	char str[KSYM_SYMBOL_LEN];
3791 3792 3793 3794
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

D
Dmitry Safonov 已提交
3795 3796 3797 3798 3799 3800
	if (mod_g) {
		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;

		/* blank module name to match all modules */
		if (!mod_g->len) {
			/* blank module globbing: modname xor exclude_mod */
3801
			if (!exclude_mod != !modname)
D
Dmitry Safonov 已提交
3802 3803 3804 3805
				goto func_match;
			return 0;
		}

3806 3807 3808 3809 3810 3811 3812 3813
		/*
		 * exclude_mod is set to trace everything but the given
		 * module. If it is set and the module matches, then
		 * return 0. If it is not set, and the module doesn't match
		 * also return 0. Otherwise, check the function to see if
		 * that matches.
		 */
		if (!mod_matches == !exclude_mod)
3814
			return 0;
D
Dmitry Safonov 已提交
3815
func_match:
3816
		/* blank search means to match all funcs in the mod */
3817
		if (!func_g->len)
3818 3819
			return 1;
	}
3820

3821
	return ftrace_match(str, func_g);
3822 3823
}

3824
static int
3825
match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3826 3827 3828
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3829
	struct ftrace_glob func_g = { .type = MATCH_FULL };
D
Dmitry Safonov 已提交
3830 3831 3832
	struct ftrace_glob mod_g = { .type = MATCH_FULL };
	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
	int exclude_mod = 0;
3833
	int found = 0;
3834
	int ret;
3835
	int clear_filter = 0;
3836

D
Dmitry Safonov 已提交
3837
	if (func) {
3838 3839 3840
		func_g.type = filter_parse_regex(func, len, &func_g.search,
						 &clear_filter);
		func_g.len = strlen(func_g.search);
3841
	}
3842

D
Dmitry Safonov 已提交
3843 3844 3845 3846
	if (mod) {
		mod_g.type = filter_parse_regex(mod, strlen(mod),
				&mod_g.search, &exclude_mod);
		mod_g.len = strlen(mod_g.search);
3847
	}
3848

3849
	mutex_lock(&ftrace_lock);
3850

3851 3852
	if (unlikely(ftrace_disabled))
		goto out_unlock;
3853

3854
	do_for_each_ftrace_rec(pg, rec) {
3855 3856 3857 3858

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
3859
		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3860
			ret = enter_record(hash, rec, clear_filter);
3861 3862 3863 3864
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
3865
			found = 1;
3866 3867
		}
	} while_for_each_ftrace_rec();
3868
 out_unlock:
3869
	mutex_unlock(&ftrace_lock);
3870 3871

	return found;
3872 3873
}

3874
static int
3875
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3876
{
3877
	return match_records(hash, buff, len, NULL);
3878 3879
}

3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929
static void ftrace_ops_update_code(struct ftrace_ops *ops,
				   struct ftrace_ops_hash *old_hash)
{
	struct ftrace_ops *op;

	if (!ftrace_enabled)
		return;

	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
		return;
	}

	/*
	 * If this is the shared global_ops filter, then we need to
	 * check if there is another ops that shares it, is enabled.
	 * If so, we still need to run the modify code.
	 */
	if (ops->func_hash != &global_ops.local_hash)
		return;

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->func_hash == &global_ops.local_hash &&
		    op->flags & FTRACE_OPS_FL_ENABLED) {
			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
			/* Only need to do this once */
			return;
		}
	} while_for_each_ftrace_op(op);
}

static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
					   struct ftrace_hash **orig_hash,
					   struct ftrace_hash *hash,
					   int enable)
{
	struct ftrace_ops_hash old_hash_ops;
	struct ftrace_hash *old_hash;
	int ret;

	old_hash = *orig_hash;
	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
	if (!ret) {
		ftrace_ops_update_code(ops, &old_hash_ops);
		free_ftrace_hash_rcu(old_hash);
	}
	return ret;
}
3930

3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968
static bool module_exists(const char *module)
{
	/* All modules have the symbol __this_module */
	const char this_mod[] = "__this_module";
	const int modname_size = MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 1;
	char modname[modname_size + 1];
	unsigned long val;
	int n;

	n = snprintf(modname, modname_size + 1, "%s:%s", module, this_mod);

	if (n > modname_size)
		return false;

	val = module_kallsyms_lookup_name(modname);
	return val != 0;
}

static int cache_mod(struct trace_array *tr,
		     const char *func, char *module, int enable)
{
	struct ftrace_mod_load *ftrace_mod, *n;
	struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
	int ret;

	mutex_lock(&ftrace_lock);

	/* We do not cache inverse filters */
	if (func[0] == '!') {
		func++;
		ret = -EINVAL;

		/* Look to remove this hash */
		list_for_each_entry_safe(ftrace_mod, n, head, list) {
			if (strcmp(ftrace_mod->module, module) != 0)
				continue;

			/* no func matches all */
3969
			if (strcmp(func, "*") == 0 ||
3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992
			    (ftrace_mod->func &&
			     strcmp(ftrace_mod->func, func) == 0)) {
				ret = 0;
				free_ftrace_mod(ftrace_mod);
				continue;
			}
		}
		goto out;
	}

	ret = -EINVAL;
	/* We only care about modules that have not been loaded yet */
	if (module_exists(module))
		goto out;

	/* Save this string off, and execute it when the module is loaded */
	ret = ftrace_add_mod(tr, func, module, enable);
 out:
	mutex_unlock(&ftrace_lock);

	return ret;
}

3993 3994 3995 3996
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable);

3997
#ifdef CONFIG_MODULES
3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016
static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
			     char *mod, bool enable)
{
	struct ftrace_mod_load *ftrace_mod, *n;
	struct ftrace_hash **orig_hash, *new_hash;
	LIST_HEAD(process_mods);
	char *func;
	int ret;

	mutex_lock(&ops->func_hash->regex_lock);

	if (enable)
		orig_hash = &ops->func_hash->filter_hash;
	else
		orig_hash = &ops->func_hash->notrace_hash;

	new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
					      *orig_hash);
	if (!new_hash)
4017
		goto out; /* warn? */
4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052

	mutex_lock(&ftrace_lock);

	list_for_each_entry_safe(ftrace_mod, n, head, list) {

		if (strcmp(ftrace_mod->module, mod) != 0)
			continue;

		if (ftrace_mod->func)
			func = kstrdup(ftrace_mod->func, GFP_KERNEL);
		else
			func = kstrdup("*", GFP_KERNEL);

		if (!func) /* warn? */
			continue;

		list_del(&ftrace_mod->list);
		list_add(&ftrace_mod->list, &process_mods);

		/* Use the newly allocated func, as it may be "*" */
		kfree(ftrace_mod->func);
		ftrace_mod->func = func;
	}

	mutex_unlock(&ftrace_lock);

	list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {

		func = ftrace_mod->func;

		/* Grabs ftrace_lock, which is why we have this extra step */
		match_records(new_hash, func, strlen(func), mod);
		free_ftrace_mod(ftrace_mod);
	}

4053 4054 4055
	if (enable && list_empty(head))
		new_hash->flags &= ~FTRACE_HASH_FL_MOD;

4056 4057 4058 4059 4060 4061
	mutex_lock(&ftrace_lock);

	ret = ftrace_hash_move_and_update_ops(ops, orig_hash,
					      new_hash, enable);
	mutex_unlock(&ftrace_lock);

4062
 out:
4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087
	mutex_unlock(&ops->func_hash->regex_lock);

	free_ftrace_hash(new_hash);
}

static void process_cached_mods(const char *mod_name)
{
	struct trace_array *tr;
	char *mod;

	mod = kstrdup(mod_name, GFP_KERNEL);
	if (!mod)
		return;

	mutex_lock(&trace_types_lock);
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (!list_empty(&tr->mod_trace))
			process_mod_list(&tr->mod_trace, tr->ops, mod, true);
		if (!list_empty(&tr->mod_notrace))
			process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
	}
	mutex_unlock(&trace_types_lock);

	kfree(mod);
}
4088
#endif
4089

4090 4091 4092 4093 4094 4095
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
4096
ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
4097
		    char *func_orig, char *cmd, char *module, int enable)
4098
{
4099
	char *func;
4100
	int ret;
4101

4102 4103 4104 4105 4106
	/* match_records() modifies func, and we need the original */
	func = kstrdup(func_orig, GFP_KERNEL);
	if (!func)
		return -ENOMEM;

4107 4108 4109 4110 4111 4112 4113
	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */
4114
	ret = match_records(hash, func, strlen(func), module);
4115 4116
	kfree(func);

4117
	if (!ret)
4118
		return cache_mod(tr, func_orig, module, enable);
4119 4120 4121
	if (ret < 0)
		return ret;
	return 0;
4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
4133
core_initcall(ftrace_mod_cmd_init);
4134

4135
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
4136
				      struct ftrace_ops *op, struct pt_regs *pt_regs)
4137
{
4138
	struct ftrace_probe_ops *probe_ops;
4139
	struct ftrace_func_probe *probe;
4140

4141 4142
	probe = container_of(op, struct ftrace_func_probe, ops);
	probe_ops = probe->probe_ops;
4143 4144 4145 4146 4147 4148

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
4149
	preempt_disable_notrace();
4150
	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
4151
	preempt_enable_notrace();
4152 4153
}

4154 4155 4156
struct ftrace_func_map {
	struct ftrace_func_entry	entry;
	void				*data;
4157 4158
};

4159 4160 4161
struct ftrace_func_mapper {
	struct ftrace_hash		hash;
};
4162

4163 4164 4165 4166 4167 4168
/**
 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
 *
 * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
 */
struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
4169
{
4170
	struct ftrace_hash *hash;
4171

4172 4173 4174 4175 4176 4177 4178 4179
	/*
	 * The mapper is simply a ftrace_hash, but since the entries
	 * in the hash are not ftrace_func_entry type, we define it
	 * as a separate structure.
	 */
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	return (struct ftrace_func_mapper *)hash;
}
4180

4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196
/**
 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: the instruction pointer to find the data for
 *
 * Returns the data mapped to @ip if found otherwise NULL. The return
 * is actually the address of the mapper data pointer. The address is
 * returned for use cases where the data is no bigger than a long, and
 * the user can use the data pointer as its data instead of having to
 * allocate more memory for the reference.
 */
void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
				  unsigned long ip)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
4197

4198 4199 4200
	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;
4201

4202 4203
	map = (struct ftrace_func_map *)entry;
	return &map->data;
4204 4205
}

4206 4207 4208 4209 4210 4211 4212 4213 4214 4215
/**
 * ftrace_func_mapper_add_ip - Map some data to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to map @data to
 * @data: The data to map to @ip
 *
 * Returns 0 on succes otherwise an error.
 */
int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
			      unsigned long ip, void *data)
4216
{
4217 4218
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
4219

4220 4221 4222
	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (entry)
		return -EBUSY;
4223

4224 4225 4226
	map = kmalloc(sizeof(*map), GFP_KERNEL);
	if (!map)
		return -ENOMEM;
4227

4228 4229
	map->entry.ip = ip;
	map->data = data;
4230

4231
	__add_hash_entry(&mapper->hash, &map->entry);
4232

4233 4234
	return 0;
}
4235

4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247
/**
 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to remove the data from
 *
 * Returns the data if it is found, otherwise NULL.
 * Note, if the data pointer is used as the data itself, (see 
 * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
 * if the data pointer was set to zero.
 */
void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
				   unsigned long ip)
4248
{
4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	void *data;

	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;

	map = (struct ftrace_func_map *)entry;
	data = map->data;

	remove_hash_entry(&mapper->hash, entry);
4261
	kfree(entry);
4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294

	return data;
}

/**
 * free_ftrace_func_mapper - free a mapping of ips and data
 * @mapper: The mapper that has the ip maps
 * @free_func: A function to be called on each data item.
 *
 * This is used to free the function mapper. The @free_func is optional
 * and can be used if the data needs to be freed as well.
 */
void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
			     ftrace_mapper_func free_func)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	struct hlist_head *hhd;
	int size = 1 << mapper->hash.size_bits;
	int i;

	if (free_func && mapper->hash.count) {
		for (i = 0; i < size; i++) {
			hhd = &mapper->hash.buckets[i];
			hlist_for_each_entry(entry, hhd, hlist) {
				map = (struct ftrace_func_map *)entry;
				free_func(map);
			}
		}
	}
	free_ftrace_hash(&mapper->hash);
}

4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307
static void release_probe(struct ftrace_func_probe *probe)
{
	struct ftrace_probe_ops *probe_ops;

	mutex_lock(&ftrace_lock);

	WARN_ON(probe->ref <= 0);

	/* Subtract the ref that was used to protect this instance */
	probe->ref--;

	if (!probe->ref) {
		probe_ops = probe->probe_ops;
4308 4309 4310 4311 4312 4313
		/*
		 * Sending zero as ip tells probe_ops to free
		 * the probe->data itself
		 */
		if (probe_ops->free)
			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326
		list_del(&probe->list);
		kfree(probe);
	}
	mutex_unlock(&ftrace_lock);
}

static void acquire_probe_locked(struct ftrace_func_probe *probe)
{
	/*
	 * Add one ref to keep it from being freed when releasing the
	 * ftrace_lock mutex.
	 */
	probe->ref++;
4327 4328 4329
}

int
4330
register_ftrace_function_probe(char *glob, struct trace_array *tr,
4331 4332
			       struct ftrace_probe_ops *probe_ops,
			       void *data)
4333
{
4334
	struct ftrace_func_entry *entry;
4335
	struct ftrace_func_probe *probe;
4336 4337
	struct ftrace_hash **orig_hash;
	struct ftrace_hash *old_hash;
4338
	struct ftrace_hash *hash;
4339
	int count = 0;
4340
	int size;
4341
	int ret;
4342
	int i;
4343

4344
	if (WARN_ON(!tr))
4345 4346
		return -EINVAL;

4347 4348
	/* We do not support '!' for function probes */
	if (WARN_ON(glob[0] == '!'))
4349 4350
		return -EINVAL;

4351

4352 4353 4354 4355 4356
	mutex_lock(&ftrace_lock);
	/* Check if the probe_ops is already registered */
	list_for_each_entry(probe, &tr->func_probes, list) {
		if (probe->probe_ops == probe_ops)
			break;
4357
	}
4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368
	if (&probe->list == &tr->func_probes) {
		probe = kzalloc(sizeof(*probe), GFP_KERNEL);
		if (!probe) {
			mutex_unlock(&ftrace_lock);
			return -ENOMEM;
		}
		probe->probe_ops = probe_ops;
		probe->ops.func = function_trace_probe_call;
		probe->tr = tr;
		ftrace_ops_init(&probe->ops);
		list_add(&probe->list, &tr->func_probes);
4369
	}
4370

4371
	acquire_probe_locked(probe);
4372

4373
	mutex_unlock(&ftrace_lock);
4374

4375
	mutex_lock(&probe->ops.func_hash->regex_lock);
4376

4377
	orig_hash = &probe->ops.func_hash->filter_hash;
4378 4379
	old_hash = *orig_hash;
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4380

4381
	ret = ftrace_match_records(hash, glob, strlen(glob));
4382

4383 4384 4385
	/* Nothing found? */
	if (!ret)
		ret = -EINVAL;
4386

4387 4388
	if (ret < 0)
		goto out;
4389

4390 4391 4392 4393
	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
			if (ftrace_lookup_ip(old_hash, entry->ip))
4394
				continue;
4395 4396 4397 4398 4399
			/*
			 * The caller might want to do something special
			 * for each function we find. We call the callback
			 * to give the caller an opportunity to do so.
			 */
4400 4401
			if (probe_ops->init) {
				ret = probe_ops->init(probe_ops, tr,
4402 4403 4404 4405 4406 4407 4408
						      entry->ip, data,
						      &probe->data);
				if (ret < 0) {
					if (probe_ops->free && count)
						probe_ops->free(probe_ops, tr,
								0, probe->data);
					probe->data = NULL;
4409
					goto out;
4410
				}
4411
			}
4412
			count++;
4413
		}
4414
	}
4415

4416
	mutex_lock(&ftrace_lock);
4417

4418 4419 4420 4421 4422
	if (!count) {
		/* Nothing was added? */
		ret = -EINVAL;
		goto out_unlock;
	}
4423

4424 4425
	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
					      hash, 1);
4426
	if (ret < 0)
4427
		goto err_unlock;
4428

4429 4430
	/* One ref for each new function traced */
	probe->ref += count;
4431

4432 4433
	if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
		ret = ftrace_startup(&probe->ops, 0);
4434

4435
 out_unlock:
4436
	mutex_unlock(&ftrace_lock);
4437

4438
	if (!ret)
4439
		ret = count;
4440
 out:
4441
	mutex_unlock(&probe->ops.func_hash->regex_lock);
4442
	free_ftrace_hash(hash);
4443

4444
	release_probe(probe);
4445

4446
	return ret;
4447

4448
 err_unlock:
4449
	if (!probe_ops->free || !count)
4450 4451 4452 4453 4454 4455 4456
		goto out_unlock;

	/* Failed to do the move, need to call the free functions */
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
			if (ftrace_lookup_ip(old_hash, entry->ip))
				continue;
4457
			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4458 4459 4460
		}
	}
	goto out_unlock;
4461 4462
}

4463
int
4464 4465
unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
				      struct ftrace_probe_ops *probe_ops)
4466
{
4467
	struct ftrace_ops_hash old_hash_ops;
4468
	struct ftrace_func_entry *entry;
4469
	struct ftrace_func_probe *probe;
4470
	struct ftrace_glob func_g;
4471 4472 4473
	struct ftrace_hash **orig_hash;
	struct ftrace_hash *old_hash;
	struct ftrace_hash *hash = NULL;
4474
	struct hlist_node *tmp;
4475
	struct hlist_head hhd;
4476
	char str[KSYM_SYMBOL_LEN];
4477 4478
	int count = 0;
	int i, ret = -ENODEV;
4479
	int size;
4480

4481
	if (!glob || !strlen(glob) || !strcmp(glob, "*"))
4482
		func_g.search = NULL;
4483
	else {
4484 4485
		int not;

4486 4487 4488 4489
		func_g.type = filter_parse_regex(glob, strlen(glob),
						 &func_g.search, &not);
		func_g.len = strlen(func_g.search);
		func_g.search = glob;
4490

S
Steven Rostedt 已提交
4491
		/* we do not support '!' for function probes */
4492
		if (WARN_ON(not))
4493
			return -EINVAL;
4494 4495
	}

4496 4497 4498 4499 4500
	mutex_lock(&ftrace_lock);
	/* Check if the probe_ops is already registered */
	list_for_each_entry(probe, &tr->func_probes, list) {
		if (probe->probe_ops == probe_ops)
			break;
4501
	}
4502 4503 4504 4505 4506 4507 4508 4509 4510 4511
	if (&probe->list == &tr->func_probes)
		goto err_unlock_ftrace;

	ret = -EINVAL;
	if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
		goto err_unlock_ftrace;

	acquire_probe_locked(probe);

	mutex_unlock(&ftrace_lock);
4512

4513
	mutex_lock(&probe->ops.func_hash->regex_lock);
4514

4515
	orig_hash = &probe->ops.func_hash->filter_hash;
4516 4517 4518 4519
	old_hash = *orig_hash;

	if (ftrace_hash_empty(old_hash))
		goto out_unlock;
4520

4521 4522 4523 4524
	old_hash_ops.filter_hash = old_hash;
	/* Probes only have filters */
	old_hash_ops.notrace_hash = NULL;

4525
	ret = -ENOMEM;
4526
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4527 4528 4529
	if (!hash)
		goto out_unlock;

4530
	INIT_HLIST_HEAD(&hhd);
4531

4532 4533 4534
	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
4535

4536
			if (func_g.search) {
4537 4538
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
4539
				if (!ftrace_match(str, &func_g))
4540 4541
					continue;
			}
4542
			count++;
4543 4544
			remove_hash_entry(hash, entry);
			hlist_add_head(&entry->hlist, &hhd);
4545 4546
		}
	}
4547 4548

	/* Nothing found? */
4549
	if (!count) {
4550 4551 4552 4553
		ret = -EINVAL;
		goto out_unlock;
	}

4554
	mutex_lock(&ftrace_lock);
4555

4556
	WARN_ON(probe->ref < count);
4557

4558
	probe->ref -= count;
4559

4560 4561 4562 4563
	if (ftrace_hash_empty(hash))
		ftrace_shutdown(&probe->ops, 0);

	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4564
					      hash, 1);
4565 4566

	/* still need to update the function call sites */
4567
	if (ftrace_enabled && !ftrace_hash_empty(hash))
4568
		ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
4569
				       &old_hash_ops);
4570
	synchronize_sched();
4571

4572 4573
	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
		hlist_del(&entry->hlist);
4574
		if (probe_ops->free)
4575
			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4576
		kfree(entry);
4577
	}
4578
	mutex_unlock(&ftrace_lock);
4579

4580
 out_unlock:
4581
	mutex_unlock(&probe->ops.func_hash->regex_lock);
4582
	free_ftrace_hash(hash);
4583

4584
	release_probe(probe);
4585

4586
	return ret;
4587

4588 4589
 err_unlock_ftrace:
	mutex_unlock(&ftrace_lock);
4590
	return ret;
4591 4592
}

4593 4594 4595 4596 4597 4598 4599 4600
void clear_ftrace_function_probes(struct trace_array *tr)
{
	struct ftrace_func_probe *probe, *n;

	list_for_each_entry_safe(probe, n, &tr->func_probes, list)
		unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
}

4601 4602 4603
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

4604 4605 4606 4607 4608
/*
 * Currently we only register ftrace commands from __init, so mark this
 * __init too.
 */
__init int register_ftrace_command(struct ftrace_func_command *cmd)
4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4627 4628 4629 4630 4631
/*
 * Currently we only unregister ftrace commands from __init, so mark
 * this __init too.
 */
__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4650
static int ftrace_process_regex(struct ftrace_iterator *iter,
4651
				char *buff, int len, int enable)
4652
{
4653
	struct ftrace_hash *hash = iter->hash;
4654
	struct trace_array *tr = iter->ops->private;
4655
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
4656
	struct ftrace_func_command *p;
4657
	int ret = -EINVAL;
4658 4659 4660 4661

	func = strsep(&next, ":");

	if (!next) {
4662
		ret = ftrace_match_records(hash, func, len);
4663 4664 4665 4666 4667
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
4668 4669
	}

4670
	/* command found */
4671 4672 4673

	command = strsep(&next, ":");

4674 4675 4676
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
4677
			ret = p->func(tr, hash, func, command, next, enable);
4678 4679
			goto out_unlock;
		}
4680
	}
4681 4682
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
4683

4684
	return ret;
4685 4686
}

I
Ingo Molnar 已提交
4687
static ssize_t
4688 4689
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
4690 4691
{
	struct ftrace_iterator *iter;
4692 4693
	struct trace_parser *parser;
	ssize_t ret, read;
4694

4695
	if (!cnt)
4696 4697 4698 4699 4700 4701 4702 4703
		return 0;

	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

4704
	if (unlikely(ftrace_disabled))
4705 4706 4707
		return -ENODEV;

	/* iter->hash is a local copy, so we don't need regex_lock */
4708

4709 4710
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
4711

4712
	if (read >= 0 && trace_parser_loaded(parser) &&
4713
	    !trace_parser_cont(parser)) {
4714
		ret = ftrace_process_regex(iter, parser->buffer,
4715
					   parser->idx, enable);
4716
		trace_parser_clear(parser);
4717
		if (ret < 0)
4718
			goto out;
4719
	}
4720 4721

	ret = read;
4722
 out:
4723 4724 4725
	return ret;
}

4726
ssize_t
4727 4728 4729 4730 4731 4732
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

4733
ssize_t
4734 4735 4736 4737 4738 4739
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

4740
static int
4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761
ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
	struct ftrace_func_entry *entry;

	if (!ftrace_location(ip))
		return -EINVAL;

	if (remove) {
		entry = ftrace_lookup_ip(hash, ip);
		if (!entry)
			return -ENOENT;
		free_hash_entry(hash, entry);
		return 0;
	}

	return add_hash_entry(hash, ip);
}

static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
		unsigned long ip, int remove, int reset, int enable)
4762
{
4763
	struct ftrace_hash **orig_hash;
4764
	struct ftrace_hash *hash;
4765
	int ret;
4766

4767
	if (unlikely(ftrace_disabled))
4768
		return -ENODEV;
4769

4770
	mutex_lock(&ops->func_hash->regex_lock);
4771

4772
	if (enable)
4773
		orig_hash = &ops->func_hash->filter_hash;
4774
	else
4775
		orig_hash = &ops->func_hash->notrace_hash;
4776

4777 4778 4779 4780 4781
	if (reset)
		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	else
		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);

4782 4783 4784 4785
	if (!hash) {
		ret = -ENOMEM;
		goto out_regex_unlock;
	}
4786

4787 4788 4789 4790
	if (buf && !ftrace_match_records(hash, buf, len)) {
		ret = -EINVAL;
		goto out_regex_unlock;
	}
4791 4792 4793 4794 4795
	if (ip) {
		ret = ftrace_match_addr(hash, ip, remove);
		if (ret < 0)
			goto out_regex_unlock;
	}
4796 4797

	mutex_lock(&ftrace_lock);
4798
	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
4799 4800
	mutex_unlock(&ftrace_lock);

4801
 out_regex_unlock:
4802
	mutex_unlock(&ops->func_hash->regex_lock);
4803 4804 4805

	free_ftrace_hash(hash);
	return ret;
4806 4807
}

4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
		int reset, int enable)
{
	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
}

/**
 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
 * @ops - the ops to set the filter with
 * @ip - the address to add to or remove from the filter.
 * @remove - non zero to remove the ip from the filter
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled
 * If @ip is NULL, it failes to update filter.
 */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
			 int remove, int reset)
{
4828
	ftrace_ops_init(ops);
4829 4830 4831 4832
	return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);

4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849
/**
 * ftrace_ops_set_global_filter - setup ops to use global filters
 * @ops - the ops which will use the global filters
 *
 * ftrace users who need global function trace filtering should call this.
 * It can set the global filter only if ops were not initialized before.
 */
void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
{
	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
		return;

	ftrace_ops_init(ops);
	ops->func_hash = &global_ops.local_hash;
}
EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);

4850 4851 4852 4853 4854 4855 4856
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
{
	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
}

4857 4858
/**
 * ftrace_set_filter - set a function to filter on in ftrace
4859 4860 4861 4862 4863 4864 4865 4866
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4867
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4868 4869
		       int len, int reset)
{
4870
	ftrace_ops_init(ops);
4871
	return ftrace_set_regex(ops, buf, len, reset, 1);
4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4886
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4887 4888
			int len, int reset)
{
4889
	ftrace_ops_init(ops);
4890
	return ftrace_set_regex(ops, buf, len, reset, 0);
4891 4892 4893
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
4894
 * ftrace_set_global_filter - set a function to filter on with global tracers
4895 4896 4897 4898 4899 4900 4901
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4902
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4903
{
4904
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4905
}
4906
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4907

4908
/**
4909
 * ftrace_set_global_notrace - set a function to not trace with global tracers
4910 4911 4912 4913 4914 4915 4916 4917
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4918
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4919
{
4920
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4921
}
4922
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4923

4924 4925 4926 4927 4928 4929 4930
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

4931 4932 4933
/* Used by function selftest to not test if filter is set */
bool ftrace_filter_param __initdata;

4934 4935
static int __init set_ftrace_notrace(char *str)
{
4936
	ftrace_filter_param = true;
4937
	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4938 4939 4940 4941 4942 4943
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
4944
	ftrace_filter_param = true;
4945
	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4946 4947 4948 4949
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

4950
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4951
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4952
static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4953
static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
4954

4955 4956 4957
static unsigned long save_global_trampoline;
static unsigned long save_global_flags;

4958 4959
static int __init set_graph_function(char *str)
{
4960
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4961 4962 4963 4964
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

4965 4966 4967 4968 4969 4970 4971
static int __init set_graph_notrace_function(char *str)
{
	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);

4972 4973 4974 4975 4976 4977 4978 4979
static int __init set_graph_max_depth_function(char *str)
{
	if (!str)
		return 0;
	fgraph_max_depth = simple_strtoul(str, NULL, 0);
	return 1;
}
__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
4980 4981

static void __init set_ftrace_early_graph(char *buf, int enable)
4982 4983 4984
{
	int ret;
	char *func;
4985
	struct ftrace_hash *hash;
4986

4987 4988 4989
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	if (WARN_ON(!hash))
		return;
4990 4991 4992 4993

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
4994
		ret = ftrace_graph_set_hash(hash, func);
4995 4996 4997 4998
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
4999 5000 5001 5002 5003

	if (enable)
		ftrace_graph_hash = hash;
	else
		ftrace_graph_notrace_hash = hash;
5004 5005 5006
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5007 5008
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
5009 5010 5011
{
	char *func;

5012 5013
	ftrace_ops_init(ops);

5014 5015
	while (buf) {
		func = strsep(&buf, ",");
5016
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
5017 5018 5019 5020 5021 5022
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
5023
		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
5024
	if (ftrace_notrace_buf[0])
5025
		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
5026 5027
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
5028 5029 5030
		set_ftrace_early_graph(ftrace_graph_buf, 1);
	if (ftrace_graph_notrace_buf[0])
		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
5031
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5032 5033
}

5034
int ftrace_regex_release(struct inode *inode, struct file *file)
5035 5036 5037
{
	struct seq_file *m = (struct seq_file *)file->private_data;
	struct ftrace_iterator *iter;
5038
	struct ftrace_hash **orig_hash;
5039
	struct trace_parser *parser;
5040
	int filter_hash;
5041
	int ret;
5042 5043 5044 5045 5046 5047 5048

	if (file->f_mode & FMODE_READ) {
		iter = m->private;
		seq_release(inode, file);
	} else
		iter = file->private_data;

5049 5050 5051
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
5052
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
5053 5054
	}

5055 5056
	trace_parser_put(parser);

5057
	mutex_lock(&iter->ops->func_hash->regex_lock);
5058

5059
	if (file->f_mode & FMODE_WRITE) {
5060 5061
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

5062
		if (filter_hash) {
5063
			orig_hash = &iter->ops->func_hash->filter_hash;
5064
			if (iter->tr && !list_empty(&iter->tr->mod_trace))
5065 5066
				iter->hash->flags |= FTRACE_HASH_FL_MOD;
		} else
5067
			orig_hash = &iter->ops->func_hash->notrace_hash;
5068

5069
		mutex_lock(&ftrace_lock);
5070 5071
		ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
						      iter->hash, filter_hash);
5072
		mutex_unlock(&ftrace_lock);
5073 5074 5075
	} else {
		/* For read only, the hash is the ops hash */
		iter->hash = NULL;
5076
	}
5077

5078
	mutex_unlock(&iter->ops->func_hash->regex_lock);
5079 5080
	free_ftrace_hash(iter->hash);
	kfree(iter);
5081

5082 5083 5084
	return 0;
}

5085
static const struct file_operations ftrace_avail_fops = {
5086 5087 5088
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
5089
	.release = seq_release_private,
5090 5091
};

5092 5093 5094 5095 5096 5097 5098
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

5099
static const struct file_operations ftrace_filter_fops = {
5100
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
5101
	.read = seq_read,
5102
	.write = ftrace_filter_write,
5103
	.llseek = tracing_lseek,
5104
	.release = ftrace_regex_release,
5105 5106
};

5107
static const struct file_operations ftrace_notrace_fops = {
5108
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
5109
	.read = seq_read,
5110
	.write = ftrace_notrace_write,
5111
	.llseek = tracing_lseek,
5112
	.release = ftrace_regex_release,
5113 5114
};

5115 5116 5117 5118
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

5119 5120 5121 5122 5123 5124 5125
struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH;
struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH;

enum graph_filter_type {
	GRAPH_FILTER_NOTRACE	= 0,
	GRAPH_FILTER_FUNCTION,
};
5126

5127 5128
#define FTRACE_GRAPH_EMPTY	((void *)1)

5129
struct ftrace_graph_data {
5130 5131 5132 5133 5134 5135 5136
	struct ftrace_hash		*hash;
	struct ftrace_func_entry	*entry;
	int				idx;   /* for hash table iteration */
	enum graph_filter_type		type;
	struct ftrace_hash		*new_hash;
	const struct seq_operations	*seq_ops;
	struct trace_parser		parser;
5137 5138
};

5139
static void *
5140
__g_next(struct seq_file *m, loff_t *pos)
5141
{
5142
	struct ftrace_graph_data *fgd = m->private;
5143 5144 5145
	struct ftrace_func_entry *entry = fgd->entry;
	struct hlist_head *head;
	int i, idx = fgd->idx;
5146

5147
	if (*pos >= fgd->hash->count)
5148
		return NULL;
5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167

	if (entry) {
		hlist_for_each_entry_continue(entry, hlist) {
			fgd->entry = entry;
			return entry;
		}

		idx++;
	}

	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
		head = &fgd->hash->buckets[i];
		hlist_for_each_entry(entry, head, hlist) {
			fgd->entry = entry;
			fgd->idx = i;
			return entry;
		}
	}
	return NULL;
5168
}
5169

5170 5171 5172 5173 5174
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
5175 5176 5177 5178
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
5179 5180
	struct ftrace_graph_data *fgd = m->private;

5181 5182
	mutex_lock(&graph_lock);

5183 5184 5185 5186 5187 5188 5189
	if (fgd->type == GRAPH_FILTER_FUNCTION)
		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
	else
		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));

5190
	/* Nothing, tell g_show to print all functions are enabled */
5191
	if (ftrace_hash_empty(fgd->hash) && !*pos)
5192
		return FTRACE_GRAPH_EMPTY;
5193

5194 5195
	fgd->idx = 0;
	fgd->entry = NULL;
5196
	return __g_next(m, pos);
5197 5198 5199 5200 5201 5202 5203 5204 5205
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
5206
	struct ftrace_func_entry *entry = v;
5207

5208
	if (!entry)
5209 5210
		return 0;

5211
	if (entry == FTRACE_GRAPH_EMPTY) {
5212 5213
		struct ftrace_graph_data *fgd = m->private;

5214
		if (fgd->type == GRAPH_FILTER_FUNCTION)
5215
			seq_puts(m, "#### all functions enabled ####\n");
5216
		else
5217
			seq_puts(m, "#### no functions disabled ####\n");
5218 5219 5220
		return 0;
	}

5221
	seq_printf(m, "%ps\n", (void *)entry->ip);
5222 5223 5224 5225

	return 0;
}

J
James Morris 已提交
5226
static const struct seq_operations ftrace_graph_seq_ops = {
5227 5228 5229 5230 5231 5232 5233
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
5234 5235
__ftrace_graph_open(struct inode *inode, struct file *file,
		    struct ftrace_graph_data *fgd)
5236 5237
{
	int ret = 0;
5238
	struct ftrace_hash *new_hash = NULL;
5239

5240 5241 5242
	if (file->f_mode & FMODE_WRITE) {
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

5243 5244 5245
		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
			return -ENOMEM;

5246 5247 5248 5249 5250 5251 5252 5253 5254
		if (file->f_flags & O_TRUNC)
			new_hash = alloc_ftrace_hash(size_bits);
		else
			new_hash = alloc_and_copy_ftrace_hash(size_bits,
							      fgd->hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}
5255 5256
	}

5257
	if (file->f_mode & FMODE_READ) {
5258
		ret = seq_open(file, &ftrace_graph_seq_ops);
5259 5260 5261
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = fgd;
5262 5263 5264 5265
		} else {
			/* Failed */
			free_ftrace_hash(new_hash);
			new_hash = NULL;
5266 5267 5268
		}
	} else
		file->private_data = fgd;
5269

5270
out:
5271 5272 5273
	if (ret < 0 && file->f_mode & FMODE_WRITE)
		trace_parser_put(&fgd->parser);

5274
	fgd->new_hash = new_hash;
5275 5276 5277 5278 5279 5280 5281 5282

	/*
	 * All uses of fgd->hash must be taken with the graph_lock
	 * held. The graph_lock is going to be released, so force
	 * fgd->hash to be reinitialized when it is taken again.
	 */
	fgd->hash = NULL;

5283 5284 5285
	return ret;
}

5286 5287 5288 5289
static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
5290
	int ret;
5291 5292 5293 5294 5295 5296 5297 5298

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

5299 5300
	mutex_lock(&graph_lock);

5301 5302
	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
5303
	fgd->type = GRAPH_FILTER_FUNCTION;
5304 5305
	fgd->seq_ops = &ftrace_graph_seq_ops;

5306 5307 5308 5309 5310 5311
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
5312 5313
}

5314 5315 5316 5317
static int
ftrace_graph_notrace_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
5318
	int ret;
5319 5320 5321 5322 5323 5324 5325 5326

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

5327 5328
	mutex_lock(&graph_lock);

5329 5330
	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
5331
	fgd->type = GRAPH_FILTER_NOTRACE;
5332 5333
	fgd->seq_ops = &ftrace_graph_seq_ops;

5334 5335 5336 5337 5338 5339
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
5340 5341
}

5342 5343 5344
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
5345
	struct ftrace_graph_data *fgd;
5346 5347 5348
	struct ftrace_hash *old_hash, *new_hash;
	struct trace_parser *parser;
	int ret = 0;
5349

5350 5351 5352
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;

5353
		fgd = m->private;
5354
		seq_release(inode, file);
5355
	} else {
5356
		fgd = file->private_data;
5357 5358
	}

5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398

	if (file->f_mode & FMODE_WRITE) {

		parser = &fgd->parser;

		if (trace_parser_loaded((parser))) {
			parser->buffer[parser->idx] = 0;
			ret = ftrace_graph_set_hash(fgd->new_hash,
						    parser->buffer);
		}

		trace_parser_put(parser);

		new_hash = __ftrace_hash_move(fgd->new_hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}

		mutex_lock(&graph_lock);

		if (fgd->type == GRAPH_FILTER_FUNCTION) {
			old_hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_hash, new_hash);
		} else {
			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
		}

		mutex_unlock(&graph_lock);

		/* Wait till all users are no longer using the old hash */
		synchronize_sched();

		free_ftrace_hash(old_hash);
	}

 out:
5399
	free_ftrace_hash(fgd->new_hash);
5400 5401
	kfree(fgd);

5402
	return ret;
5403 5404
}

5405
static int
5406
ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
5407
{
5408
	struct ftrace_glob func_g;
5409 5410
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
5411
	struct ftrace_func_entry *entry;
5412
	int fail = 1;
5413
	int not;
5414

5415
	/* decode regex */
5416 5417
	func_g.type = filter_parse_regex(buffer, strlen(buffer),
					 &func_g.search, &not);
5418

5419
	func_g.len = strlen(func_g.search);
5420

5421
	mutex_lock(&ftrace_lock);
5422 5423 5424 5425 5426 5427

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

5428 5429
	do_for_each_ftrace_rec(pg, rec) {

5430 5431 5432
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
5433
		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
5434
			entry = ftrace_lookup_ip(hash, rec->ip);
5435 5436 5437

			if (!not) {
				fail = 0;
5438 5439 5440 5441 5442

				if (entry)
					continue;
				if (add_hash_entry(hash, rec->ip) < 0)
					goto out;
5443
			} else {
5444 5445
				if (entry) {
					free_hash_entry(hash, entry);
5446 5447 5448
					fail = 0;
				}
			}
5449
		}
5450
	} while_for_each_ftrace_rec();
5451
out:
5452
	mutex_unlock(&ftrace_lock);
5453

5454 5455 5456 5457
	if (fail)
		return -EINVAL;

	return 0;
5458 5459 5460 5461 5462 5463
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5464
	ssize_t read, ret = 0;
5465
	struct ftrace_graph_data *fgd = file->private_data;
5466
	struct trace_parser *parser;
5467

5468
	if (!cnt)
5469 5470
		return 0;

5471 5472 5473 5474 5475 5476
	/* Read mode uses seq functions */
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		fgd = m->private;
	}

5477
	parser = &fgd->parser;
5478

5479
	read = trace_get_user(parser, ubuf, cnt, ppos);
5480

5481 5482
	if (read >= 0 && trace_parser_loaded(parser) &&
	    !trace_parser_cont(parser)) {
5483

5484
		ret = ftrace_graph_set_hash(fgd->new_hash,
5485 5486
					    parser->buffer);
		trace_parser_clear(parser);
5487 5488
	}

5489 5490
	if (!ret)
		ret = read;
5491

5492 5493 5494 5495
	return ret;
}

static const struct file_operations ftrace_graph_fops = {
5496 5497 5498
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5499
	.llseek		= tracing_lseek,
5500
	.release	= ftrace_graph_release,
5501
};
5502 5503 5504 5505 5506

static const struct file_operations ftrace_graph_notrace_fops = {
	.open		= ftrace_graph_notrace_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5507
	.llseek		= tracing_lseek,
5508 5509
	.release	= ftrace_graph_release,
};
5510 5511
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541
void ftrace_create_filter_files(struct ftrace_ops *ops,
				struct dentry *parent)
{

	trace_create_file("set_ftrace_filter", 0644, parent,
			  ops, &ftrace_filter_fops);

	trace_create_file("set_ftrace_notrace", 0644, parent,
			  ops, &ftrace_notrace_fops);
}

/*
 * The name "destroy_filter_files" is really a misnomer. Although
 * in the future, it may actualy delete the files, but this is
 * really intended to make sure the ops passed in are disabled
 * and that when this function returns, the caller is free to
 * free the ops.
 *
 * The "destroy" name is only to match the "create" name that this
 * should be paired with.
 */
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
	mutex_lock(&ftrace_lock);
	if (ops->flags & FTRACE_OPS_FL_ENABLED)
		ftrace_shutdown(ops, 0);
	ops->flags |= FTRACE_OPS_FL_DELETED;
	mutex_unlock(&ftrace_lock);
}

5542
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
5543 5544
{

5545 5546
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
5547

5548 5549 5550
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

5551
	ftrace_create_filter_files(&global_ops, d_tracer);
5552

5553
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5554
	trace_create_file("set_graph_function", 0444, d_tracer,
5555 5556
				    NULL,
				    &ftrace_graph_fops);
5557 5558 5559
	trace_create_file("set_graph_notrace", 0444, d_tracer,
				    NULL,
				    &ftrace_graph_notrace_fops);
5560 5561
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5562 5563 5564
	return 0;
}

5565
static int ftrace_cmp_ips(const void *a, const void *b)
5566
{
5567 5568
	const unsigned long *ipa = a;
	const unsigned long *ipb = b;
5569

5570 5571 5572 5573 5574 5575 5576
	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

5577
static int ftrace_process_locs(struct module *mod,
5578
			       unsigned long *start,
5579 5580
			       unsigned long *end)
{
5581
	struct ftrace_page *start_pg;
5582
	struct ftrace_page *pg;
5583
	struct dyn_ftrace *rec;
5584
	unsigned long count;
5585 5586
	unsigned long *p;
	unsigned long addr;
5587
	unsigned long flags = 0; /* Shut up gcc */
5588 5589 5590 5591 5592 5593 5594
	int ret = -ENOMEM;

	count = end - start;

	if (!count)
		return 0;

5595
	sort(start, count, sizeof(*start),
5596
	     ftrace_cmp_ips, NULL);
5597

5598 5599
	start_pg = ftrace_allocate_pages(count);
	if (!start_pg)
5600
		return -ENOMEM;
5601

S
Steven Rostedt 已提交
5602
	mutex_lock(&ftrace_lock);
5603

5604 5605 5606 5607 5608
	/*
	 * Core and each module needs their own pages, as
	 * modules will free them when they are removed.
	 * Force a new page to be allocated for modules.
	 */
5609 5610 5611
	if (!mod) {
		WARN_ON(ftrace_pages || ftrace_pages_start);
		/* First initialization */
5612
		ftrace_pages = ftrace_pages_start = start_pg;
5613
	} else {
5614
		if (!ftrace_pages)
5615
			goto out;
5616

5617 5618 5619 5620
		if (WARN_ON(ftrace_pages->next)) {
			/* Hmm, we have free pages? */
			while (ftrace_pages->next)
				ftrace_pages = ftrace_pages->next;
5621
		}
5622

5623
		ftrace_pages->next = start_pg;
5624 5625
	}

5626
	p = start;
5627
	pg = start_pg;
5628 5629
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
5630 5631 5632 5633 5634 5635 5636 5637
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
5638 5639 5640 5641 5642 5643 5644 5645 5646 5647

		if (pg->index == pg->size) {
			/* We should have allocated enough */
			if (WARN_ON(!pg->next))
				break;
			pg = pg->next;
		}

		rec = &pg->records[pg->index++];
		rec->ip = addr;
5648 5649
	}

5650 5651 5652 5653 5654 5655
	/* We should have used all pages */
	WARN_ON(pg->next);

	/* Assign the last page to ftrace_pages */
	ftrace_pages = pg;

5656
	/*
5657 5658 5659 5660 5661 5662
	 * We only need to disable interrupts on start up
	 * because we are modifying code that an interrupt
	 * may execute, and the modification is not atomic.
	 * But for modules, nothing runs the code we modify
	 * until we are finished with it, and there's no
	 * reason to cause large interrupt latencies while we do it.
5663
	 */
5664 5665
	if (!mod)
		local_irq_save(flags);
5666
	ftrace_update_code(mod, start_pg);
5667 5668
	if (!mod)
		local_irq_restore(flags);
5669 5670
	ret = 0;
 out:
S
Steven Rostedt 已提交
5671
	mutex_unlock(&ftrace_lock);
5672

5673
	return ret;
5674 5675
}

5676
#ifdef CONFIG_MODULES
5677 5678 5679

#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)

5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692
static int referenced_filters(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
		if (ops_references_rec(ops, rec))
		    cnt++;
	}

	return cnt;
}

5693
void ftrace_release_mod(struct module *mod)
5694 5695
{
	struct dyn_ftrace *rec;
5696
	struct ftrace_page **last_pg;
5697
	struct ftrace_page *pg;
5698
	int order;
5699

5700 5701
	mutex_lock(&ftrace_lock);

5702
	if (ftrace_disabled)
5703
		goto out_unlock;
5704

5705 5706 5707 5708 5709 5710 5711
	/*
	 * Each module has its own ftrace_pages, remove
	 * them from the list.
	 */
	last_pg = &ftrace_pages_start;
	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
		rec = &pg->records[0];
5712
		if (within_module_core(rec->ip, mod)) {
5713
			/*
5714 5715
			 * As core pages are first, the first
			 * page should never be a module page.
5716
			 */
5717 5718 5719 5720 5721 5722 5723
			if (WARN_ON(pg == ftrace_pages_start))
				goto out_unlock;

			/* Check if we are deleting the last page */
			if (pg == ftrace_pages)
				ftrace_pages = next_to_ftrace_page(last_pg);

5724
			ftrace_update_tot_cnt -= pg->index;
5725
			*last_pg = pg->next;
5726 5727 5728
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
5729 5730 5731
		} else
			last_pg = &pg->next;
	}
5732
 out_unlock:
5733 5734 5735
	mutex_unlock(&ftrace_lock);
}

5736
void ftrace_module_enable(struct module *mod)
5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	mutex_lock(&ftrace_lock);

	if (ftrace_disabled)
		goto out_unlock;

	/*
	 * If the tracing is enabled, go ahead and enable the record.
	 *
	 * The reason not to enable the record immediatelly is the
	 * inherent check of ftrace_make_nop/ftrace_make_call for
	 * correct previous instructions.  Making first the NOP
	 * conversion puts the module to the correct state, thus
	 * passing the ftrace_make_call check.
	 *
	 * We also delay this to after the module code already set the
	 * text to read-only, as we now need to set it back to read-write
	 * so that we can modify the text.
	 */
	if (ftrace_start_up)
		ftrace_arch_code_modify_prepare();

	do_for_each_ftrace_rec(pg, rec) {
		int cnt;
		/*
		 * do_for_each_ftrace_rec() is a double loop.
		 * module text shares the pg. If a record is
		 * not part of this module, then skip this pg,
		 * which the "break" will do.
		 */
		if (!within_module_core(rec->ip, mod))
			break;

		cnt = 0;

		/*
		 * When adding a module, we need to check if tracers are
		 * currently enabled and if they are, and can trace this record,
		 * we need to enable the module functions as well as update the
		 * reference counts for those function records.
		 */
		if (ftrace_start_up)
			cnt += referenced_filters(rec);

		/* This clears FTRACE_FL_DISABLED */
		rec->flags = cnt;

		if (ftrace_start_up && cnt) {
			int failed = __ftrace_replace_code(rec, 1);
			if (failed) {
				ftrace_bug(failed, rec);
				goto out_loop;
			}
		}

	} while_for_each_ftrace_rec();

 out_loop:
	if (ftrace_start_up)
		ftrace_arch_code_modify_post_process();

 out_unlock:
	mutex_unlock(&ftrace_lock);
5803 5804

	process_cached_mods(mod->name);
5805 5806
}

5807
void ftrace_module_init(struct module *mod)
5808
{
5809
	if (ftrace_disabled || !mod->num_ftrace_callsites)
5810
		return;
5811

5812 5813
	ftrace_process_locs(mod, mod->ftrace_callsites,
			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5814
}
5815 5816
#endif /* CONFIG_MODULES */

5817
void __init ftrace_free_init_mem(void)
5818
{
5819 5820
	unsigned long start = (unsigned long)(&__init_begin);
	unsigned long end = (unsigned long)(&__init_end);
5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842
	struct ftrace_page **last_pg = &ftrace_pages_start;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	struct dyn_ftrace key;
	int order;

	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */

	mutex_lock(&ftrace_lock);

	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
			continue;
 again:
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (!rec)
			continue;
		pg->index--;
5843
		ftrace_update_tot_cnt--;
5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861
		if (!pg->index) {
			*last_pg = pg->next;
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
			pg = container_of(last_pg, struct ftrace_page, next);
			if (!(*last_pg))
				ftrace_pages = pg;
			continue;
		}
		memmove(rec, rec + 1,
			(pg->index - (rec - pg->records)) * sizeof(*rec));
		/* More than one function may be in this block */
		goto again;
	}
	mutex_unlock(&ftrace_lock);
}

5862 5863
void __init ftrace_init(void)
{
5864 5865
	extern unsigned long __start_mcount_loc[];
	extern unsigned long __stop_mcount_loc[];
5866
	unsigned long count, flags;
5867 5868 5869
	int ret;

	local_irq_save(flags);
5870
	ret = ftrace_dyn_arch_init();
5871
	local_irq_restore(flags);
5872
	if (ret)
5873 5874 5875
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;
5876 5877
	if (!count) {
		pr_info("ftrace: No functions to be traced?\n");
5878
		goto failed;
5879 5880 5881 5882
	}

	pr_info("ftrace: allocating %ld entries in %ld pages\n",
		count, count / ENTRIES_PER_PAGE + 1);
5883 5884 5885

	last_ftrace_enabled = ftrace_enabled = 1;

5886
	ret = ftrace_process_locs(NULL,
5887
				  __start_mcount_loc,
5888 5889
				  __stop_mcount_loc);

5890 5891
	set_ftrace_early_filters();

5892 5893 5894 5895 5896
	return;
 failed:
	ftrace_disabled = 1;
}

5897 5898 5899 5900 5901 5902 5903 5904 5905 5906
/* Do nothing if arch does not support this */
void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
	arch_ftrace_update_trampoline(ops);
}

5907 5908 5909
void ftrace_init_trace_array(struct trace_array *tr)
{
	INIT_LIST_HEAD(&tr->func_probes);
5910 5911
	INIT_LIST_HEAD(&tr->mod_trace);
	INIT_LIST_HEAD(&tr->mod_notrace);
5912
}
5913
#else
5914

5915
static struct ftrace_ops global_ops = {
5916
	.func			= ftrace_stub,
5917 5918 5919
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				  FTRACE_OPS_FL_INITIALIZED |
				  FTRACE_OPS_FL_PID,
5920 5921
};

5922 5923 5924 5925 5926
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
5927
core_initcall(ftrace_nodyn_init);
5928

5929
static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
5930
static inline void ftrace_startup_enable(int command) { }
5931
static inline void ftrace_startup_all(int command) { }
5932
/* Keep as macros so we do not need to define the commands */
5933 5934 5935 5936 5937 5938
# define ftrace_startup(ops, command)					\
	({								\
		int ___ret = __register_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
5939
	})
5940 5941 5942 5943 5944 5945 5946
# define ftrace_shutdown(ops, command)					\
	({								\
		int ___ret = __unregister_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
	})
5947

I
Ingo Molnar 已提交
5948 5949
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
5950 5951

static inline int
5952
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
5953 5954 5955 5956
{
	return 1;
}

5957 5958 5959 5960
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

5961 5962
#endif /* CONFIG_DYNAMIC_FTRACE */

5963 5964 5965 5966
__init void ftrace_init_global_array_ops(struct trace_array *tr)
{
	tr->ops = &global_ops;
	tr->ops->private = tr;
5967
	ftrace_init_trace_array(tr);
5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986
}

void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
{
	/* If we filter on pids, update to use the pid function */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		if (WARN_ON(tr->ops->func != ftrace_stub))
			printk("ftrace ops had %pS for function\n",
			       tr->ops->func);
	}
	tr->ops->func = func;
	tr->ops->private = tr;
}

void ftrace_reset_array_ops(struct trace_array *tr)
{
	tr->ops->func = ftrace_stub;
}

5987 5988
static inline void
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5989
		       struct ftrace_ops *ignored, struct pt_regs *regs)
5990
{
5991
	struct ftrace_ops *op;
5992
	int bit;
5993

5994 5995 5996
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;
5997

5998 5999 6000 6001 6002
	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
6003

6004
	do_for_each_ftrace_op(op, ftrace_ops_list) {
6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018
		/*
		 * Check the following for each ops before calling their func:
		 *  if RCU flag is set, then rcu_is_watching() must be true
		 *  if PER_CPU is set, then ftrace_function_local_disable()
		 *                          must be false
		 *  Otherwise test if the ip matches the ops filter
		 *
		 * If any of the above fails then the op->func() is not executed.
		 */
		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
		    (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
		     !ftrace_function_local_disabled(op)) &&
		    ftrace_ops_test(op, ip, regs)) {
		    
6019 6020
			if (FTRACE_WARN_ON(!op->func)) {
				pr_warn("op=%p %pS\n", op, op);
6021 6022
				goto out;
			}
6023
			op->func(ip, parent_ip, op, regs);
6024
		}
6025
	} while_for_each_ftrace_op(op);
6026
out:
6027
	preempt_enable_notrace();
6028
	trace_clear_recursion(bit);
6029 6030
}

6031 6032 6033 6034 6035
/*
 * Some archs only support passing ip and parent_ip. Even though
 * the list function ignores the op parameter, we do not want any
 * C side effects, where a function is called without the caller
 * sending a third parameter.
6036 6037 6038
 * Archs are to support both the regs and ftrace_ops at the same time.
 * If they support ftrace_ops, it is assumed they support regs.
 * If call backs want to use regs, they must either check for regs
6039 6040
 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
6041
 * An architecture can pass partial regs with ftrace_ops and still
L
Li Bin 已提交
6042
 * set the ARCH_SUPPORTS_FTRACE_OPS.
6043 6044 6045
 */
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
6046
				 struct ftrace_ops *op, struct pt_regs *regs)
6047
{
6048
	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
6049 6050 6051 6052
}
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
{
6053
	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
6054 6055 6056
}
#endif

6057 6058
/*
 * If there's only one function registered but it does not support
6059 6060
 * recursion, needs RCU protection and/or requires per cpu handling, then
 * this function will be called by the mcount trampoline.
6061
 */
6062
static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
6063 6064 6065 6066
				   struct ftrace_ops *op, struct pt_regs *regs)
{
	int bit;

6067 6068 6069
	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
		return;

6070 6071 6072 6073
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;

6074
	preempt_disable_notrace();
6075

6076 6077 6078 6079 6080 6081
	if (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
	    !ftrace_function_local_disabled(op)) {
		op->func(ip, parent_ip, op, regs);
	}

	preempt_enable_notrace();
6082 6083 6084
	trace_clear_recursion(bit);
}

6085 6086 6087 6088 6089 6090 6091
/**
 * ftrace_ops_get_func - get the function a trampoline should call
 * @ops: the ops to get the function for
 *
 * Normally the mcount trampoline will call the ops->func, but there
 * are times that it should not. For example, if the ops does not
 * have its own recursion protection, then it should call the
6092
 * ftrace_ops_assist_func() instead.
6093 6094 6095 6096 6097 6098
 *
 * Returns the function that the trampoline should call for @ops.
 */
ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
{
	/*
6099 6100
	 * If the function does not handle recursion, needs to be RCU safe,
	 * or does per cpu logic, then we need to call the assist handler.
6101
	 */
6102 6103 6104
	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
	    ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU))
		return ftrace_ops_assist_func;
6105 6106 6107 6108

	return ops->func;
}

6109 6110 6111
static void
ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
		    struct task_struct *prev, struct task_struct *next)
S
Steven Rostedt 已提交
6112
{
6113 6114
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;
S
Steven Rostedt 已提交
6115

6116
	pid_list = rcu_dereference_sched(tr->function_pids);
6117

6118 6119
	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, next));
S
Steven Rostedt 已提交
6120 6121
}

6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158
static void
ftrace_pid_follow_sched_process_fork(void *data,
				     struct task_struct *self,
				     struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, self, task);
}

static void
ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, NULL, task);
}

void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
{
	if (enable) {
		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						  tr);
		register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						  tr);
	} else {
		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						    tr);
		unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						    tr);
	}
}

6159
static void clear_ftrace_pids(struct trace_array *tr)
6160
{
6161 6162
	struct trace_pid_list *pid_list;
	int cpu;
6163

6164 6165 6166 6167
	pid_list = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));
	if (!pid_list)
		return;
6168

6169
	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
6170

6171 6172
	for_each_possible_cpu(cpu)
		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
S
Steven Rostedt 已提交
6173

6174
	rcu_assign_pointer(tr->function_pids, NULL);
S
Steven Rostedt 已提交
6175

6176 6177
	/* Wait till all users are no longer using pid filtering */
	synchronize_sched();
6178

6179
	trace_free_pid_list(pid_list);
6180 6181
}

6182 6183 6184 6185 6186 6187 6188 6189 6190
void ftrace_clear_pids(struct trace_array *tr)
{
	mutex_lock(&ftrace_lock);

	clear_ftrace_pids(tr);

	mutex_unlock(&ftrace_lock);
}

6191
static void ftrace_pid_reset(struct trace_array *tr)
6192
{
6193
	mutex_lock(&ftrace_lock);
6194
	clear_ftrace_pids(tr);
S
Steven Rostedt 已提交
6195

6196
	ftrace_update_pid_func();
6197
	ftrace_startup_all(0);
6198 6199 6200 6201

	mutex_unlock(&ftrace_lock);
}

6202 6203
/* Greater than any max PID */
#define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
6204

6205
static void *fpid_start(struct seq_file *m, loff_t *pos)
6206
	__acquires(RCU)
6207
{
6208 6209 6210
	struct trace_pid_list *pid_list;
	struct trace_array *tr = m->private;

6211
	mutex_lock(&ftrace_lock);
6212 6213 6214
	rcu_read_lock_sched();

	pid_list = rcu_dereference_sched(tr->function_pids);
6215

6216 6217
	if (!pid_list)
		return !(*pos) ? FTRACE_NO_PIDS : NULL;
6218

6219
	return trace_pid_start(pid_list, pos);
6220 6221 6222 6223
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
6224 6225 6226 6227
	struct trace_array *tr = m->private;
	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);

	if (v == FTRACE_NO_PIDS)
6228 6229
		return NULL;

6230
	return trace_pid_next(pid_list, v, pos);
6231 6232 6233
}

static void fpid_stop(struct seq_file *m, void *p)
6234
	__releases(RCU)
6235
{
6236
	rcu_read_unlock_sched();
6237 6238 6239 6240 6241
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
6242
	if (v == FTRACE_NO_PIDS) {
6243
		seq_puts(m, "no pid\n");
6244 6245 6246
		return 0;
	}

6247
	return trace_pid_show(m, v);
6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
6260 6261
	struct trace_array *tr = inode->i_private;
	struct seq_file *m;
6262 6263
	int ret = 0;

6264 6265 6266
	if (trace_array_get(tr) < 0)
		return -ENODEV;

6267 6268
	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
6269
		ftrace_pid_reset(tr);
6270

6271 6272 6273 6274 6275 6276 6277 6278
	ret = seq_open(file, &ftrace_pid_sops);
	if (ret < 0) {
		trace_array_put(tr);
	} else {
		m = file->private_data;
		/* copy tr over to seq ops */
		m->private = tr;
	}
6279 6280 6281 6282

	return ret;
}

6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298
static void ignore_task_cpu(void *data)
{
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;

	/*
	 * This function is called by on_each_cpu() while the
	 * event_mutex is held.
	 */
	pid_list = rcu_dereference_protected(tr->function_pids,
					     mutex_is_locked(&ftrace_lock));

	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, current));
}

6299 6300 6301 6302
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
6303 6304 6305 6306 6307
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
	struct trace_pid_list *filtered_pids = NULL;
	struct trace_pid_list *pid_list;
	ssize_t ret;
6308

6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319
	if (!cnt)
		return 0;

	mutex_lock(&ftrace_lock);

	filtered_pids = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));

	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
	if (ret < 0)
		goto out;
6320

6321
	rcu_assign_pointer(tr->function_pids, pid_list);
6322

6323 6324 6325 6326 6327 6328 6329
	if (filtered_pids) {
		synchronize_sched();
		trace_free_pid_list(filtered_pids);
	} else if (pid_list) {
		/* Register a probe to set whether to ignore the tracing of a task */
		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
	}
6330

6331
	/*
6332 6333 6334
	 * Ignoring of pids is done at task switch. But we have to
	 * check for those tasks that are currently running.
	 * Always do this in case a pid was appended or removed.
6335
	 */
6336
	on_each_cpu(ignore_task_cpu, tr, 1);
6337

6338 6339 6340 6341
	ftrace_update_pid_func();
	ftrace_startup_all(0);
 out:
	mutex_unlock(&ftrace_lock);
6342

6343 6344
	if (ret > 0)
		*ppos += ret;
6345

6346
	return ret;
6347
}
6348

6349 6350 6351
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
6352
	struct trace_array *tr = inode->i_private;
6353

6354 6355 6356
	trace_array_put(tr);

	return seq_release(inode, file);
6357 6358
}

6359
static const struct file_operations ftrace_pid_fops = {
6360 6361 6362
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
6363
	.llseek		= tracing_lseek,
6364
	.release	= ftrace_pid_release,
6365 6366
};

6367
void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6368
{
6369
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
6370
			    tr, &ftrace_pid_fops);
6371 6372
}

6373 6374 6375 6376 6377 6378 6379 6380 6381 6382
void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
					 struct dentry *d_tracer)
{
	/* Only the top level directory has the dyn_tracefs and profile */
	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));

	ftrace_init_dyn_tracefs(d_tracer);
	ftrace_profile_tracefs(d_tracer);
}

S
Steven Rostedt 已提交
6383
/**
6384
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
6385 6386 6387 6388 6389
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
6390
void ftrace_kill(void)
S
Steven Rostedt 已提交
6391 6392 6393 6394 6395 6396
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

6397 6398 6399 6400 6401 6402 6403 6404
/**
 * Test if ftrace is dead or not.
 */
int ftrace_is_dead(void)
{
	return ftrace_disabled;
}

6405
/**
6406 6407
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
6408
 *
6409 6410 6411 6412 6413 6414
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
6415
 */
6416
int register_ftrace_function(struct ftrace_ops *ops)
6417
{
6418
	int ret = -1;
6419

6420 6421
	ftrace_ops_init(ops);

S
Steven Rostedt 已提交
6422
	mutex_lock(&ftrace_lock);
6423

6424
	ret = ftrace_startup(ops, 0);
6425

S
Steven Rostedt 已提交
6426
	mutex_unlock(&ftrace_lock);
6427

6428
	return ret;
6429
}
6430
EXPORT_SYMBOL_GPL(register_ftrace_function);
6431 6432

/**
6433
 * unregister_ftrace_function - unregister a function for profiling.
6434 6435 6436 6437 6438 6439 6440 6441
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
6442
	mutex_lock(&ftrace_lock);
6443
	ret = ftrace_shutdown(ops, 0);
S
Steven Rostedt 已提交
6444
	mutex_unlock(&ftrace_lock);
6445 6446 6447

	return ret;
}
6448
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
6449

I
Ingo Molnar 已提交
6450
int
6451
ftrace_enable_sysctl(struct ctl_table *table, int write,
6452
		     void __user *buffer, size_t *lenp,
6453 6454
		     loff_t *ppos)
{
6455
	int ret = -ENODEV;
6456

S
Steven Rostedt 已提交
6457
	mutex_lock(&ftrace_lock);
6458

6459 6460 6461 6462
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
6463

6464
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
6465 6466
		goto out;

6467
	last_ftrace_enabled = !!ftrace_enabled;
6468 6469 6470 6471

	if (ftrace_enabled) {

		/* we are starting ftrace again */
6472 6473
		if (rcu_dereference_protected(ftrace_ops_list,
			lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
6474
			update_ftrace_function();
6475

6476 6477
		ftrace_startup_sysctl();

6478 6479 6480 6481 6482 6483 6484 6485
	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
6486
	mutex_unlock(&ftrace_lock);
6487
	return ret;
6488
}
6489

6490
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
6491

6492 6493 6494 6495
static struct ftrace_ops graph_ops = {
	.func			= ftrace_stub,
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				   FTRACE_OPS_FL_INITIALIZED |
6496
				   FTRACE_OPS_FL_PID |
6497 6498 6499
				   FTRACE_OPS_FL_STUB,
#ifdef FTRACE_GRAPH_TRAMP_ADDR
	.trampoline		= FTRACE_GRAPH_TRAMP_ADDR,
6500
	/* trampoline_size is only needed for dynamically allocated tramps */
6501 6502 6503 6504
#endif
	ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
};

6505 6506 6507 6508 6509 6510 6511 6512 6513 6514
void ftrace_graph_sleep_time_control(bool enable)
{
	fgraph_sleep_time = enable;
}

void ftrace_graph_graph_time_control(bool enable)
{
	fgraph_graph_time = enable;
}

6515 6516 6517 6518 6519
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

6520 6521 6522
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
6523
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
6524
static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

6546
	read_lock(&tasklist_lock);
6547 6548 6549 6550 6551 6552 6553
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
6554
			atomic_set(&t->tracing_graph_pause, 0);
6555
			atomic_set(&t->trace_overrun, 0);
6556 6557 6558 6559
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
6560 6561 6562 6563
		}
	} while_each_thread(g, t);

unlock:
6564
	read_unlock(&tasklist_lock);
6565 6566 6567 6568 6569 6570
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

6571
static void
6572
ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
6573
			struct task_struct *prev, struct task_struct *next)
6574 6575 6576 6577
{
	unsigned long long timestamp;
	int index;

6578 6579 6580 6581
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
6582
	if (fgraph_sleep_time)
6583 6584
		return;

6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

6603
/* Allocate a return stack for each task */
6604
static int start_graph_tracing(void)
6605 6606
{
	struct ftrace_ret_stack **ret_stack_list;
6607
	int ret, cpu;
6608 6609 6610 6611 6612 6613 6614 6615

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

6616
	/* The cpu_boot init_task->ret_stack will never be freed */
6617 6618
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
6619
			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
6620
	}
6621

6622 6623 6624 6625
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

6626
	if (!ret) {
6627
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6628 6629 6630 6631 6632
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

6633 6634 6635 6636
	kfree(ret_stack_list);
	return ret;
}

6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673
static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
{
	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
		return 0;
	return __ftrace_graph_entry(trace);
}

/*
 * The function graph tracer should only trace the functions defined
 * by set_ftrace_filter and set_ftrace_notrace. If another function
 * tracer ops is registered, the graph tracer requires testing the
 * function against the global ops, and not just trace any function
 * that any ftrace_ops registered.
 */
static void update_function_graph_func(void)
{
6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692
	struct ftrace_ops *op;
	bool do_test = false;

	/*
	 * The graph and global ops share the same set of functions
	 * to test. If any other ops is on the list, then
	 * the graph tracing needs to test if its the function
	 * it should call.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op != &global_ops && op != &graph_ops &&
		    op != &ftrace_list_end) {
			do_test = true;
			/* in double loop, break out with goto */
			goto out;
		}
	} while_for_each_ftrace_op(op);
 out:
	if (do_test)
6693
		ftrace_graph_entry = ftrace_graph_entry_test;
6694 6695
	else
		ftrace_graph_entry = __ftrace_graph_entry;
6696 6697
}

6698 6699 6700 6701
static struct notifier_block ftrace_suspend_notifier = {
	.notifier_call = ftrace_suspend_notifier_call,
};

6702 6703
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
6704
{
6705 6706
	int ret = 0;

S
Steven Rostedt 已提交
6707
	mutex_lock(&ftrace_lock);
6708

6709
	/* we currently allow only one tracer registered at a time */
6710
	if (ftrace_graph_active) {
6711 6712 6713 6714
		ret = -EBUSY;
		goto out;
	}

6715 6716
	register_pm_notifier(&ftrace_suspend_notifier);

6717
	ftrace_graph_active++;
6718
	ret = start_graph_tracing();
6719
	if (ret) {
6720
		ftrace_graph_active--;
6721 6722
		goto out;
	}
6723

6724
	ftrace_graph_return = retfunc;
6725 6726 6727 6728 6729 6730 6731 6732 6733 6734

	/*
	 * Update the indirect function to the entryfunc, and the
	 * function that gets called to the entry_test first. Then
	 * call the update fgraph entry function to determine if
	 * the entryfunc should be called directly or not.
	 */
	__ftrace_graph_entry = entryfunc;
	ftrace_graph_entry = ftrace_graph_entry_test;
	update_function_graph_func();
6735

6736
	ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
6737
out:
S
Steven Rostedt 已提交
6738
	mutex_unlock(&ftrace_lock);
6739
	return ret;
6740 6741
}

6742
void unregister_ftrace_graph(void)
6743
{
S
Steven Rostedt 已提交
6744
	mutex_lock(&ftrace_lock);
6745

6746
	if (unlikely(!ftrace_graph_active))
6747 6748
		goto out;

6749
	ftrace_graph_active--;
6750
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
6751
	ftrace_graph_entry = ftrace_graph_entry_stub;
6752
	__ftrace_graph_entry = ftrace_graph_entry_stub;
6753
	ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
6754
	unregister_pm_notifier(&ftrace_suspend_notifier);
6755
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6756

6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767
#ifdef CONFIG_DYNAMIC_FTRACE
	/*
	 * Function graph does not allocate the trampoline, but
	 * other global_ops do. We need to reset the ALLOC_TRAMP flag
	 * if one was used.
	 */
	global_ops.trampoline = save_global_trampoline;
	if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP)
		global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
#endif

6768
 out:
S
Steven Rostedt 已提交
6769
	mutex_unlock(&ftrace_lock);
6770
}
6771

6772 6773 6774 6775 6776 6777 6778 6779
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);

static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
	atomic_set(&t->tracing_graph_pause, 0);
	atomic_set(&t->trace_overrun, 0);
	t->ftrace_timestamp = 0;
L
Lucas De Marchi 已提交
6780
	/* make curr_ret_stack visible before we add the ret_stack */
6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814
	smp_wmb();
	t->ret_stack = ret_stack;
}

/*
 * Allocate a return stack for the idle task. May be the first
 * time through, or it may be done by CPU hotplug online.
 */
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
{
	t->curr_ret_stack = -1;
	/*
	 * The idle task has no parent, it either has its own
	 * stack or no stack at all.
	 */
	if (t->ret_stack)
		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));

	if (ftrace_graph_active) {
		struct ftrace_ret_stack *ret_stack;

		ret_stack = per_cpu(idle_ret_stack, cpu);
		if (!ret_stack) {
			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
					    * sizeof(struct ftrace_ret_stack),
					    GFP_KERNEL);
			if (!ret_stack)
				return;
			per_cpu(idle_ret_stack, cpu) = ret_stack;
		}
		graph_init_task(t, ret_stack);
	}
}

6815
/* Allocate a return stack for newly created task */
6816
void ftrace_graph_init_task(struct task_struct *t)
6817
{
6818 6819
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;
6820
	t->curr_ret_stack = -1;
6821

6822
	if (ftrace_graph_active) {
6823 6824 6825
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
6826 6827
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
6828
		if (!ret_stack)
6829
			return;
6830
		graph_init_task(t, ret_stack);
6831
	}
6832 6833
}

6834
void ftrace_graph_exit_task(struct task_struct *t)
6835
{
6836 6837
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

6838
	t->ret_stack = NULL;
6839 6840 6841 6842
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
6843
}
6844
#endif