ftrace.c 106.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
13
 *  Copyright (C) 2004 Nadia Yvette Chambers
14 15
 */

16 17 18
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
#include <linux/kallsyms.h>
19
#include <linux/seq_file.h>
20
#include <linux/suspend.h>
21
#include <linux/debugfs.h>
22
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
23
#include <linux/kthread.h>
24
#include <linux/uaccess.h>
25
#include <linux/bsearch.h>
26
#include <linux/module.h>
I
Ingo Molnar 已提交
27
#include <linux/ftrace.h>
28
#include <linux/sysctl.h>
29
#include <linux/slab.h>
30
#include <linux/ctype.h>
31
#include <linux/sort.h>
32
#include <linux/list.h>
33
#include <linux/hash.h>
34
#include <linux/rcupdate.h>
35

36
#include <trace/events/sched.h>
37

38
#include <asm/setup.h>
39

40
#include "trace_output.h"
S
Steven Rostedt 已提交
41
#include "trace_stat.h"
42

43
#define FTRACE_WARN_ON(cond)			\
44 45 46
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
47
			ftrace_kill();		\
48 49
		___r;				\
	})
50 51

#define FTRACE_WARN_ON_ONCE(cond)		\
52 53 54
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
55
			ftrace_kill();		\
56 57
		___r;				\
	})
58

59 60 61
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
62 63
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
64

65 66
#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)

67 68
static struct ftrace_ops ftrace_list_end __read_mostly = {
	.func		= ftrace_stub,
69
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE,
70 71
};

72 73
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
74
static int last_ftrace_enabled;
75

76
/* Quick disabling of function tracer. */
77 78 79 80
int function_trace_stop __read_mostly;

/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
81

82 83 84 85 86 87 88
/* List for set_ftrace_pid's pids. */
LIST_HEAD(ftrace_pids);
struct ftrace_pid {
	struct list_head list;
	struct pid *pid;
};

89 90 91 92 93 94
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

95
static DEFINE_MUTEX(ftrace_lock);
96

97
static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
98
static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
99
static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
100
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
101
ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
102
static struct ftrace_ops global_ops;
103
static struct ftrace_ops control_ops;
104

105 106
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
107
				 struct ftrace_ops *op, struct pt_regs *regs);
108 109 110 111 112
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
113

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
/*
 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
 * can use rcu_dereference_raw() is that elements removed from this list
 * are simply leaked, so there is no need to interact with a grace-period
 * mechanism.  The rcu_dereference_raw() calls are needed to handle
 * concurrent insertions into the ftrace_global_list.
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
#define do_for_each_ftrace_op(op, list)			\
	op = rcu_dereference_raw(list);			\
	do

/*
 * Optimized for just a single item in the list (as that is the normal case).
 */
#define while_for_each_ftrace_op(op)				\
	while (likely(op = rcu_dereference_raw((op)->next)) &&	\
	       unlikely((op) != &ftrace_list_end))

134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
/**
 * ftrace_nr_registered_ops - return number of ops registered
 *
 * Returns the number of ftrace_ops registered and tracing functions
 */
int ftrace_nr_registered_ops(void)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	mutex_lock(&ftrace_lock);

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next)
		cnt++;

	mutex_unlock(&ftrace_lock);

	return cnt;
}

155 156
static void
ftrace_global_list_func(unsigned long ip, unsigned long parent_ip,
157
			struct ftrace_ops *op, struct pt_regs *regs)
158
{
159 160
	int bit;

161 162
	bit = trace_test_and_set_recursion(TRACE_GLOBAL_START, TRACE_GLOBAL_MAX);
	if (bit < 0)
163
		return;
164

165
	do_for_each_ftrace_op(op, ftrace_global_list) {
166
		op->func(ip, parent_ip, op, regs);
167
	} while_for_each_ftrace_op(op);
168 169

	trace_clear_recursion(bit);
170 171
}

172
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
173
			    struct ftrace_ops *op, struct pt_regs *regs)
174
{
175
	if (!test_tsk_trace_trace(current))
176 177
		return;

178
	ftrace_pid_function(ip, parent_ip, op, regs);
179 180 181 182 183 184 185 186 187
}

static void set_ftrace_pid_function(ftrace_func_t func)
{
	/* do not set ftrace_pid_function to itself! */
	if (func != ftrace_pid_func)
		ftrace_pid_function = func;
}

188
/**
189
 * clear_ftrace_function - reset the ftrace function
190
 *
191 192
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
193
 */
194
void clear_ftrace_function(void)
195
{
196
	ftrace_trace_function = ftrace_stub;
197
	ftrace_pid_function = ftrace_stub;
198 199
}

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
static void control_ops_disable_all(struct ftrace_ops *ops)
{
	int cpu;

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(ops->disabled, cpu) = 1;
}

static int control_ops_alloc(struct ftrace_ops *ops)
{
	int __percpu *disabled;

	disabled = alloc_percpu(int);
	if (!disabled)
		return -ENOMEM;

	ops->disabled = disabled;
	control_ops_disable_all(ops);
	return 0;
}

static void control_ops_free(struct ftrace_ops *ops)
{
	free_percpu(ops->disabled);
}

226
static void update_global_ops(void)
227 228 229 230 231 232 233 234
{
	ftrace_func_t func;

	/*
	 * If there's only one function registered, then call that
	 * function directly. Otherwise, we need to iterate over the
	 * registered callers.
	 */
235
	if (ftrace_global_list == &ftrace_list_end ||
236
	    ftrace_global_list->next == &ftrace_list_end) {
237
		func = ftrace_global_list->func;
238 239 240 241 242 243 244 245 246 247 248
		/*
		 * As we are calling the function directly.
		 * If it does not have recursion protection,
		 * the function_trace_op needs to be updated
		 * accordingly.
		 */
		if (ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE)
			global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
		else
			global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE;
	} else {
249
		func = ftrace_global_list_func;
250 251 252 253
		/* The list has its own recursion protection. */
		global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE;
	}

254 255 256 257 258 259

	/* If we filter on pids, update to use the pid function */
	if (!list_empty(&ftrace_pids)) {
		set_ftrace_pid_function(func);
		func = ftrace_pid_func;
	}
260 261 262 263 264 265 266 267 268 269

	global_ops.func = func;
}

static void update_ftrace_function(void)
{
	ftrace_func_t func;

	update_global_ops();

270 271
	/*
	 * If we are at the end of the list and this ops is
272 273
	 * recursion safe and not dynamic and the arch supports passing ops,
	 * then have the mcount trampoline call the function directly.
274
	 */
275
	if (ftrace_ops_list == &ftrace_list_end ||
276
	    (ftrace_ops_list->next == &ftrace_list_end &&
277
	     !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) &&
278
	     (ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) &&
279
	     !FTRACE_FORCE_LIST_FUNC)) {
280 281 282 283 284
		/* Set the ftrace_ops that the arch callback uses */
		if (ftrace_ops_list == &global_ops)
			function_trace_op = ftrace_global_list;
		else
			function_trace_op = ftrace_ops_list;
285
		func = ftrace_ops_list->func;
286 287 288
	} else {
		/* Just use the default ftrace_ops */
		function_trace_op = &ftrace_list_end;
289
		func = ftrace_ops_list_func;
290
	}
291

292 293 294
	ftrace_trace_function = func;
}

295
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
296
{
297
	ops->next = *list;
298
	/*
299
	 * We are entering ops into the list but another
300 301
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
302
	 * the ops pointer included into the list.
303
	 */
304
	rcu_assign_pointer(*list, ops);
305 306
}

307
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
308 309 310 311
{
	struct ftrace_ops **p;

	/*
312 313
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
314
	 */
315 316
	if (*list == ops && ops->next == &ftrace_list_end) {
		*list = &ftrace_list_end;
S
Steven Rostedt 已提交
317
		return 0;
318 319
	}

320
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
321 322 323
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
324 325
	if (*p != ops)
		return -1;
326 327

	*p = (*p)->next;
328 329
	return 0;
}
330

331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
static void add_ftrace_list_ops(struct ftrace_ops **list,
				struct ftrace_ops *main_ops,
				struct ftrace_ops *ops)
{
	int first = *list == &ftrace_list_end;
	add_ftrace_ops(list, ops);
	if (first)
		add_ftrace_ops(&ftrace_ops_list, main_ops);
}

static int remove_ftrace_list_ops(struct ftrace_ops **list,
				  struct ftrace_ops *main_ops,
				  struct ftrace_ops *ops)
{
	int ret = remove_ftrace_ops(list, ops);
	if (!ret && *list == &ftrace_list_end)
		ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
	return ret;
}

351 352
static int __register_ftrace_function(struct ftrace_ops *ops)
{
353
	if (unlikely(ftrace_disabled))
354 355 356 357 358
		return -ENODEV;

	if (FTRACE_WARN_ON(ops == &global_ops))
		return -EINVAL;

359 360 361
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

362 363 364 365
	/* We don't support both control and global flags set. */
	if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
		return -EINVAL;

366
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
367 368 369 370 371 372 373 374 375 376 377 378 379
	/*
	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
	 */
	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
		return -EINVAL;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif

380 381 382
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

383
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
384
		add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);
385
		ops->flags |= FTRACE_OPS_FL_ENABLED;
386 387 388 389
	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
		if (control_ops_alloc(ops))
			return -ENOMEM;
		add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
390 391 392
	} else
		add_ftrace_ops(&ftrace_ops_list, ops);

393 394 395 396 397 398 399 400 401 402 403 404 405
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

	if (ftrace_disabled)
		return -ENODEV;

406 407 408
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

409 410 411
	if (FTRACE_WARN_ON(ops == &global_ops))
		return -EINVAL;

412
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
413 414
		ret = remove_ftrace_list_ops(&ftrace_global_list,
					     &global_ops, ops);
415 416
		if (!ret)
			ops->flags &= ~FTRACE_OPS_FL_ENABLED;
417 418 419 420 421 422 423 424 425 426 427 428 429
	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
		ret = remove_ftrace_list_ops(&ftrace_control_list,
					     &control_ops, ops);
		if (!ret) {
			/*
			 * The ftrace_ops is now removed from the list,
			 * so there'll be no new users. We must ensure
			 * all current users are done before we free
			 * the control data.
			 */
			synchronize_sched();
			control_ops_free(ops);
		}
430 431 432
	} else
		ret = remove_ftrace_ops(&ftrace_ops_list, ops);

433 434
	if (ret < 0)
		return ret;
435

436 437
	if (ftrace_enabled)
		update_ftrace_function();
438

439 440 441 442 443 444 445
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
	 */
	if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
		synchronize_sched();

S
Steven Rostedt 已提交
446
	return 0;
447 448
}

449 450
static void ftrace_update_pid_func(void)
{
451
	/* Only do something if we are tracing something */
452
	if (ftrace_trace_function == ftrace_stub)
453
		return;
454

455
	update_ftrace_function();
456 457
}

458 459 460 461 462
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
463 464
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
465
	unsigned long long		time_squared;
466
#endif
467 468
};

469 470 471 472
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
473 474
};

475 476 477 478 479 480 481 482
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

483 484
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
485

486 487
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
488

489 490 491 492
static int ftrace_profile_bits __read_mostly;
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
493 494
static DEFINE_MUTEX(ftrace_profile_lock);

495
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
496 497 498

#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */

S
Steven Rostedt 已提交
499 500 501
static void *
function_stat_next(void *v, int idx)
{
502 503
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
504

505
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
506 507

 again:
L
Li Zefan 已提交
508 509 510
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
511 512 513 514 515
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
516 517
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
518 519 520 521 522 523 524
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
525 526 527 528 529 530 531
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
532 533
}

534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
550 551
static int function_stat_cmp(void *p1, void *p2)
{
552 553
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
554 555 556 557 558 559 560 561

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
562
#endif
S
Steven Rostedt 已提交
563 564 565

static int function_stat_headers(struct seq_file *m)
{
566
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
567
	seq_printf(m, "  Function                               "
568
		   "Hit    Time            Avg             s^2\n"
569
		      "  --------                               "
570
		   "---    ----            ---             ---\n");
571
#else
S
Steven Rostedt 已提交
572 573
	seq_printf(m, "  Function                               Hit\n"
		      "  --------                               ---\n");
574
#endif
S
Steven Rostedt 已提交
575 576 577 578 579
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
580
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
581
	char str[KSYM_SYMBOL_LEN];
582
	int ret = 0;
583
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
584 585
	static struct trace_seq s;
	unsigned long long avg;
586
	unsigned long long stddev;
587
#endif
588 589 590 591 592 593 594
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
S
Steven Rostedt 已提交
595 596

	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
597 598 599 600
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	seq_printf(m, "    ");
601 602 603
	avg = rec->time;
	do_div(avg, rec->counter);

604 605 606 607 608 609 610 611 612 613 614 615
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
		stddev = rec->time_squared - rec->counter * avg * avg;
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
		do_div(stddev, (rec->counter - 1) * 1000);
	}

616 617 618 619
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
620 621
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
622 623 624
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
625 626
out:
	mutex_unlock(&ftrace_profile_lock);
S
Steven Rostedt 已提交
627

628
	return ret;
S
Steven Rostedt 已提交
629 630
}

631
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
632
{
633
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
634

635
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
636

637 638 639 640
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
641 642
	}

643
	memset(stat->hash, 0,
644 645
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
646

647
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
648 649
{
	struct ftrace_profile_page *pg;
650 651
	int functions;
	int pages;
652
	int i;
S
Steven Rostedt 已提交
653

654
	/* If we already allocated, do nothing */
655
	if (stat->pages)
656
		return 0;
S
Steven Rostedt 已提交
657

658 659
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
660
		return -ENOMEM;
S
Steven Rostedt 已提交
661

662 663 664 665 666 667 668 669 670 671 672 673 674
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

675
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
676

677 678 679
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

	for (i = 0; i < pages; i++) {
680 681
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
682
			goto out_free;
683 684 685 686
		pg = pg->next;
	}

	return 0;
687 688 689 690 691 692 693 694 695 696 697 698 699 700 701

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	free_page((unsigned long)stat->pages);
	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
702 703
}

704
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
705
{
706
	struct ftrace_profile_stat *stat;
707
	int size;
S
Steven Rostedt 已提交
708

709 710 711
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
712
		/* If the profile is already created, simply reset it */
713
		ftrace_profile_reset(stat);
714 715
		return 0;
	}
S
Steven Rostedt 已提交
716

717 718 719 720 721
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
722

723
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
724

725
	if (!stat->hash)
726 727
		return -ENOMEM;

728 729
	if (!ftrace_profile_bits) {
		size--;
730

731 732 733
		for (; size; size >>= 1)
			ftrace_profile_bits++;
	}
734

735
	/* Preallocate the function profiling pages */
736 737 738
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
739 740 741 742
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
743 744
}

745 746 747 748 749 750 751 752 753 754 755 756 757 758
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

	for_each_online_cpu(cpu) {
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

759
/* interrupts must be disabled */
760 761
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
762
{
763
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
764 765 766 767 768
	struct hlist_head *hhd;
	struct hlist_node *n;
	unsigned long key;

	key = hash_long(ip, ftrace_profile_bits);
769
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
770 771 772 773 774 775

	if (hlist_empty(hhd))
		return NULL;

	hlist_for_each_entry_rcu(rec, n, hhd, node) {
		if (rec->ip == ip)
776 777 778 779 780 781
			return rec;
	}

	return NULL;
}

782 783
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
784 785 786 787
{
	unsigned long key;

	key = hash_long(rec->ip, ftrace_profile_bits);
788
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
789 790
}

791 792 793
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
794
static struct ftrace_profile *
795
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
796 797 798
{
	struct ftrace_profile *rec = NULL;

799
	/* prevent recursion (from NMIs) */
800
	if (atomic_inc_return(&stat->disabled) != 1)
801 802 803
		goto out;

	/*
804 805
	 * Try to find the function again since an NMI
	 * could have added it
806
	 */
807
	rec = ftrace_find_profiled_func(stat, ip);
808
	if (rec)
809
		goto out;
810

811 812 813 814
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
815
	}
816

817
	rec = &stat->pages->records[stat->pages->index++];
818
	rec->ip = ip;
819
	ftrace_add_profile(stat, rec);
820

S
Steven Rostedt 已提交
821
 out:
822
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
823 824 825 826 827

	return rec;
}

static void
828
function_profile_call(unsigned long ip, unsigned long parent_ip,
829
		      struct ftrace_ops *ops, struct pt_regs *regs)
S
Steven Rostedt 已提交
830
{
831
	struct ftrace_profile_stat *stat;
832
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
833 834 835 836 837 838
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
839 840

	stat = &__get_cpu_var(ftrace_profile_stats);
841
	if (!stat->hash || !ftrace_profile_enabled)
842 843 844
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
845
	if (!rec) {
846
		rec = ftrace_profile_alloc(stat, ip);
847 848 849
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
850 851 852 853 854 855

	rec->counter++;
 out:
	local_irq_restore(flags);
}

856 857 858
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
859
	function_profile_call(trace->func, 0, NULL, NULL);
860 861 862 863 864
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
865
	struct ftrace_profile_stat *stat;
866
	unsigned long long calltime;
867
	struct ftrace_profile *rec;
868
	unsigned long flags;
869 870

	local_irq_save(flags);
871
	stat = &__get_cpu_var(ftrace_profile_stats);
872
	if (!stat->hash || !ftrace_profile_enabled)
873 874
		goto out;

875 876 877 878
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
	calltime = trace->rettime - trace->calltime;

	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

896
	rec = ftrace_find_profiled_func(stat, trace->func);
897
	if (rec) {
898
		rec->time += calltime;
899 900
		rec->time_squared += calltime * calltime;
	}
901

902
 out:
903 904 905 906 907 908 909 910 911 912 913 914 915 916
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
917
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
918
	.func		= function_profile_call,
919
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE,
S
Steven Rostedt 已提交
920 921
};

922 923 924 925 926 927 928 929 930 931 932
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
933 934 935 936 937 938 939
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

940 941
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
S
Steven Rostedt 已提交
942 943 944 945 946 947 948
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
949 950 951 952 953 954
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

955 956 957 958 959
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
960 961 962
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
963 964 965 966
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
967
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
968 969
		}
	}
970
 out:
S
Steven Rostedt 已提交
971 972
	mutex_unlock(&ftrace_profile_lock);

973
	*ppos += cnt;
S
Steven Rostedt 已提交
974 975 976 977

	return cnt;
}

978 979 980 981
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
982
	char buf[64];		/* big enough to hold a number */
983 984 985 986 987 988
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
989 990 991 992
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
993
	.llseek		= default_llseek,
S
Steven Rostedt 已提交
994 995
};

996 997
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
998 999 1000 1001 1002 1003
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
1004 1005
};

1006
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1007
{
1008
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
1009
	struct dentry *entry;
1010
	char *name;
S
Steven Rostedt 已提交
1011
	int ret;
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

		/* allocate enough for function name + cpu number */
		name = kmalloc(32, GFP_KERNEL);
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		snprintf(name, 32, "function%d", cpu);
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
	}

	entry = debugfs_create_file("function_profile_enabled", 0644,
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
		pr_warning("Could not create debugfs "
			   "'function_profile_enabled' entry\n");
}

#else /* CONFIG_FUNCTION_PROFILER */
1050
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1051 1052 1053 1054
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

#ifdef CONFIG_DYNAMIC_FTRACE

#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;

struct ftrace_func_probe {
	struct hlist_node	node;
	struct ftrace_probe_ops	*ops;
	unsigned long		flags;
	unsigned long		ip;
	void			*data;
	struct rcu_head		rcu;
};

1074 1075 1076 1077 1078 1079 1080 1081 1082
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

struct ftrace_hash {
	unsigned long		size_bits;
	struct hlist_head	*buckets;
	unsigned long		count;
1083
	struct rcu_head		rcu;
1084 1085
};

1086 1087 1088 1089 1090 1091 1092 1093 1094
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
1095
};
1096
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1097

1098
static struct ftrace_ops global_ops = {
1099
	.func			= ftrace_stub,
1100 1101
	.notrace_hash		= EMPTY_HASH,
	.filter_hash		= EMPTY_HASH,
1102
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE,
1103 1104
};

1105 1106 1107 1108
static DEFINE_MUTEX(ftrace_regex_lock);

struct ftrace_page {
	struct ftrace_page	*next;
1109
	struct dyn_ftrace	*records;
1110
	int			index;
1111
	int			size;
1112 1113
};

1114 1115
static struct ftrace_page *ftrace_new_pgs;

1116 1117
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1118 1119 1120 1121 1122 1123 1124

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

1125 1126 1127 1128 1129
static bool ftrace_hash_empty(struct ftrace_hash *hash)
{
	return !hash || !hash->count;
}

1130 1131 1132 1133 1134 1135 1136 1137
static struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;
	struct hlist_node *n;

1138
	if (ftrace_hash_empty(hash))
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
		return NULL;

	if (hash->size_bits > 0)
		key = hash_long(ip, hash->size_bits);
	else
		key = 0;

	hhd = &hash->buckets[key];

	hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1155 1156
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1157 1158 1159 1160 1161
{
	struct hlist_head *hhd;
	unsigned long key;

	if (hash->size_bits)
1162
		key = hash_long(entry->ip, hash->size_bits);
1163 1164 1165 1166 1167 1168
	else
		key = 0;

	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1181 1182 1183 1184 1185

	return 0;
}

static void
1186
free_hash_entry(struct ftrace_hash *hash,
1187 1188 1189 1190 1191 1192 1193
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1194 1195 1196 1197 1198 1199 1200 1201
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	hash->count--;
}

1202 1203 1204 1205 1206 1207 1208 1209
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
	struct hlist_node *tp, *tn;
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1210 1211 1212
	if (!hash->count)
		return;

1213 1214 1215
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
		hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1216
			free_hash_entry(hash, entry);
1217 1218 1219 1220
	}
	FTRACE_WARN_ON(hash->count);
}

1221 1222 1223 1224 1225 1226 1227 1228 1229
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}

1245 1246 1247 1248 1249 1250
void ftrace_free_filter(struct ftrace_ops *ops)
{
	free_ftrace_hash(ops->filter_hash);
	free_ftrace_hash(ops->notrace_hash);
}

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
1261
	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	struct hlist_node *tp;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

	/* Empty hash? */
1288
	if (ftrace_hash_empty(hash))
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

1309 1310 1311 1312 1313
static void
ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
static void
ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);

1314
static int
1315 1316
ftrace_hash_move(struct ftrace_ops *ops, int enable,
		 struct ftrace_hash **dst, struct ftrace_hash *src)
1317 1318 1319 1320
{
	struct ftrace_func_entry *entry;
	struct hlist_node *tp, *tn;
	struct hlist_head *hhd;
1321 1322
	struct ftrace_hash *old_hash;
	struct ftrace_hash *new_hash;
1323 1324 1325
	unsigned long key;
	int size = src->count;
	int bits = 0;
1326
	int ret;
1327 1328
	int i;

1329 1330 1331 1332 1333 1334
	/*
	 * Remove the current set, update the hash and add
	 * them back.
	 */
	ftrace_hash_rec_disable(ops, enable);

1335 1336 1337 1338 1339
	/*
	 * If the new source is empty, just free dst and assign it
	 * the empty_hash.
	 */
	if (!src->count) {
1340 1341
		free_ftrace_hash_rcu(*dst);
		rcu_assign_pointer(*dst, EMPTY_HASH);
1342 1343 1344
		/* still need to update the function records */
		ret = 0;
		goto out;
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
	}

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1357
	ret = -ENOMEM;
1358 1359
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
1360
		goto out;
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370

	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
		hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
			if (bits > 0)
				key = hash_long(entry->ip, bits);
			else
				key = 0;
			remove_hash_entry(src, entry);
1371
			__add_hash_entry(new_hash, entry);
1372 1373 1374
		}
	}

1375 1376 1377 1378
	old_hash = *dst;
	rcu_assign_pointer(*dst, new_hash);
	free_ftrace_hash_rcu(old_hash);

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388
	ret = 0;
 out:
	/*
	 * Enable regardless of ret:
	 *  On success, we enable the new hash.
	 *  On failure, we re-enable the original hash.
	 */
	ftrace_hash_rec_enable(ops, enable);

	return ret;
1389 1390
}

1391 1392 1393 1394 1395 1396 1397 1398
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1399 1400 1401
 *
 * This needs to be called with preemption disabled as
 * the hashes are freed with call_rcu_sched().
1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
 */
static int
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
{
	struct ftrace_hash *filter_hash;
	struct ftrace_hash *notrace_hash;
	int ret;

	filter_hash = rcu_dereference_raw(ops->filter_hash);
	notrace_hash = rcu_dereference_raw(ops->notrace_hash);

1413
	if ((ftrace_hash_empty(filter_hash) ||
1414
	     ftrace_lookup_ip(filter_hash, ip)) &&
1415
	    (ftrace_hash_empty(notrace_hash) ||
1416 1417 1418 1419 1420 1421 1422 1423
	     !ftrace_lookup_ip(notrace_hash, ip)))
		ret = 1;
	else
		ret = 0;

	return ret;
}

1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1438 1439 1440

static int ftrace_cmp_recs(const void *a, const void *b)
{
1441 1442
	const struct dyn_ftrace *key = a;
	const struct dyn_ftrace *rec = b;
1443

1444
	if (key->flags < rec->ip)
1445
		return -1;
1446 1447
	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
		return 1;
1448 1449 1450
	return 0;
}

1451
static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1452 1453 1454
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1455
	struct dyn_ftrace key;
1456

1457 1458
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */
1459 1460

	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1461 1462
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1463
			continue;
1464 1465 1466 1467
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (rec)
1468
			return rec->ip;
1469
	}
1470 1471 1472 1473

	return 0;
}

1474 1475 1476 1477
/**
 * ftrace_location - return true if the ip giving is a traced location
 * @ip: the instruction pointer to check
 *
1478
 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1479 1480 1481 1482
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1483
unsigned long ftrace_location(unsigned long ip)
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499
{
	return ftrace_location_range(ip, ip);
}

/**
 * ftrace_text_reserved - return true if range contains an ftrace location
 * @start: start of range to search
 * @end: end of range to search (inclusive). @end points to the last byte to check.
 *
 * Returns 1 if @start and @end contains a ftrace location.
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
int ftrace_text_reserved(void *start, void *end)
{
1500 1501 1502 1503 1504 1505
	unsigned long ret;

	ret = ftrace_location_range((unsigned long)start,
				    (unsigned long)end);

	return (int)!!ret;
1506 1507
}

1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	int count = 0;
	int all = 0;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return;

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
		hash = ops->filter_hash;
		other_hash = ops->notrace_hash;
1537
		if (ftrace_hash_empty(hash))
1538 1539 1540 1541 1542 1543 1544 1545 1546
			all = 1;
	} else {
		inc = !inc;
		hash = ops->notrace_hash;
		other_hash = ops->filter_hash;
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1547
		if (ftrace_hash_empty(hash))
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
			return;
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1561
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1562 1563
				match = 1;
		} else {
1564 1565
			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1566 1567 1568 1569 1570 1571 1572

			/*
			 *
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
1573
				 (in_other_hash || ftrace_hash_empty(other_hash)))
1574 1575 1576 1577 1578 1579 1580 1581 1582
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
				return;
1583 1584 1585 1586 1587 1588
			/*
			 * If any ops wants regs saved for this function
			 * then all ops will get saved regs.
			 */
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
				rec->flags |= FTRACE_FL_REGS;
1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
		} else {
			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
				return;
			rec->flags--;
		}
		count++;
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
			return;
	} while_for_each_ftrace_rec();
}

static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
				    int filter_hash)
{
	__ftrace_hash_rec_update(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
				   int filter_hash)
{
	__ftrace_hash_rec_update(ops, filter_hash, 1);
}

1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
static void print_ip_ins(const char *fmt, unsigned char *p)
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
/**
 * ftrace_bug - report and shutdown function tracer
 * @failed: The failed type (EFAULT, EINVAL, EPERM)
 * @ip: The address that failed
 *
 * The arch code that enables or disables the function tracing
 * can call ftrace_bug() when it has detected a problem in
 * modifying the code. @failed should be one of either:
 * EFAULT - if the problem happens on reading the @ip address
 * EINVAL - if what is read at @ip is not what was expected
 * EPERM - if the problem happens on writting to the @ip address
 */
void ftrace_bug(int failed, unsigned long ip)
1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
{
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
		print_ip_ins(" actual: ", (unsigned char *)ip);
		printk(KERN_CONT "\n");
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
}

1662
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
1663
{
1664
	unsigned long flag = 0UL;
1665

S
Steven Rostedt 已提交
1666
	/*
1667
	 * If we are updating calls:
S
Steven Rostedt 已提交
1668
	 *
1669 1670
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
S
Steven Rostedt 已提交
1671
	 *
1672 1673
	 *   Otherwise we make sure its disabled.
	 *
1674
	 * If we are disabling calls, then disable all records that
1675
	 * are enabled.
S
Steven Rostedt 已提交
1676
	 */
1677
	if (enable && (rec->flags & ~FTRACE_FL_MASK))
1678
		flag = FTRACE_FL_ENABLED;
S
Steven Rostedt 已提交
1679

1680 1681 1682 1683 1684 1685 1686 1687 1688
	/*
	 * If enabling and the REGS flag does not match the REGS_EN, then
	 * do not ignore this record. Set flags to fail the compare against
	 * ENABLED.
	 */
	if (flag &&
	    (!(rec->flags & FTRACE_FL_REGS) != !(rec->flags & FTRACE_FL_REGS_EN)))
		flag |= FTRACE_FL_REGS;

1689 1690
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1691
		return FTRACE_UPDATE_IGNORE;
S
Steven Rostedt 已提交
1692

1693
	if (flag) {
1694 1695 1696 1697
		/* Save off if rec is being enabled (for return value) */
		flag ^= rec->flags & FTRACE_FL_ENABLED;

		if (update) {
1698
			rec->flags |= FTRACE_FL_ENABLED;
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722
			if (flag & FTRACE_FL_REGS) {
				if (rec->flags & FTRACE_FL_REGS)
					rec->flags |= FTRACE_FL_REGS_EN;
				else
					rec->flags &= ~FTRACE_FL_REGS_EN;
			}
		}

		/*
		 * If this record is being updated from a nop, then
		 *   return UPDATE_MAKE_CALL.
		 * Otherwise, if the EN flag is set, then return
		 *   UPDATE_MODIFY_CALL_REGS to tell the caller to convert
		 *   from the non-save regs, to a save regs function.
		 * Otherwise,
		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
		 *   from the save regs, to a non-save regs function.
		 */
		if (flag & FTRACE_FL_ENABLED)
			return FTRACE_UPDATE_MAKE_CALL;
		else if (rec->flags & FTRACE_FL_REGS_EN)
			return FTRACE_UPDATE_MODIFY_CALL_REGS;
		else
			return FTRACE_UPDATE_MODIFY_CALL;
1723 1724
	}

1725 1726 1727 1728 1729 1730 1731 1732
	if (update) {
		/* If there's no more users, clear all flags */
		if (!(rec->flags & ~FTRACE_FL_MASK))
			rec->flags = 0;
		else
			/* Just disable the record (keep REGS state) */
			rec->flags &= ~FTRACE_FL_ENABLED;
	}
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766

	return FTRACE_UPDATE_MAKE_NOP;
}

/**
 * ftrace_update_record, set a record that now is tracing or not
 * @rec: the record to update
 * @enable: set to 1 if the record is tracing, zero to force disable
 *
 * The records that represent all functions that can be traced need
 * to be updated when tracing has been enabled.
 */
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 1);
}

/**
 * ftrace_test_record, check if the record has been enabled or not
 * @rec: the record to test
 * @enable: set to 1 to check if enabled, 0 if it is disabled
 *
 * The arch code may need to test if a record is already set to
 * tracing to determine how to modify the function code that it
 * represents.
 */
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 0);
}

static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
1767
	unsigned long ftrace_old_addr;
1768 1769 1770 1771 1772
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_update_record(rec, enable);

1773 1774 1775 1776 1777
	if (rec->flags & FTRACE_FL_REGS)
		ftrace_addr = (unsigned long)FTRACE_REGS_ADDR;
	else
		ftrace_addr = (unsigned long)FTRACE_ADDR;

1778 1779 1780 1781 1782
	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
1783
		return ftrace_make_call(rec, ftrace_addr);
1784 1785 1786

	case FTRACE_UPDATE_MAKE_NOP:
		return ftrace_make_nop(NULL, rec, ftrace_addr);
1787 1788 1789 1790 1791 1792 1793 1794 1795

	case FTRACE_UPDATE_MODIFY_CALL_REGS:
	case FTRACE_UPDATE_MODIFY_CALL:
		if (rec->flags & FTRACE_FL_REGS)
			ftrace_old_addr = (unsigned long)FTRACE_ADDR;
		else
			ftrace_old_addr = (unsigned long)FTRACE_REGS_ADDR;

		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
1796 1797
	}

1798
	return -1; /* unknow ftrace bug */
1799 1800
}

1801
void __weak ftrace_replace_code(int enable)
1802 1803 1804
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
1805
	int failed;
1806

1807 1808 1809
	if (unlikely(ftrace_disabled))
		return;

1810
	do_for_each_ftrace_rec(pg, rec) {
1811
		failed = __ftrace_replace_code(rec, enable);
1812
		if (failed) {
1813 1814 1815
			ftrace_bug(failed, rec->ip);
			/* Stop processing */
			return;
1816
		}
1817
	} while_for_each_ftrace_rec();
1818 1819
}

1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891
struct ftrace_rec_iter {
	struct ftrace_page	*pg;
	int			index;
};

/**
 * ftrace_rec_iter_start, start up iterating over traced functions
 *
 * Returns an iterator handle that is used to iterate over all
 * the records that represent address locations where functions
 * are traced.
 *
 * May return NULL if no records are available.
 */
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
	/*
	 * We only use a single iterator.
	 * Protected by the ftrace_lock mutex.
	 */
	static struct ftrace_rec_iter ftrace_rec_iter;
	struct ftrace_rec_iter *iter = &ftrace_rec_iter;

	iter->pg = ftrace_pages_start;
	iter->index = 0;

	/* Could have empty pages */
	while (iter->pg && !iter->pg->index)
		iter->pg = iter->pg->next;

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_next, get the next record to process.
 * @iter: The handle to the iterator.
 *
 * Returns the next iterator after the given iterator @iter.
 */
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
	iter->index++;

	if (iter->index >= iter->pg->index) {
		iter->pg = iter->pg->next;
		iter->index = 0;

		/* Could have empty pages */
		while (iter->pg && !iter->pg->index)
			iter->pg = iter->pg->next;
	}

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_record, get the record at the iterator location
 * @iter: The current iterator location
 *
 * Returns the record that the current @iter is at.
 */
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
	return &iter->pg->records[iter->index];
}

1892
static int
1893
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1894 1895
{
	unsigned long ip;
1896
	int ret;
1897 1898 1899

	ip = rec->ip;

1900 1901 1902
	if (unlikely(ftrace_disabled))
		return 0;

1903
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1904
	if (ret) {
1905
		ftrace_bug(ret, ip);
1906
		return 0;
1907
	}
1908
	return 1;
1909 1910
}

1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

1929
void ftrace_modify_all_code(int command)
1930
{
1931
	if (command & FTRACE_UPDATE_CALLS)
1932
		ftrace_replace_code(1);
1933
	else if (command & FTRACE_DISABLE_CALLS)
1934 1935
		ftrace_replace_code(0);

1936
	if (command & FTRACE_UPDATE_TRACE_FUNC)
1937 1938
		ftrace_update_ftrace_func(ftrace_trace_function);

1939
	if (command & FTRACE_START_FUNC_RET)
1940
		ftrace_enable_ftrace_graph_caller();
1941
	else if (command & FTRACE_STOP_FUNC_RET)
1942
		ftrace_disable_ftrace_graph_caller();
1943 1944 1945 1946 1947 1948 1949
}

static int __ftrace_modify_code(void *data)
{
	int *command = data;

	ftrace_modify_all_code(*command);
1950

1951
	return 0;
1952 1953
}

1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
/**
 * ftrace_run_stop_machine, go back to the stop machine method
 * @command: The command to tell ftrace what to do
 *
 * If an arch needs to fall back to the stop machine method, the
 * it can call this function.
 */
void ftrace_run_stop_machine(int command)
{
	stop_machine(__ftrace_modify_code, &command, NULL);
}

/**
 * arch_ftrace_update_code, modify the code to trace or not trace
 * @command: The command that needs to be done
 *
 * Archs can override this function if it does not need to
 * run stop_machine() to modify code.
 */
void __weak arch_ftrace_update_code(int command)
{
	ftrace_run_stop_machine(command);
}

I
Ingo Molnar 已提交
1978
static void ftrace_run_update_code(int command)
1979
{
1980 1981 1982 1983 1984 1985
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;
1986 1987 1988 1989 1990
	/*
	 * Do not call function tracer while we update the code.
	 * We are in stop machine.
	 */
	function_trace_stop++;
1991

1992 1993 1994 1995 1996 1997 1998 1999 2000
	/*
	 * By default we use stop_machine() to modify the code.
	 * But archs can do what ever they want as long as it
	 * is safe. The stop_machine() is the safest, but also
	 * produces the most overhead.
	 */
	arch_ftrace_update_code(command);

	function_trace_stop--;
2001 2002 2003

	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
2004 2005
}

2006
static ftrace_func_t saved_ftrace_func;
2007
static int ftrace_start_up;
2008
static int global_start_up;
2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021

static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
2022

2023
static int ftrace_startup(struct ftrace_ops *ops, int command)
2024
{
2025 2026
	bool hash_enable = true;

2027
	if (unlikely(ftrace_disabled))
2028
		return -ENODEV;
2029

2030
	ftrace_start_up++;
2031
	command |= FTRACE_UPDATE_CALLS;
2032

2033 2034 2035 2036 2037 2038 2039 2040 2041
	/* ops marked global share the filter hashes */
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
		ops = &global_ops;
		/* Don't update hash if global is already set */
		if (global_start_up)
			hash_enable = false;
		global_start_up++;
	}

2042
	ops->flags |= FTRACE_OPS_FL_ENABLED;
2043
	if (hash_enable)
2044 2045
		ftrace_hash_rec_enable(ops, 1);

2046
	ftrace_startup_enable(command);
2047 2048

	return 0;
2049 2050
}

2051
static void ftrace_shutdown(struct ftrace_ops *ops, int command)
2052
{
2053 2054
	bool hash_disable = true;

2055 2056 2057
	if (unlikely(ftrace_disabled))
		return;

2058
	ftrace_start_up--;
2059 2060 2061 2062 2063 2064 2065
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
		ops = &global_ops;
		global_start_up--;
		WARN_ON_ONCE(global_start_up < 0);
		/* Don't update hash if global still has users */
		if (global_start_up) {
			WARN_ON_ONCE(!ftrace_start_up);
			hash_disable = false;
		}
	}

	if (hash_disable)
2078 2079
		ftrace_hash_rec_disable(ops, 1);

2080
	if (ops != &global_ops || !global_start_up)
2081
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2082

2083
	command |= FTRACE_UPDATE_CALLS;
2084

2085 2086 2087 2088
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
2089

2090
	if (!command || !ftrace_enabled)
S
Steven Rostedt 已提交
2091
		return;
2092 2093

	ftrace_run_update_code(command);
2094 2095
}

I
Ingo Molnar 已提交
2096
static void ftrace_startup_sysctl(void)
2097
{
2098 2099 2100
	if (unlikely(ftrace_disabled))
		return;

2101 2102
	/* Force update next time */
	saved_ftrace_func = NULL;
2103 2104
	/* ftrace_start_up is true if we want ftrace running */
	if (ftrace_start_up)
2105
		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
2106 2107
}

I
Ingo Molnar 已提交
2108
static void ftrace_shutdown_sysctl(void)
2109
{
2110 2111 2112
	if (unlikely(ftrace_disabled))
		return;

2113 2114
	/* ftrace_start_up is true if ftrace is running */
	if (ftrace_start_up)
2115
		ftrace_run_update_code(FTRACE_DISABLE_CALLS);
2116 2117
}

2118 2119 2120 2121
static cycle_t		ftrace_update_time;
static unsigned long	ftrace_update_cnt;
unsigned long		ftrace_update_tot_cnt;

2122 2123 2124 2125 2126
static int ops_traces_mod(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash;

	hash = ops->filter_hash;
2127
	return ftrace_hash_empty(hash);
2128 2129
}

2130
static int ftrace_update_code(struct module *mod)
2131
{
2132
	struct ftrace_page *pg;
2133
	struct dyn_ftrace *p;
2134
	cycle_t start, stop;
2135
	unsigned long ref = 0;
2136
	int i;
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153

	/*
	 * When adding a module, we need to check if tracers are
	 * currently enabled and if they are set to trace all functions.
	 * If they are, we need to enable the module functions as well
	 * as update the reference counts for those function records.
	 */
	if (mod) {
		struct ftrace_ops *ops;

		for (ops = ftrace_ops_list;
		     ops != &ftrace_list_end; ops = ops->next) {
			if (ops->flags & FTRACE_OPS_FL_ENABLED &&
			    ops_traces_mod(ops))
				ref++;
		}
	}
2154

I
Ingo Molnar 已提交
2155
	start = ftrace_now(raw_smp_processor_id());
2156 2157
	ftrace_update_cnt = 0;

2158
	for (pg = ftrace_new_pgs; pg; pg = pg->next) {
2159

2160 2161 2162 2163
		for (i = 0; i < pg->index; i++) {
			/* If something went wrong, bail without enabling anything */
			if (unlikely(ftrace_disabled))
				return -1;
2164

2165 2166
			p = &pg->records[i];
			p->flags = ref;
2167

2168 2169 2170 2171 2172 2173
			/*
			 * Do the initial record conversion from mcount jump
			 * to the NOP instructions.
			 */
			if (!ftrace_code_disable(mod, p))
				break;
2174

2175
			ftrace_update_cnt++;
2176

2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
			/*
			 * If the tracing is enabled, go ahead and enable the record.
			 *
			 * The reason not to enable the record immediatelly is the
			 * inherent check of ftrace_make_nop/ftrace_make_call for
			 * correct previous instructions.  Making first the NOP
			 * conversion puts the module to the correct state, thus
			 * passing the ftrace_make_call check.
			 */
			if (ftrace_start_up && ref) {
				int failed = __ftrace_replace_code(p, 1);
				if (failed)
					ftrace_bug(failed, p->ip);
			}
2191
		}
2192 2193
	}

2194 2195
	ftrace_new_pgs = NULL;

I
Ingo Molnar 已提交
2196
	stop = ftrace_now(raw_smp_processor_id());
2197 2198 2199
	ftrace_update_time = stop - start;
	ftrace_update_tot_cnt += ftrace_update_cnt;

2200 2201 2202
	return 0;
}

2203
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2204
{
2205
	int order;
2206 2207
	int cnt;

2208 2209 2210 2211
	if (WARN_ON(!count))
		return -EINVAL;

	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2212 2213

	/*
2214 2215
	 * We want to fill as much as possible. No more than a page
	 * may be empty.
2216
	 */
2217 2218
	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
		order--;
2219

2220 2221
 again:
	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2222

2223 2224 2225 2226 2227 2228 2229
	if (!pg->records) {
		/* if we can't allocate this size, try something smaller */
		if (!order)
			return -ENOMEM;
		order >>= 1;
		goto again;
	}
2230

2231 2232
	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
	pg->size = cnt;
2233

2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266
	if (cnt > count)
		cnt = count;

	return cnt;
}

static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
	struct ftrace_page *start_pg;
	struct ftrace_page *pg;
	int order;
	int cnt;

	if (!num_to_init)
		return 0;

	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
	if (!pg)
		return NULL;

	/*
	 * Try to allocate as much as possible in one continues
	 * location that fills in all of the space. We want to
	 * waste as little space as possible.
	 */
	for (;;) {
		cnt = ftrace_allocate_records(pg, num_to_init);
		if (cnt < 0)
			goto free_pages;

		num_to_init -= cnt;
		if (!num_to_init)
2267 2268
			break;

2269 2270 2271 2272
		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
		if (!pg->next)
			goto free_pages;

2273 2274 2275
		pg = pg->next;
	}

2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302
	return start_pg;

 free_pages:
	while (start_pg) {
		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		start_pg = pg->next;
		kfree(pg);
		pg = start_pg;
	}
	pr_info("ftrace: FAILED to allocate memory for functions\n");
	return NULL;
}

static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
{
	int cnt;

	if (!num_to_init) {
		pr_info("ftrace: No functions to be traced?\n");
		return -1;
	}

	cnt = num_to_init / ENTRIES_PER_PAGE;
	pr_info("ftrace: allocating %ld entries in %d pages\n",
		num_to_init, cnt + 1);

2303 2304 2305
	return 0;
}

2306 2307 2308
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
2309
	loff_t				pos;
2310 2311 2312 2313 2314
	loff_t				func_pos;
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
	struct trace_parser		parser;
2315
	struct ftrace_hash		*hash;
2316
	struct ftrace_ops		*ops;
2317 2318 2319
	int				hidx;
	int				idx;
	unsigned			flags;
2320 2321
};

2322
static void *
2323
t_hash_next(struct seq_file *m, loff_t *pos)
2324 2325
{
	struct ftrace_iterator *iter = m->private;
2326
	struct hlist_node *hnd = NULL;
2327 2328 2329
	struct hlist_head *hhd;

	(*pos)++;
2330
	iter->pos = *pos;
2331

2332 2333
	if (iter->probe)
		hnd = &iter->probe->node;
2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355
 retry:
	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
		return NULL;

	hhd = &ftrace_func_hash[iter->hidx];

	if (hlist_empty(hhd)) {
		iter->hidx++;
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
			iter->hidx++;
			goto retry;
		}
	}

2356 2357 2358 2359 2360 2361
	if (WARN_ON_ONCE(!hnd))
		return NULL;

	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);

	return iter;
2362 2363 2364 2365 2366 2367
}

static void *t_hash_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
2368 2369
	loff_t l;

2370 2371 2372
	if (!(iter->flags & FTRACE_ITER_DO_HASH))
		return NULL;

2373 2374
	if (iter->func_pos > *pos)
		return NULL;
2375

L
Li Zefan 已提交
2376
	iter->hidx = 0;
2377
	for (l = 0; l <= (*pos - iter->func_pos); ) {
2378
		p = t_hash_next(m, &l);
L
Li Zefan 已提交
2379 2380 2381
		if (!p)
			break;
	}
2382 2383 2384
	if (!p)
		return NULL;

2385 2386 2387
	/* Only set this if we have an item */
	iter->flags |= FTRACE_ITER_HASH;

2388
	return iter;
2389 2390
}

2391 2392
static int
t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
2393
{
S
Steven Rostedt 已提交
2394
	struct ftrace_func_probe *rec;
2395

2396 2397 2398
	rec = iter->probe;
	if (WARN_ON_ONCE(!rec))
		return -EIO;
2399

2400 2401 2402
	if (rec->ops->print)
		return rec->ops->print(m, rec->ip, rec->ops, rec->data);

2403
	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
2404 2405 2406 2407 2408 2409 2410 2411

	if (rec->data)
		seq_printf(m, ":%p", rec->data);
	seq_putc(m, '\n');

	return 0;
}

I
Ingo Molnar 已提交
2412
static void *
2413 2414 2415
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
2416
	struct ftrace_ops *ops = iter->ops;
2417 2418
	struct dyn_ftrace *rec = NULL;

2419 2420 2421
	if (unlikely(ftrace_disabled))
		return NULL;

2422
	if (iter->flags & FTRACE_ITER_HASH)
2423
		return t_hash_next(m, pos);
2424

2425
	(*pos)++;
2426
	iter->pos = iter->func_pos = *pos;
2427

2428
	if (iter->flags & FTRACE_ITER_PRINTALL)
2429
		return t_hash_start(m, pos);
2430

2431 2432 2433 2434 2435 2436 2437 2438 2439
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
2440
		if (((iter->flags & FTRACE_ITER_FILTER) &&
2441
		     !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
S
Steven Rostedt 已提交
2442

2443
		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
2444 2445 2446 2447 2448
		     !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
		     !(rec->flags & ~FTRACE_FL_MASK))) {

2449 2450 2451 2452 2453
			rec = NULL;
			goto retry;
		}
	}

2454
	if (!rec)
2455
		return t_hash_start(m, pos);
2456 2457 2458 2459

	iter->func = rec;

	return iter;
2460 2461
}

2462 2463 2464 2465
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
2466
	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
2467 2468 2469 2470 2471
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
2472
	struct ftrace_ops *ops = iter->ops;
2473
	void *p = NULL;
2474
	loff_t l;
2475

2476
	mutex_lock(&ftrace_lock);
2477 2478 2479 2480

	if (unlikely(ftrace_disabled))
		return NULL;

2481 2482 2483 2484 2485 2486
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

2487 2488 2489 2490 2491
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
2492 2493
	if (iter->flags & FTRACE_ITER_FILTER &&
	    ftrace_hash_empty(ops->filter_hash)) {
2494
		if (*pos > 0)
2495
			return t_hash_start(m, pos);
2496
		iter->flags |= FTRACE_ITER_PRINTALL;
2497 2498
		/* reset in case of seek/pread */
		iter->flags &= ~FTRACE_ITER_HASH;
2499 2500 2501
		return iter;
	}

2502 2503 2504
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_start(m, pos);

2505 2506 2507 2508 2509
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
2510 2511 2512 2513 2514 2515
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
		p = t_next(m, p, &l);
		if (!p)
			break;
2516
	}
2517

2518 2519
	if (!p)
		return t_hash_start(m, pos);
2520 2521

	return iter;
2522 2523 2524 2525
}

static void t_stop(struct seq_file *m, void *p)
{
2526
	mutex_unlock(&ftrace_lock);
2527 2528 2529 2530
}

static int t_show(struct seq_file *m, void *v)
{
2531
	struct ftrace_iterator *iter = m->private;
2532
	struct dyn_ftrace *rec;
2533

2534
	if (iter->flags & FTRACE_ITER_HASH)
2535
		return t_hash_show(m, iter);
2536

2537 2538 2539 2540 2541
	if (iter->flags & FTRACE_ITER_PRINTALL) {
		seq_printf(m, "#### all functions enabled ####\n");
		return 0;
	}

2542 2543
	rec = iter->func;

2544 2545 2546
	if (!rec)
		return 0;

2547 2548
	seq_printf(m, "%ps", (void *)rec->ip);
	if (iter->flags & FTRACE_ITER_ENABLED)
2549 2550 2551
		seq_printf(m, " (%ld)%s",
			   rec->flags & ~FTRACE_FL_MASK,
			   rec->flags & FTRACE_FL_REGS ? " R" : "");
2552
	seq_printf(m, "\n");
2553 2554 2555 2556

	return 0;
}

J
James Morris 已提交
2557
static const struct seq_operations show_ftrace_seq_ops = {
2558 2559 2560 2561 2562 2563
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
2564
static int
2565 2566 2567 2568
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

2569 2570 2571
	if (unlikely(ftrace_disabled))
		return -ENODEV;

2572 2573 2574 2575
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
	if (iter) {
		iter->pg = ftrace_pages_start;
		iter->ops = &global_ops;
I
Ingo Molnar 已提交
2576
	}
2577

2578
	return iter ? 0 : -ENOMEM;
2579 2580
}

2581 2582 2583 2584 2585 2586 2587 2588
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

	if (unlikely(ftrace_disabled))
		return -ENODEV;

2589 2590 2591 2592 2593
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
	if (iter) {
		iter->pg = ftrace_pages_start;
		iter->flags = FTRACE_ITER_ENABLED;
		iter->ops = &global_ops;
2594 2595
	}

2596
	return iter ? 0 : -ENOMEM;
2597 2598
}

2599
static void ftrace_filter_reset(struct ftrace_hash *hash)
2600
{
2601
	mutex_lock(&ftrace_lock);
2602
	ftrace_hash_clear(hash);
2603
	mutex_unlock(&ftrace_lock);
2604 2605
}

2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
/**
 * ftrace_regex_open - initialize function tracer filter files
 * @ops: The ftrace_ops that hold the hash filters
 * @flag: The type of filter to process
 * @inode: The inode, usually passed in to your open routine
 * @file: The file, usually passed in to your open routine
 *
 * ftrace_regex_open() initializes the filter files for the
 * @ops. Depending on @flag it may process the filter hash or
 * the notrace hash of @ops. With this called from the open
 * routine, you can use ftrace_filter_write() for the write
 * routine if @flag has FTRACE_ITER_FILTER set, or
 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
 * ftrace_regex_lseek() should be used as the lseek routine, and
 * release must call ftrace_regex_release().
 */
int
2623
ftrace_regex_open(struct ftrace_ops *ops, int flag,
2624
		  struct inode *inode, struct file *file)
2625 2626
{
	struct ftrace_iterator *iter;
2627
	struct ftrace_hash *hash;
2628 2629
	int ret = 0;

2630 2631 2632
	if (unlikely(ftrace_disabled))
		return -ENODEV;

2633 2634 2635 2636
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

2637 2638 2639 2640 2641
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

2642 2643 2644 2645 2646
	if (flag & FTRACE_ITER_NOTRACE)
		hash = ops->notrace_hash;
	else
		hash = ops->filter_hash;

2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660
	iter->ops = ops;
	iter->flags = flag;

	if (file->f_mode & FMODE_WRITE) {
		mutex_lock(&ftrace_lock);
		iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
		mutex_unlock(&ftrace_lock);

		if (!iter->hash) {
			trace_parser_put(&iter->parser);
			kfree(iter);
			return -ENOMEM;
		}
	}
2661

2662
	mutex_lock(&ftrace_regex_lock);
2663

2664
	if ((file->f_mode & FMODE_WRITE) &&
2665
	    (file->f_flags & O_TRUNC))
2666
		ftrace_filter_reset(iter->hash);
2667 2668 2669 2670 2671 2672 2673 2674

	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
2675
		} else {
2676 2677
			/* Failed */
			free_ftrace_hash(iter->hash);
2678
			trace_parser_put(&iter->parser);
2679
			kfree(iter);
2680
		}
2681 2682
	} else
		file->private_data = iter;
2683
	mutex_unlock(&ftrace_regex_lock);
2684 2685 2686 2687

	return ret;
}

2688 2689 2690
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
2691 2692 2693
	return ftrace_regex_open(&global_ops,
			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
			inode, file);
2694 2695 2696 2697 2698
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
2699
	return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2700
				 inode, file);
2701 2702
}

2703
loff_t
2704
ftrace_regex_lseek(struct file *file, loff_t offset, int whence)
2705 2706 2707 2708
{
	loff_t ret;

	if (file->f_mode & FMODE_READ)
2709
		ret = seq_lseek(file, offset, whence);
2710 2711 2712 2713 2714 2715
	else
		file->f_pos = ret = 1;

	return ret;
}

2716
static int ftrace_match(char *str, char *regex, int len, int type)
2717 2718
{
	int matched = 0;
2719
	int slen;
2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734

	switch (type) {
	case MATCH_FULL:
		if (strcmp(str, regex) == 0)
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
		if (strncmp(str, regex, len) == 0)
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
		if (strstr(str, regex))
			matched = 1;
		break;
	case MATCH_END_ONLY:
2735 2736
		slen = strlen(str);
		if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2737 2738 2739 2740 2741 2742 2743
			matched = 1;
		break;
	}

	return matched;
}

2744
static int
2745
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2746
{
2747 2748 2749
	struct ftrace_func_entry *entry;
	int ret = 0;

2750 2751 2752 2753 2754
	entry = ftrace_lookup_ip(hash, rec->ip);
	if (not) {
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
2755

2756
		free_hash_entry(hash, entry);
2757 2758 2759 2760
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
2761

2762
		ret = add_hash_entry(hash, rec->ip);
2763 2764
	}
	return ret;
2765 2766
}

2767
static int
2768 2769
ftrace_match_record(struct dyn_ftrace *rec, char *mod,
		    char *regex, int len, int type)
2770 2771
{
	char str[KSYM_SYMBOL_LEN];
2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

	if (mod) {
		/* module lookup requires matching the module */
		if (!modname || strcmp(modname, mod))
			return 0;

		/* blank search means to match all funcs in the mod */
		if (!len)
			return 1;
	}
2785 2786 2787 2788

	return ftrace_match(str, regex, len, type);
}

2789 2790 2791
static int
match_records(struct ftrace_hash *hash, char *buff,
	      int len, char *mod, int not)
2792
{
2793
	unsigned search_len = 0;
2794 2795
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
2796 2797
	int type = MATCH_FULL;
	char *search = buff;
2798
	int found = 0;
2799
	int ret;
2800

2801 2802 2803 2804
	if (len) {
		type = filter_parse_regex(buff, len, &search, &not);
		search_len = strlen(search);
	}
2805

2806
	mutex_lock(&ftrace_lock);
2807

2808 2809
	if (unlikely(ftrace_disabled))
		goto out_unlock;
2810

2811
	do_for_each_ftrace_rec(pg, rec) {
2812
		if (ftrace_match_record(rec, mod, search, search_len, type)) {
2813
			ret = enter_record(hash, rec, not);
2814 2815 2816 2817
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
2818
			found = 1;
2819 2820
		}
	} while_for_each_ftrace_rec();
2821
 out_unlock:
2822
	mutex_unlock(&ftrace_lock);
2823 2824

	return found;
2825 2826
}

2827
static int
2828
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2829
{
2830
	return match_records(hash, buff, len, NULL, 0);
2831 2832
}

2833 2834
static int
ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2835 2836
{
	int not = 0;
S
Steven Rostedt 已提交
2837

2838 2839 2840 2841 2842 2843 2844 2845 2846 2847
	/* blank or '*' mean the same */
	if (strcmp(buff, "*") == 0)
		buff[0] = 0;

	/* handle the case of 'dont filter this module' */
	if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
		buff[0] = 0;
		not = 1;
	}

2848
	return match_records(hash, buff, strlen(buff), mod, not);
2849 2850
}

2851 2852 2853 2854 2855 2856
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
2857 2858
ftrace_mod_callback(struct ftrace_hash *hash,
		    char *func, char *cmd, char *param, int enable)
2859 2860
{
	char *mod;
2861
	int ret = -EINVAL;
2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872

	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */

	/* we must have a module name */
	if (!param)
2873
		return ret;
2874 2875 2876

	mod = strsep(&param, ":");
	if (!strlen(mod))
2877
		return ret;
2878

2879
	ret = ftrace_match_module_records(hash, func, mod);
2880 2881 2882 2883 2884 2885
	if (!ret)
		ret = -EINVAL;
	if (ret < 0)
		return ret;

	return 0;
2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
2897
core_initcall(ftrace_mod_cmd_init);
2898

2899
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
2900
				      struct ftrace_ops *op, struct pt_regs *pt_regs)
2901
{
S
Steven Rostedt 已提交
2902
	struct ftrace_func_probe *entry;
2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918
	struct hlist_head *hhd;
	struct hlist_node *n;
	unsigned long key;

	key = hash_long(ip, FTRACE_HASH_BITS);

	hhd = &ftrace_func_hash[key];

	if (hlist_empty(hhd))
		return;

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
2919
	preempt_disable_notrace();
2920 2921 2922 2923
	hlist_for_each_entry_rcu(entry, n, hhd, node) {
		if (entry->ip == ip)
			entry->ops->func(ip, parent_ip, &entry->data);
	}
2924
	preempt_enable_notrace();
2925 2926
}

S
Steven Rostedt 已提交
2927
static struct ftrace_ops trace_probe_ops __read_mostly =
2928
{
2929
	.func		= function_trace_probe_call,
2930 2931
};

S
Steven Rostedt 已提交
2932
static int ftrace_probe_registered;
2933

S
Steven Rostedt 已提交
2934
static void __enable_ftrace_function_probe(void)
2935
{
2936
	int ret;
2937 2938
	int i;

S
Steven Rostedt 已提交
2939
	if (ftrace_probe_registered)
2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			break;
	}
	/* Nothing registered? */
	if (i == FTRACE_FUNC_HASHSIZE)
		return;

2951 2952
	ret = __register_ftrace_function(&trace_probe_ops);
	if (!ret)
2953
		ret = ftrace_startup(&trace_probe_ops, 0);
2954

S
Steven Rostedt 已提交
2955
	ftrace_probe_registered = 1;
2956 2957
}

S
Steven Rostedt 已提交
2958
static void __disable_ftrace_function_probe(void)
2959
{
2960
	int ret;
2961 2962
	int i;

S
Steven Rostedt 已提交
2963
	if (!ftrace_probe_registered)
2964 2965 2966 2967 2968 2969 2970 2971 2972
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			return;
	}

	/* no more funcs left */
2973 2974 2975 2976
	ret = __unregister_ftrace_function(&trace_probe_ops);
	if (!ret)
		ftrace_shutdown(&trace_probe_ops, 0);

S
Steven Rostedt 已提交
2977
	ftrace_probe_registered = 0;
2978 2979 2980 2981 2982
}


static void ftrace_free_entry_rcu(struct rcu_head *rhp)
{
S
Steven Rostedt 已提交
2983 2984
	struct ftrace_func_probe *entry =
		container_of(rhp, struct ftrace_func_probe, rcu);
2985 2986 2987 2988 2989 2990 2991 2992

	if (entry->ops->free)
		entry->ops->free(&entry->data);
	kfree(entry);
}


int
S
Steven Rostedt 已提交
2993
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2994 2995
			      void *data)
{
S
Steven Rostedt 已提交
2996
	struct ftrace_func_probe *entry;
2997 2998 2999
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	int type, len, not;
S
Steven Rostedt 已提交
3000
	unsigned long key;
3001 3002 3003
	int count = 0;
	char *search;

3004
	type = filter_parse_regex(glob, strlen(glob), &search, &not);
3005 3006
	len = strlen(search);

S
Steven Rostedt 已提交
3007
	/* we do not support '!' for function probes */
3008 3009 3010 3011 3012
	if (WARN_ON(not))
		return -EINVAL;

	mutex_lock(&ftrace_lock);

3013 3014
	if (unlikely(ftrace_disabled))
		goto out_unlock;
3015

3016
	do_for_each_ftrace_rec(pg, rec) {
3017

3018
		if (!ftrace_match_record(rec, NULL, search, len, type))
3019 3020 3021 3022
			continue;

		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
		if (!entry) {
S
Steven Rostedt 已提交
3023
			/* If we did not process any, then return error */
3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052
			if (!count)
				count = -ENOMEM;
			goto out_unlock;
		}

		count++;

		entry->data = data;

		/*
		 * The caller might want to do something special
		 * for each function we find. We call the callback
		 * to give the caller an opportunity to do so.
		 */
		if (ops->callback) {
			if (ops->callback(rec->ip, &entry->data) < 0) {
				/* caller does not like this func */
				kfree(entry);
				continue;
			}
		}

		entry->ops = ops;
		entry->ip = rec->ip;

		key = hash_long(entry->ip, FTRACE_HASH_BITS);
		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);

	} while_for_each_ftrace_rec();
S
Steven Rostedt 已提交
3053
	__enable_ftrace_function_probe();
3054 3055 3056 3057 3058 3059 3060 3061

 out_unlock:
	mutex_unlock(&ftrace_lock);

	return count;
}

enum {
S
Steven Rostedt 已提交
3062 3063
	PROBE_TEST_FUNC		= 1,
	PROBE_TEST_DATA		= 2
3064 3065 3066
};

static void
S
Steven Rostedt 已提交
3067
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3068 3069
				  void *data, int flags)
{
S
Steven Rostedt 已提交
3070
	struct ftrace_func_probe *entry;
3071 3072 3073 3074 3075 3076
	struct hlist_node *n, *tmp;
	char str[KSYM_SYMBOL_LEN];
	int type = MATCH_FULL;
	int i, len = 0;
	char *search;

3077
	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
3078
		glob = NULL;
3079
	else if (glob) {
3080 3081
		int not;

3082
		type = filter_parse_regex(glob, strlen(glob), &search, &not);
3083 3084
		len = strlen(search);

S
Steven Rostedt 已提交
3085
		/* we do not support '!' for function probes */
3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096
		if (WARN_ON(not))
			return;
	}

	mutex_lock(&ftrace_lock);
	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];

		hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {

			/* break up if statements for readability */
S
Steven Rostedt 已提交
3097
			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3098 3099
				continue;

S
Steven Rostedt 已提交
3100
			if ((flags & PROBE_TEST_DATA) && entry->data != data)
3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114
				continue;

			/* do this last, since it is the most expensive */
			if (glob) {
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
				if (!ftrace_match(str, glob, len, type))
					continue;
			}

			hlist_del(&entry->node);
			call_rcu(&entry->rcu, ftrace_free_entry_rcu);
		}
	}
S
Steven Rostedt 已提交
3115
	__disable_ftrace_function_probe();
3116 3117 3118 3119
	mutex_unlock(&ftrace_lock);
}

void
S
Steven Rostedt 已提交
3120
unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3121 3122
				void *data)
{
S
Steven Rostedt 已提交
3123 3124
	__unregister_ftrace_function_probe(glob, ops, data,
					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
3125 3126 3127
}

void
S
Steven Rostedt 已提交
3128
unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
3129
{
S
Steven Rostedt 已提交
3130
	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
3131 3132
}

S
Steven Rostedt 已提交
3133
void unregister_ftrace_function_probe_all(char *glob)
3134
{
S
Steven Rostedt 已提交
3135
	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
3136 3137
}

3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

int register_ftrace_command(struct ftrace_func_command *cmd)
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

int unregister_ftrace_command(struct ftrace_func_command *cmd)
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

3179 3180
static int ftrace_process_regex(struct ftrace_hash *hash,
				char *buff, int len, int enable)
3181
{
3182
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
3183
	struct ftrace_func_command *p;
3184
	int ret = -EINVAL;
3185 3186 3187 3188

	func = strsep(&next, ":");

	if (!next) {
3189
		ret = ftrace_match_records(hash, func, len);
3190 3191 3192 3193 3194
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
3195 3196
	}

3197
	/* command found */
3198 3199 3200

	command = strsep(&next, ":");

3201 3202 3203
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
3204
			ret = p->func(hash, func, command, next, enable);
3205 3206
			goto out_unlock;
		}
3207
	}
3208 3209
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
3210

3211
	return ret;
3212 3213
}

I
Ingo Molnar 已提交
3214
static ssize_t
3215 3216
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
3217 3218
{
	struct ftrace_iterator *iter;
3219 3220
	struct trace_parser *parser;
	ssize_t ret, read;
3221

3222
	if (!cnt)
3223 3224
		return 0;

3225
	mutex_lock(&ftrace_regex_lock);
3226

3227 3228 3229 3230
	ret = -ENODEV;
	if (unlikely(ftrace_disabled))
		goto out_unlock;

3231 3232 3233 3234 3235 3236
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

3237 3238
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
3239

3240
	if (read >= 0 && trace_parser_loaded(parser) &&
3241
	    !trace_parser_cont(parser)) {
3242
		ret = ftrace_process_regex(iter->hash, parser->buffer,
3243
					   parser->idx, enable);
3244
		trace_parser_clear(parser);
3245
		if (ret)
3246
			goto out_unlock;
3247
	}
3248 3249

	ret = read;
3250
out_unlock:
3251
	mutex_unlock(&ftrace_regex_lock);
3252

3253 3254 3255
	return ret;
}

3256
ssize_t
3257 3258 3259 3260 3261 3262
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

3263
ssize_t
3264 3265 3266 3267 3268 3269
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

3270
static int
3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291
ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
	struct ftrace_func_entry *entry;

	if (!ftrace_location(ip))
		return -EINVAL;

	if (remove) {
		entry = ftrace_lookup_ip(hash, ip);
		if (!entry)
			return -ENOENT;
		free_hash_entry(hash, entry);
		return 0;
	}

	return add_hash_entry(hash, ip);
}

static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
		unsigned long ip, int remove, int reset, int enable)
3292
{
3293
	struct ftrace_hash **orig_hash;
3294
	struct ftrace_hash *hash;
3295
	int ret;
3296

3297 3298 3299 3300
	/* All global ops uses the global ops filters */
	if (ops->flags & FTRACE_OPS_FL_GLOBAL)
		ops = &global_ops;

3301
	if (unlikely(ftrace_disabled))
3302
		return -ENODEV;
3303

3304
	if (enable)
3305
		orig_hash = &ops->filter_hash;
3306
	else
3307 3308 3309 3310 3311
		orig_hash = &ops->notrace_hash;

	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
	if (!hash)
		return -ENOMEM;
3312

3313 3314
	mutex_lock(&ftrace_regex_lock);
	if (reset)
3315
		ftrace_filter_reset(hash);
3316 3317 3318 3319
	if (buf && !ftrace_match_records(hash, buf, len)) {
		ret = -EINVAL;
		goto out_regex_unlock;
	}
3320 3321 3322 3323 3324
	if (ip) {
		ret = ftrace_match_addr(hash, ip, remove);
		if (ret < 0)
			goto out_regex_unlock;
	}
3325 3326

	mutex_lock(&ftrace_lock);
3327
	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3328 3329
	if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
	    && ftrace_enabled)
3330
		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3331

3332 3333
	mutex_unlock(&ftrace_lock);

3334
 out_regex_unlock:
3335
	mutex_unlock(&ftrace_regex_lock);
3336 3337 3338

	free_ftrace_hash(hash);
	return ret;
3339 3340
}

3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
		int reset, int enable)
{
	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
}

/**
 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
 * @ops - the ops to set the filter with
 * @ip - the address to add to or remove from the filter.
 * @remove - non zero to remove the ip from the filter
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled
 * If @ip is NULL, it failes to update filter.
 */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
			 int remove, int reset)
{
	return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);

static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
{
	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
}

3372 3373
/**
 * ftrace_set_filter - set a function to filter on in ftrace
3374 3375 3376 3377 3378 3379 3380 3381
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
3382
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
3383 3384
		       int len, int reset)
{
3385
	return ftrace_set_regex(ops, buf, len, reset, 1);
3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
3400
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
3401 3402
			int len, int reset)
{
3403
	return ftrace_set_regex(ops, buf, len, reset, 0);
3404 3405 3406 3407 3408
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
 * ftrace_set_filter - set a function to filter on in ftrace
 * @ops - the ops to set the filter with
3409 3410 3411 3412 3413 3414 3415
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
3416
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3417
{
3418
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
3419
}
3420
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3421

3422 3423
/**
 * ftrace_set_notrace - set a function to not trace in ftrace
3424
 * @ops - the ops to set the notrace filter with
3425 3426 3427 3428 3429 3430 3431 3432
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
3433
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3434
{
3435
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
3436
}
3437
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3438

3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

static int __init set_ftrace_notrace(char *str)
{
	strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
	strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

3460
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3461
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3462 3463
static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);

3464 3465
static int __init set_graph_function(char *str)
{
3466
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

static void __init set_ftrace_early_graph(char *buf)
{
	int ret;
	char *func;

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
				      func);
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

3488 3489
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3490 3491 3492 3493 3494
{
	char *func;

	while (buf) {
		func = strsep(&buf, ",");
3495
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
3496 3497 3498 3499 3500 3501
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
3502
		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
3503
	if (ftrace_notrace_buf[0])
3504
		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
3505 3506 3507 3508
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
		set_ftrace_early_graph(ftrace_graph_buf);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3509 3510
}

3511
int ftrace_regex_release(struct inode *inode, struct file *file)
3512 3513 3514
{
	struct seq_file *m = (struct seq_file *)file->private_data;
	struct ftrace_iterator *iter;
3515
	struct ftrace_hash **orig_hash;
3516
	struct trace_parser *parser;
3517
	int filter_hash;
3518
	int ret;
3519

3520
	mutex_lock(&ftrace_regex_lock);
3521 3522 3523 3524 3525 3526 3527
	if (file->f_mode & FMODE_READ) {
		iter = m->private;

		seq_release(inode, file);
	} else
		iter = file->private_data;

3528 3529 3530
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
3531
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3532 3533
	}

3534 3535
	trace_parser_put(parser);

3536
	if (file->f_mode & FMODE_WRITE) {
3537 3538 3539
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

		if (filter_hash)
3540
			orig_hash = &iter->ops->filter_hash;
3541 3542
		else
			orig_hash = &iter->ops->notrace_hash;
3543

3544
		mutex_lock(&ftrace_lock);
3545 3546 3547 3548
		ret = ftrace_hash_move(iter->ops, filter_hash,
				       orig_hash, iter->hash);
		if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
		    && ftrace_enabled)
3549
			ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3550

3551 3552
		mutex_unlock(&ftrace_lock);
	}
3553 3554
	free_ftrace_hash(iter->hash);
	kfree(iter);
3555

3556
	mutex_unlock(&ftrace_regex_lock);
3557 3558 3559
	return 0;
}

3560
static const struct file_operations ftrace_avail_fops = {
3561 3562 3563
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
3564
	.release = seq_release_private,
3565 3566
};

3567 3568 3569 3570 3571 3572 3573
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

3574
static const struct file_operations ftrace_filter_fops = {
3575
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
3576
	.read = seq_read,
3577
	.write = ftrace_filter_write,
3578
	.llseek = ftrace_regex_lseek,
3579
	.release = ftrace_regex_release,
3580 3581
};

3582
static const struct file_operations ftrace_notrace_fops = {
3583
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
3584
	.read = seq_read,
3585 3586
	.write = ftrace_notrace_write,
	.llseek = ftrace_regex_lseek,
3587
	.release = ftrace_regex_release,
3588 3589
};

3590 3591 3592 3593 3594
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

int ftrace_graph_count;
3595
int ftrace_graph_filter_enabled;
3596 3597 3598
unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;

static void *
3599
__g_next(struct seq_file *m, loff_t *pos)
3600
{
3601
	if (*pos >= ftrace_graph_count)
3602
		return NULL;
3603
	return &ftrace_graph_funcs[*pos];
3604
}
3605

3606 3607 3608 3609 3610
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
3611 3612 3613 3614 3615 3616
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&graph_lock);

3617
	/* Nothing, tell g_show to print all functions are enabled */
3618
	if (!ftrace_graph_filter_enabled && !*pos)
3619 3620
		return (void *)1;

3621
	return __g_next(m, pos);
3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
	unsigned long *ptr = v;

	if (!ptr)
		return 0;

3636 3637 3638 3639 3640
	if (ptr == (unsigned long *)1) {
		seq_printf(m, "#### all functions enabled ####\n");
		return 0;
	}

3641
	seq_printf(m, "%ps\n", (void *)*ptr);
3642 3643 3644 3645

	return 0;
}

J
James Morris 已提交
3646
static const struct seq_operations ftrace_graph_seq_ops = {
3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	int ret = 0;

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	mutex_lock(&graph_lock);
	if ((file->f_mode & FMODE_WRITE) &&
3663
	    (file->f_flags & O_TRUNC)) {
3664
		ftrace_graph_filter_enabled = 0;
3665 3666 3667
		ftrace_graph_count = 0;
		memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
	}
3668
	mutex_unlock(&graph_lock);
3669

3670
	if (file->f_mode & FMODE_READ)
3671 3672 3673 3674 3675
		ret = seq_open(file, &ftrace_graph_seq_ops);

	return ret;
}

3676 3677 3678 3679 3680 3681 3682 3683
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
	if (file->f_mode & FMODE_READ)
		seq_release(inode, file);
	return 0;
}

3684
static int
3685
ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3686 3687 3688
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
3689
	int search_len;
3690
	int fail = 1;
3691 3692 3693 3694
	int type, not;
	char *search;
	bool exists;
	int i;
3695

3696
	/* decode regex */
3697
	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3698 3699
	if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
		return -EBUSY;
3700 3701 3702

	search_len = strlen(search);

3703
	mutex_lock(&ftrace_lock);
3704 3705 3706 3707 3708 3709

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

3710 3711
	do_for_each_ftrace_rec(pg, rec) {

3712
		if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3713
			/* if it is in the array */
3714
			exists = false;
3715
			for (i = 0; i < *idx; i++) {
3716 3717
				if (array[i] == rec->ip) {
					exists = true;
3718 3719
					break;
				}
3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735
			}

			if (!not) {
				fail = 0;
				if (!exists) {
					array[(*idx)++] = rec->ip;
					if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
						goto out;
				}
			} else {
				if (exists) {
					array[i] = array[--(*idx)];
					array[*idx] = 0;
					fail = 0;
				}
			}
3736
		}
3737
	} while_for_each_ftrace_rec();
3738
out:
3739
	mutex_unlock(&ftrace_lock);
3740

3741 3742 3743 3744 3745
	if (fail)
		return -EINVAL;

	ftrace_graph_filter_enabled = 1;
	return 0;
3746 3747 3748 3749 3750 3751
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
3752
	struct trace_parser parser;
3753
	ssize_t read, ret;
3754

3755
	if (!cnt)
3756 3757 3758 3759
		return 0;

	mutex_lock(&graph_lock);

3760 3761
	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
		ret = -ENOMEM;
3762
		goto out_unlock;
3763 3764
	}

3765
	read = trace_get_user(&parser, ubuf, cnt, ppos);
3766

3767
	if (read >= 0 && trace_parser_loaded((&parser))) {
3768 3769 3770
		parser.buffer[parser.idx] = 0;

		/* we allow only one expression at a time */
3771
		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3772
					parser.buffer);
3773
		if (ret)
3774
			goto out_free;
3775 3776 3777
	}

	ret = read;
3778 3779

out_free:
3780
	trace_parser_put(&parser);
3781
out_unlock:
3782 3783 3784 3785 3786 3787
	mutex_unlock(&graph_lock);

	return ret;
}

static const struct file_operations ftrace_graph_fops = {
3788 3789 3790 3791
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
	.release	= ftrace_graph_release,
3792
	.llseek		= seq_lseek,
3793 3794 3795
};
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

3796
static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3797 3798
{

3799 3800
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
3801

3802 3803 3804
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

3805 3806
	trace_create_file("set_ftrace_filter", 0644, d_tracer,
			NULL, &ftrace_filter_fops);
3807

3808
	trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3809
				    NULL, &ftrace_notrace_fops);
3810

3811
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3812
	trace_create_file("set_graph_function", 0444, d_tracer,
3813 3814 3815 3816
				    NULL,
				    &ftrace_graph_fops);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

3817 3818 3819
	return 0;
}

3820
static int ftrace_cmp_ips(const void *a, const void *b)
3821
{
3822 3823
	const unsigned long *ipa = a;
	const unsigned long *ipb = b;
3824

3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840
	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

static void ftrace_swap_ips(void *a, void *b, int size)
{
	unsigned long *ipa = a;
	unsigned long *ipb = b;
	unsigned long t;

	t = *ipa;
	*ipa = *ipb;
	*ipb = t;
3841 3842
}

3843
static int ftrace_process_locs(struct module *mod,
3844
			       unsigned long *start,
3845 3846
			       unsigned long *end)
{
3847
	struct ftrace_page *start_pg;
3848
	struct ftrace_page *pg;
3849
	struct dyn_ftrace *rec;
3850
	unsigned long count;
3851 3852
	unsigned long *p;
	unsigned long addr;
3853
	unsigned long flags = 0; /* Shut up gcc */
3854 3855 3856 3857 3858 3859 3860
	int ret = -ENOMEM;

	count = end - start;

	if (!count)
		return 0;

3861 3862 3863
	sort(start, count, sizeof(*start),
	     ftrace_cmp_ips, ftrace_swap_ips);

3864 3865
	start_pg = ftrace_allocate_pages(count);
	if (!start_pg)
3866
		return -ENOMEM;
3867

S
Steven Rostedt 已提交
3868
	mutex_lock(&ftrace_lock);
3869

3870 3871 3872 3873 3874
	/*
	 * Core and each module needs their own pages, as
	 * modules will free them when they are removed.
	 * Force a new page to be allocated for modules.
	 */
3875 3876 3877
	if (!mod) {
		WARN_ON(ftrace_pages || ftrace_pages_start);
		/* First initialization */
3878
		ftrace_pages = ftrace_pages_start = start_pg;
3879
	} else {
3880
		if (!ftrace_pages)
3881
			goto out;
3882

3883 3884 3885 3886
		if (WARN_ON(ftrace_pages->next)) {
			/* Hmm, we have free pages? */
			while (ftrace_pages->next)
				ftrace_pages = ftrace_pages->next;
3887
		}
3888

3889
		ftrace_pages->next = start_pg;
3890 3891
	}

3892
	p = start;
3893
	pg = start_pg;
3894 3895
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
3896 3897 3898 3899 3900 3901 3902 3903
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
3904 3905 3906 3907 3908 3909 3910 3911 3912 3913

		if (pg->index == pg->size) {
			/* We should have allocated enough */
			if (WARN_ON(!pg->next))
				break;
			pg = pg->next;
		}

		rec = &pg->records[pg->index++];
		rec->ip = addr;
3914 3915
	}

3916 3917 3918 3919 3920 3921
	/* We should have used all pages */
	WARN_ON(pg->next);

	/* Assign the last page to ftrace_pages */
	ftrace_pages = pg;

3922
	/* These new locations need to be initialized */
3923
	ftrace_new_pgs = start_pg;
3924

3925
	/*
3926 3927 3928 3929 3930 3931
	 * We only need to disable interrupts on start up
	 * because we are modifying code that an interrupt
	 * may execute, and the modification is not atomic.
	 * But for modules, nothing runs the code we modify
	 * until we are finished with it, and there's no
	 * reason to cause large interrupt latencies while we do it.
3932
	 */
3933 3934
	if (!mod)
		local_irq_save(flags);
3935
	ftrace_update_code(mod);
3936 3937
	if (!mod)
		local_irq_restore(flags);
3938 3939
	ret = 0;
 out:
S
Steven Rostedt 已提交
3940
	mutex_unlock(&ftrace_lock);
3941

3942
	return ret;
3943 3944
}

3945
#ifdef CONFIG_MODULES
3946 3947 3948

#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)

3949
void ftrace_release_mod(struct module *mod)
3950 3951
{
	struct dyn_ftrace *rec;
3952
	struct ftrace_page **last_pg;
3953
	struct ftrace_page *pg;
3954
	int order;
3955

3956 3957
	mutex_lock(&ftrace_lock);

3958
	if (ftrace_disabled)
3959
		goto out_unlock;
3960

3961 3962 3963 3964 3965 3966 3967
	/*
	 * Each module has its own ftrace_pages, remove
	 * them from the list.
	 */
	last_pg = &ftrace_pages_start;
	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
		rec = &pg->records[0];
3968
		if (within_module_core(rec->ip, mod)) {
3969
			/*
3970 3971
			 * As core pages are first, the first
			 * page should never be a module page.
3972
			 */
3973 3974 3975 3976 3977 3978 3979 3980
			if (WARN_ON(pg == ftrace_pages_start))
				goto out_unlock;

			/* Check if we are deleting the last page */
			if (pg == ftrace_pages)
				ftrace_pages = next_to_ftrace_page(last_pg);

			*last_pg = pg->next;
3981 3982 3983
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
3984 3985 3986
		} else
			last_pg = &pg->next;
	}
3987
 out_unlock:
3988 3989 3990 3991 3992
	mutex_unlock(&ftrace_lock);
}

static void ftrace_init_module(struct module *mod,
			       unsigned long *start, unsigned long *end)
3993
{
3994
	if (ftrace_disabled || start == end)
3995
		return;
3996
	ftrace_process_locs(mod, start, end);
3997 3998
}

3999 4000
static int ftrace_module_notify_enter(struct notifier_block *self,
				      unsigned long val, void *data)
4001 4002 4003
{
	struct module *mod = data;

4004
	if (val == MODULE_STATE_COMING)
4005 4006 4007
		ftrace_init_module(mod, mod->ftrace_callsites,
				   mod->ftrace_callsites +
				   mod->num_ftrace_callsites);
4008 4009 4010 4011 4012 4013 4014 4015 4016
	return 0;
}

static int ftrace_module_notify_exit(struct notifier_block *self,
				     unsigned long val, void *data)
{
	struct module *mod = data;

	if (val == MODULE_STATE_GOING)
4017
		ftrace_release_mod(mod);
4018 4019 4020 4021

	return 0;
}
#else
4022 4023 4024 4025 4026 4027 4028
static int ftrace_module_notify_enter(struct notifier_block *self,
				      unsigned long val, void *data)
{
	return 0;
}
static int ftrace_module_notify_exit(struct notifier_block *self,
				     unsigned long val, void *data)
4029 4030 4031 4032 4033
{
	return 0;
}
#endif /* CONFIG_MODULES */

4034 4035
struct notifier_block ftrace_module_enter_nb = {
	.notifier_call = ftrace_module_notify_enter,
4036
	.priority = INT_MAX,	/* Run before anything that can use kprobes */
4037 4038
};

4039 4040 4041 4042 4043
struct notifier_block ftrace_module_exit_nb = {
	.notifier_call = ftrace_module_notify_exit,
	.priority = INT_MIN,	/* Run after anything that can remove kprobes */
};

4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070
extern unsigned long __start_mcount_loc[];
extern unsigned long __stop_mcount_loc[];

void __init ftrace_init(void)
{
	unsigned long count, addr, flags;
	int ret;

	/* Keep the ftrace pointer to the stub */
	addr = (unsigned long)ftrace_stub;

	local_irq_save(flags);
	ftrace_dyn_arch_init(&addr);
	local_irq_restore(flags);

	/* ftrace_dyn_arch_init places the return code in addr */
	if (addr)
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;

	ret = ftrace_dyn_table_alloc(count);
	if (ret)
		goto failed;

	last_ftrace_enabled = ftrace_enabled = 1;

4071
	ret = ftrace_process_locs(NULL,
4072
				  __start_mcount_loc,
4073 4074
				  __stop_mcount_loc);

4075 4076 4077 4078 4079
	ret = register_module_notifier(&ftrace_module_enter_nb);
	if (ret)
		pr_warning("Failed to register trace ftrace module enter notifier\n");

	ret = register_module_notifier(&ftrace_module_exit_nb);
4080
	if (ret)
4081
		pr_warning("Failed to register trace ftrace module exit notifier\n");
4082

4083 4084
	set_ftrace_early_filters();

4085 4086 4087 4088 4089
	return;
 failed:
	ftrace_disabled = 1;
}

4090
#else
4091

4092
static struct ftrace_ops global_ops = {
4093
	.func			= ftrace_stub,
4094
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE,
4095 4096
};

4097 4098 4099 4100 4101
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
4102
core_initcall(ftrace_nodyn_init);
4103

4104 4105
static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
static inline void ftrace_startup_enable(int command) { }
4106
/* Keep as macros so we do not need to define the commands */
4107 4108 4109 4110 4111
# define ftrace_startup(ops, command)			\
	({						\
		(ops)->flags |= FTRACE_OPS_FL_ENABLED;	\
		0;					\
	})
4112
# define ftrace_shutdown(ops, command)	do { } while (0)
I
Ingo Molnar 已提交
4113 4114
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
4115 4116 4117 4118 4119 4120 4121

static inline int
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
{
	return 1;
}

4122 4123
#endif /* CONFIG_DYNAMIC_FTRACE */

4124
static void
4125
ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
4126
			struct ftrace_ops *op, struct pt_regs *regs)
4127 4128 4129 4130 4131 4132 4133 4134 4135 4136
{
	if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))
		return;

	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
	trace_recursion_set(TRACE_CONTROL_BIT);
4137
	do_for_each_ftrace_op(op, ftrace_control_list) {
4138 4139
		if (!ftrace_function_local_disabled(op) &&
		    ftrace_ops_test(op, ip))
4140
			op->func(ip, parent_ip, op, regs);
4141
	} while_for_each_ftrace_op(op);
4142 4143 4144 4145 4146 4147
	trace_recursion_clear(TRACE_CONTROL_BIT);
	preempt_enable_notrace();
}

static struct ftrace_ops control_ops = {
	.func = ftrace_ops_control_func,
4148
	.flags = FTRACE_OPS_FL_RECURSION_SAFE,
4149 4150
};

4151 4152
static inline void
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4153
		       struct ftrace_ops *ignored, struct pt_regs *regs)
4154
{
4155
	struct ftrace_ops *op;
4156
	int bit;
4157

4158 4159 4160
	if (function_trace_stop)
		return;

4161 4162 4163
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;
4164

4165 4166 4167 4168 4169
	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
4170
	do_for_each_ftrace_op(op, ftrace_ops_list) {
4171
		if (ftrace_ops_test(op, ip))
4172
			op->func(ip, parent_ip, op, regs);
4173
	} while_for_each_ftrace_op(op);
4174
	preempt_enable_notrace();
4175
	trace_clear_recursion(bit);
4176 4177
}

4178 4179 4180 4181 4182
/*
 * Some archs only support passing ip and parent_ip. Even though
 * the list function ignores the op parameter, we do not want any
 * C side effects, where a function is called without the caller
 * sending a third parameter.
4183 4184 4185
 * Archs are to support both the regs and ftrace_ops at the same time.
 * If they support ftrace_ops, it is assumed they support regs.
 * If call backs want to use regs, they must either check for regs
4186 4187
 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
4188 4189
 * An architecture can pass partial regs with ftrace_ops and still
 * set the ARCH_SUPPORT_FTARCE_OPS.
4190 4191 4192
 */
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4193
				 struct ftrace_ops *op, struct pt_regs *regs)
4194
{
4195
	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
4196 4197 4198 4199
}
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
{
4200
	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
4201 4202 4203
}
#endif

4204
static void clear_ftrace_swapper(void)
S
Steven Rostedt 已提交
4205 4206
{
	struct task_struct *p;
4207
	int cpu;
S
Steven Rostedt 已提交
4208

4209 4210 4211
	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
S
Steven Rostedt 已提交
4212
		clear_tsk_trace_trace(p);
4213 4214 4215
	}
	put_online_cpus();
}
S
Steven Rostedt 已提交
4216

4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227
static void set_ftrace_swapper(void)
{
	struct task_struct *p;
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
		set_tsk_trace_trace(p);
	}
	put_online_cpus();
S
Steven Rostedt 已提交
4228 4229
}

4230 4231 4232 4233
static void clear_ftrace_pid(struct pid *pid)
{
	struct task_struct *p;

4234
	rcu_read_lock();
4235 4236 4237
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		clear_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
4238 4239
	rcu_read_unlock();

4240 4241 4242 4243
	put_pid(pid);
}

static void set_ftrace_pid(struct pid *pid)
S
Steven Rostedt 已提交
4244 4245 4246
{
	struct task_struct *p;

4247
	rcu_read_lock();
S
Steven Rostedt 已提交
4248 4249 4250
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		set_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
4251
	rcu_read_unlock();
S
Steven Rostedt 已提交
4252 4253
}

4254
static void clear_ftrace_pid_task(struct pid *pid)
4255
{
4256
	if (pid == ftrace_swapper_pid)
4257 4258
		clear_ftrace_swapper();
	else
4259
		clear_ftrace_pid(pid);
4260 4261 4262 4263 4264 4265 4266 4267 4268 4269
}

static void set_ftrace_pid_task(struct pid *pid)
{
	if (pid == ftrace_swapper_pid)
		set_ftrace_swapper();
	else
		set_ftrace_pid(pid);
}

4270
static int ftrace_pid_add(int p)
4271
{
S
Steven Rostedt 已提交
4272
	struct pid *pid;
4273 4274
	struct ftrace_pid *fpid;
	int ret = -EINVAL;
4275

4276
	mutex_lock(&ftrace_lock);
4277

4278 4279 4280 4281
	if (!p)
		pid = ftrace_swapper_pid;
	else
		pid = find_get_pid(p);
4282

4283 4284
	if (!pid)
		goto out;
4285

4286
	ret = 0;
4287

4288 4289 4290
	list_for_each_entry(fpid, &ftrace_pids, list)
		if (fpid->pid == pid)
			goto out_put;
S
Steven Rostedt 已提交
4291

4292
	ret = -ENOMEM;
4293

4294 4295 4296
	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
	if (!fpid)
		goto out_put;
4297

4298 4299
	list_add(&fpid->list, &ftrace_pids);
	fpid->pid = pid;
4300

4301
	set_ftrace_pid_task(pid);
S
Steven Rostedt 已提交
4302

4303 4304 4305 4306 4307 4308 4309 4310 4311
	ftrace_update_pid_func();
	ftrace_startup_enable(0);

	mutex_unlock(&ftrace_lock);
	return 0;

out_put:
	if (pid != ftrace_swapper_pid)
		put_pid(pid);
S
Steven Rostedt 已提交
4312

4313 4314 4315 4316 4317 4318 4319 4320
out:
	mutex_unlock(&ftrace_lock);
	return ret;
}

static void ftrace_pid_reset(void)
{
	struct ftrace_pid *fpid, *safe;
S
Steven Rostedt 已提交
4321

4322 4323 4324 4325 4326 4327 4328 4329
	mutex_lock(&ftrace_lock);
	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
		struct pid *pid = fpid->pid;

		clear_ftrace_pid_task(pid);

		list_del(&fpid->list);
		kfree(fpid);
4330 4331 4332 4333 4334
	}

	ftrace_update_pid_func();
	ftrace_startup_enable(0);

S
Steven Rostedt 已提交
4335
	mutex_unlock(&ftrace_lock);
4336
}
4337

4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399
static void *fpid_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&ftrace_lock);

	if (list_empty(&ftrace_pids) && (!*pos))
		return (void *) 1;

	return seq_list_start(&ftrace_pids, *pos);
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
	if (v == (void *)1)
		return NULL;

	return seq_list_next(v, &ftrace_pids, pos);
}

static void fpid_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);

	if (v == (void *)1) {
		seq_printf(m, "no pid\n");
		return 0;
	}

	if (fpid->pid == ftrace_swapper_pid)
		seq_printf(m, "swapper tasks\n");
	else
		seq_printf(m, "%u\n", pid_vnr(fpid->pid));

	return 0;
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
	int ret = 0;

	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
		ftrace_pid_reset();

	if (file->f_mode & FMODE_READ)
		ret = seq_open(file, &ftrace_pid_sops);

	return ret;
}

4400 4401 4402 4403
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
4404
	char buf[64], *tmp;
4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415
	long val;
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

4416 4417 4418 4419
	/*
	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
	 * to clean the filter quietly.
	 */
4420 4421
	tmp = strstrip(buf);
	if (strlen(tmp) == 0)
4422 4423
		return 1;

4424
	ret = kstrtol(tmp, 10, &val);
4425 4426 4427
	if (ret < 0)
		return ret;

4428
	ret = ftrace_pid_add(val);
4429

4430 4431
	return ret ? ret : cnt;
}
4432

4433 4434 4435 4436 4437
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
	if (file->f_mode & FMODE_READ)
		seq_release(inode, file);
4438

4439
	return 0;
4440 4441
}

4442
static const struct file_operations ftrace_pid_fops = {
4443 4444 4445 4446 4447
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= ftrace_pid_release,
4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459
};

static __init int ftrace_init_debugfs(void)
{
	struct dentry *d_tracer;

	d_tracer = tracing_init_dentry();
	if (!d_tracer)
		return 0;

	ftrace_init_dyn_debugfs(d_tracer);

4460 4461
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
			    NULL, &ftrace_pid_fops);
4462 4463 4464

	ftrace_profile_debugfs(d_tracer);

4465 4466 4467 4468
	return 0;
}
fs_initcall(ftrace_init_debugfs);

S
Steven Rostedt 已提交
4469
/**
4470
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
4471 4472 4473 4474 4475
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
4476
void ftrace_kill(void)
S
Steven Rostedt 已提交
4477 4478 4479 4480 4481 4482
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

4483 4484 4485 4486 4487 4488 4489 4490
/**
 * Test if ftrace is dead or not.
 */
int ftrace_is_dead(void)
{
	return ftrace_disabled;
}

4491
/**
4492 4493
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
4494
 *
4495 4496 4497 4498 4499 4500
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
4501
 */
4502
int register_ftrace_function(struct ftrace_ops *ops)
4503
{
4504
	int ret = -1;
4505

S
Steven Rostedt 已提交
4506
	mutex_lock(&ftrace_lock);
4507

4508
	ret = __register_ftrace_function(ops);
4509
	if (!ret)
4510
		ret = ftrace_startup(ops, 0);
4511

S
Steven Rostedt 已提交
4512
	mutex_unlock(&ftrace_lock);
4513

4514
	return ret;
4515
}
4516
EXPORT_SYMBOL_GPL(register_ftrace_function);
4517 4518

/**
4519
 * unregister_ftrace_function - unregister a function for profiling.
4520 4521 4522 4523 4524 4525 4526 4527
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
4528
	mutex_lock(&ftrace_lock);
4529
	ret = __unregister_ftrace_function(ops);
4530 4531
	if (!ret)
		ftrace_shutdown(ops, 0);
S
Steven Rostedt 已提交
4532
	mutex_unlock(&ftrace_lock);
4533 4534 4535

	return ret;
}
4536
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
4537

I
Ingo Molnar 已提交
4538
int
4539
ftrace_enable_sysctl(struct ctl_table *table, int write,
4540
		     void __user *buffer, size_t *lenp,
4541 4542
		     loff_t *ppos)
{
4543
	int ret = -ENODEV;
4544

S
Steven Rostedt 已提交
4545
	mutex_lock(&ftrace_lock);
4546

4547 4548 4549 4550
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
4551

4552
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
4553 4554
		goto out;

4555
	last_ftrace_enabled = !!ftrace_enabled;
4556 4557 4558 4559 4560 4561

	if (ftrace_enabled) {

		ftrace_startup_sysctl();

		/* we are starting ftrace again */
4562 4563 4564
		if (ftrace_ops_list != &ftrace_list_end) {
			if (ftrace_ops_list->next == &ftrace_list_end)
				ftrace_trace_function = ftrace_ops_list->func;
4565
			else
4566
				ftrace_trace_function = ftrace_ops_list_func;
4567 4568 4569 4570 4571 4572 4573 4574 4575 4576
		}

	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
4577
	mutex_unlock(&ftrace_lock);
4578
	return ret;
4579
}
4580

4581
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4582

4583
static int ftrace_graph_active;
4584
static struct notifier_block ftrace_suspend_notifier;
4585

4586 4587 4588 4589 4590
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

4591 4592 4593
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
4594
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	unsigned long flags;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

	read_lock_irqsave(&tasklist_lock, flags);
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
4625
			atomic_set(&t->tracing_graph_pause, 0);
4626
			atomic_set(&t->trace_overrun, 0);
4627 4628 4629 4630
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641
		}
	} while_each_thread(g, t);

unlock:
	read_unlock_irqrestore(&tasklist_lock, flags);
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

4642
static void
4643 4644
ftrace_graph_probe_sched_switch(void *ignore,
			struct task_struct *prev, struct task_struct *next)
4645 4646 4647 4648
{
	unsigned long long timestamp;
	int index;

4649 4650 4651 4652 4653 4654 4655
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
	if (trace_flags & TRACE_ITER_SLEEP_TIME)
		return;

4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

4674
/* Allocate a return stack for each task */
4675
static int start_graph_tracing(void)
4676 4677
{
	struct ftrace_ret_stack **ret_stack_list;
4678
	int ret, cpu;
4679 4680 4681 4682 4683 4684 4685 4686

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

4687
	/* The cpu_boot init_task->ret_stack will never be freed */
4688 4689
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
4690
			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4691
	}
4692

4693 4694 4695 4696
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

4697
	if (!ret) {
4698
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4699 4700 4701 4702 4703
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

4704 4705 4706 4707
	kfree(ret_stack_list);
	return ret;
}

4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

4729 4730
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
4731
{
4732 4733
	int ret = 0;

S
Steven Rostedt 已提交
4734
	mutex_lock(&ftrace_lock);
4735

4736
	/* we currently allow only one tracer registered at a time */
4737
	if (ftrace_graph_active) {
4738 4739 4740 4741
		ret = -EBUSY;
		goto out;
	}

4742 4743 4744
	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
	register_pm_notifier(&ftrace_suspend_notifier);

4745
	ftrace_graph_active++;
4746
	ret = start_graph_tracing();
4747
	if (ret) {
4748
		ftrace_graph_active--;
4749 4750
		goto out;
	}
4751

4752 4753
	ftrace_graph_return = retfunc;
	ftrace_graph_entry = entryfunc;
4754

4755
	ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
4756 4757

out:
S
Steven Rostedt 已提交
4758
	mutex_unlock(&ftrace_lock);
4759
	return ret;
4760 4761
}

4762
void unregister_ftrace_graph(void)
4763
{
S
Steven Rostedt 已提交
4764
	mutex_lock(&ftrace_lock);
4765

4766
	if (unlikely(!ftrace_graph_active))
4767 4768
		goto out;

4769
	ftrace_graph_active--;
4770
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4771
	ftrace_graph_entry = ftrace_graph_entry_stub;
4772
	ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
4773
	unregister_pm_notifier(&ftrace_suspend_notifier);
4774
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4775

4776
 out:
S
Steven Rostedt 已提交
4777
	mutex_unlock(&ftrace_lock);
4778
}
4779

4780 4781 4782 4783 4784 4785 4786 4787
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);

static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
	atomic_set(&t->tracing_graph_pause, 0);
	atomic_set(&t->trace_overrun, 0);
	t->ftrace_timestamp = 0;
L
Lucas De Marchi 已提交
4788
	/* make curr_ret_stack visible before we add the ret_stack */
4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822
	smp_wmb();
	t->ret_stack = ret_stack;
}

/*
 * Allocate a return stack for the idle task. May be the first
 * time through, or it may be done by CPU hotplug online.
 */
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
{
	t->curr_ret_stack = -1;
	/*
	 * The idle task has no parent, it either has its own
	 * stack or no stack at all.
	 */
	if (t->ret_stack)
		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));

	if (ftrace_graph_active) {
		struct ftrace_ret_stack *ret_stack;

		ret_stack = per_cpu(idle_ret_stack, cpu);
		if (!ret_stack) {
			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
					    * sizeof(struct ftrace_ret_stack),
					    GFP_KERNEL);
			if (!ret_stack)
				return;
			per_cpu(idle_ret_stack, cpu) = ret_stack;
		}
		graph_init_task(t, ret_stack);
	}
}

4823
/* Allocate a return stack for newly created task */
4824
void ftrace_graph_init_task(struct task_struct *t)
4825
{
4826 4827
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;
4828
	t->curr_ret_stack = -1;
4829

4830
	if (ftrace_graph_active) {
4831 4832 4833
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4834 4835
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
4836
		if (!ret_stack)
4837
			return;
4838
		graph_init_task(t, ret_stack);
4839
	}
4840 4841
}

4842
void ftrace_graph_exit_task(struct task_struct *t)
4843
{
4844 4845
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

4846
	t->ret_stack = NULL;
4847 4848 4849 4850
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
4851
}
S
Steven Rostedt 已提交
4852 4853 4854 4855 4856

void ftrace_graph_stop(void)
{
	ftrace_stop();
}
4857
#endif