ftrace.c 87.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
 *  Copyright (C) 2004 William Lee Irwin III
 */

16 17 18
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
#include <linux/kallsyms.h>
19
#include <linux/seq_file.h>
20
#include <linux/suspend.h>
21
#include <linux/debugfs.h>
22
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
23
#include <linux/kthread.h>
24
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
25
#include <linux/ftrace.h>
26
#include <linux/sysctl.h>
27
#include <linux/slab.h>
28
#include <linux/ctype.h>
29
#include <linux/list.h>
30
#include <linux/hash.h>
31
#include <linux/rcupdate.h>
32

33
#include <trace/events/sched.h>
34

35
#include <asm/ftrace.h>
36
#include <asm/setup.h>
37

38
#include "trace_output.h"
S
Steven Rostedt 已提交
39
#include "trace_stat.h"
40

41
#define FTRACE_WARN_ON(cond)			\
42 43 44
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
45
			ftrace_kill();		\
46 47
		___r;				\
	})
48 49

#define FTRACE_WARN_ON_ONCE(cond)		\
50 51 52
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
53
			ftrace_kill();		\
54 55
		___r;				\
	})
56

57 58 59
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
60 61
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
62

63 64
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
65
static int last_ftrace_enabled;
66

67 68 69
/* Quick disabling of function tracer. */
int function_trace_stop;

70 71 72 73 74 75 76
/* List for set_ftrace_pid's pids. */
LIST_HEAD(ftrace_pids);
struct ftrace_pid {
	struct list_head list;
	struct pid *pid;
};

77 78 79 80 81 82
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

83
static DEFINE_MUTEX(ftrace_lock);
84

85 86
static struct ftrace_ops ftrace_list_end __read_mostly =
{
87
	.func		= ftrace_stub,
88 89
};

90 91
static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
92
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
93
ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
94
ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
95
static struct ftrace_ops global_ops;
96

97 98 99
static void
ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);

100
/*
101
 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
102 103 104
 * can use rcu_dereference_raw() is that elements removed from this list
 * are simply leaked, so there is no need to interact with a grace-period
 * mechanism.  The rcu_dereference_raw() calls are needed to handle
105
 * concurrent insertions into the ftrace_global_list.
106 107 108
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
109 110
static void ftrace_global_list_func(unsigned long ip,
				    unsigned long parent_ip)
111
{
112 113 114 115
	struct ftrace_ops *op;

	if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
		return;
116

117 118
	trace_recursion_set(TRACE_GLOBAL_BIT);
	op = rcu_dereference_raw(ftrace_global_list); /*see above*/
119 120
	while (op != &ftrace_list_end) {
		op->func(ip, parent_ip);
121
		op = rcu_dereference_raw(op->next); /*see above*/
122
	};
123
	trace_recursion_clear(TRACE_GLOBAL_BIT);
124 125
}

126 127
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
{
128
	if (!test_tsk_trace_trace(current))
129 130 131 132 133 134 135 136 137 138 139 140
		return;

	ftrace_pid_function(ip, parent_ip);
}

static void set_ftrace_pid_function(ftrace_func_t func)
{
	/* do not set ftrace_pid_function to itself! */
	if (func != ftrace_pid_func)
		ftrace_pid_function = func;
}

141
/**
142
 * clear_ftrace_function - reset the ftrace function
143
 *
144 145
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
146
 */
147
void clear_ftrace_function(void)
148
{
149
	ftrace_trace_function = ftrace_stub;
150
	__ftrace_trace_function = ftrace_stub;
151
	ftrace_pid_function = ftrace_stub;
152 153
}

154 155 156 157 158 159 160 161 162 163 164 165 166 167
#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
/*
 * For those archs that do not test ftrace_trace_stop in their
 * mcount call site, we need to do it from C.
 */
static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
{
	if (function_trace_stop)
		return;

	__ftrace_trace_function(ip, parent_ip);
}
#endif

168
static void update_global_ops(void)
169 170 171 172 173 174 175 176
{
	ftrace_func_t func;

	/*
	 * If there's only one function registered, then call that
	 * function directly. Otherwise, we need to iterate over the
	 * registered callers.
	 */
177 178 179
	if (ftrace_global_list == &ftrace_list_end ||
	    ftrace_global_list->next == &ftrace_list_end)
		func = ftrace_global_list->func;
180
	else
181
		func = ftrace_global_list_func;
182 183 184 185 186 187

	/* If we filter on pids, update to use the pid function */
	if (!list_empty(&ftrace_pids)) {
		set_ftrace_pid_function(func);
		func = ftrace_pid_func;
	}
188 189 190 191 192 193 194 195 196 197

	global_ops.func = func;
}

static void update_ftrace_function(void)
{
	ftrace_func_t func;

	update_global_ops();

198 199 200 201 202
	/*
	 * If we are at the end of the list and this ops is
	 * not dynamic, then have the mcount trampoline call
	 * the function directly
	 */
203
	if (ftrace_ops_list == &ftrace_list_end ||
204 205
	    (ftrace_ops_list->next == &ftrace_list_end &&
	     !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
206 207 208
		func = ftrace_ops_list->func;
	else
		func = ftrace_ops_list_func;
209

210 211 212 213 214 215 216 217
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
	ftrace_trace_function = func;
#else
	__ftrace_trace_function = func;
	ftrace_trace_function = ftrace_test_stop_func;
#endif
}

218
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
219
{
220
	ops->next = *list;
221
	/*
222
	 * We are entering ops into the list but another
223 224
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
225
	 * the ops pointer included into the list.
226
	 */
227
	rcu_assign_pointer(*list, ops);
228 229
}

230
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
231 232 233 234
{
	struct ftrace_ops **p;

	/*
235 236
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
237
	 */
238 239
	if (*list == ops && ops->next == &ftrace_list_end) {
		*list = &ftrace_list_end;
S
Steven Rostedt 已提交
240
		return 0;
241 242
	}

243
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
244 245 246
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
247 248
	if (*p != ops)
		return -1;
249 250

	*p = (*p)->next;
251 252
	return 0;
}
253

254 255 256 257 258 259 260 261
static int __register_ftrace_function(struct ftrace_ops *ops)
{
	if (ftrace_disabled)
		return -ENODEV;

	if (FTRACE_WARN_ON(ops == &global_ops))
		return -EINVAL;

262 263 264
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

265 266 267
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

268 269 270 271 272 273 274 275 276
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
		int first = ftrace_global_list == &ftrace_list_end;
		add_ftrace_ops(&ftrace_global_list, ops);
		ops->flags |= FTRACE_OPS_FL_ENABLED;
		if (first)
			add_ftrace_ops(&ftrace_ops_list, &global_ops);
	} else
		add_ftrace_ops(&ftrace_ops_list, ops);

277 278 279 280 281 282 283 284 285 286 287 288 289
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

	if (ftrace_disabled)
		return -ENODEV;

290 291 292
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

293 294 295
	if (FTRACE_WARN_ON(ops == &global_ops))
		return -EINVAL;

296 297 298 299 300 301 302 303 304
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
		ret = remove_ftrace_ops(&ftrace_global_list, ops);
		if (!ret && ftrace_global_list == &ftrace_list_end)
			ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops);
		if (!ret)
			ops->flags &= ~FTRACE_OPS_FL_ENABLED;
	} else
		ret = remove_ftrace_ops(&ftrace_ops_list, ops);

305 306
	if (ret < 0)
		return ret;
307

308 309
	if (ftrace_enabled)
		update_ftrace_function();
310

311 312 313 314 315 316 317
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
	 */
	if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
		synchronize_sched();

S
Steven Rostedt 已提交
318
	return 0;
319 320
}

321 322
static void ftrace_update_pid_func(void)
{
323
	/* Only do something if we are tracing something */
324
	if (ftrace_trace_function == ftrace_stub)
325
		return;
326

327
	update_ftrace_function();
328 329
}

330 331 332 333 334
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
335 336
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
337
	unsigned long long		time_squared;
338
#endif
339 340
};

341 342 343 344
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
345 346
};

347 348 349 350 351 352 353 354
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

355 356
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
357

358 359
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
360

361 362 363 364
static int ftrace_profile_bits __read_mostly;
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
365 366
static DEFINE_MUTEX(ftrace_profile_lock);

367
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
368 369 370

#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */

S
Steven Rostedt 已提交
371 372 373
static void *
function_stat_next(void *v, int idx)
{
374 375
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
376

377
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
378 379

 again:
L
Li Zefan 已提交
380 381 382
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
383 384 385 386 387
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
388 389
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
390 391 392 393 394 395 396
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
397 398 399 400 401 402 403
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
404 405
}

406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
422 423
static int function_stat_cmp(void *p1, void *p2)
{
424 425
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
426 427 428 429 430 431 432 433

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
434
#endif
S
Steven Rostedt 已提交
435 436 437

static int function_stat_headers(struct seq_file *m)
{
438
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
439
	seq_printf(m, "  Function                               "
440
		   "Hit    Time            Avg             s^2\n"
441
		      "  --------                               "
442
		   "---    ----            ---             ---\n");
443
#else
S
Steven Rostedt 已提交
444 445
	seq_printf(m, "  Function                               Hit\n"
		      "  --------                               ---\n");
446
#endif
S
Steven Rostedt 已提交
447 448 449 450 451
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
452
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
453
	char str[KSYM_SYMBOL_LEN];
454
	int ret = 0;
455
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
456 457
	static struct trace_seq s;
	unsigned long long avg;
458
	unsigned long long stddev;
459
#endif
460 461 462 463 464 465 466
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
S
Steven Rostedt 已提交
467 468

	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
469 470 471 472
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	seq_printf(m, "    ");
473 474 475
	avg = rec->time;
	do_div(avg, rec->counter);

476 477 478 479 480 481 482 483 484 485 486 487
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
		stddev = rec->time_squared - rec->counter * avg * avg;
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
		do_div(stddev, (rec->counter - 1) * 1000);
	}

488 489 490 491
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
492 493
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
494 495 496
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
497 498
out:
	mutex_unlock(&ftrace_profile_lock);
S
Steven Rostedt 已提交
499

500
	return ret;
S
Steven Rostedt 已提交
501 502
}

503
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
504
{
505
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
506

507
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
508

509 510 511 512
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
513 514
	}

515
	memset(stat->hash, 0,
516 517
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
518

519
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
520 521
{
	struct ftrace_profile_page *pg;
522 523
	int functions;
	int pages;
524
	int i;
S
Steven Rostedt 已提交
525

526
	/* If we already allocated, do nothing */
527
	if (stat->pages)
528
		return 0;
S
Steven Rostedt 已提交
529

530 531
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
532
		return -ENOMEM;
S
Steven Rostedt 已提交
533

534 535 536 537 538 539 540 541 542 543 544 545 546
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

547
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
548

549 550 551
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

	for (i = 0; i < pages; i++) {
552 553
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
554
			goto out_free;
555 556 557 558
		pg = pg->next;
	}

	return 0;
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	free_page((unsigned long)stat->pages);
	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
574 575
}

576
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
577
{
578
	struct ftrace_profile_stat *stat;
579
	int size;
S
Steven Rostedt 已提交
580

581 582 583
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
584
		/* If the profile is already created, simply reset it */
585
		ftrace_profile_reset(stat);
586 587
		return 0;
	}
S
Steven Rostedt 已提交
588

589 590 591 592 593
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
594

595
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
596

597
	if (!stat->hash)
598 599
		return -ENOMEM;

600 601
	if (!ftrace_profile_bits) {
		size--;
602

603 604 605
		for (; size; size >>= 1)
			ftrace_profile_bits++;
	}
606

607
	/* Preallocate the function profiling pages */
608 609 610
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
611 612 613 614
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
615 616
}

617 618 619 620 621 622 623 624 625 626 627 628 629 630
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

	for_each_online_cpu(cpu) {
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

631
/* interrupts must be disabled */
632 633
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
634
{
635
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
636 637 638 639 640
	struct hlist_head *hhd;
	struct hlist_node *n;
	unsigned long key;

	key = hash_long(ip, ftrace_profile_bits);
641
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
642 643 644 645 646 647

	if (hlist_empty(hhd))
		return NULL;

	hlist_for_each_entry_rcu(rec, n, hhd, node) {
		if (rec->ip == ip)
648 649 650 651 652 653
			return rec;
	}

	return NULL;
}

654 655
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
656 657 658 659
{
	unsigned long key;

	key = hash_long(rec->ip, ftrace_profile_bits);
660
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
661 662
}

663 664 665
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
666
static struct ftrace_profile *
667
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
668 669 670
{
	struct ftrace_profile *rec = NULL;

671
	/* prevent recursion (from NMIs) */
672
	if (atomic_inc_return(&stat->disabled) != 1)
673 674 675
		goto out;

	/*
676 677
	 * Try to find the function again since an NMI
	 * could have added it
678
	 */
679
	rec = ftrace_find_profiled_func(stat, ip);
680
	if (rec)
681
		goto out;
682

683 684 685 686
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
687
	}
688

689
	rec = &stat->pages->records[stat->pages->index++];
690
	rec->ip = ip;
691
	ftrace_add_profile(stat, rec);
692

S
Steven Rostedt 已提交
693
 out:
694
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
695 696 697 698 699 700 701

	return rec;
}

static void
function_profile_call(unsigned long ip, unsigned long parent_ip)
{
702
	struct ftrace_profile_stat *stat;
703
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
704 705 706 707 708 709
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
710 711

	stat = &__get_cpu_var(ftrace_profile_stats);
712
	if (!stat->hash || !ftrace_profile_enabled)
713 714 715
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
716
	if (!rec) {
717
		rec = ftrace_profile_alloc(stat, ip);
718 719 720
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
721 722 723 724 725 726

	rec->counter++;
 out:
	local_irq_restore(flags);
}

727 728 729 730 731 732 733 734 735
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
	function_profile_call(trace->func, 0);
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
736
	struct ftrace_profile_stat *stat;
737
	unsigned long long calltime;
738
	struct ftrace_profile *rec;
739
	unsigned long flags;
740 741

	local_irq_save(flags);
742
	stat = &__get_cpu_var(ftrace_profile_stats);
743
	if (!stat->hash || !ftrace_profile_enabled)
744 745
		goto out;

746 747 748 749
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766
	calltime = trace->rettime - trace->calltime;

	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

767
	rec = ftrace_find_profiled_func(stat, trace->func);
768
	if (rec) {
769
		rec->time += calltime;
770 771
		rec->time_squared += calltime * calltime;
	}
772

773
 out:
774 775 776 777 778 779 780 781 782 783 784 785 786 787
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
S
Steven Rostedt 已提交
788 789
static struct ftrace_ops ftrace_profile_ops __read_mostly =
{
790
	.func		= function_profile_call,
S
Steven Rostedt 已提交
791 792
};

793 794 795 796 797 798 799 800 801 802 803
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
804 805 806 807 808
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
809
	char buf[64];		/* big enough to hold a number */
S
Steven Rostedt 已提交
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	ret = strict_strtoul(buf, 10, &val);
	if (ret < 0)
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
829 830 831 832 833 834
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

835 836 837 838 839
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
840 841 842
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
843 844 845 846
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
847
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
848 849
		}
	}
850
 out:
S
Steven Rostedt 已提交
851 852
	mutex_unlock(&ftrace_profile_lock);

853
	*ppos += cnt;
S
Steven Rostedt 已提交
854 855 856 857

	return cnt;
}

858 859 860 861
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
862
	char buf[64];		/* big enough to hold a number */
863 864 865 866 867 868
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
869 870 871 872
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
873
	.llseek		= default_llseek,
S
Steven Rostedt 已提交
874 875
};

876 877
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
878 879 880 881 882 883
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
884 885
};

886
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
887
{
888
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
889
	struct dentry *entry;
890
	char *name;
S
Steven Rostedt 已提交
891
	int ret;
892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

		/* allocate enough for function name + cpu number */
		name = kmalloc(32, GFP_KERNEL);
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		snprintf(name, 32, "function%d", cpu);
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
920 921 922 923 924 925 926 927 928 929
	}

	entry = debugfs_create_file("function_profile_enabled", 0644,
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
		pr_warning("Could not create debugfs "
			   "'function_profile_enabled' entry\n");
}

#else /* CONFIG_FUNCTION_PROFILER */
930
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
931 932 933 934
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

#ifdef CONFIG_DYNAMIC_FTRACE

#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;

struct ftrace_func_probe {
	struct hlist_node	node;
	struct ftrace_probe_ops	*ops;
	unsigned long		flags;
	unsigned long		ip;
	void			*data;
	struct rcu_head		rcu;
};

enum {
	FTRACE_ENABLE_CALLS		= (1 << 0),
	FTRACE_DISABLE_CALLS		= (1 << 1),
	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
958 959
	FTRACE_START_FUNC_RET		= (1 << 3),
	FTRACE_STOP_FUNC_RET		= (1 << 4),
960
};
961 962 963 964 965 966 967 968 969
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

struct ftrace_hash {
	unsigned long		size_bits;
	struct hlist_head	*buckets;
	unsigned long		count;
970
	struct rcu_head		rcu;
971 972
};

973 974 975 976 977 978 979 980 981
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
982
};
983
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
984

985
static struct ftrace_ops global_ops = {
986
	.func			= ftrace_stub,
987 988
	.notrace_hash		= EMPTY_HASH,
	.filter_hash		= EMPTY_HASH,
989 990
};

991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
static struct dyn_ftrace *ftrace_new_addrs;

static DEFINE_MUTEX(ftrace_regex_lock);

struct ftrace_page {
	struct ftrace_page	*next;
	int			index;
	struct dyn_ftrace	records[];
};

#define ENTRIES_PER_PAGE \
  ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

static struct dyn_ftrace *ftrace_free_records;

1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
static struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;
	struct hlist_node *n;

	if (!hash->count)
		return NULL;

	if (hash->size_bits > 0)
		key = hash_long(ip, hash->size_bits);
	else
		key = 0;

	hhd = &hash->buckets[key];

	hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1037 1038
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1039 1040 1041 1042 1043
{
	struct hlist_head *hhd;
	unsigned long key;

	if (hash->size_bits)
1044
		key = hash_long(entry->ip, hash->size_bits);
1045 1046 1047 1048 1049 1050
	else
		key = 0;

	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1063 1064 1065 1066 1067

	return 0;
}

static void
1068
free_hash_entry(struct ftrace_hash *hash,
1069 1070 1071 1072 1073 1074 1075
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1076 1077 1078 1079 1080 1081 1082 1083
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	hash->count--;
}

1084 1085 1086 1087 1088 1089 1090 1091
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
	struct hlist_node *tp, *tn;
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1092 1093 1094
	if (!hash->count)
		return;

1095 1096 1097
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
		hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1098
			free_hash_entry(hash, entry);
1099 1100 1101 1102
	}
	FTRACE_WARN_ON(hash->count);
}

1103 1104 1105 1106 1107 1108 1109 1110 1111
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}

1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
	hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL);

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	struct hlist_node *tp;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

	/* Empty hash? */
	if (!hash || !hash->count)
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

static int
ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src)
{
	struct ftrace_func_entry *entry;
	struct hlist_node *tp, *tn;
	struct hlist_head *hhd;
1191 1192
	struct ftrace_hash *old_hash;
	struct ftrace_hash *new_hash;
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202
	unsigned long key;
	int size = src->count;
	int bits = 0;
	int i;

	/*
	 * If the new source is empty, just free dst and assign it
	 * the empty_hash.
	 */
	if (!src->count) {
1203 1204
		free_ftrace_hash_rcu(*dst);
		rcu_assign_pointer(*dst, EMPTY_HASH);
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217
		return 0;
	}

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1218 1219 1220
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
		return -ENOMEM;
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230

	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
		hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
			if (bits > 0)
				key = hash_long(entry->ip, bits);
			else
				key = 0;
			remove_hash_entry(src, entry);
1231
			__add_hash_entry(new_hash, entry);
1232 1233 1234
		}
	}

1235 1236 1237 1238
	old_hash = *dst;
	rcu_assign_pointer(*dst, new_hash);
	free_ftrace_hash_rcu(old_hash);

1239 1240 1241
	return 0;
}

1242 1243 1244 1245 1246 1247 1248 1249
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1250 1251 1252
 *
 * This needs to be called with preemption disabled as
 * the hashes are freed with call_rcu_sched().
1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
 */
static int
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
{
	struct ftrace_hash *filter_hash;
	struct ftrace_hash *notrace_hash;
	int ret;

	filter_hash = rcu_dereference_raw(ops->filter_hash);
	notrace_hash = rcu_dereference_raw(ops->notrace_hash);

	if ((!filter_hash || !filter_hash->count ||
	     ftrace_lookup_ip(filter_hash, ip)) &&
	    (!notrace_hash || !notrace_hash->count ||
	     !ftrace_lookup_ip(notrace_hash, ip)))
		ret = 1;
	else
		ret = 0;

	return ret;
}

1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	int count = 0;
	int all = 0;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return;

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
		hash = ops->filter_hash;
		other_hash = ops->notrace_hash;
1318
		if (!hash || !hash->count)
1319 1320 1321 1322 1323 1324 1325 1326 1327
			all = 1;
	} else {
		inc = !inc;
		hash = ops->notrace_hash;
		other_hash = ops->filter_hash;
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1328
		if (hash && !hash->count)
1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341
			return;
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1342
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1343 1344
				match = 1;
		} else {
1345 1346
			in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip);
1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387

			/*
			 *
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
				 (in_other_hash || !other_hash->count))
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
				return;
		} else {
			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
				return;
			rec->flags--;
		}
		count++;
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
			return;
	} while_for_each_ftrace_rec();
}

static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
				    int filter_hash)
{
	__ftrace_hash_rec_update(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
				   int filter_hash)
{
	__ftrace_hash_rec_update(ops, filter_hash, 1);
}

I
Ingo Molnar 已提交
1388
static void ftrace_free_rec(struct dyn_ftrace *rec)
1389
{
1390
	rec->freelist = ftrace_free_records;
1391 1392 1393 1394
	ftrace_free_records = rec;
	rec->flags |= FTRACE_FL_FREE;
}

I
Ingo Molnar 已提交
1395
static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
1396
{
1397 1398 1399 1400 1401 1402 1403
	struct dyn_ftrace *rec;

	/* First check for freed records */
	if (ftrace_free_records) {
		rec = ftrace_free_records;

		if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
1404
			FTRACE_WARN_ON_ONCE(1);
1405 1406 1407 1408
			ftrace_free_records = NULL;
			return NULL;
		}

1409
		ftrace_free_records = rec->freelist;
1410 1411 1412 1413
		memset(rec, 0, sizeof(*rec));
		return rec;
	}

1414
	if (ftrace_pages->index == ENTRIES_PER_PAGE) {
S
Steven Rostedt 已提交
1415 1416 1417 1418 1419 1420 1421
		if (!ftrace_pages->next) {
			/* allocate another page */
			ftrace_pages->next =
				(void *)get_zeroed_page(GFP_KERNEL);
			if (!ftrace_pages->next)
				return NULL;
		}
1422 1423 1424 1425 1426 1427
		ftrace_pages = ftrace_pages->next;
	}

	return &ftrace_pages->records[ftrace_pages->index++];
}

S
Steven Rostedt 已提交
1428
static struct dyn_ftrace *
1429
ftrace_record_ip(unsigned long ip)
1430
{
S
Steven Rostedt 已提交
1431
	struct dyn_ftrace *rec;
1432

1433
	if (ftrace_disabled)
S
Steven Rostedt 已提交
1434
		return NULL;
1435

S
Steven Rostedt 已提交
1436 1437 1438
	rec = ftrace_alloc_dyn_node(ip);
	if (!rec)
		return NULL;
1439

S
Steven Rostedt 已提交
1440
	rec->ip = ip;
1441
	rec->newlist = ftrace_new_addrs;
1442
	ftrace_new_addrs = rec;
1443

S
Steven Rostedt 已提交
1444
	return rec;
1445 1446
}

1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
static void print_ip_ins(const char *fmt, unsigned char *p)
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1457
static void ftrace_bug(int failed, unsigned long ip)
1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
{
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
		print_ip_ins(" actual: ", (unsigned char *)ip);
		printk(KERN_CONT "\n");
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
}

1484

1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499
/* Return 1 if the address range is reserved for ftrace */
int ftrace_text_reserved(void *start, void *end)
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	do_for_each_ftrace_rec(pg, rec) {
		if (rec->ip <= (unsigned long)end &&
		    rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
			return 1;
	} while_for_each_ftrace_rec();
	return 0;
}


1500
static int
1501
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1502
{
1503
	unsigned long ftrace_addr;
1504
	unsigned long flag = 0UL;
1505

1506
	ftrace_addr = (unsigned long)FTRACE_ADDR;
1507

S
Steven Rostedt 已提交
1508
	/*
1509
	 * If we are enabling tracing:
S
Steven Rostedt 已提交
1510
	 *
1511 1512
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
S
Steven Rostedt 已提交
1513
	 *
1514 1515 1516 1517
	 *   Otherwise we make sure its disabled.
	 *
	 * If we are disabling tracing, then disable all records that
	 * are enabled.
S
Steven Rostedt 已提交
1518
	 */
1519 1520
	if (enable && (rec->flags & ~FTRACE_FL_MASK))
		flag = FTRACE_FL_ENABLED;
S
Steven Rostedt 已提交
1521

1522 1523 1524
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
		return 0;
S
Steven Rostedt 已提交
1525

1526 1527 1528
	if (flag) {
		rec->flags |= FTRACE_FL_ENABLED;
		return ftrace_make_call(rec, ftrace_addr);
1529 1530
	}

1531 1532
	rec->flags &= ~FTRACE_FL_ENABLED;
	return ftrace_make_nop(NULL, rec, ftrace_addr);
1533 1534
}

I
Ingo Molnar 已提交
1535
static void ftrace_replace_code(int enable)
1536 1537 1538
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
1539
	int failed;
1540

1541 1542 1543
	if (unlikely(ftrace_disabled))
		return;

1544
	do_for_each_ftrace_rec(pg, rec) {
1545 1546
		/* Skip over free records */
		if (rec->flags & FTRACE_FL_FREE)
1547 1548 1549
			continue;

		failed = __ftrace_replace_code(rec, enable);
1550
		if (failed) {
1551 1552 1553
			ftrace_bug(failed, rec->ip);
			/* Stop processing */
			return;
1554
		}
1555
	} while_for_each_ftrace_rec();
1556 1557
}

1558
static int
1559
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1560 1561
{
	unsigned long ip;
1562
	int ret;
1563 1564 1565

	ip = rec->ip;

1566 1567 1568
	if (unlikely(ftrace_disabled))
		return 0;

1569
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1570
	if (ret) {
1571
		ftrace_bug(ret, ip);
1572
		return 0;
1573
	}
1574
	return 1;
1575 1576
}

1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

I
Ingo Molnar 已提交
1595
static int __ftrace_modify_code(void *data)
1596
{
1597 1598
	int *command = data;

1599
	if (*command & FTRACE_ENABLE_CALLS)
1600
		ftrace_replace_code(1);
1601
	else if (*command & FTRACE_DISABLE_CALLS)
1602 1603 1604 1605 1606
		ftrace_replace_code(0);

	if (*command & FTRACE_UPDATE_TRACE_FUNC)
		ftrace_update_ftrace_func(ftrace_trace_function);

1607 1608 1609 1610 1611
	if (*command & FTRACE_START_FUNC_RET)
		ftrace_enable_ftrace_graph_caller();
	else if (*command & FTRACE_STOP_FUNC_RET)
		ftrace_disable_ftrace_graph_caller();

1612
	return 0;
1613 1614
}

I
Ingo Molnar 已提交
1615
static void ftrace_run_update_code(int command)
1616
{
1617 1618 1619 1620 1621 1622 1623
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;

R
Rusty Russell 已提交
1624
	stop_machine(__ftrace_modify_code, &command, NULL);
1625 1626 1627

	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
1628 1629
}

1630
static ftrace_func_t saved_ftrace_func;
1631
static int ftrace_start_up;
1632
static int global_start_up;
1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645

static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
1646

1647
static int ftrace_startup(struct ftrace_ops *ops, int command)
1648
{
1649 1650
	bool hash_enable = true;

1651
	if (unlikely(ftrace_disabled))
1652
		return -ENODEV;
1653

1654
	ftrace_start_up++;
S
Steven Rostedt 已提交
1655
	command |= FTRACE_ENABLE_CALLS;
1656

1657 1658 1659 1660 1661 1662 1663 1664 1665
	/* ops marked global share the filter hashes */
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
		ops = &global_ops;
		/* Don't update hash if global is already set */
		if (global_start_up)
			hash_enable = false;
		global_start_up++;
	}

1666
	ops->flags |= FTRACE_OPS_FL_ENABLED;
1667
	if (hash_enable)
1668 1669
		ftrace_hash_rec_enable(ops, 1);

1670
	ftrace_startup_enable(command);
1671 1672

	return 0;
1673 1674
}

1675
static void ftrace_shutdown(struct ftrace_ops *ops, int command)
1676
{
1677 1678
	bool hash_disable = true;

1679 1680 1681
	if (unlikely(ftrace_disabled))
		return;

1682
	ftrace_start_up--;
1683 1684 1685 1686 1687 1688 1689
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
		ops = &global_ops;
		global_start_up--;
		WARN_ON_ONCE(global_start_up < 0);
		/* Don't update hash if global still has users */
		if (global_start_up) {
			WARN_ON_ONCE(!ftrace_start_up);
			hash_disable = false;
		}
	}

	if (hash_disable)
1702 1703
		ftrace_hash_rec_disable(ops, 1);

1704
	if (ops != &global_ops || !global_start_up)
1705
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
1706 1707 1708

	if (!ftrace_start_up)
		command |= FTRACE_DISABLE_CALLS;
1709

1710 1711 1712 1713
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
1714

1715
	if (!command || !ftrace_enabled)
S
Steven Rostedt 已提交
1716
		return;
1717 1718

	ftrace_run_update_code(command);
1719 1720
}

I
Ingo Molnar 已提交
1721
static void ftrace_startup_sysctl(void)
1722
{
1723 1724 1725
	if (unlikely(ftrace_disabled))
		return;

1726 1727
	/* Force update next time */
	saved_ftrace_func = NULL;
1728 1729
	/* ftrace_start_up is true if we want ftrace running */
	if (ftrace_start_up)
1730
		ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1731 1732
}

I
Ingo Molnar 已提交
1733
static void ftrace_shutdown_sysctl(void)
1734
{
1735 1736 1737
	if (unlikely(ftrace_disabled))
		return;

1738 1739
	/* ftrace_start_up is true if ftrace is running */
	if (ftrace_start_up)
1740
		ftrace_run_update_code(FTRACE_DISABLE_CALLS);
1741 1742
}

1743 1744 1745 1746
static cycle_t		ftrace_update_time;
static unsigned long	ftrace_update_cnt;
unsigned long		ftrace_update_tot_cnt;

1747
static int ftrace_update_code(struct module *mod)
1748
{
1749
	struct dyn_ftrace *p;
1750
	cycle_t start, stop;
1751

I
Ingo Molnar 已提交
1752
	start = ftrace_now(raw_smp_processor_id());
1753 1754
	ftrace_update_cnt = 0;

1755
	while (ftrace_new_addrs) {
1756

S
Steven Rostedt 已提交
1757 1758 1759
		/* If something went wrong, bail without enabling anything */
		if (unlikely(ftrace_disabled))
			return -1;
1760

1761
		p = ftrace_new_addrs;
1762
		ftrace_new_addrs = p->newlist;
1763
		p->flags = 0L;
1764

1765
		/*
L
Lucas De Marchi 已提交
1766
		 * Do the initial record conversion from mcount jump
1767 1768 1769
		 * to the NOP instructions.
		 */
		if (!ftrace_code_disable(mod, p)) {
S
Steven Rostedt 已提交
1770
			ftrace_free_rec(p);
1771 1772
			/* Game over */
			break;
1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
		}

		ftrace_update_cnt++;

		/*
		 * If the tracing is enabled, go ahead and enable the record.
		 *
		 * The reason not to enable the record immediatelly is the
		 * inherent check of ftrace_make_nop/ftrace_make_call for
		 * correct previous instructions.  Making first the NOP
		 * conversion puts the module to the correct state, thus
		 * passing the ftrace_make_call check.
		 */
		if (ftrace_start_up) {
			int failed = __ftrace_replace_code(p, 1);
			if (failed) {
				ftrace_bug(failed, p->ip);
				ftrace_free_rec(p);
			}
		}
1793 1794
	}

I
Ingo Molnar 已提交
1795
	stop = ftrace_now(raw_smp_processor_id());
1796 1797 1798
	ftrace_update_time = stop - start;
	ftrace_update_tot_cnt += ftrace_update_cnt;

1799 1800 1801
	return 0;
}

1802
static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828
{
	struct ftrace_page *pg;
	int cnt;
	int i;

	/* allocate a few pages */
	ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
	if (!ftrace_pages_start)
		return -1;

	/*
	 * Allocate a few more pages.
	 *
	 * TODO: have some parser search vmlinux before
	 *   final linking to find all calls to ftrace.
	 *   Then we can:
	 *    a) know how many pages to allocate.
	 *     and/or
	 *    b) set up the table then.
	 *
	 *  The dynamic code is still necessary for
	 *  modules.
	 */

	pg = ftrace_pages = ftrace_pages_start;

1829
	cnt = num_to_init / ENTRIES_PER_PAGE;
S
Steven Rostedt 已提交
1830
	pr_info("ftrace: allocating %ld entries in %d pages\n",
1831
		num_to_init, cnt + 1);
1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845

	for (i = 0; i < cnt; i++) {
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);

		/* If we fail, we'll try later anyway */
		if (!pg->next)
			break;

		pg = pg->next;
	}

	return 0;
}

1846 1847
enum {
	FTRACE_ITER_FILTER	= (1 << 0),
1848
	FTRACE_ITER_NOTRACE	= (1 << 1),
S
Steven Rostedt 已提交
1849 1850
	FTRACE_ITER_PRINTALL	= (1 << 2),
	FTRACE_ITER_HASH	= (1 << 3),
1851
	FTRACE_ITER_ENABLED	= (1 << 4),
1852 1853 1854 1855 1856
};

#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
1857
	loff_t				pos;
1858 1859 1860 1861 1862
	loff_t				func_pos;
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
	struct trace_parser		parser;
1863
	struct ftrace_hash		*hash;
1864
	struct ftrace_ops		*ops;
1865 1866 1867
	int				hidx;
	int				idx;
	unsigned			flags;
1868 1869
};

1870
static void *
1871
t_hash_next(struct seq_file *m, loff_t *pos)
1872 1873
{
	struct ftrace_iterator *iter = m->private;
1874
	struct hlist_node *hnd = NULL;
1875 1876 1877
	struct hlist_head *hhd;

	(*pos)++;
1878
	iter->pos = *pos;
1879

1880 1881
	if (iter->probe)
		hnd = &iter->probe->node;
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
 retry:
	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
		return NULL;

	hhd = &ftrace_func_hash[iter->hidx];

	if (hlist_empty(hhd)) {
		iter->hidx++;
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
			iter->hidx++;
			goto retry;
		}
	}

1904 1905 1906 1907 1908 1909
	if (WARN_ON_ONCE(!hnd))
		return NULL;

	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);

	return iter;
1910 1911 1912 1913 1914 1915
}

static void *t_hash_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
1916 1917
	loff_t l;

1918 1919
	if (iter->func_pos > *pos)
		return NULL;
1920

L
Li Zefan 已提交
1921
	iter->hidx = 0;
1922
	for (l = 0; l <= (*pos - iter->func_pos); ) {
1923
		p = t_hash_next(m, &l);
L
Li Zefan 已提交
1924 1925 1926
		if (!p)
			break;
	}
1927 1928 1929
	if (!p)
		return NULL;

1930 1931 1932
	/* Only set this if we have an item */
	iter->flags |= FTRACE_ITER_HASH;

1933
	return iter;
1934 1935
}

1936 1937
static int
t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
1938
{
S
Steven Rostedt 已提交
1939
	struct ftrace_func_probe *rec;
1940

1941 1942 1943
	rec = iter->probe;
	if (WARN_ON_ONCE(!rec))
		return -EIO;
1944

1945 1946 1947
	if (rec->ops->print)
		return rec->ops->print(m, rec->ip, rec->ops, rec->data);

1948
	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1949 1950 1951 1952 1953 1954 1955 1956

	if (rec->data)
		seq_printf(m, ":%p", rec->data);
	seq_putc(m, '\n');

	return 0;
}

I
Ingo Molnar 已提交
1957
static void *
1958 1959 1960
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
1961
	struct ftrace_ops *ops = &global_ops;
1962 1963
	struct dyn_ftrace *rec = NULL;

1964 1965 1966
	if (unlikely(ftrace_disabled))
		return NULL;

1967
	if (iter->flags & FTRACE_ITER_HASH)
1968
		return t_hash_next(m, pos);
1969

1970
	(*pos)++;
1971
	iter->pos = iter->func_pos = *pos;
1972

1973
	if (iter->flags & FTRACE_ITER_PRINTALL)
1974
		return t_hash_start(m, pos);
1975

1976 1977 1978 1979 1980 1981 1982 1983 1984
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
1985 1986
		if ((rec->flags & FTRACE_FL_FREE) ||

S
Steven Rostedt 已提交
1987
		    ((iter->flags & FTRACE_ITER_FILTER) &&
1988
		     !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
S
Steven Rostedt 已提交
1989

1990
		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
1991 1992 1993 1994 1995
		     !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
		     !(rec->flags & ~FTRACE_FL_MASK))) {

1996 1997 1998 1999 2000
			rec = NULL;
			goto retry;
		}
	}

2001
	if (!rec)
2002
		return t_hash_start(m, pos);
2003 2004 2005 2006

	iter->func = rec;

	return iter;
2007 2008
}

2009 2010 2011 2012 2013
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
	iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
2014 2015 2016 2017 2018
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
2019
	struct ftrace_ops *ops = &global_ops;
2020
	void *p = NULL;
2021
	loff_t l;
2022

2023
	mutex_lock(&ftrace_lock);
2024 2025 2026 2027

	if (unlikely(ftrace_disabled))
		return NULL;

2028 2029 2030 2031 2032 2033
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

2034 2035 2036 2037 2038
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
2039
	if (iter->flags & FTRACE_ITER_FILTER && !ops->filter_hash->count) {
2040
		if (*pos > 0)
2041
			return t_hash_start(m, pos);
2042
		iter->flags |= FTRACE_ITER_PRINTALL;
2043 2044
		/* reset in case of seek/pread */
		iter->flags &= ~FTRACE_ITER_HASH;
2045 2046 2047
		return iter;
	}

2048 2049 2050
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_start(m, pos);

2051 2052 2053 2054 2055
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
2056 2057 2058 2059 2060 2061
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
		p = t_next(m, p, &l);
		if (!p)
			break;
2062
	}
2063

2064 2065 2066
	if (!p) {
		if (iter->flags & FTRACE_ITER_FILTER)
			return t_hash_start(m, pos);
2067

2068 2069 2070 2071
		return NULL;
	}

	return iter;
2072 2073 2074 2075
}

static void t_stop(struct seq_file *m, void *p)
{
2076
	mutex_unlock(&ftrace_lock);
2077 2078 2079 2080
}

static int t_show(struct seq_file *m, void *v)
{
2081
	struct ftrace_iterator *iter = m->private;
2082
	struct dyn_ftrace *rec;
2083

2084
	if (iter->flags & FTRACE_ITER_HASH)
2085
		return t_hash_show(m, iter);
2086

2087 2088 2089 2090 2091
	if (iter->flags & FTRACE_ITER_PRINTALL) {
		seq_printf(m, "#### all functions enabled ####\n");
		return 0;
	}

2092 2093
	rec = iter->func;

2094 2095 2096
	if (!rec)
		return 0;

2097 2098 2099 2100 2101
	seq_printf(m, "%ps", (void *)rec->ip);
	if (iter->flags & FTRACE_ITER_ENABLED)
		seq_printf(m, " (%ld)",
			   rec->flags & ~FTRACE_FL_MASK);
	seq_printf(m, "\n");
2102 2103 2104 2105

	return 0;
}

J
James Morris 已提交
2106
static const struct seq_operations show_ftrace_seq_ops = {
2107 2108 2109 2110 2111 2112
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
2113
static int
2114 2115 2116 2117 2118
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;
	int ret;

2119 2120 2121
	if (unlikely(ftrace_disabled))
		return -ENODEV;

2122 2123 2124 2125 2126 2127 2128 2129 2130
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

	iter->pg = ftrace_pages_start;

	ret = seq_open(file, &show_ftrace_seq_ops);
	if (!ret) {
		struct seq_file *m = file->private_data;
I
Ingo Molnar 已提交
2131

2132
		m->private = iter;
I
Ingo Molnar 已提交
2133
	} else {
2134
		kfree(iter);
I
Ingo Molnar 已提交
2135
	}
2136 2137 2138 2139

	return ret;
}

2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;
	int ret;

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

	iter->pg = ftrace_pages_start;
	iter->flags = FTRACE_ITER_ENABLED;

	ret = seq_open(file, &show_ftrace_seq_ops);
	if (!ret) {
		struct seq_file *m = file->private_data;

		m->private = iter;
	} else {
		kfree(iter);
	}

	return ret;
}

2168
static void ftrace_filter_reset(struct ftrace_hash *hash)
2169
{
2170
	mutex_lock(&ftrace_lock);
2171
	ftrace_hash_clear(hash);
2172
	mutex_unlock(&ftrace_lock);
2173 2174
}

I
Ingo Molnar 已提交
2175
static int
2176
ftrace_regex_open(struct ftrace_ops *ops, int flag,
2177
		  struct inode *inode, struct file *file)
2178 2179
{
	struct ftrace_iterator *iter;
2180
	struct ftrace_hash *hash;
2181 2182
	int ret = 0;

2183 2184 2185
	if (unlikely(ftrace_disabled))
		return -ENODEV;

2186 2187 2188 2189
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

2190 2191 2192 2193 2194
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

2195 2196 2197 2198 2199
	if (flag & FTRACE_ITER_NOTRACE)
		hash = ops->notrace_hash;
	else
		hash = ops->filter_hash;

2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
	iter->ops = ops;
	iter->flags = flag;

	if (file->f_mode & FMODE_WRITE) {
		mutex_lock(&ftrace_lock);
		iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
		mutex_unlock(&ftrace_lock);

		if (!iter->hash) {
			trace_parser_put(&iter->parser);
			kfree(iter);
			return -ENOMEM;
		}
	}
2214

2215
	mutex_lock(&ftrace_regex_lock);
2216

2217
	if ((file->f_mode & FMODE_WRITE) &&
2218
	    (file->f_flags & O_TRUNC))
2219
		ftrace_filter_reset(iter->hash);
2220 2221 2222 2223 2224 2225 2226 2227

	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
2228
		} else {
2229 2230
			/* Failed */
			free_ftrace_hash(iter->hash);
2231
			trace_parser_put(&iter->parser);
2232
			kfree(iter);
2233
		}
2234 2235
	} else
		file->private_data = iter;
2236
	mutex_unlock(&ftrace_regex_lock);
2237 2238 2239 2240

	return ret;
}

2241 2242 2243
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
2244
	return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER,
2245
				 inode, file);
2246 2247 2248 2249 2250
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
2251
	return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2252
				 inode, file);
2253 2254
}

I
Ingo Molnar 已提交
2255
static loff_t
2256
ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
{
	loff_t ret;

	if (file->f_mode & FMODE_READ)
		ret = seq_lseek(file, offset, origin);
	else
		file->f_pos = ret = 1;

	return ret;
}

2268
static int ftrace_match(char *str, char *regex, int len, int type)
2269 2270
{
	int matched = 0;
2271
	int slen;
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286

	switch (type) {
	case MATCH_FULL:
		if (strcmp(str, regex) == 0)
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
		if (strncmp(str, regex, len) == 0)
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
		if (strstr(str, regex))
			matched = 1;
		break;
	case MATCH_END_ONLY:
2287 2288
		slen = strlen(str);
		if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2289 2290 2291 2292 2293 2294 2295
			matched = 1;
		break;
	}

	return matched;
}

2296
static int
2297
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2298
{
2299 2300 2301
	struct ftrace_func_entry *entry;
	int ret = 0;

2302 2303 2304 2305 2306
	entry = ftrace_lookup_ip(hash, rec->ip);
	if (not) {
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
2307

2308
		free_hash_entry(hash, entry);
2309 2310 2311 2312
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
2313

2314
		ret = add_hash_entry(hash, rec->ip);
2315 2316
	}
	return ret;
2317 2318
}

2319
static int
2320 2321
ftrace_match_record(struct dyn_ftrace *rec, char *mod,
		    char *regex, int len, int type)
2322 2323
{
	char str[KSYM_SYMBOL_LEN];
2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

	if (mod) {
		/* module lookup requires matching the module */
		if (!modname || strcmp(modname, mod))
			return 0;

		/* blank search means to match all funcs in the mod */
		if (!len)
			return 1;
	}
2337 2338 2339 2340

	return ftrace_match(str, regex, len, type);
}

2341 2342 2343
static int
match_records(struct ftrace_hash *hash, char *buff,
	      int len, char *mod, int not)
2344
{
2345
	unsigned search_len = 0;
2346 2347
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
2348 2349
	int type = MATCH_FULL;
	char *search = buff;
2350
	int found = 0;
2351
	int ret;
2352

2353 2354 2355 2356
	if (len) {
		type = filter_parse_regex(buff, len, &search, &not);
		search_len = strlen(search);
	}
2357

2358
	mutex_lock(&ftrace_lock);
2359

2360 2361
	if (unlikely(ftrace_disabled))
		goto out_unlock;
2362

2363 2364
	do_for_each_ftrace_rec(pg, rec) {

2365
		if (ftrace_match_record(rec, mod, search, search_len, type)) {
2366
			ret = enter_record(hash, rec, not);
2367 2368 2369 2370
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
2371
			found = 1;
2372 2373
		}
	} while_for_each_ftrace_rec();
2374
 out_unlock:
2375
	mutex_unlock(&ftrace_lock);
2376 2377

	return found;
2378 2379
}

2380
static int
2381
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2382
{
2383
	return match_records(hash, buff, len, NULL, 0);
2384 2385
}

2386 2387
static int
ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2388 2389
{
	int not = 0;
S
Steven Rostedt 已提交
2390

2391 2392 2393 2394 2395 2396 2397 2398 2399 2400
	/* blank or '*' mean the same */
	if (strcmp(buff, "*") == 0)
		buff[0] = 0;

	/* handle the case of 'dont filter this module' */
	if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
		buff[0] = 0;
		not = 1;
	}

2401
	return match_records(hash, buff, strlen(buff), mod, not);
2402 2403
}

2404 2405 2406 2407 2408 2409 2410 2411
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
{
2412
	struct ftrace_ops *ops = &global_ops;
2413
	struct ftrace_hash *hash;
2414
	char *mod;
2415
	int ret = -EINVAL;
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426

	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */

	/* we must have a module name */
	if (!param)
2427
		return ret;
2428 2429 2430

	mod = strsep(&param, ":");
	if (!strlen(mod))
2431
		return ret;
2432

2433
	if (enable)
2434
		hash = ops->filter_hash;
2435
	else
2436
		hash = ops->notrace_hash;
2437 2438

	ret = ftrace_match_module_records(hash, func, mod);
2439 2440 2441 2442 2443 2444
	if (!ret)
		ret = -EINVAL;
	if (ret < 0)
		return ret;

	return 0;
2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
device_initcall(ftrace_mod_cmd_init);

2458
static void
S
Steven Rostedt 已提交
2459
function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
2460
{
S
Steven Rostedt 已提交
2461
	struct ftrace_func_probe *entry;
2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477
	struct hlist_head *hhd;
	struct hlist_node *n;
	unsigned long key;

	key = hash_long(ip, FTRACE_HASH_BITS);

	hhd = &ftrace_func_hash[key];

	if (hlist_empty(hhd))
		return;

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
2478
	preempt_disable_notrace();
2479 2480 2481 2482
	hlist_for_each_entry_rcu(entry, n, hhd, node) {
		if (entry->ip == ip)
			entry->ops->func(ip, parent_ip, &entry->data);
	}
2483
	preempt_enable_notrace();
2484 2485
}

S
Steven Rostedt 已提交
2486
static struct ftrace_ops trace_probe_ops __read_mostly =
2487
{
2488
	.func		= function_trace_probe_call,
2489 2490
};

S
Steven Rostedt 已提交
2491
static int ftrace_probe_registered;
2492

S
Steven Rostedt 已提交
2493
static void __enable_ftrace_function_probe(void)
2494
{
2495
	int ret;
2496 2497
	int i;

S
Steven Rostedt 已提交
2498
	if (ftrace_probe_registered)
2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			break;
	}
	/* Nothing registered? */
	if (i == FTRACE_FUNC_HASHSIZE)
		return;

2510 2511
	ret = __register_ftrace_function(&trace_probe_ops);
	if (!ret)
2512
		ret = ftrace_startup(&trace_probe_ops, 0);
2513

S
Steven Rostedt 已提交
2514
	ftrace_probe_registered = 1;
2515 2516
}

S
Steven Rostedt 已提交
2517
static void __disable_ftrace_function_probe(void)
2518
{
2519
	int ret;
2520 2521
	int i;

S
Steven Rostedt 已提交
2522
	if (!ftrace_probe_registered)
2523 2524 2525 2526 2527 2528 2529 2530 2531
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			return;
	}

	/* no more funcs left */
2532 2533 2534 2535
	ret = __unregister_ftrace_function(&trace_probe_ops);
	if (!ret)
		ftrace_shutdown(&trace_probe_ops, 0);

S
Steven Rostedt 已提交
2536
	ftrace_probe_registered = 0;
2537 2538 2539 2540 2541
}


static void ftrace_free_entry_rcu(struct rcu_head *rhp)
{
S
Steven Rostedt 已提交
2542 2543
	struct ftrace_func_probe *entry =
		container_of(rhp, struct ftrace_func_probe, rcu);
2544 2545 2546 2547 2548 2549 2550 2551

	if (entry->ops->free)
		entry->ops->free(&entry->data);
	kfree(entry);
}


int
S
Steven Rostedt 已提交
2552
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2553 2554
			      void *data)
{
S
Steven Rostedt 已提交
2555
	struct ftrace_func_probe *entry;
2556 2557 2558
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	int type, len, not;
S
Steven Rostedt 已提交
2559
	unsigned long key;
2560 2561 2562
	int count = 0;
	char *search;

2563
	type = filter_parse_regex(glob, strlen(glob), &search, &not);
2564 2565
	len = strlen(search);

S
Steven Rostedt 已提交
2566
	/* we do not support '!' for function probes */
2567 2568 2569 2570 2571
	if (WARN_ON(not))
		return -EINVAL;

	mutex_lock(&ftrace_lock);

2572 2573
	if (unlikely(ftrace_disabled))
		goto out_unlock;
2574

2575
	do_for_each_ftrace_rec(pg, rec) {
2576

2577
		if (!ftrace_match_record(rec, NULL, search, len, type))
2578 2579 2580 2581
			continue;

		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
		if (!entry) {
S
Steven Rostedt 已提交
2582
			/* If we did not process any, then return error */
2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611
			if (!count)
				count = -ENOMEM;
			goto out_unlock;
		}

		count++;

		entry->data = data;

		/*
		 * The caller might want to do something special
		 * for each function we find. We call the callback
		 * to give the caller an opportunity to do so.
		 */
		if (ops->callback) {
			if (ops->callback(rec->ip, &entry->data) < 0) {
				/* caller does not like this func */
				kfree(entry);
				continue;
			}
		}

		entry->ops = ops;
		entry->ip = rec->ip;

		key = hash_long(entry->ip, FTRACE_HASH_BITS);
		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);

	} while_for_each_ftrace_rec();
S
Steven Rostedt 已提交
2612
	__enable_ftrace_function_probe();
2613 2614 2615 2616 2617 2618 2619 2620

 out_unlock:
	mutex_unlock(&ftrace_lock);

	return count;
}

enum {
S
Steven Rostedt 已提交
2621 2622
	PROBE_TEST_FUNC		= 1,
	PROBE_TEST_DATA		= 2
2623 2624 2625
};

static void
S
Steven Rostedt 已提交
2626
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2627 2628
				  void *data, int flags)
{
S
Steven Rostedt 已提交
2629
	struct ftrace_func_probe *entry;
2630 2631 2632 2633 2634 2635
	struct hlist_node *n, *tmp;
	char str[KSYM_SYMBOL_LEN];
	int type = MATCH_FULL;
	int i, len = 0;
	char *search;

2636
	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2637
		glob = NULL;
2638
	else if (glob) {
2639 2640
		int not;

2641
		type = filter_parse_regex(glob, strlen(glob), &search, &not);
2642 2643
		len = strlen(search);

S
Steven Rostedt 已提交
2644
		/* we do not support '!' for function probes */
2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655
		if (WARN_ON(not))
			return;
	}

	mutex_lock(&ftrace_lock);
	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];

		hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {

			/* break up if statements for readability */
S
Steven Rostedt 已提交
2656
			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2657 2658
				continue;

S
Steven Rostedt 已提交
2659
			if ((flags & PROBE_TEST_DATA) && entry->data != data)
2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673
				continue;

			/* do this last, since it is the most expensive */
			if (glob) {
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
				if (!ftrace_match(str, glob, len, type))
					continue;
			}

			hlist_del(&entry->node);
			call_rcu(&entry->rcu, ftrace_free_entry_rcu);
		}
	}
S
Steven Rostedt 已提交
2674
	__disable_ftrace_function_probe();
2675 2676 2677 2678
	mutex_unlock(&ftrace_lock);
}

void
S
Steven Rostedt 已提交
2679
unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2680 2681
				void *data)
{
S
Steven Rostedt 已提交
2682 2683
	__unregister_ftrace_function_probe(glob, ops, data,
					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
2684 2685 2686
}

void
S
Steven Rostedt 已提交
2687
unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2688
{
S
Steven Rostedt 已提交
2689
	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2690 2691
}

S
Steven Rostedt 已提交
2692
void unregister_ftrace_function_probe_all(char *glob)
2693
{
S
Steven Rostedt 已提交
2694
	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2695 2696
}

2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

int register_ftrace_command(struct ftrace_func_command *cmd)
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

int unregister_ftrace_command(struct ftrace_func_command *cmd)
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

2738 2739
static int ftrace_process_regex(struct ftrace_hash *hash,
				char *buff, int len, int enable)
2740
{
2741
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
2742
	struct ftrace_func_command *p;
2743
	int ret = -EINVAL;
2744 2745 2746 2747

	func = strsep(&next, ":");

	if (!next) {
2748
		ret = ftrace_match_records(hash, func, len);
2749 2750 2751 2752 2753
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
2754 2755
	}

2756
	/* command found */
2757 2758 2759

	command = strsep(&next, ":");

2760 2761 2762 2763 2764 2765
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
			ret = p->func(func, command, next, enable);
			goto out_unlock;
		}
2766
	}
2767 2768
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
2769

2770
	return ret;
2771 2772
}

I
Ingo Molnar 已提交
2773
static ssize_t
2774 2775
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
2776 2777
{
	struct ftrace_iterator *iter;
2778 2779
	struct trace_parser *parser;
	ssize_t ret, read;
2780

2781
	if (!cnt)
2782 2783
		return 0;

2784
	mutex_lock(&ftrace_regex_lock);
2785

2786 2787 2788 2789
	ret = -ENODEV;
	if (unlikely(ftrace_disabled))
		goto out_unlock;

2790 2791 2792 2793 2794 2795
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

2796 2797
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
2798

2799
	if (read >= 0 && trace_parser_loaded(parser) &&
2800
	    !trace_parser_cont(parser)) {
2801
		ret = ftrace_process_regex(iter->hash, parser->buffer,
2802
					   parser->idx, enable);
2803
		trace_parser_clear(parser);
2804
		if (ret)
2805
			goto out_unlock;
2806
	}
2807 2808

	ret = read;
2809
out_unlock:
2810
	mutex_unlock(&ftrace_regex_lock);
2811

2812 2813 2814
	return ret;
}

2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828
static ssize_t
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

static ssize_t
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

2829
static int
2830 2831
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
2832
{
2833
	struct ftrace_hash **orig_hash;
2834
	struct ftrace_hash *hash;
2835
	int ret;
2836

2837 2838 2839 2840
	/* All global ops uses the global ops filters */
	if (ops->flags & FTRACE_OPS_FL_GLOBAL)
		ops = &global_ops;

2841
	if (unlikely(ftrace_disabled))
2842
		return -ENODEV;
2843

2844
	if (enable)
2845
		orig_hash = &ops->filter_hash;
2846
	else
2847 2848 2849 2850 2851
		orig_hash = &ops->notrace_hash;

	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
	if (!hash)
		return -ENOMEM;
2852

2853 2854
	mutex_lock(&ftrace_regex_lock);
	if (reset)
2855
		ftrace_filter_reset(hash);
2856
	if (buf)
2857
		ftrace_match_records(hash, buf, len);
2858 2859 2860 2861 2862

	mutex_lock(&ftrace_lock);
	ret = ftrace_hash_move(orig_hash, hash);
	mutex_unlock(&ftrace_lock);

2863
	mutex_unlock(&ftrace_regex_lock);
2864 2865 2866

	free_ftrace_hash(hash);
	return ret;
2867 2868
}

2869 2870
/**
 * ftrace_set_filter - set a function to filter on in ftrace
2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
		       int len, int reset)
{
	ftrace_set_regex(ops, buf, len, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
			int len, int reset)
{
	ftrace_set_regex(ops, buf, len, reset, 0);
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
 * ftrace_set_filter - set a function to filter on in ftrace
 * @ops - the ops to set the filter with
2906 2907 2908 2909 2910 2911 2912
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
2913
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
2914
{
2915
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
2916
}
2917
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
2918

2919 2920
/**
 * ftrace_set_notrace - set a function to not trace in ftrace
2921
 * @ops - the ops to set the notrace filter with
2922 2923 2924 2925 2926 2927 2928 2929
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
2930
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
2931
{
2932
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
2933
}
2934
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
2935

2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

static int __init set_ftrace_notrace(char *str)
{
	strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
	strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

2957
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2958
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2959 2960
static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);

2961 2962
static int __init set_graph_function(char *str)
{
2963
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

static void __init set_ftrace_early_graph(char *buf)
{
	int ret;
	char *func;

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
				      func);
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

2985 2986
static void __init
set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
2987 2988 2989 2990 2991
{
	char *func;

	while (buf) {
		func = strsep(&buf, ",");
2992
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
2993 2994 2995 2996 2997 2998
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
2999
		set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1);
3000
	if (ftrace_notrace_buf[0])
3001
		set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0);
3002 3003 3004 3005
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
		set_ftrace_early_graph(ftrace_graph_buf);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3006 3007
}

I
Ingo Molnar 已提交
3008
static int
3009
ftrace_regex_release(struct inode *inode, struct file *file)
3010 3011 3012
{
	struct seq_file *m = (struct seq_file *)file->private_data;
	struct ftrace_iterator *iter;
3013
	struct ftrace_hash **orig_hash;
3014
	struct trace_parser *parser;
3015
	int filter_hash;
3016
	int ret;
3017

3018
	mutex_lock(&ftrace_regex_lock);
3019 3020 3021 3022 3023 3024 3025
	if (file->f_mode & FMODE_READ) {
		iter = m->private;

		seq_release(inode, file);
	} else
		iter = file->private_data;

3026 3027 3028
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
3029
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3030 3031
	}

3032 3033
	trace_parser_put(parser);

3034
	if (file->f_mode & FMODE_WRITE) {
3035 3036 3037
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

		if (filter_hash)
3038
			orig_hash = &iter->ops->filter_hash;
3039 3040
		else
			orig_hash = &iter->ops->notrace_hash;
3041

3042
		mutex_lock(&ftrace_lock);
3043 3044 3045 3046 3047
		/*
		 * Remove the current set, update the hash and add
		 * them back.
		 */
		ftrace_hash_rec_disable(iter->ops, filter_hash);
3048
		ret = ftrace_hash_move(orig_hash, iter->hash);
3049 3050 3051 3052 3053 3054
		if (!ret) {
			ftrace_hash_rec_enable(iter->ops, filter_hash);
			if (iter->ops->flags & FTRACE_OPS_FL_ENABLED
			    && ftrace_enabled)
				ftrace_run_update_code(FTRACE_ENABLE_CALLS);
		}
3055 3056
		mutex_unlock(&ftrace_lock);
	}
3057 3058
	free_ftrace_hash(iter->hash);
	kfree(iter);
3059

3060
	mutex_unlock(&ftrace_regex_lock);
3061 3062 3063
	return 0;
}

3064
static const struct file_operations ftrace_avail_fops = {
3065 3066 3067
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
3068
	.release = seq_release_private,
3069 3070
};

3071 3072 3073 3074 3075 3076 3077
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

3078
static const struct file_operations ftrace_filter_fops = {
3079
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
3080
	.read = seq_read,
3081
	.write = ftrace_filter_write,
3082
	.llseek = ftrace_regex_lseek,
3083
	.release = ftrace_regex_release,
3084 3085
};

3086
static const struct file_operations ftrace_notrace_fops = {
3087
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
3088
	.read = seq_read,
3089 3090
	.write = ftrace_notrace_write,
	.llseek = ftrace_regex_lseek,
3091
	.release = ftrace_regex_release,
3092 3093
};

3094 3095 3096 3097 3098
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

int ftrace_graph_count;
3099
int ftrace_graph_filter_enabled;
3100 3101 3102
unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;

static void *
3103
__g_next(struct seq_file *m, loff_t *pos)
3104
{
3105
	if (*pos >= ftrace_graph_count)
3106
		return NULL;
3107
	return &ftrace_graph_funcs[*pos];
3108
}
3109

3110 3111 3112 3113 3114
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
3115 3116 3117 3118 3119 3120
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&graph_lock);

3121
	/* Nothing, tell g_show to print all functions are enabled */
3122
	if (!ftrace_graph_filter_enabled && !*pos)
3123 3124
		return (void *)1;

3125
	return __g_next(m, pos);
3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
	unsigned long *ptr = v;

	if (!ptr)
		return 0;

3140 3141 3142 3143 3144
	if (ptr == (unsigned long *)1) {
		seq_printf(m, "#### all functions enabled ####\n");
		return 0;
	}

3145
	seq_printf(m, "%ps\n", (void *)*ptr);
3146 3147 3148 3149

	return 0;
}

J
James Morris 已提交
3150
static const struct seq_operations ftrace_graph_seq_ops = {
3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	int ret = 0;

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	mutex_lock(&graph_lock);
	if ((file->f_mode & FMODE_WRITE) &&
3167
	    (file->f_flags & O_TRUNC)) {
3168
		ftrace_graph_filter_enabled = 0;
3169 3170 3171
		ftrace_graph_count = 0;
		memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
	}
3172
	mutex_unlock(&graph_lock);
3173

3174
	if (file->f_mode & FMODE_READ)
3175 3176 3177 3178 3179
		ret = seq_open(file, &ftrace_graph_seq_ops);

	return ret;
}

3180 3181 3182 3183 3184 3185 3186 3187
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
	if (file->f_mode & FMODE_READ)
		seq_release(inode, file);
	return 0;
}

3188
static int
3189
ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3190 3191 3192
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
3193
	int search_len;
3194
	int fail = 1;
3195 3196 3197 3198
	int type, not;
	char *search;
	bool exists;
	int i;
3199

3200
	/* decode regex */
3201
	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3202 3203
	if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
		return -EBUSY;
3204 3205 3206

	search_len = strlen(search);

3207
	mutex_lock(&ftrace_lock);
3208 3209 3210 3211 3212 3213

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

3214 3215
	do_for_each_ftrace_rec(pg, rec) {

3216
		if (rec->flags & FTRACE_FL_FREE)
3217 3218
			continue;

3219
		if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3220
			/* if it is in the array */
3221
			exists = false;
3222
			for (i = 0; i < *idx; i++) {
3223 3224
				if (array[i] == rec->ip) {
					exists = true;
3225 3226
					break;
				}
3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242
			}

			if (!not) {
				fail = 0;
				if (!exists) {
					array[(*idx)++] = rec->ip;
					if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
						goto out;
				}
			} else {
				if (exists) {
					array[i] = array[--(*idx)];
					array[*idx] = 0;
					fail = 0;
				}
			}
3243
		}
3244
	} while_for_each_ftrace_rec();
3245
out:
3246
	mutex_unlock(&ftrace_lock);
3247

3248 3249 3250 3251 3252
	if (fail)
		return -EINVAL;

	ftrace_graph_filter_enabled = 1;
	return 0;
3253 3254 3255 3256 3257 3258
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
3259
	struct trace_parser parser;
3260
	ssize_t read, ret;
3261

3262
	if (!cnt)
3263 3264 3265 3266
		return 0;

	mutex_lock(&graph_lock);

3267 3268
	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
		ret = -ENOMEM;
3269
		goto out_unlock;
3270 3271
	}

3272
	read = trace_get_user(&parser, ubuf, cnt, ppos);
3273

3274
	if (read >= 0 && trace_parser_loaded((&parser))) {
3275 3276 3277
		parser.buffer[parser.idx] = 0;

		/* we allow only one expression at a time */
3278
		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3279
					parser.buffer);
3280
		if (ret)
3281
			goto out_free;
3282 3283 3284
	}

	ret = read;
3285 3286

out_free:
3287
	trace_parser_put(&parser);
3288
out_unlock:
3289 3290 3291 3292 3293 3294
	mutex_unlock(&graph_lock);

	return ret;
}

static const struct file_operations ftrace_graph_fops = {
3295 3296 3297 3298
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
	.release	= ftrace_graph_release,
3299
	.llseek		= seq_lseek,
3300 3301 3302
};
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

3303
static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3304 3305
{

3306 3307
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
3308

3309 3310 3311
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

3312 3313
	trace_create_file("set_ftrace_filter", 0644, d_tracer,
			NULL, &ftrace_filter_fops);
3314

3315
	trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3316
				    NULL, &ftrace_notrace_fops);
3317

3318
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3319
	trace_create_file("set_graph_function", 0444, d_tracer,
3320 3321 3322 3323
				    NULL,
				    &ftrace_graph_fops);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

3324 3325 3326
	return 0;
}

3327
static int ftrace_process_locs(struct module *mod,
3328
			       unsigned long *start,
3329 3330 3331 3332
			       unsigned long *end)
{
	unsigned long *p;
	unsigned long addr;
3333
	unsigned long flags;
3334

S
Steven Rostedt 已提交
3335
	mutex_lock(&ftrace_lock);
3336 3337 3338
	p = start;
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
3339 3340 3341 3342 3343 3344 3345 3346
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
3347 3348 3349
		ftrace_record_ip(addr);
	}

3350 3351 3352 3353 3354
	/*
	 * Disable interrupts to prevent interrupts from executing
	 * code that is being modified.
	 */
	local_irq_save(flags);
3355
	ftrace_update_code(mod);
3356
	local_irq_restore(flags);
S
Steven Rostedt 已提交
3357
	mutex_unlock(&ftrace_lock);
3358 3359 3360 3361

	return 0;
}

3362
#ifdef CONFIG_MODULES
3363
void ftrace_release_mod(struct module *mod)
3364 3365 3366 3367
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

3368 3369
	mutex_lock(&ftrace_lock);

3370
	if (ftrace_disabled)
3371
		goto out_unlock;
3372 3373

	do_for_each_ftrace_rec(pg, rec) {
3374
		if (within_module_core(rec->ip, mod)) {
3375 3376 3377 3378 3379 3380 3381 3382
			/*
			 * rec->ip is changed in ftrace_free_rec()
			 * It should not between s and e if record was freed.
			 */
			FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
			ftrace_free_rec(rec);
		}
	} while_for_each_ftrace_rec();
3383
 out_unlock:
3384 3385 3386 3387 3388
	mutex_unlock(&ftrace_lock);
}

static void ftrace_init_module(struct module *mod,
			       unsigned long *start, unsigned long *end)
3389
{
3390
	if (ftrace_disabled || start == end)
3391
		return;
3392
	ftrace_process_locs(mod, start, end);
3393 3394
}

3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406
static int ftrace_module_notify(struct notifier_block *self,
				unsigned long val, void *data)
{
	struct module *mod = data;

	switch (val) {
	case MODULE_STATE_COMING:
		ftrace_init_module(mod, mod->ftrace_callsites,
				   mod->ftrace_callsites +
				   mod->num_ftrace_callsites);
		break;
	case MODULE_STATE_GOING:
3407
		ftrace_release_mod(mod);
3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425
		break;
	}

	return 0;
}
#else
static int ftrace_module_notify(struct notifier_block *self,
				unsigned long val, void *data)
{
	return 0;
}
#endif /* CONFIG_MODULES */

struct notifier_block ftrace_module_nb = {
	.notifier_call = ftrace_module_notify,
	.priority = 0,
};

3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452
extern unsigned long __start_mcount_loc[];
extern unsigned long __stop_mcount_loc[];

void __init ftrace_init(void)
{
	unsigned long count, addr, flags;
	int ret;

	/* Keep the ftrace pointer to the stub */
	addr = (unsigned long)ftrace_stub;

	local_irq_save(flags);
	ftrace_dyn_arch_init(&addr);
	local_irq_restore(flags);

	/* ftrace_dyn_arch_init places the return code in addr */
	if (addr)
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;

	ret = ftrace_dyn_table_alloc(count);
	if (ret)
		goto failed;

	last_ftrace_enabled = ftrace_enabled = 1;

3453
	ret = ftrace_process_locs(NULL,
3454
				  __start_mcount_loc,
3455 3456
				  __stop_mcount_loc);

3457
	ret = register_module_notifier(&ftrace_module_nb);
3458
	if (ret)
3459 3460
		pr_warning("Failed to register trace ftrace module notifier\n");

3461 3462
	set_ftrace_early_filters();

3463 3464 3465 3466 3467
	return;
 failed:
	ftrace_disabled = 1;
}

3468
#else
3469

3470
static struct ftrace_ops global_ops = {
3471 3472 3473
	.func			= ftrace_stub,
};

3474 3475 3476 3477 3478 3479 3480
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
device_initcall(ftrace_nodyn_init);

3481 3482
static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
static inline void ftrace_startup_enable(int command) { }
3483
/* Keep as macros so we do not need to define the commands */
3484 3485 3486 3487 3488
# define ftrace_startup(ops, command)			\
	({						\
		(ops)->flags |= FTRACE_OPS_FL_ENABLED;	\
		0;					\
	})
3489
# define ftrace_shutdown(ops, command)	do { } while (0)
I
Ingo Molnar 已提交
3490 3491
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
3492 3493 3494 3495 3496 3497 3498

static inline int
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
{
	return 1;
}

3499 3500
#endif /* CONFIG_DYNAMIC_FTRACE */

3501 3502 3503
static void
ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
{
3504
	struct ftrace_ops *op;
3505

3506 3507 3508 3509
	if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
		return;

	trace_recursion_set(TRACE_INTERNAL_BIT);
3510 3511 3512 3513 3514 3515
	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
	op = rcu_dereference_raw(ftrace_ops_list);
3516 3517 3518 3519 3520
	while (op != &ftrace_list_end) {
		if (ftrace_ops_test(op, ip))
			op->func(ip, parent_ip);
		op = rcu_dereference_raw(op->next);
	};
3521
	preempt_enable_notrace();
3522
	trace_recursion_clear(TRACE_INTERNAL_BIT);
3523 3524
}

3525
static void clear_ftrace_swapper(void)
S
Steven Rostedt 已提交
3526 3527
{
	struct task_struct *p;
3528
	int cpu;
S
Steven Rostedt 已提交
3529

3530 3531 3532
	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
S
Steven Rostedt 已提交
3533
		clear_tsk_trace_trace(p);
3534 3535 3536
	}
	put_online_cpus();
}
S
Steven Rostedt 已提交
3537

3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548
static void set_ftrace_swapper(void)
{
	struct task_struct *p;
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
		set_tsk_trace_trace(p);
	}
	put_online_cpus();
S
Steven Rostedt 已提交
3549 3550
}

3551 3552 3553 3554
static void clear_ftrace_pid(struct pid *pid)
{
	struct task_struct *p;

3555
	rcu_read_lock();
3556 3557 3558
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		clear_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
3559 3560
	rcu_read_unlock();

3561 3562 3563 3564
	put_pid(pid);
}

static void set_ftrace_pid(struct pid *pid)
S
Steven Rostedt 已提交
3565 3566 3567
{
	struct task_struct *p;

3568
	rcu_read_lock();
S
Steven Rostedt 已提交
3569 3570 3571
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		set_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
3572
	rcu_read_unlock();
S
Steven Rostedt 已提交
3573 3574
}

3575
static void clear_ftrace_pid_task(struct pid *pid)
3576
{
3577
	if (pid == ftrace_swapper_pid)
3578 3579
		clear_ftrace_swapper();
	else
3580
		clear_ftrace_pid(pid);
3581 3582 3583 3584 3585 3586 3587 3588 3589 3590
}

static void set_ftrace_pid_task(struct pid *pid)
{
	if (pid == ftrace_swapper_pid)
		set_ftrace_swapper();
	else
		set_ftrace_pid(pid);
}

3591
static int ftrace_pid_add(int p)
3592
{
S
Steven Rostedt 已提交
3593
	struct pid *pid;
3594 3595
	struct ftrace_pid *fpid;
	int ret = -EINVAL;
3596

3597
	mutex_lock(&ftrace_lock);
3598

3599 3600 3601 3602
	if (!p)
		pid = ftrace_swapper_pid;
	else
		pid = find_get_pid(p);
3603

3604 3605
	if (!pid)
		goto out;
3606

3607
	ret = 0;
3608

3609 3610 3611
	list_for_each_entry(fpid, &ftrace_pids, list)
		if (fpid->pid == pid)
			goto out_put;
S
Steven Rostedt 已提交
3612

3613
	ret = -ENOMEM;
3614

3615 3616 3617
	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
	if (!fpid)
		goto out_put;
3618

3619 3620
	list_add(&fpid->list, &ftrace_pids);
	fpid->pid = pid;
3621

3622
	set_ftrace_pid_task(pid);
S
Steven Rostedt 已提交
3623

3624 3625 3626 3627 3628 3629 3630 3631 3632
	ftrace_update_pid_func();
	ftrace_startup_enable(0);

	mutex_unlock(&ftrace_lock);
	return 0;

out_put:
	if (pid != ftrace_swapper_pid)
		put_pid(pid);
S
Steven Rostedt 已提交
3633

3634 3635 3636 3637 3638 3639 3640 3641
out:
	mutex_unlock(&ftrace_lock);
	return ret;
}

static void ftrace_pid_reset(void)
{
	struct ftrace_pid *fpid, *safe;
S
Steven Rostedt 已提交
3642

3643 3644 3645 3646 3647 3648 3649 3650
	mutex_lock(&ftrace_lock);
	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
		struct pid *pid = fpid->pid;

		clear_ftrace_pid_task(pid);

		list_del(&fpid->list);
		kfree(fpid);
3651 3652 3653 3654 3655
	}

	ftrace_update_pid_func();
	ftrace_startup_enable(0);

S
Steven Rostedt 已提交
3656
	mutex_unlock(&ftrace_lock);
3657
}
3658

3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720
static void *fpid_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&ftrace_lock);

	if (list_empty(&ftrace_pids) && (!*pos))
		return (void *) 1;

	return seq_list_start(&ftrace_pids, *pos);
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
	if (v == (void *)1)
		return NULL;

	return seq_list_next(v, &ftrace_pids, pos);
}

static void fpid_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);

	if (v == (void *)1) {
		seq_printf(m, "no pid\n");
		return 0;
	}

	if (fpid->pid == ftrace_swapper_pid)
		seq_printf(m, "swapper tasks\n");
	else
		seq_printf(m, "%u\n", pid_vnr(fpid->pid));

	return 0;
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
	int ret = 0;

	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
		ftrace_pid_reset();

	if (file->f_mode & FMODE_READ)
		ret = seq_open(file, &ftrace_pid_sops);

	return ret;
}

3721 3722 3723 3724
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
3725
	char buf[64], *tmp;
3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736
	long val;
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

3737 3738 3739 3740
	/*
	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
	 * to clean the filter quietly.
	 */
3741 3742
	tmp = strstrip(buf);
	if (strlen(tmp) == 0)
3743 3744
		return 1;

3745
	ret = strict_strtol(tmp, 10, &val);
3746 3747 3748
	if (ret < 0)
		return ret;

3749
	ret = ftrace_pid_add(val);
3750

3751 3752
	return ret ? ret : cnt;
}
3753

3754 3755 3756 3757 3758
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
	if (file->f_mode & FMODE_READ)
		seq_release(inode, file);
3759

3760
	return 0;
3761 3762
}

3763
static const struct file_operations ftrace_pid_fops = {
3764 3765 3766 3767 3768
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= ftrace_pid_release,
3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780
};

static __init int ftrace_init_debugfs(void)
{
	struct dentry *d_tracer;

	d_tracer = tracing_init_dentry();
	if (!d_tracer)
		return 0;

	ftrace_init_dyn_debugfs(d_tracer);

3781 3782
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
			    NULL, &ftrace_pid_fops);
3783 3784 3785

	ftrace_profile_debugfs(d_tracer);

3786 3787 3788 3789
	return 0;
}
fs_initcall(ftrace_init_debugfs);

S
Steven Rostedt 已提交
3790
/**
3791
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
3792 3793 3794 3795 3796
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
3797
void ftrace_kill(void)
S
Steven Rostedt 已提交
3798 3799 3800 3801 3802 3803
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

3804
/**
3805 3806
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
3807
 *
3808 3809 3810 3811 3812 3813
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
3814
 */
3815
int register_ftrace_function(struct ftrace_ops *ops)
3816
{
3817
	int ret = -1;
3818

S
Steven Rostedt 已提交
3819
	mutex_lock(&ftrace_lock);
3820

3821 3822 3823
	if (unlikely(ftrace_disabled))
		goto out_unlock;

3824
	ret = __register_ftrace_function(ops);
3825
	if (!ret)
3826
		ret = ftrace_startup(ops, 0);
3827

3828

3829
 out_unlock:
S
Steven Rostedt 已提交
3830
	mutex_unlock(&ftrace_lock);
3831
	return ret;
3832
}
3833
EXPORT_SYMBOL_GPL(register_ftrace_function);
3834 3835

/**
3836
 * unregister_ftrace_function - unregister a function for profiling.
3837 3838 3839 3840 3841 3842 3843 3844
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
3845
	mutex_lock(&ftrace_lock);
3846
	ret = __unregister_ftrace_function(ops);
3847 3848
	if (!ret)
		ftrace_shutdown(ops, 0);
S
Steven Rostedt 已提交
3849
	mutex_unlock(&ftrace_lock);
3850 3851 3852

	return ret;
}
3853
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
3854

I
Ingo Molnar 已提交
3855
int
3856
ftrace_enable_sysctl(struct ctl_table *table, int write,
3857
		     void __user *buffer, size_t *lenp,
3858 3859
		     loff_t *ppos)
{
3860
	int ret = -ENODEV;
3861

S
Steven Rostedt 已提交
3862
	mutex_lock(&ftrace_lock);
3863

3864 3865 3866 3867
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
3868

3869
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3870 3871
		goto out;

3872
	last_ftrace_enabled = !!ftrace_enabled;
3873 3874 3875 3876 3877 3878

	if (ftrace_enabled) {

		ftrace_startup_sysctl();

		/* we are starting ftrace again */
3879 3880 3881
		if (ftrace_ops_list != &ftrace_list_end) {
			if (ftrace_ops_list->next == &ftrace_list_end)
				ftrace_trace_function = ftrace_ops_list->func;
3882
			else
3883
				ftrace_trace_function = ftrace_ops_list_func;
3884 3885 3886 3887 3888 3889 3890 3891 3892 3893
		}

	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
3894
	mutex_unlock(&ftrace_lock);
3895
	return ret;
3896
}
3897

3898
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3899

3900
static int ftrace_graph_active;
3901
static struct notifier_block ftrace_suspend_notifier;
3902

3903 3904 3905 3906 3907
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

3908 3909 3910
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
3911
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	unsigned long flags;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

	read_lock_irqsave(&tasklist_lock, flags);
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
3942
			atomic_set(&t->tracing_graph_pause, 0);
3943
			atomic_set(&t->trace_overrun, 0);
3944 3945 3946 3947
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958
		}
	} while_each_thread(g, t);

unlock:
	read_unlock_irqrestore(&tasklist_lock, flags);
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

3959
static void
3960 3961
ftrace_graph_probe_sched_switch(void *ignore,
			struct task_struct *prev, struct task_struct *next)
3962 3963 3964 3965
{
	unsigned long long timestamp;
	int index;

3966 3967 3968 3969 3970 3971 3972
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
	if (trace_flags & TRACE_ITER_SLEEP_TIME)
		return;

3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

3991
/* Allocate a return stack for each task */
3992
static int start_graph_tracing(void)
3993 3994
{
	struct ftrace_ret_stack **ret_stack_list;
3995
	int ret, cpu;
3996 3997 3998 3999 4000 4001 4002 4003

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

4004
	/* The cpu_boot init_task->ret_stack will never be freed */
4005 4006
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
4007
			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4008
	}
4009

4010 4011 4012 4013
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

4014
	if (!ret) {
4015
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4016 4017 4018 4019 4020
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

4021 4022 4023 4024
	kfree(ret_stack_list);
	return ret;
}

4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

4046 4047
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
4048
{
4049 4050
	int ret = 0;

S
Steven Rostedt 已提交
4051
	mutex_lock(&ftrace_lock);
4052

4053
	/* we currently allow only one tracer registered at a time */
4054
	if (ftrace_graph_active) {
4055 4056 4057 4058
		ret = -EBUSY;
		goto out;
	}

4059 4060 4061
	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
	register_pm_notifier(&ftrace_suspend_notifier);

4062
	ftrace_graph_active++;
4063
	ret = start_graph_tracing();
4064
	if (ret) {
4065
		ftrace_graph_active--;
4066 4067
		goto out;
	}
4068

4069 4070
	ftrace_graph_return = retfunc;
	ftrace_graph_entry = entryfunc;
4071

4072
	ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
4073 4074

out:
S
Steven Rostedt 已提交
4075
	mutex_unlock(&ftrace_lock);
4076
	return ret;
4077 4078
}

4079
void unregister_ftrace_graph(void)
4080
{
S
Steven Rostedt 已提交
4081
	mutex_lock(&ftrace_lock);
4082

4083
	if (unlikely(!ftrace_graph_active))
4084 4085
		goto out;

4086
	ftrace_graph_active--;
4087
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4088
	ftrace_graph_entry = ftrace_graph_entry_stub;
4089
	ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
4090
	unregister_pm_notifier(&ftrace_suspend_notifier);
4091
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4092

4093
 out:
S
Steven Rostedt 已提交
4094
	mutex_unlock(&ftrace_lock);
4095
}
4096

4097 4098 4099 4100 4101 4102 4103 4104
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);

static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
	atomic_set(&t->tracing_graph_pause, 0);
	atomic_set(&t->trace_overrun, 0);
	t->ftrace_timestamp = 0;
L
Lucas De Marchi 已提交
4105
	/* make curr_ret_stack visible before we add the ret_stack */
4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139
	smp_wmb();
	t->ret_stack = ret_stack;
}

/*
 * Allocate a return stack for the idle task. May be the first
 * time through, or it may be done by CPU hotplug online.
 */
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
{
	t->curr_ret_stack = -1;
	/*
	 * The idle task has no parent, it either has its own
	 * stack or no stack at all.
	 */
	if (t->ret_stack)
		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));

	if (ftrace_graph_active) {
		struct ftrace_ret_stack *ret_stack;

		ret_stack = per_cpu(idle_ret_stack, cpu);
		if (!ret_stack) {
			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
					    * sizeof(struct ftrace_ret_stack),
					    GFP_KERNEL);
			if (!ret_stack)
				return;
			per_cpu(idle_ret_stack, cpu) = ret_stack;
		}
		graph_init_task(t, ret_stack);
	}
}

4140
/* Allocate a return stack for newly created task */
4141
void ftrace_graph_init_task(struct task_struct *t)
4142
{
4143 4144
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;
4145
	t->curr_ret_stack = -1;
4146

4147
	if (ftrace_graph_active) {
4148 4149 4150
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4151 4152
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
4153
		if (!ret_stack)
4154
			return;
4155
		graph_init_task(t, ret_stack);
4156
	}
4157 4158
}

4159
void ftrace_graph_exit_task(struct task_struct *t)
4160
{
4161 4162
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

4163
	t->ret_stack = NULL;
4164 4165 4166 4167
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
4168
}
S
Steven Rostedt 已提交
4169 4170 4171 4172 4173

void ftrace_graph_stop(void)
{
	ftrace_stop();
}
4174
#endif