ftrace.c 70.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
 *  Copyright (C) 2004 William Lee Irwin III
 */

16 17 18
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
#include <linux/kallsyms.h>
19
#include <linux/seq_file.h>
20
#include <linux/suspend.h>
21
#include <linux/debugfs.h>
22
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
23
#include <linux/kthread.h>
24
#include <linux/uaccess.h>
25
#include <linux/kprobes.h>
I
Ingo Molnar 已提交
26
#include <linux/ftrace.h>
27
#include <linux/sysctl.h>
28
#include <linux/ctype.h>
29
#include <linux/list.h>
30
#include <linux/hash.h>
31

32
#include <trace/events/sched.h>
33

34
#include <asm/ftrace.h>
35
#include <asm/setup.h>
36

37
#include "trace_output.h"
S
Steven Rostedt 已提交
38
#include "trace_stat.h"
39

40 41 42 43 44 45 46 47 48 49 50 51
#define FTRACE_WARN_ON(cond)			\
	do {					\
		if (WARN_ON(cond))		\
			ftrace_kill();		\
	} while (0)

#define FTRACE_WARN_ON_ONCE(cond)		\
	do {					\
		if (WARN_ON_ONCE(cond))		\
			ftrace_kill();		\
	} while (0)

52 53 54 55
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)

56 57
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
58
static int last_ftrace_enabled;
59

60 61 62
/* Quick disabling of function tracer. */
int function_trace_stop;

63 64 65 66 67 68 69
/* List for set_ftrace_pid's pids. */
LIST_HEAD(ftrace_pids);
struct ftrace_pid {
	struct list_head list;
	struct pid *pid;
};

70 71 72 73 74 75
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

76
static DEFINE_MUTEX(ftrace_lock);
77

78 79
static struct ftrace_ops ftrace_list_end __read_mostly =
{
80
	.func		= ftrace_stub,
81 82 83 84
};

static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
85
ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
86
ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
87

88 89 90 91
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
#endif

I
Ingo Molnar 已提交
92
static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
93 94 95 96 97 98 99 100 101 102 103 104 105 106
{
	struct ftrace_ops *op = ftrace_list;

	/* in case someone actually ports this to alpha! */
	read_barrier_depends();

	while (op != &ftrace_list_end) {
		/* silly alpha */
		read_barrier_depends();
		op->func(ip, parent_ip);
		op = op->next;
	};
}

107 108
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
{
109
	if (!test_tsk_trace_trace(current))
110 111 112 113 114 115 116 117 118 119 120 121
		return;

	ftrace_pid_function(ip, parent_ip);
}

static void set_ftrace_pid_function(ftrace_func_t func)
{
	/* do not set ftrace_pid_function to itself! */
	if (func != ftrace_pid_func)
		ftrace_pid_function = func;
}

122
/**
123
 * clear_ftrace_function - reset the ftrace function
124
 *
125 126
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
127
 */
128
void clear_ftrace_function(void)
129
{
130
	ftrace_trace_function = ftrace_stub;
131
	__ftrace_trace_function = ftrace_stub;
132
	ftrace_pid_function = ftrace_stub;
133 134
}

135 136 137 138 139 140 141 142 143 144 145 146 147 148
#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
/*
 * For those archs that do not test ftrace_trace_stop in their
 * mcount call site, we need to do it from C.
 */
static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
{
	if (function_trace_stop)
		return;

	__ftrace_trace_function(ip, parent_ip);
}
#endif

I
Ingo Molnar 已提交
149
static int __register_ftrace_function(struct ftrace_ops *ops)
150
{
151 152 153 154 155 156 157 158 159
	ops->next = ftrace_list;
	/*
	 * We are entering ops into the ftrace_list but another
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
	 * the ops pointer included into the ftrace_list.
	 */
	smp_wmb();
	ftrace_list = ops;
160

161
	if (ftrace_enabled) {
162 163 164 165 166 167 168
		ftrace_func_t func;

		if (ops->next == &ftrace_list_end)
			func = ops->func;
		else
			func = ftrace_list_func;

169
		if (!list_empty(&ftrace_pids)) {
170 171 172 173
			set_ftrace_pid_function(func);
			func = ftrace_pid_func;
		}

174 175 176 177
		/*
		 * For one func, simply call it directly.
		 * For more than one func, call the chain.
		 */
178
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
179
		ftrace_trace_function = func;
180
#else
181
		__ftrace_trace_function = func;
182 183
		ftrace_trace_function = ftrace_test_stop_func;
#endif
184
	}
185

186 187 188
	return 0;
}

I
Ingo Molnar 已提交
189
static int __unregister_ftrace_function(struct ftrace_ops *ops)
190 191 192 193
{
	struct ftrace_ops **p;

	/*
194 195
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
196 197 198 199
	 */
	if (ftrace_list == ops && ops->next == &ftrace_list_end) {
		ftrace_trace_function = ftrace_stub;
		ftrace_list = &ftrace_list_end;
S
Steven Rostedt 已提交
200
		return 0;
201 202 203 204 205 206
	}

	for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
207 208
	if (*p != ops)
		return -1;
209 210 211

	*p = (*p)->next;

212 213
	if (ftrace_enabled) {
		/* If we only have one func left, then call that directly */
214 215 216
		if (ftrace_list->next == &ftrace_list_end) {
			ftrace_func_t func = ftrace_list->func;

217
			if (!list_empty(&ftrace_pids)) {
218 219 220 221 222 223 224 225 226
				set_ftrace_pid_function(func);
				func = ftrace_pid_func;
			}
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
			ftrace_trace_function = func;
#else
			__ftrace_trace_function = func;
#endif
		}
227
	}
228

S
Steven Rostedt 已提交
229
	return 0;
230 231
}

232 233 234 235 236
static void ftrace_update_pid_func(void)
{
	ftrace_func_t func;

	if (ftrace_trace_function == ftrace_stub)
237
		return;
238

239
#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
240
	func = ftrace_trace_function;
241 242 243
#else
	func = __ftrace_trace_function;
#endif
244

245
	if (!list_empty(&ftrace_pids)) {
246 247 248
		set_ftrace_pid_function(func);
		func = ftrace_pid_func;
	} else {
249 250
		if (func == ftrace_pid_func)
			func = ftrace_pid_function;
251 252 253 254 255 256 257 258 259
	}

#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
	ftrace_trace_function = func;
#else
	__ftrace_trace_function = func;
#endif
}

260 261 262 263 264
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
265 266 267
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
#endif
268 269
};

270 271 272 273
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
274 275
};

276 277 278 279 280 281 282 283
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

284 285
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
286

287 288
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
289

290 291 292 293
static int ftrace_profile_bits __read_mostly;
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
294 295
static DEFINE_MUTEX(ftrace_profile_lock);

296
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
297 298 299

#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */

S
Steven Rostedt 已提交
300 301 302
static void *
function_stat_next(void *v, int idx)
{
303 304
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
305

306
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
307 308

 again:
L
Li Zefan 已提交
309 310 311
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
312 313 314 315 316
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
317 318
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
319 320 321 322 323 324 325
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
326 327 328 329 330 331 332
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
333 334
}

335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
351 352
static int function_stat_cmp(void *p1, void *p2)
{
353 354
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
355 356 357 358 359 360 361 362

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
363
#endif
S
Steven Rostedt 已提交
364 365 366

static int function_stat_headers(struct seq_file *m)
{
367
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
368 369 370 371
	seq_printf(m, "  Function                               "
		   "Hit    Time            Avg\n"
		      "  --------                               "
		   "---    ----            ---\n");
372
#else
S
Steven Rostedt 已提交
373 374
	seq_printf(m, "  Function                               Hit\n"
		      "  --------                               ---\n");
375
#endif
S
Steven Rostedt 已提交
376 377 378 379 380
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
381
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
382
	char str[KSYM_SYMBOL_LEN];
383 384
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	static DEFINE_MUTEX(mutex);
385 386
	static struct trace_seq s;
	unsigned long long avg;
387
#endif
S
Steven Rostedt 已提交
388 389

	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
390 391 392 393
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	seq_printf(m, "    ");
394 395 396 397 398 399 400 401
	avg = rec->time;
	do_div(avg, rec->counter);

	mutex_lock(&mutex);
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
402 403 404 405
	trace_print_seq(m, &s);
	mutex_unlock(&mutex);
#endif
	seq_putc(m, '\n');
S
Steven Rostedt 已提交
406 407 408 409

	return 0;
}

410
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
411
{
412
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
413

414
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
415

416 417 418 419
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
420 421
	}

422
	memset(stat->hash, 0,
423 424
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
425

426
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
427 428
{
	struct ftrace_profile_page *pg;
429 430
	int functions;
	int pages;
431
	int i;
S
Steven Rostedt 已提交
432

433
	/* If we already allocated, do nothing */
434
	if (stat->pages)
435
		return 0;
S
Steven Rostedt 已提交
436

437 438
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
439
		return -ENOMEM;
S
Steven Rostedt 已提交
440

441 442 443 444 445 446 447 448 449 450 451 452 453
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

454
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
455

456 457 458
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

	for (i = 0; i < pages; i++) {
459 460
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
461
			goto out_free;
462 463 464 465
		pg = pg->next;
	}

	return 0;
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	free_page((unsigned long)stat->pages);
	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
481 482
}

483
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
484
{
485
	struct ftrace_profile_stat *stat;
486
	int size;
S
Steven Rostedt 已提交
487

488 489 490
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
491
		/* If the profile is already created, simply reset it */
492
		ftrace_profile_reset(stat);
493 494
		return 0;
	}
S
Steven Rostedt 已提交
495

496 497 498 499 500
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
501

502
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
503

504
	if (!stat->hash)
505 506
		return -ENOMEM;

507 508
	if (!ftrace_profile_bits) {
		size--;
509

510 511 512
		for (; size; size >>= 1)
			ftrace_profile_bits++;
	}
513

514
	/* Preallocate the function profiling pages */
515 516 517
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
518 519 520 521
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
522 523
}

524 525 526 527 528 529 530 531 532 533 534 535 536 537
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

	for_each_online_cpu(cpu) {
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

538
/* interrupts must be disabled */
539 540
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
541
{
542
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
543 544 545 546 547
	struct hlist_head *hhd;
	struct hlist_node *n;
	unsigned long key;

	key = hash_long(ip, ftrace_profile_bits);
548
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
549 550 551 552 553 554

	if (hlist_empty(hhd))
		return NULL;

	hlist_for_each_entry_rcu(rec, n, hhd, node) {
		if (rec->ip == ip)
555 556 557 558 559 560
			return rec;
	}

	return NULL;
}

561 562
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
563 564 565 566
{
	unsigned long key;

	key = hash_long(rec->ip, ftrace_profile_bits);
567
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
568 569
}

570 571 572
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
573
static struct ftrace_profile *
574
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
575 576 577
{
	struct ftrace_profile *rec = NULL;

578
	/* prevent recursion (from NMIs) */
579
	if (atomic_inc_return(&stat->disabled) != 1)
580 581 582
		goto out;

	/*
583 584
	 * Try to find the function again since an NMI
	 * could have added it
585
	 */
586
	rec = ftrace_find_profiled_func(stat, ip);
587
	if (rec)
588
		goto out;
589

590 591 592 593
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
594
	}
595

596
	rec = &stat->pages->records[stat->pages->index++];
597
	rec->ip = ip;
598
	ftrace_add_profile(stat, rec);
599

S
Steven Rostedt 已提交
600
 out:
601
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
602 603 604 605 606 607 608

	return rec;
}

static void
function_profile_call(unsigned long ip, unsigned long parent_ip)
{
609
	struct ftrace_profile_stat *stat;
610
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
611 612 613 614 615 616
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
617 618

	stat = &__get_cpu_var(ftrace_profile_stats);
619
	if (!stat->hash || !ftrace_profile_enabled)
620 621 622
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
623
	if (!rec) {
624
		rec = ftrace_profile_alloc(stat, ip);
625 626 627
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
628 629 630 631 632 633

	rec->counter++;
 out:
	local_irq_restore(flags);
}

634 635 636 637 638 639 640 641 642
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
	function_profile_call(trace->func, 0);
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
643
	struct ftrace_profile_stat *stat;
644
	unsigned long long calltime;
645
	struct ftrace_profile *rec;
646
	unsigned long flags;
647 648

	local_irq_save(flags);
649
	stat = &__get_cpu_var(ftrace_profile_stats);
650
	if (!stat->hash || !ftrace_profile_enabled)
651 652
		goto out;

653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
	calltime = trace->rettime - trace->calltime;

	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

670
	rec = ftrace_find_profiled_func(stat, trace->func);
671
	if (rec)
672 673
		rec->time += calltime;

674
 out:
675 676 677 678 679 680 681 682 683 684 685 686 687 688
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
S
Steven Rostedt 已提交
689 690
static struct ftrace_ops ftrace_profile_ops __read_mostly =
{
691
	.func		= function_profile_call,
S
Steven Rostedt 已提交
692 693
};

694 695 696 697 698 699 700 701 702 703 704
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
705 706 707 708 709
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
710
	char buf[64];		/* big enough to hold a number */
S
Steven Rostedt 已提交
711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	ret = strict_strtoul(buf, 10, &val);
	if (ret < 0)
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
730 731 732 733 734 735
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

736 737 738 739 740
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
741 742 743
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
744 745 746 747
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
748
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
749 750
		}
	}
751
 out:
S
Steven Rostedt 已提交
752 753
	mutex_unlock(&ftrace_profile_lock);

754
	*ppos += cnt;
S
Steven Rostedt 已提交
755 756 757 758

	return cnt;
}

759 760 761 762
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
763
	char buf[64];		/* big enough to hold a number */
764 765 766 767 768 769
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
770 771 772 773 774 775
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
};

776 777
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
778 779 780 781 782 783
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
784 785
};

786
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
787
{
788
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
789
	struct dentry *entry;
790
	char *name;
S
Steven Rostedt 已提交
791
	int ret;
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

		/* allocate enough for function name + cpu number */
		name = kmalloc(32, GFP_KERNEL);
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		snprintf(name, 32, "function%d", cpu);
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
820 821 822 823 824 825 826 827 828 829
	}

	entry = debugfs_create_file("function_profile_enabled", 0644,
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
		pr_warning("Could not create debugfs "
			   "'function_profile_enabled' entry\n");
}

#else /* CONFIG_FUNCTION_PROFILER */
830
static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
831 832 833 834
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

#ifdef CONFIG_DYNAMIC_FTRACE

#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;

struct ftrace_func_probe {
	struct hlist_node	node;
	struct ftrace_probe_ops	*ops;
	unsigned long		flags;
	unsigned long		ip;
	void			*data;
	struct rcu_head		rcu;
};

enum {
	FTRACE_ENABLE_CALLS		= (1 << 0),
	FTRACE_DISABLE_CALLS		= (1 << 1),
	FTRACE_UPDATE_TRACE_FUNC	= (1 << 2),
	FTRACE_ENABLE_MCOUNT		= (1 << 3),
	FTRACE_DISABLE_MCOUNT		= (1 << 4),
	FTRACE_START_FUNC_RET		= (1 << 5),
	FTRACE_STOP_FUNC_RET		= (1 << 6),
};

static int ftrace_filtered;

static struct dyn_ftrace *ftrace_new_addrs;

static DEFINE_MUTEX(ftrace_regex_lock);

struct ftrace_page {
	struct ftrace_page	*next;
	int			index;
	struct dyn_ftrace	records[];
};

#define ENTRIES_PER_PAGE \
  ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

static struct dyn_ftrace *ftrace_free_records;

/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

A
Abhishek Sagar 已提交
901
#ifdef CONFIG_KPROBES
902 903 904

static int frozen_record_count;

A
Abhishek Sagar 已提交
905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
static inline void freeze_record(struct dyn_ftrace *rec)
{
	if (!(rec->flags & FTRACE_FL_FROZEN)) {
		rec->flags |= FTRACE_FL_FROZEN;
		frozen_record_count++;
	}
}

static inline void unfreeze_record(struct dyn_ftrace *rec)
{
	if (rec->flags & FTRACE_FL_FROZEN) {
		rec->flags &= ~FTRACE_FL_FROZEN;
		frozen_record_count--;
	}
}

static inline int record_frozen(struct dyn_ftrace *rec)
{
	return rec->flags & FTRACE_FL_FROZEN;
}
#else
# define freeze_record(rec)			({ 0; })
# define unfreeze_record(rec)			({ 0; })
# define record_frozen(rec)			({ 0; })
#endif /* CONFIG_KPROBES */

I
Ingo Molnar 已提交
931
static void ftrace_free_rec(struct dyn_ftrace *rec)
932
{
933
	rec->freelist = ftrace_free_records;
934 935 936 937
	ftrace_free_records = rec;
	rec->flags |= FTRACE_FL_FREE;
}

I
Ingo Molnar 已提交
938
static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
939
{
940 941 942 943 944 945 946
	struct dyn_ftrace *rec;

	/* First check for freed records */
	if (ftrace_free_records) {
		rec = ftrace_free_records;

		if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
947
			FTRACE_WARN_ON_ONCE(1);
948 949 950 951
			ftrace_free_records = NULL;
			return NULL;
		}

952
		ftrace_free_records = rec->freelist;
953 954 955 956
		memset(rec, 0, sizeof(*rec));
		return rec;
	}

957
	if (ftrace_pages->index == ENTRIES_PER_PAGE) {
S
Steven Rostedt 已提交
958 959 960 961 962 963 964
		if (!ftrace_pages->next) {
			/* allocate another page */
			ftrace_pages->next =
				(void *)get_zeroed_page(GFP_KERNEL);
			if (!ftrace_pages->next)
				return NULL;
		}
965 966 967 968 969 970
		ftrace_pages = ftrace_pages->next;
	}

	return &ftrace_pages->records[ftrace_pages->index++];
}

S
Steven Rostedt 已提交
971
static struct dyn_ftrace *
972
ftrace_record_ip(unsigned long ip)
973
{
S
Steven Rostedt 已提交
974
	struct dyn_ftrace *rec;
975

976
	if (ftrace_disabled)
S
Steven Rostedt 已提交
977
		return NULL;
978

S
Steven Rostedt 已提交
979 980 981
	rec = ftrace_alloc_dyn_node(ip);
	if (!rec)
		return NULL;
982

S
Steven Rostedt 已提交
983
	rec->ip = ip;
984
	rec->newlist = ftrace_new_addrs;
985
	ftrace_new_addrs = rec;
986

S
Steven Rostedt 已提交
987
	return rec;
988 989
}

990 991 992 993 994 995 996 997 998 999
static void print_ip_ins(const char *fmt, unsigned char *p)
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1000
static void ftrace_bug(int failed, unsigned long ip)
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
{
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
		print_ip_ins(" actual: ", (unsigned char *)ip);
		printk(KERN_CONT "\n");
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
}

1027

1028
static int
1029
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1030
{
1031
	unsigned long ftrace_addr;
1032
	unsigned long flag = 0UL;
1033

1034
	ftrace_addr = (unsigned long)FTRACE_ADDR;
1035

S
Steven Rostedt 已提交
1036
	/*
1037 1038
	 * If this record is not to be traced or we want to disable it,
	 * then disable it.
S
Steven Rostedt 已提交
1039
	 *
1040
	 * If we want to enable it and filtering is off, then enable it.
S
Steven Rostedt 已提交
1041
	 *
1042 1043
	 * If we want to enable it and filtering is on, enable it only if
	 * it's filtered
S
Steven Rostedt 已提交
1044
	 */
1045 1046 1047 1048
	if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
		if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
			flag = FTRACE_FL_ENABLED;
	}
S
Steven Rostedt 已提交
1049

1050 1051 1052
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
		return 0;
S
Steven Rostedt 已提交
1053

1054 1055 1056
	if (flag) {
		rec->flags |= FTRACE_FL_ENABLED;
		return ftrace_make_call(rec, ftrace_addr);
1057 1058
	}

1059 1060
	rec->flags &= ~FTRACE_FL_ENABLED;
	return ftrace_make_nop(NULL, rec, ftrace_addr);
1061 1062
}

I
Ingo Molnar 已提交
1063
static void ftrace_replace_code(int enable)
1064 1065 1066
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
1067
	int failed;
1068

1069 1070
	do_for_each_ftrace_rec(pg, rec) {
		/*
1071 1072
		 * Skip over free records, records that have
		 * failed and not converted.
1073 1074
		 */
		if (rec->flags & FTRACE_FL_FREE ||
1075
		    rec->flags & FTRACE_FL_FAILED ||
1076
		    !(rec->flags & FTRACE_FL_CONVERTED))
1077 1078 1079 1080 1081 1082 1083 1084 1085
			continue;

		/* ignore updates to this record's mcount site */
		if (get_kprobe((void *)rec->ip)) {
			freeze_record(rec);
			continue;
		} else {
			unfreeze_record(rec);
		}
1086

1087
		failed = __ftrace_replace_code(rec, enable);
1088
		if (failed) {
1089
			rec->flags |= FTRACE_FL_FAILED;
1090 1091 1092
			ftrace_bug(failed, rec->ip);
			/* Stop processing */
			return;
1093
		}
1094
	} while_for_each_ftrace_rec();
1095 1096
}

1097
static int
1098
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1099 1100
{
	unsigned long ip;
1101
	int ret;
1102 1103 1104

	ip = rec->ip;

1105
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1106
	if (ret) {
1107
		ftrace_bug(ret, ip);
1108
		rec->flags |= FTRACE_FL_FAILED;
1109
		return 0;
1110
	}
1111
	return 1;
1112 1113
}

1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

I
Ingo Molnar 已提交
1132
static int __ftrace_modify_code(void *data)
1133
{
1134 1135
	int *command = data;

1136
	if (*command & FTRACE_ENABLE_CALLS)
1137
		ftrace_replace_code(1);
1138
	else if (*command & FTRACE_DISABLE_CALLS)
1139 1140 1141 1142 1143
		ftrace_replace_code(0);

	if (*command & FTRACE_UPDATE_TRACE_FUNC)
		ftrace_update_ftrace_func(ftrace_trace_function);

1144 1145 1146 1147 1148
	if (*command & FTRACE_START_FUNC_RET)
		ftrace_enable_ftrace_graph_caller();
	else if (*command & FTRACE_STOP_FUNC_RET)
		ftrace_disable_ftrace_graph_caller();

1149
	return 0;
1150 1151
}

I
Ingo Molnar 已提交
1152
static void ftrace_run_update_code(int command)
1153
{
1154 1155 1156 1157 1158 1159 1160
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;

R
Rusty Russell 已提交
1161
	stop_machine(__ftrace_modify_code, &command, NULL);
1162 1163 1164

	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
1165 1166
}

1167
static ftrace_func_t saved_ftrace_func;
1168
static int ftrace_start_up;
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181

static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
1182

1183
static void ftrace_startup(int command)
1184
{
1185 1186 1187
	if (unlikely(ftrace_disabled))
		return;

1188
	ftrace_start_up++;
S
Steven Rostedt 已提交
1189
	command |= FTRACE_ENABLE_CALLS;
1190

1191
	ftrace_startup_enable(command);
1192 1193
}

1194
static void ftrace_shutdown(int command)
1195
{
1196 1197 1198
	if (unlikely(ftrace_disabled))
		return;

1199
	ftrace_start_up--;
1200 1201 1202 1203 1204 1205 1206
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

1207
	if (!ftrace_start_up)
1208
		command |= FTRACE_DISABLE_CALLS;
1209

1210 1211 1212 1213
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
1214

1215
	if (!command || !ftrace_enabled)
S
Steven Rostedt 已提交
1216
		return;
1217 1218

	ftrace_run_update_code(command);
1219 1220
}

I
Ingo Molnar 已提交
1221
static void ftrace_startup_sysctl(void)
1222
{
1223 1224
	int command = FTRACE_ENABLE_MCOUNT;

1225 1226 1227
	if (unlikely(ftrace_disabled))
		return;

1228 1229
	/* Force update next time */
	saved_ftrace_func = NULL;
1230 1231
	/* ftrace_start_up is true if we want ftrace running */
	if (ftrace_start_up)
1232 1233 1234
		command |= FTRACE_ENABLE_CALLS;

	ftrace_run_update_code(command);
1235 1236
}

I
Ingo Molnar 已提交
1237
static void ftrace_shutdown_sysctl(void)
1238
{
1239 1240
	int command = FTRACE_DISABLE_MCOUNT;

1241 1242 1243
	if (unlikely(ftrace_disabled))
		return;

1244 1245
	/* ftrace_start_up is true if ftrace is running */
	if (ftrace_start_up)
1246 1247 1248
		command |= FTRACE_DISABLE_CALLS;

	ftrace_run_update_code(command);
1249 1250
}

1251 1252 1253 1254
static cycle_t		ftrace_update_time;
static unsigned long	ftrace_update_cnt;
unsigned long		ftrace_update_tot_cnt;

1255
static int ftrace_update_code(struct module *mod)
1256
{
1257
	struct dyn_ftrace *p;
1258
	cycle_t start, stop;
1259

I
Ingo Molnar 已提交
1260
	start = ftrace_now(raw_smp_processor_id());
1261 1262
	ftrace_update_cnt = 0;

1263
	while (ftrace_new_addrs) {
1264

S
Steven Rostedt 已提交
1265 1266 1267
		/* If something went wrong, bail without enabling anything */
		if (unlikely(ftrace_disabled))
			return -1;
1268

1269
		p = ftrace_new_addrs;
1270
		ftrace_new_addrs = p->newlist;
1271
		p->flags = 0L;
1272

1273 1274 1275 1276 1277
		/*
		 * Do the initial record convertion from mcount jump
		 * to the NOP instructions.
		 */
		if (!ftrace_code_disable(mod, p)) {
S
Steven Rostedt 已提交
1278
			ftrace_free_rec(p);
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
			continue;
		}

		p->flags |= FTRACE_FL_CONVERTED;
		ftrace_update_cnt++;

		/*
		 * If the tracing is enabled, go ahead and enable the record.
		 *
		 * The reason not to enable the record immediatelly is the
		 * inherent check of ftrace_make_nop/ftrace_make_call for
		 * correct previous instructions.  Making first the NOP
		 * conversion puts the module to the correct state, thus
		 * passing the ftrace_make_call check.
		 */
		if (ftrace_start_up) {
			int failed = __ftrace_replace_code(p, 1);
			if (failed) {
				ftrace_bug(failed, p->ip);
				ftrace_free_rec(p);
			}
		}
1301 1302
	}

I
Ingo Molnar 已提交
1303
	stop = ftrace_now(raw_smp_processor_id());
1304 1305 1306
	ftrace_update_time = stop - start;
	ftrace_update_tot_cnt += ftrace_update_cnt;

1307 1308 1309
	return 0;
}

1310
static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
{
	struct ftrace_page *pg;
	int cnt;
	int i;

	/* allocate a few pages */
	ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
	if (!ftrace_pages_start)
		return -1;

	/*
	 * Allocate a few more pages.
	 *
	 * TODO: have some parser search vmlinux before
	 *   final linking to find all calls to ftrace.
	 *   Then we can:
	 *    a) know how many pages to allocate.
	 *     and/or
	 *    b) set up the table then.
	 *
	 *  The dynamic code is still necessary for
	 *  modules.
	 */

	pg = ftrace_pages = ftrace_pages_start;

1337
	cnt = num_to_init / ENTRIES_PER_PAGE;
S
Steven Rostedt 已提交
1338
	pr_info("ftrace: allocating %ld entries in %d pages\n",
1339
		num_to_init, cnt + 1);
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353

	for (i = 0; i < cnt; i++) {
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);

		/* If we fail, we'll try later anyway */
		if (!pg->next)
			break;

		pg = pg->next;
	}

	return 0;
}

1354 1355
enum {
	FTRACE_ITER_FILTER	= (1 << 0),
1356 1357 1358 1359
	FTRACE_ITER_NOTRACE	= (1 << 1),
	FTRACE_ITER_FAILURES	= (1 << 2),
	FTRACE_ITER_PRINTALL	= (1 << 3),
	FTRACE_ITER_HASH	= (1 << 4),
1360 1361 1362 1363 1364 1365
};

#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
	struct ftrace_page	*pg;
1366
	int			hidx;
1367
	int			idx;
1368
	unsigned		flags;
1369
	struct trace_parser	parser;
1370 1371
};

1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
static void *
t_hash_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	struct hlist_node *hnd = v;
	struct hlist_head *hhd;

	WARN_ON(!(iter->flags & FTRACE_ITER_HASH));

	(*pos)++;

 retry:
	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
		return NULL;

	hhd = &ftrace_func_hash[iter->hidx];

	if (hlist_empty(hhd)) {
		iter->hidx++;
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
			iter->hidx++;
			goto retry;
		}
	}

	return hnd;
}

static void *t_hash_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
1412 1413 1414 1415
	loff_t l;

	if (!(iter->flags & FTRACE_ITER_HASH))
		*pos = 0;
1416 1417 1418

	iter->flags |= FTRACE_ITER_HASH;

L
Li Zefan 已提交
1419 1420 1421 1422 1423 1424 1425
	iter->hidx = 0;
	for (l = 0; l <= *pos; ) {
		p = t_hash_next(m, p, &l);
		if (!p)
			break;
	}
	return p;
1426 1427 1428 1429
}

static int t_hash_show(struct seq_file *m, void *v)
{
S
Steven Rostedt 已提交
1430
	struct ftrace_func_probe *rec;
1431 1432
	struct hlist_node *hnd = v;

S
Steven Rostedt 已提交
1433
	rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1434

1435 1436 1437
	if (rec->ops->print)
		return rec->ops->print(m, rec->ip, rec->ops, rec->data);

1438
	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1439 1440 1441 1442 1443 1444 1445 1446

	if (rec->data)
		seq_printf(m, ":%p", rec->data);
	seq_putc(m, '\n');

	return 0;
}

I
Ingo Molnar 已提交
1447
static void *
1448 1449 1450 1451 1452
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	struct dyn_ftrace *rec = NULL;

1453 1454 1455
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_next(m, v, pos);

1456 1457
	(*pos)++;

1458 1459 1460
	if (iter->flags & FTRACE_ITER_PRINTALL)
		return NULL;

1461 1462 1463 1464 1465 1466 1467 1468 1469
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
1470 1471 1472
		if ((rec->flags & FTRACE_FL_FREE) ||

		    (!(iter->flags & FTRACE_ITER_FAILURES) &&
1473 1474 1475
		     (rec->flags & FTRACE_FL_FAILED)) ||

		    ((iter->flags & FTRACE_ITER_FAILURES) &&
1476
		     !(rec->flags & FTRACE_FL_FAILED)) ||
1477

S
Steven Rostedt 已提交
1478 1479 1480
		    ((iter->flags & FTRACE_ITER_FILTER) &&
		     !(rec->flags & FTRACE_FL_FILTER)) ||

1481 1482
		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
		     !(rec->flags & FTRACE_FL_NOTRACE))) {
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
			rec = NULL;
			goto retry;
		}
	}

	return rec;
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
1495
	loff_t l;
1496

1497
	mutex_lock(&ftrace_lock);
1498 1499 1500 1501 1502 1503 1504
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
	if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
		if (*pos > 0)
1505
			return t_hash_start(m, pos);
1506 1507 1508 1509
		iter->flags |= FTRACE_ITER_PRINTALL;
		return iter;
	}

1510 1511 1512
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_start(m, pos);

1513 1514 1515 1516 1517 1518
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
		p = t_next(m, p, &l);
		if (!p)
			break;
1519
	}
1520

1521
	if (!p && iter->flags & FTRACE_ITER_FILTER)
1522 1523
		return t_hash_start(m, pos);

1524 1525 1526 1527 1528
	return p;
}

static void t_stop(struct seq_file *m, void *p)
{
1529
	mutex_unlock(&ftrace_lock);
1530 1531 1532 1533
}

static int t_show(struct seq_file *m, void *v)
{
1534
	struct ftrace_iterator *iter = m->private;
1535 1536
	struct dyn_ftrace *rec = v;

1537 1538 1539
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_show(m, v);

1540 1541 1542 1543 1544
	if (iter->flags & FTRACE_ITER_PRINTALL) {
		seq_printf(m, "#### all functions enabled ####\n");
		return 0;
	}

1545 1546 1547
	if (!rec)
		return 0;

1548
	seq_printf(m, "%ps\n", (void *)rec->ip);
1549 1550 1551 1552

	return 0;
}

J
James Morris 已提交
1553
static const struct seq_operations show_ftrace_seq_ops = {
1554 1555 1556 1557 1558 1559
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
1560
static int
1561 1562 1563 1564 1565
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;
	int ret;

1566 1567 1568
	if (unlikely(ftrace_disabled))
		return -ENODEV;

1569 1570 1571 1572 1573 1574 1575 1576 1577
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

	iter->pg = ftrace_pages_start;

	ret = seq_open(file, &show_ftrace_seq_ops);
	if (!ret) {
		struct seq_file *m = file->private_data;
I
Ingo Molnar 已提交
1578

1579
		m->private = iter;
I
Ingo Molnar 已提交
1580
	} else {
1581
		kfree(iter);
I
Ingo Molnar 已提交
1582
	}
1583 1584 1585 1586

	return ret;
}

1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604
static int
ftrace_failures_open(struct inode *inode, struct file *file)
{
	int ret;
	struct seq_file *m;
	struct ftrace_iterator *iter;

	ret = ftrace_avail_open(inode, file);
	if (!ret) {
		m = (struct seq_file *)file->private_data;
		iter = (struct ftrace_iterator *)m->private;
		iter->flags = FTRACE_ITER_FAILURES;
	}

	return ret;
}


1605
static void ftrace_filter_reset(int enable)
1606 1607 1608
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1609
	unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1610

1611
	mutex_lock(&ftrace_lock);
1612 1613
	if (enable)
		ftrace_filtered = 0;
1614 1615 1616 1617 1618
	do_for_each_ftrace_rec(pg, rec) {
		if (rec->flags & FTRACE_FL_FAILED)
			continue;
		rec->flags &= ~type;
	} while_for_each_ftrace_rec();
1619
	mutex_unlock(&ftrace_lock);
1620 1621
}

I
Ingo Molnar 已提交
1622
static int
1623
ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1624 1625 1626 1627
{
	struct ftrace_iterator *iter;
	int ret = 0;

1628 1629 1630
	if (unlikely(ftrace_disabled))
		return -ENODEV;

1631 1632 1633 1634
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

1635 1636 1637 1638 1639
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

1640
	mutex_lock(&ftrace_regex_lock);
1641
	if ((file->f_mode & FMODE_WRITE) &&
1642
	    (file->f_flags & O_TRUNC))
1643
		ftrace_filter_reset(enable);
1644 1645 1646

	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;
1647 1648
		iter->flags = enable ? FTRACE_ITER_FILTER :
			FTRACE_ITER_NOTRACE;
1649 1650 1651 1652 1653

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
1654 1655
		} else {
			trace_parser_put(&iter->parser);
1656
			kfree(iter);
1657
		}
1658 1659
	} else
		file->private_data = iter;
1660
	mutex_unlock(&ftrace_regex_lock);
1661 1662 1663 1664

	return ret;
}

1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
	return ftrace_regex_open(inode, file, 1);
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
	return ftrace_regex_open(inode, file, 0);
}

I
Ingo Molnar 已提交
1677
static loff_t
1678
ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
{
	loff_t ret;

	if (file->f_mode & FMODE_READ)
		ret = seq_lseek(file, offset, origin);
	else
		file->f_pos = ret = 1;

	return ret;
}

1690
static int ftrace_match(char *str, char *regex, int len, int type)
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
{
	int matched = 0;
	char *ptr;

	switch (type) {
	case MATCH_FULL:
		if (strcmp(str, regex) == 0)
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
		if (strncmp(str, regex, len) == 0)
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
		if (strstr(str, regex))
			matched = 1;
		break;
	case MATCH_END_ONLY:
		ptr = strstr(str, regex);
		if (ptr && (ptr[len] == 0))
			matched = 1;
		break;
	}

	return matched;
}

1718 1719 1720 1721 1722 1723 1724 1725 1726
static int
ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
{
	char str[KSYM_SYMBOL_LEN];

	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
	return ftrace_match(str, regex, len, type);
}

1727 1728
static void ftrace_match_records(char *buff, int len, int enable)
{
S
Steven Rostedt 已提交
1729
	unsigned int search_len;
1730 1731
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
S
Steven Rostedt 已提交
1732 1733
	unsigned long flag;
	char *search;
1734 1735 1736
	int type;
	int not;

S
Steven Rostedt 已提交
1737
	flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1738
	type = filter_parse_regex(buff, len, &search, &not);
1739 1740 1741

	search_len = strlen(search);

1742
	mutex_lock(&ftrace_lock);
1743 1744 1745 1746
	do_for_each_ftrace_rec(pg, rec) {

		if (rec->flags & FTRACE_FL_FAILED)
			continue;
1747 1748

		if (ftrace_match_record(rec, search, search_len, type)) {
1749 1750 1751 1752 1753
			if (not)
				rec->flags &= ~flag;
			else
				rec->flags |= flag;
		}
1754 1755 1756 1757 1758 1759
		/*
		 * Only enable filtering if we have a function that
		 * is filtered on.
		 */
		if (enable && (rec->flags & FTRACE_FL_FILTER))
			ftrace_filtered = 1;
1760
	} while_for_each_ftrace_rec();
1761
	mutex_unlock(&ftrace_lock);
1762 1763
}

1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784
static int
ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
			   char *regex, int len, int type)
{
	char str[KSYM_SYMBOL_LEN];
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

	if (!modname || strcmp(modname, mod))
		return 0;

	/* blank search means to match all funcs in the mod */
	if (len)
		return ftrace_match(str, regex, len, type);
	else
		return 1;
}

static void ftrace_match_module_records(char *buff, char *mod, int enable)
{
S
Steven Rostedt 已提交
1785
	unsigned search_len = 0;
1786 1787 1788
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	int type = MATCH_FULL;
S
Steven Rostedt 已提交
1789 1790
	char *search = buff;
	unsigned long flag;
1791 1792
	int not = 0;

S
Steven Rostedt 已提交
1793 1794
	flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;

1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
	/* blank or '*' mean the same */
	if (strcmp(buff, "*") == 0)
		buff[0] = 0;

	/* handle the case of 'dont filter this module' */
	if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
		buff[0] = 0;
		not = 1;
	}

	if (strlen(buff)) {
1806
		type = filter_parse_regex(buff, strlen(buff), &search, &not);
1807 1808 1809
		search_len = strlen(search);
	}

1810
	mutex_lock(&ftrace_lock);
1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822
	do_for_each_ftrace_rec(pg, rec) {

		if (rec->flags & FTRACE_FL_FAILED)
			continue;

		if (ftrace_match_module_record(rec, mod,
					       search, search_len, type)) {
			if (not)
				rec->flags &= ~flag;
			else
				rec->flags |= flag;
		}
1823 1824
		if (enable && (rec->flags & FTRACE_FL_FILTER))
			ftrace_filtered = 1;
1825 1826

	} while_for_each_ftrace_rec();
1827
	mutex_unlock(&ftrace_lock);
1828 1829
}

1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
{
	char *mod;

	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */

	/* we must have a module name */
	if (!param)
		return -EINVAL;

	mod = strsep(&param, ":");
	if (!strlen(mod))
		return -EINVAL;

	ftrace_match_module_records(func, mod, enable);
	return 0;
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
device_initcall(ftrace_mod_cmd_init);

1871
static void
S
Steven Rostedt 已提交
1872
function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1873
{
S
Steven Rostedt 已提交
1874
	struct ftrace_func_probe *entry;
1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
	struct hlist_head *hhd;
	struct hlist_node *n;
	unsigned long key;
	int resched;

	key = hash_long(ip, FTRACE_HASH_BITS);

	hhd = &ftrace_func_hash[key];

	if (hlist_empty(hhd))
		return;

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
	resched = ftrace_preempt_disable();
	hlist_for_each_entry_rcu(entry, n, hhd, node) {
		if (entry->ip == ip)
			entry->ops->func(ip, parent_ip, &entry->data);
	}
	ftrace_preempt_enable(resched);
}

S
Steven Rostedt 已提交
1900
static struct ftrace_ops trace_probe_ops __read_mostly =
1901
{
1902
	.func		= function_trace_probe_call,
1903 1904
};

S
Steven Rostedt 已提交
1905
static int ftrace_probe_registered;
1906

S
Steven Rostedt 已提交
1907
static void __enable_ftrace_function_probe(void)
1908 1909 1910
{
	int i;

S
Steven Rostedt 已提交
1911
	if (ftrace_probe_registered)
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			break;
	}
	/* Nothing registered? */
	if (i == FTRACE_FUNC_HASHSIZE)
		return;

S
Steven Rostedt 已提交
1923
	__register_ftrace_function(&trace_probe_ops);
1924
	ftrace_startup(0);
S
Steven Rostedt 已提交
1925
	ftrace_probe_registered = 1;
1926 1927
}

S
Steven Rostedt 已提交
1928
static void __disable_ftrace_function_probe(void)
1929 1930 1931
{
	int i;

S
Steven Rostedt 已提交
1932
	if (!ftrace_probe_registered)
1933 1934 1935 1936 1937 1938 1939 1940 1941
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			return;
	}

	/* no more funcs left */
S
Steven Rostedt 已提交
1942
	__unregister_ftrace_function(&trace_probe_ops);
1943
	ftrace_shutdown(0);
S
Steven Rostedt 已提交
1944
	ftrace_probe_registered = 0;
1945 1946 1947 1948 1949
}


static void ftrace_free_entry_rcu(struct rcu_head *rhp)
{
S
Steven Rostedt 已提交
1950 1951
	struct ftrace_func_probe *entry =
		container_of(rhp, struct ftrace_func_probe, rcu);
1952 1953 1954 1955 1956 1957 1958 1959

	if (entry->ops->free)
		entry->ops->free(&entry->data);
	kfree(entry);
}


int
S
Steven Rostedt 已提交
1960
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1961 1962
			      void *data)
{
S
Steven Rostedt 已提交
1963
	struct ftrace_func_probe *entry;
1964 1965 1966
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	int type, len, not;
S
Steven Rostedt 已提交
1967
	unsigned long key;
1968 1969 1970
	int count = 0;
	char *search;

1971
	type = filter_parse_regex(glob, strlen(glob), &search, &not);
1972 1973
	len = strlen(search);

S
Steven Rostedt 已提交
1974
	/* we do not support '!' for function probes */
1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
	if (WARN_ON(not))
		return -EINVAL;

	mutex_lock(&ftrace_lock);
	do_for_each_ftrace_rec(pg, rec) {

		if (rec->flags & FTRACE_FL_FAILED)
			continue;

		if (!ftrace_match_record(rec, search, len, type))
			continue;

		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
		if (!entry) {
S
Steven Rostedt 已提交
1989
			/* If we did not process any, then return error */
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018
			if (!count)
				count = -ENOMEM;
			goto out_unlock;
		}

		count++;

		entry->data = data;

		/*
		 * The caller might want to do something special
		 * for each function we find. We call the callback
		 * to give the caller an opportunity to do so.
		 */
		if (ops->callback) {
			if (ops->callback(rec->ip, &entry->data) < 0) {
				/* caller does not like this func */
				kfree(entry);
				continue;
			}
		}

		entry->ops = ops;
		entry->ip = rec->ip;

		key = hash_long(entry->ip, FTRACE_HASH_BITS);
		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);

	} while_for_each_ftrace_rec();
S
Steven Rostedt 已提交
2019
	__enable_ftrace_function_probe();
2020 2021 2022 2023 2024 2025 2026 2027

 out_unlock:
	mutex_unlock(&ftrace_lock);

	return count;
}

enum {
S
Steven Rostedt 已提交
2028 2029
	PROBE_TEST_FUNC		= 1,
	PROBE_TEST_DATA		= 2
2030 2031 2032
};

static void
S
Steven Rostedt 已提交
2033
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2034 2035
				  void *data, int flags)
{
S
Steven Rostedt 已提交
2036
	struct ftrace_func_probe *entry;
2037 2038 2039 2040 2041 2042
	struct hlist_node *n, *tmp;
	char str[KSYM_SYMBOL_LEN];
	int type = MATCH_FULL;
	int i, len = 0;
	char *search;

2043
	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2044
		glob = NULL;
2045
	else if (glob) {
2046 2047
		int not;

2048
		type = filter_parse_regex(glob, strlen(glob), &search, &not);
2049 2050
		len = strlen(search);

S
Steven Rostedt 已提交
2051
		/* we do not support '!' for function probes */
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062
		if (WARN_ON(not))
			return;
	}

	mutex_lock(&ftrace_lock);
	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];

		hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {

			/* break up if statements for readability */
S
Steven Rostedt 已提交
2063
			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2064 2065
				continue;

S
Steven Rostedt 已提交
2066
			if ((flags & PROBE_TEST_DATA) && entry->data != data)
2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080
				continue;

			/* do this last, since it is the most expensive */
			if (glob) {
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
				if (!ftrace_match(str, glob, len, type))
					continue;
			}

			hlist_del(&entry->node);
			call_rcu(&entry->rcu, ftrace_free_entry_rcu);
		}
	}
S
Steven Rostedt 已提交
2081
	__disable_ftrace_function_probe();
2082 2083 2084 2085
	mutex_unlock(&ftrace_lock);
}

void
S
Steven Rostedt 已提交
2086
unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2087 2088
				void *data)
{
S
Steven Rostedt 已提交
2089 2090
	__unregister_ftrace_function_probe(glob, ops, data,
					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
2091 2092 2093
}

void
S
Steven Rostedt 已提交
2094
unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2095
{
S
Steven Rostedt 已提交
2096
	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2097 2098
}

S
Steven Rostedt 已提交
2099
void unregister_ftrace_function_probe_all(char *glob)
2100
{
S
Steven Rostedt 已提交
2101
	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2102 2103
}

2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

int register_ftrace_command(struct ftrace_func_command *cmd)
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

int unregister_ftrace_command(struct ftrace_func_command *cmd)
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

2145 2146
static int ftrace_process_regex(char *buff, int len, int enable)
{
2147
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
2148
	struct ftrace_func_command *p;
2149
	int ret = -EINVAL;
2150 2151 2152 2153 2154 2155 2156 2157

	func = strsep(&next, ":");

	if (!next) {
		ftrace_match_records(func, len, enable);
		return 0;
	}

2158
	/* command found */
2159 2160 2161

	command = strsep(&next, ":");

2162 2163 2164 2165 2166 2167
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
			ret = p->func(func, command, next, enable);
			goto out_unlock;
		}
2168
	}
2169 2170
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
2171

2172
	return ret;
2173 2174
}

I
Ingo Molnar 已提交
2175
static ssize_t
2176 2177
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
2178 2179
{
	struct ftrace_iterator *iter;
2180 2181
	struct trace_parser *parser;
	ssize_t ret, read;
2182

2183
	if (!cnt)
2184 2185
		return 0;

2186
	mutex_lock(&ftrace_regex_lock);
2187 2188 2189 2190 2191 2192 2193

	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

2194 2195
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
2196

2197
	if (read >= 0 && trace_parser_loaded(parser) &&
2198 2199 2200
	    !trace_parser_cont(parser)) {
		ret = ftrace_process_regex(parser->buffer,
					   parser->idx, enable);
2201
		if (ret)
2202
			goto out_unlock;
2203

2204
		trace_parser_clear(parser);
2205
	}
2206 2207

	ret = read;
2208
out_unlock:
2209
	mutex_unlock(&ftrace_regex_lock);
2210

2211 2212 2213
	return ret;
}

2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
static ssize_t
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

static ssize_t
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

static void
ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
{
	if (unlikely(ftrace_disabled))
		return;

	mutex_lock(&ftrace_regex_lock);
	if (reset)
		ftrace_filter_reset(enable);
	if (buf)
2238
		ftrace_match_records(buf, len, enable);
2239 2240 2241
	mutex_unlock(&ftrace_regex_lock);
}

2242 2243 2244 2245 2246 2247 2248 2249 2250
/**
 * ftrace_set_filter - set a function to filter on in ftrace
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
I
Ingo Molnar 已提交
2251
void ftrace_set_filter(unsigned char *buf, int len, int reset)
2252
{
2253 2254
	ftrace_set_regex(buf, len, reset, 1);
}
2255

2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268
/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
void ftrace_set_notrace(unsigned char *buf, int len, int reset)
{
	ftrace_set_regex(buf, len, reset, 0);
2269 2270
}

2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

static int __init set_ftrace_notrace(char *str)
{
	strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
	strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

2292
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2293
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2294 2295
static int __init set_graph_function(char *str)
{
2296
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

static void __init set_ftrace_early_graph(char *buf)
{
	int ret;
	char *func;

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
				      func);
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
static void __init set_ftrace_early_filter(char *buf, int enable)
{
	char *func;

	while (buf) {
		func = strsep(&buf, ",");
		ftrace_set_regex(func, strlen(func), 0, enable);
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
		set_ftrace_early_filter(ftrace_filter_buf, 1);
	if (ftrace_notrace_buf[0])
		set_ftrace_early_filter(ftrace_notrace_buf, 0);
2334 2335 2336 2337
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
		set_ftrace_early_graph(ftrace_graph_buf);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2338 2339
}

I
Ingo Molnar 已提交
2340
static int
2341
ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2342 2343 2344
{
	struct seq_file *m = (struct seq_file *)file->private_data;
	struct ftrace_iterator *iter;
2345
	struct trace_parser *parser;
2346

2347
	mutex_lock(&ftrace_regex_lock);
2348 2349 2350 2351 2352 2353 2354
	if (file->f_mode & FMODE_READ) {
		iter = m->private;

		seq_release(inode, file);
	} else
		iter = file->private_data;

2355 2356 2357 2358
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
		ftrace_match_records(parser->buffer, parser->idx, enable);
2359 2360
	}

S
Steven Rostedt 已提交
2361
	mutex_lock(&ftrace_lock);
2362
	if (ftrace_start_up && ftrace_enabled)
2363
		ftrace_run_update_code(FTRACE_ENABLE_CALLS);
S
Steven Rostedt 已提交
2364
	mutex_unlock(&ftrace_lock);
2365

2366
	trace_parser_put(parser);
2367
	kfree(iter);
2368

2369
	mutex_unlock(&ftrace_regex_lock);
2370 2371 2372
	return 0;
}

2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384
static int
ftrace_filter_release(struct inode *inode, struct file *file)
{
	return ftrace_regex_release(inode, file, 1);
}

static int
ftrace_notrace_release(struct inode *inode, struct file *file)
{
	return ftrace_regex_release(inode, file, 0);
}

2385
static const struct file_operations ftrace_avail_fops = {
2386 2387 2388
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
2389
	.release = seq_release_private,
2390 2391
};

2392
static const struct file_operations ftrace_failures_fops = {
2393 2394 2395
	.open = ftrace_failures_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
2396
	.release = seq_release_private,
2397 2398
};

2399
static const struct file_operations ftrace_filter_fops = {
2400
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
2401
	.read = seq_read,
2402
	.write = ftrace_filter_write,
2403
	.llseek = ftrace_regex_lseek,
2404 2405 2406
	.release = ftrace_filter_release,
};

2407
static const struct file_operations ftrace_notrace_fops = {
2408
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
2409
	.read = seq_read,
2410 2411 2412 2413 2414
	.write = ftrace_notrace_write,
	.llseek = ftrace_regex_lseek,
	.release = ftrace_notrace_release,
};

2415 2416 2417 2418 2419 2420 2421 2422
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

int ftrace_graph_count;
unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;

static void *
2423
__g_next(struct seq_file *m, loff_t *pos)
2424
{
2425
	if (*pos >= ftrace_graph_count)
2426
		return NULL;
2427
	return &ftrace_graph_funcs[*pos];
2428
}
2429

2430 2431 2432 2433 2434
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
2435 2436 2437 2438 2439 2440
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&graph_lock);

2441 2442 2443 2444
	/* Nothing, tell g_show to print all functions are enabled */
	if (!ftrace_graph_count && !*pos)
		return (void *)1;

2445
	return __g_next(m, pos);
2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
	unsigned long *ptr = v;

	if (!ptr)
		return 0;

2460 2461 2462 2463 2464
	if (ptr == (unsigned long *)1) {
		seq_printf(m, "#### all functions enabled ####\n");
		return 0;
	}

2465
	seq_printf(m, "%ps\n", (void *)*ptr);
2466 2467 2468 2469

	return 0;
}

J
James Morris 已提交
2470
static const struct seq_operations ftrace_graph_seq_ops = {
2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	int ret = 0;

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	mutex_lock(&graph_lock);
	if ((file->f_mode & FMODE_WRITE) &&
2487
	    (file->f_flags & O_TRUNC)) {
2488 2489 2490
		ftrace_graph_count = 0;
		memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
	}
2491
	mutex_unlock(&graph_lock);
2492

2493
	if (file->f_mode & FMODE_READ)
2494 2495 2496 2497 2498
		ret = seq_open(file, &ftrace_graph_seq_ops);

	return ret;
}

2499 2500 2501 2502 2503 2504 2505 2506
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
	if (file->f_mode & FMODE_READ)
		seq_release(inode, file);
	return 0;
}

2507
static int
2508
ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2509 2510 2511
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
2512
	int search_len;
2513
	int found = 0;
2514 2515 2516 2517
	int type, not;
	char *search;
	bool exists;
	int i;
2518 2519 2520 2521

	if (ftrace_disabled)
		return -ENODEV;

2522
	/* decode regex */
2523
	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2524 2525 2526 2527 2528
	if (not)
		return -EINVAL;

	search_len = strlen(search);

2529
	mutex_lock(&ftrace_lock);
2530 2531
	do_for_each_ftrace_rec(pg, rec) {

2532 2533 2534
		if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
			break;

2535 2536 2537
		if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
			continue;

2538 2539 2540 2541 2542 2543
		if (ftrace_match_record(rec, search, search_len, type)) {
			/* ensure it is not already in the array */
			exists = false;
			for (i = 0; i < *idx; i++)
				if (array[i] == rec->ip) {
					exists = true;
2544 2545
					break;
				}
2546 2547 2548 2549
			if (!exists) {
				array[(*idx)++] = rec->ip;
				found = 1;
			}
2550
		}
2551
	} while_for_each_ftrace_rec();
2552

2553
	mutex_unlock(&ftrace_lock);
2554 2555 2556 2557 2558 2559 2560 2561

	return found ? 0 : -EINVAL;
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
2562
	struct trace_parser parser;
2563
	ssize_t read, ret;
2564 2565 2566 2567 2568 2569 2570 2571

	if (!cnt || cnt < 0)
		return 0;

	mutex_lock(&graph_lock);

	if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
		ret = -EBUSY;
2572
		goto out_unlock;
2573 2574
	}

2575 2576
	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
		ret = -ENOMEM;
2577
		goto out_unlock;
2578 2579
	}

2580
	read = trace_get_user(&parser, ubuf, cnt, ppos);
2581

2582
	if (read >= 0 && trace_parser_loaded((&parser))) {
2583 2584 2585
		parser.buffer[parser.idx] = 0;

		/* we allow only one expression at a time */
2586
		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2587
					parser.buffer);
2588
		if (ret)
2589
			goto out_free;
2590 2591 2592
	}

	ret = read;
2593 2594

out_free:
2595
	trace_parser_put(&parser);
2596
out_unlock:
2597 2598 2599 2600 2601 2602
	mutex_unlock(&graph_lock);

	return ret;
}

static const struct file_operations ftrace_graph_fops = {
2603 2604 2605 2606
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
	.release	= ftrace_graph_release,
2607 2608 2609
};
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

2610
static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2611 2612
{

2613 2614
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
2615

2616 2617
	trace_create_file("failures", 0444,
			d_tracer, NULL, &ftrace_failures_fops);
2618

2619 2620
	trace_create_file("set_ftrace_filter", 0644, d_tracer,
			NULL, &ftrace_filter_fops);
2621

2622
	trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2623
				    NULL, &ftrace_notrace_fops);
2624

2625
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2626
	trace_create_file("set_graph_function", 0444, d_tracer,
2627 2628 2629 2630
				    NULL,
				    &ftrace_graph_fops);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

2631 2632 2633
	return 0;
}

2634
static int ftrace_process_locs(struct module *mod,
2635
			       unsigned long *start,
2636 2637 2638 2639 2640 2641
			       unsigned long *end)
{
	unsigned long *p;
	unsigned long addr;
	unsigned long flags;

S
Steven Rostedt 已提交
2642
	mutex_lock(&ftrace_lock);
2643 2644 2645
	p = start;
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
2646 2647 2648 2649 2650 2651 2652 2653
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
2654 2655 2656
		ftrace_record_ip(addr);
	}

S
Steven Rostedt 已提交
2657
	/* disable interrupts to prevent kstop machine */
2658
	local_irq_save(flags);
2659
	ftrace_update_code(mod);
2660
	local_irq_restore(flags);
S
Steven Rostedt 已提交
2661
	mutex_unlock(&ftrace_lock);
2662 2663 2664 2665

	return 0;
}

2666
#ifdef CONFIG_MODULES
2667
void ftrace_release_mod(struct module *mod)
2668 2669 2670 2671
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

2672
	if (ftrace_disabled)
2673 2674 2675 2676
		return;

	mutex_lock(&ftrace_lock);
	do_for_each_ftrace_rec(pg, rec) {
2677
		if (within_module_core(rec->ip, mod)) {
2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690
			/*
			 * rec->ip is changed in ftrace_free_rec()
			 * It should not between s and e if record was freed.
			 */
			FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
			ftrace_free_rec(rec);
		}
	} while_for_each_ftrace_rec();
	mutex_unlock(&ftrace_lock);
}

static void ftrace_init_module(struct module *mod,
			       unsigned long *start, unsigned long *end)
2691
{
2692
	if (ftrace_disabled || start == end)
2693
		return;
2694
	ftrace_process_locs(mod, start, end);
2695 2696
}

2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708
static int ftrace_module_notify(struct notifier_block *self,
				unsigned long val, void *data)
{
	struct module *mod = data;

	switch (val) {
	case MODULE_STATE_COMING:
		ftrace_init_module(mod, mod->ftrace_callsites,
				   mod->ftrace_callsites +
				   mod->num_ftrace_callsites);
		break;
	case MODULE_STATE_GOING:
2709
		ftrace_release_mod(mod);
2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727
		break;
	}

	return 0;
}
#else
static int ftrace_module_notify(struct notifier_block *self,
				unsigned long val, void *data)
{
	return 0;
}
#endif /* CONFIG_MODULES */

struct notifier_block ftrace_module_nb = {
	.notifier_call = ftrace_module_notify,
	.priority = 0,
};

2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754
extern unsigned long __start_mcount_loc[];
extern unsigned long __stop_mcount_loc[];

void __init ftrace_init(void)
{
	unsigned long count, addr, flags;
	int ret;

	/* Keep the ftrace pointer to the stub */
	addr = (unsigned long)ftrace_stub;

	local_irq_save(flags);
	ftrace_dyn_arch_init(&addr);
	local_irq_restore(flags);

	/* ftrace_dyn_arch_init places the return code in addr */
	if (addr)
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;

	ret = ftrace_dyn_table_alloc(count);
	if (ret)
		goto failed;

	last_ftrace_enabled = ftrace_enabled = 1;

2755
	ret = ftrace_process_locs(NULL,
2756
				  __start_mcount_loc,
2757 2758
				  __stop_mcount_loc);

2759
	ret = register_module_notifier(&ftrace_module_nb);
2760
	if (ret)
2761 2762
		pr_warning("Failed to register trace ftrace module notifier\n");

2763 2764
	set_ftrace_early_filters();

2765 2766 2767 2768 2769
	return;
 failed:
	ftrace_disabled = 1;
}

2770
#else
2771 2772 2773 2774 2775 2776 2777 2778

static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
device_initcall(ftrace_nodyn_init);

2779 2780
static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
static inline void ftrace_startup_enable(int command) { }
2781 2782 2783
/* Keep as macros so we do not need to define the commands */
# define ftrace_startup(command)	do { } while (0)
# define ftrace_shutdown(command)	do { } while (0)
I
Ingo Molnar 已提交
2784 2785
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
2786 2787
#endif /* CONFIG_DYNAMIC_FTRACE */

2788
static void clear_ftrace_swapper(void)
S
Steven Rostedt 已提交
2789 2790
{
	struct task_struct *p;
2791
	int cpu;
S
Steven Rostedt 已提交
2792

2793 2794 2795
	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
S
Steven Rostedt 已提交
2796
		clear_tsk_trace_trace(p);
2797 2798 2799
	}
	put_online_cpus();
}
S
Steven Rostedt 已提交
2800

2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
static void set_ftrace_swapper(void)
{
	struct task_struct *p;
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
		set_tsk_trace_trace(p);
	}
	put_online_cpus();
S
Steven Rostedt 已提交
2812 2813
}

2814 2815 2816 2817
static void clear_ftrace_pid(struct pid *pid)
{
	struct task_struct *p;

2818
	rcu_read_lock();
2819 2820 2821
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		clear_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
2822 2823
	rcu_read_unlock();

2824 2825 2826 2827
	put_pid(pid);
}

static void set_ftrace_pid(struct pid *pid)
S
Steven Rostedt 已提交
2828 2829 2830
{
	struct task_struct *p;

2831
	rcu_read_lock();
S
Steven Rostedt 已提交
2832 2833 2834
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		set_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
2835
	rcu_read_unlock();
S
Steven Rostedt 已提交
2836 2837
}

2838
static void clear_ftrace_pid_task(struct pid *pid)
2839
{
2840
	if (pid == ftrace_swapper_pid)
2841 2842
		clear_ftrace_swapper();
	else
2843
		clear_ftrace_pid(pid);
2844 2845 2846 2847 2848 2849 2850 2851 2852 2853
}

static void set_ftrace_pid_task(struct pid *pid)
{
	if (pid == ftrace_swapper_pid)
		set_ftrace_swapper();
	else
		set_ftrace_pid(pid);
}

2854
static int ftrace_pid_add(int p)
2855
{
S
Steven Rostedt 已提交
2856
	struct pid *pid;
2857 2858
	struct ftrace_pid *fpid;
	int ret = -EINVAL;
2859

2860
	mutex_lock(&ftrace_lock);
2861

2862 2863 2864 2865
	if (!p)
		pid = ftrace_swapper_pid;
	else
		pid = find_get_pid(p);
2866

2867 2868
	if (!pid)
		goto out;
2869

2870
	ret = 0;
2871

2872 2873 2874
	list_for_each_entry(fpid, &ftrace_pids, list)
		if (fpid->pid == pid)
			goto out_put;
S
Steven Rostedt 已提交
2875

2876
	ret = -ENOMEM;
2877

2878 2879 2880
	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
	if (!fpid)
		goto out_put;
2881

2882 2883
	list_add(&fpid->list, &ftrace_pids);
	fpid->pid = pid;
2884

2885
	set_ftrace_pid_task(pid);
S
Steven Rostedt 已提交
2886

2887 2888 2889 2890 2891 2892 2893 2894 2895
	ftrace_update_pid_func();
	ftrace_startup_enable(0);

	mutex_unlock(&ftrace_lock);
	return 0;

out_put:
	if (pid != ftrace_swapper_pid)
		put_pid(pid);
S
Steven Rostedt 已提交
2896

2897 2898 2899 2900 2901 2902 2903 2904
out:
	mutex_unlock(&ftrace_lock);
	return ret;
}

static void ftrace_pid_reset(void)
{
	struct ftrace_pid *fpid, *safe;
S
Steven Rostedt 已提交
2905

2906 2907 2908 2909 2910 2911 2912 2913
	mutex_lock(&ftrace_lock);
	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
		struct pid *pid = fpid->pid;

		clear_ftrace_pid_task(pid);

		list_del(&fpid->list);
		kfree(fpid);
2914 2915 2916 2917 2918
	}

	ftrace_update_pid_func();
	ftrace_startup_enable(0);

S
Steven Rostedt 已提交
2919
	mutex_unlock(&ftrace_lock);
2920
}
2921

2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983
static void *fpid_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&ftrace_lock);

	if (list_empty(&ftrace_pids) && (!*pos))
		return (void *) 1;

	return seq_list_start(&ftrace_pids, *pos);
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
	if (v == (void *)1)
		return NULL;

	return seq_list_next(v, &ftrace_pids, pos);
}

static void fpid_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);

	if (v == (void *)1) {
		seq_printf(m, "no pid\n");
		return 0;
	}

	if (fpid->pid == ftrace_swapper_pid)
		seq_printf(m, "swapper tasks\n");
	else
		seq_printf(m, "%u\n", pid_vnr(fpid->pid));

	return 0;
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
	int ret = 0;

	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
		ftrace_pid_reset();

	if (file->f_mode & FMODE_READ)
		ret = seq_open(file, &ftrace_pid_sops);

	return ret;
}

2984 2985 2986 2987
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
2988
	char buf[64], *tmp;
2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999
	long val;
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

3000 3001 3002 3003
	/*
	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
	 * to clean the filter quietly.
	 */
3004 3005
	tmp = strstrip(buf);
	if (strlen(tmp) == 0)
3006 3007
		return 1;

3008
	ret = strict_strtol(tmp, 10, &val);
3009 3010 3011
	if (ret < 0)
		return ret;

3012
	ret = ftrace_pid_add(val);
3013

3014 3015
	return ret ? ret : cnt;
}
3016

3017 3018 3019 3020 3021
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
	if (file->f_mode & FMODE_READ)
		seq_release(inode, file);
3022

3023
	return 0;
3024 3025
}

3026
static const struct file_operations ftrace_pid_fops = {
3027 3028 3029 3030 3031
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= ftrace_pid_release,
3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043
};

static __init int ftrace_init_debugfs(void)
{
	struct dentry *d_tracer;

	d_tracer = tracing_init_dentry();
	if (!d_tracer)
		return 0;

	ftrace_init_dyn_debugfs(d_tracer);

3044 3045
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
			    NULL, &ftrace_pid_fops);
3046 3047 3048

	ftrace_profile_debugfs(d_tracer);

3049 3050 3051 3052
	return 0;
}
fs_initcall(ftrace_init_debugfs);

S
Steven Rostedt 已提交
3053
/**
3054
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
3055 3056 3057 3058 3059
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
3060
void ftrace_kill(void)
S
Steven Rostedt 已提交
3061 3062 3063 3064 3065 3066
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

3067
/**
3068 3069
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
3070
 *
3071 3072 3073 3074 3075 3076
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
3077
 */
3078
int register_ftrace_function(struct ftrace_ops *ops)
3079
{
3080 3081
	int ret;

3082 3083 3084
	if (unlikely(ftrace_disabled))
		return -1;

S
Steven Rostedt 已提交
3085
	mutex_lock(&ftrace_lock);
3086

3087
	ret = __register_ftrace_function(ops);
3088
	ftrace_startup(0);
3089

S
Steven Rostedt 已提交
3090
	mutex_unlock(&ftrace_lock);
3091
	return ret;
3092 3093 3094
}

/**
3095
 * unregister_ftrace_function - unregister a function for profiling.
3096 3097 3098 3099 3100 3101 3102 3103
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
3104
	mutex_lock(&ftrace_lock);
3105
	ret = __unregister_ftrace_function(ops);
3106
	ftrace_shutdown(0);
S
Steven Rostedt 已提交
3107
	mutex_unlock(&ftrace_lock);
3108 3109 3110 3111

	return ret;
}

I
Ingo Molnar 已提交
3112
int
3113
ftrace_enable_sysctl(struct ctl_table *table, int write,
3114
		     void __user *buffer, size_t *lenp,
3115 3116 3117 3118
		     loff_t *ppos)
{
	int ret;

3119 3120 3121
	if (unlikely(ftrace_disabled))
		return -ENODEV;

S
Steven Rostedt 已提交
3122
	mutex_lock(&ftrace_lock);
3123

3124
	ret  = proc_dointvec(table, write, buffer, lenp, ppos);
3125

3126
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3127 3128
		goto out;

3129
	last_ftrace_enabled = !!ftrace_enabled;
3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150

	if (ftrace_enabled) {

		ftrace_startup_sysctl();

		/* we are starting ftrace again */
		if (ftrace_list != &ftrace_list_end) {
			if (ftrace_list->next == &ftrace_list_end)
				ftrace_trace_function = ftrace_list->func;
			else
				ftrace_trace_function = ftrace_list_func;
		}

	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
3151
	mutex_unlock(&ftrace_lock);
3152
	return ret;
3153
}
3154

3155
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3156

3157
static int ftrace_graph_active;
3158
static struct notifier_block ftrace_suspend_notifier;
3159

3160 3161 3162 3163 3164
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

3165 3166 3167
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
3168
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	unsigned long flags;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

	read_lock_irqsave(&tasklist_lock, flags);
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
3199
			atomic_set(&t->tracing_graph_pause, 0);
3200
			atomic_set(&t->trace_overrun, 0);
3201 3202 3203 3204
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215
		}
	} while_each_thread(g, t);

unlock:
	read_unlock_irqrestore(&tasklist_lock, flags);
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

3216 3217 3218 3219 3220 3221 3222
static void
ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
				struct task_struct *next)
{
	unsigned long long timestamp;
	int index;

3223 3224 3225 3226 3227 3228 3229
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
	if (trace_flags & TRACE_ITER_SLEEP_TIME)
		return;

3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

3248
/* Allocate a return stack for each task */
3249
static int start_graph_tracing(void)
3250 3251
{
	struct ftrace_ret_stack **ret_stack_list;
3252
	int ret, cpu;
3253 3254 3255 3256 3257 3258 3259 3260

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

3261
	/* The cpu_boot init_task->ret_stack will never be freed */
3262 3263 3264 3265
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
			ftrace_graph_init_task(idle_task(cpu));
	}
3266

3267 3268 3269 3270
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

3271 3272 3273 3274 3275 3276 3277
	if (!ret) {
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

3278 3279 3280 3281
	kfree(ret_stack_list);
	return ret;
}

3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

3303 3304
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
3305
{
3306 3307
	int ret = 0;

S
Steven Rostedt 已提交
3308
	mutex_lock(&ftrace_lock);
3309

3310
	/* we currently allow only one tracer registered at a time */
3311
	if (ftrace_graph_active) {
3312 3313 3314 3315
		ret = -EBUSY;
		goto out;
	}

3316 3317 3318
	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
	register_pm_notifier(&ftrace_suspend_notifier);

3319
	ftrace_graph_active++;
3320
	ret = start_graph_tracing();
3321
	if (ret) {
3322
		ftrace_graph_active--;
3323 3324
		goto out;
	}
3325

3326 3327
	ftrace_graph_return = retfunc;
	ftrace_graph_entry = entryfunc;
3328

3329
	ftrace_startup(FTRACE_START_FUNC_RET);
3330 3331

out:
S
Steven Rostedt 已提交
3332
	mutex_unlock(&ftrace_lock);
3333
	return ret;
3334 3335
}

3336
void unregister_ftrace_graph(void)
3337
{
S
Steven Rostedt 已提交
3338
	mutex_lock(&ftrace_lock);
3339

3340
	if (unlikely(!ftrace_graph_active))
3341 3342
		goto out;

3343
	ftrace_graph_active--;
3344
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3345
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3346
	ftrace_graph_entry = ftrace_graph_entry_stub;
3347
	ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3348
	unregister_pm_notifier(&ftrace_suspend_notifier);
3349

3350
 out:
S
Steven Rostedt 已提交
3351
	mutex_unlock(&ftrace_lock);
3352
}
3353 3354

/* Allocate a return stack for newly created task */
3355
void ftrace_graph_init_task(struct task_struct *t)
3356
{
3357 3358 3359
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;

3360
	if (ftrace_graph_active) {
3361 3362 3363
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3364 3365
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
3366
		if (!ret_stack)
3367 3368
			return;
		t->curr_ret_stack = -1;
3369
		atomic_set(&t->tracing_graph_pause, 0);
3370
		atomic_set(&t->trace_overrun, 0);
3371
		t->ftrace_timestamp = 0;
3372 3373 3374
		/* make curr_ret_stack visable before we add the ret_stack */
		smp_wmb();
		t->ret_stack = ret_stack;
3375
	}
3376 3377
}

3378
void ftrace_graph_exit_task(struct task_struct *t)
3379
{
3380 3381
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

3382
	t->ret_stack = NULL;
3383 3384 3385 3386
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
3387
}
S
Steven Rostedt 已提交
3388 3389 3390 3391 3392

void ftrace_graph_stop(void)
{
	ftrace_stop();
}
3393
#endif