ftrace.c 145.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
13
 *  Copyright (C) 2004 Nadia Yvette Chambers
14 15
 */

16 17
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
18
#include <linux/sched/task.h>
19
#include <linux/kallsyms.h>
20
#include <linux/seq_file.h>
21
#include <linux/suspend.h>
22
#include <linux/tracefs.h>
23
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
24
#include <linux/kthread.h>
25
#include <linux/uaccess.h>
26
#include <linux/bsearch.h>
27
#include <linux/module.h>
I
Ingo Molnar 已提交
28
#include <linux/ftrace.h>
29
#include <linux/sysctl.h>
30
#include <linux/slab.h>
31
#include <linux/ctype.h>
32
#include <linux/sort.h>
33
#include <linux/list.h>
34
#include <linux/hash.h>
35
#include <linux/rcupdate.h>
36

37
#include <trace/events/sched.h>
38

39
#include <asm/setup.h>
40

41
#include "trace_output.h"
S
Steven Rostedt 已提交
42
#include "trace_stat.h"
43

44
#define FTRACE_WARN_ON(cond)			\
45 46 47
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
48
			ftrace_kill();		\
49 50
		___r;				\
	})
51 52

#define FTRACE_WARN_ON_ONCE(cond)		\
53 54 55
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
56
			ftrace_kill();		\
57 58
		___r;				\
	})
59

60 61 62
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
63 64
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
65

66
#ifdef CONFIG_DYNAMIC_FTRACE
67 68 69
#define INIT_OPS_HASH(opsname)	\
	.func_hash		= &opsname.local_hash,			\
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
70 71 72
#define ASSIGN_OPS_HASH(opsname, val) \
	.func_hash		= val, \
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
73
#else
74
#define INIT_OPS_HASH(opsname)
75
#define ASSIGN_OPS_HASH(opsname, val)
76 77
#endif

78 79
static struct ftrace_ops ftrace_list_end __read_mostly = {
	.func		= ftrace_stub,
80
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
81
	INIT_OPS_HASH(ftrace_list_end)
82 83
};

84 85
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
86
static int last_ftrace_enabled;
87

88 89
/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
90 91
/* What to set function_trace_op to */
static struct ftrace_ops *set_function_trace_op;
92

93
static bool ftrace_pids_enabled(struct ftrace_ops *ops)
94
{
95 96 97 98 99 100 101 102
	struct trace_array *tr;

	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
		return false;

	tr = ops->private;

	return tr->function_pids != NULL;
103 104 105 106
}

static void ftrace_update_trampoline(struct ftrace_ops *ops);

107 108 109 110 111 112
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

113
static DEFINE_MUTEX(ftrace_lock);
114

115
static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
116
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
117
static struct ftrace_ops global_ops;
118

119 120
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
121
				 struct ftrace_ops *op, struct pt_regs *regs);
122 123 124 125 126
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
127

128 129
/*
 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
130
 * can use rcu_dereference_raw_notrace() is that elements removed from this list
131
 * are simply leaked, so there is no need to interact with a grace-period
132
 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
133 134 135 136 137
 * concurrent insertions into the ftrace_global_list.
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
#define do_for_each_ftrace_op(op, list)			\
138
	op = rcu_dereference_raw_notrace(list);			\
139 140 141 142 143 144
	do

/*
 * Optimized for just a single item in the list (as that is the normal case).
 */
#define while_for_each_ftrace_op(op)				\
145
	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
146 147
	       unlikely((op) != &ftrace_list_end))

148 149 150 151
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
152 153
		mutex_init(&ops->local_hash.regex_lock);
		ops->func_hash = &ops->local_hash;
154 155 156 157 158
		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
	}
#endif
}

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
/**
 * ftrace_nr_registered_ops - return number of ops registered
 *
 * Returns the number of ftrace_ops registered and tracing functions
 */
int ftrace_nr_registered_ops(void)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	mutex_lock(&ftrace_lock);

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next)
		cnt++;

	mutex_unlock(&ftrace_lock);

	return cnt;
}

180
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
181
			    struct ftrace_ops *op, struct pt_regs *regs)
182
{
183 184 185
	struct trace_array *tr = op->private;

	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
186 187
		return;

188
	op->saved_func(ip, parent_ip, op, regs);
189 190
}

191
/**
192
 * clear_ftrace_function - reset the ftrace function
193
 *
194 195
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
196
 */
197
void clear_ftrace_function(void)
198
{
199 200 201
	ftrace_trace_function = ftrace_stub;
}

202
static void per_cpu_ops_disable_all(struct ftrace_ops *ops)
203 204 205 206 207 208 209
{
	int cpu;

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(ops->disabled, cpu) = 1;
}

210
static int per_cpu_ops_alloc(struct ftrace_ops *ops)
211 212 213
{
	int __percpu *disabled;

214 215 216
	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU)))
		return -EINVAL;

217 218 219 220 221
	disabled = alloc_percpu(int);
	if (!disabled)
		return -ENOMEM;

	ops->disabled = disabled;
222
	per_cpu_ops_disable_all(ops);
223 224 225
	return 0;
}

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
static void ftrace_sync(struct work_struct *work)
{
	/*
	 * This function is just a stub to implement a hard force
	 * of synchronize_sched(). This requires synchronizing
	 * tasks even in userspace and idle.
	 *
	 * Yes, function tracing is rude.
	 */
}

static void ftrace_sync_ipi(void *data)
{
	/* Probably not needed, but do it anyway */
	smp_rmb();
}

243 244
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static void update_function_graph_func(void);
245 246 247 248 249

/* Both enabled by default (can be cleared by function_graph tracer flags */
static bool fgraph_sleep_time = true;
static bool fgraph_graph_time = true;

250 251 252 253
#else
static inline void update_function_graph_func(void) { }
#endif

254 255 256 257

static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
	/*
258
	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
259 260
	 * then it needs to call the list anyway.
	 */
261 262
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU |
			  FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC)
263 264 265 266 267
		return ftrace_ops_list_func;

	return ftrace_ops_get_func(ops);
}

268 269 270 271
static void update_ftrace_function(void)
{
	ftrace_func_t func;

272 273 274 275 276 277 278 279 280 281 282
	/*
	 * Prepare the ftrace_ops that the arch callback will use.
	 * If there's only one ftrace_ops registered, the ftrace_ops_list
	 * will point to the ops we want.
	 */
	set_function_trace_op = ftrace_ops_list;

	/* If there's no ftrace_ops registered, just call the stub function */
	if (ftrace_ops_list == &ftrace_list_end) {
		func = ftrace_stub;

283 284
	/*
	 * If we are at the end of the list and this ops is
285 286
	 * recursion safe and not dynamic and the arch supports passing ops,
	 * then have the mcount trampoline call the function directly.
287
	 */
288
	} else if (ftrace_ops_list->next == &ftrace_list_end) {
289
		func = ftrace_ops_get_list_func(ftrace_ops_list);
290

291 292
	} else {
		/* Just use the default ftrace_ops */
293
		set_function_trace_op = &ftrace_list_end;
294
		func = ftrace_ops_list_func;
295
	}
296

297 298
	update_function_graph_func();

299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
	/* If there's no change, then do nothing more here */
	if (ftrace_trace_function == func)
		return;

	/*
	 * If we are using the list function, it doesn't care
	 * about the function_trace_ops.
	 */
	if (func == ftrace_ops_list_func) {
		ftrace_trace_function = func;
		/*
		 * Don't even bother setting function_trace_ops,
		 * it would be racy to do so anyway.
		 */
		return;
	}

#ifndef CONFIG_DYNAMIC_FTRACE
	/*
	 * For static tracing, we need to be a bit more careful.
	 * The function change takes affect immediately. Thus,
	 * we need to coorditate the setting of the function_trace_ops
	 * with the setting of the ftrace_trace_function.
	 *
	 * Set the function to the list ops, which will call the
	 * function we want, albeit indirectly, but it handles the
	 * ftrace_ops and doesn't depend on function_trace_op.
	 */
	ftrace_trace_function = ftrace_ops_list_func;
	/*
	 * Make sure all CPUs see this. Yes this is slow, but static
	 * tracing is slow and nasty to have enabled.
	 */
	schedule_on_each_cpu(ftrace_sync);
	/* Now all cpus are using the list ops. */
	function_trace_op = set_function_trace_op;
	/* Make sure the function_trace_op is visible on all CPUs */
	smp_wmb();
	/* Nasty way to force a rmb on all cpus */
	smp_call_function(ftrace_sync_ipi, NULL, 1);
	/* OK, we are all set to update the ftrace_trace_function now! */
#endif /* !CONFIG_DYNAMIC_FTRACE */

342 343 344
	ftrace_trace_function = func;
}

345 346 347 348 349
int using_ftrace_ops_list_func(void)
{
	return ftrace_trace_function == ftrace_ops_list_func;
}

350
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
351
{
352
	ops->next = *list;
353
	/*
354
	 * We are entering ops into the list but another
355 356
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
357
	 * the ops pointer included into the list.
358
	 */
359
	rcu_assign_pointer(*list, ops);
360 361
}

362
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
363 364 365 366
{
	struct ftrace_ops **p;

	/*
367 368
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
369
	 */
370 371
	if (*list == ops && ops->next == &ftrace_list_end) {
		*list = &ftrace_list_end;
S
Steven Rostedt 已提交
372
		return 0;
373 374
	}

375
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
376 377 378
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
379 380
	if (*p != ops)
		return -1;
381 382

	*p = (*p)->next;
383 384
	return 0;
}
385

386 387
static void ftrace_update_trampoline(struct ftrace_ops *ops);

388 389
static int __register_ftrace_function(struct ftrace_ops *ops)
{
390 391 392
	if (ops->flags & FTRACE_OPS_FL_DELETED)
		return -EINVAL;

393 394 395
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

396
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
397 398 399 400 401 402 403 404 405 406 407 408 409
	/*
	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
	 */
	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
		return -EINVAL;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif

410 411 412
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

413 414
	if (ops->flags & FTRACE_OPS_FL_PER_CPU) {
		if (per_cpu_ops_alloc(ops))
415
			return -ENOMEM;
416 417 418
	}

	add_ftrace_ops(&ftrace_ops_list, ops);
419

420 421 422
	/* Always save the function, and reset at unregistering */
	ops->saved_func = ops->func;

423
	if (ftrace_pids_enabled(ops))
424 425
		ops->func = ftrace_pid_func;

426 427
	ftrace_update_trampoline(ops);

428 429 430 431 432 433 434 435 436 437
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

438 439 440
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

441
	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
442

443 444
	if (ret < 0)
		return ret;
445

446 447
	if (ftrace_enabled)
		update_ftrace_function();
448

449 450
	ops->func = ops->saved_func;

S
Steven Rostedt 已提交
451
	return 0;
452 453
}

454 455
static void ftrace_update_pid_func(void)
{
456 457
	struct ftrace_ops *op;

458
	/* Only do something if we are tracing something */
459
	if (ftrace_trace_function == ftrace_stub)
460
		return;
461

462 463
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->flags & FTRACE_OPS_FL_PID) {
464 465
			op->func = ftrace_pids_enabled(op) ?
				ftrace_pid_func : op->saved_func;
466 467 468 469
			ftrace_update_trampoline(op);
		}
	} while_for_each_ftrace_op(op);

470
	update_ftrace_function();
471 472
}

473 474 475 476 477
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
478 479
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
480
	unsigned long long		time_squared;
481
#endif
482 483
};

484 485 486 487
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
488 489
};

490 491 492 493 494 495 496 497
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

498 499
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
500

501 502
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
503

504 505 506
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
507 508
static DEFINE_MUTEX(ftrace_profile_lock);

509
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
510

511 512
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
513

S
Steven Rostedt 已提交
514 515 516
static void *
function_stat_next(void *v, int idx)
{
517 518
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
519

520
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
521 522

 again:
L
Li Zefan 已提交
523 524 525
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
526 527 528 529 530
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
531 532
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
533 534 535 536 537 538 539
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
540 541 542 543 544 545 546
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
547 548
}

549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
565 566
static int function_stat_cmp(void *p1, void *p2)
{
567 568
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
569 570 571 572 573 574 575 576

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
577
#endif
S
Steven Rostedt 已提交
578 579 580

static int function_stat_headers(struct seq_file *m)
{
581
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
582 583 584 585
	seq_puts(m, "  Function                               "
		 "Hit    Time            Avg             s^2\n"
		    "  --------                               "
		 "---    ----            ---             ---\n");
586
#else
587 588
	seq_puts(m, "  Function                               Hit\n"
		    "  --------                               ---\n");
589
#endif
S
Steven Rostedt 已提交
590 591 592 593 594
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
595
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
596
	char str[KSYM_SYMBOL_LEN];
597
	int ret = 0;
598
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
599 600
	static struct trace_seq s;
	unsigned long long avg;
601
	unsigned long long stddev;
602
#endif
603 604 605 606 607 608 609
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
S
Steven Rostedt 已提交
610

611 612 613 614 615 616 617
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	avg = rec->time;
	do_div(avg, rec->counter);
	if (tracing_thresh && (avg < tracing_thresh))
		goto out;
#endif

S
Steven Rostedt 已提交
618
	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
619 620 621
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
622
	seq_puts(m, "    ");
623

624 625 626 627
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
628 629 630 631 632 633 634
		/*
		 * Apply Welford's method:
		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
		 */
		stddev = rec->counter * rec->time_squared -
			 rec->time * rec->time;

635 636 637 638
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
639
		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
640 641
	}

642 643 644 645
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
646 647
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
648 649 650
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
651 652
out:
	mutex_unlock(&ftrace_profile_lock);
S
Steven Rostedt 已提交
653

654
	return ret;
S
Steven Rostedt 已提交
655 656
}

657
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
658
{
659
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
660

661
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
662

663 664 665 666
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
667 668
	}

669
	memset(stat->hash, 0,
670 671
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
672

673
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
674 675
{
	struct ftrace_profile_page *pg;
676 677
	int functions;
	int pages;
678
	int i;
S
Steven Rostedt 已提交
679

680
	/* If we already allocated, do nothing */
681
	if (stat->pages)
682
		return 0;
S
Steven Rostedt 已提交
683

684 685
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
686
		return -ENOMEM;
S
Steven Rostedt 已提交
687

688 689 690 691 692 693 694 695 696 697 698 699 700
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

701
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
702

703 704
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

705
	for (i = 1; i < pages; i++) {
706 707
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
708
			goto out_free;
709 710 711 712
		pg = pg->next;
	}

	return 0;
713 714 715 716 717 718 719 720 721 722 723 724 725 726

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
727 728
}

729
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
730
{
731
	struct ftrace_profile_stat *stat;
732
	int size;
S
Steven Rostedt 已提交
733

734 735 736
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
737
		/* If the profile is already created, simply reset it */
738
		ftrace_profile_reset(stat);
739 740
		return 0;
	}
S
Steven Rostedt 已提交
741

742 743 744 745 746
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
747

748
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
749

750
	if (!stat->hash)
751 752
		return -ENOMEM;

753
	/* Preallocate the function profiling pages */
754 755 756
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
757 758 759 760
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
761 762
}

763 764 765 766 767
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

768
	for_each_possible_cpu(cpu) {
769 770 771 772 773 774 775 776
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

777
/* interrupts must be disabled */
778 779
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
780
{
781
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
782 783 784
	struct hlist_head *hhd;
	unsigned long key;

785
	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
786
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
787 788 789 790

	if (hlist_empty(hhd))
		return NULL;

791
	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
S
Steven Rostedt 已提交
792
		if (rec->ip == ip)
793 794 795 796 797 798
			return rec;
	}

	return NULL;
}

799 800
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
801 802 803
{
	unsigned long key;

804
	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
805
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
806 807
}

808 809 810
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
811
static struct ftrace_profile *
812
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
813 814 815
{
	struct ftrace_profile *rec = NULL;

816
	/* prevent recursion (from NMIs) */
817
	if (atomic_inc_return(&stat->disabled) != 1)
818 819 820
		goto out;

	/*
821 822
	 * Try to find the function again since an NMI
	 * could have added it
823
	 */
824
	rec = ftrace_find_profiled_func(stat, ip);
825
	if (rec)
826
		goto out;
827

828 829 830 831
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
832
	}
833

834
	rec = &stat->pages->records[stat->pages->index++];
835
	rec->ip = ip;
836
	ftrace_add_profile(stat, rec);
837

S
Steven Rostedt 已提交
838
 out:
839
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
840 841 842 843 844

	return rec;
}

static void
845
function_profile_call(unsigned long ip, unsigned long parent_ip,
846
		      struct ftrace_ops *ops, struct pt_regs *regs)
S
Steven Rostedt 已提交
847
{
848
	struct ftrace_profile_stat *stat;
849
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
850 851 852 853 854 855
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
856

857
	stat = this_cpu_ptr(&ftrace_profile_stats);
858
	if (!stat->hash || !ftrace_profile_enabled)
859 860 861
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
862
	if (!rec) {
863
		rec = ftrace_profile_alloc(stat, ip);
864 865 866
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
867 868 869 870 871 872

	rec->counter++;
 out:
	local_irq_restore(flags);
}

873 874 875
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
876 877
	int index = trace->depth;

878
	function_profile_call(trace->func, 0, NULL, NULL);
879 880 881 882

	if (index >= 0 && index < FTRACE_RETFUNC_DEPTH)
		current->ret_stack[index].subtime = 0;

883 884 885 886 887
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
888
	struct ftrace_profile_stat *stat;
889
	unsigned long long calltime;
890
	struct ftrace_profile *rec;
891
	unsigned long flags;
892 893

	local_irq_save(flags);
894
	stat = this_cpu_ptr(&ftrace_profile_stats);
895
	if (!stat->hash || !ftrace_profile_enabled)
896 897
		goto out;

898 899 900 901
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

902 903
	calltime = trace->rettime - trace->calltime;

904
	if (!fgraph_graph_time) {
905 906 907 908 909 910 911 912 913 914 915 916 917 918
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

919
	rec = ftrace_find_profiled_func(stat, trace->func);
920
	if (rec) {
921
		rec->time += calltime;
922 923
		rec->time_squared += calltime * calltime;
	}
924

925
 out:
926 927 928 929 930 931 932 933 934 935 936 937 938 939
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
940
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
941
	.func		= function_profile_call,
942
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
943
	INIT_OPS_HASH(ftrace_profile_ops)
S
Steven Rostedt 已提交
944 945
};

946 947 948 949 950 951 952 953 954 955 956
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
957 958 959 960 961 962 963
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

964 965
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
S
Steven Rostedt 已提交
966 967 968 969 970 971 972
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
973 974 975 976 977 978
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

979 980 981 982 983
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
984 985 986
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
987 988 989 990
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
991
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
992 993
		}
	}
994
 out:
S
Steven Rostedt 已提交
995 996
	mutex_unlock(&ftrace_profile_lock);

997
	*ppos += cnt;
S
Steven Rostedt 已提交
998 999 1000 1001

	return cnt;
}

1002 1003 1004 1005
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
1006
	char buf[64];		/* big enough to hold a number */
1007 1008 1009 1010 1011 1012
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
1013 1014 1015 1016
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
1017
	.llseek		= default_llseek,
S
Steven Rostedt 已提交
1018 1019
};

1020 1021
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
1022 1023 1024 1025 1026 1027
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
1028 1029
};

1030
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1031
{
1032
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
1033
	struct dentry *entry;
1034
	char *name;
S
Steven Rostedt 已提交
1035
	int ret;
1036 1037 1038 1039 1040
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

1041
		name = kasprintf(GFP_KERNEL, "function%d", cpu);
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
1062 1063
	}

1064
	entry = tracefs_create_file("function_profile_enabled", 0644,
S
Steven Rostedt 已提交
1065 1066
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
1067
		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
S
Steven Rostedt 已提交
1068 1069 1070
}

#else /* CONFIG_FUNCTION_PROFILER */
1071
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1072 1073 1074 1075
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

1076 1077
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

1078 1079 1080 1081 1082 1083
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int ftrace_graph_active;
#else
# define ftrace_graph_active 0
#endif

1084 1085
#ifdef CONFIG_DYNAMIC_FTRACE

1086 1087
static struct ftrace_ops *removed_ops;

1088 1089 1090 1091 1092 1093
/*
 * Set when doing a global update, like enabling all recs or disabling them.
 * It is not set when just updating a single ftrace_ops.
 */
static bool update_all_ops;

1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;

struct ftrace_func_probe {
	struct hlist_node	node;
	struct ftrace_probe_ops	*ops;
	unsigned long		flags;
	unsigned long		ip;
	void			*data;
1106
	struct list_head	free_list;
1107 1108
};

1109 1110 1111 1112 1113
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

1114 1115 1116 1117 1118 1119 1120 1121 1122
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
1123
};
1124
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1125

1126
static struct ftrace_ops global_ops = {
1127 1128 1129 1130 1131
	.func				= ftrace_stub,
	.local_hash.notrace_hash	= EMPTY_HASH,
	.local_hash.filter_hash		= EMPTY_HASH,
	INIT_OPS_HASH(global_ops)
	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1132 1133
					  FTRACE_OPS_FL_INITIALIZED |
					  FTRACE_OPS_FL_PID,
1134 1135
};

1136 1137
/*
 * This is used by __kernel_text_address() to return true if the
1138
 * address is on a dynamically allocated trampoline that would
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
 * not return true for either core_kernel_text() or
 * is_module_text_address().
 */
bool is_ftrace_trampoline(unsigned long addr)
{
	struct ftrace_ops *op;
	bool ret = false;

	/*
	 * Some of the ops may be dynamically allocated,
	 * they are freed after a synchronize_sched().
	 */
	preempt_disable_notrace();

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/*
		 * This is to check for dynamically allocated trampolines.
		 * Trampolines that are in kernel text will have
		 * core_kernel_text() return true.
		 */
		if (op->trampoline && op->trampoline_size)
			if (addr >= op->trampoline &&
			    addr < op->trampoline + op->trampoline_size) {
				ret = true;
				goto out;
			}
	} while_for_each_ftrace_op(op);

 out:
	preempt_enable_notrace();

	return ret;
}

1173 1174
struct ftrace_page {
	struct ftrace_page	*next;
1175
	struct dyn_ftrace	*records;
1176
	int			index;
1177
	int			size;
1178 1179
};

1180 1181
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1182 1183 1184 1185 1186 1187 1188

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

1189 1190 1191 1192 1193 1194 1195 1196 1197
static __always_inline unsigned long
ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
{
	if (hash->size_bits > 0)
		return hash_long(ip, hash->size_bits);

	return 0;
}

1198 1199 1200
/* Only use this function if ftrace_hash_empty() has already been tested */
static __always_inline struct ftrace_func_entry *
__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1201 1202 1203 1204 1205
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;

1206
	key = ftrace_hash_key(hash, ip);
1207 1208
	hhd = &hash->buckets[key];

1209
	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1210 1211 1212 1213 1214 1215
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
/**
 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
 * @hash: The hash to look at
 * @ip: The instruction pointer to test
 *
 * Search a given @hash to see if a given instruction pointer (@ip)
 * exists in it.
 *
 * Returns the entry that holds the @ip if found. NULL otherwise.
 */
struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	if (ftrace_hash_empty(hash))
		return NULL;

	return __ftrace_lookup_ip(hash, ip);
}

1235 1236
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1237 1238 1239 1240
{
	struct hlist_head *hhd;
	unsigned long key;

1241
	key = ftrace_hash_key(hash, entry->ip);
1242 1243 1244
	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1257 1258 1259 1260 1261

	return 0;
}

static void
1262
free_hash_entry(struct ftrace_hash *hash,
1263 1264 1265 1266 1267 1268 1269
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1270 1271 1272 1273 1274 1275 1276 1277
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	hash->count--;
}

1278 1279 1280
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
1281
	struct hlist_node *tn;
1282 1283 1284 1285
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1286 1287 1288
	if (!hash->count)
		return;

1289 1290
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
1291
		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1292
			free_hash_entry(hash, entry);
1293 1294 1295 1296
	}
	FTRACE_WARN_ON(hash->count);
}

1297 1298 1299 1300 1301 1302 1303 1304 1305
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}

1321 1322
void ftrace_free_filter(struct ftrace_ops *ops)
{
1323
	ftrace_ops_init(ops);
1324 1325
	free_ftrace_hash(ops->func_hash->filter_hash);
	free_ftrace_hash(ops->func_hash->notrace_hash);
1326 1327
}

1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
1338
	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

	/* Empty hash? */
1364
	if (ftrace_hash_empty(hash))
1365 1366 1367 1368
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
1369
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

1385
static void
1386
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1387
static void
1388
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1389

1390 1391 1392
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash);

1393 1394
static struct ftrace_hash *
__ftrace_hash_move(struct ftrace_hash *src)
1395 1396
{
	struct ftrace_func_entry *entry;
1397
	struct hlist_node *tn;
1398
	struct hlist_head *hhd;
1399
	struct ftrace_hash *new_hash;
1400 1401 1402 1403 1404
	int size = src->count;
	int bits = 0;
	int i;

	/*
1405
	 * If the new source is empty, just return the empty_hash.
1406
	 */
1407 1408
	if (!src->count)
		return EMPTY_HASH;
1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1420 1421
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
1422
		return NULL;
1423 1424 1425 1426

	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
1427
		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1428
			remove_hash_entry(src, entry);
1429
			__add_hash_entry(new_hash, entry);
1430 1431 1432
		}
	}

1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
	return new_hash;
}

static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
		 struct ftrace_hash **dst, struct ftrace_hash *src)
{
	struct ftrace_hash *new_hash;
	int ret;

	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
		return -EINVAL;

	new_hash = __ftrace_hash_move(src);
	if (!new_hash)
		return -ENOMEM;

1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
	if (enable) {
		/* IPMODIFY should be updated only when filter_hash updating */
		ret = ftrace_hash_ipmodify_update(ops, new_hash);
		if (ret < 0) {
			free_ftrace_hash(new_hash);
			return ret;
		}
	}

1461 1462 1463 1464
	/*
	 * Remove the current set, update the hash and add
	 * them back.
	 */
1465
	ftrace_hash_rec_disable_modify(ops, enable);
1466

1467 1468
	rcu_assign_pointer(*dst, new_hash);

1469
	ftrace_hash_rec_enable_modify(ops, enable);
1470

1471
	return 0;
1472 1473
}

1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
static bool hash_contains_ip(unsigned long ip,
			     struct ftrace_ops_hash *hash)
{
	/*
	 * The function record is a match if it exists in the filter
	 * hash and not in the notrace hash. Note, an emty hash is
	 * considered a match for the filter hash, but an empty
	 * notrace hash is considered not in the notrace hash.
	 */
	return (ftrace_hash_empty(hash->filter_hash) ||
1484
		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1485
		(ftrace_hash_empty(hash->notrace_hash) ||
1486
		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1487 1488
}

1489 1490 1491 1492 1493 1494 1495 1496
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1497 1498 1499
 *
 * This needs to be called with preemption disabled as
 * the hashes are freed with call_rcu_sched().
1500 1501
 */
static int
1502
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1503
{
1504
	struct ftrace_ops_hash hash;
1505 1506
	int ret;

1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	/*
	 * There's a small race when adding ops that the ftrace handler
	 * that wants regs, may be called without them. We can not
	 * allow that handler to be called if regs is NULL.
	 */
	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
		return 0;
#endif

1517 1518
	hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
	hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
1519

1520
	if (hash_contains_ip(ip, &hash))
1521 1522 1523 1524 1525 1526 1527
		ret = 1;
	else
		ret = 0;

	return ret;
}

1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1542 1543 1544

static int ftrace_cmp_recs(const void *a, const void *b)
{
1545 1546
	const struct dyn_ftrace *key = a;
	const struct dyn_ftrace *rec = b;
1547

1548
	if (key->flags < rec->ip)
1549
		return -1;
1550 1551
	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
		return 1;
1552 1553 1554
	return 0;
}

1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
/**
 * ftrace_location_range - return the first address of a traced location
 *	if it touches the given ip range
 * @start: start of range to search.
 * @end: end of range to search (inclusive). @end points to the last byte
 *	to check.
 *
 * Returns rec->ip if the related ftrace location is a least partly within
 * the given address range. That is, the first address of the instruction
 * that is either a NOP or call to the function tracer. It checks the ftrace
 * internal tables to determine if the address belongs or not.
 */
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1568 1569 1570
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1571
	struct dyn_ftrace key;
1572

1573 1574
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */
1575 1576

	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1577 1578
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1579
			continue;
1580 1581 1582 1583
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (rec)
1584
			return rec->ip;
1585
	}
1586 1587 1588 1589

	return 0;
}

1590 1591 1592 1593
/**
 * ftrace_location - return true if the ip giving is a traced location
 * @ip: the instruction pointer to check
 *
1594
 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1595 1596 1597 1598
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1599
unsigned long ftrace_location(unsigned long ip)
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613
{
	return ftrace_location_range(ip, ip);
}

/**
 * ftrace_text_reserved - return true if range contains an ftrace location
 * @start: start of range to search
 * @end: end of range to search (inclusive). @end points to the last byte to check.
 *
 * Returns 1 if @start and @end contains a ftrace location.
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1614
int ftrace_text_reserved(const void *start, const void *end)
1615
{
1616 1617 1618 1619 1620 1621
	unsigned long ret;

	ret = ftrace_location_range((unsigned long)start,
				    (unsigned long)end);

	return (int)!!ret;
1622 1623
}

1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
/* Test if ops registered to this rec needs regs */
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	bool keep_regs = false;

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next) {
		/* pass rec in as regs to have non-NULL val */
		if (ftrace_ops_test(ops, rec->ip, rec)) {
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				keep_regs = true;
				break;
			}
		}
	}

	return  keep_regs;
}

1644
static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1645 1646 1647 1648 1649 1650 1651
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1652
	bool update = false;
1653 1654 1655 1656 1657
	int count = 0;
	int all = 0;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1658
		return false;
1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
1672 1673
		hash = ops->func_hash->filter_hash;
		other_hash = ops->func_hash->notrace_hash;
1674
		if (ftrace_hash_empty(hash))
1675 1676 1677
			all = 1;
	} else {
		inc = !inc;
1678 1679
		hash = ops->func_hash->notrace_hash;
		other_hash = ops->func_hash->filter_hash;
1680 1681 1682 1683
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1684
		if (ftrace_hash_empty(hash))
1685
			return false;
1686 1687 1688 1689 1690 1691 1692
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

1693 1694 1695
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1696 1697 1698 1699 1700
		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1701
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1702 1703
				match = 1;
		} else {
1704 1705
			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1706 1707

			/*
1708 1709
			 * If filter_hash is set, we want to match all functions
			 * that are in the hash but not in the other hash.
1710
			 *
1711 1712 1713 1714 1715
			 * If filter_hash is not set, then we are decrementing.
			 * That means we match anything that is in the hash
			 * and also in the other_hash. That is, we need to turn
			 * off functions in the other hash because they are disabled
			 * by this hash.
1716 1717 1718 1719
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
1720
				 (in_other_hash || ftrace_hash_empty(other_hash)))
1721 1722 1723 1724 1725 1726 1727
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
1728
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1729
				return false;
1730 1731 1732 1733 1734 1735

			/*
			 * If there's only a single callback registered to a
			 * function, and the ops has a trampoline registered
			 * for it, then we can call it directly.
			 */
1736
			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1737
				rec->flags |= FTRACE_FL_TRAMP;
1738
			else
1739 1740 1741
				/*
				 * If we are adding another function callback
				 * to this function, and the previous had a
1742 1743
				 * custom trampoline in use, then we need to go
				 * back to the default trampoline.
1744
				 */
1745
				rec->flags &= ~FTRACE_FL_TRAMP;
1746

1747 1748 1749 1750 1751 1752
			/*
			 * If any ops wants regs saved for this function
			 * then all ops will get saved regs.
			 */
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
				rec->flags |= FTRACE_FL_REGS;
1753
		} else {
1754
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1755
				return false;
1756
			rec->flags--;
1757

1758 1759 1760 1761 1762 1763
			/*
			 * If the rec had REGS enabled and the ops that is
			 * being removed had REGS set, then see if there is
			 * still any ops for this record that wants regs.
			 * If not, we can stop recording them.
			 */
1764
			if (ftrace_rec_count(rec) > 0 &&
1765 1766 1767 1768 1769
			    rec->flags & FTRACE_FL_REGS &&
			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				if (!test_rec_ops_needs_regs(rec))
					rec->flags &= ~FTRACE_FL_REGS;
			}
1770

1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
			/*
			 * If the rec had TRAMP enabled, then it needs to
			 * be cleared. As TRAMP can only be enabled iff
			 * there is only a single ops attached to it.
			 * In otherwords, always disable it on decrementing.
			 * In the future, we may set it if rec count is
			 * decremented to one, and the ops that is left
			 * has a trampoline.
			 */
			rec->flags &= ~FTRACE_FL_TRAMP;

1782 1783 1784 1785
			/*
			 * flags will be cleared in ftrace_check_record()
			 * if rec count is zero.
			 */
1786 1787
		}
		count++;
1788 1789 1790 1791

		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;

1792 1793
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
1794
			return update;
1795
	} while_for_each_ftrace_rec();
1796 1797

	return update;
1798 1799
}

1800
static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1801 1802
				    int filter_hash)
{
1803
	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1804 1805
}

1806
static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1807 1808
				   int filter_hash)
{
1809
	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1810 1811
}

1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
					  int filter_hash, int inc)
{
	struct ftrace_ops *op;

	__ftrace_hash_rec_update(ops, filter_hash, inc);

	if (ops->func_hash != &global_ops.local_hash)
		return;

	/*
	 * If the ops shares the global_ops hash, then we need to update
	 * all ops that are enabled and use this hash.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* Already done */
		if (op == ops)
			continue;
		if (op->func_hash == &global_ops.local_hash)
			__ftrace_hash_rec_update(op, filter_hash, inc);
	} while_for_each_ftrace_op(op);
}

static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
					   int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
					  int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}

1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
/*
 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
 * or no-needed to update, -EBUSY if it detects a conflict of the flag
 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
 * Note that old_hash and new_hash has below meanings
 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
 *  - If the hash is EMPTY_HASH, it hits nothing
 *  - Anything else hits the recs which match the hash entries.
 */
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
					 struct ftrace_hash *old_hash,
					 struct ftrace_hash *new_hash)
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec, *end = NULL;
	int in_old, in_new;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
		return 0;

	/*
	 * Since the IPMODIFY is a very address sensitive action, we do not
	 * allow ftrace_ops to set all functions to new hash.
	 */
	if (!new_hash || !old_hash)
		return -EINVAL;

	/* Update rec->flags */
	do_for_each_ftrace_rec(pg, rec) {
1880 1881 1882 1883

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
		/* We need to update only differences of filter_hash */
		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new) {
			/* New entries must ensure no others are using it */
			if (rec->flags & FTRACE_FL_IPMODIFY)
				goto rollback;
			rec->flags |= FTRACE_FL_IPMODIFY;
		} else /* Removed entry */
			rec->flags &= ~FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

	return 0;

rollback:
	end = rec;

	/* Roll back what we did above */
	do_for_each_ftrace_rec(pg, rec) {
1906 1907 1908 1909

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962
		if (rec == end)
			goto err_out;

		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new)
			rec->flags &= ~FTRACE_FL_IPMODIFY;
		else
			rec->flags |= FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

err_out:
	return -EBUSY;
}

static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}

/* Disabling always succeeds */
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}

static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash)
{
	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(old_hash))
		old_hash = NULL;

	if (ftrace_hash_empty(new_hash))
		new_hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}

1963
static void print_ip_ins(const char *fmt, const unsigned char *p)
1964 1965 1966 1967 1968 1969 1970 1971 1972
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1973 1974
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1975 1976
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1977

1978
enum ftrace_bug_type ftrace_bug_type;
1979
const void *ftrace_expected;
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000

static void print_bug_type(void)
{
	switch (ftrace_bug_type) {
	case FTRACE_BUG_UNKNOWN:
		break;
	case FTRACE_BUG_INIT:
		pr_info("Initializing ftrace call sites\n");
		break;
	case FTRACE_BUG_NOP:
		pr_info("Setting ftrace call site to NOP\n");
		break;
	case FTRACE_BUG_CALL:
		pr_info("Setting ftrace call site to call ftrace function\n");
		break;
	case FTRACE_BUG_UPDATE:
		pr_info("Updating ftrace call site to call a different ftrace function\n");
		break;
	}
}

2001 2002 2003
/**
 * ftrace_bug - report and shutdown function tracer
 * @failed: The failed type (EFAULT, EINVAL, EPERM)
2004
 * @rec: The record that failed
2005 2006 2007 2008 2009 2010 2011 2012
 *
 * The arch code that enables or disables the function tracing
 * can call ftrace_bug() when it has detected a problem in
 * modifying the code. @failed should be one of either:
 * EFAULT - if the problem happens on reading the @ip address
 * EINVAL - if what is read at @ip is not what was expected
 * EPERM - if the problem happens on writting to the @ip address
 */
2013
void ftrace_bug(int failed, struct dyn_ftrace *rec)
2014
{
2015 2016
	unsigned long ip = rec ? rec->ip : 0;

2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
2027
		print_ip_ins(" actual:   ", (unsigned char *)ip);
2028
		pr_cont("\n");
2029 2030 2031 2032
		if (ftrace_expected) {
			print_ip_ins(" expected: ", ftrace_expected);
			pr_cont("\n");
		}
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
2044
	print_bug_type();
2045 2046 2047 2048 2049 2050 2051 2052
	if (rec) {
		struct ftrace_ops *ops = NULL;

		pr_info("ftrace record flags: %lx\n", rec->flags);
		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
			ops = ftrace_find_tramp_ops_any(rec);
2053 2054 2055 2056 2057 2058 2059 2060
			if (ops) {
				do {
					pr_cont("\ttramp: %pS (%pS)",
						(void *)ops->trampoline,
						(void *)ops->func);
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
2061 2062 2063 2064
				pr_cont("\ttramp: ERROR!");

		}
		ip = ftrace_get_addr_curr(rec);
2065
		pr_cont("\n expected tramp: %lx\n", ip);
2066
	}
2067 2068
}

2069
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2070
{
2071
	unsigned long flag = 0UL;
2072

2073 2074
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2075 2076 2077
	if (rec->flags & FTRACE_FL_DISABLED)
		return FTRACE_UPDATE_IGNORE;

S
Steven Rostedt 已提交
2078
	/*
2079
	 * If we are updating calls:
S
Steven Rostedt 已提交
2080
	 *
2081 2082
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
S
Steven Rostedt 已提交
2083
	 *
2084 2085
	 *   Otherwise we make sure its disabled.
	 *
2086
	 * If we are disabling calls, then disable all records that
2087
	 * are enabled.
S
Steven Rostedt 已提交
2088
	 */
2089
	if (enable && ftrace_rec_count(rec))
2090
		flag = FTRACE_FL_ENABLED;
S
Steven Rostedt 已提交
2091

2092
	/*
2093 2094 2095
	 * If enabling and the REGS flag does not match the REGS_EN, or
	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
	 * this record. Set flags to fail the compare against ENABLED.
2096
	 */
2097 2098 2099 2100 2101 2102 2103 2104 2105
	if (flag) {
		if (!(rec->flags & FTRACE_FL_REGS) != 
		    !(rec->flags & FTRACE_FL_REGS_EN))
			flag |= FTRACE_FL_REGS;

		if (!(rec->flags & FTRACE_FL_TRAMP) != 
		    !(rec->flags & FTRACE_FL_TRAMP_EN))
			flag |= FTRACE_FL_TRAMP;
	}
2106

2107 2108
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2109
		return FTRACE_UPDATE_IGNORE;
S
Steven Rostedt 已提交
2110

2111
	if (flag) {
2112 2113 2114 2115
		/* Save off if rec is being enabled (for return value) */
		flag ^= rec->flags & FTRACE_FL_ENABLED;

		if (update) {
2116
			rec->flags |= FTRACE_FL_ENABLED;
2117 2118 2119 2120 2121 2122
			if (flag & FTRACE_FL_REGS) {
				if (rec->flags & FTRACE_FL_REGS)
					rec->flags |= FTRACE_FL_REGS_EN;
				else
					rec->flags &= ~FTRACE_FL_REGS_EN;
			}
2123 2124 2125 2126 2127 2128
			if (flag & FTRACE_FL_TRAMP) {
				if (rec->flags & FTRACE_FL_TRAMP)
					rec->flags |= FTRACE_FL_TRAMP_EN;
				else
					rec->flags &= ~FTRACE_FL_TRAMP_EN;
			}
2129 2130 2131 2132 2133 2134 2135
		}

		/*
		 * If this record is being updated from a nop, then
		 *   return UPDATE_MAKE_CALL.
		 * Otherwise,
		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2136
		 *   from the save regs, to a non-save regs function or
2137
		 *   vice versa, or from a trampoline call.
2138
		 */
2139 2140
		if (flag & FTRACE_FL_ENABLED) {
			ftrace_bug_type = FTRACE_BUG_CALL;
2141
			return FTRACE_UPDATE_MAKE_CALL;
2142
		}
2143

2144
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2145
		return FTRACE_UPDATE_MODIFY_CALL;
2146 2147
	}

2148 2149
	if (update) {
		/* If there's no more users, clear all flags */
2150
		if (!ftrace_rec_count(rec))
2151 2152
			rec->flags = 0;
		else
2153 2154 2155 2156 2157 2158
			/*
			 * Just disable the record, but keep the ops TRAMP
			 * and REGS states. The _EN flags must be disabled though.
			 */
			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
					FTRACE_FL_REGS_EN);
2159
	}
2160

2161
	ftrace_bug_type = FTRACE_BUG_NOP;
2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
	return FTRACE_UPDATE_MAKE_NOP;
}

/**
 * ftrace_update_record, set a record that now is tracing or not
 * @rec: the record to update
 * @enable: set to 1 if the record is tracing, zero to force disable
 *
 * The records that represent all functions that can be traced need
 * to be updated when tracing has been enabled.
 */
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 1);
}

/**
 * ftrace_test_record, check if the record has been enabled or not
 * @rec: the record to test
 * @enable: set to 1 to check if enabled, 0 if it is disabled
 *
 * The arch code may need to test if a record is already set to
 * tracing to determine how to modify the function code that it
 * represents.
 */
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 0);
}

2192 2193 2194 2195
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2196
	unsigned long ip = rec->ip;
2197 2198 2199 2200 2201 2202

	do_for_each_ftrace_op(op, ftrace_ops_list) {

		if (!op->trampoline)
			continue;

2203
		if (hash_contains_ip(ip, op->func_hash))
2204 2205 2206 2207 2208 2209
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
			   struct ftrace_ops *op)
{
	unsigned long ip = rec->ip;

	while_for_each_ftrace_op(op) {

		if (!op->trampoline)
			continue;

		if (hash_contains_ip(ip, op->func_hash))
			return op;
	} 

	return NULL;
}

2228 2229 2230 2231
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2232
	unsigned long ip = rec->ip;
2233

2234 2235 2236 2237 2238 2239 2240 2241
	/*
	 * Need to check removed ops first.
	 * If they are being removed, and this rec has a tramp,
	 * and this rec is in the ops list, then it would be the
	 * one with the tramp.
	 */
	if (removed_ops) {
		if (hash_contains_ip(ip, &removed_ops->old_hash))
2242 2243 2244
			return removed_ops;
	}

2245 2246 2247 2248 2249 2250 2251
	/*
	 * Need to find the current trampoline for a rec.
	 * Now, a trampoline is only attached to a rec if there
	 * was a single 'ops' attached to it. But this can be called
	 * when we are adding another op to the rec or removing the
	 * current one. Thus, if the op is being added, we can
	 * ignore it because it hasn't attached itself to the rec
2252 2253 2254 2255 2256 2257 2258 2259 2260 2261
	 * yet.
	 *
	 * If an ops is being modified (hooking to different functions)
	 * then we don't care about the new functions that are being
	 * added, just the old ones (that are probably being removed).
	 *
	 * If we are adding an ops to a function that already is using
	 * a trampoline, it needs to be removed (trampolines are only
	 * for single ops connected), then an ops that is not being
	 * modified also needs to be checked.
2262
	 */
2263
	do_for_each_ftrace_op(op, ftrace_ops_list) {
2264 2265 2266 2267 2268 2269 2270 2271 2272

		if (!op->trampoline)
			continue;

		/*
		 * If the ops is being added, it hasn't gotten to
		 * the point to be removed from this tree yet.
		 */
		if (op->flags & FTRACE_OPS_FL_ADDING)
2273 2274
			continue;

2275

2276
		/*
2277 2278 2279
		 * If the ops is being modified and is in the old
		 * hash, then it is probably being removed from this
		 * function.
2280 2281 2282
		 */
		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, &op->old_hash))
2283
			return op;
2284 2285 2286 2287 2288 2289 2290 2291
		/*
		 * If the ops is not being added or modified, and it's
		 * in its normal filter hash, then this must be the one
		 * we want!
		 */
		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, op->func_hash))
			return op;
2292 2293 2294 2295 2296 2297 2298 2299 2300 2301

	} while_for_each_ftrace_op(op);

	return NULL;
}

static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2302
	unsigned long ip = rec->ip;
2303 2304 2305

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* pass rec in as regs to have non-NULL val */
2306
		if (hash_contains_ip(ip, op->func_hash))
2307 2308 2309 2310 2311 2312
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324
/**
 * ftrace_get_addr_new - Get the call address to set to
 * @rec:  The ftrace record descriptor
 *
 * If the record has the FTRACE_FL_REGS set, that means that it
 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
 * is not not set, then it wants to convert to the normal callback.
 *
 * Returns the address of the trampoline to set to
 */
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
2325 2326 2327 2328 2329 2330
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP) {
		ops = ftrace_find_tramp_ops_new(rec);
		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2331 2332
			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
				(void *)rec->ip, (void *)rec->ip, rec->flags);
2333 2334 2335 2336 2337 2338
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
	if (rec->flags & FTRACE_FL_REGS)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

/**
 * ftrace_get_addr_curr - Get the call address that is already there
 * @rec:  The ftrace record descriptor
 *
 * The FTRACE_FL_REGS_EN is set when the record already points to
 * a function that saves all the regs. Basically the '_EN' version
 * represents the current state of the function.
 *
 * Returns the address of the trampoline that is currently being called
 */
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
2357 2358 2359 2360 2361 2362
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP_EN) {
		ops = ftrace_find_tramp_ops_curr(rec);
		if (FTRACE_WARN_ON(!ops)) {
2363 2364
			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
				(void *)rec->ip, (void *)rec->ip);
2365 2366 2367 2368 2369 2370
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2371 2372 2373 2374 2375 2376
	if (rec->flags & FTRACE_FL_REGS_EN)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

2377 2378 2379
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
2380
	unsigned long ftrace_old_addr;
2381 2382 2383
	unsigned long ftrace_addr;
	int ret;

2384
	ftrace_addr = ftrace_get_addr_new(rec);
2385

2386 2387 2388 2389
	/* This needs to be done before we call ftrace_update_record */
	ftrace_old_addr = ftrace_get_addr_curr(rec);

	ret = ftrace_update_record(rec, enable);
2390

2391 2392
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2393 2394 2395 2396 2397
	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
2398
		ftrace_bug_type = FTRACE_BUG_CALL;
2399
		return ftrace_make_call(rec, ftrace_addr);
2400 2401

	case FTRACE_UPDATE_MAKE_NOP:
2402
		ftrace_bug_type = FTRACE_BUG_NOP;
2403
		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2404 2405

	case FTRACE_UPDATE_MODIFY_CALL:
2406
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2407
		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2408 2409
	}

2410
	return -1; /* unknow ftrace bug */
2411 2412
}

2413
void __weak ftrace_replace_code(int enable)
2414 2415 2416
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
2417
	int failed;
2418

2419 2420 2421
	if (unlikely(ftrace_disabled))
		return;

2422
	do_for_each_ftrace_rec(pg, rec) {
2423 2424 2425 2426

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

2427
		failed = __ftrace_replace_code(rec, enable);
2428
		if (failed) {
2429
			ftrace_bug(failed, rec);
2430 2431
			/* Stop processing */
			return;
2432
		}
2433
	} while_for_each_ftrace_rec();
2434 2435
}

2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507
struct ftrace_rec_iter {
	struct ftrace_page	*pg;
	int			index;
};

/**
 * ftrace_rec_iter_start, start up iterating over traced functions
 *
 * Returns an iterator handle that is used to iterate over all
 * the records that represent address locations where functions
 * are traced.
 *
 * May return NULL if no records are available.
 */
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
	/*
	 * We only use a single iterator.
	 * Protected by the ftrace_lock mutex.
	 */
	static struct ftrace_rec_iter ftrace_rec_iter;
	struct ftrace_rec_iter *iter = &ftrace_rec_iter;

	iter->pg = ftrace_pages_start;
	iter->index = 0;

	/* Could have empty pages */
	while (iter->pg && !iter->pg->index)
		iter->pg = iter->pg->next;

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_next, get the next record to process.
 * @iter: The handle to the iterator.
 *
 * Returns the next iterator after the given iterator @iter.
 */
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
	iter->index++;

	if (iter->index >= iter->pg->index) {
		iter->pg = iter->pg->next;
		iter->index = 0;

		/* Could have empty pages */
		while (iter->pg && !iter->pg->index)
			iter->pg = iter->pg->next;
	}

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_record, get the record at the iterator location
 * @iter: The current iterator location
 *
 * Returns the record that the current @iter is at.
 */
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
	return &iter->pg->records[iter->index];
}

2508
static int
2509
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2510
{
2511
	int ret;
2512

2513 2514 2515
	if (unlikely(ftrace_disabled))
		return 0;

2516
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2517
	if (ret) {
2518
		ftrace_bug_type = FTRACE_BUG_INIT;
2519
		ftrace_bug(ret, rec);
2520
		return 0;
2521
	}
2522
	return 1;
2523 2524
}

2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

2543
void ftrace_modify_all_code(int command)
2544
{
2545
	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2546
	int err = 0;
2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557

	/*
	 * If the ftrace_caller calls a ftrace_ops func directly,
	 * we need to make sure that it only traces functions it
	 * expects to trace. When doing the switch of functions,
	 * we need to update to the ftrace_ops_list_func first
	 * before the transition between old and new calls are set,
	 * as the ftrace_ops_list_func will check the ops hashes
	 * to make sure the ops are having the right functions
	 * traced.
	 */
2558 2559 2560 2561 2562
	if (update) {
		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
		if (FTRACE_WARN_ON(err))
			return;
	}
2563

2564
	if (command & FTRACE_UPDATE_CALLS)
2565
		ftrace_replace_code(1);
2566
	else if (command & FTRACE_DISABLE_CALLS)
2567 2568
		ftrace_replace_code(0);

2569 2570 2571 2572 2573 2574
	if (update && ftrace_trace_function != ftrace_ops_list_func) {
		function_trace_op = set_function_trace_op;
		smp_wmb();
		/* If irqs are disabled, we are in stop machine */
		if (!irqs_disabled())
			smp_call_function(ftrace_sync_ipi, NULL, 1);
2575 2576 2577
		err = ftrace_update_ftrace_func(ftrace_trace_function);
		if (FTRACE_WARN_ON(err))
			return;
2578
	}
2579

2580
	if (command & FTRACE_START_FUNC_RET)
2581
		err = ftrace_enable_ftrace_graph_caller();
2582
	else if (command & FTRACE_STOP_FUNC_RET)
2583 2584
		err = ftrace_disable_ftrace_graph_caller();
	FTRACE_WARN_ON(err);
2585 2586 2587 2588 2589 2590 2591
}

static int __ftrace_modify_code(void *data)
{
	int *command = data;

	ftrace_modify_all_code(*command);
2592

2593
	return 0;
2594 2595
}

2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619
/**
 * ftrace_run_stop_machine, go back to the stop machine method
 * @command: The command to tell ftrace what to do
 *
 * If an arch needs to fall back to the stop machine method, the
 * it can call this function.
 */
void ftrace_run_stop_machine(int command)
{
	stop_machine(__ftrace_modify_code, &command, NULL);
}

/**
 * arch_ftrace_update_code, modify the code to trace or not trace
 * @command: The command that needs to be done
 *
 * Archs can override this function if it does not need to
 * run stop_machine() to modify code.
 */
void __weak arch_ftrace_update_code(int command)
{
	ftrace_run_stop_machine(command);
}

I
Ingo Molnar 已提交
2620
static void ftrace_run_update_code(int command)
2621
{
2622 2623 2624 2625 2626 2627 2628
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;

2629 2630 2631 2632 2633 2634 2635 2636
	/*
	 * By default we use stop_machine() to modify the code.
	 * But archs can do what ever they want as long as it
	 * is safe. The stop_machine() is the safest, but also
	 * produces the most overhead.
	 */
	arch_ftrace_update_code(command);

2637 2638
	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
2639 2640
}

2641
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2642
				   struct ftrace_ops_hash *old_hash)
2643 2644
{
	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2645 2646
	ops->old_hash.filter_hash = old_hash->filter_hash;
	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2647
	ftrace_run_update_code(command);
2648
	ops->old_hash.filter_hash = NULL;
2649
	ops->old_hash.notrace_hash = NULL;
2650 2651 2652
	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}

2653
static ftrace_func_t saved_ftrace_func;
2654
static int ftrace_start_up;
2655

2656 2657 2658 2659
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}

2660
static void per_cpu_ops_free(struct ftrace_ops *ops)
2661 2662 2663 2664
{
	free_percpu(ops->disabled);
}

2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676
static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
2677

2678 2679 2680 2681 2682 2683 2684
static void ftrace_startup_all(int command)
{
	update_all_ops = true;
	ftrace_startup_enable(command);
	update_all_ops = false;
}

2685
static int ftrace_startup(struct ftrace_ops *ops, int command)
2686
{
2687
	int ret;
2688

2689
	if (unlikely(ftrace_disabled))
2690
		return -ENODEV;
2691

2692 2693 2694 2695
	ret = __register_ftrace_function(ops);
	if (ret)
		return ret;

2696
	ftrace_start_up++;
2697

2698 2699 2700 2701 2702 2703 2704 2705 2706
	/*
	 * Note that ftrace probes uses this to start up
	 * and modify functions it will probe. But we still
	 * set the ADDING flag for modification, as probes
	 * do not have trampolines. If they add them in the
	 * future, then the probes will need to distinguish
	 * between adding and updating probes.
	 */
	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2707

2708 2709 2710 2711 2712 2713 2714 2715 2716
	ret = ftrace_hash_ipmodify_enable(ops);
	if (ret < 0) {
		/* Rollback registration process */
		__unregister_ftrace_function(ops);
		ftrace_start_up--;
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
		return ret;
	}

2717 2718
	if (ftrace_hash_rec_enable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2719

2720
	ftrace_startup_enable(command);
2721

2722 2723
	ops->flags &= ~FTRACE_OPS_FL_ADDING;

2724
	return 0;
2725 2726
}

2727
static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2728
{
2729
	int ret;
2730

2731
	if (unlikely(ftrace_disabled))
2732 2733 2734 2735 2736
		return -ENODEV;

	ret = __unregister_ftrace_function(ops);
	if (ret)
		return ret;
2737

2738
	ftrace_start_up--;
2739 2740 2741 2742 2743 2744 2745
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

2746 2747
	/* Disabling ipmodify never fails */
	ftrace_hash_ipmodify_disable(ops);
2748

2749 2750
	if (ftrace_hash_rec_disable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2751

2752
	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2753

2754 2755 2756 2757
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
2758

2759 2760
	if (!command || !ftrace_enabled) {
		/*
2761
		 * If these are per_cpu ops, they still need their
2762 2763 2764 2765
		 * per_cpu field freed. Since, function tracing is
		 * not currently active, we can just free them
		 * without synchronizing all CPUs.
		 */
2766 2767
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2768
		return 0;
2769
	}
2770

2771 2772 2773 2774
	/*
	 * If the ops uses a trampoline, then it needs to be
	 * tested first on update.
	 */
2775
	ops->flags |= FTRACE_OPS_FL_REMOVING;
2776 2777
	removed_ops = ops;

2778 2779 2780 2781
	/* The trampoline logic checks the old hashes */
	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;

2782
	ftrace_run_update_code(command);
2783

2784 2785 2786 2787 2788 2789 2790 2791 2792
	/*
	 * If there's no more ops registered with ftrace, run a
	 * sanity check to make sure all rec flags are cleared.
	 */
	if (ftrace_ops_list == &ftrace_list_end) {
		struct ftrace_page *pg;
		struct dyn_ftrace *rec;

		do_for_each_ftrace_rec(pg, rec) {
2793
			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2794 2795 2796 2797 2798
				pr_warn("  %pS flags:%lx\n",
					(void *)rec->ip, rec->flags);
		} while_for_each_ftrace_rec();
	}

2799 2800 2801 2802
	ops->old_hash.filter_hash = NULL;
	ops->old_hash.notrace_hash = NULL;

	removed_ops = NULL;
2803
	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2804

2805 2806 2807
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
2808
	 * The same goes for freeing the per_cpu data of the per_cpu
2809 2810 2811 2812 2813 2814 2815 2816 2817 2818
	 * ops.
	 *
	 * Again, normal synchronize_sched() is not good enough.
	 * We need to do a hard force of sched synchronization.
	 * This is because we use preempt_disable() to do RCU, but
	 * the function tracers can be called where RCU is not watching
	 * (like before user_exit()). We can not rely on the RCU
	 * infrastructure to do the synchronization, thus we must do it
	 * ourselves.
	 */
2819
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) {
2820 2821
		schedule_on_each_cpu(ftrace_sync);

2822 2823
		arch_ftrace_trampoline_free(ops);

2824 2825
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2826 2827
	}

2828
	return 0;
2829 2830
}

I
Ingo Molnar 已提交
2831
static void ftrace_startup_sysctl(void)
2832
{
2833 2834
	int command;

2835 2836 2837
	if (unlikely(ftrace_disabled))
		return;

2838 2839
	/* Force update next time */
	saved_ftrace_func = NULL;
2840
	/* ftrace_start_up is true if we want ftrace running */
2841 2842 2843 2844
	if (ftrace_start_up) {
		command = FTRACE_UPDATE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_START_FUNC_RET;
2845
		ftrace_startup_enable(command);
2846
	}
2847 2848
}

I
Ingo Molnar 已提交
2849
static void ftrace_shutdown_sysctl(void)
2850
{
2851 2852
	int command;

2853 2854 2855
	if (unlikely(ftrace_disabled))
		return;

2856
	/* ftrace_start_up is true if ftrace is running */
2857 2858 2859 2860 2861 2862
	if (ftrace_start_up) {
		command = FTRACE_DISABLE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_STOP_FUNC_RET;
		ftrace_run_update_code(command);
	}
2863 2864
}

2865
static u64		ftrace_update_time;
2866 2867
unsigned long		ftrace_update_tot_cnt;

2868
static inline int ops_traces_mod(struct ftrace_ops *ops)
2869
{
2870 2871 2872 2873
	/*
	 * Filter_hash being empty will default to trace module.
	 * But notrace hash requires a test of individual module functions.
	 */
2874 2875
	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
		ftrace_hash_empty(ops->func_hash->notrace_hash);
2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891
}

/*
 * Check if the current ops references the record.
 *
 * If the ops traces all functions, then it was already accounted for.
 * If the ops does not trace the current record function, skip it.
 * If the ops ignores the function via notrace filter, skip it.
 */
static inline bool
ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	/* If ops isn't enabled, ignore it */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

2892
	/* If ops traces all then it includes this function */
2893
	if (ops_traces_mod(ops))
2894
		return 1;
2895 2896

	/* The function must be in the filter */
2897
	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2898
	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2899
		return 0;
2900

2901
	/* If in notrace hash, we ignore it too */
2902
	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2903 2904 2905 2906 2907
		return 0;

	return 1;
}

2908
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2909
{
2910
	struct ftrace_page *pg;
2911
	struct dyn_ftrace *p;
2912
	u64 start, stop;
2913
	unsigned long update_cnt = 0;
2914
	unsigned long rec_flags = 0;
2915
	int i;
2916

2917 2918
	start = ftrace_now(raw_smp_processor_id());

2919
	/*
2920 2921 2922 2923 2924 2925 2926 2927 2928
	 * When a module is loaded, this function is called to convert
	 * the calls to mcount in its text to nops, and also to create
	 * an entry in the ftrace data. Now, if ftrace is activated
	 * after this call, but before the module sets its text to
	 * read-only, the modification of enabling ftrace can fail if
	 * the read-only is done while ftrace is converting the calls.
	 * To prevent this, the module's records are set as disabled
	 * and will be enabled after the call to set the module's text
	 * to read-only.
2929
	 */
2930 2931
	if (mod)
		rec_flags |= FTRACE_FL_DISABLED;
2932

2933
	for (pg = new_pgs; pg; pg = pg->next) {
2934

2935
		for (i = 0; i < pg->index; i++) {
2936

2937 2938 2939
			/* If something went wrong, bail without enabling anything */
			if (unlikely(ftrace_disabled))
				return -1;
2940

2941
			p = &pg->records[i];
2942
			p->flags = rec_flags;
2943

2944 2945 2946 2947 2948 2949
			/*
			 * Do the initial record conversion from mcount jump
			 * to the NOP instructions.
			 */
			if (!ftrace_code_disable(mod, p))
				break;
2950

2951
			update_cnt++;
2952
		}
2953 2954
	}

I
Ingo Molnar 已提交
2955
	stop = ftrace_now(raw_smp_processor_id());
2956
	ftrace_update_time = stop - start;
2957
	ftrace_update_tot_cnt += update_cnt;
2958

2959 2960 2961
	return 0;
}

2962
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2963
{
2964
	int order;
2965 2966
	int cnt;

2967 2968 2969 2970
	if (WARN_ON(!count))
		return -EINVAL;

	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2971 2972

	/*
2973 2974
	 * We want to fill as much as possible. No more than a page
	 * may be empty.
2975
	 */
2976 2977
	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
		order--;
2978

2979 2980
 again:
	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2981

2982 2983 2984 2985 2986 2987 2988
	if (!pg->records) {
		/* if we can't allocate this size, try something smaller */
		if (!order)
			return -ENOMEM;
		order >>= 1;
		goto again;
	}
2989

2990 2991
	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
	pg->size = cnt;
2992

2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025
	if (cnt > count)
		cnt = count;

	return cnt;
}

static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
	struct ftrace_page *start_pg;
	struct ftrace_page *pg;
	int order;
	int cnt;

	if (!num_to_init)
		return 0;

	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
	if (!pg)
		return NULL;

	/*
	 * Try to allocate as much as possible in one continues
	 * location that fills in all of the space. We want to
	 * waste as little space as possible.
	 */
	for (;;) {
		cnt = ftrace_allocate_records(pg, num_to_init);
		if (cnt < 0)
			goto free_pages;

		num_to_init -= cnt;
		if (!num_to_init)
3026 3027
			break;

3028 3029 3030 3031
		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
		if (!pg->next)
			goto free_pages;

3032 3033 3034
		pg = pg->next;
	}

3035 3036 3037
	return start_pg;

 free_pages:
3038 3039
	pg = start_pg;
	while (pg) {
3040 3041 3042 3043 3044 3045 3046 3047 3048 3049
		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		start_pg = pg->next;
		kfree(pg);
		pg = start_pg;
	}
	pr_info("ftrace: FAILED to allocate memory for functions\n");
	return NULL;
}

3050 3051 3052
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
3053
	loff_t				pos;
3054 3055 3056 3057 3058
	loff_t				func_pos;
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
	struct trace_parser		parser;
3059
	struct ftrace_hash		*hash;
3060
	struct ftrace_ops		*ops;
3061 3062 3063
	int				hidx;
	int				idx;
	unsigned			flags;
3064 3065
};

3066
static void *
3067
t_hash_next(struct seq_file *m, loff_t *pos)
3068 3069
{
	struct ftrace_iterator *iter = m->private;
3070
	struct hlist_node *hnd = NULL;
3071 3072 3073
	struct hlist_head *hhd;

	(*pos)++;
3074
	iter->pos = *pos;
3075

3076 3077
	if (iter->probe)
		hnd = &iter->probe->node;
3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099
 retry:
	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
		return NULL;

	hhd = &ftrace_func_hash[iter->hidx];

	if (hlist_empty(hhd)) {
		iter->hidx++;
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
			iter->hidx++;
			goto retry;
		}
	}

3100 3101 3102 3103 3104 3105
	if (WARN_ON_ONCE(!hnd))
		return NULL;

	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);

	return iter;
3106 3107 3108 3109 3110 3111
}

static void *t_hash_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
3112 3113
	loff_t l;

3114 3115 3116
	if (!(iter->flags & FTRACE_ITER_DO_HASH))
		return NULL;

3117 3118
	if (iter->func_pos > *pos)
		return NULL;
3119

L
Li Zefan 已提交
3120
	iter->hidx = 0;
3121
	for (l = 0; l <= (*pos - iter->func_pos); ) {
3122
		p = t_hash_next(m, &l);
L
Li Zefan 已提交
3123 3124 3125
		if (!p)
			break;
	}
3126 3127 3128
	if (!p)
		return NULL;

3129 3130 3131
	/* Only set this if we have an item */
	iter->flags |= FTRACE_ITER_HASH;

3132
	return iter;
3133 3134
}

3135 3136
static int
t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
3137
{
S
Steven Rostedt 已提交
3138
	struct ftrace_func_probe *rec;
3139

3140 3141 3142
	rec = iter->probe;
	if (WARN_ON_ONCE(!rec))
		return -EIO;
3143

3144 3145 3146
	if (rec->ops->print)
		return rec->ops->print(m, rec->ip, rec->ops, rec->data);

3147
	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
3148 3149 3150 3151 3152 3153 3154 3155

	if (rec->data)
		seq_printf(m, ":%p", rec->data);
	seq_putc(m, '\n');

	return 0;
}

I
Ingo Molnar 已提交
3156
static void *
3157 3158 3159
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3160
	struct ftrace_ops *ops = iter->ops;
3161 3162
	struct dyn_ftrace *rec = NULL;

3163 3164 3165
	if (unlikely(ftrace_disabled))
		return NULL;

3166
	if (iter->flags & FTRACE_ITER_HASH)
3167
		return t_hash_next(m, pos);
3168

3169
	(*pos)++;
3170
	iter->pos = iter->func_pos = *pos;
3171

3172
	if (iter->flags & FTRACE_ITER_PRINTALL)
3173
		return t_hash_start(m, pos);
3174

3175 3176 3177 3178 3179 3180 3181 3182 3183
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
3184
		if (((iter->flags & FTRACE_ITER_FILTER) &&
3185
		     !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) ||
S
Steven Rostedt 已提交
3186

3187
		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
3188
		     !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) ||
3189 3190

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3191
		     !(rec->flags & FTRACE_FL_ENABLED))) {
3192

3193 3194 3195 3196 3197
			rec = NULL;
			goto retry;
		}
	}

3198
	if (!rec)
3199
		return t_hash_start(m, pos);
3200 3201 3202 3203

	iter->func = rec;

	return iter;
3204 3205
}

3206 3207 3208 3209
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
3210
	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
3211 3212 3213 3214 3215
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3216
	struct ftrace_ops *ops = iter->ops;
3217
	void *p = NULL;
3218
	loff_t l;
3219

3220
	mutex_lock(&ftrace_lock);
3221 3222 3223 3224

	if (unlikely(ftrace_disabled))
		return NULL;

3225 3226 3227 3228 3229 3230
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

3231 3232 3233 3234 3235
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
3236
	if ((iter->flags & FTRACE_ITER_FILTER &&
3237
	     ftrace_hash_empty(ops->func_hash->filter_hash)) ||
3238
	    (iter->flags & FTRACE_ITER_NOTRACE &&
3239
	     ftrace_hash_empty(ops->func_hash->notrace_hash))) {
3240
		if (*pos > 0)
3241
			return t_hash_start(m, pos);
3242
		iter->flags |= FTRACE_ITER_PRINTALL;
3243 3244
		/* reset in case of seek/pread */
		iter->flags &= ~FTRACE_ITER_HASH;
3245 3246 3247
		return iter;
	}

3248 3249 3250
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_start(m, pos);

3251 3252 3253 3254 3255
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
3256 3257 3258 3259 3260 3261
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
		p = t_next(m, p, &l);
		if (!p)
			break;
3262
	}
3263

3264 3265
	if (!p)
		return t_hash_start(m, pos);
3266 3267

	return iter;
3268 3269 3270 3271
}

static void t_stop(struct seq_file *m, void *p)
{
3272
	mutex_unlock(&ftrace_lock);
3273 3274
}

3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	return NULL;
}

static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
				struct dyn_ftrace *rec)
{
	void *ptr;

	ptr = arch_ftrace_trampoline_func(ops, rec);
	if (ptr)
		seq_printf(m, " ->%pS", ptr);
}

3291 3292
static int t_show(struct seq_file *m, void *v)
{
3293
	struct ftrace_iterator *iter = m->private;
3294
	struct dyn_ftrace *rec;
3295

3296
	if (iter->flags & FTRACE_ITER_HASH)
3297
		return t_hash_show(m, iter);
3298

3299
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3300
		if (iter->flags & FTRACE_ITER_NOTRACE)
3301
			seq_puts(m, "#### no functions disabled ####\n");
3302
		else
3303
			seq_puts(m, "#### all functions enabled ####\n");
3304 3305 3306
		return 0;
	}

3307 3308
	rec = iter->func;

3309 3310 3311
	if (!rec)
		return 0;

3312
	seq_printf(m, "%ps", (void *)rec->ip);
3313
	if (iter->flags & FTRACE_ITER_ENABLED) {
3314
		struct ftrace_ops *ops;
3315

3316
		seq_printf(m, " (%ld)%s%s",
3317
			   ftrace_rec_count(rec),
3318 3319
			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
3320
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3321
			ops = ftrace_find_tramp_ops_any(rec);
3322 3323 3324 3325 3326
			if (ops) {
				do {
					seq_printf(m, "\ttramp: %pS (%pS)",
						   (void *)ops->trampoline,
						   (void *)ops->func);
3327
					add_trampoline_func(m, ops, rec);
3328 3329 3330
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
3331
				seq_puts(m, "\ttramp: ERROR!");
3332 3333
		} else {
			add_trampoline_func(m, NULL, rec);
3334 3335 3336
		}
	}	

3337
	seq_putc(m, '\n');
3338 3339 3340 3341

	return 0;
}

J
James Morris 已提交
3342
static const struct seq_operations show_ftrace_seq_ops = {
3343 3344 3345 3346 3347 3348
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
3349
static int
3350 3351 3352 3353
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3354 3355 3356
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3357
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3358 3359
	if (!iter)
		return -ENOMEM;
3360

3361 3362 3363 3364
	iter->pg = ftrace_pages_start;
	iter->ops = &global_ops;

	return 0;
3365 3366
}

3367 3368 3369 3370 3371
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3372
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3373 3374
	if (!iter)
		return -ENOMEM;
3375

3376 3377 3378 3379 3380
	iter->pg = ftrace_pages_start;
	iter->flags = FTRACE_ITER_ENABLED;
	iter->ops = &global_ops;

	return 0;
3381 3382
}

3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
/**
 * ftrace_regex_open - initialize function tracer filter files
 * @ops: The ftrace_ops that hold the hash filters
 * @flag: The type of filter to process
 * @inode: The inode, usually passed in to your open routine
 * @file: The file, usually passed in to your open routine
 *
 * ftrace_regex_open() initializes the filter files for the
 * @ops. Depending on @flag it may process the filter hash or
 * the notrace hash of @ops. With this called from the open
 * routine, you can use ftrace_filter_write() for the write
 * routine if @flag has FTRACE_ITER_FILTER set, or
 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3396
 * tracing_lseek() should be used as the lseek routine, and
3397 3398 3399
 * release must call ftrace_regex_release().
 */
int
3400
ftrace_regex_open(struct ftrace_ops *ops, int flag,
3401
		  struct inode *inode, struct file *file)
3402 3403
{
	struct ftrace_iterator *iter;
3404
	struct ftrace_hash *hash;
3405 3406
	int ret = 0;

3407 3408
	ftrace_ops_init(ops);

3409 3410 3411
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3412 3413 3414 3415
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

3416 3417 3418 3419 3420
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

3421 3422 3423
	iter->ops = ops;
	iter->flags = flag;

3424
	mutex_lock(&ops->func_hash->regex_lock);
3425

3426
	if (flag & FTRACE_ITER_NOTRACE)
3427
		hash = ops->func_hash->notrace_hash;
3428
	else
3429
		hash = ops->func_hash->filter_hash;
3430

3431
	if (file->f_mode & FMODE_WRITE) {
3432 3433 3434 3435 3436 3437 3438
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

		if (file->f_flags & O_TRUNC)
			iter->hash = alloc_ftrace_hash(size_bits);
		else
			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);

3439 3440 3441
		if (!iter->hash) {
			trace_parser_put(&iter->parser);
			kfree(iter);
3442 3443
			ret = -ENOMEM;
			goto out_unlock;
3444 3445
		}
	}
3446

3447 3448 3449 3450 3451 3452 3453
	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
3454
		} else {
3455 3456
			/* Failed */
			free_ftrace_hash(iter->hash);
3457
			trace_parser_put(&iter->parser);
3458
			kfree(iter);
3459
		}
3460 3461
	} else
		file->private_data = iter;
3462 3463

 out_unlock:
3464
	mutex_unlock(&ops->func_hash->regex_lock);
3465 3466 3467 3468

	return ret;
}

3469 3470 3471
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
3472 3473 3474
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops,
3475 3476
			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
			inode, file);
3477 3478 3479 3480 3481
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
3482 3483 3484
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3485
				 inode, file);
3486 3487
}

3488 3489 3490 3491 3492 3493 3494
/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
struct ftrace_glob {
	char *search;
	unsigned len;
	int type;
};

3495 3496 3497 3498 3499 3500 3501 3502 3503 3504
/*
 * If symbols in an architecture don't correspond exactly to the user-visible
 * name of what they represent, it is possible to define this function to
 * perform the necessary adjustments.
*/
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
	return str;
}

3505
static int ftrace_match(char *str, struct ftrace_glob *g)
3506 3507
{
	int matched = 0;
3508
	int slen;
3509

3510 3511
	str = arch_ftrace_match_adjust(str, g->search);

3512
	switch (g->type) {
3513
	case MATCH_FULL:
3514
		if (strcmp(str, g->search) == 0)
3515 3516 3517
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
3518
		if (strncmp(str, g->search, g->len) == 0)
3519 3520 3521
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
3522
		if (strstr(str, g->search))
3523 3524 3525
			matched = 1;
		break;
	case MATCH_END_ONLY:
3526
		slen = strlen(str);
3527 3528
		if (slen >= g->len &&
		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3529 3530
			matched = 1;
		break;
3531 3532 3533 3534
	case MATCH_GLOB:
		if (glob_match(g->search, str))
			matched = 1;
		break;
3535 3536 3537 3538 3539
	}

	return matched;
}

3540
static int
3541
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3542
{
3543 3544 3545
	struct ftrace_func_entry *entry;
	int ret = 0;

3546
	entry = ftrace_lookup_ip(hash, rec->ip);
3547
	if (clear_filter) {
3548 3549 3550
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
3551

3552
		free_hash_entry(hash, entry);
3553 3554 3555 3556
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
3557

3558
		ret = add_hash_entry(hash, rec->ip);
3559 3560
	}
	return ret;
3561 3562
}

3563
static int
D
Dmitry Safonov 已提交
3564 3565
ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
		struct ftrace_glob *mod_g, int exclude_mod)
3566 3567
{
	char str[KSYM_SYMBOL_LEN];
3568 3569 3570 3571
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

D
Dmitry Safonov 已提交
3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591
	if (mod_g) {
		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;

		/* blank module name to match all modules */
		if (!mod_g->len) {
			/* blank module globbing: modname xor exclude_mod */
			if ((!exclude_mod) != (!modname))
				goto func_match;
			return 0;
		}

		/* not matching the module */
		if (!modname || !mod_matches) {
			if (exclude_mod)
				goto func_match;
			else
				return 0;
		}

		if (mod_matches && exclude_mod)
3592 3593
			return 0;

D
Dmitry Safonov 已提交
3594
func_match:
3595
		/* blank search means to match all funcs in the mod */
3596
		if (!func_g->len)
3597 3598
			return 1;
	}
3599

3600
	return ftrace_match(str, func_g);
3601 3602
}

3603
static int
3604
match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3605 3606 3607
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3608
	struct ftrace_glob func_g = { .type = MATCH_FULL };
D
Dmitry Safonov 已提交
3609 3610 3611
	struct ftrace_glob mod_g = { .type = MATCH_FULL };
	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
	int exclude_mod = 0;
3612
	int found = 0;
3613
	int ret;
3614
	int clear_filter;
3615

D
Dmitry Safonov 已提交
3616
	if (func) {
3617 3618 3619
		func_g.type = filter_parse_regex(func, len, &func_g.search,
						 &clear_filter);
		func_g.len = strlen(func_g.search);
3620
	}
3621

D
Dmitry Safonov 已提交
3622 3623 3624 3625
	if (mod) {
		mod_g.type = filter_parse_regex(mod, strlen(mod),
				&mod_g.search, &exclude_mod);
		mod_g.len = strlen(mod_g.search);
3626
	}
3627

3628
	mutex_lock(&ftrace_lock);
3629

3630 3631
	if (unlikely(ftrace_disabled))
		goto out_unlock;
3632

3633
	do_for_each_ftrace_rec(pg, rec) {
3634 3635 3636 3637

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
3638
		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3639
			ret = enter_record(hash, rec, clear_filter);
3640 3641 3642 3643
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
3644
			found = 1;
3645 3646
		}
	} while_for_each_ftrace_rec();
3647
 out_unlock:
3648
	mutex_unlock(&ftrace_lock);
3649 3650

	return found;
3651 3652
}

3653
static int
3654
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3655
{
3656
	return match_records(hash, buff, len, NULL);
3657 3658 3659
}


3660 3661 3662 3663 3664 3665
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
3666
ftrace_mod_callback(struct ftrace_hash *hash,
3667
		    char *func, char *cmd, char *module, int enable)
3668
{
3669
	int ret;
3670 3671 3672 3673 3674 3675 3676 3677

	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */
3678
	ret = match_records(hash, func, strlen(func), module);
3679
	if (!ret)
3680
		return -EINVAL;
3681 3682 3683
	if (ret < 0)
		return ret;
	return 0;
3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
3695
core_initcall(ftrace_mod_cmd_init);
3696

3697
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
3698
				      struct ftrace_ops *op, struct pt_regs *pt_regs)
3699
{
S
Steven Rostedt 已提交
3700
	struct ftrace_func_probe *entry;
3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715
	struct hlist_head *hhd;
	unsigned long key;

	key = hash_long(ip, FTRACE_HASH_BITS);

	hhd = &ftrace_func_hash[key];

	if (hlist_empty(hhd))
		return;

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
3716
	preempt_disable_notrace();
3717
	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
3718 3719 3720
		if (entry->ip == ip)
			entry->ops->func(ip, parent_ip, &entry->data);
	}
3721
	preempt_enable_notrace();
3722 3723
}

S
Steven Rostedt 已提交
3724
static struct ftrace_ops trace_probe_ops __read_mostly =
3725
{
3726
	.func		= function_trace_probe_call,
3727
	.flags		= FTRACE_OPS_FL_INITIALIZED,
3728
	INIT_OPS_HASH(trace_probe_ops)
3729 3730
};

S
Steven Rostedt 已提交
3731
static int ftrace_probe_registered;
3732

3733
static void __enable_ftrace_function_probe(struct ftrace_ops_hash *old_hash)
3734
{
3735
	int ret;
3736 3737
	int i;

3738 3739 3740
	if (ftrace_probe_registered) {
		/* still need to update the function call sites */
		if (ftrace_enabled)
3741 3742
			ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS,
					       old_hash);
3743
		return;
3744
	}
3745 3746 3747 3748 3749 3750 3751 3752 3753 3754

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			break;
	}
	/* Nothing registered? */
	if (i == FTRACE_FUNC_HASHSIZE)
		return;

3755
	ret = ftrace_startup(&trace_probe_ops, 0);
3756

S
Steven Rostedt 已提交
3757
	ftrace_probe_registered = 1;
3758 3759
}

S
Steven Rostedt 已提交
3760
static void __disable_ftrace_function_probe(void)
3761 3762 3763
{
	int i;

S
Steven Rostedt 已提交
3764
	if (!ftrace_probe_registered)
3765 3766 3767 3768 3769 3770 3771 3772 3773
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			return;
	}

	/* no more funcs left */
3774
	ftrace_shutdown(&trace_probe_ops, 0);
3775

S
Steven Rostedt 已提交
3776
	ftrace_probe_registered = 0;
3777 3778 3779
}


3780
static void ftrace_free_entry(struct ftrace_func_probe *entry)
3781 3782
{
	if (entry->ops->free)
3783
		entry->ops->free(entry->ops, entry->ip, &entry->data);
3784 3785 3786 3787
	kfree(entry);
}

int
S
Steven Rostedt 已提交
3788
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3789 3790
			      void *data)
{
3791
	struct ftrace_ops_hash old_hash_ops;
S
Steven Rostedt 已提交
3792
	struct ftrace_func_probe *entry;
3793
	struct ftrace_glob func_g;
3794
	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
3795
	struct ftrace_hash *old_hash = *orig_hash;
3796
	struct ftrace_hash *hash;
3797 3798
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3799
	int not;
S
Steven Rostedt 已提交
3800
	unsigned long key;
3801
	int count = 0;
3802
	int ret;
3803

3804 3805 3806
	func_g.type = filter_parse_regex(glob, strlen(glob),
			&func_g.search, &not);
	func_g.len = strlen(func_g.search);
3807

S
Steven Rostedt 已提交
3808
	/* we do not support '!' for function probes */
3809 3810 3811
	if (WARN_ON(not))
		return -EINVAL;

3812
	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
3813

3814 3815 3816 3817
	old_hash_ops.filter_hash = old_hash;
	/* Probes only have filters */
	old_hash_ops.notrace_hash = NULL;

3818
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
3819 3820
	if (!hash) {
		count = -ENOMEM;
3821
		goto out;
3822 3823 3824 3825
	}

	if (unlikely(ftrace_disabled)) {
		count = -ENODEV;
3826
		goto out;
3827
	}
3828

3829 3830
	mutex_lock(&ftrace_lock);

3831
	do_for_each_ftrace_rec(pg, rec) {
3832

3833 3834 3835
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
3836
		if (!ftrace_match_record(rec, &func_g, NULL, 0))
3837 3838 3839 3840
			continue;

		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
		if (!entry) {
S
Steven Rostedt 已提交
3841
			/* If we did not process any, then return error */
3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855
			if (!count)
				count = -ENOMEM;
			goto out_unlock;
		}

		count++;

		entry->data = data;

		/*
		 * The caller might want to do something special
		 * for each function we find. We call the callback
		 * to give the caller an opportunity to do so.
		 */
3856 3857
		if (ops->init) {
			if (ops->init(ops, rec->ip, &entry->data) < 0) {
3858 3859 3860 3861 3862 3863
				/* caller does not like this func */
				kfree(entry);
				continue;
			}
		}

3864 3865 3866 3867 3868 3869 3870
		ret = enter_record(hash, rec, 0);
		if (ret < 0) {
			kfree(entry);
			count = ret;
			goto out_unlock;
		}

3871 3872 3873 3874 3875 3876 3877
		entry->ops = ops;
		entry->ip = rec->ip;

		key = hash_long(entry->ip, FTRACE_HASH_BITS);
		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);

	} while_for_each_ftrace_rec();
3878 3879

	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3880

3881
	__enable_ftrace_function_probe(&old_hash_ops);
3882

3883 3884 3885
	if (!ret)
		free_ftrace_hash_rcu(old_hash);
	else
3886 3887
		count = ret;

3888
 out_unlock:
3889 3890
	mutex_unlock(&ftrace_lock);
 out:
3891
	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
3892
	free_ftrace_hash(hash);
3893 3894 3895 3896 3897

	return count;
}

enum {
S
Steven Rostedt 已提交
3898 3899
	PROBE_TEST_FUNC		= 1,
	PROBE_TEST_DATA		= 2
3900 3901 3902
};

static void
S
Steven Rostedt 已提交
3903
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3904 3905
				  void *data, int flags)
{
3906
	struct ftrace_func_entry *rec_entry;
S
Steven Rostedt 已提交
3907
	struct ftrace_func_probe *entry;
3908
	struct ftrace_func_probe *p;
3909
	struct ftrace_glob func_g;
3910
	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
3911
	struct ftrace_hash *old_hash = *orig_hash;
3912
	struct list_head free_list;
3913
	struct ftrace_hash *hash;
3914
	struct hlist_node *tmp;
3915
	char str[KSYM_SYMBOL_LEN];
3916
	int i, ret;
3917

3918
	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
3919
		func_g.search = NULL;
3920
	else if (glob) {
3921 3922
		int not;

3923 3924 3925 3926
		func_g.type = filter_parse_regex(glob, strlen(glob),
						 &func_g.search, &not);
		func_g.len = strlen(func_g.search);
		func_g.search = glob;
3927

S
Steven Rostedt 已提交
3928
		/* we do not support '!' for function probes */
3929 3930 3931 3932
		if (WARN_ON(not))
			return;
	}

3933
	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
3934 3935 3936 3937 3938 3939

	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
	if (!hash)
		/* Hmm, should report this somehow */
		goto out_unlock;

3940 3941
	INIT_LIST_HEAD(&free_list);

3942 3943 3944
	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];

3945
		hlist_for_each_entry_safe(entry, tmp, hhd, node) {
3946 3947

			/* break up if statements for readability */
S
Steven Rostedt 已提交
3948
			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3949 3950
				continue;

S
Steven Rostedt 已提交
3951
			if ((flags & PROBE_TEST_DATA) && entry->data != data)
3952 3953 3954
				continue;

			/* do this last, since it is the most expensive */
3955
			if (func_g.search) {
3956 3957
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
3958
				if (!ftrace_match(str, &func_g))
3959 3960 3961
					continue;
			}

3962 3963 3964 3965 3966
			rec_entry = ftrace_lookup_ip(hash, entry->ip);
			/* It is possible more than one entry had this ip */
			if (rec_entry)
				free_hash_entry(hash, rec_entry);

3967
			hlist_del_rcu(&entry->node);
3968
			list_add(&entry->free_list, &free_list);
3969 3970
		}
	}
3971
	mutex_lock(&ftrace_lock);
S
Steven Rostedt 已提交
3972
	__disable_ftrace_function_probe();
3973 3974 3975 3976
	/*
	 * Remove after the disable is called. Otherwise, if the last
	 * probe is removed, a null hash means *all enabled*.
	 */
3977
	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3978
	synchronize_sched();
3979 3980 3981
	if (!ret)
		free_ftrace_hash_rcu(old_hash);

3982 3983 3984 3985
	list_for_each_entry_safe(entry, p, &free_list, free_list) {
		list_del(&entry->free_list);
		ftrace_free_entry(entry);
	}
3986
	mutex_unlock(&ftrace_lock);
3987

3988
 out_unlock:
3989
	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
3990
	free_ftrace_hash(hash);
3991 3992 3993
}

void
S
Steven Rostedt 已提交
3994
unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3995 3996
				void *data)
{
S
Steven Rostedt 已提交
3997 3998
	__unregister_ftrace_function_probe(glob, ops, data,
					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
3999 4000 4001
}

void
S
Steven Rostedt 已提交
4002
unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
4003
{
S
Steven Rostedt 已提交
4004
	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
4005 4006
}

S
Steven Rostedt 已提交
4007
void unregister_ftrace_function_probe_all(char *glob)
4008
{
S
Steven Rostedt 已提交
4009
	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
4010 4011
}

4012 4013 4014
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

4015 4016 4017 4018 4019
/*
 * Currently we only register ftrace commands from __init, so mark this
 * __init too.
 */
__init int register_ftrace_command(struct ftrace_func_command *cmd)
4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4038 4039 4040 4041 4042
/*
 * Currently we only unregister ftrace commands from __init, so mark
 * this __init too.
 */
__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4061 4062
static int ftrace_process_regex(struct ftrace_hash *hash,
				char *buff, int len, int enable)
4063
{
4064
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
4065
	struct ftrace_func_command *p;
4066
	int ret = -EINVAL;
4067 4068 4069 4070

	func = strsep(&next, ":");

	if (!next) {
4071
		ret = ftrace_match_records(hash, func, len);
4072 4073 4074 4075 4076
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
4077 4078
	}

4079
	/* command found */
4080 4081 4082

	command = strsep(&next, ":");

4083 4084 4085
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
4086
			ret = p->func(hash, func, command, next, enable);
4087 4088
			goto out_unlock;
		}
4089
	}
4090 4091
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
4092

4093
	return ret;
4094 4095
}

I
Ingo Molnar 已提交
4096
static ssize_t
4097 4098
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
4099 4100
{
	struct ftrace_iterator *iter;
4101 4102
	struct trace_parser *parser;
	ssize_t ret, read;
4103

4104
	if (!cnt)
4105 4106 4107 4108 4109 4110 4111 4112
		return 0;

	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

4113
	if (unlikely(ftrace_disabled))
4114 4115 4116
		return -ENODEV;

	/* iter->hash is a local copy, so we don't need regex_lock */
4117

4118 4119
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
4120

4121
	if (read >= 0 && trace_parser_loaded(parser) &&
4122
	    !trace_parser_cont(parser)) {
4123
		ret = ftrace_process_regex(iter->hash, parser->buffer,
4124
					   parser->idx, enable);
4125
		trace_parser_clear(parser);
4126
		if (ret < 0)
4127
			goto out;
4128
	}
4129 4130

	ret = read;
4131
 out:
4132 4133 4134
	return ret;
}

4135
ssize_t
4136 4137 4138 4139 4140 4141
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

4142
ssize_t
4143 4144 4145 4146 4147 4148
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

4149
static int
4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167
ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
	struct ftrace_func_entry *entry;

	if (!ftrace_location(ip))
		return -EINVAL;

	if (remove) {
		entry = ftrace_lookup_ip(hash, ip);
		if (!entry)
			return -ENOENT;
		free_hash_entry(hash, entry);
		return 0;
	}

	return add_hash_entry(hash, ip);
}

4168
static void ftrace_ops_update_code(struct ftrace_ops *ops,
4169
				   struct ftrace_ops_hash *old_hash)
4170
{
4171 4172 4173 4174 4175 4176
	struct ftrace_ops *op;

	if (!ftrace_enabled)
		return;

	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
4177
		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196
		return;
	}

	/*
	 * If this is the shared global_ops filter, then we need to
	 * check if there is another ops that shares it, is enabled.
	 * If so, we still need to run the modify code.
	 */
	if (ops->func_hash != &global_ops.local_hash)
		return;

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->func_hash == &global_ops.local_hash &&
		    op->flags & FTRACE_OPS_FL_ENABLED) {
			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
			/* Only need to do this once */
			return;
		}
	} while_for_each_ftrace_op(op);
4197 4198
}

4199 4200 4201
static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
		unsigned long ip, int remove, int reset, int enable)
4202
{
4203
	struct ftrace_hash **orig_hash;
4204
	struct ftrace_ops_hash old_hash_ops;
4205
	struct ftrace_hash *old_hash;
4206
	struct ftrace_hash *hash;
4207
	int ret;
4208

4209
	if (unlikely(ftrace_disabled))
4210
		return -ENODEV;
4211

4212
	mutex_lock(&ops->func_hash->regex_lock);
4213

4214
	if (enable)
4215
		orig_hash = &ops->func_hash->filter_hash;
4216
	else
4217
		orig_hash = &ops->func_hash->notrace_hash;
4218

4219 4220 4221 4222 4223
	if (reset)
		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	else
		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);

4224 4225 4226 4227
	if (!hash) {
		ret = -ENOMEM;
		goto out_regex_unlock;
	}
4228

4229 4230 4231 4232
	if (buf && !ftrace_match_records(hash, buf, len)) {
		ret = -EINVAL;
		goto out_regex_unlock;
	}
4233 4234 4235 4236 4237
	if (ip) {
		ret = ftrace_match_addr(hash, ip, remove);
		if (ret < 0)
			goto out_regex_unlock;
	}
4238 4239

	mutex_lock(&ftrace_lock);
4240
	old_hash = *orig_hash;
4241 4242
	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
4243
	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
4244
	if (!ret) {
4245
		ftrace_ops_update_code(ops, &old_hash_ops);
4246 4247
		free_ftrace_hash_rcu(old_hash);
	}
4248 4249
	mutex_unlock(&ftrace_lock);

4250
 out_regex_unlock:
4251
	mutex_unlock(&ops->func_hash->regex_lock);
4252 4253 4254

	free_ftrace_hash(hash);
	return ret;
4255 4256
}

4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
		int reset, int enable)
{
	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
}

/**
 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
 * @ops - the ops to set the filter with
 * @ip - the address to add to or remove from the filter.
 * @remove - non zero to remove the ip from the filter
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled
 * If @ip is NULL, it failes to update filter.
 */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
			 int remove, int reset)
{
4277
	ftrace_ops_init(ops);
4278 4279 4280 4281
	return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);

4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298
/**
 * ftrace_ops_set_global_filter - setup ops to use global filters
 * @ops - the ops which will use the global filters
 *
 * ftrace users who need global function trace filtering should call this.
 * It can set the global filter only if ops were not initialized before.
 */
void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
{
	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
		return;

	ftrace_ops_init(ops);
	ops->func_hash = &global_ops.local_hash;
}
EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);

4299 4300 4301 4302 4303 4304 4305
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
{
	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
}

4306 4307
/**
 * ftrace_set_filter - set a function to filter on in ftrace
4308 4309 4310 4311 4312 4313 4314 4315
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4316
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4317 4318
		       int len, int reset)
{
4319
	ftrace_ops_init(ops);
4320
	return ftrace_set_regex(ops, buf, len, reset, 1);
4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4335
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4336 4337
			int len, int reset)
{
4338
	ftrace_ops_init(ops);
4339
	return ftrace_set_regex(ops, buf, len, reset, 0);
4340 4341 4342
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
4343
 * ftrace_set_global_filter - set a function to filter on with global tracers
4344 4345 4346 4347 4348 4349 4350
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4351
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4352
{
4353
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4354
}
4355
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4356

4357
/**
4358
 * ftrace_set_global_notrace - set a function to not trace with global tracers
4359 4360 4361 4362 4363 4364 4365 4366
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4367
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4368
{
4369
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4370
}
4371
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4372

4373 4374 4375 4376 4377 4378 4379
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

4380 4381 4382
/* Used by function selftest to not test if filter is set */
bool ftrace_filter_param __initdata;

4383 4384
static int __init set_ftrace_notrace(char *str)
{
4385
	ftrace_filter_param = true;
4386
	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4387 4388 4389 4390 4391 4392
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
4393
	ftrace_filter_param = true;
4394
	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4395 4396 4397 4398
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

4399
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4400
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4401
static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4402
static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
4403

4404 4405 4406
static unsigned long save_global_trampoline;
static unsigned long save_global_flags;

4407 4408
static int __init set_graph_function(char *str)
{
4409
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4410 4411 4412 4413
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

4414 4415 4416 4417 4418 4419 4420
static int __init set_graph_notrace_function(char *str)
{
	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);

4421 4422 4423 4424 4425 4426 4427 4428
static int __init set_graph_max_depth_function(char *str)
{
	if (!str)
		return 0;
	fgraph_max_depth = simple_strtoul(str, NULL, 0);
	return 1;
}
__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
4429 4430

static void __init set_ftrace_early_graph(char *buf, int enable)
4431 4432 4433
{
	int ret;
	char *func;
4434
	struct ftrace_hash *hash;
4435

4436 4437 4438
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	if (WARN_ON(!hash))
		return;
4439 4440 4441 4442

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
4443
		ret = ftrace_graph_set_hash(hash, func);
4444 4445 4446 4447
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
4448 4449 4450 4451 4452

	if (enable)
		ftrace_graph_hash = hash;
	else
		ftrace_graph_notrace_hash = hash;
4453 4454 4455
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4456 4457
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4458 4459 4460
{
	char *func;

4461 4462
	ftrace_ops_init(ops);

4463 4464
	while (buf) {
		func = strsep(&buf, ",");
4465
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
4466 4467 4468 4469 4470 4471
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
4472
		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
4473
	if (ftrace_notrace_buf[0])
4474
		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
4475 4476
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
4477 4478 4479
		set_ftrace_early_graph(ftrace_graph_buf, 1);
	if (ftrace_graph_notrace_buf[0])
		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
4480
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4481 4482
}

4483
int ftrace_regex_release(struct inode *inode, struct file *file)
4484 4485
{
	struct seq_file *m = (struct seq_file *)file->private_data;
4486
	struct ftrace_ops_hash old_hash_ops;
4487
	struct ftrace_iterator *iter;
4488
	struct ftrace_hash **orig_hash;
4489
	struct ftrace_hash *old_hash;
4490
	struct trace_parser *parser;
4491
	int filter_hash;
4492
	int ret;
4493 4494 4495 4496 4497 4498 4499

	if (file->f_mode & FMODE_READ) {
		iter = m->private;
		seq_release(inode, file);
	} else
		iter = file->private_data;

4500 4501 4502
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
4503
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
4504 4505
	}

4506 4507
	trace_parser_put(parser);

4508
	mutex_lock(&iter->ops->func_hash->regex_lock);
4509

4510
	if (file->f_mode & FMODE_WRITE) {
4511 4512 4513
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

		if (filter_hash)
4514
			orig_hash = &iter->ops->func_hash->filter_hash;
4515
		else
4516
			orig_hash = &iter->ops->func_hash->notrace_hash;
4517

4518
		mutex_lock(&ftrace_lock);
4519
		old_hash = *orig_hash;
4520 4521
		old_hash_ops.filter_hash = iter->ops->func_hash->filter_hash;
		old_hash_ops.notrace_hash = iter->ops->func_hash->notrace_hash;
4522 4523
		ret = ftrace_hash_move(iter->ops, filter_hash,
				       orig_hash, iter->hash);
4524
		if (!ret) {
4525
			ftrace_ops_update_code(iter->ops, &old_hash_ops);
4526 4527
			free_ftrace_hash_rcu(old_hash);
		}
4528 4529
		mutex_unlock(&ftrace_lock);
	}
4530

4531
	mutex_unlock(&iter->ops->func_hash->regex_lock);
4532 4533
	free_ftrace_hash(iter->hash);
	kfree(iter);
4534

4535 4536 4537
	return 0;
}

4538
static const struct file_operations ftrace_avail_fops = {
4539 4540 4541
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
4542
	.release = seq_release_private,
4543 4544
};

4545 4546 4547 4548 4549 4550 4551
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

4552
static const struct file_operations ftrace_filter_fops = {
4553
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
4554
	.read = seq_read,
4555
	.write = ftrace_filter_write,
4556
	.llseek = tracing_lseek,
4557
	.release = ftrace_regex_release,
4558 4559
};

4560
static const struct file_operations ftrace_notrace_fops = {
4561
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
4562
	.read = seq_read,
4563
	.write = ftrace_notrace_write,
4564
	.llseek = tracing_lseek,
4565
	.release = ftrace_regex_release,
4566 4567
};

4568 4569 4570 4571
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

4572 4573 4574 4575 4576 4577 4578
struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH;
struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH;

enum graph_filter_type {
	GRAPH_FILTER_NOTRACE	= 0,
	GRAPH_FILTER_FUNCTION,
};
4579

4580 4581
#define FTRACE_GRAPH_EMPTY	((void *)1)

4582
struct ftrace_graph_data {
4583 4584 4585 4586 4587 4588 4589
	struct ftrace_hash		*hash;
	struct ftrace_func_entry	*entry;
	int				idx;   /* for hash table iteration */
	enum graph_filter_type		type;
	struct ftrace_hash		*new_hash;
	const struct seq_operations	*seq_ops;
	struct trace_parser		parser;
4590 4591
};

4592
static void *
4593
__g_next(struct seq_file *m, loff_t *pos)
4594
{
4595
	struct ftrace_graph_data *fgd = m->private;
4596 4597 4598
	struct ftrace_func_entry *entry = fgd->entry;
	struct hlist_head *head;
	int i, idx = fgd->idx;
4599

4600
	if (*pos >= fgd->hash->count)
4601
		return NULL;
4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620

	if (entry) {
		hlist_for_each_entry_continue(entry, hlist) {
			fgd->entry = entry;
			return entry;
		}

		idx++;
	}

	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
		head = &fgd->hash->buckets[i];
		hlist_for_each_entry(entry, head, hlist) {
			fgd->entry = entry;
			fgd->idx = i;
			return entry;
		}
	}
	return NULL;
4621
}
4622

4623 4624 4625 4626 4627
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
4628 4629 4630 4631
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
4632 4633
	struct ftrace_graph_data *fgd = m->private;

4634 4635
	mutex_lock(&graph_lock);

4636 4637 4638 4639 4640 4641 4642
	if (fgd->type == GRAPH_FILTER_FUNCTION)
		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
	else
		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));

4643
	/* Nothing, tell g_show to print all functions are enabled */
4644
	if (ftrace_hash_empty(fgd->hash) && !*pos)
4645
		return FTRACE_GRAPH_EMPTY;
4646

4647 4648
	fgd->idx = 0;
	fgd->entry = NULL;
4649
	return __g_next(m, pos);
4650 4651 4652 4653 4654 4655 4656 4657 4658
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
4659
	struct ftrace_func_entry *entry = v;
4660

4661
	if (!entry)
4662 4663
		return 0;

4664
	if (entry == FTRACE_GRAPH_EMPTY) {
4665 4666
		struct ftrace_graph_data *fgd = m->private;

4667
		if (fgd->type == GRAPH_FILTER_FUNCTION)
4668
			seq_puts(m, "#### all functions enabled ####\n");
4669
		else
4670
			seq_puts(m, "#### no functions disabled ####\n");
4671 4672 4673
		return 0;
	}

4674
	seq_printf(m, "%ps\n", (void *)entry->ip);
4675 4676 4677 4678

	return 0;
}

J
James Morris 已提交
4679
static const struct seq_operations ftrace_graph_seq_ops = {
4680 4681 4682 4683 4684 4685 4686
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
4687 4688
__ftrace_graph_open(struct inode *inode, struct file *file,
		    struct ftrace_graph_data *fgd)
4689 4690
{
	int ret = 0;
4691
	struct ftrace_hash *new_hash = NULL;
4692

4693 4694 4695
	if (file->f_mode & FMODE_WRITE) {
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

4696 4697 4698
		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
			return -ENOMEM;

4699 4700 4701 4702 4703 4704 4705 4706 4707
		if (file->f_flags & O_TRUNC)
			new_hash = alloc_ftrace_hash(size_bits);
		else
			new_hash = alloc_and_copy_ftrace_hash(size_bits,
							      fgd->hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}
4708 4709
	}

4710
	if (file->f_mode & FMODE_READ) {
4711
		ret = seq_open(file, &ftrace_graph_seq_ops);
4712 4713 4714
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = fgd;
4715 4716 4717 4718
		} else {
			/* Failed */
			free_ftrace_hash(new_hash);
			new_hash = NULL;
4719 4720 4721
		}
	} else
		file->private_data = fgd;
4722

4723
out:
4724 4725 4726
	if (ret < 0 && file->f_mode & FMODE_WRITE)
		trace_parser_put(&fgd->parser);

4727
	fgd->new_hash = new_hash;
4728 4729 4730 4731 4732 4733 4734 4735

	/*
	 * All uses of fgd->hash must be taken with the graph_lock
	 * held. The graph_lock is going to be released, so force
	 * fgd->hash to be reinitialized when it is taken again.
	 */
	fgd->hash = NULL;

4736 4737 4738
	return ret;
}

4739 4740 4741 4742
static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
4743
	int ret;
4744 4745 4746 4747 4748 4749 4750 4751

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

4752 4753
	mutex_lock(&graph_lock);

4754 4755
	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
4756
	fgd->type = GRAPH_FILTER_FUNCTION;
4757 4758
	fgd->seq_ops = &ftrace_graph_seq_ops;

4759 4760 4761 4762 4763 4764
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
4765 4766
}

4767 4768 4769 4770
static int
ftrace_graph_notrace_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
4771
	int ret;
4772 4773 4774 4775 4776 4777 4778 4779

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

4780 4781
	mutex_lock(&graph_lock);

4782 4783
	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
4784
	fgd->type = GRAPH_FILTER_NOTRACE;
4785 4786
	fgd->seq_ops = &ftrace_graph_seq_ops;

4787 4788 4789 4790 4791 4792
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
4793 4794
}

4795 4796 4797
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
4798
	struct ftrace_graph_data *fgd;
4799 4800 4801
	struct ftrace_hash *old_hash, *new_hash;
	struct trace_parser *parser;
	int ret = 0;
4802

4803 4804 4805
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;

4806
		fgd = m->private;
4807
		seq_release(inode, file);
4808
	} else {
4809
		fgd = file->private_data;
4810 4811
	}

4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851

	if (file->f_mode & FMODE_WRITE) {

		parser = &fgd->parser;

		if (trace_parser_loaded((parser))) {
			parser->buffer[parser->idx] = 0;
			ret = ftrace_graph_set_hash(fgd->new_hash,
						    parser->buffer);
		}

		trace_parser_put(parser);

		new_hash = __ftrace_hash_move(fgd->new_hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}

		mutex_lock(&graph_lock);

		if (fgd->type == GRAPH_FILTER_FUNCTION) {
			old_hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_hash, new_hash);
		} else {
			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
		}

		mutex_unlock(&graph_lock);

		/* Wait till all users are no longer using the old hash */
		synchronize_sched();

		free_ftrace_hash(old_hash);
	}

 out:
4852 4853 4854
	kfree(fgd->new_hash);
	kfree(fgd);

4855
	return ret;
4856 4857
}

4858
static int
4859
ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
4860
{
4861
	struct ftrace_glob func_g;
4862 4863
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
4864
	struct ftrace_func_entry *entry;
4865
	int fail = 1;
4866
	int not;
4867

4868
	/* decode regex */
4869 4870
	func_g.type = filter_parse_regex(buffer, strlen(buffer),
					 &func_g.search, &not);
4871

4872
	func_g.len = strlen(func_g.search);
4873

4874
	mutex_lock(&ftrace_lock);
4875 4876 4877 4878 4879 4880

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

4881 4882
	do_for_each_ftrace_rec(pg, rec) {

4883 4884 4885
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
4886
		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
4887
			entry = ftrace_lookup_ip(hash, rec->ip);
4888 4889 4890

			if (!not) {
				fail = 0;
4891 4892 4893 4894 4895

				if (entry)
					continue;
				if (add_hash_entry(hash, rec->ip) < 0)
					goto out;
4896
			} else {
4897 4898
				if (entry) {
					free_hash_entry(hash, entry);
4899 4900 4901
					fail = 0;
				}
			}
4902
		}
4903
	} while_for_each_ftrace_rec();
4904
out:
4905
	mutex_unlock(&ftrace_lock);
4906

4907 4908 4909 4910
	if (fail)
		return -EINVAL;

	return 0;
4911 4912 4913 4914 4915 4916
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
4917
	ssize_t read, ret = 0;
4918
	struct ftrace_graph_data *fgd = file->private_data;
4919
	struct trace_parser *parser;
4920

4921
	if (!cnt)
4922 4923
		return 0;

4924 4925 4926 4927 4928 4929
	/* Read mode uses seq functions */
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		fgd = m->private;
	}

4930
	parser = &fgd->parser;
4931

4932
	read = trace_get_user(parser, ubuf, cnt, ppos);
4933

4934 4935
	if (read >= 0 && trace_parser_loaded(parser) &&
	    !trace_parser_cont(parser)) {
4936

4937
		ret = ftrace_graph_set_hash(fgd->new_hash,
4938 4939
					    parser->buffer);
		trace_parser_clear(parser);
4940 4941
	}

4942 4943
	if (!ret)
		ret = read;
4944

4945 4946 4947 4948
	return ret;
}

static const struct file_operations ftrace_graph_fops = {
4949 4950 4951
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
4952
	.llseek		= tracing_lseek,
4953
	.release	= ftrace_graph_release,
4954
};
4955 4956 4957 4958 4959

static const struct file_operations ftrace_graph_notrace_fops = {
	.open		= ftrace_graph_notrace_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
4960
	.llseek		= tracing_lseek,
4961 4962
	.release	= ftrace_graph_release,
};
4963 4964
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994
void ftrace_create_filter_files(struct ftrace_ops *ops,
				struct dentry *parent)
{

	trace_create_file("set_ftrace_filter", 0644, parent,
			  ops, &ftrace_filter_fops);

	trace_create_file("set_ftrace_notrace", 0644, parent,
			  ops, &ftrace_notrace_fops);
}

/*
 * The name "destroy_filter_files" is really a misnomer. Although
 * in the future, it may actualy delete the files, but this is
 * really intended to make sure the ops passed in are disabled
 * and that when this function returns, the caller is free to
 * free the ops.
 *
 * The "destroy" name is only to match the "create" name that this
 * should be paired with.
 */
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
	mutex_lock(&ftrace_lock);
	if (ops->flags & FTRACE_OPS_FL_ENABLED)
		ftrace_shutdown(ops, 0);
	ops->flags |= FTRACE_OPS_FL_DELETED;
	mutex_unlock(&ftrace_lock);
}

4995
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
4996 4997
{

4998 4999
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
5000

5001 5002 5003
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

5004
	ftrace_create_filter_files(&global_ops, d_tracer);
5005

5006
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5007
	trace_create_file("set_graph_function", 0444, d_tracer,
5008 5009
				    NULL,
				    &ftrace_graph_fops);
5010 5011 5012
	trace_create_file("set_graph_notrace", 0444, d_tracer,
				    NULL,
				    &ftrace_graph_notrace_fops);
5013 5014
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5015 5016 5017
	return 0;
}

5018
static int ftrace_cmp_ips(const void *a, const void *b)
5019
{
5020 5021
	const unsigned long *ipa = a;
	const unsigned long *ipb = b;
5022

5023 5024 5025 5026 5027 5028 5029
	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

5030
static int ftrace_process_locs(struct module *mod,
5031
			       unsigned long *start,
5032 5033
			       unsigned long *end)
{
5034
	struct ftrace_page *start_pg;
5035
	struct ftrace_page *pg;
5036
	struct dyn_ftrace *rec;
5037
	unsigned long count;
5038 5039
	unsigned long *p;
	unsigned long addr;
5040
	unsigned long flags = 0; /* Shut up gcc */
5041 5042 5043 5044 5045 5046 5047
	int ret = -ENOMEM;

	count = end - start;

	if (!count)
		return 0;

5048
	sort(start, count, sizeof(*start),
5049
	     ftrace_cmp_ips, NULL);
5050

5051 5052
	start_pg = ftrace_allocate_pages(count);
	if (!start_pg)
5053
		return -ENOMEM;
5054

S
Steven Rostedt 已提交
5055
	mutex_lock(&ftrace_lock);
5056

5057 5058 5059 5060 5061
	/*
	 * Core and each module needs their own pages, as
	 * modules will free them when they are removed.
	 * Force a new page to be allocated for modules.
	 */
5062 5063 5064
	if (!mod) {
		WARN_ON(ftrace_pages || ftrace_pages_start);
		/* First initialization */
5065
		ftrace_pages = ftrace_pages_start = start_pg;
5066
	} else {
5067
		if (!ftrace_pages)
5068
			goto out;
5069

5070 5071 5072 5073
		if (WARN_ON(ftrace_pages->next)) {
			/* Hmm, we have free pages? */
			while (ftrace_pages->next)
				ftrace_pages = ftrace_pages->next;
5074
		}
5075

5076
		ftrace_pages->next = start_pg;
5077 5078
	}

5079
	p = start;
5080
	pg = start_pg;
5081 5082
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
5083 5084 5085 5086 5087 5088 5089 5090
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
5091 5092 5093 5094 5095 5096 5097 5098 5099 5100

		if (pg->index == pg->size) {
			/* We should have allocated enough */
			if (WARN_ON(!pg->next))
				break;
			pg = pg->next;
		}

		rec = &pg->records[pg->index++];
		rec->ip = addr;
5101 5102
	}

5103 5104 5105 5106 5107 5108
	/* We should have used all pages */
	WARN_ON(pg->next);

	/* Assign the last page to ftrace_pages */
	ftrace_pages = pg;

5109
	/*
5110 5111 5112 5113 5114 5115
	 * We only need to disable interrupts on start up
	 * because we are modifying code that an interrupt
	 * may execute, and the modification is not atomic.
	 * But for modules, nothing runs the code we modify
	 * until we are finished with it, and there's no
	 * reason to cause large interrupt latencies while we do it.
5116
	 */
5117 5118
	if (!mod)
		local_irq_save(flags);
5119
	ftrace_update_code(mod, start_pg);
5120 5121
	if (!mod)
		local_irq_restore(flags);
5122 5123
	ret = 0;
 out:
S
Steven Rostedt 已提交
5124
	mutex_unlock(&ftrace_lock);
5125

5126
	return ret;
5127 5128
}

5129
#ifdef CONFIG_MODULES
5130 5131 5132

#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)

5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145
static int referenced_filters(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
		if (ops_references_rec(ops, rec))
		    cnt++;
	}

	return cnt;
}

5146
void ftrace_release_mod(struct module *mod)
5147 5148
{
	struct dyn_ftrace *rec;
5149
	struct ftrace_page **last_pg;
5150
	struct ftrace_page *pg;
5151
	int order;
5152

5153 5154
	mutex_lock(&ftrace_lock);

5155
	if (ftrace_disabled)
5156
		goto out_unlock;
5157

5158 5159 5160 5161 5162 5163 5164
	/*
	 * Each module has its own ftrace_pages, remove
	 * them from the list.
	 */
	last_pg = &ftrace_pages_start;
	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
		rec = &pg->records[0];
5165
		if (within_module_core(rec->ip, mod)) {
5166
			/*
5167 5168
			 * As core pages are first, the first
			 * page should never be a module page.
5169
			 */
5170 5171 5172 5173 5174 5175 5176 5177
			if (WARN_ON(pg == ftrace_pages_start))
				goto out_unlock;

			/* Check if we are deleting the last page */
			if (pg == ftrace_pages)
				ftrace_pages = next_to_ftrace_page(last_pg);

			*last_pg = pg->next;
5178 5179 5180
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
5181 5182 5183
		} else
			last_pg = &pg->next;
	}
5184
 out_unlock:
5185 5186 5187
	mutex_unlock(&ftrace_lock);
}

5188
void ftrace_module_enable(struct module *mod)
5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	mutex_lock(&ftrace_lock);

	if (ftrace_disabled)
		goto out_unlock;

	/*
	 * If the tracing is enabled, go ahead and enable the record.
	 *
	 * The reason not to enable the record immediatelly is the
	 * inherent check of ftrace_make_nop/ftrace_make_call for
	 * correct previous instructions.  Making first the NOP
	 * conversion puts the module to the correct state, thus
	 * passing the ftrace_make_call check.
	 *
	 * We also delay this to after the module code already set the
	 * text to read-only, as we now need to set it back to read-write
	 * so that we can modify the text.
	 */
	if (ftrace_start_up)
		ftrace_arch_code_modify_prepare();

	do_for_each_ftrace_rec(pg, rec) {
		int cnt;
		/*
		 * do_for_each_ftrace_rec() is a double loop.
		 * module text shares the pg. If a record is
		 * not part of this module, then skip this pg,
		 * which the "break" will do.
		 */
		if (!within_module_core(rec->ip, mod))
			break;

		cnt = 0;

		/*
		 * When adding a module, we need to check if tracers are
		 * currently enabled and if they are, and can trace this record,
		 * we need to enable the module functions as well as update the
		 * reference counts for those function records.
		 */
		if (ftrace_start_up)
			cnt += referenced_filters(rec);

		/* This clears FTRACE_FL_DISABLED */
		rec->flags = cnt;

		if (ftrace_start_up && cnt) {
			int failed = __ftrace_replace_code(rec, 1);
			if (failed) {
				ftrace_bug(failed, rec);
				goto out_loop;
			}
		}

	} while_for_each_ftrace_rec();

 out_loop:
	if (ftrace_start_up)
		ftrace_arch_code_modify_post_process();

 out_unlock:
	mutex_unlock(&ftrace_lock);
}

5257
void ftrace_module_init(struct module *mod)
5258
{
5259
	if (ftrace_disabled || !mod->num_ftrace_callsites)
5260
		return;
5261

5262 5263
	ftrace_process_locs(mod, mod->ftrace_callsites,
			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5264
}
5265 5266
#endif /* CONFIG_MODULES */

5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310
void ftrace_free_mem(void *start_ptr, void *end_ptr)
{
	unsigned long start = (unsigned long)start_ptr;
	unsigned long end = (unsigned long)end_ptr;
	struct ftrace_page **last_pg = &ftrace_pages_start;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	struct dyn_ftrace key;
	int order;

	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */

	mutex_lock(&ftrace_lock);

	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
			continue;
 again:
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (!rec)
			continue;
		pg->index--;
		if (!pg->index) {
			*last_pg = pg->next;
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
			pg = container_of(last_pg, struct ftrace_page, next);
			if (!(*last_pg))
				ftrace_pages = pg;
			continue;
		}
		memmove(rec, rec + 1,
			(pg->index - (rec - pg->records)) * sizeof(*rec));
		/* More than one function may be in this block */
		goto again;
	}
	mutex_unlock(&ftrace_lock);
}

5311 5312
void __init ftrace_init(void)
{
5313 5314
	extern unsigned long __start_mcount_loc[];
	extern unsigned long __stop_mcount_loc[];
5315
	unsigned long count, flags;
5316 5317 5318
	int ret;

	local_irq_save(flags);
5319
	ret = ftrace_dyn_arch_init();
5320
	local_irq_restore(flags);
5321
	if (ret)
5322 5323 5324
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;
5325 5326
	if (!count) {
		pr_info("ftrace: No functions to be traced?\n");
5327
		goto failed;
5328 5329 5330 5331
	}

	pr_info("ftrace: allocating %ld entries in %ld pages\n",
		count, count / ENTRIES_PER_PAGE + 1);
5332 5333 5334

	last_ftrace_enabled = ftrace_enabled = 1;

5335
	ret = ftrace_process_locs(NULL,
5336
				  __start_mcount_loc,
5337 5338
				  __stop_mcount_loc);

5339 5340
	set_ftrace_early_filters();

5341 5342 5343 5344 5345
	return;
 failed:
	ftrace_disabled = 1;
}

5346 5347 5348 5349 5350 5351 5352
/* Do nothing if arch does not support this */
void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363

/*
 * Currently there's no safe way to free a trampoline when the kernel
 * is configured with PREEMPT. That is because a task could be preempted
 * when it jumped to the trampoline, it may be preempted for a long time
 * depending on the system load, and currently there's no way to know
 * when it will be off the trampoline. If the trampoline is freed
 * too early, when the task runs again, it will be executing on freed
 * memory and crash.
 */
#ifdef CONFIG_PREEMPT
5364 5365 5366
	/* Currently, only non dynamic ops can have a trampoline */
	if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
		return;
5367
#endif
5368 5369 5370 5371

	arch_ftrace_update_trampoline(ops);
}

5372
#else
5373

5374
static struct ftrace_ops global_ops = {
5375
	.func			= ftrace_stub,
5376 5377 5378
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				  FTRACE_OPS_FL_INITIALIZED |
				  FTRACE_OPS_FL_PID,
5379 5380
};

5381 5382 5383 5384 5385
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
5386
core_initcall(ftrace_nodyn_init);
5387

5388
static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
5389
static inline void ftrace_startup_enable(int command) { }
5390
static inline void ftrace_startup_all(int command) { }
5391
/* Keep as macros so we do not need to define the commands */
5392 5393 5394 5395 5396 5397
# define ftrace_startup(ops, command)					\
	({								\
		int ___ret = __register_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
5398
	})
5399 5400 5401 5402 5403 5404 5405
# define ftrace_shutdown(ops, command)					\
	({								\
		int ___ret = __unregister_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
	})
5406

I
Ingo Molnar 已提交
5407 5408
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
5409 5410

static inline int
5411
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
5412 5413 5414 5415
{
	return 1;
}

5416 5417 5418 5419
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

5420 5421
#endif /* CONFIG_DYNAMIC_FTRACE */

5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444
__init void ftrace_init_global_array_ops(struct trace_array *tr)
{
	tr->ops = &global_ops;
	tr->ops->private = tr;
}

void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
{
	/* If we filter on pids, update to use the pid function */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		if (WARN_ON(tr->ops->func != ftrace_stub))
			printk("ftrace ops had %pS for function\n",
			       tr->ops->func);
	}
	tr->ops->func = func;
	tr->ops->private = tr;
}

void ftrace_reset_array_ops(struct trace_array *tr)
{
	tr->ops->func = ftrace_stub;
}

5445 5446
static inline void
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5447
		       struct ftrace_ops *ignored, struct pt_regs *regs)
5448
{
5449
	struct ftrace_ops *op;
5450
	int bit;
5451

5452 5453 5454
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;
5455

5456 5457 5458 5459 5460
	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
5461

5462
	do_for_each_ftrace_op(op, ftrace_ops_list) {
5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476
		/*
		 * Check the following for each ops before calling their func:
		 *  if RCU flag is set, then rcu_is_watching() must be true
		 *  if PER_CPU is set, then ftrace_function_local_disable()
		 *                          must be false
		 *  Otherwise test if the ip matches the ops filter
		 *
		 * If any of the above fails then the op->func() is not executed.
		 */
		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
		    (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
		     !ftrace_function_local_disabled(op)) &&
		    ftrace_ops_test(op, ip, regs)) {
		    
5477 5478
			if (FTRACE_WARN_ON(!op->func)) {
				pr_warn("op=%p %pS\n", op, op);
5479 5480
				goto out;
			}
5481
			op->func(ip, parent_ip, op, regs);
5482
		}
5483
	} while_for_each_ftrace_op(op);
5484
out:
5485
	preempt_enable_notrace();
5486
	trace_clear_recursion(bit);
5487 5488
}

5489 5490 5491 5492 5493
/*
 * Some archs only support passing ip and parent_ip. Even though
 * the list function ignores the op parameter, we do not want any
 * C side effects, where a function is called without the caller
 * sending a third parameter.
5494 5495 5496
 * Archs are to support both the regs and ftrace_ops at the same time.
 * If they support ftrace_ops, it is assumed they support regs.
 * If call backs want to use regs, they must either check for regs
5497 5498
 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
5499
 * An architecture can pass partial regs with ftrace_ops and still
L
Li Bin 已提交
5500
 * set the ARCH_SUPPORTS_FTRACE_OPS.
5501 5502 5503
 */
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5504
				 struct ftrace_ops *op, struct pt_regs *regs)
5505
{
5506
	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
5507 5508 5509 5510
}
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
{
5511
	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
5512 5513 5514
}
#endif

5515 5516
/*
 * If there's only one function registered but it does not support
5517 5518
 * recursion, needs RCU protection and/or requires per cpu handling, then
 * this function will be called by the mcount trampoline.
5519
 */
5520
static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
5521 5522 5523 5524
				   struct ftrace_ops *op, struct pt_regs *regs)
{
	int bit;

5525 5526 5527
	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
		return;

5528 5529 5530 5531
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;

5532
	preempt_disable_notrace();
5533

5534 5535 5536 5537 5538 5539
	if (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
	    !ftrace_function_local_disabled(op)) {
		op->func(ip, parent_ip, op, regs);
	}

	preempt_enable_notrace();
5540 5541 5542
	trace_clear_recursion(bit);
}

5543 5544 5545 5546 5547 5548 5549
/**
 * ftrace_ops_get_func - get the function a trampoline should call
 * @ops: the ops to get the function for
 *
 * Normally the mcount trampoline will call the ops->func, but there
 * are times that it should not. For example, if the ops does not
 * have its own recursion protection, then it should call the
5550
 * ftrace_ops_assist_func() instead.
5551 5552 5553 5554 5555 5556
 *
 * Returns the function that the trampoline should call for @ops.
 */
ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
{
	/*
5557 5558
	 * If the function does not handle recursion, needs to be RCU safe,
	 * or does per cpu logic, then we need to call the assist handler.
5559
	 */
5560 5561 5562
	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
	    ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU))
		return ftrace_ops_assist_func;
5563 5564 5565 5566

	return ops->func;
}

5567 5568 5569
static void
ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
		    struct task_struct *prev, struct task_struct *next)
S
Steven Rostedt 已提交
5570
{
5571 5572
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;
S
Steven Rostedt 已提交
5573

5574
	pid_list = rcu_dereference_sched(tr->function_pids);
5575

5576 5577
	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, next));
S
Steven Rostedt 已提交
5578 5579
}

5580
static void clear_ftrace_pids(struct trace_array *tr)
5581
{
5582 5583
	struct trace_pid_list *pid_list;
	int cpu;
5584

5585 5586 5587 5588
	pid_list = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));
	if (!pid_list)
		return;
5589

5590
	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
5591

5592 5593
	for_each_possible_cpu(cpu)
		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
S
Steven Rostedt 已提交
5594

5595
	rcu_assign_pointer(tr->function_pids, NULL);
S
Steven Rostedt 已提交
5596

5597 5598
	/* Wait till all users are no longer using pid filtering */
	synchronize_sched();
5599

5600
	trace_free_pid_list(pid_list);
5601 5602
}

5603
static void ftrace_pid_reset(struct trace_array *tr)
5604
{
5605
	mutex_lock(&ftrace_lock);
5606
	clear_ftrace_pids(tr);
S
Steven Rostedt 已提交
5607

5608
	ftrace_update_pid_func();
5609
	ftrace_startup_all(0);
5610 5611 5612 5613

	mutex_unlock(&ftrace_lock);
}

5614 5615
/* Greater than any max PID */
#define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
5616

5617
static void *fpid_start(struct seq_file *m, loff_t *pos)
5618
	__acquires(RCU)
5619
{
5620 5621 5622
	struct trace_pid_list *pid_list;
	struct trace_array *tr = m->private;

5623
	mutex_lock(&ftrace_lock);
5624 5625 5626
	rcu_read_lock_sched();

	pid_list = rcu_dereference_sched(tr->function_pids);
5627

5628 5629
	if (!pid_list)
		return !(*pos) ? FTRACE_NO_PIDS : NULL;
5630

5631
	return trace_pid_start(pid_list, pos);
5632 5633 5634 5635
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
5636 5637 5638 5639
	struct trace_array *tr = m->private;
	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);

	if (v == FTRACE_NO_PIDS)
5640 5641
		return NULL;

5642
	return trace_pid_next(pid_list, v, pos);
5643 5644 5645
}

static void fpid_stop(struct seq_file *m, void *p)
5646
	__releases(RCU)
5647
{
5648
	rcu_read_unlock_sched();
5649 5650 5651 5652 5653
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
5654
	if (v == FTRACE_NO_PIDS) {
5655
		seq_puts(m, "no pid\n");
5656 5657 5658
		return 0;
	}

5659
	return trace_pid_show(m, v);
5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
5672 5673
	struct trace_array *tr = inode->i_private;
	struct seq_file *m;
5674 5675
	int ret = 0;

5676 5677 5678
	if (trace_array_get(tr) < 0)
		return -ENODEV;

5679 5680
	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
5681
		ftrace_pid_reset(tr);
5682

5683 5684 5685 5686 5687 5688 5689 5690
	ret = seq_open(file, &ftrace_pid_sops);
	if (ret < 0) {
		trace_array_put(tr);
	} else {
		m = file->private_data;
		/* copy tr over to seq ops */
		m->private = tr;
	}
5691 5692 5693 5694

	return ret;
}

5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710
static void ignore_task_cpu(void *data)
{
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;

	/*
	 * This function is called by on_each_cpu() while the
	 * event_mutex is held.
	 */
	pid_list = rcu_dereference_protected(tr->function_pids,
					     mutex_is_locked(&ftrace_lock));

	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, current));
}

5711 5712 5713 5714
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5715 5716 5717 5718 5719
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
	struct trace_pid_list *filtered_pids = NULL;
	struct trace_pid_list *pid_list;
	ssize_t ret;
5720

5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731
	if (!cnt)
		return 0;

	mutex_lock(&ftrace_lock);

	filtered_pids = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));

	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
	if (ret < 0)
		goto out;
5732

5733
	rcu_assign_pointer(tr->function_pids, pid_list);
5734

5735 5736 5737 5738 5739 5740 5741
	if (filtered_pids) {
		synchronize_sched();
		trace_free_pid_list(filtered_pids);
	} else if (pid_list) {
		/* Register a probe to set whether to ignore the tracing of a task */
		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
	}
5742

5743
	/*
5744 5745 5746
	 * Ignoring of pids is done at task switch. But we have to
	 * check for those tasks that are currently running.
	 * Always do this in case a pid was appended or removed.
5747
	 */
5748
	on_each_cpu(ignore_task_cpu, tr, 1);
5749

5750 5751 5752 5753
	ftrace_update_pid_func();
	ftrace_startup_all(0);
 out:
	mutex_unlock(&ftrace_lock);
5754

5755 5756
	if (ret > 0)
		*ppos += ret;
5757

5758
	return ret;
5759
}
5760

5761 5762 5763
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
5764
	struct trace_array *tr = inode->i_private;
5765

5766 5767 5768
	trace_array_put(tr);

	return seq_release(inode, file);
5769 5770
}

5771
static const struct file_operations ftrace_pid_fops = {
5772 5773 5774
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
5775
	.llseek		= tracing_lseek,
5776
	.release	= ftrace_pid_release,
5777 5778
};

5779
void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
5780
{
5781
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
5782
			    tr, &ftrace_pid_fops);
5783 5784
}

5785 5786 5787 5788 5789 5790 5791 5792 5793 5794
void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
					 struct dentry *d_tracer)
{
	/* Only the top level directory has the dyn_tracefs and profile */
	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));

	ftrace_init_dyn_tracefs(d_tracer);
	ftrace_profile_tracefs(d_tracer);
}

S
Steven Rostedt 已提交
5795
/**
5796
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
5797 5798 5799 5800 5801
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
5802
void ftrace_kill(void)
S
Steven Rostedt 已提交
5803 5804 5805 5806 5807 5808
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

5809 5810 5811 5812 5813 5814 5815 5816
/**
 * Test if ftrace is dead or not.
 */
int ftrace_is_dead(void)
{
	return ftrace_disabled;
}

5817
/**
5818 5819
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
5820
 *
5821 5822 5823 5824 5825 5826
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
5827
 */
5828
int register_ftrace_function(struct ftrace_ops *ops)
5829
{
5830
	int ret = -1;
5831

5832 5833
	ftrace_ops_init(ops);

S
Steven Rostedt 已提交
5834
	mutex_lock(&ftrace_lock);
5835

5836
	ret = ftrace_startup(ops, 0);
5837

S
Steven Rostedt 已提交
5838
	mutex_unlock(&ftrace_lock);
5839

5840
	return ret;
5841
}
5842
EXPORT_SYMBOL_GPL(register_ftrace_function);
5843 5844

/**
5845
 * unregister_ftrace_function - unregister a function for profiling.
5846 5847 5848 5849 5850 5851 5852 5853
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
5854
	mutex_lock(&ftrace_lock);
5855
	ret = ftrace_shutdown(ops, 0);
S
Steven Rostedt 已提交
5856
	mutex_unlock(&ftrace_lock);
5857 5858 5859

	return ret;
}
5860
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
5861

I
Ingo Molnar 已提交
5862
int
5863
ftrace_enable_sysctl(struct ctl_table *table, int write,
5864
		     void __user *buffer, size_t *lenp,
5865 5866
		     loff_t *ppos)
{
5867
	int ret = -ENODEV;
5868

S
Steven Rostedt 已提交
5869
	mutex_lock(&ftrace_lock);
5870

5871 5872 5873 5874
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
5875

5876
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
5877 5878
		goto out;

5879
	last_ftrace_enabled = !!ftrace_enabled;
5880 5881 5882 5883

	if (ftrace_enabled) {

		/* we are starting ftrace again */
5884 5885
		if (ftrace_ops_list != &ftrace_list_end)
			update_ftrace_function();
5886

5887 5888
		ftrace_startup_sysctl();

5889 5890 5891 5892 5893 5894 5895 5896
	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
5897
	mutex_unlock(&ftrace_lock);
5898
	return ret;
5899
}
5900

5901
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5902

5903 5904 5905 5906
static struct ftrace_ops graph_ops = {
	.func			= ftrace_stub,
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				   FTRACE_OPS_FL_INITIALIZED |
5907
				   FTRACE_OPS_FL_PID |
5908 5909 5910
				   FTRACE_OPS_FL_STUB,
#ifdef FTRACE_GRAPH_TRAMP_ADDR
	.trampoline		= FTRACE_GRAPH_TRAMP_ADDR,
5911
	/* trampoline_size is only needed for dynamically allocated tramps */
5912 5913 5914 5915
#endif
	ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
};

5916 5917 5918 5919 5920 5921 5922 5923 5924 5925
void ftrace_graph_sleep_time_control(bool enable)
{
	fgraph_sleep_time = enable;
}

void ftrace_graph_graph_time_control(bool enable)
{
	fgraph_graph_time = enable;
}

5926 5927 5928 5929 5930
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

5931 5932 5933
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
5934
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
5935
static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

5957
	read_lock(&tasklist_lock);
5958 5959 5960 5961 5962 5963 5964
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
5965
			atomic_set(&t->tracing_graph_pause, 0);
5966
			atomic_set(&t->trace_overrun, 0);
5967 5968 5969 5970
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
5971 5972 5973 5974
		}
	} while_each_thread(g, t);

unlock:
5975
	read_unlock(&tasklist_lock);
5976 5977 5978 5979 5980 5981
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

5982
static void
5983
ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
5984
			struct task_struct *prev, struct task_struct *next)
5985 5986 5987 5988
{
	unsigned long long timestamp;
	int index;

5989 5990 5991 5992
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
5993
	if (fgraph_sleep_time)
5994 5995
		return;

5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

6014
/* Allocate a return stack for each task */
6015
static int start_graph_tracing(void)
6016 6017
{
	struct ftrace_ret_stack **ret_stack_list;
6018
	int ret, cpu;
6019 6020 6021 6022 6023 6024 6025 6026

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

6027
	/* The cpu_boot init_task->ret_stack will never be freed */
6028 6029
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
6030
			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
6031
	}
6032

6033 6034 6035 6036
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

6037
	if (!ret) {
6038
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6039 6040 6041 6042 6043
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

6044 6045 6046 6047
	kfree(ret_stack_list);
	return ret;
}

6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084
static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
{
	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
		return 0;
	return __ftrace_graph_entry(trace);
}

/*
 * The function graph tracer should only trace the functions defined
 * by set_ftrace_filter and set_ftrace_notrace. If another function
 * tracer ops is registered, the graph tracer requires testing the
 * function against the global ops, and not just trace any function
 * that any ftrace_ops registered.
 */
static void update_function_graph_func(void)
{
6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103
	struct ftrace_ops *op;
	bool do_test = false;

	/*
	 * The graph and global ops share the same set of functions
	 * to test. If any other ops is on the list, then
	 * the graph tracing needs to test if its the function
	 * it should call.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op != &global_ops && op != &graph_ops &&
		    op != &ftrace_list_end) {
			do_test = true;
			/* in double loop, break out with goto */
			goto out;
		}
	} while_for_each_ftrace_op(op);
 out:
	if (do_test)
6104
		ftrace_graph_entry = ftrace_graph_entry_test;
6105 6106
	else
		ftrace_graph_entry = __ftrace_graph_entry;
6107 6108
}

6109 6110 6111 6112
static struct notifier_block ftrace_suspend_notifier = {
	.notifier_call = ftrace_suspend_notifier_call,
};

6113 6114
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
6115
{
6116 6117
	int ret = 0;

S
Steven Rostedt 已提交
6118
	mutex_lock(&ftrace_lock);
6119

6120
	/* we currently allow only one tracer registered at a time */
6121
	if (ftrace_graph_active) {
6122 6123 6124 6125
		ret = -EBUSY;
		goto out;
	}

6126 6127
	register_pm_notifier(&ftrace_suspend_notifier);

6128
	ftrace_graph_active++;
6129
	ret = start_graph_tracing();
6130
	if (ret) {
6131
		ftrace_graph_active--;
6132 6133
		goto out;
	}
6134

6135
	ftrace_graph_return = retfunc;
6136 6137 6138 6139 6140 6141 6142 6143 6144 6145

	/*
	 * Update the indirect function to the entryfunc, and the
	 * function that gets called to the entry_test first. Then
	 * call the update fgraph entry function to determine if
	 * the entryfunc should be called directly or not.
	 */
	__ftrace_graph_entry = entryfunc;
	ftrace_graph_entry = ftrace_graph_entry_test;
	update_function_graph_func();
6146

6147
	ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
6148
out:
S
Steven Rostedt 已提交
6149
	mutex_unlock(&ftrace_lock);
6150
	return ret;
6151 6152
}

6153
void unregister_ftrace_graph(void)
6154
{
S
Steven Rostedt 已提交
6155
	mutex_lock(&ftrace_lock);
6156

6157
	if (unlikely(!ftrace_graph_active))
6158 6159
		goto out;

6160
	ftrace_graph_active--;
6161
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
6162
	ftrace_graph_entry = ftrace_graph_entry_stub;
6163
	__ftrace_graph_entry = ftrace_graph_entry_stub;
6164
	ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
6165
	unregister_pm_notifier(&ftrace_suspend_notifier);
6166
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6167

6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178
#ifdef CONFIG_DYNAMIC_FTRACE
	/*
	 * Function graph does not allocate the trampoline, but
	 * other global_ops do. We need to reset the ALLOC_TRAMP flag
	 * if one was used.
	 */
	global_ops.trampoline = save_global_trampoline;
	if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP)
		global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
#endif

6179
 out:
S
Steven Rostedt 已提交
6180
	mutex_unlock(&ftrace_lock);
6181
}
6182

6183 6184 6185 6186 6187 6188 6189 6190
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);

static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
	atomic_set(&t->tracing_graph_pause, 0);
	atomic_set(&t->trace_overrun, 0);
	t->ftrace_timestamp = 0;
L
Lucas De Marchi 已提交
6191
	/* make curr_ret_stack visible before we add the ret_stack */
6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225
	smp_wmb();
	t->ret_stack = ret_stack;
}

/*
 * Allocate a return stack for the idle task. May be the first
 * time through, or it may be done by CPU hotplug online.
 */
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
{
	t->curr_ret_stack = -1;
	/*
	 * The idle task has no parent, it either has its own
	 * stack or no stack at all.
	 */
	if (t->ret_stack)
		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));

	if (ftrace_graph_active) {
		struct ftrace_ret_stack *ret_stack;

		ret_stack = per_cpu(idle_ret_stack, cpu);
		if (!ret_stack) {
			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
					    * sizeof(struct ftrace_ret_stack),
					    GFP_KERNEL);
			if (!ret_stack)
				return;
			per_cpu(idle_ret_stack, cpu) = ret_stack;
		}
		graph_init_task(t, ret_stack);
	}
}

6226
/* Allocate a return stack for newly created task */
6227
void ftrace_graph_init_task(struct task_struct *t)
6228
{
6229 6230
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;
6231
	t->curr_ret_stack = -1;
6232

6233
	if (ftrace_graph_active) {
6234 6235 6236
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
6237 6238
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
6239
		if (!ret_stack)
6240
			return;
6241
		graph_init_task(t, ret_stack);
6242
	}
6243 6244
}

6245
void ftrace_graph_exit_task(struct task_struct *t)
6246
{
6247 6248
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

6249
	t->ret_stack = NULL;
6250 6251 6252 6253
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
6254
}
6255
#endif