ftrace.c 139.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
13
 *  Copyright (C) 2004 Nadia Yvette Chambers
14 15
 */

16 17 18
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
#include <linux/kallsyms.h>
19
#include <linux/seq_file.h>
20
#include <linux/suspend.h>
21
#include <linux/tracefs.h>
22
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
23
#include <linux/kthread.h>
24
#include <linux/uaccess.h>
25
#include <linux/bsearch.h>
26
#include <linux/module.h>
I
Ingo Molnar 已提交
27
#include <linux/ftrace.h>
28
#include <linux/sysctl.h>
29
#include <linux/slab.h>
30
#include <linux/ctype.h>
31
#include <linux/sort.h>
32
#include <linux/list.h>
33
#include <linux/hash.h>
34
#include <linux/rcupdate.h>
35

36
#include <trace/events/sched.h>
37

38
#include <asm/setup.h>
39

40
#include "trace_output.h"
S
Steven Rostedt 已提交
41
#include "trace_stat.h"
42

43
#define FTRACE_WARN_ON(cond)			\
44 45 46
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
47
			ftrace_kill();		\
48 49
		___r;				\
	})
50 51

#define FTRACE_WARN_ON_ONCE(cond)		\
52 53 54
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
55
			ftrace_kill();		\
56 57
		___r;				\
	})
58

59 60 61
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
62 63
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
64

65
#ifdef CONFIG_DYNAMIC_FTRACE
66 67 68
#define INIT_OPS_HASH(opsname)	\
	.func_hash		= &opsname.local_hash,			\
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
69 70 71
#define ASSIGN_OPS_HASH(opsname, val) \
	.func_hash		= val, \
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
72
#else
73
#define INIT_OPS_HASH(opsname)
74
#define ASSIGN_OPS_HASH(opsname, val)
75 76
#endif

77 78
static struct ftrace_ops ftrace_list_end __read_mostly = {
	.func		= ftrace_stub,
79
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
80
	INIT_OPS_HASH(ftrace_list_end)
81 82
};

83 84
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
85
static int last_ftrace_enabled;
86

87 88
/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
89 90
/* What to set function_trace_op to */
static struct ftrace_ops *set_function_trace_op;
91

92 93 94 95 96 97 98
/* List for set_ftrace_pid's pids. */
LIST_HEAD(ftrace_pids);
struct ftrace_pid {
	struct list_head list;
	struct pid *pid;
};

99 100 101 102 103 104 105
static bool ftrace_pids_enabled(void)
{
	return !list_empty(&ftrace_pids);
}

static void ftrace_update_trampoline(struct ftrace_ops *ops);

106 107 108 109 110 111
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

112
static DEFINE_MUTEX(ftrace_lock);
113

114
static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
115
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
116
static struct ftrace_ops global_ops;
117

118 119
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
120
				 struct ftrace_ops *op, struct pt_regs *regs);
121 122 123 124 125
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
126

127 128
/*
 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
129
 * can use rcu_dereference_raw_notrace() is that elements removed from this list
130
 * are simply leaked, so there is no need to interact with a grace-period
131
 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
132 133 134 135 136
 * concurrent insertions into the ftrace_global_list.
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
#define do_for_each_ftrace_op(op, list)			\
137
	op = rcu_dereference_raw_notrace(list);			\
138 139 140 141 142 143
	do

/*
 * Optimized for just a single item in the list (as that is the normal case).
 */
#define while_for_each_ftrace_op(op)				\
144
	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
145 146
	       unlikely((op) != &ftrace_list_end))

147 148 149 150
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
151 152
		mutex_init(&ops->local_hash.regex_lock);
		ops->func_hash = &ops->local_hash;
153 154 155 156 157
		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
	}
#endif
}

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
/**
 * ftrace_nr_registered_ops - return number of ops registered
 *
 * Returns the number of ftrace_ops registered and tracing functions
 */
int ftrace_nr_registered_ops(void)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	mutex_lock(&ftrace_lock);

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next)
		cnt++;

	mutex_unlock(&ftrace_lock);

	return cnt;
}

179
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
180
			    struct ftrace_ops *op, struct pt_regs *regs)
181
{
182
	if (!test_tsk_trace_trace(current))
183 184
		return;

185
	op->saved_func(ip, parent_ip, op, regs);
186 187
}

188
/**
189
 * clear_ftrace_function - reset the ftrace function
190
 *
191 192
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
193
 */
194
void clear_ftrace_function(void)
195
{
196 197 198
	ftrace_trace_function = ftrace_stub;
}

199
static void per_cpu_ops_disable_all(struct ftrace_ops *ops)
200 201 202 203 204 205 206
{
	int cpu;

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(ops->disabled, cpu) = 1;
}

207
static int per_cpu_ops_alloc(struct ftrace_ops *ops)
208 209 210
{
	int __percpu *disabled;

211 212 213
	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU)))
		return -EINVAL;

214 215 216 217 218
	disabled = alloc_percpu(int);
	if (!disabled)
		return -ENOMEM;

	ops->disabled = disabled;
219
	per_cpu_ops_disable_all(ops);
220 221 222
	return 0;
}

223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
static void ftrace_sync(struct work_struct *work)
{
	/*
	 * This function is just a stub to implement a hard force
	 * of synchronize_sched(). This requires synchronizing
	 * tasks even in userspace and idle.
	 *
	 * Yes, function tracing is rude.
	 */
}

static void ftrace_sync_ipi(void *data)
{
	/* Probably not needed, but do it anyway */
	smp_rmb();
}

240 241
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static void update_function_graph_func(void);
242 243 244 245 246

/* Both enabled by default (can be cleared by function_graph tracer flags */
static bool fgraph_sleep_time = true;
static bool fgraph_graph_time = true;

247 248 249 250
#else
static inline void update_function_graph_func(void) { }
#endif

251 252 253 254

static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
	/*
255
	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
256 257
	 * then it needs to call the list anyway.
	 */
258 259
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU |
			  FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC)
260 261 262 263 264
		return ftrace_ops_list_func;

	return ftrace_ops_get_func(ops);
}

265 266 267 268
static void update_ftrace_function(void)
{
	ftrace_func_t func;

269 270 271 272 273 274 275 276 277 278 279
	/*
	 * Prepare the ftrace_ops that the arch callback will use.
	 * If there's only one ftrace_ops registered, the ftrace_ops_list
	 * will point to the ops we want.
	 */
	set_function_trace_op = ftrace_ops_list;

	/* If there's no ftrace_ops registered, just call the stub function */
	if (ftrace_ops_list == &ftrace_list_end) {
		func = ftrace_stub;

280 281
	/*
	 * If we are at the end of the list and this ops is
282 283
	 * recursion safe and not dynamic and the arch supports passing ops,
	 * then have the mcount trampoline call the function directly.
284
	 */
285
	} else if (ftrace_ops_list->next == &ftrace_list_end) {
286
		func = ftrace_ops_get_list_func(ftrace_ops_list);
287

288 289
	} else {
		/* Just use the default ftrace_ops */
290
		set_function_trace_op = &ftrace_list_end;
291
		func = ftrace_ops_list_func;
292
	}
293

294 295
	update_function_graph_func();

296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
	/* If there's no change, then do nothing more here */
	if (ftrace_trace_function == func)
		return;

	/*
	 * If we are using the list function, it doesn't care
	 * about the function_trace_ops.
	 */
	if (func == ftrace_ops_list_func) {
		ftrace_trace_function = func;
		/*
		 * Don't even bother setting function_trace_ops,
		 * it would be racy to do so anyway.
		 */
		return;
	}

#ifndef CONFIG_DYNAMIC_FTRACE
	/*
	 * For static tracing, we need to be a bit more careful.
	 * The function change takes affect immediately. Thus,
	 * we need to coorditate the setting of the function_trace_ops
	 * with the setting of the ftrace_trace_function.
	 *
	 * Set the function to the list ops, which will call the
	 * function we want, albeit indirectly, but it handles the
	 * ftrace_ops and doesn't depend on function_trace_op.
	 */
	ftrace_trace_function = ftrace_ops_list_func;
	/*
	 * Make sure all CPUs see this. Yes this is slow, but static
	 * tracing is slow and nasty to have enabled.
	 */
	schedule_on_each_cpu(ftrace_sync);
	/* Now all cpus are using the list ops. */
	function_trace_op = set_function_trace_op;
	/* Make sure the function_trace_op is visible on all CPUs */
	smp_wmb();
	/* Nasty way to force a rmb on all cpus */
	smp_call_function(ftrace_sync_ipi, NULL, 1);
	/* OK, we are all set to update the ftrace_trace_function now! */
#endif /* !CONFIG_DYNAMIC_FTRACE */

339 340 341
	ftrace_trace_function = func;
}

342 343 344 345 346
int using_ftrace_ops_list_func(void)
{
	return ftrace_trace_function == ftrace_ops_list_func;
}

347
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
348
{
349
	ops->next = *list;
350
	/*
351
	 * We are entering ops into the list but another
352 353
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
354
	 * the ops pointer included into the list.
355
	 */
356
	rcu_assign_pointer(*list, ops);
357 358
}

359
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
360 361 362 363
{
	struct ftrace_ops **p;

	/*
364 365
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
366
	 */
367 368
	if (*list == ops && ops->next == &ftrace_list_end) {
		*list = &ftrace_list_end;
S
Steven Rostedt 已提交
369
		return 0;
370 371
	}

372
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
373 374 375
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
376 377
	if (*p != ops)
		return -1;
378 379

	*p = (*p)->next;
380 381
	return 0;
}
382

383 384
static void ftrace_update_trampoline(struct ftrace_ops *ops);

385 386
static int __register_ftrace_function(struct ftrace_ops *ops)
{
387 388 389
	if (ops->flags & FTRACE_OPS_FL_DELETED)
		return -EINVAL;

390 391 392
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

393
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
394 395 396 397 398 399 400 401 402 403 404 405 406
	/*
	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
	 */
	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
		return -EINVAL;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif

407 408 409
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

410 411
	if (ops->flags & FTRACE_OPS_FL_PER_CPU) {
		if (per_cpu_ops_alloc(ops))
412
			return -ENOMEM;
413 414 415
	}

	add_ftrace_ops(&ftrace_ops_list, ops);
416

417 418 419 420 421 422
	/* Always save the function, and reset at unregistering */
	ops->saved_func = ops->func;

	if (ops->flags & FTRACE_OPS_FL_PID && ftrace_pids_enabled())
		ops->func = ftrace_pid_func;

423 424
	ftrace_update_trampoline(ops);

425 426 427 428 429 430 431 432 433 434
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

435 436 437
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

438
	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
439

440 441
	if (ret < 0)
		return ret;
442

443 444
	if (ftrace_enabled)
		update_ftrace_function();
445

446 447
	ops->func = ops->saved_func;

S
Steven Rostedt 已提交
448
	return 0;
449 450
}

451 452
static void ftrace_update_pid_func(void)
{
453 454 455
	bool enabled = ftrace_pids_enabled();
	struct ftrace_ops *op;

456
	/* Only do something if we are tracing something */
457
	if (ftrace_trace_function == ftrace_stub)
458
		return;
459

460 461 462 463 464 465 466 467
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->flags & FTRACE_OPS_FL_PID) {
			op->func = enabled ? ftrace_pid_func :
				op->saved_func;
			ftrace_update_trampoline(op);
		}
	} while_for_each_ftrace_op(op);

468
	update_ftrace_function();
469 470
}

471 472 473 474 475
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
476 477
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
478
	unsigned long long		time_squared;
479
#endif
480 481
};

482 483 484 485
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
486 487
};

488 489 490 491 492 493 494 495
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

496 497
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
498

499 500
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
501

502 503 504
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
505 506
static DEFINE_MUTEX(ftrace_profile_lock);

507
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
508

509 510
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
511

S
Steven Rostedt 已提交
512 513 514
static void *
function_stat_next(void *v, int idx)
{
515 516
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
517

518
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
519 520

 again:
L
Li Zefan 已提交
521 522 523
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
524 525 526 527 528
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
529 530
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
531 532 533 534 535 536 537
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
538 539 540 541 542 543 544
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
545 546
}

547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
563 564
static int function_stat_cmp(void *p1, void *p2)
{
565 566
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
567 568 569 570 571 572 573 574

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
575
#endif
S
Steven Rostedt 已提交
576 577 578

static int function_stat_headers(struct seq_file *m)
{
579
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
580 581 582 583
	seq_puts(m, "  Function                               "
		 "Hit    Time            Avg             s^2\n"
		    "  --------                               "
		 "---    ----            ---             ---\n");
584
#else
585 586
	seq_puts(m, "  Function                               Hit\n"
		    "  --------                               ---\n");
587
#endif
S
Steven Rostedt 已提交
588 589 590 591 592
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
593
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
594
	char str[KSYM_SYMBOL_LEN];
595
	int ret = 0;
596
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
597 598
	static struct trace_seq s;
	unsigned long long avg;
599
	unsigned long long stddev;
600
#endif
601 602 603 604 605 606 607
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
S
Steven Rostedt 已提交
608

609 610 611 612 613 614 615
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	avg = rec->time;
	do_div(avg, rec->counter);
	if (tracing_thresh && (avg < tracing_thresh))
		goto out;
#endif

S
Steven Rostedt 已提交
616
	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
617 618 619
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
620
	seq_puts(m, "    ");
621

622 623 624 625
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
626 627 628 629 630 631 632
		/*
		 * Apply Welford's method:
		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
		 */
		stddev = rec->counter * rec->time_squared -
			 rec->time * rec->time;

633 634 635 636
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
637
		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
638 639
	}

640 641 642 643
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
644 645
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
646 647 648
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
649 650
out:
	mutex_unlock(&ftrace_profile_lock);
S
Steven Rostedt 已提交
651

652
	return ret;
S
Steven Rostedt 已提交
653 654
}

655
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
656
{
657
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
658

659
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
660

661 662 663 664
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
665 666
	}

667
	memset(stat->hash, 0,
668 669
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
670

671
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
672 673
{
	struct ftrace_profile_page *pg;
674 675
	int functions;
	int pages;
676
	int i;
S
Steven Rostedt 已提交
677

678
	/* If we already allocated, do nothing */
679
	if (stat->pages)
680
		return 0;
S
Steven Rostedt 已提交
681

682 683
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
684
		return -ENOMEM;
S
Steven Rostedt 已提交
685

686 687 688 689 690 691 692 693 694 695 696 697 698
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

699
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
700

701 702
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

703
	for (i = 1; i < pages; i++) {
704 705
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
706
			goto out_free;
707 708 709 710
		pg = pg->next;
	}

	return 0;
711 712 713 714 715 716 717 718 719 720 721 722 723 724

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
725 726
}

727
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
728
{
729
	struct ftrace_profile_stat *stat;
730
	int size;
S
Steven Rostedt 已提交
731

732 733 734
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
735
		/* If the profile is already created, simply reset it */
736
		ftrace_profile_reset(stat);
737 738
		return 0;
	}
S
Steven Rostedt 已提交
739

740 741 742 743 744
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
745

746
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
747

748
	if (!stat->hash)
749 750
		return -ENOMEM;

751
	/* Preallocate the function profiling pages */
752 753 754
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
755 756 757 758
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
759 760
}

761 762 763 764 765
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

766
	for_each_possible_cpu(cpu) {
767 768 769 770 771 772 773 774
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

775
/* interrupts must be disabled */
776 777
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
778
{
779
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
780 781 782
	struct hlist_head *hhd;
	unsigned long key;

783
	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
784
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
785 786 787 788

	if (hlist_empty(hhd))
		return NULL;

789
	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
S
Steven Rostedt 已提交
790
		if (rec->ip == ip)
791 792 793 794 795 796
			return rec;
	}

	return NULL;
}

797 798
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
799 800 801
{
	unsigned long key;

802
	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
803
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
804 805
}

806 807 808
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
809
static struct ftrace_profile *
810
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
811 812 813
{
	struct ftrace_profile *rec = NULL;

814
	/* prevent recursion (from NMIs) */
815
	if (atomic_inc_return(&stat->disabled) != 1)
816 817 818
		goto out;

	/*
819 820
	 * Try to find the function again since an NMI
	 * could have added it
821
	 */
822
	rec = ftrace_find_profiled_func(stat, ip);
823
	if (rec)
824
		goto out;
825

826 827 828 829
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
830
	}
831

832
	rec = &stat->pages->records[stat->pages->index++];
833
	rec->ip = ip;
834
	ftrace_add_profile(stat, rec);
835

S
Steven Rostedt 已提交
836
 out:
837
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
838 839 840 841 842

	return rec;
}

static void
843
function_profile_call(unsigned long ip, unsigned long parent_ip,
844
		      struct ftrace_ops *ops, struct pt_regs *regs)
S
Steven Rostedt 已提交
845
{
846
	struct ftrace_profile_stat *stat;
847
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
848 849 850 851 852 853
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
854

855
	stat = this_cpu_ptr(&ftrace_profile_stats);
856
	if (!stat->hash || !ftrace_profile_enabled)
857 858 859
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
860
	if (!rec) {
861
		rec = ftrace_profile_alloc(stat, ip);
862 863 864
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
865 866 867 868 869 870

	rec->counter++;
 out:
	local_irq_restore(flags);
}

871 872 873
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
874
	function_profile_call(trace->func, 0, NULL, NULL);
875 876 877 878 879
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
880
	struct ftrace_profile_stat *stat;
881
	unsigned long long calltime;
882
	struct ftrace_profile *rec;
883
	unsigned long flags;
884 885

	local_irq_save(flags);
886
	stat = this_cpu_ptr(&ftrace_profile_stats);
887
	if (!stat->hash || !ftrace_profile_enabled)
888 889
		goto out;

890 891 892 893
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

894 895
	calltime = trace->rettime - trace->calltime;

896
	if (!fgraph_graph_time) {
897 898 899 900 901 902 903 904 905 906 907 908 909 910
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

911
	rec = ftrace_find_profiled_func(stat, trace->func);
912
	if (rec) {
913
		rec->time += calltime;
914 915
		rec->time_squared += calltime * calltime;
	}
916

917
 out:
918 919 920 921 922 923 924 925 926 927 928 929 930 931
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
932
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
933
	.func		= function_profile_call,
934
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
935
	INIT_OPS_HASH(ftrace_profile_ops)
S
Steven Rostedt 已提交
936 937
};

938 939 940 941 942 943 944 945 946 947 948
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
949 950 951 952 953 954 955
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

956 957
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
S
Steven Rostedt 已提交
958 959 960 961 962 963 964
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
965 966 967 968 969 970
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

971 972 973 974 975
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
976 977 978
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
979 980 981 982
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
983
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
984 985
		}
	}
986
 out:
S
Steven Rostedt 已提交
987 988
	mutex_unlock(&ftrace_profile_lock);

989
	*ppos += cnt;
S
Steven Rostedt 已提交
990 991 992 993

	return cnt;
}

994 995 996 997
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
998
	char buf[64];		/* big enough to hold a number */
999 1000 1001 1002 1003 1004
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
1005 1006 1007 1008
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
1009
	.llseek		= default_llseek,
S
Steven Rostedt 已提交
1010 1011
};

1012 1013
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
1014 1015 1016 1017 1018 1019
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
1020 1021
};

1022
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1023
{
1024
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
1025
	struct dentry *entry;
1026
	char *name;
S
Steven Rostedt 已提交
1027
	int ret;
1028 1029 1030 1031 1032
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

1033
		name = kasprintf(GFP_KERNEL, "function%d", cpu);
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
1054 1055
	}

1056
	entry = tracefs_create_file("function_profile_enabled", 0644,
S
Steven Rostedt 已提交
1057 1058
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
1059
		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
S
Steven Rostedt 已提交
1060 1061 1062
}

#else /* CONFIG_FUNCTION_PROFILER */
1063
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1064 1065 1066 1067
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

1068 1069
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

1070 1071 1072 1073 1074 1075
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int ftrace_graph_active;
#else
# define ftrace_graph_active 0
#endif

1076 1077
#ifdef CONFIG_DYNAMIC_FTRACE

1078 1079
static struct ftrace_ops *removed_ops;

1080 1081 1082 1083 1084 1085
/*
 * Set when doing a global update, like enabling all recs or disabling them.
 * It is not set when just updating a single ftrace_ops.
 */
static bool update_all_ops;

1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;

struct ftrace_func_probe {
	struct hlist_node	node;
	struct ftrace_probe_ops	*ops;
	unsigned long		flags;
	unsigned long		ip;
	void			*data;
1098
	struct list_head	free_list;
1099 1100
};

1101 1102 1103 1104 1105 1106 1107 1108 1109
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

struct ftrace_hash {
	unsigned long		size_bits;
	struct hlist_head	*buckets;
	unsigned long		count;
1110
	struct rcu_head		rcu;
1111 1112
};

1113 1114 1115 1116 1117 1118 1119 1120 1121
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
1122
};
1123
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1124

1125
static struct ftrace_ops global_ops = {
1126 1127 1128 1129 1130
	.func				= ftrace_stub,
	.local_hash.notrace_hash	= EMPTY_HASH,
	.local_hash.filter_hash		= EMPTY_HASH,
	INIT_OPS_HASH(global_ops)
	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1131 1132
					  FTRACE_OPS_FL_INITIALIZED |
					  FTRACE_OPS_FL_PID,
1133 1134
};

1135 1136
/*
 * This is used by __kernel_text_address() to return true if the
1137
 * address is on a dynamically allocated trampoline that would
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
 * not return true for either core_kernel_text() or
 * is_module_text_address().
 */
bool is_ftrace_trampoline(unsigned long addr)
{
	struct ftrace_ops *op;
	bool ret = false;

	/*
	 * Some of the ops may be dynamically allocated,
	 * they are freed after a synchronize_sched().
	 */
	preempt_disable_notrace();

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/*
		 * This is to check for dynamically allocated trampolines.
		 * Trampolines that are in kernel text will have
		 * core_kernel_text() return true.
		 */
		if (op->trampoline && op->trampoline_size)
			if (addr >= op->trampoline &&
			    addr < op->trampoline + op->trampoline_size) {
				ret = true;
				goto out;
			}
	} while_for_each_ftrace_op(op);

 out:
	preempt_enable_notrace();

	return ret;
}

1172 1173
struct ftrace_page {
	struct ftrace_page	*next;
1174
	struct dyn_ftrace	*records;
1175
	int			index;
1176
	int			size;
1177 1178
};

1179 1180
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1181 1182 1183 1184 1185 1186 1187

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

1188
static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash)
1189 1190 1191 1192
{
	return !hash || !hash->count;
}

1193 1194 1195 1196 1197 1198 1199
static struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;

1200
	if (ftrace_hash_empty(hash))
1201 1202 1203 1204 1205 1206 1207 1208 1209
		return NULL;

	if (hash->size_bits > 0)
		key = hash_long(ip, hash->size_bits);
	else
		key = 0;

	hhd = &hash->buckets[key];

1210
	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1211 1212 1213 1214 1215 1216
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1217 1218
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1219 1220 1221 1222 1223
{
	struct hlist_head *hhd;
	unsigned long key;

	if (hash->size_bits)
1224
		key = hash_long(entry->ip, hash->size_bits);
1225 1226 1227 1228 1229 1230
	else
		key = 0;

	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1243 1244 1245 1246 1247

	return 0;
}

static void
1248
free_hash_entry(struct ftrace_hash *hash,
1249 1250 1251 1252 1253 1254 1255
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1256 1257 1258 1259 1260 1261 1262 1263
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	hash->count--;
}

1264 1265 1266
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
1267
	struct hlist_node *tn;
1268 1269 1270 1271
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1272 1273 1274
	if (!hash->count)
		return;

1275 1276
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
1277
		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1278
			free_hash_entry(hash, entry);
1279 1280 1281 1282
	}
	FTRACE_WARN_ON(hash->count);
}

1283 1284 1285 1286 1287 1288 1289 1290 1291
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}

1307 1308
void ftrace_free_filter(struct ftrace_ops *ops)
{
1309
	ftrace_ops_init(ops);
1310 1311
	free_ftrace_hash(ops->func_hash->filter_hash);
	free_ftrace_hash(ops->func_hash->notrace_hash);
1312 1313
}

1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
1324
	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

	/* Empty hash? */
1350
	if (ftrace_hash_empty(hash))
1351 1352 1353 1354
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
1355
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

1371
static void
1372
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1373
static void
1374
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1375

1376 1377 1378
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash);

1379
static int
1380 1381
ftrace_hash_move(struct ftrace_ops *ops, int enable,
		 struct ftrace_hash **dst, struct ftrace_hash *src)
1382 1383
{
	struct ftrace_func_entry *entry;
1384
	struct hlist_node *tn;
1385
	struct hlist_head *hhd;
1386
	struct ftrace_hash *new_hash;
1387 1388
	int size = src->count;
	int bits = 0;
1389
	int ret;
1390 1391
	int i;

1392 1393 1394 1395
	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
		return -EINVAL;

1396 1397 1398 1399 1400
	/*
	 * If the new source is empty, just free dst and assign it
	 * the empty_hash.
	 */
	if (!src->count) {
1401 1402
		new_hash = EMPTY_HASH;
		goto update;
1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414
	}

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1415 1416
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
1417
		return -ENOMEM;
1418 1419 1420 1421

	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
1422
		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1423
			remove_hash_entry(src, entry);
1424
			__add_hash_entry(new_hash, entry);
1425 1426 1427
		}
	}

1428
update:
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
	if (enable) {
		/* IPMODIFY should be updated only when filter_hash updating */
		ret = ftrace_hash_ipmodify_update(ops, new_hash);
		if (ret < 0) {
			free_ftrace_hash(new_hash);
			return ret;
		}
	}

1439 1440 1441 1442
	/*
	 * Remove the current set, update the hash and add
	 * them back.
	 */
1443
	ftrace_hash_rec_disable_modify(ops, enable);
1444

1445 1446
	rcu_assign_pointer(*dst, new_hash);

1447
	ftrace_hash_rec_enable_modify(ops, enable);
1448

1449
	return 0;
1450 1451
}

1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
static bool hash_contains_ip(unsigned long ip,
			     struct ftrace_ops_hash *hash)
{
	/*
	 * The function record is a match if it exists in the filter
	 * hash and not in the notrace hash. Note, an emty hash is
	 * considered a match for the filter hash, but an empty
	 * notrace hash is considered not in the notrace hash.
	 */
	return (ftrace_hash_empty(hash->filter_hash) ||
		ftrace_lookup_ip(hash->filter_hash, ip)) &&
		(ftrace_hash_empty(hash->notrace_hash) ||
		 !ftrace_lookup_ip(hash->notrace_hash, ip));
}

1467 1468 1469 1470 1471 1472 1473 1474
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1475 1476 1477
 *
 * This needs to be called with preemption disabled as
 * the hashes are freed with call_rcu_sched().
1478 1479
 */
static int
1480
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1481
{
1482
	struct ftrace_ops_hash hash;
1483 1484
	int ret;

1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	/*
	 * There's a small race when adding ops that the ftrace handler
	 * that wants regs, may be called without them. We can not
	 * allow that handler to be called if regs is NULL.
	 */
	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
		return 0;
#endif

1495 1496
	hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
	hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
1497

1498
	if (hash_contains_ip(ip, &hash))
1499 1500 1501 1502 1503 1504 1505
		ret = 1;
	else
		ret = 0;

	return ret;
}

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1520 1521 1522

static int ftrace_cmp_recs(const void *a, const void *b)
{
1523 1524
	const struct dyn_ftrace *key = a;
	const struct dyn_ftrace *rec = b;
1525

1526
	if (key->flags < rec->ip)
1527
		return -1;
1528 1529
	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
		return 1;
1530 1531 1532
	return 0;
}

1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545
/**
 * ftrace_location_range - return the first address of a traced location
 *	if it touches the given ip range
 * @start: start of range to search.
 * @end: end of range to search (inclusive). @end points to the last byte
 *	to check.
 *
 * Returns rec->ip if the related ftrace location is a least partly within
 * the given address range. That is, the first address of the instruction
 * that is either a NOP or call to the function tracer. It checks the ftrace
 * internal tables to determine if the address belongs or not.
 */
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1546 1547 1548
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1549
	struct dyn_ftrace key;
1550

1551 1552
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */
1553 1554

	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1555 1556
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1557
			continue;
1558 1559 1560 1561
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (rec)
1562
			return rec->ip;
1563
	}
1564 1565 1566 1567

	return 0;
}

1568 1569 1570 1571
/**
 * ftrace_location - return true if the ip giving is a traced location
 * @ip: the instruction pointer to check
 *
1572
 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1573 1574 1575 1576
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1577
unsigned long ftrace_location(unsigned long ip)
1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
{
	return ftrace_location_range(ip, ip);
}

/**
 * ftrace_text_reserved - return true if range contains an ftrace location
 * @start: start of range to search
 * @end: end of range to search (inclusive). @end points to the last byte to check.
 *
 * Returns 1 if @start and @end contains a ftrace location.
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1592
int ftrace_text_reserved(const void *start, const void *end)
1593
{
1594 1595 1596 1597 1598 1599
	unsigned long ret;

	ret = ftrace_location_range((unsigned long)start,
				    (unsigned long)end);

	return (int)!!ret;
1600 1601
}

1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
/* Test if ops registered to this rec needs regs */
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	bool keep_regs = false;

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next) {
		/* pass rec in as regs to have non-NULL val */
		if (ftrace_ops_test(ops, rec->ip, rec)) {
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				keep_regs = true;
				break;
			}
		}
	}

	return  keep_regs;
}

1622
static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1623 1624 1625 1626 1627 1628 1629
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1630
	bool update = false;
1631 1632 1633 1634 1635
	int count = 0;
	int all = 0;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1636
		return false;
1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
1650 1651
		hash = ops->func_hash->filter_hash;
		other_hash = ops->func_hash->notrace_hash;
1652
		if (ftrace_hash_empty(hash))
1653 1654 1655
			all = 1;
	} else {
		inc = !inc;
1656 1657
		hash = ops->func_hash->notrace_hash;
		other_hash = ops->func_hash->filter_hash;
1658 1659 1660 1661
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1662
		if (ftrace_hash_empty(hash))
1663
			return false;
1664 1665 1666 1667 1668 1669 1670
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

1671 1672 1673
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1674 1675 1676 1677 1678
		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1679
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1680 1681
				match = 1;
		} else {
1682 1683
			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1684 1685

			/*
1686 1687
			 * If filter_hash is set, we want to match all functions
			 * that are in the hash but not in the other hash.
1688
			 *
1689 1690 1691 1692 1693
			 * If filter_hash is not set, then we are decrementing.
			 * That means we match anything that is in the hash
			 * and also in the other_hash. That is, we need to turn
			 * off functions in the other hash because they are disabled
			 * by this hash.
1694 1695 1696 1697
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
1698
				 (in_other_hash || ftrace_hash_empty(other_hash)))
1699 1700 1701 1702 1703 1704 1705
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
1706
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1707
				return false;
1708 1709 1710 1711 1712 1713

			/*
			 * If there's only a single callback registered to a
			 * function, and the ops has a trampoline registered
			 * for it, then we can call it directly.
			 */
1714
			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1715
				rec->flags |= FTRACE_FL_TRAMP;
1716
			else
1717 1718 1719
				/*
				 * If we are adding another function callback
				 * to this function, and the previous had a
1720 1721
				 * custom trampoline in use, then we need to go
				 * back to the default trampoline.
1722
				 */
1723
				rec->flags &= ~FTRACE_FL_TRAMP;
1724

1725 1726 1727 1728 1729 1730
			/*
			 * If any ops wants regs saved for this function
			 * then all ops will get saved regs.
			 */
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
				rec->flags |= FTRACE_FL_REGS;
1731
		} else {
1732
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1733
				return false;
1734
			rec->flags--;
1735

1736 1737 1738 1739 1740 1741
			/*
			 * If the rec had REGS enabled and the ops that is
			 * being removed had REGS set, then see if there is
			 * still any ops for this record that wants regs.
			 * If not, we can stop recording them.
			 */
1742
			if (ftrace_rec_count(rec) > 0 &&
1743 1744 1745 1746 1747
			    rec->flags & FTRACE_FL_REGS &&
			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				if (!test_rec_ops_needs_regs(rec))
					rec->flags &= ~FTRACE_FL_REGS;
			}
1748

1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
			/*
			 * If the rec had TRAMP enabled, then it needs to
			 * be cleared. As TRAMP can only be enabled iff
			 * there is only a single ops attached to it.
			 * In otherwords, always disable it on decrementing.
			 * In the future, we may set it if rec count is
			 * decremented to one, and the ops that is left
			 * has a trampoline.
			 */
			rec->flags &= ~FTRACE_FL_TRAMP;

1760 1761 1762 1763
			/*
			 * flags will be cleared in ftrace_check_record()
			 * if rec count is zero.
			 */
1764 1765
		}
		count++;
1766 1767 1768 1769

		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;

1770 1771
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
1772
			return update;
1773
	} while_for_each_ftrace_rec();
1774 1775

	return update;
1776 1777
}

1778
static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1779 1780
				    int filter_hash)
{
1781
	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1782 1783
}

1784
static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1785 1786
				   int filter_hash)
{
1787
	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1788 1789
}

1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
					  int filter_hash, int inc)
{
	struct ftrace_ops *op;

	__ftrace_hash_rec_update(ops, filter_hash, inc);

	if (ops->func_hash != &global_ops.local_hash)
		return;

	/*
	 * If the ops shares the global_ops hash, then we need to update
	 * all ops that are enabled and use this hash.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* Already done */
		if (op == ops)
			continue;
		if (op->func_hash == &global_ops.local_hash)
			__ftrace_hash_rec_update(op, filter_hash, inc);
	} while_for_each_ftrace_op(op);
}

static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
					   int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
					  int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}

1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932
/*
 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
 * or no-needed to update, -EBUSY if it detects a conflict of the flag
 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
 * Note that old_hash and new_hash has below meanings
 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
 *  - If the hash is EMPTY_HASH, it hits nothing
 *  - Anything else hits the recs which match the hash entries.
 */
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
					 struct ftrace_hash *old_hash,
					 struct ftrace_hash *new_hash)
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec, *end = NULL;
	int in_old, in_new;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
		return 0;

	/*
	 * Since the IPMODIFY is a very address sensitive action, we do not
	 * allow ftrace_ops to set all functions to new hash.
	 */
	if (!new_hash || !old_hash)
		return -EINVAL;

	/* Update rec->flags */
	do_for_each_ftrace_rec(pg, rec) {
		/* We need to update only differences of filter_hash */
		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new) {
			/* New entries must ensure no others are using it */
			if (rec->flags & FTRACE_FL_IPMODIFY)
				goto rollback;
			rec->flags |= FTRACE_FL_IPMODIFY;
		} else /* Removed entry */
			rec->flags &= ~FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

	return 0;

rollback:
	end = rec;

	/* Roll back what we did above */
	do_for_each_ftrace_rec(pg, rec) {
		if (rec == end)
			goto err_out;

		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new)
			rec->flags &= ~FTRACE_FL_IPMODIFY;
		else
			rec->flags |= FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

err_out:
	return -EBUSY;
}

static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}

/* Disabling always succeeds */
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}

static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash)
{
	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(old_hash))
		old_hash = NULL;

	if (ftrace_hash_empty(new_hash))
		new_hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}

1933
static void print_ip_ins(const char *fmt, const unsigned char *p)
1934 1935 1936 1937 1938 1939 1940 1941 1942
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1943 1944
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1945 1946
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1947

1948
enum ftrace_bug_type ftrace_bug_type;
1949
const void *ftrace_expected;
1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970

static void print_bug_type(void)
{
	switch (ftrace_bug_type) {
	case FTRACE_BUG_UNKNOWN:
		break;
	case FTRACE_BUG_INIT:
		pr_info("Initializing ftrace call sites\n");
		break;
	case FTRACE_BUG_NOP:
		pr_info("Setting ftrace call site to NOP\n");
		break;
	case FTRACE_BUG_CALL:
		pr_info("Setting ftrace call site to call ftrace function\n");
		break;
	case FTRACE_BUG_UPDATE:
		pr_info("Updating ftrace call site to call a different ftrace function\n");
		break;
	}
}

1971 1972 1973
/**
 * ftrace_bug - report and shutdown function tracer
 * @failed: The failed type (EFAULT, EINVAL, EPERM)
1974
 * @rec: The record that failed
1975 1976 1977 1978 1979 1980 1981 1982
 *
 * The arch code that enables or disables the function tracing
 * can call ftrace_bug() when it has detected a problem in
 * modifying the code. @failed should be one of either:
 * EFAULT - if the problem happens on reading the @ip address
 * EINVAL - if what is read at @ip is not what was expected
 * EPERM - if the problem happens on writting to the @ip address
 */
1983
void ftrace_bug(int failed, struct dyn_ftrace *rec)
1984
{
1985 1986
	unsigned long ip = rec ? rec->ip : 0;

1987 1988 1989 1990 1991 1992 1993 1994 1995 1996
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
1997
		print_ip_ins(" actual:   ", (unsigned char *)ip);
1998
		pr_cont("\n");
1999 2000 2001 2002
		if (ftrace_expected) {
			print_ip_ins(" expected: ", ftrace_expected);
			pr_cont("\n");
		}
2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
2014
	print_bug_type();
2015 2016 2017 2018 2019 2020 2021 2022
	if (rec) {
		struct ftrace_ops *ops = NULL;

		pr_info("ftrace record flags: %lx\n", rec->flags);
		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
			ops = ftrace_find_tramp_ops_any(rec);
2023 2024 2025 2026 2027 2028 2029 2030
			if (ops) {
				do {
					pr_cont("\ttramp: %pS (%pS)",
						(void *)ops->trampoline,
						(void *)ops->func);
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
2031 2032 2033 2034
				pr_cont("\ttramp: ERROR!");

		}
		ip = ftrace_get_addr_curr(rec);
2035
		pr_cont("\n expected tramp: %lx\n", ip);
2036
	}
2037 2038
}

2039
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2040
{
2041
	unsigned long flag = 0UL;
2042

2043 2044
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2045 2046 2047
	if (rec->flags & FTRACE_FL_DISABLED)
		return FTRACE_UPDATE_IGNORE;

S
Steven Rostedt 已提交
2048
	/*
2049
	 * If we are updating calls:
S
Steven Rostedt 已提交
2050
	 *
2051 2052
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
S
Steven Rostedt 已提交
2053
	 *
2054 2055
	 *   Otherwise we make sure its disabled.
	 *
2056
	 * If we are disabling calls, then disable all records that
2057
	 * are enabled.
S
Steven Rostedt 已提交
2058
	 */
2059
	if (enable && ftrace_rec_count(rec))
2060
		flag = FTRACE_FL_ENABLED;
S
Steven Rostedt 已提交
2061

2062
	/*
2063 2064 2065
	 * If enabling and the REGS flag does not match the REGS_EN, or
	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
	 * this record. Set flags to fail the compare against ENABLED.
2066
	 */
2067 2068 2069 2070 2071 2072 2073 2074 2075
	if (flag) {
		if (!(rec->flags & FTRACE_FL_REGS) != 
		    !(rec->flags & FTRACE_FL_REGS_EN))
			flag |= FTRACE_FL_REGS;

		if (!(rec->flags & FTRACE_FL_TRAMP) != 
		    !(rec->flags & FTRACE_FL_TRAMP_EN))
			flag |= FTRACE_FL_TRAMP;
	}
2076

2077 2078
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2079
		return FTRACE_UPDATE_IGNORE;
S
Steven Rostedt 已提交
2080

2081
	if (flag) {
2082 2083 2084 2085
		/* Save off if rec is being enabled (for return value) */
		flag ^= rec->flags & FTRACE_FL_ENABLED;

		if (update) {
2086
			rec->flags |= FTRACE_FL_ENABLED;
2087 2088 2089 2090 2091 2092
			if (flag & FTRACE_FL_REGS) {
				if (rec->flags & FTRACE_FL_REGS)
					rec->flags |= FTRACE_FL_REGS_EN;
				else
					rec->flags &= ~FTRACE_FL_REGS_EN;
			}
2093 2094 2095 2096 2097 2098
			if (flag & FTRACE_FL_TRAMP) {
				if (rec->flags & FTRACE_FL_TRAMP)
					rec->flags |= FTRACE_FL_TRAMP_EN;
				else
					rec->flags &= ~FTRACE_FL_TRAMP_EN;
			}
2099 2100 2101 2102 2103 2104 2105
		}

		/*
		 * If this record is being updated from a nop, then
		 *   return UPDATE_MAKE_CALL.
		 * Otherwise,
		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2106
		 *   from the save regs, to a non-save regs function or
2107
		 *   vice versa, or from a trampoline call.
2108
		 */
2109 2110
		if (flag & FTRACE_FL_ENABLED) {
			ftrace_bug_type = FTRACE_BUG_CALL;
2111
			return FTRACE_UPDATE_MAKE_CALL;
2112
		}
2113

2114
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2115
		return FTRACE_UPDATE_MODIFY_CALL;
2116 2117
	}

2118 2119
	if (update) {
		/* If there's no more users, clear all flags */
2120
		if (!ftrace_rec_count(rec))
2121 2122
			rec->flags = 0;
		else
2123 2124 2125 2126 2127 2128
			/*
			 * Just disable the record, but keep the ops TRAMP
			 * and REGS states. The _EN flags must be disabled though.
			 */
			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
					FTRACE_FL_REGS_EN);
2129
	}
2130

2131
	ftrace_bug_type = FTRACE_BUG_NOP;
2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161
	return FTRACE_UPDATE_MAKE_NOP;
}

/**
 * ftrace_update_record, set a record that now is tracing or not
 * @rec: the record to update
 * @enable: set to 1 if the record is tracing, zero to force disable
 *
 * The records that represent all functions that can be traced need
 * to be updated when tracing has been enabled.
 */
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 1);
}

/**
 * ftrace_test_record, check if the record has been enabled or not
 * @rec: the record to test
 * @enable: set to 1 to check if enabled, 0 if it is disabled
 *
 * The arch code may need to test if a record is already set to
 * tracing to determine how to modify the function code that it
 * represents.
 */
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 0);
}

2162 2163 2164 2165
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2166
	unsigned long ip = rec->ip;
2167 2168 2169 2170 2171 2172

	do_for_each_ftrace_op(op, ftrace_ops_list) {

		if (!op->trampoline)
			continue;

2173
		if (hash_contains_ip(ip, op->func_hash))
2174 2175 2176 2177 2178 2179
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
			   struct ftrace_ops *op)
{
	unsigned long ip = rec->ip;

	while_for_each_ftrace_op(op) {

		if (!op->trampoline)
			continue;

		if (hash_contains_ip(ip, op->func_hash))
			return op;
	} 

	return NULL;
}

2198 2199 2200 2201
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2202
	unsigned long ip = rec->ip;
2203

2204 2205 2206 2207 2208 2209 2210 2211
	/*
	 * Need to check removed ops first.
	 * If they are being removed, and this rec has a tramp,
	 * and this rec is in the ops list, then it would be the
	 * one with the tramp.
	 */
	if (removed_ops) {
		if (hash_contains_ip(ip, &removed_ops->old_hash))
2212 2213 2214
			return removed_ops;
	}

2215 2216 2217 2218 2219 2220 2221
	/*
	 * Need to find the current trampoline for a rec.
	 * Now, a trampoline is only attached to a rec if there
	 * was a single 'ops' attached to it. But this can be called
	 * when we are adding another op to the rec or removing the
	 * current one. Thus, if the op is being added, we can
	 * ignore it because it hasn't attached itself to the rec
2222 2223 2224 2225 2226 2227 2228 2229 2230 2231
	 * yet.
	 *
	 * If an ops is being modified (hooking to different functions)
	 * then we don't care about the new functions that are being
	 * added, just the old ones (that are probably being removed).
	 *
	 * If we are adding an ops to a function that already is using
	 * a trampoline, it needs to be removed (trampolines are only
	 * for single ops connected), then an ops that is not being
	 * modified also needs to be checked.
2232
	 */
2233
	do_for_each_ftrace_op(op, ftrace_ops_list) {
2234 2235 2236 2237 2238 2239 2240 2241 2242

		if (!op->trampoline)
			continue;

		/*
		 * If the ops is being added, it hasn't gotten to
		 * the point to be removed from this tree yet.
		 */
		if (op->flags & FTRACE_OPS_FL_ADDING)
2243 2244
			continue;

2245

2246
		/*
2247 2248 2249
		 * If the ops is being modified and is in the old
		 * hash, then it is probably being removed from this
		 * function.
2250 2251 2252
		 */
		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, &op->old_hash))
2253
			return op;
2254 2255 2256 2257 2258 2259 2260 2261
		/*
		 * If the ops is not being added or modified, and it's
		 * in its normal filter hash, then this must be the one
		 * we want!
		 */
		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, op->func_hash))
			return op;
2262 2263 2264 2265 2266 2267 2268 2269 2270 2271

	} while_for_each_ftrace_op(op);

	return NULL;
}

static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2272
	unsigned long ip = rec->ip;
2273 2274 2275

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* pass rec in as regs to have non-NULL val */
2276
		if (hash_contains_ip(ip, op->func_hash))
2277 2278 2279 2280 2281 2282
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294
/**
 * ftrace_get_addr_new - Get the call address to set to
 * @rec:  The ftrace record descriptor
 *
 * If the record has the FTRACE_FL_REGS set, that means that it
 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
 * is not not set, then it wants to convert to the normal callback.
 *
 * Returns the address of the trampoline to set to
 */
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
2295 2296 2297 2298 2299 2300
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP) {
		ops = ftrace_find_tramp_ops_new(rec);
		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2301 2302
			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
				(void *)rec->ip, (void *)rec->ip, rec->flags);
2303 2304 2305 2306 2307 2308
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326
	if (rec->flags & FTRACE_FL_REGS)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

/**
 * ftrace_get_addr_curr - Get the call address that is already there
 * @rec:  The ftrace record descriptor
 *
 * The FTRACE_FL_REGS_EN is set when the record already points to
 * a function that saves all the regs. Basically the '_EN' version
 * represents the current state of the function.
 *
 * Returns the address of the trampoline that is currently being called
 */
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
2327 2328 2329 2330 2331 2332
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP_EN) {
		ops = ftrace_find_tramp_ops_curr(rec);
		if (FTRACE_WARN_ON(!ops)) {
2333 2334
			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
				(void *)rec->ip, (void *)rec->ip);
2335 2336 2337 2338 2339 2340
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2341 2342 2343 2344 2345 2346
	if (rec->flags & FTRACE_FL_REGS_EN)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

2347 2348 2349
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
2350
	unsigned long ftrace_old_addr;
2351 2352 2353
	unsigned long ftrace_addr;
	int ret;

2354
	ftrace_addr = ftrace_get_addr_new(rec);
2355

2356 2357 2358 2359
	/* This needs to be done before we call ftrace_update_record */
	ftrace_old_addr = ftrace_get_addr_curr(rec);

	ret = ftrace_update_record(rec, enable);
2360

2361 2362
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2363 2364 2365 2366 2367
	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
2368
		ftrace_bug_type = FTRACE_BUG_CALL;
2369
		return ftrace_make_call(rec, ftrace_addr);
2370 2371

	case FTRACE_UPDATE_MAKE_NOP:
2372
		ftrace_bug_type = FTRACE_BUG_NOP;
2373
		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2374 2375

	case FTRACE_UPDATE_MODIFY_CALL:
2376
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2377
		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2378 2379
	}

2380
	return -1; /* unknow ftrace bug */
2381 2382
}

2383
void __weak ftrace_replace_code(int enable)
2384 2385 2386
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
2387
	int failed;
2388

2389 2390 2391
	if (unlikely(ftrace_disabled))
		return;

2392
	do_for_each_ftrace_rec(pg, rec) {
2393
		failed = __ftrace_replace_code(rec, enable);
2394
		if (failed) {
2395
			ftrace_bug(failed, rec);
2396 2397
			/* Stop processing */
			return;
2398
		}
2399
	} while_for_each_ftrace_rec();
2400 2401
}

2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473
struct ftrace_rec_iter {
	struct ftrace_page	*pg;
	int			index;
};

/**
 * ftrace_rec_iter_start, start up iterating over traced functions
 *
 * Returns an iterator handle that is used to iterate over all
 * the records that represent address locations where functions
 * are traced.
 *
 * May return NULL if no records are available.
 */
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
	/*
	 * We only use a single iterator.
	 * Protected by the ftrace_lock mutex.
	 */
	static struct ftrace_rec_iter ftrace_rec_iter;
	struct ftrace_rec_iter *iter = &ftrace_rec_iter;

	iter->pg = ftrace_pages_start;
	iter->index = 0;

	/* Could have empty pages */
	while (iter->pg && !iter->pg->index)
		iter->pg = iter->pg->next;

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_next, get the next record to process.
 * @iter: The handle to the iterator.
 *
 * Returns the next iterator after the given iterator @iter.
 */
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
	iter->index++;

	if (iter->index >= iter->pg->index) {
		iter->pg = iter->pg->next;
		iter->index = 0;

		/* Could have empty pages */
		while (iter->pg && !iter->pg->index)
			iter->pg = iter->pg->next;
	}

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_record, get the record at the iterator location
 * @iter: The current iterator location
 *
 * Returns the record that the current @iter is at.
 */
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
	return &iter->pg->records[iter->index];
}

2474
static int
2475
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2476
{
2477
	int ret;
2478

2479 2480 2481
	if (unlikely(ftrace_disabled))
		return 0;

2482
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2483
	if (ret) {
2484
		ftrace_bug_type = FTRACE_BUG_INIT;
2485
		ftrace_bug(ret, rec);
2486
		return 0;
2487
	}
2488
	return 1;
2489 2490
}

2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

2509
void ftrace_modify_all_code(int command)
2510
{
2511
	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2512
	int err = 0;
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523

	/*
	 * If the ftrace_caller calls a ftrace_ops func directly,
	 * we need to make sure that it only traces functions it
	 * expects to trace. When doing the switch of functions,
	 * we need to update to the ftrace_ops_list_func first
	 * before the transition between old and new calls are set,
	 * as the ftrace_ops_list_func will check the ops hashes
	 * to make sure the ops are having the right functions
	 * traced.
	 */
2524 2525 2526 2527 2528
	if (update) {
		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
		if (FTRACE_WARN_ON(err))
			return;
	}
2529

2530
	if (command & FTRACE_UPDATE_CALLS)
2531
		ftrace_replace_code(1);
2532
	else if (command & FTRACE_DISABLE_CALLS)
2533 2534
		ftrace_replace_code(0);

2535 2536 2537 2538 2539 2540
	if (update && ftrace_trace_function != ftrace_ops_list_func) {
		function_trace_op = set_function_trace_op;
		smp_wmb();
		/* If irqs are disabled, we are in stop machine */
		if (!irqs_disabled())
			smp_call_function(ftrace_sync_ipi, NULL, 1);
2541 2542 2543
		err = ftrace_update_ftrace_func(ftrace_trace_function);
		if (FTRACE_WARN_ON(err))
			return;
2544
	}
2545

2546
	if (command & FTRACE_START_FUNC_RET)
2547
		err = ftrace_enable_ftrace_graph_caller();
2548
	else if (command & FTRACE_STOP_FUNC_RET)
2549 2550
		err = ftrace_disable_ftrace_graph_caller();
	FTRACE_WARN_ON(err);
2551 2552 2553 2554 2555 2556 2557
}

static int __ftrace_modify_code(void *data)
{
	int *command = data;

	ftrace_modify_all_code(*command);
2558

2559
	return 0;
2560 2561
}

2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585
/**
 * ftrace_run_stop_machine, go back to the stop machine method
 * @command: The command to tell ftrace what to do
 *
 * If an arch needs to fall back to the stop machine method, the
 * it can call this function.
 */
void ftrace_run_stop_machine(int command)
{
	stop_machine(__ftrace_modify_code, &command, NULL);
}

/**
 * arch_ftrace_update_code, modify the code to trace or not trace
 * @command: The command that needs to be done
 *
 * Archs can override this function if it does not need to
 * run stop_machine() to modify code.
 */
void __weak arch_ftrace_update_code(int command)
{
	ftrace_run_stop_machine(command);
}

I
Ingo Molnar 已提交
2586
static void ftrace_run_update_code(int command)
2587
{
2588 2589 2590 2591 2592 2593 2594
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;

2595 2596 2597 2598 2599 2600 2601 2602
	/*
	 * By default we use stop_machine() to modify the code.
	 * But archs can do what ever they want as long as it
	 * is safe. The stop_machine() is the safest, but also
	 * produces the most overhead.
	 */
	arch_ftrace_update_code(command);

2603 2604
	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
2605 2606
}

2607
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2608
				   struct ftrace_ops_hash *old_hash)
2609 2610
{
	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2611 2612
	ops->old_hash.filter_hash = old_hash->filter_hash;
	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2613
	ftrace_run_update_code(command);
2614
	ops->old_hash.filter_hash = NULL;
2615
	ops->old_hash.notrace_hash = NULL;
2616 2617 2618
	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}

2619
static ftrace_func_t saved_ftrace_func;
2620
static int ftrace_start_up;
2621

2622 2623 2624 2625
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}

2626
static void per_cpu_ops_free(struct ftrace_ops *ops)
2627 2628 2629 2630
{
	free_percpu(ops->disabled);
}

2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642
static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
2643

2644 2645 2646 2647 2648 2649 2650
static void ftrace_startup_all(int command)
{
	update_all_ops = true;
	ftrace_startup_enable(command);
	update_all_ops = false;
}

2651
static int ftrace_startup(struct ftrace_ops *ops, int command)
2652
{
2653
	int ret;
2654

2655
	if (unlikely(ftrace_disabled))
2656
		return -ENODEV;
2657

2658 2659 2660 2661
	ret = __register_ftrace_function(ops);
	if (ret)
		return ret;

2662
	ftrace_start_up++;
2663

2664 2665 2666 2667 2668 2669 2670 2671 2672
	/*
	 * Note that ftrace probes uses this to start up
	 * and modify functions it will probe. But we still
	 * set the ADDING flag for modification, as probes
	 * do not have trampolines. If they add them in the
	 * future, then the probes will need to distinguish
	 * between adding and updating probes.
	 */
	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2673

2674 2675 2676 2677 2678 2679 2680 2681 2682
	ret = ftrace_hash_ipmodify_enable(ops);
	if (ret < 0) {
		/* Rollback registration process */
		__unregister_ftrace_function(ops);
		ftrace_start_up--;
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
		return ret;
	}

2683 2684
	if (ftrace_hash_rec_enable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2685

2686
	ftrace_startup_enable(command);
2687

2688 2689
	ops->flags &= ~FTRACE_OPS_FL_ADDING;

2690
	return 0;
2691 2692
}

2693
static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2694
{
2695
	int ret;
2696

2697
	if (unlikely(ftrace_disabled))
2698 2699 2700 2701 2702
		return -ENODEV;

	ret = __unregister_ftrace_function(ops);
	if (ret)
		return ret;
2703

2704
	ftrace_start_up--;
2705 2706 2707 2708 2709 2710 2711
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

2712 2713
	/* Disabling ipmodify never fails */
	ftrace_hash_ipmodify_disable(ops);
2714

2715 2716
	if (ftrace_hash_rec_disable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2717

2718
	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2719

2720 2721 2722 2723
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
2724

2725 2726
	if (!command || !ftrace_enabled) {
		/*
2727
		 * If these are per_cpu ops, they still need their
2728 2729 2730 2731
		 * per_cpu field freed. Since, function tracing is
		 * not currently active, we can just free them
		 * without synchronizing all CPUs.
		 */
2732 2733
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2734
		return 0;
2735
	}
2736

2737 2738 2739 2740
	/*
	 * If the ops uses a trampoline, then it needs to be
	 * tested first on update.
	 */
2741
	ops->flags |= FTRACE_OPS_FL_REMOVING;
2742 2743
	removed_ops = ops;

2744 2745 2746 2747
	/* The trampoline logic checks the old hashes */
	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;

2748
	ftrace_run_update_code(command);
2749

2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764
	/*
	 * If there's no more ops registered with ftrace, run a
	 * sanity check to make sure all rec flags are cleared.
	 */
	if (ftrace_ops_list == &ftrace_list_end) {
		struct ftrace_page *pg;
		struct dyn_ftrace *rec;

		do_for_each_ftrace_rec(pg, rec) {
			if (FTRACE_WARN_ON_ONCE(rec->flags))
				pr_warn("  %pS flags:%lx\n",
					(void *)rec->ip, rec->flags);
		} while_for_each_ftrace_rec();
	}

2765 2766 2767 2768
	ops->old_hash.filter_hash = NULL;
	ops->old_hash.notrace_hash = NULL;

	removed_ops = NULL;
2769
	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2770

2771 2772 2773
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
2774
	 * The same goes for freeing the per_cpu data of the per_cpu
2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
	 * ops.
	 *
	 * Again, normal synchronize_sched() is not good enough.
	 * We need to do a hard force of sched synchronization.
	 * This is because we use preempt_disable() to do RCU, but
	 * the function tracers can be called where RCU is not watching
	 * (like before user_exit()). We can not rely on the RCU
	 * infrastructure to do the synchronization, thus we must do it
	 * ourselves.
	 */
2785
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) {
2786 2787
		schedule_on_each_cpu(ftrace_sync);

2788 2789
		arch_ftrace_trampoline_free(ops);

2790 2791
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2792 2793
	}

2794
	return 0;
2795 2796
}

I
Ingo Molnar 已提交
2797
static void ftrace_startup_sysctl(void)
2798
{
2799 2800
	int command;

2801 2802 2803
	if (unlikely(ftrace_disabled))
		return;

2804 2805
	/* Force update next time */
	saved_ftrace_func = NULL;
2806
	/* ftrace_start_up is true if we want ftrace running */
2807 2808 2809 2810
	if (ftrace_start_up) {
		command = FTRACE_UPDATE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_START_FUNC_RET;
2811
		ftrace_startup_enable(command);
2812
	}
2813 2814
}

I
Ingo Molnar 已提交
2815
static void ftrace_shutdown_sysctl(void)
2816
{
2817 2818
	int command;

2819 2820 2821
	if (unlikely(ftrace_disabled))
		return;

2822
	/* ftrace_start_up is true if ftrace is running */
2823 2824 2825 2826 2827 2828
	if (ftrace_start_up) {
		command = FTRACE_DISABLE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_STOP_FUNC_RET;
		ftrace_run_update_code(command);
	}
2829 2830
}

2831 2832 2833
static cycle_t		ftrace_update_time;
unsigned long		ftrace_update_tot_cnt;

2834
static inline int ops_traces_mod(struct ftrace_ops *ops)
2835
{
2836 2837 2838 2839
	/*
	 * Filter_hash being empty will default to trace module.
	 * But notrace hash requires a test of individual module functions.
	 */
2840 2841
	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
		ftrace_hash_empty(ops->func_hash->notrace_hash);
2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857
}

/*
 * Check if the current ops references the record.
 *
 * If the ops traces all functions, then it was already accounted for.
 * If the ops does not trace the current record function, skip it.
 * If the ops ignores the function via notrace filter, skip it.
 */
static inline bool
ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	/* If ops isn't enabled, ignore it */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

2858
	/* If ops traces all then it includes this function */
2859
	if (ops_traces_mod(ops))
2860
		return 1;
2861 2862

	/* The function must be in the filter */
2863 2864
	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
	    !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2865
		return 0;
2866

2867
	/* If in notrace hash, we ignore it too */
2868
	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2869 2870 2871 2872 2873
		return 0;

	return 1;
}

2874
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2875
{
2876
	struct ftrace_page *pg;
2877
	struct dyn_ftrace *p;
2878
	cycle_t start, stop;
2879
	unsigned long update_cnt = 0;
2880
	unsigned long rec_flags = 0;
2881
	int i;
2882

2883 2884
	start = ftrace_now(raw_smp_processor_id());

2885
	/*
2886 2887 2888 2889 2890 2891 2892 2893 2894
	 * When a module is loaded, this function is called to convert
	 * the calls to mcount in its text to nops, and also to create
	 * an entry in the ftrace data. Now, if ftrace is activated
	 * after this call, but before the module sets its text to
	 * read-only, the modification of enabling ftrace can fail if
	 * the read-only is done while ftrace is converting the calls.
	 * To prevent this, the module's records are set as disabled
	 * and will be enabled after the call to set the module's text
	 * to read-only.
2895
	 */
2896 2897
	if (mod)
		rec_flags |= FTRACE_FL_DISABLED;
2898

2899
	for (pg = new_pgs; pg; pg = pg->next) {
2900

2901
		for (i = 0; i < pg->index; i++) {
2902

2903 2904 2905
			/* If something went wrong, bail without enabling anything */
			if (unlikely(ftrace_disabled))
				return -1;
2906

2907
			p = &pg->records[i];
2908
			p->flags = rec_flags;
2909

2910 2911 2912 2913 2914 2915
			/*
			 * Do the initial record conversion from mcount jump
			 * to the NOP instructions.
			 */
			if (!ftrace_code_disable(mod, p))
				break;
2916

2917
			update_cnt++;
2918
		}
2919 2920
	}

I
Ingo Molnar 已提交
2921
	stop = ftrace_now(raw_smp_processor_id());
2922
	ftrace_update_time = stop - start;
2923
	ftrace_update_tot_cnt += update_cnt;
2924

2925 2926 2927
	return 0;
}

2928
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2929
{
2930
	int order;
2931 2932
	int cnt;

2933 2934 2935 2936
	if (WARN_ON(!count))
		return -EINVAL;

	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2937 2938

	/*
2939 2940
	 * We want to fill as much as possible. No more than a page
	 * may be empty.
2941
	 */
2942 2943
	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
		order--;
2944

2945 2946
 again:
	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2947

2948 2949 2950 2951 2952 2953 2954
	if (!pg->records) {
		/* if we can't allocate this size, try something smaller */
		if (!order)
			return -ENOMEM;
		order >>= 1;
		goto again;
	}
2955

2956 2957
	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
	pg->size = cnt;
2958

2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991
	if (cnt > count)
		cnt = count;

	return cnt;
}

static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
	struct ftrace_page *start_pg;
	struct ftrace_page *pg;
	int order;
	int cnt;

	if (!num_to_init)
		return 0;

	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
	if (!pg)
		return NULL;

	/*
	 * Try to allocate as much as possible in one continues
	 * location that fills in all of the space. We want to
	 * waste as little space as possible.
	 */
	for (;;) {
		cnt = ftrace_allocate_records(pg, num_to_init);
		if (cnt < 0)
			goto free_pages;

		num_to_init -= cnt;
		if (!num_to_init)
2992 2993
			break;

2994 2995 2996 2997
		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
		if (!pg->next)
			goto free_pages;

2998 2999 3000
		pg = pg->next;
	}

3001 3002 3003
	return start_pg;

 free_pages:
3004 3005
	pg = start_pg;
	while (pg) {
3006 3007 3008 3009 3010 3011 3012 3013 3014 3015
		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		start_pg = pg->next;
		kfree(pg);
		pg = start_pg;
	}
	pr_info("ftrace: FAILED to allocate memory for functions\n");
	return NULL;
}

3016 3017 3018
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
3019
	loff_t				pos;
3020 3021 3022 3023 3024
	loff_t				func_pos;
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
	struct trace_parser		parser;
3025
	struct ftrace_hash		*hash;
3026
	struct ftrace_ops		*ops;
3027 3028 3029
	int				hidx;
	int				idx;
	unsigned			flags;
3030 3031
};

3032
static void *
3033
t_hash_next(struct seq_file *m, loff_t *pos)
3034 3035
{
	struct ftrace_iterator *iter = m->private;
3036
	struct hlist_node *hnd = NULL;
3037 3038 3039
	struct hlist_head *hhd;

	(*pos)++;
3040
	iter->pos = *pos;
3041

3042 3043
	if (iter->probe)
		hnd = &iter->probe->node;
3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065
 retry:
	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
		return NULL;

	hhd = &ftrace_func_hash[iter->hidx];

	if (hlist_empty(hhd)) {
		iter->hidx++;
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
			iter->hidx++;
			goto retry;
		}
	}

3066 3067 3068 3069 3070 3071
	if (WARN_ON_ONCE(!hnd))
		return NULL;

	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);

	return iter;
3072 3073 3074 3075 3076 3077
}

static void *t_hash_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
3078 3079
	loff_t l;

3080 3081 3082
	if (!(iter->flags & FTRACE_ITER_DO_HASH))
		return NULL;

3083 3084
	if (iter->func_pos > *pos)
		return NULL;
3085

L
Li Zefan 已提交
3086
	iter->hidx = 0;
3087
	for (l = 0; l <= (*pos - iter->func_pos); ) {
3088
		p = t_hash_next(m, &l);
L
Li Zefan 已提交
3089 3090 3091
		if (!p)
			break;
	}
3092 3093 3094
	if (!p)
		return NULL;

3095 3096 3097
	/* Only set this if we have an item */
	iter->flags |= FTRACE_ITER_HASH;

3098
	return iter;
3099 3100
}

3101 3102
static int
t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
3103
{
S
Steven Rostedt 已提交
3104
	struct ftrace_func_probe *rec;
3105

3106 3107 3108
	rec = iter->probe;
	if (WARN_ON_ONCE(!rec))
		return -EIO;
3109

3110 3111 3112
	if (rec->ops->print)
		return rec->ops->print(m, rec->ip, rec->ops, rec->data);

3113
	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
3114 3115 3116 3117 3118 3119 3120 3121

	if (rec->data)
		seq_printf(m, ":%p", rec->data);
	seq_putc(m, '\n');

	return 0;
}

I
Ingo Molnar 已提交
3122
static void *
3123 3124 3125
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3126
	struct ftrace_ops *ops = iter->ops;
3127 3128
	struct dyn_ftrace *rec = NULL;

3129 3130 3131
	if (unlikely(ftrace_disabled))
		return NULL;

3132
	if (iter->flags & FTRACE_ITER_HASH)
3133
		return t_hash_next(m, pos);
3134

3135
	(*pos)++;
3136
	iter->pos = iter->func_pos = *pos;
3137

3138
	if (iter->flags & FTRACE_ITER_PRINTALL)
3139
		return t_hash_start(m, pos);
3140

3141 3142 3143 3144 3145 3146 3147 3148 3149
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
3150
		if (((iter->flags & FTRACE_ITER_FILTER) &&
3151
		     !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) ||
S
Steven Rostedt 已提交
3152

3153
		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
3154
		     !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) ||
3155 3156

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3157
		     !(rec->flags & FTRACE_FL_ENABLED))) {
3158

3159 3160 3161 3162 3163
			rec = NULL;
			goto retry;
		}
	}

3164
	if (!rec)
3165
		return t_hash_start(m, pos);
3166 3167 3168 3169

	iter->func = rec;

	return iter;
3170 3171
}

3172 3173 3174 3175
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
3176
	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
3177 3178 3179 3180 3181
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3182
	struct ftrace_ops *ops = iter->ops;
3183
	void *p = NULL;
3184
	loff_t l;
3185

3186
	mutex_lock(&ftrace_lock);
3187 3188 3189 3190

	if (unlikely(ftrace_disabled))
		return NULL;

3191 3192 3193 3194 3195 3196
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

3197 3198 3199 3200 3201
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
3202
	if ((iter->flags & FTRACE_ITER_FILTER &&
3203
	     ftrace_hash_empty(ops->func_hash->filter_hash)) ||
3204
	    (iter->flags & FTRACE_ITER_NOTRACE &&
3205
	     ftrace_hash_empty(ops->func_hash->notrace_hash))) {
3206
		if (*pos > 0)
3207
			return t_hash_start(m, pos);
3208
		iter->flags |= FTRACE_ITER_PRINTALL;
3209 3210
		/* reset in case of seek/pread */
		iter->flags &= ~FTRACE_ITER_HASH;
3211 3212 3213
		return iter;
	}

3214 3215 3216
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_start(m, pos);

3217 3218 3219 3220 3221
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
3222 3223 3224 3225 3226 3227
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
		p = t_next(m, p, &l);
		if (!p)
			break;
3228
	}
3229

3230 3231
	if (!p)
		return t_hash_start(m, pos);
3232 3233

	return iter;
3234 3235 3236 3237
}

static void t_stop(struct seq_file *m, void *p)
{
3238
	mutex_unlock(&ftrace_lock);
3239 3240
}

3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	return NULL;
}

static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
				struct dyn_ftrace *rec)
{
	void *ptr;

	ptr = arch_ftrace_trampoline_func(ops, rec);
	if (ptr)
		seq_printf(m, " ->%pS", ptr);
}

3257 3258
static int t_show(struct seq_file *m, void *v)
{
3259
	struct ftrace_iterator *iter = m->private;
3260
	struct dyn_ftrace *rec;
3261

3262
	if (iter->flags & FTRACE_ITER_HASH)
3263
		return t_hash_show(m, iter);
3264

3265
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3266
		if (iter->flags & FTRACE_ITER_NOTRACE)
3267
			seq_puts(m, "#### no functions disabled ####\n");
3268
		else
3269
			seq_puts(m, "#### all functions enabled ####\n");
3270 3271 3272
		return 0;
	}

3273 3274
	rec = iter->func;

3275 3276 3277
	if (!rec)
		return 0;

3278
	seq_printf(m, "%ps", (void *)rec->ip);
3279
	if (iter->flags & FTRACE_ITER_ENABLED) {
3280
		struct ftrace_ops *ops;
3281

3282
		seq_printf(m, " (%ld)%s%s",
3283
			   ftrace_rec_count(rec),
3284 3285
			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
3286
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3287
			ops = ftrace_find_tramp_ops_any(rec);
3288 3289 3290 3291 3292
			if (ops) {
				do {
					seq_printf(m, "\ttramp: %pS (%pS)",
						   (void *)ops->trampoline,
						   (void *)ops->func);
3293
					add_trampoline_func(m, ops, rec);
3294 3295 3296
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
3297
				seq_puts(m, "\ttramp: ERROR!");
3298 3299
		} else {
			add_trampoline_func(m, NULL, rec);
3300 3301 3302
		}
	}	

3303
	seq_putc(m, '\n');
3304 3305 3306 3307

	return 0;
}

J
James Morris 已提交
3308
static const struct seq_operations show_ftrace_seq_ops = {
3309 3310 3311 3312 3313 3314
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
3315
static int
3316 3317 3318 3319
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3320 3321 3322
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3323 3324 3325 3326
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
	if (iter) {
		iter->pg = ftrace_pages_start;
		iter->ops = &global_ops;
I
Ingo Molnar 已提交
3327
	}
3328

3329
	return iter ? 0 : -ENOMEM;
3330 3331
}

3332 3333 3334 3335 3336
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3337 3338 3339 3340 3341
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
	if (iter) {
		iter->pg = ftrace_pages_start;
		iter->flags = FTRACE_ITER_ENABLED;
		iter->ops = &global_ops;
3342 3343
	}

3344
	return iter ? 0 : -ENOMEM;
3345 3346
}

3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359
/**
 * ftrace_regex_open - initialize function tracer filter files
 * @ops: The ftrace_ops that hold the hash filters
 * @flag: The type of filter to process
 * @inode: The inode, usually passed in to your open routine
 * @file: The file, usually passed in to your open routine
 *
 * ftrace_regex_open() initializes the filter files for the
 * @ops. Depending on @flag it may process the filter hash or
 * the notrace hash of @ops. With this called from the open
 * routine, you can use ftrace_filter_write() for the write
 * routine if @flag has FTRACE_ITER_FILTER set, or
 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3360
 * tracing_lseek() should be used as the lseek routine, and
3361 3362 3363
 * release must call ftrace_regex_release().
 */
int
3364
ftrace_regex_open(struct ftrace_ops *ops, int flag,
3365
		  struct inode *inode, struct file *file)
3366 3367
{
	struct ftrace_iterator *iter;
3368
	struct ftrace_hash *hash;
3369 3370
	int ret = 0;

3371 3372
	ftrace_ops_init(ops);

3373 3374 3375
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3376 3377 3378 3379
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

3380 3381 3382 3383 3384
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

3385 3386 3387
	iter->ops = ops;
	iter->flags = flag;

3388
	mutex_lock(&ops->func_hash->regex_lock);
3389

3390
	if (flag & FTRACE_ITER_NOTRACE)
3391
		hash = ops->func_hash->notrace_hash;
3392
	else
3393
		hash = ops->func_hash->filter_hash;
3394

3395
	if (file->f_mode & FMODE_WRITE) {
3396 3397 3398 3399 3400 3401 3402
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

		if (file->f_flags & O_TRUNC)
			iter->hash = alloc_ftrace_hash(size_bits);
		else
			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);

3403 3404 3405
		if (!iter->hash) {
			trace_parser_put(&iter->parser);
			kfree(iter);
3406 3407
			ret = -ENOMEM;
			goto out_unlock;
3408 3409
		}
	}
3410

3411 3412 3413 3414 3415 3416 3417
	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
3418
		} else {
3419 3420
			/* Failed */
			free_ftrace_hash(iter->hash);
3421
			trace_parser_put(&iter->parser);
3422
			kfree(iter);
3423
		}
3424 3425
	} else
		file->private_data = iter;
3426 3427

 out_unlock:
3428
	mutex_unlock(&ops->func_hash->regex_lock);
3429 3430 3431 3432

	return ret;
}

3433 3434 3435
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
3436 3437 3438
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops,
3439 3440
			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
			inode, file);
3441 3442 3443 3444 3445
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
3446 3447 3448
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3449
				 inode, file);
3450 3451
}

3452 3453 3454 3455 3456 3457 3458
/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
struct ftrace_glob {
	char *search;
	unsigned len;
	int type;
};

3459 3460 3461 3462 3463 3464 3465 3466 3467 3468
/*
 * If symbols in an architecture don't correspond exactly to the user-visible
 * name of what they represent, it is possible to define this function to
 * perform the necessary adjustments.
*/
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
	return str;
}

3469
static int ftrace_match(char *str, struct ftrace_glob *g)
3470 3471
{
	int matched = 0;
3472
	int slen;
3473

3474 3475
	str = arch_ftrace_match_adjust(str, g->search);

3476
	switch (g->type) {
3477
	case MATCH_FULL:
3478
		if (strcmp(str, g->search) == 0)
3479 3480 3481
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
3482
		if (strncmp(str, g->search, g->len) == 0)
3483 3484 3485
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
3486
		if (strstr(str, g->search))
3487 3488 3489
			matched = 1;
		break;
	case MATCH_END_ONLY:
3490
		slen = strlen(str);
3491 3492
		if (slen >= g->len &&
		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3493 3494 3495 3496 3497 3498 3499
			matched = 1;
		break;
	}

	return matched;
}

3500
static int
3501
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3502
{
3503 3504 3505
	struct ftrace_func_entry *entry;
	int ret = 0;

3506
	entry = ftrace_lookup_ip(hash, rec->ip);
3507
	if (clear_filter) {
3508 3509 3510
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
3511

3512
		free_hash_entry(hash, entry);
3513 3514 3515 3516
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
3517

3518
		ret = add_hash_entry(hash, rec->ip);
3519 3520
	}
	return ret;
3521 3522
}

3523
static int
D
Dmitry Safonov 已提交
3524 3525
ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
		struct ftrace_glob *mod_g, int exclude_mod)
3526 3527
{
	char str[KSYM_SYMBOL_LEN];
3528 3529 3530 3531
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

D
Dmitry Safonov 已提交
3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
	if (mod_g) {
		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;

		/* blank module name to match all modules */
		if (!mod_g->len) {
			/* blank module globbing: modname xor exclude_mod */
			if ((!exclude_mod) != (!modname))
				goto func_match;
			return 0;
		}

		/* not matching the module */
		if (!modname || !mod_matches) {
			if (exclude_mod)
				goto func_match;
			else
				return 0;
		}

		if (mod_matches && exclude_mod)
3552 3553
			return 0;

D
Dmitry Safonov 已提交
3554
func_match:
3555
		/* blank search means to match all funcs in the mod */
3556
		if (!func_g->len)
3557 3558
			return 1;
	}
3559

3560
	return ftrace_match(str, func_g);
3561 3562
}

3563
static int
3564
match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3565 3566 3567
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3568
	struct ftrace_glob func_g = { .type = MATCH_FULL };
D
Dmitry Safonov 已提交
3569 3570 3571
	struct ftrace_glob mod_g = { .type = MATCH_FULL };
	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
	int exclude_mod = 0;
3572
	int found = 0;
3573
	int ret;
3574
	int clear_filter;
3575

D
Dmitry Safonov 已提交
3576
	if (func) {
3577 3578 3579
		func_g.type = filter_parse_regex(func, len, &func_g.search,
						 &clear_filter);
		func_g.len = strlen(func_g.search);
3580
	}
3581

D
Dmitry Safonov 已提交
3582 3583 3584 3585
	if (mod) {
		mod_g.type = filter_parse_regex(mod, strlen(mod),
				&mod_g.search, &exclude_mod);
		mod_g.len = strlen(mod_g.search);
3586
	}
3587

3588
	mutex_lock(&ftrace_lock);
3589

3590 3591
	if (unlikely(ftrace_disabled))
		goto out_unlock;
3592

3593
	do_for_each_ftrace_rec(pg, rec) {
D
Dmitry Safonov 已提交
3594
		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3595
			ret = enter_record(hash, rec, clear_filter);
3596 3597 3598 3599
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
3600
			found = 1;
3601 3602
		}
	} while_for_each_ftrace_rec();
3603
 out_unlock:
3604
	mutex_unlock(&ftrace_lock);
3605 3606

	return found;
3607 3608
}

3609
static int
3610
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3611
{
3612
	return match_records(hash, buff, len, NULL);
3613 3614 3615
}


3616 3617 3618 3619 3620 3621
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
3622
ftrace_mod_callback(struct ftrace_hash *hash,
3623
		    char *func, char *cmd, char *module, int enable)
3624
{
3625
	int ret;
3626 3627 3628 3629 3630 3631 3632 3633

	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */
3634
	ret = match_records(hash, func, strlen(func), module);
3635
	if (!ret)
3636
		return -EINVAL;
3637 3638 3639
	if (ret < 0)
		return ret;
	return 0;
3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
3651
core_initcall(ftrace_mod_cmd_init);
3652

3653
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
3654
				      struct ftrace_ops *op, struct pt_regs *pt_regs)
3655
{
S
Steven Rostedt 已提交
3656
	struct ftrace_func_probe *entry;
3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671
	struct hlist_head *hhd;
	unsigned long key;

	key = hash_long(ip, FTRACE_HASH_BITS);

	hhd = &ftrace_func_hash[key];

	if (hlist_empty(hhd))
		return;

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
3672
	preempt_disable_notrace();
3673
	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
3674 3675 3676
		if (entry->ip == ip)
			entry->ops->func(ip, parent_ip, &entry->data);
	}
3677
	preempt_enable_notrace();
3678 3679
}

S
Steven Rostedt 已提交
3680
static struct ftrace_ops trace_probe_ops __read_mostly =
3681
{
3682
	.func		= function_trace_probe_call,
3683
	.flags		= FTRACE_OPS_FL_INITIALIZED,
3684
	INIT_OPS_HASH(trace_probe_ops)
3685 3686
};

S
Steven Rostedt 已提交
3687
static int ftrace_probe_registered;
3688

3689
static void __enable_ftrace_function_probe(struct ftrace_ops_hash *old_hash)
3690
{
3691
	int ret;
3692 3693
	int i;

3694 3695 3696
	if (ftrace_probe_registered) {
		/* still need to update the function call sites */
		if (ftrace_enabled)
3697 3698
			ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS,
					       old_hash);
3699
		return;
3700
	}
3701 3702 3703 3704 3705 3706 3707 3708 3709 3710

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			break;
	}
	/* Nothing registered? */
	if (i == FTRACE_FUNC_HASHSIZE)
		return;

3711
	ret = ftrace_startup(&trace_probe_ops, 0);
3712

S
Steven Rostedt 已提交
3713
	ftrace_probe_registered = 1;
3714 3715
}

S
Steven Rostedt 已提交
3716
static void __disable_ftrace_function_probe(void)
3717 3718 3719
{
	int i;

S
Steven Rostedt 已提交
3720
	if (!ftrace_probe_registered)
3721 3722 3723 3724 3725 3726 3727 3728 3729
		return;

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			return;
	}

	/* no more funcs left */
3730
	ftrace_shutdown(&trace_probe_ops, 0);
3731

S
Steven Rostedt 已提交
3732
	ftrace_probe_registered = 0;
3733 3734 3735
}


3736
static void ftrace_free_entry(struct ftrace_func_probe *entry)
3737 3738
{
	if (entry->ops->free)
3739
		entry->ops->free(entry->ops, entry->ip, &entry->data);
3740 3741 3742 3743
	kfree(entry);
}

int
S
Steven Rostedt 已提交
3744
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3745 3746
			      void *data)
{
3747
	struct ftrace_ops_hash old_hash_ops;
S
Steven Rostedt 已提交
3748
	struct ftrace_func_probe *entry;
3749
	struct ftrace_glob func_g;
3750
	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
3751
	struct ftrace_hash *old_hash = *orig_hash;
3752
	struct ftrace_hash *hash;
3753 3754
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3755
	int not;
S
Steven Rostedt 已提交
3756
	unsigned long key;
3757
	int count = 0;
3758
	int ret;
3759

3760 3761 3762
	func_g.type = filter_parse_regex(glob, strlen(glob),
			&func_g.search, &not);
	func_g.len = strlen(func_g.search);
3763

S
Steven Rostedt 已提交
3764
	/* we do not support '!' for function probes */
3765 3766 3767
	if (WARN_ON(not))
		return -EINVAL;

3768
	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
3769

3770 3771 3772 3773
	old_hash_ops.filter_hash = old_hash;
	/* Probes only have filters */
	old_hash_ops.notrace_hash = NULL;

3774
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
3775 3776
	if (!hash) {
		count = -ENOMEM;
3777
		goto out;
3778 3779 3780 3781
	}

	if (unlikely(ftrace_disabled)) {
		count = -ENODEV;
3782
		goto out;
3783
	}
3784

3785 3786
	mutex_lock(&ftrace_lock);

3787
	do_for_each_ftrace_rec(pg, rec) {
3788

D
Dmitry Safonov 已提交
3789
		if (!ftrace_match_record(rec, &func_g, NULL, 0))
3790 3791 3792 3793
			continue;

		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
		if (!entry) {
S
Steven Rostedt 已提交
3794
			/* If we did not process any, then return error */
3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
			if (!count)
				count = -ENOMEM;
			goto out_unlock;
		}

		count++;

		entry->data = data;

		/*
		 * The caller might want to do something special
		 * for each function we find. We call the callback
		 * to give the caller an opportunity to do so.
		 */
3809 3810
		if (ops->init) {
			if (ops->init(ops, rec->ip, &entry->data) < 0) {
3811 3812 3813 3814 3815 3816
				/* caller does not like this func */
				kfree(entry);
				continue;
			}
		}

3817 3818 3819 3820 3821 3822 3823
		ret = enter_record(hash, rec, 0);
		if (ret < 0) {
			kfree(entry);
			count = ret;
			goto out_unlock;
		}

3824 3825 3826 3827 3828 3829 3830
		entry->ops = ops;
		entry->ip = rec->ip;

		key = hash_long(entry->ip, FTRACE_HASH_BITS);
		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);

	} while_for_each_ftrace_rec();
3831 3832

	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3833

3834
	__enable_ftrace_function_probe(&old_hash_ops);
3835

3836 3837 3838
	if (!ret)
		free_ftrace_hash_rcu(old_hash);
	else
3839 3840
		count = ret;

3841
 out_unlock:
3842 3843
	mutex_unlock(&ftrace_lock);
 out:
3844
	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
3845
	free_ftrace_hash(hash);
3846 3847 3848 3849 3850

	return count;
}

enum {
S
Steven Rostedt 已提交
3851 3852
	PROBE_TEST_FUNC		= 1,
	PROBE_TEST_DATA		= 2
3853 3854 3855
};

static void
S
Steven Rostedt 已提交
3856
__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3857 3858
				  void *data, int flags)
{
3859
	struct ftrace_func_entry *rec_entry;
S
Steven Rostedt 已提交
3860
	struct ftrace_func_probe *entry;
3861
	struct ftrace_func_probe *p;
3862
	struct ftrace_glob func_g;
3863
	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
3864
	struct ftrace_hash *old_hash = *orig_hash;
3865
	struct list_head free_list;
3866
	struct ftrace_hash *hash;
3867
	struct hlist_node *tmp;
3868
	char str[KSYM_SYMBOL_LEN];
3869
	int i, ret;
3870

3871
	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
3872
		func_g.search = NULL;
3873
	else if (glob) {
3874 3875
		int not;

3876 3877 3878 3879
		func_g.type = filter_parse_regex(glob, strlen(glob),
						 &func_g.search, &not);
		func_g.len = strlen(func_g.search);
		func_g.search = glob;
3880

S
Steven Rostedt 已提交
3881
		/* we do not support '!' for function probes */
3882 3883 3884 3885
		if (WARN_ON(not))
			return;
	}

3886
	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
3887 3888 3889 3890 3891 3892

	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
	if (!hash)
		/* Hmm, should report this somehow */
		goto out_unlock;

3893 3894
	INIT_LIST_HEAD(&free_list);

3895 3896 3897
	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];

3898
		hlist_for_each_entry_safe(entry, tmp, hhd, node) {
3899 3900

			/* break up if statements for readability */
S
Steven Rostedt 已提交
3901
			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3902 3903
				continue;

S
Steven Rostedt 已提交
3904
			if ((flags & PROBE_TEST_DATA) && entry->data != data)
3905 3906 3907
				continue;

			/* do this last, since it is the most expensive */
3908
			if (func_g.search) {
3909 3910
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
3911
				if (!ftrace_match(str, &func_g))
3912 3913 3914
					continue;
			}

3915 3916 3917 3918 3919
			rec_entry = ftrace_lookup_ip(hash, entry->ip);
			/* It is possible more than one entry had this ip */
			if (rec_entry)
				free_hash_entry(hash, rec_entry);

3920
			hlist_del_rcu(&entry->node);
3921
			list_add(&entry->free_list, &free_list);
3922 3923
		}
	}
3924
	mutex_lock(&ftrace_lock);
S
Steven Rostedt 已提交
3925
	__disable_ftrace_function_probe();
3926 3927 3928 3929
	/*
	 * Remove after the disable is called. Otherwise, if the last
	 * probe is removed, a null hash means *all enabled*.
	 */
3930
	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
3931
	synchronize_sched();
3932 3933 3934
	if (!ret)
		free_ftrace_hash_rcu(old_hash);

3935 3936 3937 3938
	list_for_each_entry_safe(entry, p, &free_list, free_list) {
		list_del(&entry->free_list);
		ftrace_free_entry(entry);
	}
3939
	mutex_unlock(&ftrace_lock);
3940

3941
 out_unlock:
3942
	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
3943
	free_ftrace_hash(hash);
3944 3945 3946
}

void
S
Steven Rostedt 已提交
3947
unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3948 3949
				void *data)
{
S
Steven Rostedt 已提交
3950 3951
	__unregister_ftrace_function_probe(glob, ops, data,
					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
3952 3953 3954
}

void
S
Steven Rostedt 已提交
3955
unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
3956
{
S
Steven Rostedt 已提交
3957
	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
3958 3959
}

S
Steven Rostedt 已提交
3960
void unregister_ftrace_function_probe_all(char *glob)
3961
{
S
Steven Rostedt 已提交
3962
	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
3963 3964
}

3965 3966 3967
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

3968 3969 3970 3971 3972
/*
 * Currently we only register ftrace commands from __init, so mark this
 * __init too.
 */
__init int register_ftrace_command(struct ftrace_func_command *cmd)
3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

3991 3992 3993 3994 3995
/*
 * Currently we only unregister ftrace commands from __init, so mark
 * this __init too.
 */
__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4014 4015
static int ftrace_process_regex(struct ftrace_hash *hash,
				char *buff, int len, int enable)
4016
{
4017
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
4018
	struct ftrace_func_command *p;
4019
	int ret = -EINVAL;
4020 4021 4022 4023

	func = strsep(&next, ":");

	if (!next) {
4024
		ret = ftrace_match_records(hash, func, len);
4025 4026 4027 4028 4029
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
4030 4031
	}

4032
	/* command found */
4033 4034 4035

	command = strsep(&next, ":");

4036 4037 4038
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
4039
			ret = p->func(hash, func, command, next, enable);
4040 4041
			goto out_unlock;
		}
4042
	}
4043 4044
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
4045

4046
	return ret;
4047 4048
}

I
Ingo Molnar 已提交
4049
static ssize_t
4050 4051
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
4052 4053
{
	struct ftrace_iterator *iter;
4054 4055
	struct trace_parser *parser;
	ssize_t ret, read;
4056

4057
	if (!cnt)
4058 4059 4060 4061 4062 4063 4064 4065
		return 0;

	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

4066
	if (unlikely(ftrace_disabled))
4067 4068 4069
		return -ENODEV;

	/* iter->hash is a local copy, so we don't need regex_lock */
4070

4071 4072
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
4073

4074
	if (read >= 0 && trace_parser_loaded(parser) &&
4075
	    !trace_parser_cont(parser)) {
4076
		ret = ftrace_process_regex(iter->hash, parser->buffer,
4077
					   parser->idx, enable);
4078
		trace_parser_clear(parser);
4079
		if (ret < 0)
4080
			goto out;
4081
	}
4082 4083

	ret = read;
4084
 out:
4085 4086 4087
	return ret;
}

4088
ssize_t
4089 4090 4091 4092 4093 4094
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

4095
ssize_t
4096 4097 4098 4099 4100 4101
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

4102
static int
4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120
ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
	struct ftrace_func_entry *entry;

	if (!ftrace_location(ip))
		return -EINVAL;

	if (remove) {
		entry = ftrace_lookup_ip(hash, ip);
		if (!entry)
			return -ENOENT;
		free_hash_entry(hash, entry);
		return 0;
	}

	return add_hash_entry(hash, ip);
}

4121
static void ftrace_ops_update_code(struct ftrace_ops *ops,
4122
				   struct ftrace_ops_hash *old_hash)
4123
{
4124 4125 4126 4127 4128 4129
	struct ftrace_ops *op;

	if (!ftrace_enabled)
		return;

	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
4130
		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149
		return;
	}

	/*
	 * If this is the shared global_ops filter, then we need to
	 * check if there is another ops that shares it, is enabled.
	 * If so, we still need to run the modify code.
	 */
	if (ops->func_hash != &global_ops.local_hash)
		return;

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->func_hash == &global_ops.local_hash &&
		    op->flags & FTRACE_OPS_FL_ENABLED) {
			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
			/* Only need to do this once */
			return;
		}
	} while_for_each_ftrace_op(op);
4150 4151
}

4152 4153 4154
static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
		unsigned long ip, int remove, int reset, int enable)
4155
{
4156
	struct ftrace_hash **orig_hash;
4157
	struct ftrace_ops_hash old_hash_ops;
4158
	struct ftrace_hash *old_hash;
4159
	struct ftrace_hash *hash;
4160
	int ret;
4161

4162
	if (unlikely(ftrace_disabled))
4163
		return -ENODEV;
4164

4165
	mutex_lock(&ops->func_hash->regex_lock);
4166

4167
	if (enable)
4168
		orig_hash = &ops->func_hash->filter_hash;
4169
	else
4170
		orig_hash = &ops->func_hash->notrace_hash;
4171

4172 4173 4174 4175 4176
	if (reset)
		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	else
		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);

4177 4178 4179 4180
	if (!hash) {
		ret = -ENOMEM;
		goto out_regex_unlock;
	}
4181

4182 4183 4184 4185
	if (buf && !ftrace_match_records(hash, buf, len)) {
		ret = -EINVAL;
		goto out_regex_unlock;
	}
4186 4187 4188 4189 4190
	if (ip) {
		ret = ftrace_match_addr(hash, ip, remove);
		if (ret < 0)
			goto out_regex_unlock;
	}
4191 4192

	mutex_lock(&ftrace_lock);
4193
	old_hash = *orig_hash;
4194 4195
	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
4196
	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
4197
	if (!ret) {
4198
		ftrace_ops_update_code(ops, &old_hash_ops);
4199 4200
		free_ftrace_hash_rcu(old_hash);
	}
4201 4202
	mutex_unlock(&ftrace_lock);

4203
 out_regex_unlock:
4204
	mutex_unlock(&ops->func_hash->regex_lock);
4205 4206 4207

	free_ftrace_hash(hash);
	return ret;
4208 4209
}

4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
		int reset, int enable)
{
	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
}

/**
 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
 * @ops - the ops to set the filter with
 * @ip - the address to add to or remove from the filter.
 * @remove - non zero to remove the ip from the filter
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled
 * If @ip is NULL, it failes to update filter.
 */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
			 int remove, int reset)
{
4230
	ftrace_ops_init(ops);
4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241
	return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);

static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
{
	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
}

4242 4243
/**
 * ftrace_set_filter - set a function to filter on in ftrace
4244 4245 4246 4247 4248 4249 4250 4251
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4252
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4253 4254
		       int len, int reset)
{
4255
	ftrace_ops_init(ops);
4256
	return ftrace_set_regex(ops, buf, len, reset, 1);
4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4271
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4272 4273
			int len, int reset)
{
4274
	ftrace_ops_init(ops);
4275
	return ftrace_set_regex(ops, buf, len, reset, 0);
4276 4277 4278
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
4279
 * ftrace_set_global_filter - set a function to filter on with global tracers
4280 4281 4282 4283 4284 4285 4286
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4287
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4288
{
4289
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4290
}
4291
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4292

4293
/**
4294
 * ftrace_set_global_notrace - set a function to not trace with global tracers
4295 4296 4297 4298 4299 4300 4301 4302
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4303
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4304
{
4305
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4306
}
4307
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4308

4309 4310 4311 4312 4313 4314 4315
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

4316 4317 4318
/* Used by function selftest to not test if filter is set */
bool ftrace_filter_param __initdata;

4319 4320
static int __init set_ftrace_notrace(char *str)
{
4321
	ftrace_filter_param = true;
4322
	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4323 4324 4325 4326 4327 4328
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
4329
	ftrace_filter_param = true;
4330
	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4331 4332 4333 4334
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

4335
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4336
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4337
static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4338
static int ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer);
4339

4340 4341 4342
static unsigned long save_global_trampoline;
static unsigned long save_global_flags;

4343 4344
static int __init set_graph_function(char *str)
{
4345
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4346 4347 4348 4349
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

4350 4351 4352 4353 4354 4355 4356 4357
static int __init set_graph_notrace_function(char *str)
{
	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);

static void __init set_ftrace_early_graph(char *buf, int enable)
4358 4359 4360
{
	int ret;
	char *func;
4361 4362 4363 4364 4365 4366 4367
	unsigned long *table = ftrace_graph_funcs;
	int *count = &ftrace_graph_count;

	if (!enable) {
		table = ftrace_graph_notrace_funcs;
		count = &ftrace_graph_notrace_count;
	}
4368 4369 4370 4371

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
4372
		ret = ftrace_set_func(table, count, FTRACE_GRAPH_MAX_FUNCS, func);
4373 4374 4375 4376 4377 4378 4379
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4380 4381
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4382 4383 4384
{
	char *func;

4385 4386
	ftrace_ops_init(ops);

4387 4388
	while (buf) {
		func = strsep(&buf, ",");
4389
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
4390 4391 4392 4393 4394 4395
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
4396
		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
4397
	if (ftrace_notrace_buf[0])
4398
		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
4399 4400
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
4401 4402 4403
		set_ftrace_early_graph(ftrace_graph_buf, 1);
	if (ftrace_graph_notrace_buf[0])
		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
4404
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4405 4406
}

4407
int ftrace_regex_release(struct inode *inode, struct file *file)
4408 4409
{
	struct seq_file *m = (struct seq_file *)file->private_data;
4410
	struct ftrace_ops_hash old_hash_ops;
4411
	struct ftrace_iterator *iter;
4412
	struct ftrace_hash **orig_hash;
4413
	struct ftrace_hash *old_hash;
4414
	struct trace_parser *parser;
4415
	int filter_hash;
4416
	int ret;
4417 4418 4419 4420 4421 4422 4423

	if (file->f_mode & FMODE_READ) {
		iter = m->private;
		seq_release(inode, file);
	} else
		iter = file->private_data;

4424 4425 4426
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
4427
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
4428 4429
	}

4430 4431
	trace_parser_put(parser);

4432
	mutex_lock(&iter->ops->func_hash->regex_lock);
4433

4434
	if (file->f_mode & FMODE_WRITE) {
4435 4436 4437
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

		if (filter_hash)
4438
			orig_hash = &iter->ops->func_hash->filter_hash;
4439
		else
4440
			orig_hash = &iter->ops->func_hash->notrace_hash;
4441

4442
		mutex_lock(&ftrace_lock);
4443
		old_hash = *orig_hash;
4444 4445
		old_hash_ops.filter_hash = iter->ops->func_hash->filter_hash;
		old_hash_ops.notrace_hash = iter->ops->func_hash->notrace_hash;
4446 4447
		ret = ftrace_hash_move(iter->ops, filter_hash,
				       orig_hash, iter->hash);
4448
		if (!ret) {
4449
			ftrace_ops_update_code(iter->ops, &old_hash_ops);
4450 4451
			free_ftrace_hash_rcu(old_hash);
		}
4452 4453
		mutex_unlock(&ftrace_lock);
	}
4454

4455
	mutex_unlock(&iter->ops->func_hash->regex_lock);
4456 4457
	free_ftrace_hash(iter->hash);
	kfree(iter);
4458

4459 4460 4461
	return 0;
}

4462
static const struct file_operations ftrace_avail_fops = {
4463 4464 4465
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
4466
	.release = seq_release_private,
4467 4468
};

4469 4470 4471 4472 4473 4474 4475
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

4476
static const struct file_operations ftrace_filter_fops = {
4477
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
4478
	.read = seq_read,
4479
	.write = ftrace_filter_write,
4480
	.llseek = tracing_lseek,
4481
	.release = ftrace_regex_release,
4482 4483
};

4484
static const struct file_operations ftrace_notrace_fops = {
4485
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
4486
	.read = seq_read,
4487
	.write = ftrace_notrace_write,
4488
	.llseek = tracing_lseek,
4489
	.release = ftrace_regex_release,
4490 4491
};

4492 4493 4494 4495 4496
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

int ftrace_graph_count;
4497
int ftrace_graph_notrace_count;
4498
unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
4499
unsigned long ftrace_graph_notrace_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
4500

4501 4502 4503 4504 4505 4506 4507
struct ftrace_graph_data {
	unsigned long *table;
	size_t size;
	int *count;
	const struct seq_operations *seq_ops;
};

4508
static void *
4509
__g_next(struct seq_file *m, loff_t *pos)
4510
{
4511 4512 4513
	struct ftrace_graph_data *fgd = m->private;

	if (*pos >= *fgd->count)
4514
		return NULL;
4515
	return &fgd->table[*pos];
4516
}
4517

4518 4519 4520 4521 4522
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
4523 4524 4525 4526
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
4527 4528
	struct ftrace_graph_data *fgd = m->private;

4529 4530
	mutex_lock(&graph_lock);

4531
	/* Nothing, tell g_show to print all functions are enabled */
4532
	if (!*fgd->count && !*pos)
4533 4534
		return (void *)1;

4535
	return __g_next(m, pos);
4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
	unsigned long *ptr = v;

	if (!ptr)
		return 0;

4550
	if (ptr == (unsigned long *)1) {
4551 4552 4553
		struct ftrace_graph_data *fgd = m->private;

		if (fgd->table == ftrace_graph_funcs)
4554
			seq_puts(m, "#### all functions enabled ####\n");
4555
		else
4556
			seq_puts(m, "#### no functions disabled ####\n");
4557 4558 4559
		return 0;
	}

4560
	seq_printf(m, "%ps\n", (void *)*ptr);
4561 4562 4563 4564

	return 0;
}

J
James Morris 已提交
4565
static const struct seq_operations ftrace_graph_seq_ops = {
4566 4567 4568 4569 4570 4571 4572
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
4573 4574
__ftrace_graph_open(struct inode *inode, struct file *file,
		    struct ftrace_graph_data *fgd)
4575 4576 4577 4578 4579
{
	int ret = 0;

	mutex_lock(&graph_lock);
	if ((file->f_mode & FMODE_WRITE) &&
4580
	    (file->f_flags & O_TRUNC)) {
4581 4582
		*fgd->count = 0;
		memset(fgd->table, 0, fgd->size * sizeof(*fgd->table));
4583
	}
4584
	mutex_unlock(&graph_lock);
4585

4586 4587 4588 4589 4590 4591 4592 4593
	if (file->f_mode & FMODE_READ) {
		ret = seq_open(file, fgd->seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = fgd;
		}
	} else
		file->private_data = fgd;
4594 4595 4596 4597

	return ret;
}

4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617
static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

	fgd->table = ftrace_graph_funcs;
	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
	fgd->count = &ftrace_graph_count;
	fgd->seq_ops = &ftrace_graph_seq_ops;

	return __ftrace_graph_open(inode, file, fgd);
}

4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637
static int
ftrace_graph_notrace_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

	fgd->table = ftrace_graph_notrace_funcs;
	fgd->size = FTRACE_GRAPH_MAX_FUNCS;
	fgd->count = &ftrace_graph_notrace_count;
	fgd->seq_ops = &ftrace_graph_seq_ops;

	return __ftrace_graph_open(inode, file, fgd);
}

4638 4639 4640
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
4641 4642 4643 4644
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;

		kfree(m->private);
4645
		seq_release(inode, file);
4646 4647 4648 4649
	} else {
		kfree(file->private_data);
	}

4650 4651 4652
	return 0;
}

4653
static int
4654
ftrace_set_func(unsigned long *array, int *idx, int size, char *buffer)
4655
{
4656
	struct ftrace_glob func_g;
4657 4658
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
4659
	int fail = 1;
4660
	int not;
4661 4662
	bool exists;
	int i;
4663

4664
	/* decode regex */
4665 4666
	func_g.type = filter_parse_regex(buffer, strlen(buffer),
					 &func_g.search, &not);
4667
	if (!not && *idx >= size)
4668
		return -EBUSY;
4669

4670
	func_g.len = strlen(func_g.search);
4671

4672
	mutex_lock(&ftrace_lock);
4673 4674 4675 4676 4677 4678

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

4679 4680
	do_for_each_ftrace_rec(pg, rec) {

D
Dmitry Safonov 已提交
4681
		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
4682
			/* if it is in the array */
4683
			exists = false;
4684
			for (i = 0; i < *idx; i++) {
4685 4686
				if (array[i] == rec->ip) {
					exists = true;
4687 4688
					break;
				}
4689 4690 4691 4692 4693 4694
			}

			if (!not) {
				fail = 0;
				if (!exists) {
					array[(*idx)++] = rec->ip;
4695
					if (*idx >= size)
4696 4697 4698 4699 4700 4701 4702 4703 4704
						goto out;
				}
			} else {
				if (exists) {
					array[i] = array[--(*idx)];
					array[*idx] = 0;
					fail = 0;
				}
			}
4705
		}
4706
	} while_for_each_ftrace_rec();
4707
out:
4708
	mutex_unlock(&ftrace_lock);
4709

4710 4711 4712 4713
	if (fail)
		return -EINVAL;

	return 0;
4714 4715 4716 4717 4718 4719
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
4720
	struct trace_parser parser;
4721
	ssize_t read, ret = 0;
4722
	struct ftrace_graph_data *fgd = file->private_data;
4723

4724
	if (!cnt)
4725 4726
		return 0;

4727 4728
	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX))
		return -ENOMEM;
4729

4730
	read = trace_get_user(&parser, ubuf, cnt, ppos);
4731

4732
	if (read >= 0 && trace_parser_loaded((&parser))) {
4733 4734
		parser.buffer[parser.idx] = 0;

4735 4736
		mutex_lock(&graph_lock);

4737
		/* we allow only one expression at a time */
4738 4739
		ret = ftrace_set_func(fgd->table, fgd->count, fgd->size,
				      parser.buffer);
4740 4741

		mutex_unlock(&graph_lock);
4742 4743
	}

4744 4745
	if (!ret)
		ret = read;
4746

4747
	trace_parser_put(&parser);
4748 4749 4750 4751 4752

	return ret;
}

static const struct file_operations ftrace_graph_fops = {
4753 4754 4755
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
4756
	.llseek		= tracing_lseek,
4757
	.release	= ftrace_graph_release,
4758
};
4759 4760 4761 4762 4763

static const struct file_operations ftrace_graph_notrace_fops = {
	.open		= ftrace_graph_notrace_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
4764
	.llseek		= tracing_lseek,
4765 4766
	.release	= ftrace_graph_release,
};
4767 4768
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798
void ftrace_create_filter_files(struct ftrace_ops *ops,
				struct dentry *parent)
{

	trace_create_file("set_ftrace_filter", 0644, parent,
			  ops, &ftrace_filter_fops);

	trace_create_file("set_ftrace_notrace", 0644, parent,
			  ops, &ftrace_notrace_fops);
}

/*
 * The name "destroy_filter_files" is really a misnomer. Although
 * in the future, it may actualy delete the files, but this is
 * really intended to make sure the ops passed in are disabled
 * and that when this function returns, the caller is free to
 * free the ops.
 *
 * The "destroy" name is only to match the "create" name that this
 * should be paired with.
 */
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
	mutex_lock(&ftrace_lock);
	if (ops->flags & FTRACE_OPS_FL_ENABLED)
		ftrace_shutdown(ops, 0);
	ops->flags |= FTRACE_OPS_FL_DELETED;
	mutex_unlock(&ftrace_lock);
}

4799
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
4800 4801
{

4802 4803
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
4804

4805 4806 4807
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

4808
	ftrace_create_filter_files(&global_ops, d_tracer);
4809

4810
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4811
	trace_create_file("set_graph_function", 0444, d_tracer,
4812 4813
				    NULL,
				    &ftrace_graph_fops);
4814 4815 4816
	trace_create_file("set_graph_notrace", 0444, d_tracer,
				    NULL,
				    &ftrace_graph_notrace_fops);
4817 4818
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4819 4820 4821
	return 0;
}

4822
static int ftrace_cmp_ips(const void *a, const void *b)
4823
{
4824 4825
	const unsigned long *ipa = a;
	const unsigned long *ipb = b;
4826

4827 4828 4829 4830 4831 4832 4833
	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

4834
static int ftrace_process_locs(struct module *mod,
4835
			       unsigned long *start,
4836 4837
			       unsigned long *end)
{
4838
	struct ftrace_page *start_pg;
4839
	struct ftrace_page *pg;
4840
	struct dyn_ftrace *rec;
4841
	unsigned long count;
4842 4843
	unsigned long *p;
	unsigned long addr;
4844
	unsigned long flags = 0; /* Shut up gcc */
4845 4846 4847 4848 4849 4850 4851
	int ret = -ENOMEM;

	count = end - start;

	if (!count)
		return 0;

4852
	sort(start, count, sizeof(*start),
4853
	     ftrace_cmp_ips, NULL);
4854

4855 4856
	start_pg = ftrace_allocate_pages(count);
	if (!start_pg)
4857
		return -ENOMEM;
4858

S
Steven Rostedt 已提交
4859
	mutex_lock(&ftrace_lock);
4860

4861 4862 4863 4864 4865
	/*
	 * Core and each module needs their own pages, as
	 * modules will free them when they are removed.
	 * Force a new page to be allocated for modules.
	 */
4866 4867 4868
	if (!mod) {
		WARN_ON(ftrace_pages || ftrace_pages_start);
		/* First initialization */
4869
		ftrace_pages = ftrace_pages_start = start_pg;
4870
	} else {
4871
		if (!ftrace_pages)
4872
			goto out;
4873

4874 4875 4876 4877
		if (WARN_ON(ftrace_pages->next)) {
			/* Hmm, we have free pages? */
			while (ftrace_pages->next)
				ftrace_pages = ftrace_pages->next;
4878
		}
4879

4880
		ftrace_pages->next = start_pg;
4881 4882
	}

4883
	p = start;
4884
	pg = start_pg;
4885 4886
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
4887 4888 4889 4890 4891 4892 4893 4894
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
4895 4896 4897 4898 4899 4900 4901 4902 4903 4904

		if (pg->index == pg->size) {
			/* We should have allocated enough */
			if (WARN_ON(!pg->next))
				break;
			pg = pg->next;
		}

		rec = &pg->records[pg->index++];
		rec->ip = addr;
4905 4906
	}

4907 4908 4909 4910 4911 4912
	/* We should have used all pages */
	WARN_ON(pg->next);

	/* Assign the last page to ftrace_pages */
	ftrace_pages = pg;

4913
	/*
4914 4915 4916 4917 4918 4919
	 * We only need to disable interrupts on start up
	 * because we are modifying code that an interrupt
	 * may execute, and the modification is not atomic.
	 * But for modules, nothing runs the code we modify
	 * until we are finished with it, and there's no
	 * reason to cause large interrupt latencies while we do it.
4920
	 */
4921 4922
	if (!mod)
		local_irq_save(flags);
4923
	ftrace_update_code(mod, start_pg);
4924 4925
	if (!mod)
		local_irq_restore(flags);
4926 4927
	ret = 0;
 out:
S
Steven Rostedt 已提交
4928
	mutex_unlock(&ftrace_lock);
4929

4930
	return ret;
4931 4932
}

4933
#ifdef CONFIG_MODULES
4934 4935 4936

#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)

4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949
static int referenced_filters(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
		if (ops_references_rec(ops, rec))
		    cnt++;
	}

	return cnt;
}

4950
void ftrace_release_mod(struct module *mod)
4951 4952
{
	struct dyn_ftrace *rec;
4953
	struct ftrace_page **last_pg;
4954
	struct ftrace_page *pg;
4955
	int order;
4956

4957 4958
	mutex_lock(&ftrace_lock);

4959
	if (ftrace_disabled)
4960
		goto out_unlock;
4961

4962 4963 4964 4965 4966 4967 4968
	/*
	 * Each module has its own ftrace_pages, remove
	 * them from the list.
	 */
	last_pg = &ftrace_pages_start;
	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
		rec = &pg->records[0];
4969
		if (within_module_core(rec->ip, mod)) {
4970
			/*
4971 4972
			 * As core pages are first, the first
			 * page should never be a module page.
4973
			 */
4974 4975 4976 4977 4978 4979 4980 4981
			if (WARN_ON(pg == ftrace_pages_start))
				goto out_unlock;

			/* Check if we are deleting the last page */
			if (pg == ftrace_pages)
				ftrace_pages = next_to_ftrace_page(last_pg);

			*last_pg = pg->next;
4982 4983 4984
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
4985 4986 4987
		} else
			last_pg = &pg->next;
	}
4988
 out_unlock:
4989 4990 4991
	mutex_unlock(&ftrace_lock);
}

4992
void ftrace_module_enable(struct module *mod)
4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	mutex_lock(&ftrace_lock);

	if (ftrace_disabled)
		goto out_unlock;

	/*
	 * If the tracing is enabled, go ahead and enable the record.
	 *
	 * The reason not to enable the record immediatelly is the
	 * inherent check of ftrace_make_nop/ftrace_make_call for
	 * correct previous instructions.  Making first the NOP
	 * conversion puts the module to the correct state, thus
	 * passing the ftrace_make_call check.
	 *
	 * We also delay this to after the module code already set the
	 * text to read-only, as we now need to set it back to read-write
	 * so that we can modify the text.
	 */
	if (ftrace_start_up)
		ftrace_arch_code_modify_prepare();

	do_for_each_ftrace_rec(pg, rec) {
		int cnt;
		/*
		 * do_for_each_ftrace_rec() is a double loop.
		 * module text shares the pg. If a record is
		 * not part of this module, then skip this pg,
		 * which the "break" will do.
		 */
		if (!within_module_core(rec->ip, mod))
			break;

		cnt = 0;

		/*
		 * When adding a module, we need to check if tracers are
		 * currently enabled and if they are, and can trace this record,
		 * we need to enable the module functions as well as update the
		 * reference counts for those function records.
		 */
		if (ftrace_start_up)
			cnt += referenced_filters(rec);

		/* This clears FTRACE_FL_DISABLED */
		rec->flags = cnt;

		if (ftrace_start_up && cnt) {
			int failed = __ftrace_replace_code(rec, 1);
			if (failed) {
				ftrace_bug(failed, rec);
				goto out_loop;
			}
		}

	} while_for_each_ftrace_rec();

 out_loop:
	if (ftrace_start_up)
		ftrace_arch_code_modify_post_process();

 out_unlock:
	mutex_unlock(&ftrace_lock);
}

5061
void ftrace_module_init(struct module *mod)
5062
{
5063
	if (ftrace_disabled || !mod->num_ftrace_callsites)
5064
		return;
5065

5066 5067
	ftrace_process_locs(mod, mod->ftrace_callsites,
			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5068
}
5069 5070
#endif /* CONFIG_MODULES */

5071 5072
void __init ftrace_init(void)
{
5073 5074
	extern unsigned long __start_mcount_loc[];
	extern unsigned long __stop_mcount_loc[];
5075
	unsigned long count, flags;
5076 5077 5078
	int ret;

	local_irq_save(flags);
5079
	ret = ftrace_dyn_arch_init();
5080
	local_irq_restore(flags);
5081
	if (ret)
5082 5083 5084
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;
5085 5086
	if (!count) {
		pr_info("ftrace: No functions to be traced?\n");
5087
		goto failed;
5088 5089 5090 5091
	}

	pr_info("ftrace: allocating %ld entries in %ld pages\n",
		count, count / ENTRIES_PER_PAGE + 1);
5092 5093 5094

	last_ftrace_enabled = ftrace_enabled = 1;

5095
	ret = ftrace_process_locs(NULL,
5096
				  __start_mcount_loc,
5097 5098
				  __stop_mcount_loc);

5099 5100
	set_ftrace_early_filters();

5101 5102 5103 5104 5105
	return;
 failed:
	ftrace_disabled = 1;
}

5106 5107 5108 5109 5110 5111 5112
/* Do nothing if arch does not support this */
void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123

/*
 * Currently there's no safe way to free a trampoline when the kernel
 * is configured with PREEMPT. That is because a task could be preempted
 * when it jumped to the trampoline, it may be preempted for a long time
 * depending on the system load, and currently there's no way to know
 * when it will be off the trampoline. If the trampoline is freed
 * too early, when the task runs again, it will be executing on freed
 * memory and crash.
 */
#ifdef CONFIG_PREEMPT
5124 5125 5126
	/* Currently, only non dynamic ops can have a trampoline */
	if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
		return;
5127
#endif
5128 5129 5130 5131

	arch_ftrace_update_trampoline(ops);
}

5132
#else
5133

5134
static struct ftrace_ops global_ops = {
5135
	.func			= ftrace_stub,
5136 5137 5138
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				  FTRACE_OPS_FL_INITIALIZED |
				  FTRACE_OPS_FL_PID,
5139 5140
};

5141 5142 5143 5144 5145
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
5146
core_initcall(ftrace_nodyn_init);
5147

5148
static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
5149
static inline void ftrace_startup_enable(int command) { }
5150
static inline void ftrace_startup_all(int command) { }
5151
/* Keep as macros so we do not need to define the commands */
5152 5153 5154 5155 5156 5157
# define ftrace_startup(ops, command)					\
	({								\
		int ___ret = __register_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
5158
	})
5159 5160 5161 5162 5163 5164 5165
# define ftrace_shutdown(ops, command)					\
	({								\
		int ___ret = __unregister_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
	})
5166

I
Ingo Molnar 已提交
5167 5168
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
5169 5170

static inline int
5171
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
5172 5173 5174 5175
{
	return 1;
}

5176 5177 5178 5179
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

5180 5181
#endif /* CONFIG_DYNAMIC_FTRACE */

5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204
__init void ftrace_init_global_array_ops(struct trace_array *tr)
{
	tr->ops = &global_ops;
	tr->ops->private = tr;
}

void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
{
	/* If we filter on pids, update to use the pid function */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		if (WARN_ON(tr->ops->func != ftrace_stub))
			printk("ftrace ops had %pS for function\n",
			       tr->ops->func);
	}
	tr->ops->func = func;
	tr->ops->private = tr;
}

void ftrace_reset_array_ops(struct trace_array *tr)
{
	tr->ops->func = ftrace_stub;
}

5205 5206
static inline void
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5207
		       struct ftrace_ops *ignored, struct pt_regs *regs)
5208
{
5209
	struct ftrace_ops *op;
5210
	int bit;
5211

5212 5213 5214
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;
5215

5216 5217 5218 5219 5220
	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
5221

5222
	do_for_each_ftrace_op(op, ftrace_ops_list) {
5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236
		/*
		 * Check the following for each ops before calling their func:
		 *  if RCU flag is set, then rcu_is_watching() must be true
		 *  if PER_CPU is set, then ftrace_function_local_disable()
		 *                          must be false
		 *  Otherwise test if the ip matches the ops filter
		 *
		 * If any of the above fails then the op->func() is not executed.
		 */
		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
		    (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
		     !ftrace_function_local_disabled(op)) &&
		    ftrace_ops_test(op, ip, regs)) {
		    
5237 5238
			if (FTRACE_WARN_ON(!op->func)) {
				pr_warn("op=%p %pS\n", op, op);
5239 5240
				goto out;
			}
5241
			op->func(ip, parent_ip, op, regs);
5242
		}
5243
	} while_for_each_ftrace_op(op);
5244
out:
5245
	preempt_enable_notrace();
5246
	trace_clear_recursion(bit);
5247 5248
}

5249 5250 5251 5252 5253
/*
 * Some archs only support passing ip and parent_ip. Even though
 * the list function ignores the op parameter, we do not want any
 * C side effects, where a function is called without the caller
 * sending a third parameter.
5254 5255 5256
 * Archs are to support both the regs and ftrace_ops at the same time.
 * If they support ftrace_ops, it is assumed they support regs.
 * If call backs want to use regs, they must either check for regs
5257 5258
 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
5259
 * An architecture can pass partial regs with ftrace_ops and still
L
Li Bin 已提交
5260
 * set the ARCH_SUPPORTS_FTRACE_OPS.
5261 5262 5263
 */
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5264
				 struct ftrace_ops *op, struct pt_regs *regs)
5265
{
5266
	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
5267 5268 5269 5270
}
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
{
5271
	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
5272 5273 5274
}
#endif

5275 5276
/*
 * If there's only one function registered but it does not support
5277 5278
 * recursion, needs RCU protection and/or requires per cpu handling, then
 * this function will be called by the mcount trampoline.
5279
 */
5280
static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
5281 5282 5283 5284
				   struct ftrace_ops *op, struct pt_regs *regs)
{
	int bit;

5285 5286 5287
	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
		return;

5288 5289 5290 5291
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;

5292
	preempt_disable_notrace();
5293

5294 5295 5296 5297 5298 5299
	if (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
	    !ftrace_function_local_disabled(op)) {
		op->func(ip, parent_ip, op, regs);
	}

	preempt_enable_notrace();
5300 5301 5302
	trace_clear_recursion(bit);
}

5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316
/**
 * ftrace_ops_get_func - get the function a trampoline should call
 * @ops: the ops to get the function for
 *
 * Normally the mcount trampoline will call the ops->func, but there
 * are times that it should not. For example, if the ops does not
 * have its own recursion protection, then it should call the
 * ftrace_ops_recurs_func() instead.
 *
 * Returns the function that the trampoline should call for @ops.
 */
ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
{
	/*
5317 5318
	 * If the function does not handle recursion, needs to be RCU safe,
	 * or does per cpu logic, then we need to call the assist handler.
5319
	 */
5320 5321 5322
	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
	    ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU))
		return ftrace_ops_assist_func;
5323 5324 5325 5326

	return ops->func;
}

5327
static void clear_ftrace_swapper(void)
S
Steven Rostedt 已提交
5328 5329
{
	struct task_struct *p;
5330
	int cpu;
S
Steven Rostedt 已提交
5331

5332 5333 5334
	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
S
Steven Rostedt 已提交
5335
		clear_tsk_trace_trace(p);
5336 5337 5338
	}
	put_online_cpus();
}
S
Steven Rostedt 已提交
5339

5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350
static void set_ftrace_swapper(void)
{
	struct task_struct *p;
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		p = idle_task(cpu);
		set_tsk_trace_trace(p);
	}
	put_online_cpus();
S
Steven Rostedt 已提交
5351 5352
}

5353 5354 5355 5356
static void clear_ftrace_pid(struct pid *pid)
{
	struct task_struct *p;

5357
	rcu_read_lock();
5358 5359 5360
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		clear_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
5361 5362
	rcu_read_unlock();

5363 5364 5365 5366
	put_pid(pid);
}

static void set_ftrace_pid(struct pid *pid)
S
Steven Rostedt 已提交
5367 5368 5369
{
	struct task_struct *p;

5370
	rcu_read_lock();
S
Steven Rostedt 已提交
5371 5372 5373
	do_each_pid_task(pid, PIDTYPE_PID, p) {
		set_tsk_trace_trace(p);
	} while_each_pid_task(pid, PIDTYPE_PID, p);
5374
	rcu_read_unlock();
S
Steven Rostedt 已提交
5375 5376
}

5377
static void clear_ftrace_pid_task(struct pid *pid)
5378
{
5379
	if (pid == ftrace_swapper_pid)
5380 5381
		clear_ftrace_swapper();
	else
5382
		clear_ftrace_pid(pid);
5383 5384 5385 5386 5387 5388 5389 5390 5391 5392
}

static void set_ftrace_pid_task(struct pid *pid)
{
	if (pid == ftrace_swapper_pid)
		set_ftrace_swapper();
	else
		set_ftrace_pid(pid);
}

5393
static int ftrace_pid_add(int p)
5394
{
S
Steven Rostedt 已提交
5395
	struct pid *pid;
5396 5397
	struct ftrace_pid *fpid;
	int ret = -EINVAL;
5398

5399
	mutex_lock(&ftrace_lock);
5400

5401 5402 5403 5404
	if (!p)
		pid = ftrace_swapper_pid;
	else
		pid = find_get_pid(p);
5405

5406 5407
	if (!pid)
		goto out;
5408

5409
	ret = 0;
5410

5411 5412 5413
	list_for_each_entry(fpid, &ftrace_pids, list)
		if (fpid->pid == pid)
			goto out_put;
S
Steven Rostedt 已提交
5414

5415
	ret = -ENOMEM;
5416

5417 5418 5419
	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
	if (!fpid)
		goto out_put;
5420

5421 5422
	list_add(&fpid->list, &ftrace_pids);
	fpid->pid = pid;
5423

5424
	set_ftrace_pid_task(pid);
S
Steven Rostedt 已提交
5425

5426
	ftrace_update_pid_func();
5427 5428

	ftrace_startup_all(0);
5429 5430 5431 5432 5433 5434 5435

	mutex_unlock(&ftrace_lock);
	return 0;

out_put:
	if (pid != ftrace_swapper_pid)
		put_pid(pid);
S
Steven Rostedt 已提交
5436

5437 5438 5439 5440 5441 5442 5443 5444
out:
	mutex_unlock(&ftrace_lock);
	return ret;
}

static void ftrace_pid_reset(void)
{
	struct ftrace_pid *fpid, *safe;
S
Steven Rostedt 已提交
5445

5446 5447 5448 5449 5450 5451 5452 5453
	mutex_lock(&ftrace_lock);
	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
		struct pid *pid = fpid->pid;

		clear_ftrace_pid_task(pid);

		list_del(&fpid->list);
		kfree(fpid);
5454 5455 5456
	}

	ftrace_update_pid_func();
5457
	ftrace_startup_all(0);
5458

S
Steven Rostedt 已提交
5459
	mutex_unlock(&ftrace_lock);
5460
}
5461

5462 5463 5464 5465
static void *fpid_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&ftrace_lock);

5466
	if (!ftrace_pids_enabled() && (!*pos))
5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489
		return (void *) 1;

	return seq_list_start(&ftrace_pids, *pos);
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
	if (v == (void *)1)
		return NULL;

	return seq_list_next(v, &ftrace_pids, pos);
}

static void fpid_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);

	if (v == (void *)1) {
5490
		seq_puts(m, "no pid\n");
5491 5492 5493 5494
		return 0;
	}

	if (fpid->pid == ftrace_swapper_pid)
5495
		seq_puts(m, "swapper tasks\n");
5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523
	else
		seq_printf(m, "%u\n", pid_vnr(fpid->pid));

	return 0;
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
	int ret = 0;

	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
		ftrace_pid_reset();

	if (file->f_mode & FMODE_READ)
		ret = seq_open(file, &ftrace_pid_sops);

	return ret;
}

5524 5525 5526 5527
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5528
	char buf[64], *tmp;
5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539
	long val;
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

5540 5541 5542 5543
	/*
	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
	 * to clean the filter quietly.
	 */
5544 5545
	tmp = strstrip(buf);
	if (strlen(tmp) == 0)
5546 5547
		return 1;

5548
	ret = kstrtol(tmp, 10, &val);
5549 5550 5551
	if (ret < 0)
		return ret;

5552
	ret = ftrace_pid_add(val);
5553

5554 5555
	return ret ? ret : cnt;
}
5556

5557 5558 5559 5560 5561
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
	if (file->f_mode & FMODE_READ)
		seq_release(inode, file);
5562

5563
	return 0;
5564 5565
}

5566
static const struct file_operations ftrace_pid_fops = {
5567 5568 5569
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
5570
	.llseek		= tracing_lseek,
5571
	.release	= ftrace_pid_release,
5572 5573
};

5574
static __init int ftrace_init_tracefs(void)
5575 5576 5577 5578
{
	struct dentry *d_tracer;

	d_tracer = tracing_init_dentry();
5579
	if (IS_ERR(d_tracer))
5580 5581
		return 0;

5582
	ftrace_init_dyn_tracefs(d_tracer);
5583

5584 5585
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
			    NULL, &ftrace_pid_fops);
5586

5587
	ftrace_profile_tracefs(d_tracer);
5588

5589 5590
	return 0;
}
5591
fs_initcall(ftrace_init_tracefs);
5592

S
Steven Rostedt 已提交
5593
/**
5594
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
5595 5596 5597 5598 5599
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
5600
void ftrace_kill(void)
S
Steven Rostedt 已提交
5601 5602 5603 5604 5605 5606
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

5607 5608 5609 5610 5611 5612 5613 5614
/**
 * Test if ftrace is dead or not.
 */
int ftrace_is_dead(void)
{
	return ftrace_disabled;
}

5615
/**
5616 5617
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
5618
 *
5619 5620 5621 5622 5623 5624
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
5625
 */
5626
int register_ftrace_function(struct ftrace_ops *ops)
5627
{
5628
	int ret = -1;
5629

5630 5631
	ftrace_ops_init(ops);

S
Steven Rostedt 已提交
5632
	mutex_lock(&ftrace_lock);
5633

5634
	ret = ftrace_startup(ops, 0);
5635

S
Steven Rostedt 已提交
5636
	mutex_unlock(&ftrace_lock);
5637

5638
	return ret;
5639
}
5640
EXPORT_SYMBOL_GPL(register_ftrace_function);
5641 5642

/**
5643
 * unregister_ftrace_function - unregister a function for profiling.
5644 5645 5646 5647 5648 5649 5650 5651
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
5652
	mutex_lock(&ftrace_lock);
5653
	ret = ftrace_shutdown(ops, 0);
S
Steven Rostedt 已提交
5654
	mutex_unlock(&ftrace_lock);
5655 5656 5657

	return ret;
}
5658
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
5659

I
Ingo Molnar 已提交
5660
int
5661
ftrace_enable_sysctl(struct ctl_table *table, int write,
5662
		     void __user *buffer, size_t *lenp,
5663 5664
		     loff_t *ppos)
{
5665
	int ret = -ENODEV;
5666

S
Steven Rostedt 已提交
5667
	mutex_lock(&ftrace_lock);
5668

5669 5670 5671 5672
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
5673

5674
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
5675 5676
		goto out;

5677
	last_ftrace_enabled = !!ftrace_enabled;
5678 5679 5680 5681

	if (ftrace_enabled) {

		/* we are starting ftrace again */
5682 5683
		if (ftrace_ops_list != &ftrace_list_end)
			update_ftrace_function();
5684

5685 5686
		ftrace_startup_sysctl();

5687 5688 5689 5690 5691 5692 5693 5694
	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
5695
	mutex_unlock(&ftrace_lock);
5696
	return ret;
5697
}
5698

5699
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5700

5701 5702 5703 5704
static struct ftrace_ops graph_ops = {
	.func			= ftrace_stub,
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				   FTRACE_OPS_FL_INITIALIZED |
5705
				   FTRACE_OPS_FL_PID |
5706 5707 5708
				   FTRACE_OPS_FL_STUB,
#ifdef FTRACE_GRAPH_TRAMP_ADDR
	.trampoline		= FTRACE_GRAPH_TRAMP_ADDR,
5709
	/* trampoline_size is only needed for dynamically allocated tramps */
5710 5711 5712 5713
#endif
	ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
};

5714 5715 5716 5717 5718 5719 5720 5721 5722 5723
void ftrace_graph_sleep_time_control(bool enable)
{
	fgraph_sleep_time = enable;
}

void ftrace_graph_graph_time_control(bool enable)
{
	fgraph_graph_time = enable;
}

5724 5725 5726 5727 5728
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

5729 5730 5731
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
5732
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
5733
static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

5755
	read_lock(&tasklist_lock);
5756 5757 5758 5759 5760 5761 5762
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
5763
			atomic_set(&t->tracing_graph_pause, 0);
5764
			atomic_set(&t->trace_overrun, 0);
5765 5766 5767 5768
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
5769 5770 5771 5772
		}
	} while_each_thread(g, t);

unlock:
5773
	read_unlock(&tasklist_lock);
5774 5775 5776 5777 5778 5779
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

5780
static void
5781
ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
5782
			struct task_struct *prev, struct task_struct *next)
5783 5784 5785 5786
{
	unsigned long long timestamp;
	int index;

5787 5788 5789 5790
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
5791
	if (fgraph_sleep_time)
5792 5793
		return;

5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

5812
/* Allocate a return stack for each task */
5813
static int start_graph_tracing(void)
5814 5815
{
	struct ftrace_ret_stack **ret_stack_list;
5816
	int ret, cpu;
5817 5818 5819 5820 5821 5822 5823 5824

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

5825
	/* The cpu_boot init_task->ret_stack will never be freed */
5826 5827
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
5828
			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
5829
	}
5830

5831 5832 5833 5834
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

5835
	if (!ret) {
5836
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5837 5838 5839 5840 5841
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

5842 5843 5844 5845
	kfree(ret_stack_list);
	return ret;
}

5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882
static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
{
	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
		return 0;
	return __ftrace_graph_entry(trace);
}

/*
 * The function graph tracer should only trace the functions defined
 * by set_ftrace_filter and set_ftrace_notrace. If another function
 * tracer ops is registered, the graph tracer requires testing the
 * function against the global ops, and not just trace any function
 * that any ftrace_ops registered.
 */
static void update_function_graph_func(void)
{
5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901
	struct ftrace_ops *op;
	bool do_test = false;

	/*
	 * The graph and global ops share the same set of functions
	 * to test. If any other ops is on the list, then
	 * the graph tracing needs to test if its the function
	 * it should call.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op != &global_ops && op != &graph_ops &&
		    op != &ftrace_list_end) {
			do_test = true;
			/* in double loop, break out with goto */
			goto out;
		}
	} while_for_each_ftrace_op(op);
 out:
	if (do_test)
5902
		ftrace_graph_entry = ftrace_graph_entry_test;
5903 5904
	else
		ftrace_graph_entry = __ftrace_graph_entry;
5905 5906
}

5907 5908 5909 5910
static struct notifier_block ftrace_suspend_notifier = {
	.notifier_call = ftrace_suspend_notifier_call,
};

5911 5912
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
5913
{
5914 5915
	int ret = 0;

S
Steven Rostedt 已提交
5916
	mutex_lock(&ftrace_lock);
5917

5918
	/* we currently allow only one tracer registered at a time */
5919
	if (ftrace_graph_active) {
5920 5921 5922 5923
		ret = -EBUSY;
		goto out;
	}

5924 5925
	register_pm_notifier(&ftrace_suspend_notifier);

5926
	ftrace_graph_active++;
5927
	ret = start_graph_tracing();
5928
	if (ret) {
5929
		ftrace_graph_active--;
5930 5931
		goto out;
	}
5932

5933
	ftrace_graph_return = retfunc;
5934 5935 5936 5937 5938 5939 5940 5941 5942 5943

	/*
	 * Update the indirect function to the entryfunc, and the
	 * function that gets called to the entry_test first. Then
	 * call the update fgraph entry function to determine if
	 * the entryfunc should be called directly or not.
	 */
	__ftrace_graph_entry = entryfunc;
	ftrace_graph_entry = ftrace_graph_entry_test;
	update_function_graph_func();
5944

5945
	ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
5946
out:
S
Steven Rostedt 已提交
5947
	mutex_unlock(&ftrace_lock);
5948
	return ret;
5949 5950
}

5951
void unregister_ftrace_graph(void)
5952
{
S
Steven Rostedt 已提交
5953
	mutex_lock(&ftrace_lock);
5954

5955
	if (unlikely(!ftrace_graph_active))
5956 5957
		goto out;

5958
	ftrace_graph_active--;
5959
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
5960
	ftrace_graph_entry = ftrace_graph_entry_stub;
5961
	__ftrace_graph_entry = ftrace_graph_entry_stub;
5962
	ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
5963
	unregister_pm_notifier(&ftrace_suspend_notifier);
5964
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
5965

5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976
#ifdef CONFIG_DYNAMIC_FTRACE
	/*
	 * Function graph does not allocate the trampoline, but
	 * other global_ops do. We need to reset the ALLOC_TRAMP flag
	 * if one was used.
	 */
	global_ops.trampoline = save_global_trampoline;
	if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP)
		global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
#endif

5977
 out:
S
Steven Rostedt 已提交
5978
	mutex_unlock(&ftrace_lock);
5979
}
5980

5981 5982 5983 5984 5985 5986 5987 5988
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);

static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
	atomic_set(&t->tracing_graph_pause, 0);
	atomic_set(&t->trace_overrun, 0);
	t->ftrace_timestamp = 0;
L
Lucas De Marchi 已提交
5989
	/* make curr_ret_stack visible before we add the ret_stack */
5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023
	smp_wmb();
	t->ret_stack = ret_stack;
}

/*
 * Allocate a return stack for the idle task. May be the first
 * time through, or it may be done by CPU hotplug online.
 */
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
{
	t->curr_ret_stack = -1;
	/*
	 * The idle task has no parent, it either has its own
	 * stack or no stack at all.
	 */
	if (t->ret_stack)
		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));

	if (ftrace_graph_active) {
		struct ftrace_ret_stack *ret_stack;

		ret_stack = per_cpu(idle_ret_stack, cpu);
		if (!ret_stack) {
			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
					    * sizeof(struct ftrace_ret_stack),
					    GFP_KERNEL);
			if (!ret_stack)
				return;
			per_cpu(idle_ret_stack, cpu) = ret_stack;
		}
		graph_init_task(t, ret_stack);
	}
}

6024
/* Allocate a return stack for newly created task */
6025
void ftrace_graph_init_task(struct task_struct *t)
6026
{
6027 6028
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;
6029
	t->curr_ret_stack = -1;
6030

6031
	if (ftrace_graph_active) {
6032 6033 6034
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
6035 6036
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
6037
		if (!ret_stack)
6038
			return;
6039
		graph_init_task(t, ret_stack);
6040
	}
6041 6042
}

6043
void ftrace_graph_exit_task(struct task_struct *t)
6044
{
6045 6046
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

6047
	t->ret_stack = NULL;
6048 6049 6050 6051
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
6052
}
6053
#endif