ftrace.c 151.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
13
 *  Copyright (C) 2004 Nadia Yvette Chambers
14 15
 */

16 17
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
18
#include <linux/sched/task.h>
19
#include <linux/kallsyms.h>
20
#include <linux/seq_file.h>
21
#include <linux/suspend.h>
22
#include <linux/tracefs.h>
23
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
24
#include <linux/kthread.h>
25
#include <linux/uaccess.h>
26
#include <linux/bsearch.h>
27
#include <linux/module.h>
I
Ingo Molnar 已提交
28
#include <linux/ftrace.h>
29
#include <linux/sysctl.h>
30
#include <linux/slab.h>
31
#include <linux/ctype.h>
32
#include <linux/sort.h>
33
#include <linux/list.h>
34
#include <linux/hash.h>
35
#include <linux/rcupdate.h>
36

37
#include <trace/events/sched.h>
38

39
#include <asm/sections.h>
40
#include <asm/setup.h>
41

42
#include "trace_output.h"
S
Steven Rostedt 已提交
43
#include "trace_stat.h"
44

45
#define FTRACE_WARN_ON(cond)			\
46 47 48
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
49
			ftrace_kill();		\
50 51
		___r;				\
	})
52 53

#define FTRACE_WARN_ON_ONCE(cond)		\
54 55 56
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
57
			ftrace_kill();		\
58 59
		___r;				\
	})
60

61 62 63
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
64 65
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
66

67
#ifdef CONFIG_DYNAMIC_FTRACE
68 69 70
#define INIT_OPS_HASH(opsname)	\
	.func_hash		= &opsname.local_hash,			\
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
71 72 73
#define ASSIGN_OPS_HASH(opsname, val) \
	.func_hash		= val, \
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
74
#else
75
#define INIT_OPS_HASH(opsname)
76
#define ASSIGN_OPS_HASH(opsname, val)
77 78
#endif

79 80
static struct ftrace_ops ftrace_list_end __read_mostly = {
	.func		= ftrace_stub,
81
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
82
	INIT_OPS_HASH(ftrace_list_end)
83 84
};

85 86
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
87
static int last_ftrace_enabled;
88

89 90
/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
91 92
/* What to set function_trace_op to */
static struct ftrace_ops *set_function_trace_op;
93

94
static bool ftrace_pids_enabled(struct ftrace_ops *ops)
95
{
96 97 98 99 100 101 102 103
	struct trace_array *tr;

	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
		return false;

	tr = ops->private;

	return tr->function_pids != NULL;
104 105 106 107
}

static void ftrace_update_trampoline(struct ftrace_ops *ops);

108 109 110 111 112 113
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

114
static DEFINE_MUTEX(ftrace_lock);
115

116
static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
117
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
118
static struct ftrace_ops global_ops;
119

120 121
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
122
				 struct ftrace_ops *op, struct pt_regs *regs);
123 124 125 126 127
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
128

129 130
/*
 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
131
 * can use rcu_dereference_raw_notrace() is that elements removed from this list
132
 * are simply leaked, so there is no need to interact with a grace-period
133
 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
134 135 136 137 138
 * concurrent insertions into the ftrace_global_list.
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
#define do_for_each_ftrace_op(op, list)			\
139
	op = rcu_dereference_raw_notrace(list);			\
140 141 142 143 144 145
	do

/*
 * Optimized for just a single item in the list (as that is the normal case).
 */
#define while_for_each_ftrace_op(op)				\
146
	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
147 148
	       unlikely((op) != &ftrace_list_end))

149 150 151 152
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
153 154
		mutex_init(&ops->local_hash.regex_lock);
		ops->func_hash = &ops->local_hash;
155 156 157 158 159
		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
	}
#endif
}

160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
/**
 * ftrace_nr_registered_ops - return number of ops registered
 *
 * Returns the number of ftrace_ops registered and tracing functions
 */
int ftrace_nr_registered_ops(void)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	mutex_lock(&ftrace_lock);

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next)
		cnt++;

	mutex_unlock(&ftrace_lock);

	return cnt;
}

181
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
182
			    struct ftrace_ops *op, struct pt_regs *regs)
183
{
184 185 186
	struct trace_array *tr = op->private;

	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
187 188
		return;

189
	op->saved_func(ip, parent_ip, op, regs);
190 191
}

192
/**
193
 * clear_ftrace_function - reset the ftrace function
194
 *
195 196
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
197
 */
198
void clear_ftrace_function(void)
199
{
200 201 202
	ftrace_trace_function = ftrace_stub;
}

203
static void per_cpu_ops_disable_all(struct ftrace_ops *ops)
204 205 206 207 208 209 210
{
	int cpu;

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(ops->disabled, cpu) = 1;
}

211
static int per_cpu_ops_alloc(struct ftrace_ops *ops)
212 213 214
{
	int __percpu *disabled;

215 216 217
	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU)))
		return -EINVAL;

218 219 220 221 222
	disabled = alloc_percpu(int);
	if (!disabled)
		return -ENOMEM;

	ops->disabled = disabled;
223
	per_cpu_ops_disable_all(ops);
224 225 226
	return 0;
}

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
static void ftrace_sync(struct work_struct *work)
{
	/*
	 * This function is just a stub to implement a hard force
	 * of synchronize_sched(). This requires synchronizing
	 * tasks even in userspace and idle.
	 *
	 * Yes, function tracing is rude.
	 */
}

static void ftrace_sync_ipi(void *data)
{
	/* Probably not needed, but do it anyway */
	smp_rmb();
}

244 245
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static void update_function_graph_func(void);
246 247 248 249 250

/* Both enabled by default (can be cleared by function_graph tracer flags */
static bool fgraph_sleep_time = true;
static bool fgraph_graph_time = true;

251 252 253 254
#else
static inline void update_function_graph_func(void) { }
#endif

255 256 257 258

static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
	/*
259
	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
260 261
	 * then it needs to call the list anyway.
	 */
262 263
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU |
			  FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC)
264 265 266 267 268
		return ftrace_ops_list_func;

	return ftrace_ops_get_func(ops);
}

269 270 271 272
static void update_ftrace_function(void)
{
	ftrace_func_t func;

273 274 275 276 277 278 279 280 281 282 283
	/*
	 * Prepare the ftrace_ops that the arch callback will use.
	 * If there's only one ftrace_ops registered, the ftrace_ops_list
	 * will point to the ops we want.
	 */
	set_function_trace_op = ftrace_ops_list;

	/* If there's no ftrace_ops registered, just call the stub function */
	if (ftrace_ops_list == &ftrace_list_end) {
		func = ftrace_stub;

284 285
	/*
	 * If we are at the end of the list and this ops is
286 287
	 * recursion safe and not dynamic and the arch supports passing ops,
	 * then have the mcount trampoline call the function directly.
288
	 */
289
	} else if (ftrace_ops_list->next == &ftrace_list_end) {
290
		func = ftrace_ops_get_list_func(ftrace_ops_list);
291

292 293
	} else {
		/* Just use the default ftrace_ops */
294
		set_function_trace_op = &ftrace_list_end;
295
		func = ftrace_ops_list_func;
296
	}
297

298 299
	update_function_graph_func();

300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
	/* If there's no change, then do nothing more here */
	if (ftrace_trace_function == func)
		return;

	/*
	 * If we are using the list function, it doesn't care
	 * about the function_trace_ops.
	 */
	if (func == ftrace_ops_list_func) {
		ftrace_trace_function = func;
		/*
		 * Don't even bother setting function_trace_ops,
		 * it would be racy to do so anyway.
		 */
		return;
	}

#ifndef CONFIG_DYNAMIC_FTRACE
	/*
	 * For static tracing, we need to be a bit more careful.
	 * The function change takes affect immediately. Thus,
	 * we need to coorditate the setting of the function_trace_ops
	 * with the setting of the ftrace_trace_function.
	 *
	 * Set the function to the list ops, which will call the
	 * function we want, albeit indirectly, but it handles the
	 * ftrace_ops and doesn't depend on function_trace_op.
	 */
	ftrace_trace_function = ftrace_ops_list_func;
	/*
	 * Make sure all CPUs see this. Yes this is slow, but static
	 * tracing is slow and nasty to have enabled.
	 */
	schedule_on_each_cpu(ftrace_sync);
	/* Now all cpus are using the list ops. */
	function_trace_op = set_function_trace_op;
	/* Make sure the function_trace_op is visible on all CPUs */
	smp_wmb();
	/* Nasty way to force a rmb on all cpus */
	smp_call_function(ftrace_sync_ipi, NULL, 1);
	/* OK, we are all set to update the ftrace_trace_function now! */
#endif /* !CONFIG_DYNAMIC_FTRACE */

343 344 345
	ftrace_trace_function = func;
}

346 347 348 349 350
int using_ftrace_ops_list_func(void)
{
	return ftrace_trace_function == ftrace_ops_list_func;
}

351
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
352
{
353
	ops->next = *list;
354
	/*
355
	 * We are entering ops into the list but another
356 357
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
358
	 * the ops pointer included into the list.
359
	 */
360
	rcu_assign_pointer(*list, ops);
361 362
}

363
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
364 365 366 367
{
	struct ftrace_ops **p;

	/*
368 369
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
370
	 */
371 372
	if (*list == ops && ops->next == &ftrace_list_end) {
		*list = &ftrace_list_end;
S
Steven Rostedt 已提交
373
		return 0;
374 375
	}

376
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
377 378 379
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
380 381
	if (*p != ops)
		return -1;
382 383

	*p = (*p)->next;
384 385
	return 0;
}
386

387 388
static void ftrace_update_trampoline(struct ftrace_ops *ops);

389 390
static int __register_ftrace_function(struct ftrace_ops *ops)
{
391 392 393
	if (ops->flags & FTRACE_OPS_FL_DELETED)
		return -EINVAL;

394 395 396
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

397
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
398 399 400 401 402 403 404 405 406 407 408 409 410
	/*
	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
	 */
	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
		return -EINVAL;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif

411 412 413
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

414 415
	if (ops->flags & FTRACE_OPS_FL_PER_CPU) {
		if (per_cpu_ops_alloc(ops))
416
			return -ENOMEM;
417 418 419
	}

	add_ftrace_ops(&ftrace_ops_list, ops);
420

421 422 423
	/* Always save the function, and reset at unregistering */
	ops->saved_func = ops->func;

424
	if (ftrace_pids_enabled(ops))
425 426
		ops->func = ftrace_pid_func;

427 428
	ftrace_update_trampoline(ops);

429 430 431 432 433 434 435 436 437 438
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

439 440 441
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

442
	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
443

444 445
	if (ret < 0)
		return ret;
446

447 448
	if (ftrace_enabled)
		update_ftrace_function();
449

450 451
	ops->func = ops->saved_func;

S
Steven Rostedt 已提交
452
	return 0;
453 454
}

455 456
static void ftrace_update_pid_func(void)
{
457 458
	struct ftrace_ops *op;

459
	/* Only do something if we are tracing something */
460
	if (ftrace_trace_function == ftrace_stub)
461
		return;
462

463 464
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->flags & FTRACE_OPS_FL_PID) {
465 466
			op->func = ftrace_pids_enabled(op) ?
				ftrace_pid_func : op->saved_func;
467 468 469 470
			ftrace_update_trampoline(op);
		}
	} while_for_each_ftrace_op(op);

471
	update_ftrace_function();
472 473
}

474 475 476 477 478
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
479 480
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
481
	unsigned long long		time_squared;
482
#endif
483 484
};

485 486 487 488
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
489 490
};

491 492 493 494 495 496 497 498
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

499 500
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
501

502 503
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
504

505 506 507
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
508 509
static DEFINE_MUTEX(ftrace_profile_lock);

510
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
511

512 513
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
514

S
Steven Rostedt 已提交
515 516 517
static void *
function_stat_next(void *v, int idx)
{
518 519
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
520

521
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
522 523

 again:
L
Li Zefan 已提交
524 525 526
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
527 528 529 530 531
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
532 533
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
534 535 536 537 538 539 540
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
541 542 543 544 545 546 547
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
548 549
}

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
566 567
static int function_stat_cmp(void *p1, void *p2)
{
568 569
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
570 571 572 573 574 575 576 577

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
578
#endif
S
Steven Rostedt 已提交
579 580 581

static int function_stat_headers(struct seq_file *m)
{
582
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
583 584 585 586
	seq_puts(m, "  Function                               "
		 "Hit    Time            Avg             s^2\n"
		    "  --------                               "
		 "---    ----            ---             ---\n");
587
#else
588 589
	seq_puts(m, "  Function                               Hit\n"
		    "  --------                               ---\n");
590
#endif
S
Steven Rostedt 已提交
591 592 593 594 595
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
596
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
597
	char str[KSYM_SYMBOL_LEN];
598
	int ret = 0;
599
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
600 601
	static struct trace_seq s;
	unsigned long long avg;
602
	unsigned long long stddev;
603
#endif
604 605 606 607 608 609 610
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
S
Steven Rostedt 已提交
611

612 613 614 615 616 617 618
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	avg = rec->time;
	do_div(avg, rec->counter);
	if (tracing_thresh && (avg < tracing_thresh))
		goto out;
#endif

S
Steven Rostedt 已提交
619
	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
620 621 622
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
623
	seq_puts(m, "    ");
624

625 626 627 628
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
629 630 631 632 633 634 635
		/*
		 * Apply Welford's method:
		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
		 */
		stddev = rec->counter * rec->time_squared -
			 rec->time * rec->time;

636 637 638 639
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
640
		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
641 642
	}

643 644 645 646
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
647 648
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
649 650 651
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
652 653
out:
	mutex_unlock(&ftrace_profile_lock);
S
Steven Rostedt 已提交
654

655
	return ret;
S
Steven Rostedt 已提交
656 657
}

658
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
659
{
660
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
661

662
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
663

664 665 666 667
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
668 669
	}

670
	memset(stat->hash, 0,
671 672
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
673

674
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
675 676
{
	struct ftrace_profile_page *pg;
677 678
	int functions;
	int pages;
679
	int i;
S
Steven Rostedt 已提交
680

681
	/* If we already allocated, do nothing */
682
	if (stat->pages)
683
		return 0;
S
Steven Rostedt 已提交
684

685 686
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
687
		return -ENOMEM;
S
Steven Rostedt 已提交
688

689 690 691 692 693 694 695 696 697 698 699 700 701
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

702
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
703

704 705
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

706
	for (i = 1; i < pages; i++) {
707 708
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
709
			goto out_free;
710 711 712 713
		pg = pg->next;
	}

	return 0;
714 715 716 717 718 719 720 721 722 723 724 725 726 727

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
728 729
}

730
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
731
{
732
	struct ftrace_profile_stat *stat;
733
	int size;
S
Steven Rostedt 已提交
734

735 736 737
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
738
		/* If the profile is already created, simply reset it */
739
		ftrace_profile_reset(stat);
740 741
		return 0;
	}
S
Steven Rostedt 已提交
742

743 744 745 746 747
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
748

749
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
750

751
	if (!stat->hash)
752 753
		return -ENOMEM;

754
	/* Preallocate the function profiling pages */
755 756 757
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
758 759 760 761
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
762 763
}

764 765 766 767 768
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

769
	for_each_possible_cpu(cpu) {
770 771 772 773 774 775 776 777
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

778
/* interrupts must be disabled */
779 780
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
781
{
782
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
783 784 785
	struct hlist_head *hhd;
	unsigned long key;

786
	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
787
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
788 789 790 791

	if (hlist_empty(hhd))
		return NULL;

792
	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
S
Steven Rostedt 已提交
793
		if (rec->ip == ip)
794 795 796 797 798 799
			return rec;
	}

	return NULL;
}

800 801
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
802 803 804
{
	unsigned long key;

805
	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
806
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
807 808
}

809 810 811
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
812
static struct ftrace_profile *
813
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
814 815 816
{
	struct ftrace_profile *rec = NULL;

817
	/* prevent recursion (from NMIs) */
818
	if (atomic_inc_return(&stat->disabled) != 1)
819 820 821
		goto out;

	/*
822 823
	 * Try to find the function again since an NMI
	 * could have added it
824
	 */
825
	rec = ftrace_find_profiled_func(stat, ip);
826
	if (rec)
827
		goto out;
828

829 830 831 832
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
833
	}
834

835
	rec = &stat->pages->records[stat->pages->index++];
836
	rec->ip = ip;
837
	ftrace_add_profile(stat, rec);
838

S
Steven Rostedt 已提交
839
 out:
840
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
841 842 843 844 845

	return rec;
}

static void
846
function_profile_call(unsigned long ip, unsigned long parent_ip,
847
		      struct ftrace_ops *ops, struct pt_regs *regs)
S
Steven Rostedt 已提交
848
{
849
	struct ftrace_profile_stat *stat;
850
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
851 852 853 854 855 856
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
857

858
	stat = this_cpu_ptr(&ftrace_profile_stats);
859
	if (!stat->hash || !ftrace_profile_enabled)
860 861 862
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
863
	if (!rec) {
864
		rec = ftrace_profile_alloc(stat, ip);
865 866 867
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
868 869 870 871 872 873

	rec->counter++;
 out:
	local_irq_restore(flags);
}

874 875 876
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
877 878
	int index = trace->depth;

879
	function_profile_call(trace->func, 0, NULL, NULL);
880 881 882 883

	if (index >= 0 && index < FTRACE_RETFUNC_DEPTH)
		current->ret_stack[index].subtime = 0;

884 885 886 887 888
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
889
	struct ftrace_profile_stat *stat;
890
	unsigned long long calltime;
891
	struct ftrace_profile *rec;
892
	unsigned long flags;
893 894

	local_irq_save(flags);
895
	stat = this_cpu_ptr(&ftrace_profile_stats);
896
	if (!stat->hash || !ftrace_profile_enabled)
897 898
		goto out;

899 900 901 902
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

903 904
	calltime = trace->rettime - trace->calltime;

905
	if (!fgraph_graph_time) {
906 907 908 909 910 911 912 913 914 915 916 917 918 919
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

920
	rec = ftrace_find_profiled_func(stat, trace->func);
921
	if (rec) {
922
		rec->time += calltime;
923 924
		rec->time_squared += calltime * calltime;
	}
925

926
 out:
927 928 929 930 931 932 933 934 935 936 937 938 939 940
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
941
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
942
	.func		= function_profile_call,
943
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
944
	INIT_OPS_HASH(ftrace_profile_ops)
S
Steven Rostedt 已提交
945 946
};

947 948 949 950 951 952 953 954 955 956 957
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
958 959 960 961 962 963 964
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

965 966
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
S
Steven Rostedt 已提交
967 968 969 970 971 972 973
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
974 975 976 977 978 979
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

980 981 982 983 984
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
985 986 987
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
988 989 990 991
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
992
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
993 994
		}
	}
995
 out:
S
Steven Rostedt 已提交
996 997
	mutex_unlock(&ftrace_profile_lock);

998
	*ppos += cnt;
S
Steven Rostedt 已提交
999 1000 1001 1002

	return cnt;
}

1003 1004 1005 1006
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
1007
	char buf[64];		/* big enough to hold a number */
1008 1009 1010 1011 1012 1013
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
1014 1015 1016 1017
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
1018
	.llseek		= default_llseek,
S
Steven Rostedt 已提交
1019 1020
};

1021 1022
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
1023 1024 1025 1026 1027 1028
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
1029 1030
};

1031
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1032
{
1033
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
1034
	struct dentry *entry;
1035
	char *name;
S
Steven Rostedt 已提交
1036
	int ret;
1037 1038 1039 1040 1041
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

1042
		name = kasprintf(GFP_KERNEL, "function%d", cpu);
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
1063 1064
	}

1065
	entry = tracefs_create_file("function_profile_enabled", 0644,
S
Steven Rostedt 已提交
1066 1067
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
1068
		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
S
Steven Rostedt 已提交
1069 1070 1071
}

#else /* CONFIG_FUNCTION_PROFILER */
1072
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1073 1074 1075 1076
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

1077 1078
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

1079 1080 1081 1082 1083 1084
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int ftrace_graph_active;
#else
# define ftrace_graph_active 0
#endif

1085 1086
#ifdef CONFIG_DYNAMIC_FTRACE

1087 1088
static struct ftrace_ops *removed_ops;

1089 1090 1091 1092 1093 1094
/*
 * Set when doing a global update, like enabling all recs or disabling them.
 * It is not set when just updating a single ftrace_ops.
 */
static bool update_all_ops;

1095 1096 1097 1098
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

1099 1100 1101 1102 1103
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

1104 1105 1106 1107 1108
struct ftrace_func_probe {
	struct ftrace_probe_ops	*probe_ops;
	struct ftrace_ops	ops;
	struct trace_array	*tr;
	struct list_head	list;
1109
	void			*data;
1110 1111 1112
	int			ref;
};

1113 1114 1115 1116 1117 1118 1119 1120 1121
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
1122
};
1123
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1124

1125
static struct ftrace_ops global_ops = {
1126 1127 1128 1129 1130
	.func				= ftrace_stub,
	.local_hash.notrace_hash	= EMPTY_HASH,
	.local_hash.filter_hash		= EMPTY_HASH,
	INIT_OPS_HASH(global_ops)
	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1131 1132
					  FTRACE_OPS_FL_INITIALIZED |
					  FTRACE_OPS_FL_PID,
1133 1134
};

1135 1136
/*
 * This is used by __kernel_text_address() to return true if the
1137
 * address is on a dynamically allocated trampoline that would
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
 * not return true for either core_kernel_text() or
 * is_module_text_address().
 */
bool is_ftrace_trampoline(unsigned long addr)
{
	struct ftrace_ops *op;
	bool ret = false;

	/*
	 * Some of the ops may be dynamically allocated,
	 * they are freed after a synchronize_sched().
	 */
	preempt_disable_notrace();

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/*
		 * This is to check for dynamically allocated trampolines.
		 * Trampolines that are in kernel text will have
		 * core_kernel_text() return true.
		 */
		if (op->trampoline && op->trampoline_size)
			if (addr >= op->trampoline &&
			    addr < op->trampoline + op->trampoline_size) {
				ret = true;
				goto out;
			}
	} while_for_each_ftrace_op(op);

 out:
	preempt_enable_notrace();

	return ret;
}

1172 1173
struct ftrace_page {
	struct ftrace_page	*next;
1174
	struct dyn_ftrace	*records;
1175
	int			index;
1176
	int			size;
1177 1178
};

1179 1180
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1181 1182 1183 1184 1185 1186 1187

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

1188 1189 1190 1191 1192 1193 1194 1195 1196
static __always_inline unsigned long
ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
{
	if (hash->size_bits > 0)
		return hash_long(ip, hash->size_bits);

	return 0;
}

1197 1198 1199
/* Only use this function if ftrace_hash_empty() has already been tested */
static __always_inline struct ftrace_func_entry *
__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1200 1201 1202 1203 1204
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;

1205
	key = ftrace_hash_key(hash, ip);
1206 1207
	hhd = &hash->buckets[key];

1208
	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1209 1210 1211 1212 1213 1214
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
/**
 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
 * @hash: The hash to look at
 * @ip: The instruction pointer to test
 *
 * Search a given @hash to see if a given instruction pointer (@ip)
 * exists in it.
 *
 * Returns the entry that holds the @ip if found. NULL otherwise.
 */
struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	if (ftrace_hash_empty(hash))
		return NULL;

	return __ftrace_lookup_ip(hash, ip);
}

1234 1235
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1236 1237 1238 1239
{
	struct hlist_head *hhd;
	unsigned long key;

1240
	key = ftrace_hash_key(hash, entry->ip);
1241 1242 1243
	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1256 1257 1258 1259 1260

	return 0;
}

static void
1261
free_hash_entry(struct ftrace_hash *hash,
1262 1263 1264 1265 1266 1267 1268
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1269 1270 1271 1272
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
1273
	hlist_del_rcu(&entry->hlist);
1274 1275 1276
	hash->count--;
}

1277 1278 1279
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
1280
	struct hlist_node *tn;
1281 1282 1283 1284
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1285 1286 1287
	if (!hash->count)
		return;

1288 1289
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
1290
		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1291
			free_hash_entry(hash, entry);
1292 1293 1294 1295
	}
	FTRACE_WARN_ON(hash->count);
}

1296 1297 1298 1299 1300 1301 1302 1303 1304
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}

1320 1321
void ftrace_free_filter(struct ftrace_ops *ops)
{
1322
	ftrace_ops_init(ops);
1323 1324
	free_ftrace_hash(ops->func_hash->filter_hash);
	free_ftrace_hash(ops->func_hash->notrace_hash);
1325 1326
}

1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
1337
	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

	/* Empty hash? */
1363
	if (ftrace_hash_empty(hash))
1364 1365 1366 1367
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
1368
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

1384
static void
1385
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1386
static void
1387
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1388

1389 1390 1391
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash);

1392 1393
static struct ftrace_hash *
__ftrace_hash_move(struct ftrace_hash *src)
1394 1395
{
	struct ftrace_func_entry *entry;
1396
	struct hlist_node *tn;
1397
	struct hlist_head *hhd;
1398
	struct ftrace_hash *new_hash;
1399 1400 1401 1402 1403
	int size = src->count;
	int bits = 0;
	int i;

	/*
1404
	 * If the new source is empty, just return the empty_hash.
1405
	 */
1406 1407
	if (!src->count)
		return EMPTY_HASH;
1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1419 1420
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
1421
		return NULL;
1422 1423 1424 1425

	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
1426
		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1427
			remove_hash_entry(src, entry);
1428
			__add_hash_entry(new_hash, entry);
1429 1430 1431
		}
	}

1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
	return new_hash;
}

static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
		 struct ftrace_hash **dst, struct ftrace_hash *src)
{
	struct ftrace_hash *new_hash;
	int ret;

	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
		return -EINVAL;

	new_hash = __ftrace_hash_move(src);
	if (!new_hash)
		return -ENOMEM;

1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
	if (enable) {
		/* IPMODIFY should be updated only when filter_hash updating */
		ret = ftrace_hash_ipmodify_update(ops, new_hash);
		if (ret < 0) {
			free_ftrace_hash(new_hash);
			return ret;
		}
	}

1460 1461 1462 1463
	/*
	 * Remove the current set, update the hash and add
	 * them back.
	 */
1464
	ftrace_hash_rec_disable_modify(ops, enable);
1465

1466 1467
	rcu_assign_pointer(*dst, new_hash);

1468
	ftrace_hash_rec_enable_modify(ops, enable);
1469

1470
	return 0;
1471 1472
}

1473 1474 1475 1476 1477 1478 1479 1480 1481 1482
static bool hash_contains_ip(unsigned long ip,
			     struct ftrace_ops_hash *hash)
{
	/*
	 * The function record is a match if it exists in the filter
	 * hash and not in the notrace hash. Note, an emty hash is
	 * considered a match for the filter hash, but an empty
	 * notrace hash is considered not in the notrace hash.
	 */
	return (ftrace_hash_empty(hash->filter_hash) ||
1483
		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1484
		(ftrace_hash_empty(hash->notrace_hash) ||
1485
		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1486 1487
}

1488 1489 1490 1491 1492 1493 1494 1495
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1496 1497 1498
 *
 * This needs to be called with preemption disabled as
 * the hashes are freed with call_rcu_sched().
1499 1500
 */
static int
1501
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1502
{
1503
	struct ftrace_ops_hash hash;
1504 1505
	int ret;

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	/*
	 * There's a small race when adding ops that the ftrace handler
	 * that wants regs, may be called without them. We can not
	 * allow that handler to be called if regs is NULL.
	 */
	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
		return 0;
#endif

1516 1517
	hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
	hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
1518

1519
	if (hash_contains_ip(ip, &hash))
1520 1521 1522 1523 1524 1525 1526
		ret = 1;
	else
		ret = 0;

	return ret;
}

1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1541 1542 1543

static int ftrace_cmp_recs(const void *a, const void *b)
{
1544 1545
	const struct dyn_ftrace *key = a;
	const struct dyn_ftrace *rec = b;
1546

1547
	if (key->flags < rec->ip)
1548
		return -1;
1549 1550
	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
		return 1;
1551 1552 1553
	return 0;
}

1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
/**
 * ftrace_location_range - return the first address of a traced location
 *	if it touches the given ip range
 * @start: start of range to search.
 * @end: end of range to search (inclusive). @end points to the last byte
 *	to check.
 *
 * Returns rec->ip if the related ftrace location is a least partly within
 * the given address range. That is, the first address of the instruction
 * that is either a NOP or call to the function tracer. It checks the ftrace
 * internal tables to determine if the address belongs or not.
 */
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1567 1568 1569
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1570
	struct dyn_ftrace key;
1571

1572 1573
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */
1574 1575

	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1576 1577
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1578
			continue;
1579 1580 1581 1582
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (rec)
1583
			return rec->ip;
1584
	}
1585 1586 1587 1588

	return 0;
}

1589 1590 1591 1592
/**
 * ftrace_location - return true if the ip giving is a traced location
 * @ip: the instruction pointer to check
 *
1593
 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1594 1595 1596 1597
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1598
unsigned long ftrace_location(unsigned long ip)
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
{
	return ftrace_location_range(ip, ip);
}

/**
 * ftrace_text_reserved - return true if range contains an ftrace location
 * @start: start of range to search
 * @end: end of range to search (inclusive). @end points to the last byte to check.
 *
 * Returns 1 if @start and @end contains a ftrace location.
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1613
int ftrace_text_reserved(const void *start, const void *end)
1614
{
1615 1616 1617 1618 1619 1620
	unsigned long ret;

	ret = ftrace_location_range((unsigned long)start,
				    (unsigned long)end);

	return (int)!!ret;
1621 1622
}

1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
/* Test if ops registered to this rec needs regs */
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	bool keep_regs = false;

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next) {
		/* pass rec in as regs to have non-NULL val */
		if (ftrace_ops_test(ops, rec->ip, rec)) {
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				keep_regs = true;
				break;
			}
		}
	}

	return  keep_regs;
}

1643
static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1644 1645 1646 1647 1648 1649 1650
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1651
	bool update = false;
1652 1653 1654 1655 1656
	int count = 0;
	int all = 0;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1657
		return false;
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
1671 1672
		hash = ops->func_hash->filter_hash;
		other_hash = ops->func_hash->notrace_hash;
1673
		if (ftrace_hash_empty(hash))
1674 1675 1676
			all = 1;
	} else {
		inc = !inc;
1677 1678
		hash = ops->func_hash->notrace_hash;
		other_hash = ops->func_hash->filter_hash;
1679 1680 1681 1682
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1683
		if (ftrace_hash_empty(hash))
1684
			return false;
1685 1686 1687 1688 1689 1690 1691
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

1692 1693 1694
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1695 1696 1697 1698 1699
		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1700
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1701 1702
				match = 1;
		} else {
1703 1704
			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1705 1706

			/*
1707 1708
			 * If filter_hash is set, we want to match all functions
			 * that are in the hash but not in the other hash.
1709
			 *
1710 1711 1712 1713 1714
			 * If filter_hash is not set, then we are decrementing.
			 * That means we match anything that is in the hash
			 * and also in the other_hash. That is, we need to turn
			 * off functions in the other hash because they are disabled
			 * by this hash.
1715 1716 1717 1718
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
1719
				 (in_other_hash || ftrace_hash_empty(other_hash)))
1720 1721 1722 1723 1724 1725 1726
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
1727
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1728
				return false;
1729 1730 1731 1732 1733 1734

			/*
			 * If there's only a single callback registered to a
			 * function, and the ops has a trampoline registered
			 * for it, then we can call it directly.
			 */
1735
			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1736
				rec->flags |= FTRACE_FL_TRAMP;
1737
			else
1738 1739 1740
				/*
				 * If we are adding another function callback
				 * to this function, and the previous had a
1741 1742
				 * custom trampoline in use, then we need to go
				 * back to the default trampoline.
1743
				 */
1744
				rec->flags &= ~FTRACE_FL_TRAMP;
1745

1746 1747 1748 1749 1750 1751
			/*
			 * If any ops wants regs saved for this function
			 * then all ops will get saved regs.
			 */
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
				rec->flags |= FTRACE_FL_REGS;
1752
		} else {
1753
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1754
				return false;
1755
			rec->flags--;
1756

1757 1758 1759 1760 1761 1762
			/*
			 * If the rec had REGS enabled and the ops that is
			 * being removed had REGS set, then see if there is
			 * still any ops for this record that wants regs.
			 * If not, we can stop recording them.
			 */
1763
			if (ftrace_rec_count(rec) > 0 &&
1764 1765 1766 1767 1768
			    rec->flags & FTRACE_FL_REGS &&
			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				if (!test_rec_ops_needs_regs(rec))
					rec->flags &= ~FTRACE_FL_REGS;
			}
1769

1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780
			/*
			 * If the rec had TRAMP enabled, then it needs to
			 * be cleared. As TRAMP can only be enabled iff
			 * there is only a single ops attached to it.
			 * In otherwords, always disable it on decrementing.
			 * In the future, we may set it if rec count is
			 * decremented to one, and the ops that is left
			 * has a trampoline.
			 */
			rec->flags &= ~FTRACE_FL_TRAMP;

1781 1782 1783 1784
			/*
			 * flags will be cleared in ftrace_check_record()
			 * if rec count is zero.
			 */
1785 1786
		}
		count++;
1787 1788 1789 1790

		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;

1791 1792
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
1793
			return update;
1794
	} while_for_each_ftrace_rec();
1795 1796

	return update;
1797 1798
}

1799
static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1800 1801
				    int filter_hash)
{
1802
	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1803 1804
}

1805
static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1806 1807
				   int filter_hash)
{
1808
	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1809 1810
}

1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
					  int filter_hash, int inc)
{
	struct ftrace_ops *op;

	__ftrace_hash_rec_update(ops, filter_hash, inc);

	if (ops->func_hash != &global_ops.local_hash)
		return;

	/*
	 * If the ops shares the global_ops hash, then we need to update
	 * all ops that are enabled and use this hash.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* Already done */
		if (op == ops)
			continue;
		if (op->func_hash == &global_ops.local_hash)
			__ftrace_hash_rec_update(op, filter_hash, inc);
	} while_for_each_ftrace_op(op);
}

static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
					   int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
					  int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}

1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
/*
 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
 * or no-needed to update, -EBUSY if it detects a conflict of the flag
 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
 * Note that old_hash and new_hash has below meanings
 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
 *  - If the hash is EMPTY_HASH, it hits nothing
 *  - Anything else hits the recs which match the hash entries.
 */
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
					 struct ftrace_hash *old_hash,
					 struct ftrace_hash *new_hash)
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec, *end = NULL;
	int in_old, in_new;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
		return 0;

	/*
	 * Since the IPMODIFY is a very address sensitive action, we do not
	 * allow ftrace_ops to set all functions to new hash.
	 */
	if (!new_hash || !old_hash)
		return -EINVAL;

	/* Update rec->flags */
	do_for_each_ftrace_rec(pg, rec) {
1879 1880 1881 1882

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
		/* We need to update only differences of filter_hash */
		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new) {
			/* New entries must ensure no others are using it */
			if (rec->flags & FTRACE_FL_IPMODIFY)
				goto rollback;
			rec->flags |= FTRACE_FL_IPMODIFY;
		} else /* Removed entry */
			rec->flags &= ~FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

	return 0;

rollback:
	end = rec;

	/* Roll back what we did above */
	do_for_each_ftrace_rec(pg, rec) {
1905 1906 1907 1908

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
		if (rec == end)
			goto err_out;

		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new)
			rec->flags &= ~FTRACE_FL_IPMODIFY;
		else
			rec->flags |= FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

err_out:
	return -EBUSY;
}

static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}

/* Disabling always succeeds */
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}

static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash)
{
	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(old_hash))
		old_hash = NULL;

	if (ftrace_hash_empty(new_hash))
		new_hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}

1962
static void print_ip_ins(const char *fmt, const unsigned char *p)
1963 1964 1965 1966 1967 1968 1969 1970 1971
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1972 1973
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1974 1975
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1976

1977
enum ftrace_bug_type ftrace_bug_type;
1978
const void *ftrace_expected;
1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999

static void print_bug_type(void)
{
	switch (ftrace_bug_type) {
	case FTRACE_BUG_UNKNOWN:
		break;
	case FTRACE_BUG_INIT:
		pr_info("Initializing ftrace call sites\n");
		break;
	case FTRACE_BUG_NOP:
		pr_info("Setting ftrace call site to NOP\n");
		break;
	case FTRACE_BUG_CALL:
		pr_info("Setting ftrace call site to call ftrace function\n");
		break;
	case FTRACE_BUG_UPDATE:
		pr_info("Updating ftrace call site to call a different ftrace function\n");
		break;
	}
}

2000 2001 2002
/**
 * ftrace_bug - report and shutdown function tracer
 * @failed: The failed type (EFAULT, EINVAL, EPERM)
2003
 * @rec: The record that failed
2004 2005 2006 2007 2008 2009 2010 2011
 *
 * The arch code that enables or disables the function tracing
 * can call ftrace_bug() when it has detected a problem in
 * modifying the code. @failed should be one of either:
 * EFAULT - if the problem happens on reading the @ip address
 * EINVAL - if what is read at @ip is not what was expected
 * EPERM - if the problem happens on writting to the @ip address
 */
2012
void ftrace_bug(int failed, struct dyn_ftrace *rec)
2013
{
2014 2015
	unsigned long ip = rec ? rec->ip : 0;

2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
2026
		print_ip_ins(" actual:   ", (unsigned char *)ip);
2027
		pr_cont("\n");
2028 2029 2030 2031
		if (ftrace_expected) {
			print_ip_ins(" expected: ", ftrace_expected);
			pr_cont("\n");
		}
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
2043
	print_bug_type();
2044 2045 2046 2047 2048 2049 2050 2051
	if (rec) {
		struct ftrace_ops *ops = NULL;

		pr_info("ftrace record flags: %lx\n", rec->flags);
		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
			ops = ftrace_find_tramp_ops_any(rec);
2052 2053 2054 2055 2056 2057 2058 2059
			if (ops) {
				do {
					pr_cont("\ttramp: %pS (%pS)",
						(void *)ops->trampoline,
						(void *)ops->func);
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
2060 2061 2062 2063
				pr_cont("\ttramp: ERROR!");

		}
		ip = ftrace_get_addr_curr(rec);
2064
		pr_cont("\n expected tramp: %lx\n", ip);
2065
	}
2066 2067
}

2068
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2069
{
2070
	unsigned long flag = 0UL;
2071

2072 2073
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2074 2075 2076
	if (rec->flags & FTRACE_FL_DISABLED)
		return FTRACE_UPDATE_IGNORE;

S
Steven Rostedt 已提交
2077
	/*
2078
	 * If we are updating calls:
S
Steven Rostedt 已提交
2079
	 *
2080 2081
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
S
Steven Rostedt 已提交
2082
	 *
2083 2084
	 *   Otherwise we make sure its disabled.
	 *
2085
	 * If we are disabling calls, then disable all records that
2086
	 * are enabled.
S
Steven Rostedt 已提交
2087
	 */
2088
	if (enable && ftrace_rec_count(rec))
2089
		flag = FTRACE_FL_ENABLED;
S
Steven Rostedt 已提交
2090

2091
	/*
2092 2093 2094
	 * If enabling and the REGS flag does not match the REGS_EN, or
	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
	 * this record. Set flags to fail the compare against ENABLED.
2095
	 */
2096 2097 2098 2099 2100 2101 2102 2103 2104
	if (flag) {
		if (!(rec->flags & FTRACE_FL_REGS) != 
		    !(rec->flags & FTRACE_FL_REGS_EN))
			flag |= FTRACE_FL_REGS;

		if (!(rec->flags & FTRACE_FL_TRAMP) != 
		    !(rec->flags & FTRACE_FL_TRAMP_EN))
			flag |= FTRACE_FL_TRAMP;
	}
2105

2106 2107
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2108
		return FTRACE_UPDATE_IGNORE;
S
Steven Rostedt 已提交
2109

2110
	if (flag) {
2111 2112 2113 2114
		/* Save off if rec is being enabled (for return value) */
		flag ^= rec->flags & FTRACE_FL_ENABLED;

		if (update) {
2115
			rec->flags |= FTRACE_FL_ENABLED;
2116 2117 2118 2119 2120 2121
			if (flag & FTRACE_FL_REGS) {
				if (rec->flags & FTRACE_FL_REGS)
					rec->flags |= FTRACE_FL_REGS_EN;
				else
					rec->flags &= ~FTRACE_FL_REGS_EN;
			}
2122 2123 2124 2125 2126 2127
			if (flag & FTRACE_FL_TRAMP) {
				if (rec->flags & FTRACE_FL_TRAMP)
					rec->flags |= FTRACE_FL_TRAMP_EN;
				else
					rec->flags &= ~FTRACE_FL_TRAMP_EN;
			}
2128 2129 2130 2131 2132 2133 2134
		}

		/*
		 * If this record is being updated from a nop, then
		 *   return UPDATE_MAKE_CALL.
		 * Otherwise,
		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2135
		 *   from the save regs, to a non-save regs function or
2136
		 *   vice versa, or from a trampoline call.
2137
		 */
2138 2139
		if (flag & FTRACE_FL_ENABLED) {
			ftrace_bug_type = FTRACE_BUG_CALL;
2140
			return FTRACE_UPDATE_MAKE_CALL;
2141
		}
2142

2143
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2144
		return FTRACE_UPDATE_MODIFY_CALL;
2145 2146
	}

2147 2148
	if (update) {
		/* If there's no more users, clear all flags */
2149
		if (!ftrace_rec_count(rec))
2150 2151
			rec->flags = 0;
		else
2152 2153 2154 2155 2156 2157
			/*
			 * Just disable the record, but keep the ops TRAMP
			 * and REGS states. The _EN flags must be disabled though.
			 */
			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
					FTRACE_FL_REGS_EN);
2158
	}
2159

2160
	ftrace_bug_type = FTRACE_BUG_NOP;
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
	return FTRACE_UPDATE_MAKE_NOP;
}

/**
 * ftrace_update_record, set a record that now is tracing or not
 * @rec: the record to update
 * @enable: set to 1 if the record is tracing, zero to force disable
 *
 * The records that represent all functions that can be traced need
 * to be updated when tracing has been enabled.
 */
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 1);
}

/**
 * ftrace_test_record, check if the record has been enabled or not
 * @rec: the record to test
 * @enable: set to 1 to check if enabled, 0 if it is disabled
 *
 * The arch code may need to test if a record is already set to
 * tracing to determine how to modify the function code that it
 * represents.
 */
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 0);
}

2191 2192 2193 2194
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2195
	unsigned long ip = rec->ip;
2196 2197 2198 2199 2200 2201

	do_for_each_ftrace_op(op, ftrace_ops_list) {

		if (!op->trampoline)
			continue;

2202
		if (hash_contains_ip(ip, op->func_hash))
2203 2204 2205 2206 2207 2208
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
			   struct ftrace_ops *op)
{
	unsigned long ip = rec->ip;

	while_for_each_ftrace_op(op) {

		if (!op->trampoline)
			continue;

		if (hash_contains_ip(ip, op->func_hash))
			return op;
	} 

	return NULL;
}

2227 2228 2229 2230
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2231
	unsigned long ip = rec->ip;
2232

2233 2234 2235 2236 2237 2238 2239 2240
	/*
	 * Need to check removed ops first.
	 * If they are being removed, and this rec has a tramp,
	 * and this rec is in the ops list, then it would be the
	 * one with the tramp.
	 */
	if (removed_ops) {
		if (hash_contains_ip(ip, &removed_ops->old_hash))
2241 2242 2243
			return removed_ops;
	}

2244 2245 2246 2247 2248 2249 2250
	/*
	 * Need to find the current trampoline for a rec.
	 * Now, a trampoline is only attached to a rec if there
	 * was a single 'ops' attached to it. But this can be called
	 * when we are adding another op to the rec or removing the
	 * current one. Thus, if the op is being added, we can
	 * ignore it because it hasn't attached itself to the rec
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
	 * yet.
	 *
	 * If an ops is being modified (hooking to different functions)
	 * then we don't care about the new functions that are being
	 * added, just the old ones (that are probably being removed).
	 *
	 * If we are adding an ops to a function that already is using
	 * a trampoline, it needs to be removed (trampolines are only
	 * for single ops connected), then an ops that is not being
	 * modified also needs to be checked.
2261
	 */
2262
	do_for_each_ftrace_op(op, ftrace_ops_list) {
2263 2264 2265 2266 2267 2268 2269 2270 2271

		if (!op->trampoline)
			continue;

		/*
		 * If the ops is being added, it hasn't gotten to
		 * the point to be removed from this tree yet.
		 */
		if (op->flags & FTRACE_OPS_FL_ADDING)
2272 2273
			continue;

2274

2275
		/*
2276 2277 2278
		 * If the ops is being modified and is in the old
		 * hash, then it is probably being removed from this
		 * function.
2279 2280 2281
		 */
		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, &op->old_hash))
2282
			return op;
2283 2284 2285 2286 2287 2288 2289 2290
		/*
		 * If the ops is not being added or modified, and it's
		 * in its normal filter hash, then this must be the one
		 * we want!
		 */
		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, op->func_hash))
			return op;
2291 2292 2293 2294 2295 2296 2297 2298 2299 2300

	} while_for_each_ftrace_op(op);

	return NULL;
}

static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2301
	unsigned long ip = rec->ip;
2302 2303 2304

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* pass rec in as regs to have non-NULL val */
2305
		if (hash_contains_ip(ip, op->func_hash))
2306 2307 2308 2309 2310 2311
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
/**
 * ftrace_get_addr_new - Get the call address to set to
 * @rec:  The ftrace record descriptor
 *
 * If the record has the FTRACE_FL_REGS set, that means that it
 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
 * is not not set, then it wants to convert to the normal callback.
 *
 * Returns the address of the trampoline to set to
 */
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
2324 2325 2326 2327 2328 2329
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP) {
		ops = ftrace_find_tramp_ops_new(rec);
		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2330 2331
			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
				(void *)rec->ip, (void *)rec->ip, rec->flags);
2332 2333 2334 2335 2336 2337
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355
	if (rec->flags & FTRACE_FL_REGS)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

/**
 * ftrace_get_addr_curr - Get the call address that is already there
 * @rec:  The ftrace record descriptor
 *
 * The FTRACE_FL_REGS_EN is set when the record already points to
 * a function that saves all the regs. Basically the '_EN' version
 * represents the current state of the function.
 *
 * Returns the address of the trampoline that is currently being called
 */
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
2356 2357 2358 2359 2360 2361
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP_EN) {
		ops = ftrace_find_tramp_ops_curr(rec);
		if (FTRACE_WARN_ON(!ops)) {
2362 2363
			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
				(void *)rec->ip, (void *)rec->ip);
2364 2365 2366 2367 2368 2369
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2370 2371 2372 2373 2374 2375
	if (rec->flags & FTRACE_FL_REGS_EN)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

2376 2377 2378
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
2379
	unsigned long ftrace_old_addr;
2380 2381 2382
	unsigned long ftrace_addr;
	int ret;

2383
	ftrace_addr = ftrace_get_addr_new(rec);
2384

2385 2386 2387 2388
	/* This needs to be done before we call ftrace_update_record */
	ftrace_old_addr = ftrace_get_addr_curr(rec);

	ret = ftrace_update_record(rec, enable);
2389

2390 2391
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2392 2393 2394 2395 2396
	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
2397
		ftrace_bug_type = FTRACE_BUG_CALL;
2398
		return ftrace_make_call(rec, ftrace_addr);
2399 2400

	case FTRACE_UPDATE_MAKE_NOP:
2401
		ftrace_bug_type = FTRACE_BUG_NOP;
2402
		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2403 2404

	case FTRACE_UPDATE_MODIFY_CALL:
2405
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2406
		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2407 2408
	}

2409
	return -1; /* unknow ftrace bug */
2410 2411
}

2412
void __weak ftrace_replace_code(int enable)
2413 2414 2415
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
2416
	int failed;
2417

2418 2419 2420
	if (unlikely(ftrace_disabled))
		return;

2421
	do_for_each_ftrace_rec(pg, rec) {
2422 2423 2424 2425

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

2426
		failed = __ftrace_replace_code(rec, enable);
2427
		if (failed) {
2428
			ftrace_bug(failed, rec);
2429 2430
			/* Stop processing */
			return;
2431
		}
2432
	} while_for_each_ftrace_rec();
2433 2434
}

2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506
struct ftrace_rec_iter {
	struct ftrace_page	*pg;
	int			index;
};

/**
 * ftrace_rec_iter_start, start up iterating over traced functions
 *
 * Returns an iterator handle that is used to iterate over all
 * the records that represent address locations where functions
 * are traced.
 *
 * May return NULL if no records are available.
 */
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
	/*
	 * We only use a single iterator.
	 * Protected by the ftrace_lock mutex.
	 */
	static struct ftrace_rec_iter ftrace_rec_iter;
	struct ftrace_rec_iter *iter = &ftrace_rec_iter;

	iter->pg = ftrace_pages_start;
	iter->index = 0;

	/* Could have empty pages */
	while (iter->pg && !iter->pg->index)
		iter->pg = iter->pg->next;

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_next, get the next record to process.
 * @iter: The handle to the iterator.
 *
 * Returns the next iterator after the given iterator @iter.
 */
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
	iter->index++;

	if (iter->index >= iter->pg->index) {
		iter->pg = iter->pg->next;
		iter->index = 0;

		/* Could have empty pages */
		while (iter->pg && !iter->pg->index)
			iter->pg = iter->pg->next;
	}

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_record, get the record at the iterator location
 * @iter: The current iterator location
 *
 * Returns the record that the current @iter is at.
 */
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
	return &iter->pg->records[iter->index];
}

2507
static int
2508
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2509
{
2510
	int ret;
2511

2512 2513 2514
	if (unlikely(ftrace_disabled))
		return 0;

2515
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2516
	if (ret) {
2517
		ftrace_bug_type = FTRACE_BUG_INIT;
2518
		ftrace_bug(ret, rec);
2519
		return 0;
2520
	}
2521
	return 1;
2522 2523
}

2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

2542
void ftrace_modify_all_code(int command)
2543
{
2544
	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2545
	int err = 0;
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556

	/*
	 * If the ftrace_caller calls a ftrace_ops func directly,
	 * we need to make sure that it only traces functions it
	 * expects to trace. When doing the switch of functions,
	 * we need to update to the ftrace_ops_list_func first
	 * before the transition between old and new calls are set,
	 * as the ftrace_ops_list_func will check the ops hashes
	 * to make sure the ops are having the right functions
	 * traced.
	 */
2557 2558 2559 2560 2561
	if (update) {
		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
		if (FTRACE_WARN_ON(err))
			return;
	}
2562

2563
	if (command & FTRACE_UPDATE_CALLS)
2564
		ftrace_replace_code(1);
2565
	else if (command & FTRACE_DISABLE_CALLS)
2566 2567
		ftrace_replace_code(0);

2568 2569 2570 2571 2572 2573
	if (update && ftrace_trace_function != ftrace_ops_list_func) {
		function_trace_op = set_function_trace_op;
		smp_wmb();
		/* If irqs are disabled, we are in stop machine */
		if (!irqs_disabled())
			smp_call_function(ftrace_sync_ipi, NULL, 1);
2574 2575 2576
		err = ftrace_update_ftrace_func(ftrace_trace_function);
		if (FTRACE_WARN_ON(err))
			return;
2577
	}
2578

2579
	if (command & FTRACE_START_FUNC_RET)
2580
		err = ftrace_enable_ftrace_graph_caller();
2581
	else if (command & FTRACE_STOP_FUNC_RET)
2582 2583
		err = ftrace_disable_ftrace_graph_caller();
	FTRACE_WARN_ON(err);
2584 2585 2586 2587 2588 2589 2590
}

static int __ftrace_modify_code(void *data)
{
	int *command = data;

	ftrace_modify_all_code(*command);
2591

2592
	return 0;
2593 2594
}

2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618
/**
 * ftrace_run_stop_machine, go back to the stop machine method
 * @command: The command to tell ftrace what to do
 *
 * If an arch needs to fall back to the stop machine method, the
 * it can call this function.
 */
void ftrace_run_stop_machine(int command)
{
	stop_machine(__ftrace_modify_code, &command, NULL);
}

/**
 * arch_ftrace_update_code, modify the code to trace or not trace
 * @command: The command that needs to be done
 *
 * Archs can override this function if it does not need to
 * run stop_machine() to modify code.
 */
void __weak arch_ftrace_update_code(int command)
{
	ftrace_run_stop_machine(command);
}

I
Ingo Molnar 已提交
2619
static void ftrace_run_update_code(int command)
2620
{
2621 2622 2623 2624 2625 2626 2627
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;

2628 2629 2630 2631 2632 2633 2634 2635
	/*
	 * By default we use stop_machine() to modify the code.
	 * But archs can do what ever they want as long as it
	 * is safe. The stop_machine() is the safest, but also
	 * produces the most overhead.
	 */
	arch_ftrace_update_code(command);

2636 2637
	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
2638 2639
}

2640
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2641
				   struct ftrace_ops_hash *old_hash)
2642 2643
{
	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2644 2645
	ops->old_hash.filter_hash = old_hash->filter_hash;
	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2646
	ftrace_run_update_code(command);
2647
	ops->old_hash.filter_hash = NULL;
2648
	ops->old_hash.notrace_hash = NULL;
2649 2650 2651
	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}

2652
static ftrace_func_t saved_ftrace_func;
2653
static int ftrace_start_up;
2654

2655 2656 2657 2658
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}

2659
static void per_cpu_ops_free(struct ftrace_ops *ops)
2660 2661 2662 2663
{
	free_percpu(ops->disabled);
}

2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675
static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
2676

2677 2678 2679 2680 2681 2682 2683
static void ftrace_startup_all(int command)
{
	update_all_ops = true;
	ftrace_startup_enable(command);
	update_all_ops = false;
}

2684
static int ftrace_startup(struct ftrace_ops *ops, int command)
2685
{
2686
	int ret;
2687

2688
	if (unlikely(ftrace_disabled))
2689
		return -ENODEV;
2690

2691 2692 2693 2694
	ret = __register_ftrace_function(ops);
	if (ret)
		return ret;

2695
	ftrace_start_up++;
2696

2697 2698 2699 2700 2701 2702 2703 2704 2705
	/*
	 * Note that ftrace probes uses this to start up
	 * and modify functions it will probe. But we still
	 * set the ADDING flag for modification, as probes
	 * do not have trampolines. If they add them in the
	 * future, then the probes will need to distinguish
	 * between adding and updating probes.
	 */
	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2706

2707 2708 2709 2710 2711 2712 2713 2714 2715
	ret = ftrace_hash_ipmodify_enable(ops);
	if (ret < 0) {
		/* Rollback registration process */
		__unregister_ftrace_function(ops);
		ftrace_start_up--;
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
		return ret;
	}

2716 2717
	if (ftrace_hash_rec_enable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2718

2719
	ftrace_startup_enable(command);
2720

2721 2722
	ops->flags &= ~FTRACE_OPS_FL_ADDING;

2723
	return 0;
2724 2725
}

2726
static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2727
{
2728
	int ret;
2729

2730
	if (unlikely(ftrace_disabled))
2731 2732 2733 2734 2735
		return -ENODEV;

	ret = __unregister_ftrace_function(ops);
	if (ret)
		return ret;
2736

2737
	ftrace_start_up--;
2738 2739 2740 2741 2742 2743 2744
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

2745 2746
	/* Disabling ipmodify never fails */
	ftrace_hash_ipmodify_disable(ops);
2747

2748 2749
	if (ftrace_hash_rec_disable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2750

2751
	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2752

2753 2754 2755 2756
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
2757

2758 2759
	if (!command || !ftrace_enabled) {
		/*
2760
		 * If these are per_cpu ops, they still need their
2761 2762 2763 2764
		 * per_cpu field freed. Since, function tracing is
		 * not currently active, we can just free them
		 * without synchronizing all CPUs.
		 */
2765 2766
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2767
		return 0;
2768
	}
2769

2770 2771 2772 2773
	/*
	 * If the ops uses a trampoline, then it needs to be
	 * tested first on update.
	 */
2774
	ops->flags |= FTRACE_OPS_FL_REMOVING;
2775 2776
	removed_ops = ops;

2777 2778 2779 2780
	/* The trampoline logic checks the old hashes */
	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;

2781
	ftrace_run_update_code(command);
2782

2783 2784 2785 2786 2787 2788 2789 2790 2791
	/*
	 * If there's no more ops registered with ftrace, run a
	 * sanity check to make sure all rec flags are cleared.
	 */
	if (ftrace_ops_list == &ftrace_list_end) {
		struct ftrace_page *pg;
		struct dyn_ftrace *rec;

		do_for_each_ftrace_rec(pg, rec) {
2792
			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2793 2794 2795 2796 2797
				pr_warn("  %pS flags:%lx\n",
					(void *)rec->ip, rec->flags);
		} while_for_each_ftrace_rec();
	}

2798 2799 2800 2801
	ops->old_hash.filter_hash = NULL;
	ops->old_hash.notrace_hash = NULL;

	removed_ops = NULL;
2802
	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2803

2804 2805 2806
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
2807
	 * The same goes for freeing the per_cpu data of the per_cpu
2808 2809
	 * ops.
	 */
2810
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) {
2811 2812 2813 2814 2815 2816 2817 2818
		/*
		 * We need to do a hard force of sched synchronization.
		 * This is because we use preempt_disable() to do RCU, but
		 * the function tracers can be called where RCU is not watching
		 * (like before user_exit()). We can not rely on the RCU
		 * infrastructure to do the synchronization, thus we must do it
		 * ourselves.
		 */
2819 2820
		schedule_on_each_cpu(ftrace_sync);

2821 2822 2823 2824 2825 2826 2827 2828 2829 2830
		/*
		 * When the kernel is preeptive, tasks can be preempted
		 * while on a ftrace trampoline. Just scheduling a task on
		 * a CPU is not good enough to flush them. Calling
		 * synchornize_rcu_tasks() will wait for those tasks to
		 * execute and either schedule voluntarily or enter user space.
		 */
		if (IS_ENABLED(CONFIG_PREEMPT))
			synchronize_rcu_tasks();

2831 2832
		arch_ftrace_trampoline_free(ops);

2833 2834
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2835 2836
	}

2837
	return 0;
2838 2839
}

I
Ingo Molnar 已提交
2840
static void ftrace_startup_sysctl(void)
2841
{
2842 2843
	int command;

2844 2845 2846
	if (unlikely(ftrace_disabled))
		return;

2847 2848
	/* Force update next time */
	saved_ftrace_func = NULL;
2849
	/* ftrace_start_up is true if we want ftrace running */
2850 2851 2852 2853
	if (ftrace_start_up) {
		command = FTRACE_UPDATE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_START_FUNC_RET;
2854
		ftrace_startup_enable(command);
2855
	}
2856 2857
}

I
Ingo Molnar 已提交
2858
static void ftrace_shutdown_sysctl(void)
2859
{
2860 2861
	int command;

2862 2863 2864
	if (unlikely(ftrace_disabled))
		return;

2865
	/* ftrace_start_up is true if ftrace is running */
2866 2867 2868 2869 2870 2871
	if (ftrace_start_up) {
		command = FTRACE_DISABLE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_STOP_FUNC_RET;
		ftrace_run_update_code(command);
	}
2872 2873
}

2874
static u64		ftrace_update_time;
2875 2876
unsigned long		ftrace_update_tot_cnt;

2877
static inline int ops_traces_mod(struct ftrace_ops *ops)
2878
{
2879 2880 2881 2882
	/*
	 * Filter_hash being empty will default to trace module.
	 * But notrace hash requires a test of individual module functions.
	 */
2883 2884
	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
		ftrace_hash_empty(ops->func_hash->notrace_hash);
2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900
}

/*
 * Check if the current ops references the record.
 *
 * If the ops traces all functions, then it was already accounted for.
 * If the ops does not trace the current record function, skip it.
 * If the ops ignores the function via notrace filter, skip it.
 */
static inline bool
ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	/* If ops isn't enabled, ignore it */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

2901
	/* If ops traces all then it includes this function */
2902
	if (ops_traces_mod(ops))
2903
		return 1;
2904 2905

	/* The function must be in the filter */
2906
	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2907
	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2908
		return 0;
2909

2910
	/* If in notrace hash, we ignore it too */
2911
	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2912 2913 2914 2915 2916
		return 0;

	return 1;
}

2917
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2918
{
2919
	struct ftrace_page *pg;
2920
	struct dyn_ftrace *p;
2921
	u64 start, stop;
2922
	unsigned long update_cnt = 0;
2923
	unsigned long rec_flags = 0;
2924
	int i;
2925

2926 2927
	start = ftrace_now(raw_smp_processor_id());

2928
	/*
2929 2930 2931 2932 2933 2934 2935 2936 2937
	 * When a module is loaded, this function is called to convert
	 * the calls to mcount in its text to nops, and also to create
	 * an entry in the ftrace data. Now, if ftrace is activated
	 * after this call, but before the module sets its text to
	 * read-only, the modification of enabling ftrace can fail if
	 * the read-only is done while ftrace is converting the calls.
	 * To prevent this, the module's records are set as disabled
	 * and will be enabled after the call to set the module's text
	 * to read-only.
2938
	 */
2939 2940
	if (mod)
		rec_flags |= FTRACE_FL_DISABLED;
2941

2942
	for (pg = new_pgs; pg; pg = pg->next) {
2943

2944
		for (i = 0; i < pg->index; i++) {
2945

2946 2947 2948
			/* If something went wrong, bail without enabling anything */
			if (unlikely(ftrace_disabled))
				return -1;
2949

2950
			p = &pg->records[i];
2951
			p->flags = rec_flags;
2952

2953 2954 2955 2956 2957 2958
			/*
			 * Do the initial record conversion from mcount jump
			 * to the NOP instructions.
			 */
			if (!ftrace_code_disable(mod, p))
				break;
2959

2960
			update_cnt++;
2961
		}
2962 2963
	}

I
Ingo Molnar 已提交
2964
	stop = ftrace_now(raw_smp_processor_id());
2965
	ftrace_update_time = stop - start;
2966
	ftrace_update_tot_cnt += update_cnt;
2967

2968 2969 2970
	return 0;
}

2971
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2972
{
2973
	int order;
2974 2975
	int cnt;

2976 2977 2978 2979
	if (WARN_ON(!count))
		return -EINVAL;

	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2980 2981

	/*
2982 2983
	 * We want to fill as much as possible. No more than a page
	 * may be empty.
2984
	 */
2985 2986
	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
		order--;
2987

2988 2989
 again:
	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2990

2991 2992 2993 2994 2995 2996 2997
	if (!pg->records) {
		/* if we can't allocate this size, try something smaller */
		if (!order)
			return -ENOMEM;
		order >>= 1;
		goto again;
	}
2998

2999 3000
	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
	pg->size = cnt;
3001

3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034
	if (cnt > count)
		cnt = count;

	return cnt;
}

static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
	struct ftrace_page *start_pg;
	struct ftrace_page *pg;
	int order;
	int cnt;

	if (!num_to_init)
		return 0;

	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
	if (!pg)
		return NULL;

	/*
	 * Try to allocate as much as possible in one continues
	 * location that fills in all of the space. We want to
	 * waste as little space as possible.
	 */
	for (;;) {
		cnt = ftrace_allocate_records(pg, num_to_init);
		if (cnt < 0)
			goto free_pages;

		num_to_init -= cnt;
		if (!num_to_init)
3035 3036
			break;

3037 3038 3039 3040
		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
		if (!pg->next)
			goto free_pages;

3041 3042 3043
		pg = pg->next;
	}

3044 3045 3046
	return start_pg;

 free_pages:
3047 3048
	pg = start_pg;
	while (pg) {
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058
		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		start_pg = pg->next;
		kfree(pg);
		pg = start_pg;
	}
	pr_info("ftrace: FAILED to allocate memory for functions\n");
	return NULL;
}

3059 3060 3061
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
3062
	loff_t				pos;
3063 3064 3065 3066
	loff_t				func_pos;
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
3067
	struct ftrace_func_entry	*probe_entry;
3068
	struct trace_parser		parser;
3069
	struct ftrace_hash		*hash;
3070
	struct ftrace_ops		*ops;
3071
	int				pidx;
3072 3073
	int				idx;
	unsigned			flags;
3074 3075
};

3076
static void *
3077
t_probe_next(struct seq_file *m, loff_t *pos)
3078 3079
{
	struct ftrace_iterator *iter = m->private;
3080
	struct trace_array *tr = iter->ops->private;
3081
	struct list_head *func_probes;
3082 3083
	struct ftrace_hash *hash;
	struct list_head *next;
3084
	struct hlist_node *hnd = NULL;
3085
	struct hlist_head *hhd;
3086
	int size;
3087 3088

	(*pos)++;
3089
	iter->pos = *pos;
3090

3091
	if (!tr)
3092 3093
		return NULL;

3094 3095
	func_probes = &tr->func_probes;
	if (list_empty(func_probes))
3096 3097
		return NULL;

3098
	if (!iter->probe) {
3099
		next = func_probes->next;
3100
		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3101 3102 3103 3104 3105 3106 3107 3108 3109 3110
	}

	if (iter->probe_entry)
		hnd = &iter->probe_entry->hlist;

	hash = iter->probe->ops.func_hash->filter_hash;
	size = 1 << hash->size_bits;

 retry:
	if (iter->pidx >= size) {
3111
		if (iter->probe->list.next == func_probes)
3112 3113
			return NULL;
		next = iter->probe->list.next;
3114
		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3115 3116 3117 3118 3119 3120
		hash = iter->probe->ops.func_hash->filter_hash;
		size = 1 << hash->size_bits;
		iter->pidx = 0;
	}

	hhd = &hash->buckets[iter->pidx];
3121 3122

	if (hlist_empty(hhd)) {
3123
		iter->pidx++;
3124 3125 3126 3127 3128 3129 3130 3131 3132
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
3133
			iter->pidx++;
3134 3135 3136 3137
			goto retry;
		}
	}

3138 3139 3140
	if (WARN_ON_ONCE(!hnd))
		return NULL;

3141
	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
3142 3143

	return iter;
3144 3145
}

3146
static void *t_probe_start(struct seq_file *m, loff_t *pos)
3147 3148 3149
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
3150 3151
	loff_t l;

3152
	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
3153 3154
		return NULL;

3155 3156
	if (iter->func_pos > *pos)
		return NULL;
3157

3158 3159 3160
	iter->probe = NULL;
	iter->probe_entry = NULL;
	iter->pidx = 0;
3161
	for (l = 0; l <= (*pos - iter->func_pos); ) {
3162
		p = t_probe_next(m, &l);
L
Li Zefan 已提交
3163 3164 3165
		if (!p)
			break;
	}
3166 3167 3168
	if (!p)
		return NULL;

3169
	/* Only set this if we have an item */
3170
	iter->flags |= FTRACE_ITER_PROBE;
3171

3172
	return iter;
3173 3174
}

3175
static int
3176
t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
3177
{
3178
	struct ftrace_func_entry *probe_entry;
3179 3180
	struct ftrace_probe_ops *probe_ops;
	struct ftrace_func_probe *probe;
3181

3182 3183
	probe = iter->probe;
	probe_entry = iter->probe_entry;
3184

3185
	if (WARN_ON_ONCE(!probe || !probe_entry))
3186
		return -EIO;
3187

3188
	probe_ops = probe->probe_ops;
3189

3190
	if (probe_ops->print)
3191
		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
3192

3193 3194
	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
		   (void *)probe_ops->func);
3195 3196 3197 3198

	return 0;
}

I
Ingo Molnar 已提交
3199
static void *
3200
t_func_next(struct seq_file *m, loff_t *pos)
3201 3202 3203 3204 3205
{
	struct ftrace_iterator *iter = m->private;
	struct dyn_ftrace *rec = NULL;

	(*pos)++;
3206

3207 3208 3209 3210 3211 3212 3213 3214 3215
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
3216 3217
		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3218 3219

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3220
		     !(rec->flags & FTRACE_FL_ENABLED))) {
3221

3222 3223 3224 3225 3226
			rec = NULL;
			goto retry;
		}
	}

3227
	if (!rec)
3228
		return NULL;
3229

3230
	iter->pos = iter->func_pos = *pos;
3231 3232 3233
	iter->func = rec;

	return iter;
3234 3235
}

3236 3237 3238 3239
static void *
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3240
	loff_t l = *pos; /* t_hash_start() must use original pos */
3241 3242 3243 3244 3245
	void *ret;

	if (unlikely(ftrace_disabled))
		return NULL;

3246 3247
	if (iter->flags & FTRACE_ITER_PROBE)
		return t_probe_next(m, pos);
3248 3249

	if (iter->flags & FTRACE_ITER_PRINTALL) {
3250
		/* next must increment pos, and t_probe_start does not */
3251
		(*pos)++;
3252
		return t_probe_start(m, &l);
3253 3254 3255 3256 3257
	}

	ret = t_func_next(m, pos);

	if (!ret)
3258
		return t_probe_start(m, &l);
3259 3260 3261 3262

	return ret;
}

3263 3264 3265 3266
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
3267
	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE);
3268 3269 3270 3271 3272 3273
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
3274
	loff_t l;
3275

3276
	mutex_lock(&ftrace_lock);
3277 3278 3279 3280

	if (unlikely(ftrace_disabled))
		return NULL;

3281 3282 3283 3284 3285 3286
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

3287 3288 3289 3290 3291
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
3292 3293
	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
	    ftrace_hash_empty(iter->hash)) {
3294
		iter->func_pos = 1; /* Account for the message */
3295
		if (*pos > 0)
3296
			return t_probe_start(m, pos);
3297
		iter->flags |= FTRACE_ITER_PRINTALL;
3298
		/* reset in case of seek/pread */
3299
		iter->flags &= ~FTRACE_ITER_PROBE;
3300 3301 3302
		return iter;
	}

3303 3304
	if (iter->flags & FTRACE_ITER_PROBE)
		return t_probe_start(m, pos);
3305

3306 3307 3308 3309 3310
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
3311 3312 3313
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
3314
		p = t_func_next(m, &l);
3315 3316
		if (!p)
			break;
3317
	}
3318

3319
	if (!p)
3320
		return t_probe_start(m, pos);
3321 3322

	return iter;
3323 3324 3325 3326
}

static void t_stop(struct seq_file *m, void *p)
{
3327
	mutex_unlock(&ftrace_lock);
3328 3329
}

3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	return NULL;
}

static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
				struct dyn_ftrace *rec)
{
	void *ptr;

	ptr = arch_ftrace_trampoline_func(ops, rec);
	if (ptr)
		seq_printf(m, " ->%pS", ptr);
}

3346 3347
static int t_show(struct seq_file *m, void *v)
{
3348
	struct ftrace_iterator *iter = m->private;
3349
	struct dyn_ftrace *rec;
3350

3351 3352
	if (iter->flags & FTRACE_ITER_PROBE)
		return t_probe_show(m, iter);
3353

3354
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3355
		if (iter->flags & FTRACE_ITER_NOTRACE)
3356
			seq_puts(m, "#### no functions disabled ####\n");
3357
		else
3358
			seq_puts(m, "#### all functions enabled ####\n");
3359 3360 3361
		return 0;
	}

3362 3363
	rec = iter->func;

3364 3365 3366
	if (!rec)
		return 0;

3367
	seq_printf(m, "%ps", (void *)rec->ip);
3368
	if (iter->flags & FTRACE_ITER_ENABLED) {
3369
		struct ftrace_ops *ops;
3370

3371
		seq_printf(m, " (%ld)%s%s",
3372
			   ftrace_rec_count(rec),
3373 3374
			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
3375
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3376
			ops = ftrace_find_tramp_ops_any(rec);
3377 3378 3379 3380 3381
			if (ops) {
				do {
					seq_printf(m, "\ttramp: %pS (%pS)",
						   (void *)ops->trampoline,
						   (void *)ops->func);
3382
					add_trampoline_func(m, ops, rec);
3383 3384 3385
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
3386
				seq_puts(m, "\ttramp: ERROR!");
3387 3388
		} else {
			add_trampoline_func(m, NULL, rec);
3389 3390 3391
		}
	}	

3392
	seq_putc(m, '\n');
3393 3394 3395 3396

	return 0;
}

J
James Morris 已提交
3397
static const struct seq_operations show_ftrace_seq_ops = {
3398 3399 3400 3401 3402 3403
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
3404
static int
3405 3406 3407 3408
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3409 3410 3411
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3412
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3413 3414
	if (!iter)
		return -ENOMEM;
3415

3416 3417 3418 3419
	iter->pg = ftrace_pages_start;
	iter->ops = &global_ops;

	return 0;
3420 3421
}

3422 3423 3424 3425 3426
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3427
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3428 3429
	if (!iter)
		return -ENOMEM;
3430

3431 3432 3433 3434 3435
	iter->pg = ftrace_pages_start;
	iter->flags = FTRACE_ITER_ENABLED;
	iter->ops = &global_ops;

	return 0;
3436 3437
}

3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450
/**
 * ftrace_regex_open - initialize function tracer filter files
 * @ops: The ftrace_ops that hold the hash filters
 * @flag: The type of filter to process
 * @inode: The inode, usually passed in to your open routine
 * @file: The file, usually passed in to your open routine
 *
 * ftrace_regex_open() initializes the filter files for the
 * @ops. Depending on @flag it may process the filter hash or
 * the notrace hash of @ops. With this called from the open
 * routine, you can use ftrace_filter_write() for the write
 * routine if @flag has FTRACE_ITER_FILTER set, or
 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3451
 * tracing_lseek() should be used as the lseek routine, and
3452 3453 3454
 * release must call ftrace_regex_release().
 */
int
3455
ftrace_regex_open(struct ftrace_ops *ops, int flag,
3456
		  struct inode *inode, struct file *file)
3457 3458
{
	struct ftrace_iterator *iter;
3459
	struct ftrace_hash *hash;
3460 3461
	int ret = 0;

3462 3463
	ftrace_ops_init(ops);

3464 3465 3466
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3467 3468 3469 3470
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

3471 3472 3473 3474 3475
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

3476 3477 3478
	iter->ops = ops;
	iter->flags = flag;

3479
	mutex_lock(&ops->func_hash->regex_lock);
3480

3481
	if (flag & FTRACE_ITER_NOTRACE)
3482
		hash = ops->func_hash->notrace_hash;
3483
	else
3484
		hash = ops->func_hash->filter_hash;
3485

3486
	if (file->f_mode & FMODE_WRITE) {
3487 3488 3489 3490 3491 3492 3493
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

		if (file->f_flags & O_TRUNC)
			iter->hash = alloc_ftrace_hash(size_bits);
		else
			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);

3494 3495 3496
		if (!iter->hash) {
			trace_parser_put(&iter->parser);
			kfree(iter);
3497 3498
			ret = -ENOMEM;
			goto out_unlock;
3499
		}
3500 3501
	} else
		iter->hash = hash;
3502

3503 3504 3505 3506 3507 3508 3509
	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
3510
		} else {
3511 3512
			/* Failed */
			free_ftrace_hash(iter->hash);
3513
			trace_parser_put(&iter->parser);
3514
			kfree(iter);
3515
		}
3516 3517
	} else
		file->private_data = iter;
3518 3519

 out_unlock:
3520
	mutex_unlock(&ops->func_hash->regex_lock);
3521 3522 3523 3524

	return ret;
}

3525 3526 3527
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
3528 3529 3530
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops,
3531
			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
3532
			inode, file);
3533 3534 3535 3536 3537
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
3538 3539 3540
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3541
				 inode, file);
3542 3543
}

3544 3545 3546 3547 3548 3549 3550
/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
struct ftrace_glob {
	char *search;
	unsigned len;
	int type;
};

3551 3552 3553 3554 3555 3556 3557 3558 3559 3560
/*
 * If symbols in an architecture don't correspond exactly to the user-visible
 * name of what they represent, it is possible to define this function to
 * perform the necessary adjustments.
*/
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
	return str;
}

3561
static int ftrace_match(char *str, struct ftrace_glob *g)
3562 3563
{
	int matched = 0;
3564
	int slen;
3565

3566 3567
	str = arch_ftrace_match_adjust(str, g->search);

3568
	switch (g->type) {
3569
	case MATCH_FULL:
3570
		if (strcmp(str, g->search) == 0)
3571 3572 3573
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
3574
		if (strncmp(str, g->search, g->len) == 0)
3575 3576 3577
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
3578
		if (strstr(str, g->search))
3579 3580 3581
			matched = 1;
		break;
	case MATCH_END_ONLY:
3582
		slen = strlen(str);
3583 3584
		if (slen >= g->len &&
		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3585 3586
			matched = 1;
		break;
3587 3588 3589 3590
	case MATCH_GLOB:
		if (glob_match(g->search, str))
			matched = 1;
		break;
3591 3592 3593 3594 3595
	}

	return matched;
}

3596
static int
3597
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3598
{
3599 3600 3601
	struct ftrace_func_entry *entry;
	int ret = 0;

3602
	entry = ftrace_lookup_ip(hash, rec->ip);
3603
	if (clear_filter) {
3604 3605 3606
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
3607

3608
		free_hash_entry(hash, entry);
3609 3610 3611 3612
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
3613

3614
		ret = add_hash_entry(hash, rec->ip);
3615 3616
	}
	return ret;
3617 3618
}

3619
static int
D
Dmitry Safonov 已提交
3620 3621
ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
		struct ftrace_glob *mod_g, int exclude_mod)
3622 3623
{
	char str[KSYM_SYMBOL_LEN];
3624 3625 3626 3627
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

D
Dmitry Safonov 已提交
3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647
	if (mod_g) {
		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;

		/* blank module name to match all modules */
		if (!mod_g->len) {
			/* blank module globbing: modname xor exclude_mod */
			if ((!exclude_mod) != (!modname))
				goto func_match;
			return 0;
		}

		/* not matching the module */
		if (!modname || !mod_matches) {
			if (exclude_mod)
				goto func_match;
			else
				return 0;
		}

		if (mod_matches && exclude_mod)
3648 3649
			return 0;

D
Dmitry Safonov 已提交
3650
func_match:
3651
		/* blank search means to match all funcs in the mod */
3652
		if (!func_g->len)
3653 3654
			return 1;
	}
3655

3656
	return ftrace_match(str, func_g);
3657 3658
}

3659
static int
3660
match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3661 3662 3663
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3664
	struct ftrace_glob func_g = { .type = MATCH_FULL };
D
Dmitry Safonov 已提交
3665 3666 3667
	struct ftrace_glob mod_g = { .type = MATCH_FULL };
	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
	int exclude_mod = 0;
3668
	int found = 0;
3669
	int ret;
3670
	int clear_filter;
3671

D
Dmitry Safonov 已提交
3672
	if (func) {
3673 3674 3675
		func_g.type = filter_parse_regex(func, len, &func_g.search,
						 &clear_filter);
		func_g.len = strlen(func_g.search);
3676
	}
3677

D
Dmitry Safonov 已提交
3678 3679 3680 3681
	if (mod) {
		mod_g.type = filter_parse_regex(mod, strlen(mod),
				&mod_g.search, &exclude_mod);
		mod_g.len = strlen(mod_g.search);
3682
	}
3683

3684
	mutex_lock(&ftrace_lock);
3685

3686 3687
	if (unlikely(ftrace_disabled))
		goto out_unlock;
3688

3689
	do_for_each_ftrace_rec(pg, rec) {
3690 3691 3692 3693

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
3694
		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3695
			ret = enter_record(hash, rec, clear_filter);
3696 3697 3698 3699
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
3700
			found = 1;
3701 3702
		}
	} while_for_each_ftrace_rec();
3703
 out_unlock:
3704
	mutex_unlock(&ftrace_lock);
3705 3706

	return found;
3707 3708
}

3709
static int
3710
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3711
{
3712
	return match_records(hash, buff, len, NULL);
3713 3714
}

3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764
static void ftrace_ops_update_code(struct ftrace_ops *ops,
				   struct ftrace_ops_hash *old_hash)
{
	struct ftrace_ops *op;

	if (!ftrace_enabled)
		return;

	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
		return;
	}

	/*
	 * If this is the shared global_ops filter, then we need to
	 * check if there is another ops that shares it, is enabled.
	 * If so, we still need to run the modify code.
	 */
	if (ops->func_hash != &global_ops.local_hash)
		return;

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->func_hash == &global_ops.local_hash &&
		    op->flags & FTRACE_OPS_FL_ENABLED) {
			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
			/* Only need to do this once */
			return;
		}
	} while_for_each_ftrace_op(op);
}

static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
					   struct ftrace_hash **orig_hash,
					   struct ftrace_hash *hash,
					   int enable)
{
	struct ftrace_ops_hash old_hash_ops;
	struct ftrace_hash *old_hash;
	int ret;

	old_hash = *orig_hash;
	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
	if (!ret) {
		ftrace_ops_update_code(ops, &old_hash_ops);
		free_ftrace_hash_rcu(old_hash);
	}
	return ret;
}
3765

3766 3767 3768 3769 3770 3771
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
3772
ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
3773
		    char *func, char *cmd, char *module, int enable)
3774
{
3775
	int ret;
3776 3777 3778 3779 3780 3781 3782 3783

	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */
3784
	ret = match_records(hash, func, strlen(func), module);
3785
	if (!ret)
3786
		return -EINVAL;
3787 3788 3789
	if (ret < 0)
		return ret;
	return 0;
3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
3801
core_initcall(ftrace_mod_cmd_init);
3802

3803
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
3804
				      struct ftrace_ops *op, struct pt_regs *pt_regs)
3805
{
3806
	struct ftrace_probe_ops *probe_ops;
3807
	struct ftrace_func_probe *probe;
3808

3809 3810
	probe = container_of(op, struct ftrace_func_probe, ops);
	probe_ops = probe->probe_ops;
3811 3812 3813 3814 3815 3816

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
3817
	preempt_disable_notrace();
3818
	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
3819
	preempt_enable_notrace();
3820 3821
}

3822 3823 3824
struct ftrace_func_map {
	struct ftrace_func_entry	entry;
	void				*data;
3825 3826
};

3827 3828 3829
struct ftrace_func_mapper {
	struct ftrace_hash		hash;
};
3830

3831 3832 3833 3834 3835 3836
/**
 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
 *
 * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
 */
struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
3837
{
3838
	struct ftrace_hash *hash;
3839

3840 3841 3842 3843 3844 3845 3846 3847
	/*
	 * The mapper is simply a ftrace_hash, but since the entries
	 * in the hash are not ftrace_func_entry type, we define it
	 * as a separate structure.
	 */
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	return (struct ftrace_func_mapper *)hash;
}
3848

3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864
/**
 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: the instruction pointer to find the data for
 *
 * Returns the data mapped to @ip if found otherwise NULL. The return
 * is actually the address of the mapper data pointer. The address is
 * returned for use cases where the data is no bigger than a long, and
 * the user can use the data pointer as its data instead of having to
 * allocate more memory for the reference.
 */
void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
				  unsigned long ip)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
3865

3866 3867 3868
	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;
3869

3870 3871
	map = (struct ftrace_func_map *)entry;
	return &map->data;
3872 3873
}

3874 3875 3876 3877 3878 3879 3880 3881 3882 3883
/**
 * ftrace_func_mapper_add_ip - Map some data to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to map @data to
 * @data: The data to map to @ip
 *
 * Returns 0 on succes otherwise an error.
 */
int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
			      unsigned long ip, void *data)
3884
{
3885 3886
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
3887

3888 3889 3890
	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (entry)
		return -EBUSY;
3891

3892 3893 3894
	map = kmalloc(sizeof(*map), GFP_KERNEL);
	if (!map)
		return -ENOMEM;
3895

3896 3897
	map->entry.ip = ip;
	map->data = data;
3898

3899
	__add_hash_entry(&mapper->hash, &map->entry);
3900

3901 3902
	return 0;
}
3903

3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915
/**
 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to remove the data from
 *
 * Returns the data if it is found, otherwise NULL.
 * Note, if the data pointer is used as the data itself, (see 
 * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
 * if the data pointer was set to zero.
 */
void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
				   unsigned long ip)
3916
{
3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	void *data;

	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;

	map = (struct ftrace_func_map *)entry;
	data = map->data;

	remove_hash_entry(&mapper->hash, entry);
3929
	kfree(entry);
3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962

	return data;
}

/**
 * free_ftrace_func_mapper - free a mapping of ips and data
 * @mapper: The mapper that has the ip maps
 * @free_func: A function to be called on each data item.
 *
 * This is used to free the function mapper. The @free_func is optional
 * and can be used if the data needs to be freed as well.
 */
void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
			     ftrace_mapper_func free_func)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	struct hlist_head *hhd;
	int size = 1 << mapper->hash.size_bits;
	int i;

	if (free_func && mapper->hash.count) {
		for (i = 0; i < size; i++) {
			hhd = &mapper->hash.buckets[i];
			hlist_for_each_entry(entry, hhd, hlist) {
				map = (struct ftrace_func_map *)entry;
				free_func(map);
			}
		}
	}
	free_ftrace_hash(&mapper->hash);
}

3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975
static void release_probe(struct ftrace_func_probe *probe)
{
	struct ftrace_probe_ops *probe_ops;

	mutex_lock(&ftrace_lock);

	WARN_ON(probe->ref <= 0);

	/* Subtract the ref that was used to protect this instance */
	probe->ref--;

	if (!probe->ref) {
		probe_ops = probe->probe_ops;
3976 3977 3978 3979 3980 3981
		/*
		 * Sending zero as ip tells probe_ops to free
		 * the probe->data itself
		 */
		if (probe_ops->free)
			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994
		list_del(&probe->list);
		kfree(probe);
	}
	mutex_unlock(&ftrace_lock);
}

static void acquire_probe_locked(struct ftrace_func_probe *probe)
{
	/*
	 * Add one ref to keep it from being freed when releasing the
	 * ftrace_lock mutex.
	 */
	probe->ref++;
3995 3996 3997
}

int
3998
register_ftrace_function_probe(char *glob, struct trace_array *tr,
3999 4000
			       struct ftrace_probe_ops *probe_ops,
			       void *data)
4001
{
4002
	struct ftrace_func_entry *entry;
4003
	struct ftrace_func_probe *probe;
4004 4005
	struct ftrace_hash **orig_hash;
	struct ftrace_hash *old_hash;
4006
	struct ftrace_hash *hash;
4007
	int count = 0;
4008
	int size;
4009
	int ret;
4010
	int i;
4011

4012
	if (WARN_ON(!tr))
4013 4014
		return -EINVAL;

4015 4016
	/* We do not support '!' for function probes */
	if (WARN_ON(glob[0] == '!'))
4017 4018
		return -EINVAL;

4019

4020 4021 4022 4023 4024
	mutex_lock(&ftrace_lock);
	/* Check if the probe_ops is already registered */
	list_for_each_entry(probe, &tr->func_probes, list) {
		if (probe->probe_ops == probe_ops)
			break;
4025
	}
4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036
	if (&probe->list == &tr->func_probes) {
		probe = kzalloc(sizeof(*probe), GFP_KERNEL);
		if (!probe) {
			mutex_unlock(&ftrace_lock);
			return -ENOMEM;
		}
		probe->probe_ops = probe_ops;
		probe->ops.func = function_trace_probe_call;
		probe->tr = tr;
		ftrace_ops_init(&probe->ops);
		list_add(&probe->list, &tr->func_probes);
4037
	}
4038

4039
	acquire_probe_locked(probe);
4040

4041
	mutex_unlock(&ftrace_lock);
4042

4043
	mutex_lock(&probe->ops.func_hash->regex_lock);
4044

4045
	orig_hash = &probe->ops.func_hash->filter_hash;
4046 4047
	old_hash = *orig_hash;
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4048

4049
	ret = ftrace_match_records(hash, glob, strlen(glob));
4050

4051 4052 4053
	/* Nothing found? */
	if (!ret)
		ret = -EINVAL;
4054

4055 4056
	if (ret < 0)
		goto out;
4057

4058 4059 4060 4061
	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
			if (ftrace_lookup_ip(old_hash, entry->ip))
4062
				continue;
4063 4064 4065 4066 4067
			/*
			 * The caller might want to do something special
			 * for each function we find. We call the callback
			 * to give the caller an opportunity to do so.
			 */
4068 4069
			if (probe_ops->init) {
				ret = probe_ops->init(probe_ops, tr,
4070 4071 4072 4073 4074 4075 4076
						      entry->ip, data,
						      &probe->data);
				if (ret < 0) {
					if (probe_ops->free && count)
						probe_ops->free(probe_ops, tr,
								0, probe->data);
					probe->data = NULL;
4077
					goto out;
4078
				}
4079
			}
4080
			count++;
4081
		}
4082
	}
4083

4084
	mutex_lock(&ftrace_lock);
4085

4086 4087 4088 4089 4090
	if (!count) {
		/* Nothing was added? */
		ret = -EINVAL;
		goto out_unlock;
	}
4091

4092 4093
	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
					      hash, 1);
4094
	if (ret < 0)
4095
		goto err_unlock;
4096

4097 4098
	/* One ref for each new function traced */
	probe->ref += count;
4099

4100 4101
	if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
		ret = ftrace_startup(&probe->ops, 0);
4102

4103
 out_unlock:
4104
	mutex_unlock(&ftrace_lock);
4105

4106
	if (!ret)
4107
		ret = count;
4108
 out:
4109
	mutex_unlock(&probe->ops.func_hash->regex_lock);
4110
	free_ftrace_hash(hash);
4111

4112
	release_probe(probe);
4113

4114
	return ret;
4115

4116
 err_unlock:
4117
	if (!probe_ops->free || !count)
4118 4119 4120 4121 4122 4123 4124
		goto out_unlock;

	/* Failed to do the move, need to call the free functions */
	for (i = 0; i < size; i++) {
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
			if (ftrace_lookup_ip(old_hash, entry->ip))
				continue;
4125
			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4126 4127 4128
		}
	}
	goto out_unlock;
4129 4130
}

4131
int
4132 4133
unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
				      struct ftrace_probe_ops *probe_ops)
4134
{
4135
	struct ftrace_ops_hash old_hash_ops;
4136
	struct ftrace_func_entry *entry;
4137
	struct ftrace_func_probe *probe;
4138
	struct ftrace_glob func_g;
4139 4140 4141
	struct ftrace_hash **orig_hash;
	struct ftrace_hash *old_hash;
	struct ftrace_hash *hash = NULL;
4142
	struct hlist_node *tmp;
4143
	struct hlist_head hhd;
4144
	char str[KSYM_SYMBOL_LEN];
4145 4146
	int count = 0;
	int i, ret = -ENODEV;
4147
	int size;
4148

4149
	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
4150
		func_g.search = NULL;
4151
	else if (glob) {
4152 4153
		int not;

4154 4155 4156 4157
		func_g.type = filter_parse_regex(glob, strlen(glob),
						 &func_g.search, &not);
		func_g.len = strlen(func_g.search);
		func_g.search = glob;
4158

S
Steven Rostedt 已提交
4159
		/* we do not support '!' for function probes */
4160
		if (WARN_ON(not))
4161
			return -EINVAL;
4162 4163
	}

4164 4165 4166 4167 4168
	mutex_lock(&ftrace_lock);
	/* Check if the probe_ops is already registered */
	list_for_each_entry(probe, &tr->func_probes, list) {
		if (probe->probe_ops == probe_ops)
			break;
4169
	}
4170 4171 4172 4173 4174 4175 4176 4177 4178 4179
	if (&probe->list == &tr->func_probes)
		goto err_unlock_ftrace;

	ret = -EINVAL;
	if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
		goto err_unlock_ftrace;

	acquire_probe_locked(probe);

	mutex_unlock(&ftrace_lock);
4180

4181
	mutex_lock(&probe->ops.func_hash->regex_lock);
4182

4183
	orig_hash = &probe->ops.func_hash->filter_hash;
4184 4185 4186 4187
	old_hash = *orig_hash;

	if (ftrace_hash_empty(old_hash))
		goto out_unlock;
4188

4189 4190 4191 4192
	old_hash_ops.filter_hash = old_hash;
	/* Probes only have filters */
	old_hash_ops.notrace_hash = NULL;

4193
	ret = -ENOMEM;
4194
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4195 4196 4197
	if (!hash)
		goto out_unlock;

4198
	INIT_HLIST_HEAD(&hhd);
4199

4200 4201 4202
	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
4203

4204
			if (func_g.search) {
4205 4206
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
4207
				if (!ftrace_match(str, &func_g))
4208 4209
					continue;
			}
4210
			count++;
4211 4212
			remove_hash_entry(hash, entry);
			hlist_add_head(&entry->hlist, &hhd);
4213 4214
		}
	}
4215 4216

	/* Nothing found? */
4217
	if (!count) {
4218 4219 4220 4221
		ret = -EINVAL;
		goto out_unlock;
	}

4222
	mutex_lock(&ftrace_lock);
4223

4224
	WARN_ON(probe->ref < count);
4225

4226
	probe->ref -= count;
4227

4228 4229 4230 4231
	if (ftrace_hash_empty(hash))
		ftrace_shutdown(&probe->ops, 0);

	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4232
					      hash, 1);
4233 4234

	/* still need to update the function call sites */
4235
	if (ftrace_enabled && !ftrace_hash_empty(hash))
4236
		ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
4237
				       &old_hash_ops);
4238
	synchronize_sched();
4239

4240 4241
	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
		hlist_del(&entry->hlist);
4242
		if (probe_ops->free)
4243
			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4244
		kfree(entry);
4245
	}
4246
	mutex_unlock(&ftrace_lock);
4247

4248
 out_unlock:
4249
	mutex_unlock(&probe->ops.func_hash->regex_lock);
4250
	free_ftrace_hash(hash);
4251

4252
	release_probe(probe);
4253

4254
	return ret;
4255

4256 4257
 err_unlock_ftrace:
	mutex_unlock(&ftrace_lock);
4258
	return ret;
4259 4260
}

4261 4262 4263
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

4264 4265 4266 4267 4268
/*
 * Currently we only register ftrace commands from __init, so mark this
 * __init too.
 */
__init int register_ftrace_command(struct ftrace_func_command *cmd)
4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4287 4288 4289 4290 4291
/*
 * Currently we only unregister ftrace commands from __init, so mark
 * this __init too.
 */
__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4310
static int ftrace_process_regex(struct ftrace_iterator *iter,
4311
				char *buff, int len, int enable)
4312
{
4313
	struct ftrace_hash *hash = iter->hash;
4314
	struct trace_array *tr = iter->ops->private;
4315
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
4316
	struct ftrace_func_command *p;
4317
	int ret = -EINVAL;
4318 4319 4320 4321

	func = strsep(&next, ":");

	if (!next) {
4322
		ret = ftrace_match_records(hash, func, len);
4323 4324 4325 4326 4327
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
4328 4329
	}

4330
	/* command found */
4331 4332 4333

	command = strsep(&next, ":");

4334 4335 4336
	if (WARN_ON_ONCE(!tr))
		return -EINVAL;

4337 4338 4339
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
4340
			ret = p->func(tr, hash, func, command, next, enable);
4341 4342
			goto out_unlock;
		}
4343
	}
4344 4345
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
4346

4347
	return ret;
4348 4349
}

I
Ingo Molnar 已提交
4350
static ssize_t
4351 4352
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
4353 4354
{
	struct ftrace_iterator *iter;
4355 4356
	struct trace_parser *parser;
	ssize_t ret, read;
4357

4358
	if (!cnt)
4359 4360 4361 4362 4363 4364 4365 4366
		return 0;

	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

4367
	if (unlikely(ftrace_disabled))
4368 4369 4370
		return -ENODEV;

	/* iter->hash is a local copy, so we don't need regex_lock */
4371

4372 4373
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
4374

4375
	if (read >= 0 && trace_parser_loaded(parser) &&
4376
	    !trace_parser_cont(parser)) {
4377
		ret = ftrace_process_regex(iter, parser->buffer,
4378
					   parser->idx, enable);
4379
		trace_parser_clear(parser);
4380
		if (ret < 0)
4381
			goto out;
4382
	}
4383 4384

	ret = read;
4385
 out:
4386 4387 4388
	return ret;
}

4389
ssize_t
4390 4391 4392 4393 4394 4395
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

4396
ssize_t
4397 4398 4399 4400 4401 4402
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

4403
static int
4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424
ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
	struct ftrace_func_entry *entry;

	if (!ftrace_location(ip))
		return -EINVAL;

	if (remove) {
		entry = ftrace_lookup_ip(hash, ip);
		if (!entry)
			return -ENOENT;
		free_hash_entry(hash, entry);
		return 0;
	}

	return add_hash_entry(hash, ip);
}

static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
		unsigned long ip, int remove, int reset, int enable)
4425
{
4426
	struct ftrace_hash **orig_hash;
4427
	struct ftrace_hash *hash;
4428
	int ret;
4429

4430
	if (unlikely(ftrace_disabled))
4431
		return -ENODEV;
4432

4433
	mutex_lock(&ops->func_hash->regex_lock);
4434

4435
	if (enable)
4436
		orig_hash = &ops->func_hash->filter_hash;
4437
	else
4438
		orig_hash = &ops->func_hash->notrace_hash;
4439

4440 4441 4442 4443 4444
	if (reset)
		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	else
		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);

4445 4446 4447 4448
	if (!hash) {
		ret = -ENOMEM;
		goto out_regex_unlock;
	}
4449

4450 4451 4452 4453
	if (buf && !ftrace_match_records(hash, buf, len)) {
		ret = -EINVAL;
		goto out_regex_unlock;
	}
4454 4455 4456 4457 4458
	if (ip) {
		ret = ftrace_match_addr(hash, ip, remove);
		if (ret < 0)
			goto out_regex_unlock;
	}
4459 4460

	mutex_lock(&ftrace_lock);
4461
	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
4462 4463
	mutex_unlock(&ftrace_lock);

4464
 out_regex_unlock:
4465
	mutex_unlock(&ops->func_hash->regex_lock);
4466 4467 4468

	free_ftrace_hash(hash);
	return ret;
4469 4470
}

4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
		int reset, int enable)
{
	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
}

/**
 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
 * @ops - the ops to set the filter with
 * @ip - the address to add to or remove from the filter.
 * @remove - non zero to remove the ip from the filter
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled
 * If @ip is NULL, it failes to update filter.
 */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
			 int remove, int reset)
{
4491
	ftrace_ops_init(ops);
4492 4493 4494 4495
	return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);

4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512
/**
 * ftrace_ops_set_global_filter - setup ops to use global filters
 * @ops - the ops which will use the global filters
 *
 * ftrace users who need global function trace filtering should call this.
 * It can set the global filter only if ops were not initialized before.
 */
void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
{
	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
		return;

	ftrace_ops_init(ops);
	ops->func_hash = &global_ops.local_hash;
}
EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);

4513 4514 4515 4516 4517 4518 4519
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
{
	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
}

4520 4521
/**
 * ftrace_set_filter - set a function to filter on in ftrace
4522 4523 4524 4525 4526 4527 4528 4529
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4530
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4531 4532
		       int len, int reset)
{
4533
	ftrace_ops_init(ops);
4534
	return ftrace_set_regex(ops, buf, len, reset, 1);
4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4549
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4550 4551
			int len, int reset)
{
4552
	ftrace_ops_init(ops);
4553
	return ftrace_set_regex(ops, buf, len, reset, 0);
4554 4555 4556
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
4557
 * ftrace_set_global_filter - set a function to filter on with global tracers
4558 4559 4560 4561 4562 4563 4564
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4565
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4566
{
4567
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4568
}
4569
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4570

4571
/**
4572
 * ftrace_set_global_notrace - set a function to not trace with global tracers
4573 4574 4575 4576 4577 4578 4579 4580
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4581
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4582
{
4583
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4584
}
4585
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4586

4587 4588 4589 4590 4591 4592 4593
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

4594 4595 4596
/* Used by function selftest to not test if filter is set */
bool ftrace_filter_param __initdata;

4597 4598
static int __init set_ftrace_notrace(char *str)
{
4599
	ftrace_filter_param = true;
4600
	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4601 4602 4603 4604 4605 4606
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
4607
	ftrace_filter_param = true;
4608
	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4609 4610 4611 4612
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

4613
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4614
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4615
static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4616
static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
4617

4618 4619 4620
static unsigned long save_global_trampoline;
static unsigned long save_global_flags;

4621 4622
static int __init set_graph_function(char *str)
{
4623
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4624 4625 4626 4627
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

4628 4629 4630 4631 4632 4633 4634
static int __init set_graph_notrace_function(char *str)
{
	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);

4635 4636 4637 4638 4639 4640 4641 4642
static int __init set_graph_max_depth_function(char *str)
{
	if (!str)
		return 0;
	fgraph_max_depth = simple_strtoul(str, NULL, 0);
	return 1;
}
__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
4643 4644

static void __init set_ftrace_early_graph(char *buf, int enable)
4645 4646 4647
{
	int ret;
	char *func;
4648
	struct ftrace_hash *hash;
4649

4650 4651 4652
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	if (WARN_ON(!hash))
		return;
4653 4654 4655 4656

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
4657
		ret = ftrace_graph_set_hash(hash, func);
4658 4659 4660 4661
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
4662 4663 4664 4665 4666

	if (enable)
		ftrace_graph_hash = hash;
	else
		ftrace_graph_notrace_hash = hash;
4667 4668 4669
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4670 4671
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4672 4673 4674
{
	char *func;

4675 4676
	ftrace_ops_init(ops);

4677 4678
	while (buf) {
		func = strsep(&buf, ",");
4679
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
4680 4681 4682 4683 4684 4685
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
4686
		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
4687
	if (ftrace_notrace_buf[0])
4688
		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
4689 4690
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
4691 4692 4693
		set_ftrace_early_graph(ftrace_graph_buf, 1);
	if (ftrace_graph_notrace_buf[0])
		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
4694
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4695 4696
}

4697
int ftrace_regex_release(struct inode *inode, struct file *file)
4698 4699 4700
{
	struct seq_file *m = (struct seq_file *)file->private_data;
	struct ftrace_iterator *iter;
4701
	struct ftrace_hash **orig_hash;
4702
	struct trace_parser *parser;
4703
	int filter_hash;
4704
	int ret;
4705 4706 4707 4708 4709 4710 4711

	if (file->f_mode & FMODE_READ) {
		iter = m->private;
		seq_release(inode, file);
	} else
		iter = file->private_data;

4712 4713 4714
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
4715
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
4716 4717
	}

4718 4719
	trace_parser_put(parser);

4720
	mutex_lock(&iter->ops->func_hash->regex_lock);
4721

4722
	if (file->f_mode & FMODE_WRITE) {
4723 4724 4725
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

		if (filter_hash)
4726
			orig_hash = &iter->ops->func_hash->filter_hash;
4727
		else
4728
			orig_hash = &iter->ops->func_hash->notrace_hash;
4729

4730
		mutex_lock(&ftrace_lock);
4731 4732
		ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
						      iter->hash, filter_hash);
4733
		mutex_unlock(&ftrace_lock);
4734 4735 4736
	} else {
		/* For read only, the hash is the ops hash */
		iter->hash = NULL;
4737
	}
4738

4739
	mutex_unlock(&iter->ops->func_hash->regex_lock);
4740 4741
	free_ftrace_hash(iter->hash);
	kfree(iter);
4742

4743 4744 4745
	return 0;
}

4746
static const struct file_operations ftrace_avail_fops = {
4747 4748 4749
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
4750
	.release = seq_release_private,
4751 4752
};

4753 4754 4755 4756 4757 4758 4759
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

4760
static const struct file_operations ftrace_filter_fops = {
4761
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
4762
	.read = seq_read,
4763
	.write = ftrace_filter_write,
4764
	.llseek = tracing_lseek,
4765
	.release = ftrace_regex_release,
4766 4767
};

4768
static const struct file_operations ftrace_notrace_fops = {
4769
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
4770
	.read = seq_read,
4771
	.write = ftrace_notrace_write,
4772
	.llseek = tracing_lseek,
4773
	.release = ftrace_regex_release,
4774 4775
};

4776 4777 4778 4779
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

4780 4781 4782 4783 4784 4785 4786
struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH;
struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH;

enum graph_filter_type {
	GRAPH_FILTER_NOTRACE	= 0,
	GRAPH_FILTER_FUNCTION,
};
4787

4788 4789
#define FTRACE_GRAPH_EMPTY	((void *)1)

4790
struct ftrace_graph_data {
4791 4792 4793 4794 4795 4796 4797
	struct ftrace_hash		*hash;
	struct ftrace_func_entry	*entry;
	int				idx;   /* for hash table iteration */
	enum graph_filter_type		type;
	struct ftrace_hash		*new_hash;
	const struct seq_operations	*seq_ops;
	struct trace_parser		parser;
4798 4799
};

4800
static void *
4801
__g_next(struct seq_file *m, loff_t *pos)
4802
{
4803
	struct ftrace_graph_data *fgd = m->private;
4804 4805 4806
	struct ftrace_func_entry *entry = fgd->entry;
	struct hlist_head *head;
	int i, idx = fgd->idx;
4807

4808
	if (*pos >= fgd->hash->count)
4809
		return NULL;
4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828

	if (entry) {
		hlist_for_each_entry_continue(entry, hlist) {
			fgd->entry = entry;
			return entry;
		}

		idx++;
	}

	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
		head = &fgd->hash->buckets[i];
		hlist_for_each_entry(entry, head, hlist) {
			fgd->entry = entry;
			fgd->idx = i;
			return entry;
		}
	}
	return NULL;
4829
}
4830

4831 4832 4833 4834 4835
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
4836 4837 4838 4839
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
4840 4841
	struct ftrace_graph_data *fgd = m->private;

4842 4843
	mutex_lock(&graph_lock);

4844 4845 4846 4847 4848 4849 4850
	if (fgd->type == GRAPH_FILTER_FUNCTION)
		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
	else
		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));

4851
	/* Nothing, tell g_show to print all functions are enabled */
4852
	if (ftrace_hash_empty(fgd->hash) && !*pos)
4853
		return FTRACE_GRAPH_EMPTY;
4854

4855 4856
	fgd->idx = 0;
	fgd->entry = NULL;
4857
	return __g_next(m, pos);
4858 4859 4860 4861 4862 4863 4864 4865 4866
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
4867
	struct ftrace_func_entry *entry = v;
4868

4869
	if (!entry)
4870 4871
		return 0;

4872
	if (entry == FTRACE_GRAPH_EMPTY) {
4873 4874
		struct ftrace_graph_data *fgd = m->private;

4875
		if (fgd->type == GRAPH_FILTER_FUNCTION)
4876
			seq_puts(m, "#### all functions enabled ####\n");
4877
		else
4878
			seq_puts(m, "#### no functions disabled ####\n");
4879 4880 4881
		return 0;
	}

4882
	seq_printf(m, "%ps\n", (void *)entry->ip);
4883 4884 4885 4886

	return 0;
}

J
James Morris 已提交
4887
static const struct seq_operations ftrace_graph_seq_ops = {
4888 4889 4890 4891 4892 4893 4894
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
4895 4896
__ftrace_graph_open(struct inode *inode, struct file *file,
		    struct ftrace_graph_data *fgd)
4897 4898
{
	int ret = 0;
4899
	struct ftrace_hash *new_hash = NULL;
4900

4901 4902 4903
	if (file->f_mode & FMODE_WRITE) {
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

4904 4905 4906
		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
			return -ENOMEM;

4907 4908 4909 4910 4911 4912 4913 4914 4915
		if (file->f_flags & O_TRUNC)
			new_hash = alloc_ftrace_hash(size_bits);
		else
			new_hash = alloc_and_copy_ftrace_hash(size_bits,
							      fgd->hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}
4916 4917
	}

4918
	if (file->f_mode & FMODE_READ) {
4919
		ret = seq_open(file, &ftrace_graph_seq_ops);
4920 4921 4922
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = fgd;
4923 4924 4925 4926
		} else {
			/* Failed */
			free_ftrace_hash(new_hash);
			new_hash = NULL;
4927 4928 4929
		}
	} else
		file->private_data = fgd;
4930

4931
out:
4932 4933 4934
	if (ret < 0 && file->f_mode & FMODE_WRITE)
		trace_parser_put(&fgd->parser);

4935
	fgd->new_hash = new_hash;
4936 4937 4938 4939 4940 4941 4942 4943

	/*
	 * All uses of fgd->hash must be taken with the graph_lock
	 * held. The graph_lock is going to be released, so force
	 * fgd->hash to be reinitialized when it is taken again.
	 */
	fgd->hash = NULL;

4944 4945 4946
	return ret;
}

4947 4948 4949 4950
static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
4951
	int ret;
4952 4953 4954 4955 4956 4957 4958 4959

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

4960 4961
	mutex_lock(&graph_lock);

4962 4963
	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
4964
	fgd->type = GRAPH_FILTER_FUNCTION;
4965 4966
	fgd->seq_ops = &ftrace_graph_seq_ops;

4967 4968 4969 4970 4971 4972
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
4973 4974
}

4975 4976 4977 4978
static int
ftrace_graph_notrace_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
4979
	int ret;
4980 4981 4982 4983 4984 4985 4986 4987

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

4988 4989
	mutex_lock(&graph_lock);

4990 4991
	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
4992
	fgd->type = GRAPH_FILTER_NOTRACE;
4993 4994
	fgd->seq_ops = &ftrace_graph_seq_ops;

4995 4996 4997 4998 4999 5000
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
5001 5002
}

5003 5004 5005
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
5006
	struct ftrace_graph_data *fgd;
5007 5008 5009
	struct ftrace_hash *old_hash, *new_hash;
	struct trace_parser *parser;
	int ret = 0;
5010

5011 5012 5013
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;

5014
		fgd = m->private;
5015
		seq_release(inode, file);
5016
	} else {
5017
		fgd = file->private_data;
5018 5019
	}

5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059

	if (file->f_mode & FMODE_WRITE) {

		parser = &fgd->parser;

		if (trace_parser_loaded((parser))) {
			parser->buffer[parser->idx] = 0;
			ret = ftrace_graph_set_hash(fgd->new_hash,
						    parser->buffer);
		}

		trace_parser_put(parser);

		new_hash = __ftrace_hash_move(fgd->new_hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}

		mutex_lock(&graph_lock);

		if (fgd->type == GRAPH_FILTER_FUNCTION) {
			old_hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_hash, new_hash);
		} else {
			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
		}

		mutex_unlock(&graph_lock);

		/* Wait till all users are no longer using the old hash */
		synchronize_sched();

		free_ftrace_hash(old_hash);
	}

 out:
5060 5061 5062
	kfree(fgd->new_hash);
	kfree(fgd);

5063
	return ret;
5064 5065
}

5066
static int
5067
ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
5068
{
5069
	struct ftrace_glob func_g;
5070 5071
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
5072
	struct ftrace_func_entry *entry;
5073
	int fail = 1;
5074
	int not;
5075

5076
	/* decode regex */
5077 5078
	func_g.type = filter_parse_regex(buffer, strlen(buffer),
					 &func_g.search, &not);
5079

5080
	func_g.len = strlen(func_g.search);
5081

5082
	mutex_lock(&ftrace_lock);
5083 5084 5085 5086 5087 5088

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

5089 5090
	do_for_each_ftrace_rec(pg, rec) {

5091 5092 5093
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
5094
		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
5095
			entry = ftrace_lookup_ip(hash, rec->ip);
5096 5097 5098

			if (!not) {
				fail = 0;
5099 5100 5101 5102 5103

				if (entry)
					continue;
				if (add_hash_entry(hash, rec->ip) < 0)
					goto out;
5104
			} else {
5105 5106
				if (entry) {
					free_hash_entry(hash, entry);
5107 5108 5109
					fail = 0;
				}
			}
5110
		}
5111
	} while_for_each_ftrace_rec();
5112
out:
5113
	mutex_unlock(&ftrace_lock);
5114

5115 5116 5117 5118
	if (fail)
		return -EINVAL;

	return 0;
5119 5120 5121 5122 5123 5124
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5125
	ssize_t read, ret = 0;
5126
	struct ftrace_graph_data *fgd = file->private_data;
5127
	struct trace_parser *parser;
5128

5129
	if (!cnt)
5130 5131
		return 0;

5132 5133 5134 5135 5136 5137
	/* Read mode uses seq functions */
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		fgd = m->private;
	}

5138
	parser = &fgd->parser;
5139

5140
	read = trace_get_user(parser, ubuf, cnt, ppos);
5141

5142 5143
	if (read >= 0 && trace_parser_loaded(parser) &&
	    !trace_parser_cont(parser)) {
5144

5145
		ret = ftrace_graph_set_hash(fgd->new_hash,
5146 5147
					    parser->buffer);
		trace_parser_clear(parser);
5148 5149
	}

5150 5151
	if (!ret)
		ret = read;
5152

5153 5154 5155 5156
	return ret;
}

static const struct file_operations ftrace_graph_fops = {
5157 5158 5159
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5160
	.llseek		= tracing_lseek,
5161
	.release	= ftrace_graph_release,
5162
};
5163 5164 5165 5166 5167

static const struct file_operations ftrace_graph_notrace_fops = {
	.open		= ftrace_graph_notrace_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5168
	.llseek		= tracing_lseek,
5169 5170
	.release	= ftrace_graph_release,
};
5171 5172
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202
void ftrace_create_filter_files(struct ftrace_ops *ops,
				struct dentry *parent)
{

	trace_create_file("set_ftrace_filter", 0644, parent,
			  ops, &ftrace_filter_fops);

	trace_create_file("set_ftrace_notrace", 0644, parent,
			  ops, &ftrace_notrace_fops);
}

/*
 * The name "destroy_filter_files" is really a misnomer. Although
 * in the future, it may actualy delete the files, but this is
 * really intended to make sure the ops passed in are disabled
 * and that when this function returns, the caller is free to
 * free the ops.
 *
 * The "destroy" name is only to match the "create" name that this
 * should be paired with.
 */
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
	mutex_lock(&ftrace_lock);
	if (ops->flags & FTRACE_OPS_FL_ENABLED)
		ftrace_shutdown(ops, 0);
	ops->flags |= FTRACE_OPS_FL_DELETED;
	mutex_unlock(&ftrace_lock);
}

5203
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
5204 5205
{

5206 5207
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
5208

5209 5210 5211
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

5212
	ftrace_create_filter_files(&global_ops, d_tracer);
5213

5214
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5215
	trace_create_file("set_graph_function", 0444, d_tracer,
5216 5217
				    NULL,
				    &ftrace_graph_fops);
5218 5219 5220
	trace_create_file("set_graph_notrace", 0444, d_tracer,
				    NULL,
				    &ftrace_graph_notrace_fops);
5221 5222
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5223 5224 5225
	return 0;
}

5226
static int ftrace_cmp_ips(const void *a, const void *b)
5227
{
5228 5229
	const unsigned long *ipa = a;
	const unsigned long *ipb = b;
5230

5231 5232 5233 5234 5235 5236 5237
	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

5238
static int ftrace_process_locs(struct module *mod,
5239
			       unsigned long *start,
5240 5241
			       unsigned long *end)
{
5242
	struct ftrace_page *start_pg;
5243
	struct ftrace_page *pg;
5244
	struct dyn_ftrace *rec;
5245
	unsigned long count;
5246 5247
	unsigned long *p;
	unsigned long addr;
5248
	unsigned long flags = 0; /* Shut up gcc */
5249 5250 5251 5252 5253 5254 5255
	int ret = -ENOMEM;

	count = end - start;

	if (!count)
		return 0;

5256
	sort(start, count, sizeof(*start),
5257
	     ftrace_cmp_ips, NULL);
5258

5259 5260
	start_pg = ftrace_allocate_pages(count);
	if (!start_pg)
5261
		return -ENOMEM;
5262

S
Steven Rostedt 已提交
5263
	mutex_lock(&ftrace_lock);
5264

5265 5266 5267 5268 5269
	/*
	 * Core and each module needs their own pages, as
	 * modules will free them when they are removed.
	 * Force a new page to be allocated for modules.
	 */
5270 5271 5272
	if (!mod) {
		WARN_ON(ftrace_pages || ftrace_pages_start);
		/* First initialization */
5273
		ftrace_pages = ftrace_pages_start = start_pg;
5274
	} else {
5275
		if (!ftrace_pages)
5276
			goto out;
5277

5278 5279 5280 5281
		if (WARN_ON(ftrace_pages->next)) {
			/* Hmm, we have free pages? */
			while (ftrace_pages->next)
				ftrace_pages = ftrace_pages->next;
5282
		}
5283

5284
		ftrace_pages->next = start_pg;
5285 5286
	}

5287
	p = start;
5288
	pg = start_pg;
5289 5290
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
5291 5292 5293 5294 5295 5296 5297 5298
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
5299 5300 5301 5302 5303 5304 5305 5306 5307 5308

		if (pg->index == pg->size) {
			/* We should have allocated enough */
			if (WARN_ON(!pg->next))
				break;
			pg = pg->next;
		}

		rec = &pg->records[pg->index++];
		rec->ip = addr;
5309 5310
	}

5311 5312 5313 5314 5315 5316
	/* We should have used all pages */
	WARN_ON(pg->next);

	/* Assign the last page to ftrace_pages */
	ftrace_pages = pg;

5317
	/*
5318 5319 5320 5321 5322 5323
	 * We only need to disable interrupts on start up
	 * because we are modifying code that an interrupt
	 * may execute, and the modification is not atomic.
	 * But for modules, nothing runs the code we modify
	 * until we are finished with it, and there's no
	 * reason to cause large interrupt latencies while we do it.
5324
	 */
5325 5326
	if (!mod)
		local_irq_save(flags);
5327
	ftrace_update_code(mod, start_pg);
5328 5329
	if (!mod)
		local_irq_restore(flags);
5330 5331
	ret = 0;
 out:
S
Steven Rostedt 已提交
5332
	mutex_unlock(&ftrace_lock);
5333

5334
	return ret;
5335 5336
}

5337
#ifdef CONFIG_MODULES
5338 5339 5340

#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)

5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353
static int referenced_filters(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
		if (ops_references_rec(ops, rec))
		    cnt++;
	}

	return cnt;
}

5354
void ftrace_release_mod(struct module *mod)
5355 5356
{
	struct dyn_ftrace *rec;
5357
	struct ftrace_page **last_pg;
5358
	struct ftrace_page *pg;
5359
	int order;
5360

5361 5362
	mutex_lock(&ftrace_lock);

5363
	if (ftrace_disabled)
5364
		goto out_unlock;
5365

5366 5367 5368 5369 5370 5371 5372
	/*
	 * Each module has its own ftrace_pages, remove
	 * them from the list.
	 */
	last_pg = &ftrace_pages_start;
	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
		rec = &pg->records[0];
5373
		if (within_module_core(rec->ip, mod)) {
5374
			/*
5375 5376
			 * As core pages are first, the first
			 * page should never be a module page.
5377
			 */
5378 5379 5380 5381 5382 5383 5384 5385
			if (WARN_ON(pg == ftrace_pages_start))
				goto out_unlock;

			/* Check if we are deleting the last page */
			if (pg == ftrace_pages)
				ftrace_pages = next_to_ftrace_page(last_pg);

			*last_pg = pg->next;
5386 5387 5388
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
5389 5390 5391
		} else
			last_pg = &pg->next;
	}
5392
 out_unlock:
5393 5394 5395
	mutex_unlock(&ftrace_lock);
}

5396
void ftrace_module_enable(struct module *mod)
5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	mutex_lock(&ftrace_lock);

	if (ftrace_disabled)
		goto out_unlock;

	/*
	 * If the tracing is enabled, go ahead and enable the record.
	 *
	 * The reason not to enable the record immediatelly is the
	 * inherent check of ftrace_make_nop/ftrace_make_call for
	 * correct previous instructions.  Making first the NOP
	 * conversion puts the module to the correct state, thus
	 * passing the ftrace_make_call check.
	 *
	 * We also delay this to after the module code already set the
	 * text to read-only, as we now need to set it back to read-write
	 * so that we can modify the text.
	 */
	if (ftrace_start_up)
		ftrace_arch_code_modify_prepare();

	do_for_each_ftrace_rec(pg, rec) {
		int cnt;
		/*
		 * do_for_each_ftrace_rec() is a double loop.
		 * module text shares the pg. If a record is
		 * not part of this module, then skip this pg,
		 * which the "break" will do.
		 */
		if (!within_module_core(rec->ip, mod))
			break;

		cnt = 0;

		/*
		 * When adding a module, we need to check if tracers are
		 * currently enabled and if they are, and can trace this record,
		 * we need to enable the module functions as well as update the
		 * reference counts for those function records.
		 */
		if (ftrace_start_up)
			cnt += referenced_filters(rec);

		/* This clears FTRACE_FL_DISABLED */
		rec->flags = cnt;

		if (ftrace_start_up && cnt) {
			int failed = __ftrace_replace_code(rec, 1);
			if (failed) {
				ftrace_bug(failed, rec);
				goto out_loop;
			}
		}

	} while_for_each_ftrace_rec();

 out_loop:
	if (ftrace_start_up)
		ftrace_arch_code_modify_post_process();

 out_unlock:
	mutex_unlock(&ftrace_lock);
}

5465
void ftrace_module_init(struct module *mod)
5466
{
5467
	if (ftrace_disabled || !mod->num_ftrace_callsites)
5468
		return;
5469

5470 5471
	ftrace_process_locs(mod, mod->ftrace_callsites,
			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5472
}
5473 5474
#endif /* CONFIG_MODULES */

5475
void __init ftrace_free_init_mem(void)
5476
{
5477 5478
	unsigned long start = (unsigned long)(&__init_begin);
	unsigned long end = (unsigned long)(&__init_end);
5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518
	struct ftrace_page **last_pg = &ftrace_pages_start;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	struct dyn_ftrace key;
	int order;

	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */

	mutex_lock(&ftrace_lock);

	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
			continue;
 again:
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (!rec)
			continue;
		pg->index--;
		if (!pg->index) {
			*last_pg = pg->next;
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
			pg = container_of(last_pg, struct ftrace_page, next);
			if (!(*last_pg))
				ftrace_pages = pg;
			continue;
		}
		memmove(rec, rec + 1,
			(pg->index - (rec - pg->records)) * sizeof(*rec));
		/* More than one function may be in this block */
		goto again;
	}
	mutex_unlock(&ftrace_lock);
}

5519 5520
void __init ftrace_init(void)
{
5521 5522
	extern unsigned long __start_mcount_loc[];
	extern unsigned long __stop_mcount_loc[];
5523
	unsigned long count, flags;
5524 5525 5526
	int ret;

	local_irq_save(flags);
5527
	ret = ftrace_dyn_arch_init();
5528
	local_irq_restore(flags);
5529
	if (ret)
5530 5531 5532
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;
5533 5534
	if (!count) {
		pr_info("ftrace: No functions to be traced?\n");
5535
		goto failed;
5536 5537 5538 5539
	}

	pr_info("ftrace: allocating %ld entries in %ld pages\n",
		count, count / ENTRIES_PER_PAGE + 1);
5540 5541 5542

	last_ftrace_enabled = ftrace_enabled = 1;

5543
	ret = ftrace_process_locs(NULL,
5544
				  __start_mcount_loc,
5545 5546
				  __stop_mcount_loc);

5547 5548
	set_ftrace_early_filters();

5549 5550 5551 5552 5553
	return;
 failed:
	ftrace_disabled = 1;
}

5554 5555 5556 5557 5558 5559 5560 5561 5562 5563
/* Do nothing if arch does not support this */
void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
	arch_ftrace_update_trampoline(ops);
}

5564 5565 5566 5567
void ftrace_init_trace_array(struct trace_array *tr)
{
	INIT_LIST_HEAD(&tr->func_probes);
}
5568
#else
5569

5570
static struct ftrace_ops global_ops = {
5571
	.func			= ftrace_stub,
5572 5573 5574
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				  FTRACE_OPS_FL_INITIALIZED |
				  FTRACE_OPS_FL_PID,
5575 5576
};

5577 5578 5579 5580 5581
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
5582
core_initcall(ftrace_nodyn_init);
5583

5584
static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
5585
static inline void ftrace_startup_enable(int command) { }
5586
static inline void ftrace_startup_all(int command) { }
5587
/* Keep as macros so we do not need to define the commands */
5588 5589 5590 5591 5592 5593
# define ftrace_startup(ops, command)					\
	({								\
		int ___ret = __register_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
5594
	})
5595 5596 5597 5598 5599 5600 5601
# define ftrace_shutdown(ops, command)					\
	({								\
		int ___ret = __unregister_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
	})
5602

I
Ingo Molnar 已提交
5603 5604
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
5605 5606

static inline int
5607
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
5608 5609 5610 5611
{
	return 1;
}

5612 5613 5614 5615
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

5616 5617
#endif /* CONFIG_DYNAMIC_FTRACE */

5618 5619 5620 5621
__init void ftrace_init_global_array_ops(struct trace_array *tr)
{
	tr->ops = &global_ops;
	tr->ops->private = tr;
5622
	ftrace_init_trace_array(tr);
5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641
}

void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
{
	/* If we filter on pids, update to use the pid function */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		if (WARN_ON(tr->ops->func != ftrace_stub))
			printk("ftrace ops had %pS for function\n",
			       tr->ops->func);
	}
	tr->ops->func = func;
	tr->ops->private = tr;
}

void ftrace_reset_array_ops(struct trace_array *tr)
{
	tr->ops->func = ftrace_stub;
}

5642 5643
static inline void
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5644
		       struct ftrace_ops *ignored, struct pt_regs *regs)
5645
{
5646
	struct ftrace_ops *op;
5647
	int bit;
5648

5649 5650 5651
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;
5652

5653 5654 5655 5656 5657
	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
5658

5659
	do_for_each_ftrace_op(op, ftrace_ops_list) {
5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673
		/*
		 * Check the following for each ops before calling their func:
		 *  if RCU flag is set, then rcu_is_watching() must be true
		 *  if PER_CPU is set, then ftrace_function_local_disable()
		 *                          must be false
		 *  Otherwise test if the ip matches the ops filter
		 *
		 * If any of the above fails then the op->func() is not executed.
		 */
		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
		    (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
		     !ftrace_function_local_disabled(op)) &&
		    ftrace_ops_test(op, ip, regs)) {
		    
5674 5675
			if (FTRACE_WARN_ON(!op->func)) {
				pr_warn("op=%p %pS\n", op, op);
5676 5677
				goto out;
			}
5678
			op->func(ip, parent_ip, op, regs);
5679
		}
5680
	} while_for_each_ftrace_op(op);
5681
out:
5682
	preempt_enable_notrace();
5683
	trace_clear_recursion(bit);
5684 5685
}

5686 5687 5688 5689 5690
/*
 * Some archs only support passing ip and parent_ip. Even though
 * the list function ignores the op parameter, we do not want any
 * C side effects, where a function is called without the caller
 * sending a third parameter.
5691 5692 5693
 * Archs are to support both the regs and ftrace_ops at the same time.
 * If they support ftrace_ops, it is assumed they support regs.
 * If call backs want to use regs, they must either check for regs
5694 5695
 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
5696
 * An architecture can pass partial regs with ftrace_ops and still
L
Li Bin 已提交
5697
 * set the ARCH_SUPPORTS_FTRACE_OPS.
5698 5699 5700
 */
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5701
				 struct ftrace_ops *op, struct pt_regs *regs)
5702
{
5703
	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
5704 5705 5706 5707
}
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
{
5708
	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
5709 5710 5711
}
#endif

5712 5713
/*
 * If there's only one function registered but it does not support
5714 5715
 * recursion, needs RCU protection and/or requires per cpu handling, then
 * this function will be called by the mcount trampoline.
5716
 */
5717
static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
5718 5719 5720 5721
				   struct ftrace_ops *op, struct pt_regs *regs)
{
	int bit;

5722 5723 5724
	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
		return;

5725 5726 5727 5728
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;

5729
	preempt_disable_notrace();
5730

5731 5732 5733 5734 5735 5736
	if (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
	    !ftrace_function_local_disabled(op)) {
		op->func(ip, parent_ip, op, regs);
	}

	preempt_enable_notrace();
5737 5738 5739
	trace_clear_recursion(bit);
}

5740 5741 5742 5743 5744 5745 5746
/**
 * ftrace_ops_get_func - get the function a trampoline should call
 * @ops: the ops to get the function for
 *
 * Normally the mcount trampoline will call the ops->func, but there
 * are times that it should not. For example, if the ops does not
 * have its own recursion protection, then it should call the
5747
 * ftrace_ops_assist_func() instead.
5748 5749 5750 5751 5752 5753
 *
 * Returns the function that the trampoline should call for @ops.
 */
ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
{
	/*
5754 5755
	 * If the function does not handle recursion, needs to be RCU safe,
	 * or does per cpu logic, then we need to call the assist handler.
5756
	 */
5757 5758 5759
	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
	    ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU))
		return ftrace_ops_assist_func;
5760 5761 5762 5763

	return ops->func;
}

5764 5765 5766
static void
ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
		    struct task_struct *prev, struct task_struct *next)
S
Steven Rostedt 已提交
5767
{
5768 5769
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;
S
Steven Rostedt 已提交
5770

5771
	pid_list = rcu_dereference_sched(tr->function_pids);
5772

5773 5774
	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, next));
S
Steven Rostedt 已提交
5775 5776
}

5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813
static void
ftrace_pid_follow_sched_process_fork(void *data,
				     struct task_struct *self,
				     struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, self, task);
}

static void
ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, NULL, task);
}

void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
{
	if (enable) {
		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						  tr);
		register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						  tr);
	} else {
		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						    tr);
		unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						    tr);
	}
}

5814
static void clear_ftrace_pids(struct trace_array *tr)
5815
{
5816 5817
	struct trace_pid_list *pid_list;
	int cpu;
5818

5819 5820 5821 5822
	pid_list = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));
	if (!pid_list)
		return;
5823

5824
	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
5825

5826 5827
	for_each_possible_cpu(cpu)
		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
S
Steven Rostedt 已提交
5828

5829
	rcu_assign_pointer(tr->function_pids, NULL);
S
Steven Rostedt 已提交
5830

5831 5832
	/* Wait till all users are no longer using pid filtering */
	synchronize_sched();
5833

5834
	trace_free_pid_list(pid_list);
5835 5836
}

5837 5838 5839 5840 5841 5842 5843 5844 5845
void ftrace_clear_pids(struct trace_array *tr)
{
	mutex_lock(&ftrace_lock);

	clear_ftrace_pids(tr);

	mutex_unlock(&ftrace_lock);
}

5846
static void ftrace_pid_reset(struct trace_array *tr)
5847
{
5848
	mutex_lock(&ftrace_lock);
5849
	clear_ftrace_pids(tr);
S
Steven Rostedt 已提交
5850

5851
	ftrace_update_pid_func();
5852
	ftrace_startup_all(0);
5853 5854 5855 5856

	mutex_unlock(&ftrace_lock);
}

5857 5858
/* Greater than any max PID */
#define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
5859

5860
static void *fpid_start(struct seq_file *m, loff_t *pos)
5861
	__acquires(RCU)
5862
{
5863 5864 5865
	struct trace_pid_list *pid_list;
	struct trace_array *tr = m->private;

5866
	mutex_lock(&ftrace_lock);
5867 5868 5869
	rcu_read_lock_sched();

	pid_list = rcu_dereference_sched(tr->function_pids);
5870

5871 5872
	if (!pid_list)
		return !(*pos) ? FTRACE_NO_PIDS : NULL;
5873

5874
	return trace_pid_start(pid_list, pos);
5875 5876 5877 5878
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
5879 5880 5881 5882
	struct trace_array *tr = m->private;
	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);

	if (v == FTRACE_NO_PIDS)
5883 5884
		return NULL;

5885
	return trace_pid_next(pid_list, v, pos);
5886 5887 5888
}

static void fpid_stop(struct seq_file *m, void *p)
5889
	__releases(RCU)
5890
{
5891
	rcu_read_unlock_sched();
5892 5893 5894 5895 5896
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
5897
	if (v == FTRACE_NO_PIDS) {
5898
		seq_puts(m, "no pid\n");
5899 5900 5901
		return 0;
	}

5902
	return trace_pid_show(m, v);
5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
5915 5916
	struct trace_array *tr = inode->i_private;
	struct seq_file *m;
5917 5918
	int ret = 0;

5919 5920 5921
	if (trace_array_get(tr) < 0)
		return -ENODEV;

5922 5923
	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
5924
		ftrace_pid_reset(tr);
5925

5926 5927 5928 5929 5930 5931 5932 5933
	ret = seq_open(file, &ftrace_pid_sops);
	if (ret < 0) {
		trace_array_put(tr);
	} else {
		m = file->private_data;
		/* copy tr over to seq ops */
		m->private = tr;
	}
5934 5935 5936 5937

	return ret;
}

5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953
static void ignore_task_cpu(void *data)
{
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;

	/*
	 * This function is called by on_each_cpu() while the
	 * event_mutex is held.
	 */
	pid_list = rcu_dereference_protected(tr->function_pids,
					     mutex_is_locked(&ftrace_lock));

	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, current));
}

5954 5955 5956 5957
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5958 5959 5960 5961 5962
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
	struct trace_pid_list *filtered_pids = NULL;
	struct trace_pid_list *pid_list;
	ssize_t ret;
5963

5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974
	if (!cnt)
		return 0;

	mutex_lock(&ftrace_lock);

	filtered_pids = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));

	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
	if (ret < 0)
		goto out;
5975

5976
	rcu_assign_pointer(tr->function_pids, pid_list);
5977

5978 5979 5980 5981 5982 5983 5984
	if (filtered_pids) {
		synchronize_sched();
		trace_free_pid_list(filtered_pids);
	} else if (pid_list) {
		/* Register a probe to set whether to ignore the tracing of a task */
		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
	}
5985

5986
	/*
5987 5988 5989
	 * Ignoring of pids is done at task switch. But we have to
	 * check for those tasks that are currently running.
	 * Always do this in case a pid was appended or removed.
5990
	 */
5991
	on_each_cpu(ignore_task_cpu, tr, 1);
5992

5993 5994 5995 5996
	ftrace_update_pid_func();
	ftrace_startup_all(0);
 out:
	mutex_unlock(&ftrace_lock);
5997

5998 5999
	if (ret > 0)
		*ppos += ret;
6000

6001
	return ret;
6002
}
6003

6004 6005 6006
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
6007
	struct trace_array *tr = inode->i_private;
6008

6009 6010 6011
	trace_array_put(tr);

	return seq_release(inode, file);
6012 6013
}

6014
static const struct file_operations ftrace_pid_fops = {
6015 6016 6017
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
6018
	.llseek		= tracing_lseek,
6019
	.release	= ftrace_pid_release,
6020 6021
};

6022
void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6023
{
6024
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
6025
			    tr, &ftrace_pid_fops);
6026 6027
}

6028 6029 6030 6031 6032 6033 6034 6035 6036 6037
void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
					 struct dentry *d_tracer)
{
	/* Only the top level directory has the dyn_tracefs and profile */
	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));

	ftrace_init_dyn_tracefs(d_tracer);
	ftrace_profile_tracefs(d_tracer);
}

S
Steven Rostedt 已提交
6038
/**
6039
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
6040 6041 6042 6043 6044
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
6045
void ftrace_kill(void)
S
Steven Rostedt 已提交
6046 6047 6048 6049 6050 6051
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

6052 6053 6054 6055 6056 6057 6058 6059
/**
 * Test if ftrace is dead or not.
 */
int ftrace_is_dead(void)
{
	return ftrace_disabled;
}

6060
/**
6061 6062
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
6063
 *
6064 6065 6066 6067 6068 6069
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
6070
 */
6071
int register_ftrace_function(struct ftrace_ops *ops)
6072
{
6073
	int ret = -1;
6074

6075 6076
	ftrace_ops_init(ops);

S
Steven Rostedt 已提交
6077
	mutex_lock(&ftrace_lock);
6078

6079
	ret = ftrace_startup(ops, 0);
6080

S
Steven Rostedt 已提交
6081
	mutex_unlock(&ftrace_lock);
6082

6083
	return ret;
6084
}
6085
EXPORT_SYMBOL_GPL(register_ftrace_function);
6086 6087

/**
6088
 * unregister_ftrace_function - unregister a function for profiling.
6089 6090 6091 6092 6093 6094 6095 6096
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
6097
	mutex_lock(&ftrace_lock);
6098
	ret = ftrace_shutdown(ops, 0);
S
Steven Rostedt 已提交
6099
	mutex_unlock(&ftrace_lock);
6100 6101 6102

	return ret;
}
6103
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
6104

I
Ingo Molnar 已提交
6105
int
6106
ftrace_enable_sysctl(struct ctl_table *table, int write,
6107
		     void __user *buffer, size_t *lenp,
6108 6109
		     loff_t *ppos)
{
6110
	int ret = -ENODEV;
6111

S
Steven Rostedt 已提交
6112
	mutex_lock(&ftrace_lock);
6113

6114 6115 6116 6117
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
6118

6119
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
6120 6121
		goto out;

6122
	last_ftrace_enabled = !!ftrace_enabled;
6123 6124 6125 6126

	if (ftrace_enabled) {

		/* we are starting ftrace again */
6127 6128
		if (ftrace_ops_list != &ftrace_list_end)
			update_ftrace_function();
6129

6130 6131
		ftrace_startup_sysctl();

6132 6133 6134 6135 6136 6137 6138 6139
	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
6140
	mutex_unlock(&ftrace_lock);
6141
	return ret;
6142
}
6143

6144
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
6145

6146 6147 6148 6149
static struct ftrace_ops graph_ops = {
	.func			= ftrace_stub,
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				   FTRACE_OPS_FL_INITIALIZED |
6150
				   FTRACE_OPS_FL_PID |
6151 6152 6153
				   FTRACE_OPS_FL_STUB,
#ifdef FTRACE_GRAPH_TRAMP_ADDR
	.trampoline		= FTRACE_GRAPH_TRAMP_ADDR,
6154
	/* trampoline_size is only needed for dynamically allocated tramps */
6155 6156 6157 6158
#endif
	ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
};

6159 6160 6161 6162 6163 6164 6165 6166 6167 6168
void ftrace_graph_sleep_time_control(bool enable)
{
	fgraph_sleep_time = enable;
}

void ftrace_graph_graph_time_control(bool enable)
{
	fgraph_graph_time = enable;
}

6169 6170 6171 6172 6173
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

6174 6175 6176
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
6177
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
6178
static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

6200
	read_lock(&tasklist_lock);
6201 6202 6203 6204 6205 6206 6207
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
6208
			atomic_set(&t->tracing_graph_pause, 0);
6209
			atomic_set(&t->trace_overrun, 0);
6210 6211 6212 6213
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
6214 6215 6216 6217
		}
	} while_each_thread(g, t);

unlock:
6218
	read_unlock(&tasklist_lock);
6219 6220 6221 6222 6223 6224
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

6225
static void
6226
ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
6227
			struct task_struct *prev, struct task_struct *next)
6228 6229 6230 6231
{
	unsigned long long timestamp;
	int index;

6232 6233 6234 6235
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
6236
	if (fgraph_sleep_time)
6237 6238
		return;

6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

6257
/* Allocate a return stack for each task */
6258
static int start_graph_tracing(void)
6259 6260
{
	struct ftrace_ret_stack **ret_stack_list;
6261
	int ret, cpu;
6262 6263 6264 6265 6266 6267 6268 6269

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

6270
	/* The cpu_boot init_task->ret_stack will never be freed */
6271 6272
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
6273
			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
6274
	}
6275

6276 6277 6278 6279
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

6280
	if (!ret) {
6281
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6282 6283 6284 6285 6286
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

6287 6288 6289 6290
	kfree(ret_stack_list);
	return ret;
}

6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327
static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
{
	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
		return 0;
	return __ftrace_graph_entry(trace);
}

/*
 * The function graph tracer should only trace the functions defined
 * by set_ftrace_filter and set_ftrace_notrace. If another function
 * tracer ops is registered, the graph tracer requires testing the
 * function against the global ops, and not just trace any function
 * that any ftrace_ops registered.
 */
static void update_function_graph_func(void)
{
6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346
	struct ftrace_ops *op;
	bool do_test = false;

	/*
	 * The graph and global ops share the same set of functions
	 * to test. If any other ops is on the list, then
	 * the graph tracing needs to test if its the function
	 * it should call.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op != &global_ops && op != &graph_ops &&
		    op != &ftrace_list_end) {
			do_test = true;
			/* in double loop, break out with goto */
			goto out;
		}
	} while_for_each_ftrace_op(op);
 out:
	if (do_test)
6347
		ftrace_graph_entry = ftrace_graph_entry_test;
6348 6349
	else
		ftrace_graph_entry = __ftrace_graph_entry;
6350 6351
}

6352 6353 6354 6355
static struct notifier_block ftrace_suspend_notifier = {
	.notifier_call = ftrace_suspend_notifier_call,
};

6356 6357
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
6358
{
6359 6360
	int ret = 0;

S
Steven Rostedt 已提交
6361
	mutex_lock(&ftrace_lock);
6362

6363
	/* we currently allow only one tracer registered at a time */
6364
	if (ftrace_graph_active) {
6365 6366 6367 6368
		ret = -EBUSY;
		goto out;
	}

6369 6370
	register_pm_notifier(&ftrace_suspend_notifier);

6371
	ftrace_graph_active++;
6372
	ret = start_graph_tracing();
6373
	if (ret) {
6374
		ftrace_graph_active--;
6375 6376
		goto out;
	}
6377

6378
	ftrace_graph_return = retfunc;
6379 6380 6381 6382 6383 6384 6385 6386 6387 6388

	/*
	 * Update the indirect function to the entryfunc, and the
	 * function that gets called to the entry_test first. Then
	 * call the update fgraph entry function to determine if
	 * the entryfunc should be called directly or not.
	 */
	__ftrace_graph_entry = entryfunc;
	ftrace_graph_entry = ftrace_graph_entry_test;
	update_function_graph_func();
6389

6390
	ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
6391
out:
S
Steven Rostedt 已提交
6392
	mutex_unlock(&ftrace_lock);
6393
	return ret;
6394 6395
}

6396
void unregister_ftrace_graph(void)
6397
{
S
Steven Rostedt 已提交
6398
	mutex_lock(&ftrace_lock);
6399

6400
	if (unlikely(!ftrace_graph_active))
6401 6402
		goto out;

6403
	ftrace_graph_active--;
6404
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
6405
	ftrace_graph_entry = ftrace_graph_entry_stub;
6406
	__ftrace_graph_entry = ftrace_graph_entry_stub;
6407
	ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
6408
	unregister_pm_notifier(&ftrace_suspend_notifier);
6409
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6410

6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421
#ifdef CONFIG_DYNAMIC_FTRACE
	/*
	 * Function graph does not allocate the trampoline, but
	 * other global_ops do. We need to reset the ALLOC_TRAMP flag
	 * if one was used.
	 */
	global_ops.trampoline = save_global_trampoline;
	if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP)
		global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
#endif

6422
 out:
S
Steven Rostedt 已提交
6423
	mutex_unlock(&ftrace_lock);
6424
}
6425

6426 6427 6428 6429 6430 6431 6432 6433
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);

static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
	atomic_set(&t->tracing_graph_pause, 0);
	atomic_set(&t->trace_overrun, 0);
	t->ftrace_timestamp = 0;
L
Lucas De Marchi 已提交
6434
	/* make curr_ret_stack visible before we add the ret_stack */
6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468
	smp_wmb();
	t->ret_stack = ret_stack;
}

/*
 * Allocate a return stack for the idle task. May be the first
 * time through, or it may be done by CPU hotplug online.
 */
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
{
	t->curr_ret_stack = -1;
	/*
	 * The idle task has no parent, it either has its own
	 * stack or no stack at all.
	 */
	if (t->ret_stack)
		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));

	if (ftrace_graph_active) {
		struct ftrace_ret_stack *ret_stack;

		ret_stack = per_cpu(idle_ret_stack, cpu);
		if (!ret_stack) {
			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
					    * sizeof(struct ftrace_ret_stack),
					    GFP_KERNEL);
			if (!ret_stack)
				return;
			per_cpu(idle_ret_stack, cpu) = ret_stack;
		}
		graph_init_task(t, ret_stack);
	}
}

6469
/* Allocate a return stack for newly created task */
6470
void ftrace_graph_init_task(struct task_struct *t)
6471
{
6472 6473
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;
6474
	t->curr_ret_stack = -1;
6475

6476
	if (ftrace_graph_active) {
6477 6478 6479
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
6480 6481
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
6482
		if (!ret_stack)
6483
			return;
6484
		graph_init_task(t, ret_stack);
6485
	}
6486 6487
}

6488
void ftrace_graph_exit_task(struct task_struct *t)
6489
{
6490 6491
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

6492
	t->ret_stack = NULL;
6493 6494 6495 6496
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
6497
}
6498
#endif