ftrace.c 149.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Infrastructure for profiling code inserted by 'gcc -pg'.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally ported from the -rt patch by:
 *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
13
 *  Copyright (C) 2004 Nadia Yvette Chambers
14 15
 */

16 17
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
18
#include <linux/sched/task.h>
19
#include <linux/kallsyms.h>
20
#include <linux/seq_file.h>
21
#include <linux/suspend.h>
22
#include <linux/tracefs.h>
23
#include <linux/hardirq.h>
I
Ingo Molnar 已提交
24
#include <linux/kthread.h>
25
#include <linux/uaccess.h>
26
#include <linux/bsearch.h>
27
#include <linux/module.h>
I
Ingo Molnar 已提交
28
#include <linux/ftrace.h>
29
#include <linux/sysctl.h>
30
#include <linux/slab.h>
31
#include <linux/ctype.h>
32
#include <linux/sort.h>
33
#include <linux/list.h>
34
#include <linux/hash.h>
35
#include <linux/rcupdate.h>
36

37
#include <trace/events/sched.h>
38

39
#include <asm/sections.h>
40
#include <asm/setup.h>
41

42
#include "trace_output.h"
S
Steven Rostedt 已提交
43
#include "trace_stat.h"
44

45
#define FTRACE_WARN_ON(cond)			\
46 47 48
	({					\
		int ___r = cond;		\
		if (WARN_ON(___r))		\
49
			ftrace_kill();		\
50 51
		___r;				\
	})
52 53

#define FTRACE_WARN_ON_ONCE(cond)		\
54 55 56
	({					\
		int ___r = cond;		\
		if (WARN_ON_ONCE(___r))		\
57
			ftrace_kill();		\
58 59
		___r;				\
	})
60

61 62 63
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
64 65
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
66

67
#ifdef CONFIG_DYNAMIC_FTRACE
68 69 70
#define INIT_OPS_HASH(opsname)	\
	.func_hash		= &opsname.local_hash,			\
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
71 72 73
#define ASSIGN_OPS_HASH(opsname, val) \
	.func_hash		= val, \
	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
74
#else
75
#define INIT_OPS_HASH(opsname)
76
#define ASSIGN_OPS_HASH(opsname, val)
77 78
#endif

79 80
static struct ftrace_ops ftrace_list_end __read_mostly = {
	.func		= ftrace_stub,
81
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
82
	INIT_OPS_HASH(ftrace_list_end)
83 84
};

85 86
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
87
static int last_ftrace_enabled;
88

89 90
/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
91 92
/* What to set function_trace_op to */
static struct ftrace_ops *set_function_trace_op;
93

94
static bool ftrace_pids_enabled(struct ftrace_ops *ops)
95
{
96 97 98 99 100 101 102 103
	struct trace_array *tr;

	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
		return false;

	tr = ops->private;

	return tr->function_pids != NULL;
104 105 106 107
}

static void ftrace_update_trampoline(struct ftrace_ops *ops);

108 109 110 111 112 113
/*
 * ftrace_disabled is set when an anomaly is discovered.
 * ftrace_disabled is much stronger than ftrace_enabled.
 */
static int ftrace_disabled __read_mostly;

114
static DEFINE_MUTEX(ftrace_lock);
115

116
static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
117
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
118
static struct ftrace_ops global_ops;
119

120 121
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
122
				 struct ftrace_ops *op, struct pt_regs *regs);
123 124 125 126 127
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
128

129 130
/*
 * Traverse the ftrace_global_list, invoking all entries.  The reason that we
131
 * can use rcu_dereference_raw_notrace() is that elements removed from this list
132
 * are simply leaked, so there is no need to interact with a grace-period
133
 * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle
134 135 136 137 138
 * concurrent insertions into the ftrace_global_list.
 *
 * Silly Alpha and silly pointer-speculation compiler optimizations!
 */
#define do_for_each_ftrace_op(op, list)			\
139
	op = rcu_dereference_raw_notrace(list);			\
140 141 142 143 144 145
	do

/*
 * Optimized for just a single item in the list (as that is the normal case).
 */
#define while_for_each_ftrace_op(op)				\
146
	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\
147 148
	       unlikely((op) != &ftrace_list_end))

149 150 151 152
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
153 154
		mutex_init(&ops->local_hash.regex_lock);
		ops->func_hash = &ops->local_hash;
155 156 157 158 159
		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
	}
#endif
}

160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
/**
 * ftrace_nr_registered_ops - return number of ops registered
 *
 * Returns the number of ftrace_ops registered and tracing functions
 */
int ftrace_nr_registered_ops(void)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	mutex_lock(&ftrace_lock);

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next)
		cnt++;

	mutex_unlock(&ftrace_lock);

	return cnt;
}

181
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
182
			    struct ftrace_ops *op, struct pt_regs *regs)
183
{
184 185 186
	struct trace_array *tr = op->private;

	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
187 188
		return;

189
	op->saved_func(ip, parent_ip, op, regs);
190 191
}

192
/**
193
 * clear_ftrace_function - reset the ftrace function
194
 *
195 196
 * This NULLs the ftrace function and in essence stops
 * tracing.  There may be lag
197
 */
198
void clear_ftrace_function(void)
199
{
200 201 202
	ftrace_trace_function = ftrace_stub;
}

203
static void per_cpu_ops_disable_all(struct ftrace_ops *ops)
204 205 206 207 208 209 210
{
	int cpu;

	for_each_possible_cpu(cpu)
		*per_cpu_ptr(ops->disabled, cpu) = 1;
}

211
static int per_cpu_ops_alloc(struct ftrace_ops *ops)
212 213 214
{
	int __percpu *disabled;

215 216 217
	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU)))
		return -EINVAL;

218 219 220 221 222
	disabled = alloc_percpu(int);
	if (!disabled)
		return -ENOMEM;

	ops->disabled = disabled;
223
	per_cpu_ops_disable_all(ops);
224 225 226
	return 0;
}

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
static void ftrace_sync(struct work_struct *work)
{
	/*
	 * This function is just a stub to implement a hard force
	 * of synchronize_sched(). This requires synchronizing
	 * tasks even in userspace and idle.
	 *
	 * Yes, function tracing is rude.
	 */
}

static void ftrace_sync_ipi(void *data)
{
	/* Probably not needed, but do it anyway */
	smp_rmb();
}

244 245
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static void update_function_graph_func(void);
246 247 248 249 250

/* Both enabled by default (can be cleared by function_graph tracer flags */
static bool fgraph_sleep_time = true;
static bool fgraph_graph_time = true;

251 252 253 254
#else
static inline void update_function_graph_func(void) { }
#endif

255 256 257 258

static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
	/*
259
	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
260 261
	 * then it needs to call the list anyway.
	 */
262 263
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU |
			  FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC)
264 265 266 267 268
		return ftrace_ops_list_func;

	return ftrace_ops_get_func(ops);
}

269 270 271 272
static void update_ftrace_function(void)
{
	ftrace_func_t func;

273 274 275 276 277 278 279 280 281 282 283
	/*
	 * Prepare the ftrace_ops that the arch callback will use.
	 * If there's only one ftrace_ops registered, the ftrace_ops_list
	 * will point to the ops we want.
	 */
	set_function_trace_op = ftrace_ops_list;

	/* If there's no ftrace_ops registered, just call the stub function */
	if (ftrace_ops_list == &ftrace_list_end) {
		func = ftrace_stub;

284 285
	/*
	 * If we are at the end of the list and this ops is
286 287
	 * recursion safe and not dynamic and the arch supports passing ops,
	 * then have the mcount trampoline call the function directly.
288
	 */
289
	} else if (ftrace_ops_list->next == &ftrace_list_end) {
290
		func = ftrace_ops_get_list_func(ftrace_ops_list);
291

292 293
	} else {
		/* Just use the default ftrace_ops */
294
		set_function_trace_op = &ftrace_list_end;
295
		func = ftrace_ops_list_func;
296
	}
297

298 299
	update_function_graph_func();

300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
	/* If there's no change, then do nothing more here */
	if (ftrace_trace_function == func)
		return;

	/*
	 * If we are using the list function, it doesn't care
	 * about the function_trace_ops.
	 */
	if (func == ftrace_ops_list_func) {
		ftrace_trace_function = func;
		/*
		 * Don't even bother setting function_trace_ops,
		 * it would be racy to do so anyway.
		 */
		return;
	}

#ifndef CONFIG_DYNAMIC_FTRACE
	/*
	 * For static tracing, we need to be a bit more careful.
	 * The function change takes affect immediately. Thus,
	 * we need to coorditate the setting of the function_trace_ops
	 * with the setting of the ftrace_trace_function.
	 *
	 * Set the function to the list ops, which will call the
	 * function we want, albeit indirectly, but it handles the
	 * ftrace_ops and doesn't depend on function_trace_op.
	 */
	ftrace_trace_function = ftrace_ops_list_func;
	/*
	 * Make sure all CPUs see this. Yes this is slow, but static
	 * tracing is slow and nasty to have enabled.
	 */
	schedule_on_each_cpu(ftrace_sync);
	/* Now all cpus are using the list ops. */
	function_trace_op = set_function_trace_op;
	/* Make sure the function_trace_op is visible on all CPUs */
	smp_wmb();
	/* Nasty way to force a rmb on all cpus */
	smp_call_function(ftrace_sync_ipi, NULL, 1);
	/* OK, we are all set to update the ftrace_trace_function now! */
#endif /* !CONFIG_DYNAMIC_FTRACE */

343 344 345
	ftrace_trace_function = func;
}

346 347 348 349 350
int using_ftrace_ops_list_func(void)
{
	return ftrace_trace_function == ftrace_ops_list_func;
}

351
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
352
{
353
	ops->next = *list;
354
	/*
355
	 * We are entering ops into the list but another
356 357
	 * CPU might be walking that list. We need to make sure
	 * the ops->next pointer is valid before another CPU sees
358
	 * the ops pointer included into the list.
359
	 */
360
	rcu_assign_pointer(*list, ops);
361 362
}

363
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
364 365 366 367
{
	struct ftrace_ops **p;

	/*
368 369
	 * If we are removing the last function, then simply point
	 * to the ftrace_stub.
370
	 */
371 372
	if (*list == ops && ops->next == &ftrace_list_end) {
		*list = &ftrace_list_end;
S
Steven Rostedt 已提交
373
		return 0;
374 375
	}

376
	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
377 378 379
		if (*p == ops)
			break;

S
Steven Rostedt 已提交
380 381
	if (*p != ops)
		return -1;
382 383

	*p = (*p)->next;
384 385
	return 0;
}
386

387 388
static void ftrace_update_trampoline(struct ftrace_ops *ops);

389 390
static int __register_ftrace_function(struct ftrace_ops *ops)
{
391 392 393
	if (ops->flags & FTRACE_OPS_FL_DELETED)
		return -EINVAL;

394 395 396
	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
		return -EBUSY;

397
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
398 399 400 401 402 403 404 405 406 407 408 409 410
	/*
	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
	 */
	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
		return -EINVAL;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif

411 412 413
	if (!core_kernel_data((unsigned long)ops))
		ops->flags |= FTRACE_OPS_FL_DYNAMIC;

414 415
	if (ops->flags & FTRACE_OPS_FL_PER_CPU) {
		if (per_cpu_ops_alloc(ops))
416
			return -ENOMEM;
417 418 419
	}

	add_ftrace_ops(&ftrace_ops_list, ops);
420

421 422 423
	/* Always save the function, and reset at unregistering */
	ops->saved_func = ops->func;

424
	if (ftrace_pids_enabled(ops))
425 426
		ops->func = ftrace_pid_func;

427 428
	ftrace_update_trampoline(ops);

429 430 431 432 433 434 435 436 437 438
	if (ftrace_enabled)
		update_ftrace_function();

	return 0;
}

static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

439 440 441
	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
		return -EBUSY;

442
	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
443

444 445
	if (ret < 0)
		return ret;
446

447 448
	if (ftrace_enabled)
		update_ftrace_function();
449

450 451
	ops->func = ops->saved_func;

S
Steven Rostedt 已提交
452
	return 0;
453 454
}

455 456
static void ftrace_update_pid_func(void)
{
457 458
	struct ftrace_ops *op;

459
	/* Only do something if we are tracing something */
460
	if (ftrace_trace_function == ftrace_stub)
461
		return;
462

463 464
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->flags & FTRACE_OPS_FL_PID) {
465 466
			op->func = ftrace_pids_enabled(op) ?
				ftrace_pid_func : op->saved_func;
467 468 469 470
			ftrace_update_trampoline(op);
		}
	} while_for_each_ftrace_op(op);

471
	update_ftrace_function();
472 473
}

474 475 476 477 478
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
	struct hlist_node		node;
	unsigned long			ip;
	unsigned long			counter;
479 480
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	unsigned long long		time;
481
	unsigned long long		time_squared;
482
#endif
483 484
};

485 486 487 488
struct ftrace_profile_page {
	struct ftrace_profile_page	*next;
	unsigned long			index;
	struct ftrace_profile		records[];
489 490
};

491 492 493 494 495 496 497 498
struct ftrace_profile_stat {
	atomic_t			disabled;
	struct hlist_head		*hash;
	struct ftrace_profile_page	*pages;
	struct ftrace_profile_page	*start;
	struct tracer_stat		stat;
};

499 500
#define PROFILE_RECORDS_SIZE						\
	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
501

502 503
#define PROFILES_PER_PAGE					\
	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
504

505 506 507
static int ftrace_profile_enabled __read_mostly;

/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
S
Steven Rostedt 已提交
508 509
static DEFINE_MUTEX(ftrace_profile_lock);

510
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
511

512 513
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
514

S
Steven Rostedt 已提交
515 516 517
static void *
function_stat_next(void *v, int idx)
{
518 519
	struct ftrace_profile *rec = v;
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
520

521
	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
S
Steven Rostedt 已提交
522 523

 again:
L
Li Zefan 已提交
524 525 526
	if (idx != 0)
		rec++;

S
Steven Rostedt 已提交
527 528 529 530 531
	if ((void *)rec >= (void *)&pg->records[pg->index]) {
		pg = pg->next;
		if (!pg)
			return NULL;
		rec = &pg->records[0];
532 533
		if (!rec->counter)
			goto again;
S
Steven Rostedt 已提交
534 535 536 537 538 539 540
	}

	return rec;
}

static void *function_stat_start(struct tracer_stat *trace)
{
541 542 543 544 545 546 547
	struct ftrace_profile_stat *stat =
		container_of(trace, struct ftrace_profile_stat, stat);

	if (!stat || !stat->start)
		return NULL;

	return function_stat_next(&stat->start->records[0], 0);
S
Steven Rostedt 已提交
548 549
}

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;

	if (a->time < b->time)
		return -1;
	if (a->time > b->time)
		return 1;
	else
		return 0;
}
#else
/* not function graph compares against hits */
S
Steven Rostedt 已提交
566 567
static int function_stat_cmp(void *p1, void *p2)
{
568 569
	struct ftrace_profile *a = p1;
	struct ftrace_profile *b = p2;
S
Steven Rostedt 已提交
570 571 572 573 574 575 576 577

	if (a->counter < b->counter)
		return -1;
	if (a->counter > b->counter)
		return 1;
	else
		return 0;
}
578
#endif
S
Steven Rostedt 已提交
579 580 581

static int function_stat_headers(struct seq_file *m)
{
582
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
583 584 585 586
	seq_puts(m, "  Function                               "
		 "Hit    Time            Avg             s^2\n"
		    "  --------                               "
		 "---    ----            ---             ---\n");
587
#else
588 589
	seq_puts(m, "  Function                               Hit\n"
		    "  --------                               ---\n");
590
#endif
S
Steven Rostedt 已提交
591 592 593 594 595
	return 0;
}

static int function_stat_show(struct seq_file *m, void *v)
{
596
	struct ftrace_profile *rec = v;
S
Steven Rostedt 已提交
597
	char str[KSYM_SYMBOL_LEN];
598
	int ret = 0;
599
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
600 601
	static struct trace_seq s;
	unsigned long long avg;
602
	unsigned long long stddev;
603
#endif
604 605 606 607 608 609 610
	mutex_lock(&ftrace_profile_lock);

	/* we raced with function_profile_reset() */
	if (unlikely(rec->counter == 0)) {
		ret = -EBUSY;
		goto out;
	}
S
Steven Rostedt 已提交
611

612 613 614 615 616 617 618
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	avg = rec->time;
	do_div(avg, rec->counter);
	if (tracing_thresh && (avg < tracing_thresh))
		goto out;
#endif

S
Steven Rostedt 已提交
619
	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
620 621 622
	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
623
	seq_puts(m, "    ");
624

625 626 627 628
	/* Sample standard deviation (s^2) */
	if (rec->counter <= 1)
		stddev = 0;
	else {
629 630 631 632 633 634 635
		/*
		 * Apply Welford's method:
		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
		 */
		stddev = rec->counter * rec->time_squared -
			 rec->time * rec->time;

636 637 638 639
		/*
		 * Divide only 1000 for ns^2 -> us^2 conversion.
		 * trace_print_graph_duration will divide 1000 again.
		 */
640
		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
641 642
	}

643 644 645 646
	trace_seq_init(&s);
	trace_print_graph_duration(rec->time, &s);
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(avg, &s);
647 648
	trace_seq_puts(&s, "    ");
	trace_print_graph_duration(stddev, &s);
649 650 651
	trace_print_seq(m, &s);
#endif
	seq_putc(m, '\n');
652 653
out:
	mutex_unlock(&ftrace_profile_lock);
S
Steven Rostedt 已提交
654

655
	return ret;
S
Steven Rostedt 已提交
656 657
}

658
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
S
Steven Rostedt 已提交
659
{
660
	struct ftrace_profile_page *pg;
S
Steven Rostedt 已提交
661

662
	pg = stat->pages = stat->start;
S
Steven Rostedt 已提交
663

664 665 666 667
	while (pg) {
		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
		pg->index = 0;
		pg = pg->next;
S
Steven Rostedt 已提交
668 669
	}

670
	memset(stat->hash, 0,
671 672
	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
S
Steven Rostedt 已提交
673

674
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
675 676
{
	struct ftrace_profile_page *pg;
677 678
	int functions;
	int pages;
679
	int i;
S
Steven Rostedt 已提交
680

681
	/* If we already allocated, do nothing */
682
	if (stat->pages)
683
		return 0;
S
Steven Rostedt 已提交
684

685 686
	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
	if (!stat->pages)
687
		return -ENOMEM;
S
Steven Rostedt 已提交
688

689 690 691 692 693 694 695 696 697 698 699 700 701
#ifdef CONFIG_DYNAMIC_FTRACE
	functions = ftrace_update_tot_cnt;
#else
	/*
	 * We do not know the number of functions that exist because
	 * dynamic tracing is what counts them. With past experience
	 * we have around 20K functions. That should be more than enough.
	 * It is highly unlikely we will execute every function in
	 * the kernel.
	 */
	functions = 20000;
#endif

702
	pg = stat->start = stat->pages;
S
Steven Rostedt 已提交
703

704 705
	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);

706
	for (i = 1; i < pages; i++) {
707 708
		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
		if (!pg->next)
709
			goto out_free;
710 711 712 713
		pg = pg->next;
	}

	return 0;
714 715 716 717 718 719 720 721 722 723 724 725 726 727

 out_free:
	pg = stat->start;
	while (pg) {
		unsigned long tmp = (unsigned long)pg;

		pg = pg->next;
		free_page(tmp);
	}

	stat->pages = NULL;
	stat->start = NULL;

	return -ENOMEM;
S
Steven Rostedt 已提交
728 729
}

730
static int ftrace_profile_init_cpu(int cpu)
S
Steven Rostedt 已提交
731
{
732
	struct ftrace_profile_stat *stat;
733
	int size;
S
Steven Rostedt 已提交
734

735 736 737
	stat = &per_cpu(ftrace_profile_stats, cpu);

	if (stat->hash) {
738
		/* If the profile is already created, simply reset it */
739
		ftrace_profile_reset(stat);
740 741
		return 0;
	}
S
Steven Rostedt 已提交
742

743 744 745 746 747
	/*
	 * We are profiling all functions, but usually only a few thousand
	 * functions are hit. We'll make a hash of 1024 items.
	 */
	size = FTRACE_PROFILE_HASH_SIZE;
S
Steven Rostedt 已提交
748

749
	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
750

751
	if (!stat->hash)
752 753
		return -ENOMEM;

754
	/* Preallocate the function profiling pages */
755 756 757
	if (ftrace_profile_pages_init(stat) < 0) {
		kfree(stat->hash);
		stat->hash = NULL;
758 759 760 761
		return -ENOMEM;
	}

	return 0;
S
Steven Rostedt 已提交
762 763
}

764 765 766 767 768
static int ftrace_profile_init(void)
{
	int cpu;
	int ret = 0;

769
	for_each_possible_cpu(cpu) {
770 771 772 773 774 775 776 777
		ret = ftrace_profile_init_cpu(cpu);
		if (ret)
			break;
	}

	return ret;
}

778
/* interrupts must be disabled */
779 780
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
S
Steven Rostedt 已提交
781
{
782
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
783 784 785
	struct hlist_head *hhd;
	unsigned long key;

786
	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
787
	hhd = &stat->hash[key];
S
Steven Rostedt 已提交
788 789 790 791

	if (hlist_empty(hhd))
		return NULL;

792
	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
S
Steven Rostedt 已提交
793
		if (rec->ip == ip)
794 795 796 797 798 799
			return rec;
	}

	return NULL;
}

800 801
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
			       struct ftrace_profile *rec)
802 803 804
{
	unsigned long key;

805
	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
806
	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
807 808
}

809 810 811
/*
 * The memory is already allocated, this simply finds a new record to use.
 */
812
static struct ftrace_profile *
813
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
814 815 816
{
	struct ftrace_profile *rec = NULL;

817
	/* prevent recursion (from NMIs) */
818
	if (atomic_inc_return(&stat->disabled) != 1)
819 820 821
		goto out;

	/*
822 823
	 * Try to find the function again since an NMI
	 * could have added it
824
	 */
825
	rec = ftrace_find_profiled_func(stat, ip);
826
	if (rec)
827
		goto out;
828

829 830 831 832
	if (stat->pages->index == PROFILES_PER_PAGE) {
		if (!stat->pages->next)
			goto out;
		stat->pages = stat->pages->next;
S
Steven Rostedt 已提交
833
	}
834

835
	rec = &stat->pages->records[stat->pages->index++];
836
	rec->ip = ip;
837
	ftrace_add_profile(stat, rec);
838

S
Steven Rostedt 已提交
839
 out:
840
	atomic_dec(&stat->disabled);
S
Steven Rostedt 已提交
841 842 843 844 845

	return rec;
}

static void
846
function_profile_call(unsigned long ip, unsigned long parent_ip,
847
		      struct ftrace_ops *ops, struct pt_regs *regs)
S
Steven Rostedt 已提交
848
{
849
	struct ftrace_profile_stat *stat;
850
	struct ftrace_profile *rec;
S
Steven Rostedt 已提交
851 852 853 854 855 856
	unsigned long flags;

	if (!ftrace_profile_enabled)
		return;

	local_irq_save(flags);
857

858
	stat = this_cpu_ptr(&ftrace_profile_stats);
859
	if (!stat->hash || !ftrace_profile_enabled)
860 861 862
		goto out;

	rec = ftrace_find_profiled_func(stat, ip);
863
	if (!rec) {
864
		rec = ftrace_profile_alloc(stat, ip);
865 866 867
		if (!rec)
			goto out;
	}
S
Steven Rostedt 已提交
868 869 870 871 872 873

	rec->counter++;
 out:
	local_irq_restore(flags);
}

874 875 876
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
877 878
	int index = trace->depth;

879
	function_profile_call(trace->func, 0, NULL, NULL);
880 881 882 883

	if (index >= 0 && index < FTRACE_RETFUNC_DEPTH)
		current->ret_stack[index].subtime = 0;

884 885 886 887 888
	return 1;
}

static void profile_graph_return(struct ftrace_graph_ret *trace)
{
889
	struct ftrace_profile_stat *stat;
890
	unsigned long long calltime;
891
	struct ftrace_profile *rec;
892
	unsigned long flags;
893 894

	local_irq_save(flags);
895
	stat = this_cpu_ptr(&ftrace_profile_stats);
896
	if (!stat->hash || !ftrace_profile_enabled)
897 898
		goto out;

899 900 901 902
	/* If the calltime was zero'd ignore it */
	if (!trace->calltime)
		goto out;

903 904
	calltime = trace->rettime - trace->calltime;

905
	if (!fgraph_graph_time) {
906 907 908 909 910 911 912 913 914 915 916 917 918 919
		int index;

		index = trace->depth;

		/* Append this call time to the parent time to subtract */
		if (index)
			current->ret_stack[index - 1].subtime += calltime;

		if (current->ret_stack[index].subtime < calltime)
			calltime -= current->ret_stack[index].subtime;
		else
			calltime = 0;
	}

920
	rec = ftrace_find_profiled_func(stat, trace->func);
921
	if (rec) {
922
		rec->time += calltime;
923 924
		rec->time_squared += calltime * calltime;
	}
925

926
 out:
927 928 929 930 931 932 933 934 935 936 937 938 939 940
	local_irq_restore(flags);
}

static int register_ftrace_profiler(void)
{
	return register_ftrace_graph(&profile_graph_return,
				     &profile_graph_entry);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_graph();
}
#else
941
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
942
	.func		= function_profile_call,
943
	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
944
	INIT_OPS_HASH(ftrace_profile_ops)
S
Steven Rostedt 已提交
945 946
};

947 948 949 950 951 952 953 954 955 956 957
static int register_ftrace_profiler(void)
{
	return register_ftrace_function(&ftrace_profile_ops);
}

static void unregister_ftrace_profiler(void)
{
	unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

S
Steven Rostedt 已提交
958 959 960 961 962 963 964
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

965 966
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
S
Steven Rostedt 已提交
967 968 969 970 971 972 973
		return ret;

	val = !!val;

	mutex_lock(&ftrace_profile_lock);
	if (ftrace_profile_enabled ^ val) {
		if (val) {
974 975 976 977 978 979
			ret = ftrace_profile_init();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}

980 981 982 983 984
			ret = register_ftrace_profiler();
			if (ret < 0) {
				cnt = ret;
				goto out;
			}
S
Steven Rostedt 已提交
985 986 987
			ftrace_profile_enabled = 1;
		} else {
			ftrace_profile_enabled = 0;
988 989 990 991
			/*
			 * unregister_ftrace_profiler calls stop_machine
			 * so this acts like an synchronize_sched.
			 */
992
			unregister_ftrace_profiler();
S
Steven Rostedt 已提交
993 994
		}
	}
995
 out:
S
Steven Rostedt 已提交
996 997
	mutex_unlock(&ftrace_profile_lock);

998
	*ppos += cnt;
S
Steven Rostedt 已提交
999 1000 1001 1002

	return cnt;
}

1003 1004 1005 1006
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
1007
	char buf[64];		/* big enough to hold a number */
1008 1009 1010 1011 1012 1013
	int r;

	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

S
Steven Rostedt 已提交
1014 1015 1016 1017
static const struct file_operations ftrace_profile_fops = {
	.open		= tracing_open_generic,
	.read		= ftrace_profile_read,
	.write		= ftrace_profile_write,
1018
	.llseek		= default_llseek,
S
Steven Rostedt 已提交
1019 1020
};

1021 1022
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
1023 1024 1025 1026 1027 1028
	.name		= "functions",
	.stat_start	= function_stat_start,
	.stat_next	= function_stat_next,
	.stat_cmp	= function_stat_cmp,
	.stat_headers	= function_stat_headers,
	.stat_show	= function_stat_show
1029 1030
};

1031
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1032
{
1033
	struct ftrace_profile_stat *stat;
S
Steven Rostedt 已提交
1034
	struct dentry *entry;
1035
	char *name;
S
Steven Rostedt 已提交
1036
	int ret;
1037 1038 1039 1040 1041
	int cpu;

	for_each_possible_cpu(cpu) {
		stat = &per_cpu(ftrace_profile_stats, cpu);

1042
		name = kasprintf(GFP_KERNEL, "function%d", cpu);
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
		if (!name) {
			/*
			 * The files created are permanent, if something happens
			 * we still do not free memory.
			 */
			WARN(1,
			     "Could not allocate stat file for cpu %d\n",
			     cpu);
			return;
		}
		stat->stat = function_stats;
		stat->stat.name = name;
		ret = register_stat_tracer(&stat->stat);
		if (ret) {
			WARN(1,
			     "Could not register function stat for cpu %d\n",
			     cpu);
			kfree(name);
			return;
		}
S
Steven Rostedt 已提交
1063 1064
	}

1065
	entry = tracefs_create_file("function_profile_enabled", 0644,
S
Steven Rostedt 已提交
1066 1067
				    d_tracer, NULL, &ftrace_profile_fops);
	if (!entry)
1068
		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
S
Steven Rostedt 已提交
1069 1070 1071
}

#else /* CONFIG_FUNCTION_PROFILER */
1072
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
S
Steven Rostedt 已提交
1073 1074 1075 1076
{
}
#endif /* CONFIG_FUNCTION_PROFILER */

1077 1078
static struct pid * const ftrace_swapper_pid = &init_struct_pid;

1079 1080 1081 1082 1083 1084
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int ftrace_graph_active;
#else
# define ftrace_graph_active 0
#endif

1085 1086
#ifdef CONFIG_DYNAMIC_FTRACE

1087 1088
static struct ftrace_ops *removed_ops;

1089 1090 1091 1092 1093 1094
/*
 * Set when doing a global update, like enabling all recs or disabling them.
 * It is not set when just updating a single ftrace_ops.
 */
static bool update_all_ops;

1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif

static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;

struct ftrace_func_probe {
	struct hlist_node	node;
	struct ftrace_probe_ops	*ops;
	unsigned long		ip;
1105
	struct list_head	free_list;
1106 1107
};

1108 1109 1110 1111 1112
struct ftrace_func_entry {
	struct hlist_node hlist;
	unsigned long ip;
};

1113 1114 1115 1116 1117 1118 1119 1120 1121
/*
 * We make these constant because no one should touch them,
 * but they are used as the default "empty hash", to avoid allocating
 * it all the time. These are in a read only section such that if
 * anyone does try to modify it, it will cause an exception.
 */
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
	.buckets = (struct hlist_head *)empty_buckets,
1122
};
1123
#define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1124

1125
static struct ftrace_ops global_ops = {
1126 1127 1128 1129 1130
	.func				= ftrace_stub,
	.local_hash.notrace_hash	= EMPTY_HASH,
	.local_hash.filter_hash		= EMPTY_HASH,
	INIT_OPS_HASH(global_ops)
	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1131 1132
					  FTRACE_OPS_FL_INITIALIZED |
					  FTRACE_OPS_FL_PID,
1133 1134
};

1135 1136
/*
 * This is used by __kernel_text_address() to return true if the
1137
 * address is on a dynamically allocated trampoline that would
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
 * not return true for either core_kernel_text() or
 * is_module_text_address().
 */
bool is_ftrace_trampoline(unsigned long addr)
{
	struct ftrace_ops *op;
	bool ret = false;

	/*
	 * Some of the ops may be dynamically allocated,
	 * they are freed after a synchronize_sched().
	 */
	preempt_disable_notrace();

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/*
		 * This is to check for dynamically allocated trampolines.
		 * Trampolines that are in kernel text will have
		 * core_kernel_text() return true.
		 */
		if (op->trampoline && op->trampoline_size)
			if (addr >= op->trampoline &&
			    addr < op->trampoline + op->trampoline_size) {
				ret = true;
				goto out;
			}
	} while_for_each_ftrace_op(op);

 out:
	preempt_enable_notrace();

	return ret;
}

1172 1173
struct ftrace_page {
	struct ftrace_page	*next;
1174
	struct dyn_ftrace	*records;
1175
	int			index;
1176
	int			size;
1177 1178
};

1179 1180
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1181 1182 1183 1184 1185 1186 1187

/* estimate from running different kernels */
#define NR_TO_INIT		10000

static struct ftrace_page	*ftrace_pages_start;
static struct ftrace_page	*ftrace_pages;

1188 1189 1190 1191 1192 1193 1194 1195 1196
static __always_inline unsigned long
ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
{
	if (hash->size_bits > 0)
		return hash_long(ip, hash->size_bits);

	return 0;
}

1197 1198 1199
/* Only use this function if ftrace_hash_empty() has already been tested */
static __always_inline struct ftrace_func_entry *
__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1200 1201 1202 1203 1204
{
	unsigned long key;
	struct ftrace_func_entry *entry;
	struct hlist_head *hhd;

1205
	key = ftrace_hash_key(hash, ip);
1206 1207
	hhd = &hash->buckets[key];

1208
	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1209 1210 1211 1212 1213 1214
		if (entry->ip == ip)
			return entry;
	}
	return NULL;
}

1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
/**
 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
 * @hash: The hash to look at
 * @ip: The instruction pointer to test
 *
 * Search a given @hash to see if a given instruction pointer (@ip)
 * exists in it.
 *
 * Returns the entry that holds the @ip if found. NULL otherwise.
 */
struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
	if (ftrace_hash_empty(hash))
		return NULL;

	return __ftrace_lookup_ip(hash, ip);
}

1234 1235
static void __add_hash_entry(struct ftrace_hash *hash,
			     struct ftrace_func_entry *entry)
1236 1237 1238 1239
{
	struct hlist_head *hhd;
	unsigned long key;

1240
	key = ftrace_hash_key(hash, entry->ip);
1241 1242 1243
	hhd = &hash->buckets[key];
	hlist_add_head(&entry->hlist, hhd);
	hash->count++;
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
}

static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
	struct ftrace_func_entry *entry;

	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	entry->ip = ip;
	__add_hash_entry(hash, entry);
1256 1257 1258 1259 1260

	return 0;
}

static void
1261
free_hash_entry(struct ftrace_hash *hash,
1262 1263 1264 1265 1266 1267 1268
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	kfree(entry);
	hash->count--;
}

1269 1270 1271 1272 1273 1274 1275 1276
static void
remove_hash_entry(struct ftrace_hash *hash,
		  struct ftrace_func_entry *entry)
{
	hlist_del(&entry->hlist);
	hash->count--;
}

1277 1278 1279
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
	struct hlist_head *hhd;
1280
	struct hlist_node *tn;
1281 1282 1283 1284
	struct ftrace_func_entry *entry;
	int size = 1 << hash->size_bits;
	int i;

1285 1286 1287
	if (!hash->count)
		return;

1288 1289
	for (i = 0; i < size; i++) {
		hhd = &hash->buckets[i];
1290
		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1291
			free_hash_entry(hash, entry);
1292 1293 1294 1295
	}
	FTRACE_WARN_ON(hash->count);
}

1296 1297 1298 1299 1300 1301 1302 1303 1304
static void free_ftrace_hash(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	ftrace_hash_clear(hash);
	kfree(hash->buckets);
	kfree(hash);
}

1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
	struct ftrace_hash *hash;

	hash = container_of(rcu, struct ftrace_hash, rcu);
	free_ftrace_hash(hash);
}

static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
	if (!hash || hash == EMPTY_HASH)
		return;
	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}

1320 1321
void ftrace_free_filter(struct ftrace_ops *ops)
{
1322
	ftrace_ops_init(ops);
1323 1324
	free_ftrace_hash(ops->func_hash->filter_hash);
	free_ftrace_hash(ops->func_hash->notrace_hash);
1325 1326
}

1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
	struct ftrace_hash *hash;
	int size;

	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
	if (!hash)
		return NULL;

	size = 1 << size_bits;
1337
	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362

	if (!hash->buckets) {
		kfree(hash);
		return NULL;
	}

	hash->size_bits = size_bits;

	return hash;
}

static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
	struct ftrace_func_entry *entry;
	struct ftrace_hash *new_hash;
	int size;
	int ret;
	int i;

	new_hash = alloc_ftrace_hash(size_bits);
	if (!new_hash)
		return NULL;

	/* Empty hash? */
1363
	if (ftrace_hash_empty(hash))
1364 1365 1366 1367
		return new_hash;

	size = 1 << hash->size_bits;
	for (i = 0; i < size; i++) {
1368
		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
			ret = add_hash_entry(new_hash, entry->ip);
			if (ret < 0)
				goto free_hash;
		}
	}

	FTRACE_WARN_ON(new_hash->count != hash->count);

	return new_hash;

 free_hash:
	free_ftrace_hash(new_hash);
	return NULL;
}

1384
static void
1385
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1386
static void
1387
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1388

1389 1390 1391
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash);

1392 1393
static struct ftrace_hash *
__ftrace_hash_move(struct ftrace_hash *src)
1394 1395
{
	struct ftrace_func_entry *entry;
1396
	struct hlist_node *tn;
1397
	struct hlist_head *hhd;
1398
	struct ftrace_hash *new_hash;
1399 1400 1401 1402 1403
	int size = src->count;
	int bits = 0;
	int i;

	/*
1404
	 * If the new source is empty, just return the empty_hash.
1405
	 */
1406 1407
	if (!src->count)
		return EMPTY_HASH;
1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418

	/*
	 * Make the hash size about 1/2 the # found
	 */
	for (size /= 2; size; size >>= 1)
		bits++;

	/* Don't allocate too much */
	if (bits > FTRACE_HASH_MAX_BITS)
		bits = FTRACE_HASH_MAX_BITS;

1419 1420
	new_hash = alloc_ftrace_hash(bits);
	if (!new_hash)
1421
		return NULL;
1422 1423 1424 1425

	size = 1 << src->size_bits;
	for (i = 0; i < size; i++) {
		hhd = &src->buckets[i];
1426
		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1427
			remove_hash_entry(src, entry);
1428
			__add_hash_entry(new_hash, entry);
1429 1430 1431
		}
	}

1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
	return new_hash;
}

static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
		 struct ftrace_hash **dst, struct ftrace_hash *src)
{
	struct ftrace_hash *new_hash;
	int ret;

	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
		return -EINVAL;

	new_hash = __ftrace_hash_move(src);
	if (!new_hash)
		return -ENOMEM;

1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
	if (enable) {
		/* IPMODIFY should be updated only when filter_hash updating */
		ret = ftrace_hash_ipmodify_update(ops, new_hash);
		if (ret < 0) {
			free_ftrace_hash(new_hash);
			return ret;
		}
	}

1460 1461 1462 1463
	/*
	 * Remove the current set, update the hash and add
	 * them back.
	 */
1464
	ftrace_hash_rec_disable_modify(ops, enable);
1465

1466 1467
	rcu_assign_pointer(*dst, new_hash);

1468
	ftrace_hash_rec_enable_modify(ops, enable);
1469

1470
	return 0;
1471 1472
}

1473 1474 1475 1476 1477 1478 1479 1480 1481 1482
static bool hash_contains_ip(unsigned long ip,
			     struct ftrace_ops_hash *hash)
{
	/*
	 * The function record is a match if it exists in the filter
	 * hash and not in the notrace hash. Note, an emty hash is
	 * considered a match for the filter hash, but an empty
	 * notrace hash is considered not in the notrace hash.
	 */
	return (ftrace_hash_empty(hash->filter_hash) ||
1483
		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1484
		(ftrace_hash_empty(hash->notrace_hash) ||
1485
		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1486 1487
}

1488 1489 1490 1491 1492 1493 1494 1495
/*
 * Test the hashes for this ops to see if we want to call
 * the ops->func or not.
 *
 * It's a match if the ip is in the ops->filter_hash or
 * the filter_hash does not exist or is empty,
 *  AND
 * the ip is not in the ops->notrace_hash.
1496 1497 1498
 *
 * This needs to be called with preemption disabled as
 * the hashes are freed with call_rcu_sched().
1499 1500
 */
static int
1501
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1502
{
1503
	struct ftrace_ops_hash hash;
1504 1505
	int ret;

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	/*
	 * There's a small race when adding ops that the ftrace handler
	 * that wants regs, may be called without them. We can not
	 * allow that handler to be called if regs is NULL.
	 */
	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
		return 0;
#endif

1516 1517
	hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
	hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
1518

1519
	if (hash_contains_ip(ip, &hash))
1520 1521 1522 1523 1524 1525 1526
		ret = 1;
	else
		ret = 0;

	return ret;
}

1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540
/*
 * This is a double for. Do not use 'break' to break out of the loop,
 * you must use a goto.
 */
#define do_for_each_ftrace_rec(pg, rec)					\
	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
		int _____i;						\
		for (_____i = 0; _____i < pg->index; _____i++) {	\
			rec = &pg->records[_____i];

#define while_for_each_ftrace_rec()		\
		}				\
	}

1541 1542 1543

static int ftrace_cmp_recs(const void *a, const void *b)
{
1544 1545
	const struct dyn_ftrace *key = a;
	const struct dyn_ftrace *rec = b;
1546

1547
	if (key->flags < rec->ip)
1548
		return -1;
1549 1550
	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
		return 1;
1551 1552 1553
	return 0;
}

1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
/**
 * ftrace_location_range - return the first address of a traced location
 *	if it touches the given ip range
 * @start: start of range to search.
 * @end: end of range to search (inclusive). @end points to the last byte
 *	to check.
 *
 * Returns rec->ip if the related ftrace location is a least partly within
 * the given address range. That is, the first address of the instruction
 * that is either a NOP or call to the function tracer. It checks the ftrace
 * internal tables to determine if the address belongs or not.
 */
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1567 1568 1569
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1570
	struct dyn_ftrace key;
1571

1572 1573
	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */
1574 1575

	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1576 1577
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1578
			continue;
1579 1580 1581 1582
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (rec)
1583
			return rec->ip;
1584
	}
1585 1586 1587 1588

	return 0;
}

1589 1590 1591 1592
/**
 * ftrace_location - return true if the ip giving is a traced location
 * @ip: the instruction pointer to check
 *
1593
 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1594 1595 1596 1597
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1598
unsigned long ftrace_location(unsigned long ip)
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
{
	return ftrace_location_range(ip, ip);
}

/**
 * ftrace_text_reserved - return true if range contains an ftrace location
 * @start: start of range to search
 * @end: end of range to search (inclusive). @end points to the last byte to check.
 *
 * Returns 1 if @start and @end contains a ftrace location.
 * That is, the instruction that is either a NOP or call to
 * the function tracer. It checks the ftrace internal tables to
 * determine if the address belongs or not.
 */
1613
int ftrace_text_reserved(const void *start, const void *end)
1614
{
1615 1616 1617 1618 1619 1620
	unsigned long ret;

	ret = ftrace_location_range((unsigned long)start,
				    (unsigned long)end);

	return (int)!!ret;
1621 1622
}

1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
/* Test if ops registered to this rec needs regs */
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	bool keep_regs = false;

	for (ops = ftrace_ops_list;
	     ops != &ftrace_list_end; ops = ops->next) {
		/* pass rec in as regs to have non-NULL val */
		if (ftrace_ops_test(ops, rec->ip, rec)) {
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				keep_regs = true;
				break;
			}
		}
	}

	return  keep_regs;
}

1643
static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1644 1645 1646 1647 1648 1649 1650
				     int filter_hash,
				     bool inc)
{
	struct ftrace_hash *hash;
	struct ftrace_hash *other_hash;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
1651
	bool update = false;
1652 1653 1654 1655 1656
	int count = 0;
	int all = 0;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1657
		return false;
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670

	/*
	 * In the filter_hash case:
	 *   If the count is zero, we update all records.
	 *   Otherwise we just update the items in the hash.
	 *
	 * In the notrace_hash case:
	 *   We enable the update in the hash.
	 *   As disabling notrace means enabling the tracing,
	 *   and enabling notrace means disabling, the inc variable
	 *   gets inversed.
	 */
	if (filter_hash) {
1671 1672
		hash = ops->func_hash->filter_hash;
		other_hash = ops->func_hash->notrace_hash;
1673
		if (ftrace_hash_empty(hash))
1674 1675 1676
			all = 1;
	} else {
		inc = !inc;
1677 1678
		hash = ops->func_hash->notrace_hash;
		other_hash = ops->func_hash->filter_hash;
1679 1680 1681 1682
		/*
		 * If the notrace hash has no items,
		 * then there's nothing to do.
		 */
1683
		if (ftrace_hash_empty(hash))
1684
			return false;
1685 1686 1687 1688 1689 1690 1691
	}

	do_for_each_ftrace_rec(pg, rec) {
		int in_other_hash = 0;
		int in_hash = 0;
		int match = 0;

1692 1693 1694
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1695 1696 1697 1698 1699
		if (all) {
			/*
			 * Only the filter_hash affects all records.
			 * Update if the record is not in the notrace hash.
			 */
1700
			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1701 1702
				match = 1;
		} else {
1703 1704
			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1705 1706

			/*
1707 1708
			 * If filter_hash is set, we want to match all functions
			 * that are in the hash but not in the other hash.
1709
			 *
1710 1711 1712 1713 1714
			 * If filter_hash is not set, then we are decrementing.
			 * That means we match anything that is in the hash
			 * and also in the other_hash. That is, we need to turn
			 * off functions in the other hash because they are disabled
			 * by this hash.
1715 1716 1717 1718
			 */
			if (filter_hash && in_hash && !in_other_hash)
				match = 1;
			else if (!filter_hash && in_hash &&
1719
				 (in_other_hash || ftrace_hash_empty(other_hash)))
1720 1721 1722 1723 1724 1725 1726
				match = 1;
		}
		if (!match)
			continue;

		if (inc) {
			rec->flags++;
1727
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1728
				return false;
1729 1730 1731 1732 1733 1734

			/*
			 * If there's only a single callback registered to a
			 * function, and the ops has a trampoline registered
			 * for it, then we can call it directly.
			 */
1735
			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1736
				rec->flags |= FTRACE_FL_TRAMP;
1737
			else
1738 1739 1740
				/*
				 * If we are adding another function callback
				 * to this function, and the previous had a
1741 1742
				 * custom trampoline in use, then we need to go
				 * back to the default trampoline.
1743
				 */
1744
				rec->flags &= ~FTRACE_FL_TRAMP;
1745

1746 1747 1748 1749 1750 1751
			/*
			 * If any ops wants regs saved for this function
			 * then all ops will get saved regs.
			 */
			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
				rec->flags |= FTRACE_FL_REGS;
1752
		} else {
1753
			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1754
				return false;
1755
			rec->flags--;
1756

1757 1758 1759 1760 1761 1762
			/*
			 * If the rec had REGS enabled and the ops that is
			 * being removed had REGS set, then see if there is
			 * still any ops for this record that wants regs.
			 * If not, we can stop recording them.
			 */
1763
			if (ftrace_rec_count(rec) > 0 &&
1764 1765 1766 1767 1768
			    rec->flags & FTRACE_FL_REGS &&
			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
				if (!test_rec_ops_needs_regs(rec))
					rec->flags &= ~FTRACE_FL_REGS;
			}
1769

1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780
			/*
			 * If the rec had TRAMP enabled, then it needs to
			 * be cleared. As TRAMP can only be enabled iff
			 * there is only a single ops attached to it.
			 * In otherwords, always disable it on decrementing.
			 * In the future, we may set it if rec count is
			 * decremented to one, and the ops that is left
			 * has a trampoline.
			 */
			rec->flags &= ~FTRACE_FL_TRAMP;

1781 1782 1783 1784
			/*
			 * flags will be cleared in ftrace_check_record()
			 * if rec count is zero.
			 */
1785 1786
		}
		count++;
1787 1788 1789 1790

		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;

1791 1792
		/* Shortcut, if we handled all records, we are done. */
		if (!all && count == hash->count)
1793
			return update;
1794
	} while_for_each_ftrace_rec();
1795 1796

	return update;
1797 1798
}

1799
static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1800 1801
				    int filter_hash)
{
1802
	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1803 1804
}

1805
static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1806 1807
				   int filter_hash)
{
1808
	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1809 1810
}

1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
					  int filter_hash, int inc)
{
	struct ftrace_ops *op;

	__ftrace_hash_rec_update(ops, filter_hash, inc);

	if (ops->func_hash != &global_ops.local_hash)
		return;

	/*
	 * If the ops shares the global_ops hash, then we need to update
	 * all ops that are enabled and use this hash.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* Already done */
		if (op == ops)
			continue;
		if (op->func_hash == &global_ops.local_hash)
			__ftrace_hash_rec_update(op, filter_hash, inc);
	} while_for_each_ftrace_op(op);
}

static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
					   int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}

static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
					  int filter_hash)
{
	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}

1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
/*
 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
 * or no-needed to update, -EBUSY if it detects a conflict of the flag
 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
 * Note that old_hash and new_hash has below meanings
 *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
 *  - If the hash is EMPTY_HASH, it hits nothing
 *  - Anything else hits the recs which match the hash entries.
 */
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
					 struct ftrace_hash *old_hash,
					 struct ftrace_hash *new_hash)
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec, *end = NULL;
	int in_old, in_new;

	/* Only update if the ops has been registered */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
		return 0;

	/*
	 * Since the IPMODIFY is a very address sensitive action, we do not
	 * allow ftrace_ops to set all functions to new hash.
	 */
	if (!new_hash || !old_hash)
		return -EINVAL;

	/* Update rec->flags */
	do_for_each_ftrace_rec(pg, rec) {
1879 1880 1881 1882

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
		/* We need to update only differences of filter_hash */
		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new) {
			/* New entries must ensure no others are using it */
			if (rec->flags & FTRACE_FL_IPMODIFY)
				goto rollback;
			rec->flags |= FTRACE_FL_IPMODIFY;
		} else /* Removed entry */
			rec->flags &= ~FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

	return 0;

rollback:
	end = rec;

	/* Roll back what we did above */
	do_for_each_ftrace_rec(pg, rec) {
1905 1906 1907 1908

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
		if (rec == end)
			goto err_out;

		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
		if (in_old == in_new)
			continue;

		if (in_new)
			rec->flags &= ~FTRACE_FL_IPMODIFY;
		else
			rec->flags |= FTRACE_FL_IPMODIFY;
	} while_for_each_ftrace_rec();

err_out:
	return -EBUSY;
}

static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}

/* Disabling always succeeds */
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
	struct ftrace_hash *hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(hash))
		hash = NULL;

	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}

static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
				       struct ftrace_hash *new_hash)
{
	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;

	if (ftrace_hash_empty(old_hash))
		old_hash = NULL;

	if (ftrace_hash_empty(new_hash))
		new_hash = NULL;

	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}

1962
static void print_ip_ins(const char *fmt, const unsigned char *p)
1963 1964 1965 1966 1967 1968 1969 1970 1971
{
	int i;

	printk(KERN_CONT "%s", fmt);

	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}

1972 1973
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1974 1975
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1976

1977
enum ftrace_bug_type ftrace_bug_type;
1978
const void *ftrace_expected;
1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999

static void print_bug_type(void)
{
	switch (ftrace_bug_type) {
	case FTRACE_BUG_UNKNOWN:
		break;
	case FTRACE_BUG_INIT:
		pr_info("Initializing ftrace call sites\n");
		break;
	case FTRACE_BUG_NOP:
		pr_info("Setting ftrace call site to NOP\n");
		break;
	case FTRACE_BUG_CALL:
		pr_info("Setting ftrace call site to call ftrace function\n");
		break;
	case FTRACE_BUG_UPDATE:
		pr_info("Updating ftrace call site to call a different ftrace function\n");
		break;
	}
}

2000 2001 2002
/**
 * ftrace_bug - report and shutdown function tracer
 * @failed: The failed type (EFAULT, EINVAL, EPERM)
2003
 * @rec: The record that failed
2004 2005 2006 2007 2008 2009 2010 2011
 *
 * The arch code that enables or disables the function tracing
 * can call ftrace_bug() when it has detected a problem in
 * modifying the code. @failed should be one of either:
 * EFAULT - if the problem happens on reading the @ip address
 * EINVAL - if what is read at @ip is not what was expected
 * EPERM - if the problem happens on writting to the @ip address
 */
2012
void ftrace_bug(int failed, struct dyn_ftrace *rec)
2013
{
2014 2015
	unsigned long ip = rec ? rec->ip : 0;

2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
	switch (failed) {
	case -EFAULT:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on modifying ");
		print_ip_sym(ip);
		break;
	case -EINVAL:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace failed to modify ");
		print_ip_sym(ip);
2026
		print_ip_ins(" actual:   ", (unsigned char *)ip);
2027
		pr_cont("\n");
2028 2029 2030 2031
		if (ftrace_expected) {
			print_ip_ins(" expected: ", ftrace_expected);
			pr_cont("\n");
		}
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042
		break;
	case -EPERM:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on writing ");
		print_ip_sym(ip);
		break;
	default:
		FTRACE_WARN_ON_ONCE(1);
		pr_info("ftrace faulted on unknown error ");
		print_ip_sym(ip);
	}
2043
	print_bug_type();
2044 2045 2046 2047 2048 2049 2050 2051
	if (rec) {
		struct ftrace_ops *ops = NULL;

		pr_info("ftrace record flags: %lx\n", rec->flags);
		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
			ops = ftrace_find_tramp_ops_any(rec);
2052 2053 2054 2055 2056 2057 2058 2059
			if (ops) {
				do {
					pr_cont("\ttramp: %pS (%pS)",
						(void *)ops->trampoline,
						(void *)ops->func);
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
2060 2061 2062 2063
				pr_cont("\ttramp: ERROR!");

		}
		ip = ftrace_get_addr_curr(rec);
2064
		pr_cont("\n expected tramp: %lx\n", ip);
2065
	}
2066 2067
}

2068
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2069
{
2070
	unsigned long flag = 0UL;
2071

2072 2073
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2074 2075 2076
	if (rec->flags & FTRACE_FL_DISABLED)
		return FTRACE_UPDATE_IGNORE;

S
Steven Rostedt 已提交
2077
	/*
2078
	 * If we are updating calls:
S
Steven Rostedt 已提交
2079
	 *
2080 2081
	 *   If the record has a ref count, then we need to enable it
	 *   because someone is using it.
S
Steven Rostedt 已提交
2082
	 *
2083 2084
	 *   Otherwise we make sure its disabled.
	 *
2085
	 * If we are disabling calls, then disable all records that
2086
	 * are enabled.
S
Steven Rostedt 已提交
2087
	 */
2088
	if (enable && ftrace_rec_count(rec))
2089
		flag = FTRACE_FL_ENABLED;
S
Steven Rostedt 已提交
2090

2091
	/*
2092 2093 2094
	 * If enabling and the REGS flag does not match the REGS_EN, or
	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
	 * this record. Set flags to fail the compare against ENABLED.
2095
	 */
2096 2097 2098 2099 2100 2101 2102 2103 2104
	if (flag) {
		if (!(rec->flags & FTRACE_FL_REGS) != 
		    !(rec->flags & FTRACE_FL_REGS_EN))
			flag |= FTRACE_FL_REGS;

		if (!(rec->flags & FTRACE_FL_TRAMP) != 
		    !(rec->flags & FTRACE_FL_TRAMP_EN))
			flag |= FTRACE_FL_TRAMP;
	}
2105

2106 2107
	/* If the state of this record hasn't changed, then do nothing */
	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2108
		return FTRACE_UPDATE_IGNORE;
S
Steven Rostedt 已提交
2109

2110
	if (flag) {
2111 2112 2113 2114
		/* Save off if rec is being enabled (for return value) */
		flag ^= rec->flags & FTRACE_FL_ENABLED;

		if (update) {
2115
			rec->flags |= FTRACE_FL_ENABLED;
2116 2117 2118 2119 2120 2121
			if (flag & FTRACE_FL_REGS) {
				if (rec->flags & FTRACE_FL_REGS)
					rec->flags |= FTRACE_FL_REGS_EN;
				else
					rec->flags &= ~FTRACE_FL_REGS_EN;
			}
2122 2123 2124 2125 2126 2127
			if (flag & FTRACE_FL_TRAMP) {
				if (rec->flags & FTRACE_FL_TRAMP)
					rec->flags |= FTRACE_FL_TRAMP_EN;
				else
					rec->flags &= ~FTRACE_FL_TRAMP_EN;
			}
2128 2129 2130 2131 2132 2133 2134
		}

		/*
		 * If this record is being updated from a nop, then
		 *   return UPDATE_MAKE_CALL.
		 * Otherwise,
		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2135
		 *   from the save regs, to a non-save regs function or
2136
		 *   vice versa, or from a trampoline call.
2137
		 */
2138 2139
		if (flag & FTRACE_FL_ENABLED) {
			ftrace_bug_type = FTRACE_BUG_CALL;
2140
			return FTRACE_UPDATE_MAKE_CALL;
2141
		}
2142

2143
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2144
		return FTRACE_UPDATE_MODIFY_CALL;
2145 2146
	}

2147 2148
	if (update) {
		/* If there's no more users, clear all flags */
2149
		if (!ftrace_rec_count(rec))
2150 2151
			rec->flags = 0;
		else
2152 2153 2154 2155 2156 2157
			/*
			 * Just disable the record, but keep the ops TRAMP
			 * and REGS states. The _EN flags must be disabled though.
			 */
			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
					FTRACE_FL_REGS_EN);
2158
	}
2159

2160
	ftrace_bug_type = FTRACE_BUG_NOP;
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
	return FTRACE_UPDATE_MAKE_NOP;
}

/**
 * ftrace_update_record, set a record that now is tracing or not
 * @rec: the record to update
 * @enable: set to 1 if the record is tracing, zero to force disable
 *
 * The records that represent all functions that can be traced need
 * to be updated when tracing has been enabled.
 */
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 1);
}

/**
 * ftrace_test_record, check if the record has been enabled or not
 * @rec: the record to test
 * @enable: set to 1 to check if enabled, 0 if it is disabled
 *
 * The arch code may need to test if a record is already set to
 * tracing to determine how to modify the function code that it
 * represents.
 */
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
	return ftrace_check_record(rec, enable, 0);
}

2191 2192 2193 2194
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2195
	unsigned long ip = rec->ip;
2196 2197 2198 2199 2200 2201

	do_for_each_ftrace_op(op, ftrace_ops_list) {

		if (!op->trampoline)
			continue;

2202
		if (hash_contains_ip(ip, op->func_hash))
2203 2204 2205 2206 2207 2208
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
			   struct ftrace_ops *op)
{
	unsigned long ip = rec->ip;

	while_for_each_ftrace_op(op) {

		if (!op->trampoline)
			continue;

		if (hash_contains_ip(ip, op->func_hash))
			return op;
	} 

	return NULL;
}

2227 2228 2229 2230
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2231
	unsigned long ip = rec->ip;
2232

2233 2234 2235 2236 2237 2238 2239 2240
	/*
	 * Need to check removed ops first.
	 * If they are being removed, and this rec has a tramp,
	 * and this rec is in the ops list, then it would be the
	 * one with the tramp.
	 */
	if (removed_ops) {
		if (hash_contains_ip(ip, &removed_ops->old_hash))
2241 2242 2243
			return removed_ops;
	}

2244 2245 2246 2247 2248 2249 2250
	/*
	 * Need to find the current trampoline for a rec.
	 * Now, a trampoline is only attached to a rec if there
	 * was a single 'ops' attached to it. But this can be called
	 * when we are adding another op to the rec or removing the
	 * current one. Thus, if the op is being added, we can
	 * ignore it because it hasn't attached itself to the rec
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
	 * yet.
	 *
	 * If an ops is being modified (hooking to different functions)
	 * then we don't care about the new functions that are being
	 * added, just the old ones (that are probably being removed).
	 *
	 * If we are adding an ops to a function that already is using
	 * a trampoline, it needs to be removed (trampolines are only
	 * for single ops connected), then an ops that is not being
	 * modified also needs to be checked.
2261
	 */
2262
	do_for_each_ftrace_op(op, ftrace_ops_list) {
2263 2264 2265 2266 2267 2268 2269 2270 2271

		if (!op->trampoline)
			continue;

		/*
		 * If the ops is being added, it hasn't gotten to
		 * the point to be removed from this tree yet.
		 */
		if (op->flags & FTRACE_OPS_FL_ADDING)
2272 2273
			continue;

2274

2275
		/*
2276 2277 2278
		 * If the ops is being modified and is in the old
		 * hash, then it is probably being removed from this
		 * function.
2279 2280 2281
		 */
		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, &op->old_hash))
2282
			return op;
2283 2284 2285 2286 2287 2288 2289 2290
		/*
		 * If the ops is not being added or modified, and it's
		 * in its normal filter hash, then this must be the one
		 * we want!
		 */
		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
		    hash_contains_ip(ip, op->func_hash))
			return op;
2291 2292 2293 2294 2295 2296 2297 2298 2299 2300

	} while_for_each_ftrace_op(op);

	return NULL;
}

static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
	struct ftrace_ops *op;
2301
	unsigned long ip = rec->ip;
2302 2303 2304

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		/* pass rec in as regs to have non-NULL val */
2305
		if (hash_contains_ip(ip, op->func_hash))
2306 2307 2308 2309 2310 2311
			return op;
	} while_for_each_ftrace_op(op);

	return NULL;
}

2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
/**
 * ftrace_get_addr_new - Get the call address to set to
 * @rec:  The ftrace record descriptor
 *
 * If the record has the FTRACE_FL_REGS set, that means that it
 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
 * is not not set, then it wants to convert to the normal callback.
 *
 * Returns the address of the trampoline to set to
 */
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
2324 2325 2326 2327 2328 2329
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP) {
		ops = ftrace_find_tramp_ops_new(rec);
		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2330 2331
			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
				(void *)rec->ip, (void *)rec->ip, rec->flags);
2332 2333 2334 2335 2336 2337
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355
	if (rec->flags & FTRACE_FL_REGS)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

/**
 * ftrace_get_addr_curr - Get the call address that is already there
 * @rec:  The ftrace record descriptor
 *
 * The FTRACE_FL_REGS_EN is set when the record already points to
 * a function that saves all the regs. Basically the '_EN' version
 * represents the current state of the function.
 *
 * Returns the address of the trampoline that is currently being called
 */
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
2356 2357 2358 2359 2360 2361
	struct ftrace_ops *ops;

	/* Trampolines take precedence over regs */
	if (rec->flags & FTRACE_FL_TRAMP_EN) {
		ops = ftrace_find_tramp_ops_curr(rec);
		if (FTRACE_WARN_ON(!ops)) {
2362 2363
			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
				(void *)rec->ip, (void *)rec->ip);
2364 2365 2366 2367 2368 2369
			/* Ftrace is shutting down, return anything */
			return (unsigned long)FTRACE_ADDR;
		}
		return ops->trampoline;
	}

2370 2371 2372 2373 2374 2375
	if (rec->flags & FTRACE_FL_REGS_EN)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

2376 2377 2378
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
2379
	unsigned long ftrace_old_addr;
2380 2381 2382
	unsigned long ftrace_addr;
	int ret;

2383
	ftrace_addr = ftrace_get_addr_new(rec);
2384

2385 2386 2387 2388
	/* This needs to be done before we call ftrace_update_record */
	ftrace_old_addr = ftrace_get_addr_curr(rec);

	ret = ftrace_update_record(rec, enable);
2389

2390 2391
	ftrace_bug_type = FTRACE_BUG_UNKNOWN;

2392 2393 2394 2395 2396
	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
2397
		ftrace_bug_type = FTRACE_BUG_CALL;
2398
		return ftrace_make_call(rec, ftrace_addr);
2399 2400

	case FTRACE_UPDATE_MAKE_NOP:
2401
		ftrace_bug_type = FTRACE_BUG_NOP;
2402
		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2403 2404

	case FTRACE_UPDATE_MODIFY_CALL:
2405
		ftrace_bug_type = FTRACE_BUG_UPDATE;
2406
		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2407 2408
	}

2409
	return -1; /* unknow ftrace bug */
2410 2411
}

2412
void __weak ftrace_replace_code(int enable)
2413 2414 2415
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
S
Steven Rostedt 已提交
2416
	int failed;
2417

2418 2419 2420
	if (unlikely(ftrace_disabled))
		return;

2421
	do_for_each_ftrace_rec(pg, rec) {
2422 2423 2424 2425

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

2426
		failed = __ftrace_replace_code(rec, enable);
2427
		if (failed) {
2428
			ftrace_bug(failed, rec);
2429 2430
			/* Stop processing */
			return;
2431
		}
2432
	} while_for_each_ftrace_rec();
2433 2434
}

2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506
struct ftrace_rec_iter {
	struct ftrace_page	*pg;
	int			index;
};

/**
 * ftrace_rec_iter_start, start up iterating over traced functions
 *
 * Returns an iterator handle that is used to iterate over all
 * the records that represent address locations where functions
 * are traced.
 *
 * May return NULL if no records are available.
 */
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
	/*
	 * We only use a single iterator.
	 * Protected by the ftrace_lock mutex.
	 */
	static struct ftrace_rec_iter ftrace_rec_iter;
	struct ftrace_rec_iter *iter = &ftrace_rec_iter;

	iter->pg = ftrace_pages_start;
	iter->index = 0;

	/* Could have empty pages */
	while (iter->pg && !iter->pg->index)
		iter->pg = iter->pg->next;

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_next, get the next record to process.
 * @iter: The handle to the iterator.
 *
 * Returns the next iterator after the given iterator @iter.
 */
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
	iter->index++;

	if (iter->index >= iter->pg->index) {
		iter->pg = iter->pg->next;
		iter->index = 0;

		/* Could have empty pages */
		while (iter->pg && !iter->pg->index)
			iter->pg = iter->pg->next;
	}

	if (!iter->pg)
		return NULL;

	return iter;
}

/**
 * ftrace_rec_iter_record, get the record at the iterator location
 * @iter: The current iterator location
 *
 * Returns the record that the current @iter is at.
 */
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
	return &iter->pg->records[iter->index];
}

2507
static int
2508
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2509
{
2510
	int ret;
2511

2512 2513 2514
	if (unlikely(ftrace_disabled))
		return 0;

2515
	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2516
	if (ret) {
2517
		ftrace_bug_type = FTRACE_BUG_INIT;
2518
		ftrace_bug(ret, rec);
2519
		return 0;
2520
	}
2521
	return 1;
2522 2523
}

2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
/*
 * archs can override this function if they must do something
 * before the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_prepare(void)
{
	return 0;
}

/*
 * archs can override this function if they must do something
 * after the modifying code is performed.
 */
int __weak ftrace_arch_code_modify_post_process(void)
{
	return 0;
}

2542
void ftrace_modify_all_code(int command)
2543
{
2544
	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2545
	int err = 0;
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556

	/*
	 * If the ftrace_caller calls a ftrace_ops func directly,
	 * we need to make sure that it only traces functions it
	 * expects to trace. When doing the switch of functions,
	 * we need to update to the ftrace_ops_list_func first
	 * before the transition between old and new calls are set,
	 * as the ftrace_ops_list_func will check the ops hashes
	 * to make sure the ops are having the right functions
	 * traced.
	 */
2557 2558 2559 2560 2561
	if (update) {
		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
		if (FTRACE_WARN_ON(err))
			return;
	}
2562

2563
	if (command & FTRACE_UPDATE_CALLS)
2564
		ftrace_replace_code(1);
2565
	else if (command & FTRACE_DISABLE_CALLS)
2566 2567
		ftrace_replace_code(0);

2568 2569 2570 2571 2572 2573
	if (update && ftrace_trace_function != ftrace_ops_list_func) {
		function_trace_op = set_function_trace_op;
		smp_wmb();
		/* If irqs are disabled, we are in stop machine */
		if (!irqs_disabled())
			smp_call_function(ftrace_sync_ipi, NULL, 1);
2574 2575 2576
		err = ftrace_update_ftrace_func(ftrace_trace_function);
		if (FTRACE_WARN_ON(err))
			return;
2577
	}
2578

2579
	if (command & FTRACE_START_FUNC_RET)
2580
		err = ftrace_enable_ftrace_graph_caller();
2581
	else if (command & FTRACE_STOP_FUNC_RET)
2582 2583
		err = ftrace_disable_ftrace_graph_caller();
	FTRACE_WARN_ON(err);
2584 2585 2586 2587 2588 2589 2590
}

static int __ftrace_modify_code(void *data)
{
	int *command = data;

	ftrace_modify_all_code(*command);
2591

2592
	return 0;
2593 2594
}

2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618
/**
 * ftrace_run_stop_machine, go back to the stop machine method
 * @command: The command to tell ftrace what to do
 *
 * If an arch needs to fall back to the stop machine method, the
 * it can call this function.
 */
void ftrace_run_stop_machine(int command)
{
	stop_machine(__ftrace_modify_code, &command, NULL);
}

/**
 * arch_ftrace_update_code, modify the code to trace or not trace
 * @command: The command that needs to be done
 *
 * Archs can override this function if it does not need to
 * run stop_machine() to modify code.
 */
void __weak arch_ftrace_update_code(int command)
{
	ftrace_run_stop_machine(command);
}

I
Ingo Molnar 已提交
2619
static void ftrace_run_update_code(int command)
2620
{
2621 2622 2623 2624 2625 2626 2627
	int ret;

	ret = ftrace_arch_code_modify_prepare();
	FTRACE_WARN_ON(ret);
	if (ret)
		return;

2628 2629 2630 2631 2632 2633 2634 2635
	/*
	 * By default we use stop_machine() to modify the code.
	 * But archs can do what ever they want as long as it
	 * is safe. The stop_machine() is the safest, but also
	 * produces the most overhead.
	 */
	arch_ftrace_update_code(command);

2636 2637
	ret = ftrace_arch_code_modify_post_process();
	FTRACE_WARN_ON(ret);
2638 2639
}

2640
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2641
				   struct ftrace_ops_hash *old_hash)
2642 2643
{
	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2644 2645
	ops->old_hash.filter_hash = old_hash->filter_hash;
	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2646
	ftrace_run_update_code(command);
2647
	ops->old_hash.filter_hash = NULL;
2648
	ops->old_hash.notrace_hash = NULL;
2649 2650 2651
	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}

2652
static ftrace_func_t saved_ftrace_func;
2653
static int ftrace_start_up;
2654

2655 2656 2657 2658
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}

2659
static void per_cpu_ops_free(struct ftrace_ops *ops)
2660 2661 2662 2663
{
	free_percpu(ops->disabled);
}

2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675
static void ftrace_startup_enable(int command)
{
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}

	if (!command || !ftrace_enabled)
		return;

	ftrace_run_update_code(command);
}
2676

2677 2678 2679 2680 2681 2682 2683
static void ftrace_startup_all(int command)
{
	update_all_ops = true;
	ftrace_startup_enable(command);
	update_all_ops = false;
}

2684
static int ftrace_startup(struct ftrace_ops *ops, int command)
2685
{
2686
	int ret;
2687

2688
	if (unlikely(ftrace_disabled))
2689
		return -ENODEV;
2690

2691 2692 2693 2694
	ret = __register_ftrace_function(ops);
	if (ret)
		return ret;

2695
	ftrace_start_up++;
2696

2697 2698 2699 2700 2701 2702 2703 2704 2705
	/*
	 * Note that ftrace probes uses this to start up
	 * and modify functions it will probe. But we still
	 * set the ADDING flag for modification, as probes
	 * do not have trampolines. If they add them in the
	 * future, then the probes will need to distinguish
	 * between adding and updating probes.
	 */
	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2706

2707 2708 2709 2710 2711 2712 2713 2714 2715
	ret = ftrace_hash_ipmodify_enable(ops);
	if (ret < 0) {
		/* Rollback registration process */
		__unregister_ftrace_function(ops);
		ftrace_start_up--;
		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
		return ret;
	}

2716 2717
	if (ftrace_hash_rec_enable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2718

2719
	ftrace_startup_enable(command);
2720

2721 2722
	ops->flags &= ~FTRACE_OPS_FL_ADDING;

2723
	return 0;
2724 2725
}

2726
static int ftrace_shutdown(struct ftrace_ops *ops, int command)
2727
{
2728
	int ret;
2729

2730
	if (unlikely(ftrace_disabled))
2731 2732 2733 2734 2735
		return -ENODEV;

	ret = __unregister_ftrace_function(ops);
	if (ret)
		return ret;
2736

2737
	ftrace_start_up--;
2738 2739 2740 2741 2742 2743 2744
	/*
	 * Just warn in case of unbalance, no need to kill ftrace, it's not
	 * critical but the ftrace_call callers may be never nopped again after
	 * further ftrace uses.
	 */
	WARN_ON_ONCE(ftrace_start_up < 0);

2745 2746
	/* Disabling ipmodify never fails */
	ftrace_hash_ipmodify_disable(ops);
2747

2748 2749
	if (ftrace_hash_rec_disable(ops, 1))
		command |= FTRACE_UPDATE_CALLS;
2750

2751
	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2752

2753 2754 2755 2756
	if (saved_ftrace_func != ftrace_trace_function) {
		saved_ftrace_func = ftrace_trace_function;
		command |= FTRACE_UPDATE_TRACE_FUNC;
	}
2757

2758 2759
	if (!command || !ftrace_enabled) {
		/*
2760
		 * If these are per_cpu ops, they still need their
2761 2762 2763 2764
		 * per_cpu field freed. Since, function tracing is
		 * not currently active, we can just free them
		 * without synchronizing all CPUs.
		 */
2765 2766
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2767
		return 0;
2768
	}
2769

2770 2771 2772 2773
	/*
	 * If the ops uses a trampoline, then it needs to be
	 * tested first on update.
	 */
2774
	ops->flags |= FTRACE_OPS_FL_REMOVING;
2775 2776
	removed_ops = ops;

2777 2778 2779 2780
	/* The trampoline logic checks the old hashes */
	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;

2781
	ftrace_run_update_code(command);
2782

2783 2784 2785 2786 2787 2788 2789 2790 2791
	/*
	 * If there's no more ops registered with ftrace, run a
	 * sanity check to make sure all rec flags are cleared.
	 */
	if (ftrace_ops_list == &ftrace_list_end) {
		struct ftrace_page *pg;
		struct dyn_ftrace *rec;

		do_for_each_ftrace_rec(pg, rec) {
2792
			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2793 2794 2795 2796 2797
				pr_warn("  %pS flags:%lx\n",
					(void *)rec->ip, rec->flags);
		} while_for_each_ftrace_rec();
	}

2798 2799 2800 2801
	ops->old_hash.filter_hash = NULL;
	ops->old_hash.notrace_hash = NULL;

	removed_ops = NULL;
2802
	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2803

2804 2805 2806
	/*
	 * Dynamic ops may be freed, we must make sure that all
	 * callers are done before leaving this function.
2807
	 * The same goes for freeing the per_cpu data of the per_cpu
2808 2809
	 * ops.
	 */
2810
	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU)) {
2811 2812 2813 2814 2815 2816 2817 2818
		/*
		 * We need to do a hard force of sched synchronization.
		 * This is because we use preempt_disable() to do RCU, but
		 * the function tracers can be called where RCU is not watching
		 * (like before user_exit()). We can not rely on the RCU
		 * infrastructure to do the synchronization, thus we must do it
		 * ourselves.
		 */
2819 2820
		schedule_on_each_cpu(ftrace_sync);

2821 2822 2823 2824 2825 2826 2827 2828 2829 2830
		/*
		 * When the kernel is preeptive, tasks can be preempted
		 * while on a ftrace trampoline. Just scheduling a task on
		 * a CPU is not good enough to flush them. Calling
		 * synchornize_rcu_tasks() will wait for those tasks to
		 * execute and either schedule voluntarily or enter user space.
		 */
		if (IS_ENABLED(CONFIG_PREEMPT))
			synchronize_rcu_tasks();

2831 2832
		arch_ftrace_trampoline_free(ops);

2833 2834
		if (ops->flags & FTRACE_OPS_FL_PER_CPU)
			per_cpu_ops_free(ops);
2835 2836
	}

2837
	return 0;
2838 2839
}

I
Ingo Molnar 已提交
2840
static void ftrace_startup_sysctl(void)
2841
{
2842 2843
	int command;

2844 2845 2846
	if (unlikely(ftrace_disabled))
		return;

2847 2848
	/* Force update next time */
	saved_ftrace_func = NULL;
2849
	/* ftrace_start_up is true if we want ftrace running */
2850 2851 2852 2853
	if (ftrace_start_up) {
		command = FTRACE_UPDATE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_START_FUNC_RET;
2854
		ftrace_startup_enable(command);
2855
	}
2856 2857
}

I
Ingo Molnar 已提交
2858
static void ftrace_shutdown_sysctl(void)
2859
{
2860 2861
	int command;

2862 2863 2864
	if (unlikely(ftrace_disabled))
		return;

2865
	/* ftrace_start_up is true if ftrace is running */
2866 2867 2868 2869 2870 2871
	if (ftrace_start_up) {
		command = FTRACE_DISABLE_CALLS;
		if (ftrace_graph_active)
			command |= FTRACE_STOP_FUNC_RET;
		ftrace_run_update_code(command);
	}
2872 2873
}

2874
static u64		ftrace_update_time;
2875 2876
unsigned long		ftrace_update_tot_cnt;

2877
static inline int ops_traces_mod(struct ftrace_ops *ops)
2878
{
2879 2880 2881 2882
	/*
	 * Filter_hash being empty will default to trace module.
	 * But notrace hash requires a test of individual module functions.
	 */
2883 2884
	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
		ftrace_hash_empty(ops->func_hash->notrace_hash);
2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900
}

/*
 * Check if the current ops references the record.
 *
 * If the ops traces all functions, then it was already accounted for.
 * If the ops does not trace the current record function, skip it.
 * If the ops ignores the function via notrace filter, skip it.
 */
static inline bool
ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	/* If ops isn't enabled, ignore it */
	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
		return 0;

2901
	/* If ops traces all then it includes this function */
2902
	if (ops_traces_mod(ops))
2903
		return 1;
2904 2905

	/* The function must be in the filter */
2906
	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2907
	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2908
		return 0;
2909

2910
	/* If in notrace hash, we ignore it too */
2911
	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2912 2913 2914 2915 2916
		return 0;

	return 1;
}

2917
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2918
{
2919
	struct ftrace_page *pg;
2920
	struct dyn_ftrace *p;
2921
	u64 start, stop;
2922
	unsigned long update_cnt = 0;
2923
	unsigned long rec_flags = 0;
2924
	int i;
2925

2926 2927
	start = ftrace_now(raw_smp_processor_id());

2928
	/*
2929 2930 2931 2932 2933 2934 2935 2936 2937
	 * When a module is loaded, this function is called to convert
	 * the calls to mcount in its text to nops, and also to create
	 * an entry in the ftrace data. Now, if ftrace is activated
	 * after this call, but before the module sets its text to
	 * read-only, the modification of enabling ftrace can fail if
	 * the read-only is done while ftrace is converting the calls.
	 * To prevent this, the module's records are set as disabled
	 * and will be enabled after the call to set the module's text
	 * to read-only.
2938
	 */
2939 2940
	if (mod)
		rec_flags |= FTRACE_FL_DISABLED;
2941

2942
	for (pg = new_pgs; pg; pg = pg->next) {
2943

2944
		for (i = 0; i < pg->index; i++) {
2945

2946 2947 2948
			/* If something went wrong, bail without enabling anything */
			if (unlikely(ftrace_disabled))
				return -1;
2949

2950
			p = &pg->records[i];
2951
			p->flags = rec_flags;
2952

2953 2954 2955 2956 2957 2958
			/*
			 * Do the initial record conversion from mcount jump
			 * to the NOP instructions.
			 */
			if (!ftrace_code_disable(mod, p))
				break;
2959

2960
			update_cnt++;
2961
		}
2962 2963
	}

I
Ingo Molnar 已提交
2964
	stop = ftrace_now(raw_smp_processor_id());
2965
	ftrace_update_time = stop - start;
2966
	ftrace_update_tot_cnt += update_cnt;
2967

2968 2969 2970
	return 0;
}

2971
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2972
{
2973
	int order;
2974 2975
	int cnt;

2976 2977 2978 2979
	if (WARN_ON(!count))
		return -EINVAL;

	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2980 2981

	/*
2982 2983
	 * We want to fill as much as possible. No more than a page
	 * may be empty.
2984
	 */
2985 2986
	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
		order--;
2987

2988 2989
 again:
	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2990

2991 2992 2993 2994 2995 2996 2997
	if (!pg->records) {
		/* if we can't allocate this size, try something smaller */
		if (!order)
			return -ENOMEM;
		order >>= 1;
		goto again;
	}
2998

2999 3000
	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
	pg->size = cnt;
3001

3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034
	if (cnt > count)
		cnt = count;

	return cnt;
}

static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
	struct ftrace_page *start_pg;
	struct ftrace_page *pg;
	int order;
	int cnt;

	if (!num_to_init)
		return 0;

	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
	if (!pg)
		return NULL;

	/*
	 * Try to allocate as much as possible in one continues
	 * location that fills in all of the space. We want to
	 * waste as little space as possible.
	 */
	for (;;) {
		cnt = ftrace_allocate_records(pg, num_to_init);
		if (cnt < 0)
			goto free_pages;

		num_to_init -= cnt;
		if (!num_to_init)
3035 3036
			break;

3037 3038 3039 3040
		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
		if (!pg->next)
			goto free_pages;

3041 3042 3043
		pg = pg->next;
	}

3044 3045 3046
	return start_pg;

 free_pages:
3047 3048
	pg = start_pg;
	while (pg) {
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058
		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
		free_pages((unsigned long)pg->records, order);
		start_pg = pg->next;
		kfree(pg);
		pg = start_pg;
	}
	pr_info("ftrace: FAILED to allocate memory for functions\n");
	return NULL;
}

3059 3060 3061
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */

struct ftrace_iterator {
3062
	loff_t				pos;
3063 3064 3065 3066 3067
	loff_t				func_pos;
	struct ftrace_page		*pg;
	struct dyn_ftrace		*func;
	struct ftrace_func_probe	*probe;
	struct trace_parser		parser;
3068
	struct ftrace_hash		*hash;
3069
	struct ftrace_ops		*ops;
3070 3071 3072
	int				hidx;
	int				idx;
	unsigned			flags;
3073 3074
};

3075
static void *
3076
t_hash_next(struct seq_file *m, loff_t *pos)
3077 3078
{
	struct ftrace_iterator *iter = m->private;
3079
	struct hlist_node *hnd = NULL;
3080 3081 3082
	struct hlist_head *hhd;

	(*pos)++;
3083
	iter->pos = *pos;
3084

3085 3086
	if (iter->probe)
		hnd = &iter->probe->node;
3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108
 retry:
	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
		return NULL;

	hhd = &ftrace_func_hash[iter->hidx];

	if (hlist_empty(hhd)) {
		iter->hidx++;
		hnd = NULL;
		goto retry;
	}

	if (!hnd)
		hnd = hhd->first;
	else {
		hnd = hnd->next;
		if (!hnd) {
			iter->hidx++;
			goto retry;
		}
	}

3109 3110 3111 3112 3113 3114
	if (WARN_ON_ONCE(!hnd))
		return NULL;

	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);

	return iter;
3115 3116 3117 3118 3119 3120
}

static void *t_hash_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
L
Li Zefan 已提交
3121 3122
	loff_t l;

3123 3124 3125
	if (!(iter->flags & FTRACE_ITER_DO_HASH))
		return NULL;

3126 3127
	if (iter->func_pos > *pos)
		return NULL;
3128

L
Li Zefan 已提交
3129
	iter->hidx = 0;
3130
	for (l = 0; l <= (*pos - iter->func_pos); ) {
3131
		p = t_hash_next(m, &l);
L
Li Zefan 已提交
3132 3133 3134
		if (!p)
			break;
	}
3135 3136 3137
	if (!p)
		return NULL;

3138 3139 3140
	/* Only set this if we have an item */
	iter->flags |= FTRACE_ITER_HASH;

3141
	return iter;
3142 3143
}

3144 3145
static int
t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
3146
{
S
Steven Rostedt 已提交
3147
	struct ftrace_func_probe *rec;
3148

3149 3150 3151
	rec = iter->probe;
	if (WARN_ON_ONCE(!rec))
		return -EIO;
3152

3153
	if (rec->ops->print)
3154
		return rec->ops->print(m, rec->ip, rec->ops, NULL);
3155

3156
	seq_printf(m, "%ps:%ps\n", (void *)rec->ip, (void *)rec->ops->func);
3157 3158 3159 3160

	return 0;
}

I
Ingo Molnar 已提交
3161
static void *
3162
t_func_next(struct seq_file *m, loff_t *pos)
3163 3164 3165 3166 3167
{
	struct ftrace_iterator *iter = m->private;
	struct dyn_ftrace *rec = NULL;

	(*pos)++;
3168

3169 3170 3171 3172 3173 3174 3175 3176 3177
 retry:
	if (iter->idx >= iter->pg->index) {
		if (iter->pg->next) {
			iter->pg = iter->pg->next;
			iter->idx = 0;
			goto retry;
		}
	} else {
		rec = &iter->pg->records[iter->idx++];
3178 3179
		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3180 3181

		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3182
		     !(rec->flags & FTRACE_FL_ENABLED))) {
3183

3184 3185 3186 3187 3188
			rec = NULL;
			goto retry;
		}
	}

3189
	if (!rec)
3190
		return NULL;
3191

3192
	iter->pos = iter->func_pos = *pos;
3193 3194 3195
	iter->func = rec;

	return iter;
3196 3197
}

3198 3199 3200 3201
static void *
t_next(struct seq_file *m, void *v, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
3202
	loff_t l = *pos; /* t_hash_start() must use original pos */
3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213
	void *ret;

	if (unlikely(ftrace_disabled))
		return NULL;

	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_next(m, pos);

	if (iter->flags & FTRACE_ITER_PRINTALL) {
		/* next must increment pos, and t_hash_start does not */
		(*pos)++;
3214
		return t_hash_start(m, &l);
3215 3216 3217 3218 3219
	}

	ret = t_func_next(m, pos);

	if (!ret)
3220
		return t_hash_start(m, &l);
3221 3222 3223 3224

	return ret;
}

3225 3226 3227 3228
static void reset_iter_read(struct ftrace_iterator *iter)
{
	iter->pos = 0;
	iter->func_pos = 0;
3229
	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
3230 3231 3232 3233 3234 3235
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
	struct ftrace_iterator *iter = m->private;
	void *p = NULL;
3236
	loff_t l;
3237

3238
	mutex_lock(&ftrace_lock);
3239 3240 3241 3242

	if (unlikely(ftrace_disabled))
		return NULL;

3243 3244 3245 3246 3247 3248
	/*
	 * If an lseek was done, then reset and start from beginning.
	 */
	if (*pos < iter->pos)
		reset_iter_read(iter);

3249 3250 3251 3252 3253
	/*
	 * For set_ftrace_filter reading, if we have the filter
	 * off, we can short cut and just print out that all
	 * functions are enabled.
	 */
3254 3255
	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
	    ftrace_hash_empty(iter->hash)) {
3256
		iter->func_pos = 1; /* Account for the message */
3257
		if (*pos > 0)
3258
			return t_hash_start(m, pos);
3259
		iter->flags |= FTRACE_ITER_PRINTALL;
3260 3261
		/* reset in case of seek/pread */
		iter->flags &= ~FTRACE_ITER_HASH;
3262 3263 3264
		return iter;
	}

3265 3266 3267
	if (iter->flags & FTRACE_ITER_HASH)
		return t_hash_start(m, pos);

3268 3269 3270 3271 3272
	/*
	 * Unfortunately, we need to restart at ftrace_pages_start
	 * every time we let go of the ftrace_mutex. This is because
	 * those pointers can change without the lock.
	 */
3273 3274 3275
	iter->pg = ftrace_pages_start;
	iter->idx = 0;
	for (l = 0; l <= *pos; ) {
3276
		p = t_func_next(m, &l);
3277 3278
		if (!p)
			break;
3279
	}
3280

3281
	if (!p)
3282
		return t_hash_start(m, pos);
3283 3284

	return iter;
3285 3286 3287 3288
}

static void t_stop(struct seq_file *m, void *p)
{
3289
	mutex_unlock(&ftrace_lock);
3290 3291
}

3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	return NULL;
}

static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
				struct dyn_ftrace *rec)
{
	void *ptr;

	ptr = arch_ftrace_trampoline_func(ops, rec);
	if (ptr)
		seq_printf(m, " ->%pS", ptr);
}

3308 3309
static int t_show(struct seq_file *m, void *v)
{
3310
	struct ftrace_iterator *iter = m->private;
3311
	struct dyn_ftrace *rec;
3312

3313
	if (iter->flags & FTRACE_ITER_HASH)
3314
		return t_hash_show(m, iter);
3315

3316
	if (iter->flags & FTRACE_ITER_PRINTALL) {
3317
		if (iter->flags & FTRACE_ITER_NOTRACE)
3318
			seq_puts(m, "#### no functions disabled ####\n");
3319
		else
3320
			seq_puts(m, "#### all functions enabled ####\n");
3321 3322 3323
		return 0;
	}

3324 3325
	rec = iter->func;

3326 3327 3328
	if (!rec)
		return 0;

3329
	seq_printf(m, "%ps", (void *)rec->ip);
3330
	if (iter->flags & FTRACE_ITER_ENABLED) {
3331
		struct ftrace_ops *ops;
3332

3333
		seq_printf(m, " (%ld)%s%s",
3334
			   ftrace_rec_count(rec),
3335 3336
			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
3337
		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3338
			ops = ftrace_find_tramp_ops_any(rec);
3339 3340 3341 3342 3343
			if (ops) {
				do {
					seq_printf(m, "\ttramp: %pS (%pS)",
						   (void *)ops->trampoline,
						   (void *)ops->func);
3344
					add_trampoline_func(m, ops, rec);
3345 3346 3347
					ops = ftrace_find_tramp_ops_next(rec, ops);
				} while (ops);
			} else
3348
				seq_puts(m, "\ttramp: ERROR!");
3349 3350
		} else {
			add_trampoline_func(m, NULL, rec);
3351 3352 3353
		}
	}	

3354
	seq_putc(m, '\n');
3355 3356 3357 3358

	return 0;
}

J
James Morris 已提交
3359
static const struct seq_operations show_ftrace_seq_ops = {
3360 3361 3362 3363 3364 3365
	.start = t_start,
	.next = t_next,
	.stop = t_stop,
	.show = t_show,
};

I
Ingo Molnar 已提交
3366
static int
3367 3368 3369 3370
ftrace_avail_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3371 3372 3373
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3374
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3375 3376
	if (!iter)
		return -ENOMEM;
3377

3378 3379 3380 3381
	iter->pg = ftrace_pages_start;
	iter->ops = &global_ops;

	return 0;
3382 3383
}

3384 3385 3386 3387 3388
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
	struct ftrace_iterator *iter;

3389
	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3390 3391
	if (!iter)
		return -ENOMEM;
3392

3393 3394 3395 3396 3397
	iter->pg = ftrace_pages_start;
	iter->flags = FTRACE_ITER_ENABLED;
	iter->ops = &global_ops;

	return 0;
3398 3399
}

3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412
/**
 * ftrace_regex_open - initialize function tracer filter files
 * @ops: The ftrace_ops that hold the hash filters
 * @flag: The type of filter to process
 * @inode: The inode, usually passed in to your open routine
 * @file: The file, usually passed in to your open routine
 *
 * ftrace_regex_open() initializes the filter files for the
 * @ops. Depending on @flag it may process the filter hash or
 * the notrace hash of @ops. With this called from the open
 * routine, you can use ftrace_filter_write() for the write
 * routine if @flag has FTRACE_ITER_FILTER set, or
 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3413
 * tracing_lseek() should be used as the lseek routine, and
3414 3415 3416
 * release must call ftrace_regex_release().
 */
int
3417
ftrace_regex_open(struct ftrace_ops *ops, int flag,
3418
		  struct inode *inode, struct file *file)
3419 3420
{
	struct ftrace_iterator *iter;
3421
	struct ftrace_hash *hash;
3422 3423
	int ret = 0;

3424 3425
	ftrace_ops_init(ops);

3426 3427 3428
	if (unlikely(ftrace_disabled))
		return -ENODEV;

3429 3430 3431 3432
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return -ENOMEM;

3433 3434 3435 3436 3437
	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
		kfree(iter);
		return -ENOMEM;
	}

3438 3439 3440
	iter->ops = ops;
	iter->flags = flag;

3441
	mutex_lock(&ops->func_hash->regex_lock);
3442

3443
	if (flag & FTRACE_ITER_NOTRACE)
3444
		hash = ops->func_hash->notrace_hash;
3445
	else
3446
		hash = ops->func_hash->filter_hash;
3447

3448
	if (file->f_mode & FMODE_WRITE) {
3449 3450 3451 3452 3453 3454 3455
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

		if (file->f_flags & O_TRUNC)
			iter->hash = alloc_ftrace_hash(size_bits);
		else
			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);

3456 3457 3458
		if (!iter->hash) {
			trace_parser_put(&iter->parser);
			kfree(iter);
3459 3460
			ret = -ENOMEM;
			goto out_unlock;
3461
		}
3462 3463
	} else
		iter->hash = hash;
3464

3465 3466 3467 3468 3469 3470 3471
	if (file->f_mode & FMODE_READ) {
		iter->pg = ftrace_pages_start;

		ret = seq_open(file, &show_ftrace_seq_ops);
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = iter;
3472
		} else {
3473 3474
			/* Failed */
			free_ftrace_hash(iter->hash);
3475
			trace_parser_put(&iter->parser);
3476
			kfree(iter);
3477
		}
3478 3479
	} else
		file->private_data = iter;
3480 3481

 out_unlock:
3482
	mutex_unlock(&ops->func_hash->regex_lock);
3483 3484 3485 3486

	return ret;
}

3487 3488 3489
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
3490 3491 3492
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops,
3493 3494
			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
			inode, file);
3495 3496 3497 3498 3499
}

static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
3500 3501 3502
	struct ftrace_ops *ops = inode->i_private;

	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3503
				 inode, file);
3504 3505
}

3506 3507 3508 3509 3510 3511 3512
/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
struct ftrace_glob {
	char *search;
	unsigned len;
	int type;
};

3513 3514 3515 3516 3517 3518 3519 3520 3521 3522
/*
 * If symbols in an architecture don't correspond exactly to the user-visible
 * name of what they represent, it is possible to define this function to
 * perform the necessary adjustments.
*/
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
	return str;
}

3523
static int ftrace_match(char *str, struct ftrace_glob *g)
3524 3525
{
	int matched = 0;
3526
	int slen;
3527

3528 3529
	str = arch_ftrace_match_adjust(str, g->search);

3530
	switch (g->type) {
3531
	case MATCH_FULL:
3532
		if (strcmp(str, g->search) == 0)
3533 3534 3535
			matched = 1;
		break;
	case MATCH_FRONT_ONLY:
3536
		if (strncmp(str, g->search, g->len) == 0)
3537 3538 3539
			matched = 1;
		break;
	case MATCH_MIDDLE_ONLY:
3540
		if (strstr(str, g->search))
3541 3542 3543
			matched = 1;
		break;
	case MATCH_END_ONLY:
3544
		slen = strlen(str);
3545 3546
		if (slen >= g->len &&
		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3547 3548
			matched = 1;
		break;
3549 3550 3551 3552
	case MATCH_GLOB:
		if (glob_match(g->search, str))
			matched = 1;
		break;
3553 3554 3555 3556 3557
	}

	return matched;
}

3558
static int
3559
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3560
{
3561 3562 3563
	struct ftrace_func_entry *entry;
	int ret = 0;

3564
	entry = ftrace_lookup_ip(hash, rec->ip);
3565
	if (clear_filter) {
3566 3567 3568
		/* Do nothing if it doesn't exist */
		if (!entry)
			return 0;
3569

3570
		free_hash_entry(hash, entry);
3571 3572 3573 3574
	} else {
		/* Do nothing if it exists */
		if (entry)
			return 0;
3575

3576
		ret = add_hash_entry(hash, rec->ip);
3577 3578
	}
	return ret;
3579 3580
}

3581
static int
D
Dmitry Safonov 已提交
3582 3583
ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
		struct ftrace_glob *mod_g, int exclude_mod)
3584 3585
{
	char str[KSYM_SYMBOL_LEN];
3586 3587 3588 3589
	char *modname;

	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);

D
Dmitry Safonov 已提交
3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609
	if (mod_g) {
		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;

		/* blank module name to match all modules */
		if (!mod_g->len) {
			/* blank module globbing: modname xor exclude_mod */
			if ((!exclude_mod) != (!modname))
				goto func_match;
			return 0;
		}

		/* not matching the module */
		if (!modname || !mod_matches) {
			if (exclude_mod)
				goto func_match;
			else
				return 0;
		}

		if (mod_matches && exclude_mod)
3610 3611
			return 0;

D
Dmitry Safonov 已提交
3612
func_match:
3613
		/* blank search means to match all funcs in the mod */
3614
		if (!func_g->len)
3615 3616
			return 1;
	}
3617

3618
	return ftrace_match(str, func_g);
3619 3620
}

3621
static int
3622
match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3623 3624 3625
{
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
3626
	struct ftrace_glob func_g = { .type = MATCH_FULL };
D
Dmitry Safonov 已提交
3627 3628 3629
	struct ftrace_glob mod_g = { .type = MATCH_FULL };
	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
	int exclude_mod = 0;
3630
	int found = 0;
3631
	int ret;
3632
	int clear_filter;
3633

D
Dmitry Safonov 已提交
3634
	if (func) {
3635 3636 3637
		func_g.type = filter_parse_regex(func, len, &func_g.search,
						 &clear_filter);
		func_g.len = strlen(func_g.search);
3638
	}
3639

D
Dmitry Safonov 已提交
3640 3641 3642 3643
	if (mod) {
		mod_g.type = filter_parse_regex(mod, strlen(mod),
				&mod_g.search, &exclude_mod);
		mod_g.len = strlen(mod_g.search);
3644
	}
3645

3646
	mutex_lock(&ftrace_lock);
3647

3648 3649
	if (unlikely(ftrace_disabled))
		goto out_unlock;
3650

3651
	do_for_each_ftrace_rec(pg, rec) {
3652 3653 3654 3655

		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
3656
		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3657
			ret = enter_record(hash, rec, clear_filter);
3658 3659 3660 3661
			if (ret < 0) {
				found = ret;
				goto out_unlock;
			}
3662
			found = 1;
3663 3664
		}
	} while_for_each_ftrace_rec();
3665
 out_unlock:
3666
	mutex_unlock(&ftrace_lock);
3667 3668

	return found;
3669 3670
}

3671
static int
3672
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3673
{
3674
	return match_records(hash, buff, len, NULL);
3675 3676
}

3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726
static void ftrace_ops_update_code(struct ftrace_ops *ops,
				   struct ftrace_ops_hash *old_hash)
{
	struct ftrace_ops *op;

	if (!ftrace_enabled)
		return;

	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
		return;
	}

	/*
	 * If this is the shared global_ops filter, then we need to
	 * check if there is another ops that shares it, is enabled.
	 * If so, we still need to run the modify code.
	 */
	if (ops->func_hash != &global_ops.local_hash)
		return;

	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op->func_hash == &global_ops.local_hash &&
		    op->flags & FTRACE_OPS_FL_ENABLED) {
			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
			/* Only need to do this once */
			return;
		}
	} while_for_each_ftrace_op(op);
}

static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
					   struct ftrace_hash **orig_hash,
					   struct ftrace_hash *hash,
					   int enable)
{
	struct ftrace_ops_hash old_hash_ops;
	struct ftrace_hash *old_hash;
	int ret;

	old_hash = *orig_hash;
	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
	if (!ret) {
		ftrace_ops_update_code(ops, &old_hash_ops);
		free_ftrace_hash_rcu(old_hash);
	}
	return ret;
}
3727

3728 3729 3730 3731 3732 3733
/*
 * We register the module command as a template to show others how
 * to register the a command as well.
 */

static int
3734
ftrace_mod_callback(struct ftrace_hash *hash,
3735
		    char *func, char *cmd, char *module, int enable)
3736
{
3737
	int ret;
3738 3739 3740 3741 3742 3743 3744 3745

	/*
	 * cmd == 'mod' because we only registered this func
	 * for the 'mod' ftrace_func_command.
	 * But if you register one func with multiple commands,
	 * you can tell which command was used by the cmd
	 * parameter.
	 */
3746
	ret = match_records(hash, func, strlen(func), module);
3747
	if (!ret)
3748
		return -EINVAL;
3749 3750 3751
	if (ret < 0)
		return ret;
	return 0;
3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762
}

static struct ftrace_func_command ftrace_mod_cmd = {
	.name			= "mod",
	.func			= ftrace_mod_callback,
};

static int __init ftrace_mod_cmd_init(void)
{
	return register_ftrace_command(&ftrace_mod_cmd);
}
3763
core_initcall(ftrace_mod_cmd_init);
3764

3765
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
3766
				      struct ftrace_ops *op, struct pt_regs *pt_regs)
3767
{
S
Steven Rostedt 已提交
3768
	struct ftrace_func_probe *entry;
3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783
	struct hlist_head *hhd;
	unsigned long key;

	key = hash_long(ip, FTRACE_HASH_BITS);

	hhd = &ftrace_func_hash[key];

	if (hlist_empty(hhd))
		return;

	/*
	 * Disable preemption for these calls to prevent a RCU grace
	 * period. This syncs the hash iteration and freeing of items
	 * on the hash. rcu_read_lock is too dangerous here.
	 */
3784
	preempt_disable_notrace();
3785
	hlist_for_each_entry_rcu_notrace(entry, hhd, node) {
3786
		if (entry->ip == ip)
3787
			entry->ops->func(ip, parent_ip, entry->ops, NULL);
3788
	}
3789
	preempt_enable_notrace();
3790 3791
}

S
Steven Rostedt 已提交
3792
static struct ftrace_ops trace_probe_ops __read_mostly =
3793
{
3794
	.func		= function_trace_probe_call,
3795
	.flags		= FTRACE_OPS_FL_INITIALIZED,
3796
	INIT_OPS_HASH(trace_probe_ops)
3797 3798
};

S
Steven Rostedt 已提交
3799
static int ftrace_probe_registered;
3800

3801
static void __enable_ftrace_function_probe(struct ftrace_ops_hash *old_hash)
3802
{
3803
	int ret;
3804 3805
	int i;

3806 3807 3808
	if (ftrace_probe_registered) {
		/* still need to update the function call sites */
		if (ftrace_enabled)
3809 3810
			ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS,
					       old_hash);
3811
		return;
3812
	}
3813 3814 3815 3816 3817 3818 3819 3820 3821 3822

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
			break;
	}
	/* Nothing registered? */
	if (i == FTRACE_FUNC_HASHSIZE)
		return;

3823
	ret = ftrace_startup(&trace_probe_ops, 0);
3824

S
Steven Rostedt 已提交
3825
	ftrace_probe_registered = 1;
3826 3827
}

3828
static bool __disable_ftrace_function_probe(void)
3829 3830 3831
{
	int i;

S
Steven Rostedt 已提交
3832
	if (!ftrace_probe_registered)
3833
		return false;
3834 3835 3836 3837

	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];
		if (hhd->first)
3838
			return false;
3839 3840 3841
	}

	/* no more funcs left */
3842
	ftrace_shutdown(&trace_probe_ops, 0);
3843

S
Steven Rostedt 已提交
3844
	ftrace_probe_registered = 0;
3845
	return true;
3846 3847 3848
}


3849
static void ftrace_free_entry(struct ftrace_func_probe *entry)
3850 3851
{
	if (entry->ops->free)
3852
		entry->ops->free(entry->ops, entry->ip, NULL);
3853 3854 3855
	kfree(entry);
}

3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996
struct ftrace_func_map {
	struct ftrace_func_entry	entry;
	void				*data;
};

struct ftrace_func_mapper {
	struct ftrace_hash		hash;
};

/**
 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
 *
 * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
 */
struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
{
	struct ftrace_hash *hash;

	/*
	 * The mapper is simply a ftrace_hash, but since the entries
	 * in the hash are not ftrace_func_entry type, we define it
	 * as a separate structure.
	 */
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	return (struct ftrace_func_mapper *)hash;
}

/**
 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: the instruction pointer to find the data for
 *
 * Returns the data mapped to @ip if found otherwise NULL. The return
 * is actually the address of the mapper data pointer. The address is
 * returned for use cases where the data is no bigger than a long, and
 * the user can use the data pointer as its data instead of having to
 * allocate more memory for the reference.
 */
void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
				  unsigned long ip)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;

	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;

	map = (struct ftrace_func_map *)entry;
	return &map->data;
}

/**
 * ftrace_func_mapper_add_ip - Map some data to an ip
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to map @data to
 * @data: The data to map to @ip
 *
 * Returns 0 on succes otherwise an error.
 */
int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
			      unsigned long ip, void *data)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;

	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (entry)
		return -EBUSY;

	map = kmalloc(sizeof(*map), GFP_KERNEL);
	if (!map)
		return -ENOMEM;

	map->entry.ip = ip;
	map->data = data;

	__add_hash_entry(&mapper->hash, &map->entry);

	return 0;
}

/**
 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
 * @mapper: The mapper that has the ip maps
 * @ip: The instruction pointer address to remove the data from
 *
 * Returns the data if it is found, otherwise NULL.
 * Note, if the data pointer is used as the data itself, (see 
 * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
 * if the data pointer was set to zero.
 */
void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
				   unsigned long ip)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	void *data;

	entry = ftrace_lookup_ip(&mapper->hash, ip);
	if (!entry)
		return NULL;

	map = (struct ftrace_func_map *)entry;
	data = map->data;

	remove_hash_entry(&mapper->hash, entry);
	kfree(entry);

	return data;
}

/**
 * free_ftrace_func_mapper - free a mapping of ips and data
 * @mapper: The mapper that has the ip maps
 * @free_func: A function to be called on each data item.
 *
 * This is used to free the function mapper. The @free_func is optional
 * and can be used if the data needs to be freed as well.
 */
void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
			     ftrace_mapper_func free_func)
{
	struct ftrace_func_entry *entry;
	struct ftrace_func_map *map;
	struct hlist_head *hhd;
	int size = 1 << mapper->hash.size_bits;
	int i;

	if (free_func && mapper->hash.count) {
		for (i = 0; i < size; i++) {
			hhd = &mapper->hash.buckets[i];
			hlist_for_each_entry(entry, hhd, hlist) {
				map = (struct ftrace_func_map *)entry;
				free_func(map);
			}
		}
	}
	free_ftrace_hash(&mapper->hash);
}

3997
int
S
Steven Rostedt 已提交
3998
register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3999 4000
			      void *data)
{
4001
	struct ftrace_ops_hash old_hash_ops;
S
Steven Rostedt 已提交
4002
	struct ftrace_func_probe *entry;
4003
	struct ftrace_glob func_g;
4004
	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
4005
	struct ftrace_hash *old_hash = *orig_hash;
4006
	struct ftrace_hash *hash;
4007 4008
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
4009
	int not;
S
Steven Rostedt 已提交
4010
	unsigned long key;
4011
	int count = 0;
4012
	int ret;
4013

4014 4015 4016
	func_g.type = filter_parse_regex(glob, strlen(glob),
			&func_g.search, &not);
	func_g.len = strlen(func_g.search);
4017

S
Steven Rostedt 已提交
4018
	/* we do not support '!' for function probes */
4019 4020 4021
	if (WARN_ON(not))
		return -EINVAL;

4022
	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
4023

4024 4025 4026 4027
	old_hash_ops.filter_hash = old_hash;
	/* Probes only have filters */
	old_hash_ops.notrace_hash = NULL;

4028
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4029 4030
	if (!hash) {
		count = -ENOMEM;
4031
		goto out;
4032 4033 4034 4035
	}

	if (unlikely(ftrace_disabled)) {
		count = -ENODEV;
4036
		goto out;
4037
	}
4038

4039 4040
	mutex_lock(&ftrace_lock);

4041
	do_for_each_ftrace_rec(pg, rec) {
4042

4043 4044 4045
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
4046
		if (!ftrace_match_record(rec, &func_g, NULL, 0))
4047 4048 4049 4050
			continue;

		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
		if (!entry) {
S
Steven Rostedt 已提交
4051
			/* If we did not process any, then return error */
4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063
			if (!count)
				count = -ENOMEM;
			goto out_unlock;
		}

		count++;

		/*
		 * The caller might want to do something special
		 * for each function we find. We call the callback
		 * to give the caller an opportunity to do so.
		 */
4064
		if (ops->init) {
4065
			if (ops->init(ops, rec->ip, data) < 0) {
4066 4067 4068 4069 4070 4071
				/* caller does not like this func */
				kfree(entry);
				continue;
			}
		}

4072 4073 4074 4075 4076 4077 4078
		ret = enter_record(hash, rec, 0);
		if (ret < 0) {
			kfree(entry);
			count = ret;
			goto out_unlock;
		}

4079 4080 4081 4082 4083 4084 4085
		entry->ops = ops;
		entry->ip = rec->ip;

		key = hash_long(entry->ip, FTRACE_HASH_BITS);
		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);

	} while_for_each_ftrace_rec();
4086 4087

	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
4088

4089
	__enable_ftrace_function_probe(&old_hash_ops);
4090

4091 4092 4093
	if (!ret)
		free_ftrace_hash_rcu(old_hash);
	else
4094 4095
		count = ret;

4096
 out_unlock:
4097 4098
	mutex_unlock(&ftrace_lock);
 out:
4099
	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
4100
	free_ftrace_hash(hash);
4101 4102 4103 4104

	return count;
}

4105
int
4106
unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
4107
{
4108
	struct ftrace_ops_hash old_hash_ops;
4109
	struct ftrace_func_entry *rec_entry;
S
Steven Rostedt 已提交
4110
	struct ftrace_func_probe *entry;
4111
	struct ftrace_func_probe *p;
4112
	struct ftrace_glob func_g;
4113
	struct ftrace_hash **orig_hash = &trace_probe_ops.func_hash->filter_hash;
4114
	struct ftrace_hash *old_hash = *orig_hash;
4115
	struct list_head free_list;
4116
	struct ftrace_hash *hash;
4117
	struct hlist_node *tmp;
4118
	char str[KSYM_SYMBOL_LEN];
4119
	int i, ret;
4120
	bool disabled;
4121

4122
	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
4123
		func_g.search = NULL;
4124
	else if (glob) {
4125 4126
		int not;

4127 4128 4129 4130
		func_g.type = filter_parse_regex(glob, strlen(glob),
						 &func_g.search, &not);
		func_g.len = strlen(func_g.search);
		func_g.search = glob;
4131

S
Steven Rostedt 已提交
4132
		/* we do not support '!' for function probes */
4133
		if (WARN_ON(not))
4134
			return -EINVAL;
4135 4136
	}

4137
	mutex_lock(&trace_probe_ops.func_hash->regex_lock);
4138

4139 4140 4141 4142
	old_hash_ops.filter_hash = old_hash;
	/* Probes only have filters */
	old_hash_ops.notrace_hash = NULL;

4143
	ret = -ENOMEM;
4144 4145 4146 4147
	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
	if (!hash)
		goto out_unlock;

4148 4149
	INIT_LIST_HEAD(&free_list);

4150 4151 4152
	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
		struct hlist_head *hhd = &ftrace_func_hash[i];

4153
		hlist_for_each_entry_safe(entry, tmp, hhd, node) {
4154 4155

			/* break up if statements for readability */
4156
			if (entry->ops != ops)
4157 4158 4159
				continue;

			/* do this last, since it is the most expensive */
4160
			if (func_g.search) {
4161 4162
				kallsyms_lookup(entry->ip, NULL, NULL,
						NULL, str);
4163
				if (!ftrace_match(str, &func_g))
4164 4165 4166
					continue;
			}

4167 4168 4169 4170 4171
			rec_entry = ftrace_lookup_ip(hash, entry->ip);
			/* It is possible more than one entry had this ip */
			if (rec_entry)
				free_hash_entry(hash, rec_entry);

4172
			hlist_del_rcu(&entry->node);
4173
			list_add(&entry->free_list, &free_list);
4174 4175
		}
	}
4176 4177 4178 4179 4180 4181 4182

	/* Nothing found? */
	if (list_empty(&free_list)) {
		ret = -EINVAL;
		goto out_unlock;
	}

4183
	mutex_lock(&ftrace_lock);
4184
	disabled = __disable_ftrace_function_probe();
4185 4186 4187 4188
	/*
	 * Remove after the disable is called. Otherwise, if the last
	 * probe is removed, a null hash means *all enabled*.
	 */
4189
	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);
4190 4191 4192 4193 4194

	/* still need to update the function call sites */
	if (ftrace_enabled && !disabled)
		ftrace_run_modify_code(&trace_probe_ops, FTRACE_UPDATE_CALLS,
				       &old_hash_ops);
4195
	synchronize_sched();
4196 4197 4198
	if (!ret)
		free_ftrace_hash_rcu(old_hash);

4199 4200 4201 4202
	list_for_each_entry_safe(entry, p, &free_list, free_list) {
		list_del(&entry->free_list);
		ftrace_free_entry(entry);
	}
4203
	mutex_unlock(&ftrace_lock);
4204

4205
 out_unlock:
4206
	mutex_unlock(&trace_probe_ops.func_hash->regex_lock);
4207
	free_ftrace_hash(hash);
4208
	return ret;
4209 4210
}

4211 4212 4213
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);

4214 4215 4216 4217 4218
/*
 * Currently we only register ftrace commands from __init, so mark this
 * __init too.
 */
__init int register_ftrace_command(struct ftrace_func_command *cmd)
4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236
{
	struct ftrace_func_command *p;
	int ret = 0;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = -EBUSY;
			goto out_unlock;
		}
	}
	list_add(&cmd->list, &ftrace_commands);
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4237 4238 4239 4240 4241
/*
 * Currently we only unregister ftrace commands from __init, so mark
 * this __init too.
 */
__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259
{
	struct ftrace_func_command *p, *n;
	int ret = -ENODEV;

	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
		if (strcmp(cmd->name, p->name) == 0) {
			ret = 0;
			list_del_init(&p->list);
			goto out_unlock;
		}
	}
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);

	return ret;
}

4260 4261
static int ftrace_process_regex(struct ftrace_hash *hash,
				char *buff, int len, int enable)
4262
{
4263
	char *func, *command, *next = buff;
S
Steven Rostedt 已提交
4264
	struct ftrace_func_command *p;
4265
	int ret = -EINVAL;
4266 4267 4268 4269

	func = strsep(&next, ":");

	if (!next) {
4270
		ret = ftrace_match_records(hash, func, len);
4271 4272 4273 4274 4275
		if (!ret)
			ret = -EINVAL;
		if (ret < 0)
			return ret;
		return 0;
4276 4277
	}

4278
	/* command found */
4279 4280 4281

	command = strsep(&next, ":");

4282 4283 4284
	mutex_lock(&ftrace_cmd_mutex);
	list_for_each_entry(p, &ftrace_commands, list) {
		if (strcmp(p->name, command) == 0) {
4285
			ret = p->func(hash, func, command, next, enable);
4286 4287
			goto out_unlock;
		}
4288
	}
4289 4290
 out_unlock:
	mutex_unlock(&ftrace_cmd_mutex);
4291

4292
	return ret;
4293 4294
}

I
Ingo Molnar 已提交
4295
static ssize_t
4296 4297
ftrace_regex_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos, int enable)
4298 4299
{
	struct ftrace_iterator *iter;
4300 4301
	struct trace_parser *parser;
	ssize_t ret, read;
4302

4303
	if (!cnt)
4304 4305 4306 4307 4308 4309 4310 4311
		return 0;

	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		iter = m->private;
	} else
		iter = file->private_data;

4312
	if (unlikely(ftrace_disabled))
4313 4314 4315
		return -ENODEV;

	/* iter->hash is a local copy, so we don't need regex_lock */
4316

4317 4318
	parser = &iter->parser;
	read = trace_get_user(parser, ubuf, cnt, ppos);
4319

4320
	if (read >= 0 && trace_parser_loaded(parser) &&
4321
	    !trace_parser_cont(parser)) {
4322
		ret = ftrace_process_regex(iter->hash, parser->buffer,
4323
					   parser->idx, enable);
4324
		trace_parser_clear(parser);
4325
		if (ret < 0)
4326
			goto out;
4327
	}
4328 4329

	ret = read;
4330
 out:
4331 4332 4333
	return ret;
}

4334
ssize_t
4335 4336 4337 4338 4339 4340
ftrace_filter_write(struct file *file, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}

4341
ssize_t
4342 4343 4344 4345 4346 4347
ftrace_notrace_write(struct file *file, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}

4348
static int
4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369
ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
	struct ftrace_func_entry *entry;

	if (!ftrace_location(ip))
		return -EINVAL;

	if (remove) {
		entry = ftrace_lookup_ip(hash, ip);
		if (!entry)
			return -ENOENT;
		free_hash_entry(hash, entry);
		return 0;
	}

	return add_hash_entry(hash, ip);
}

static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
		unsigned long ip, int remove, int reset, int enable)
4370
{
4371
	struct ftrace_hash **orig_hash;
4372
	struct ftrace_hash *hash;
4373
	int ret;
4374

4375
	if (unlikely(ftrace_disabled))
4376
		return -ENODEV;
4377

4378
	mutex_lock(&ops->func_hash->regex_lock);
4379

4380
	if (enable)
4381
		orig_hash = &ops->func_hash->filter_hash;
4382
	else
4383
		orig_hash = &ops->func_hash->notrace_hash;
4384

4385 4386 4387 4388 4389
	if (reset)
		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	else
		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);

4390 4391 4392 4393
	if (!hash) {
		ret = -ENOMEM;
		goto out_regex_unlock;
	}
4394

4395 4396 4397 4398
	if (buf && !ftrace_match_records(hash, buf, len)) {
		ret = -EINVAL;
		goto out_regex_unlock;
	}
4399 4400 4401 4402 4403
	if (ip) {
		ret = ftrace_match_addr(hash, ip, remove);
		if (ret < 0)
			goto out_regex_unlock;
	}
4404 4405

	mutex_lock(&ftrace_lock);
4406
	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
4407 4408
	mutex_unlock(&ftrace_lock);

4409
 out_regex_unlock:
4410
	mutex_unlock(&ops->func_hash->regex_lock);
4411 4412 4413

	free_ftrace_hash(hash);
	return ret;
4414 4415
}

4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
		int reset, int enable)
{
	return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
}

/**
 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
 * @ops - the ops to set the filter with
 * @ip - the address to add to or remove from the filter.
 * @remove - non zero to remove the ip from the filter
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled
 * If @ip is NULL, it failes to update filter.
 */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
			 int remove, int reset)
{
4436
	ftrace_ops_init(ops);
4437 4438 4439 4440
	return ftrace_set_addr(ops, ip, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);

4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457
/**
 * ftrace_ops_set_global_filter - setup ops to use global filters
 * @ops - the ops which will use the global filters
 *
 * ftrace users who need global function trace filtering should call this.
 * It can set the global filter only if ops were not initialized before.
 */
void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
{
	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
		return;

	ftrace_ops_init(ops);
	ops->func_hash = &global_ops.local_hash;
}
EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);

4458 4459 4460 4461 4462 4463 4464
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
		 int reset, int enable)
{
	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
}

4465 4466
/**
 * ftrace_set_filter - set a function to filter on in ftrace
4467 4468 4469 4470 4471 4472 4473 4474
 * @ops - the ops to set the filter with
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4475
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4476 4477
		       int len, int reset)
{
4478
	ftrace_ops_init(ops);
4479
	return ftrace_set_regex(ops, buf, len, reset, 1);
4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);

/**
 * ftrace_set_notrace - set a function to not trace in ftrace
 * @ops - the ops to set the notrace filter with
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4494
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4495 4496
			int len, int reset)
{
4497
	ftrace_ops_init(ops);
4498
	return ftrace_set_regex(ops, buf, len, reset, 0);
4499 4500 4501
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
/**
4502
 * ftrace_set_global_filter - set a function to filter on with global tracers
4503 4504 4505 4506 4507 4508 4509
 * @buf - the string that holds the function filter text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Filters denote which functions should be enabled when tracing is enabled.
 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
 */
4510
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4511
{
4512
	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4513
}
4514
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4515

4516
/**
4517
 * ftrace_set_global_notrace - set a function to not trace with global tracers
4518 4519 4520 4521 4522 4523 4524 4525
 * @buf - the string that holds the function notrace text.
 * @len - the length of the string.
 * @reset - non zero to reset all filters before applying this filter.
 *
 * Notrace Filters denote which functions should not be enabled when tracing
 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
 * for tracing.
 */
4526
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4527
{
4528
	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4529
}
4530
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4531

4532 4533 4534 4535 4536 4537 4538
/*
 * command line interface to allow users to set filters on boot up.
 */
#define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;

4539 4540 4541
/* Used by function selftest to not test if filter is set */
bool ftrace_filter_param __initdata;

4542 4543
static int __init set_ftrace_notrace(char *str)
{
4544
	ftrace_filter_param = true;
4545
	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4546 4547 4548 4549 4550 4551
	return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);

static int __init set_ftrace_filter(char *str)
{
4552
	ftrace_filter_param = true;
4553
	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4554 4555 4556 4557
	return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);

4558
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4559
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4560
static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4561
static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
4562

4563 4564 4565
static unsigned long save_global_trampoline;
static unsigned long save_global_flags;

4566 4567
static int __init set_graph_function(char *str)
{
4568
	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4569 4570 4571 4572
	return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);

4573 4574 4575 4576 4577 4578 4579
static int __init set_graph_notrace_function(char *str)
{
	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
	return 1;
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);

4580 4581 4582 4583 4584 4585 4586 4587
static int __init set_graph_max_depth_function(char *str)
{
	if (!str)
		return 0;
	fgraph_max_depth = simple_strtoul(str, NULL, 0);
	return 1;
}
__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
4588 4589

static void __init set_ftrace_early_graph(char *buf, int enable)
4590 4591 4592
{
	int ret;
	char *func;
4593
	struct ftrace_hash *hash;
4594

4595 4596 4597
	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
	if (WARN_ON(!hash))
		return;
4598 4599 4600 4601

	while (buf) {
		func = strsep(&buf, ",");
		/* we allow only one expression at a time */
4602
		ret = ftrace_graph_set_hash(hash, func);
4603 4604 4605 4606
		if (ret)
			printk(KERN_DEBUG "ftrace: function %s not "
					  "traceable\n", func);
	}
4607 4608 4609 4610 4611

	if (enable)
		ftrace_graph_hash = hash;
	else
		ftrace_graph_notrace_hash = hash;
4612 4613 4614
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

4615 4616
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4617 4618 4619
{
	char *func;

4620 4621
	ftrace_ops_init(ops);

4622 4623
	while (buf) {
		func = strsep(&buf, ",");
4624
		ftrace_set_regex(ops, func, strlen(func), 0, enable);
4625 4626 4627 4628 4629 4630
	}
}

static void __init set_ftrace_early_filters(void)
{
	if (ftrace_filter_buf[0])
4631
		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
4632
	if (ftrace_notrace_buf[0])
4633
		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
4634 4635
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	if (ftrace_graph_buf[0])
4636 4637 4638
		set_ftrace_early_graph(ftrace_graph_buf, 1);
	if (ftrace_graph_notrace_buf[0])
		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
4639
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4640 4641
}

4642
int ftrace_regex_release(struct inode *inode, struct file *file)
4643 4644 4645
{
	struct seq_file *m = (struct seq_file *)file->private_data;
	struct ftrace_iterator *iter;
4646
	struct ftrace_hash **orig_hash;
4647
	struct trace_parser *parser;
4648
	int filter_hash;
4649
	int ret;
4650 4651 4652 4653 4654 4655 4656

	if (file->f_mode & FMODE_READ) {
		iter = m->private;
		seq_release(inode, file);
	} else
		iter = file->private_data;

4657 4658 4659
	parser = &iter->parser;
	if (trace_parser_loaded(parser)) {
		parser->buffer[parser->idx] = 0;
4660
		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
4661 4662
	}

4663 4664
	trace_parser_put(parser);

4665
	mutex_lock(&iter->ops->func_hash->regex_lock);
4666

4667
	if (file->f_mode & FMODE_WRITE) {
4668 4669 4670
		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);

		if (filter_hash)
4671
			orig_hash = &iter->ops->func_hash->filter_hash;
4672
		else
4673
			orig_hash = &iter->ops->func_hash->notrace_hash;
4674

4675
		mutex_lock(&ftrace_lock);
4676 4677
		ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
						      iter->hash, filter_hash);
4678
		mutex_unlock(&ftrace_lock);
4679 4680 4681
	} else {
		/* For read only, the hash is the ops hash */
		iter->hash = NULL;
4682
	}
4683

4684
	mutex_unlock(&iter->ops->func_hash->regex_lock);
4685 4686
	free_ftrace_hash(iter->hash);
	kfree(iter);
4687

4688 4689 4690
	return 0;
}

4691
static const struct file_operations ftrace_avail_fops = {
4692 4693 4694
	.open = ftrace_avail_open,
	.read = seq_read,
	.llseek = seq_lseek,
L
Li Zefan 已提交
4695
	.release = seq_release_private,
4696 4697
};

4698 4699 4700 4701 4702 4703 4704
static const struct file_operations ftrace_enabled_fops = {
	.open = ftrace_enabled_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = seq_release_private,
};

4705
static const struct file_operations ftrace_filter_fops = {
4706
	.open = ftrace_filter_open,
L
Lai Jiangshan 已提交
4707
	.read = seq_read,
4708
	.write = ftrace_filter_write,
4709
	.llseek = tracing_lseek,
4710
	.release = ftrace_regex_release,
4711 4712
};

4713
static const struct file_operations ftrace_notrace_fops = {
4714
	.open = ftrace_notrace_open,
L
Lai Jiangshan 已提交
4715
	.read = seq_read,
4716
	.write = ftrace_notrace_write,
4717
	.llseek = tracing_lseek,
4718
	.release = ftrace_regex_release,
4719 4720
};

4721 4722 4723 4724
#ifdef CONFIG_FUNCTION_GRAPH_TRACER

static DEFINE_MUTEX(graph_lock);

4725 4726 4727 4728 4729 4730 4731
struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH;
struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH;

enum graph_filter_type {
	GRAPH_FILTER_NOTRACE	= 0,
	GRAPH_FILTER_FUNCTION,
};
4732

4733 4734
#define FTRACE_GRAPH_EMPTY	((void *)1)

4735
struct ftrace_graph_data {
4736 4737 4738 4739 4740 4741 4742
	struct ftrace_hash		*hash;
	struct ftrace_func_entry	*entry;
	int				idx;   /* for hash table iteration */
	enum graph_filter_type		type;
	struct ftrace_hash		*new_hash;
	const struct seq_operations	*seq_ops;
	struct trace_parser		parser;
4743 4744
};

4745
static void *
4746
__g_next(struct seq_file *m, loff_t *pos)
4747
{
4748
	struct ftrace_graph_data *fgd = m->private;
4749 4750 4751
	struct ftrace_func_entry *entry = fgd->entry;
	struct hlist_head *head;
	int i, idx = fgd->idx;
4752

4753
	if (*pos >= fgd->hash->count)
4754
		return NULL;
4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773

	if (entry) {
		hlist_for_each_entry_continue(entry, hlist) {
			fgd->entry = entry;
			return entry;
		}

		idx++;
	}

	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
		head = &fgd->hash->buckets[i];
		hlist_for_each_entry(entry, head, hlist) {
			fgd->entry = entry;
			fgd->idx = i;
			return entry;
		}
	}
	return NULL;
4774
}
4775

4776 4777 4778 4779 4780
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
	(*pos)++;
	return __g_next(m, pos);
4781 4782 4783 4784
}

static void *g_start(struct seq_file *m, loff_t *pos)
{
4785 4786
	struct ftrace_graph_data *fgd = m->private;

4787 4788
	mutex_lock(&graph_lock);

4789 4790 4791 4792 4793 4794 4795
	if (fgd->type == GRAPH_FILTER_FUNCTION)
		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
	else
		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));

4796
	/* Nothing, tell g_show to print all functions are enabled */
4797
	if (ftrace_hash_empty(fgd->hash) && !*pos)
4798
		return FTRACE_GRAPH_EMPTY;
4799

4800 4801
	fgd->idx = 0;
	fgd->entry = NULL;
4802
	return __g_next(m, pos);
4803 4804 4805 4806 4807 4808 4809 4810 4811
}

static void g_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&graph_lock);
}

static int g_show(struct seq_file *m, void *v)
{
4812
	struct ftrace_func_entry *entry = v;
4813

4814
	if (!entry)
4815 4816
		return 0;

4817
	if (entry == FTRACE_GRAPH_EMPTY) {
4818 4819
		struct ftrace_graph_data *fgd = m->private;

4820
		if (fgd->type == GRAPH_FILTER_FUNCTION)
4821
			seq_puts(m, "#### all functions enabled ####\n");
4822
		else
4823
			seq_puts(m, "#### no functions disabled ####\n");
4824 4825 4826
		return 0;
	}

4827
	seq_printf(m, "%ps\n", (void *)entry->ip);
4828 4829 4830 4831

	return 0;
}

J
James Morris 已提交
4832
static const struct seq_operations ftrace_graph_seq_ops = {
4833 4834 4835 4836 4837 4838 4839
	.start = g_start,
	.next = g_next,
	.stop = g_stop,
	.show = g_show,
};

static int
4840 4841
__ftrace_graph_open(struct inode *inode, struct file *file,
		    struct ftrace_graph_data *fgd)
4842 4843
{
	int ret = 0;
4844
	struct ftrace_hash *new_hash = NULL;
4845

4846 4847 4848
	if (file->f_mode & FMODE_WRITE) {
		const int size_bits = FTRACE_HASH_DEFAULT_BITS;

4849 4850 4851
		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
			return -ENOMEM;

4852 4853 4854 4855 4856 4857 4858 4859 4860
		if (file->f_flags & O_TRUNC)
			new_hash = alloc_ftrace_hash(size_bits);
		else
			new_hash = alloc_and_copy_ftrace_hash(size_bits,
							      fgd->hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}
4861 4862
	}

4863
	if (file->f_mode & FMODE_READ) {
4864
		ret = seq_open(file, &ftrace_graph_seq_ops);
4865 4866 4867
		if (!ret) {
			struct seq_file *m = file->private_data;
			m->private = fgd;
4868 4869 4870 4871
		} else {
			/* Failed */
			free_ftrace_hash(new_hash);
			new_hash = NULL;
4872 4873 4874
		}
	} else
		file->private_data = fgd;
4875

4876
out:
4877 4878 4879
	if (ret < 0 && file->f_mode & FMODE_WRITE)
		trace_parser_put(&fgd->parser);

4880
	fgd->new_hash = new_hash;
4881 4882 4883 4884 4885 4886 4887 4888

	/*
	 * All uses of fgd->hash must be taken with the graph_lock
	 * held. The graph_lock is going to be released, so force
	 * fgd->hash to be reinitialized when it is taken again.
	 */
	fgd->hash = NULL;

4889 4890 4891
	return ret;
}

4892 4893 4894 4895
static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
4896
	int ret;
4897 4898 4899 4900 4901 4902 4903 4904

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

4905 4906
	mutex_lock(&graph_lock);

4907 4908
	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
4909
	fgd->type = GRAPH_FILTER_FUNCTION;
4910 4911
	fgd->seq_ops = &ftrace_graph_seq_ops;

4912 4913 4914 4915 4916 4917
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
4918 4919
}

4920 4921 4922 4923
static int
ftrace_graph_notrace_open(struct inode *inode, struct file *file)
{
	struct ftrace_graph_data *fgd;
4924
	int ret;
4925 4926 4927 4928 4929 4930 4931 4932

	if (unlikely(ftrace_disabled))
		return -ENODEV;

	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
	if (fgd == NULL)
		return -ENOMEM;

4933 4934
	mutex_lock(&graph_lock);

4935 4936
	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
4937
	fgd->type = GRAPH_FILTER_NOTRACE;
4938 4939
	fgd->seq_ops = &ftrace_graph_seq_ops;

4940 4941 4942 4943 4944 4945
	ret = __ftrace_graph_open(inode, file, fgd);
	if (ret < 0)
		kfree(fgd);

	mutex_unlock(&graph_lock);
	return ret;
4946 4947
}

4948 4949 4950
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
4951
	struct ftrace_graph_data *fgd;
4952 4953 4954
	struct ftrace_hash *old_hash, *new_hash;
	struct trace_parser *parser;
	int ret = 0;
4955

4956 4957 4958
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;

4959
		fgd = m->private;
4960
		seq_release(inode, file);
4961
	} else {
4962
		fgd = file->private_data;
4963 4964
	}

4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004

	if (file->f_mode & FMODE_WRITE) {

		parser = &fgd->parser;

		if (trace_parser_loaded((parser))) {
			parser->buffer[parser->idx] = 0;
			ret = ftrace_graph_set_hash(fgd->new_hash,
						    parser->buffer);
		}

		trace_parser_put(parser);

		new_hash = __ftrace_hash_move(fgd->new_hash);
		if (!new_hash) {
			ret = -ENOMEM;
			goto out;
		}

		mutex_lock(&graph_lock);

		if (fgd->type == GRAPH_FILTER_FUNCTION) {
			old_hash = rcu_dereference_protected(ftrace_graph_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_hash, new_hash);
		} else {
			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
					lockdep_is_held(&graph_lock));
			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
		}

		mutex_unlock(&graph_lock);

		/* Wait till all users are no longer using the old hash */
		synchronize_sched();

		free_ftrace_hash(old_hash);
	}

 out:
5005 5006 5007
	kfree(fgd->new_hash);
	kfree(fgd);

5008
	return ret;
5009 5010
}

5011
static int
5012
ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
5013
{
5014
	struct ftrace_glob func_g;
5015 5016
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;
5017
	struct ftrace_func_entry *entry;
5018
	int fail = 1;
5019
	int not;
5020

5021
	/* decode regex */
5022 5023
	func_g.type = filter_parse_regex(buffer, strlen(buffer),
					 &func_g.search, &not);
5024

5025
	func_g.len = strlen(func_g.search);
5026

5027
	mutex_lock(&ftrace_lock);
5028 5029 5030 5031 5032 5033

	if (unlikely(ftrace_disabled)) {
		mutex_unlock(&ftrace_lock);
		return -ENODEV;
	}

5034 5035
	do_for_each_ftrace_rec(pg, rec) {

5036 5037 5038
		if (rec->flags & FTRACE_FL_DISABLED)
			continue;

D
Dmitry Safonov 已提交
5039
		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
5040
			entry = ftrace_lookup_ip(hash, rec->ip);
5041 5042 5043

			if (!not) {
				fail = 0;
5044 5045 5046 5047 5048

				if (entry)
					continue;
				if (add_hash_entry(hash, rec->ip) < 0)
					goto out;
5049
			} else {
5050 5051
				if (entry) {
					free_hash_entry(hash, entry);
5052 5053 5054
					fail = 0;
				}
			}
5055
		}
5056
	} while_for_each_ftrace_rec();
5057
out:
5058
	mutex_unlock(&ftrace_lock);
5059

5060 5061 5062 5063
	if (fail)
		return -EINVAL;

	return 0;
5064 5065 5066 5067 5068 5069
}

static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5070
	ssize_t read, ret = 0;
5071
	struct ftrace_graph_data *fgd = file->private_data;
5072
	struct trace_parser *parser;
5073

5074
	if (!cnt)
5075 5076
		return 0;

5077 5078 5079 5080 5081 5082
	/* Read mode uses seq functions */
	if (file->f_mode & FMODE_READ) {
		struct seq_file *m = file->private_data;
		fgd = m->private;
	}

5083
	parser = &fgd->parser;
5084

5085
	read = trace_get_user(parser, ubuf, cnt, ppos);
5086

5087 5088
	if (read >= 0 && trace_parser_loaded(parser) &&
	    !trace_parser_cont(parser)) {
5089

5090
		ret = ftrace_graph_set_hash(fgd->new_hash,
5091 5092
					    parser->buffer);
		trace_parser_clear(parser);
5093 5094
	}

5095 5096
	if (!ret)
		ret = read;
5097

5098 5099 5100 5101
	return ret;
}

static const struct file_operations ftrace_graph_fops = {
5102 5103 5104
	.open		= ftrace_graph_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5105
	.llseek		= tracing_lseek,
5106
	.release	= ftrace_graph_release,
5107
};
5108 5109 5110 5111 5112

static const struct file_operations ftrace_graph_notrace_fops = {
	.open		= ftrace_graph_notrace_open,
	.read		= seq_read,
	.write		= ftrace_graph_write,
5113
	.llseek		= tracing_lseek,
5114 5115
	.release	= ftrace_graph_release,
};
5116 5117
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147
void ftrace_create_filter_files(struct ftrace_ops *ops,
				struct dentry *parent)
{

	trace_create_file("set_ftrace_filter", 0644, parent,
			  ops, &ftrace_filter_fops);

	trace_create_file("set_ftrace_notrace", 0644, parent,
			  ops, &ftrace_notrace_fops);
}

/*
 * The name "destroy_filter_files" is really a misnomer. Although
 * in the future, it may actualy delete the files, but this is
 * really intended to make sure the ops passed in are disabled
 * and that when this function returns, the caller is free to
 * free the ops.
 *
 * The "destroy" name is only to match the "create" name that this
 * should be paired with.
 */
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
	mutex_lock(&ftrace_lock);
	if (ops->flags & FTRACE_OPS_FL_ENABLED)
		ftrace_shutdown(ops, 0);
	ops->flags |= FTRACE_OPS_FL_DELETED;
	mutex_unlock(&ftrace_lock);
}

5148
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
5149 5150
{

5151 5152
	trace_create_file("available_filter_functions", 0444,
			d_tracer, NULL, &ftrace_avail_fops);
5153

5154 5155 5156
	trace_create_file("enabled_functions", 0444,
			d_tracer, NULL, &ftrace_enabled_fops);

5157
	ftrace_create_filter_files(&global_ops, d_tracer);
5158

5159
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5160
	trace_create_file("set_graph_function", 0444, d_tracer,
5161 5162
				    NULL,
				    &ftrace_graph_fops);
5163 5164 5165
	trace_create_file("set_graph_notrace", 0444, d_tracer,
				    NULL,
				    &ftrace_graph_notrace_fops);
5166 5167
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

5168 5169 5170
	return 0;
}

5171
static int ftrace_cmp_ips(const void *a, const void *b)
5172
{
5173 5174
	const unsigned long *ipa = a;
	const unsigned long *ipb = b;
5175

5176 5177 5178 5179 5180 5181 5182
	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

5183
static int ftrace_process_locs(struct module *mod,
5184
			       unsigned long *start,
5185 5186
			       unsigned long *end)
{
5187
	struct ftrace_page *start_pg;
5188
	struct ftrace_page *pg;
5189
	struct dyn_ftrace *rec;
5190
	unsigned long count;
5191 5192
	unsigned long *p;
	unsigned long addr;
5193
	unsigned long flags = 0; /* Shut up gcc */
5194 5195 5196 5197 5198 5199 5200
	int ret = -ENOMEM;

	count = end - start;

	if (!count)
		return 0;

5201
	sort(start, count, sizeof(*start),
5202
	     ftrace_cmp_ips, NULL);
5203

5204 5205
	start_pg = ftrace_allocate_pages(count);
	if (!start_pg)
5206
		return -ENOMEM;
5207

S
Steven Rostedt 已提交
5208
	mutex_lock(&ftrace_lock);
5209

5210 5211 5212 5213 5214
	/*
	 * Core and each module needs their own pages, as
	 * modules will free them when they are removed.
	 * Force a new page to be allocated for modules.
	 */
5215 5216 5217
	if (!mod) {
		WARN_ON(ftrace_pages || ftrace_pages_start);
		/* First initialization */
5218
		ftrace_pages = ftrace_pages_start = start_pg;
5219
	} else {
5220
		if (!ftrace_pages)
5221
			goto out;
5222

5223 5224 5225 5226
		if (WARN_ON(ftrace_pages->next)) {
			/* Hmm, we have free pages? */
			while (ftrace_pages->next)
				ftrace_pages = ftrace_pages->next;
5227
		}
5228

5229
		ftrace_pages->next = start_pg;
5230 5231
	}

5232
	p = start;
5233
	pg = start_pg;
5234 5235
	while (p < end) {
		addr = ftrace_call_adjust(*p++);
5236 5237 5238 5239 5240 5241 5242 5243
		/*
		 * Some architecture linkers will pad between
		 * the different mcount_loc sections of different
		 * object files to satisfy alignments.
		 * Skip any NULL pointers.
		 */
		if (!addr)
			continue;
5244 5245 5246 5247 5248 5249 5250 5251 5252 5253

		if (pg->index == pg->size) {
			/* We should have allocated enough */
			if (WARN_ON(!pg->next))
				break;
			pg = pg->next;
		}

		rec = &pg->records[pg->index++];
		rec->ip = addr;
5254 5255
	}

5256 5257 5258 5259 5260 5261
	/* We should have used all pages */
	WARN_ON(pg->next);

	/* Assign the last page to ftrace_pages */
	ftrace_pages = pg;

5262
	/*
5263 5264 5265 5266 5267 5268
	 * We only need to disable interrupts on start up
	 * because we are modifying code that an interrupt
	 * may execute, and the modification is not atomic.
	 * But for modules, nothing runs the code we modify
	 * until we are finished with it, and there's no
	 * reason to cause large interrupt latencies while we do it.
5269
	 */
5270 5271
	if (!mod)
		local_irq_save(flags);
5272
	ftrace_update_code(mod, start_pg);
5273 5274
	if (!mod)
		local_irq_restore(flags);
5275 5276
	ret = 0;
 out:
S
Steven Rostedt 已提交
5277
	mutex_unlock(&ftrace_lock);
5278

5279
	return ret;
5280 5281
}

5282
#ifdef CONFIG_MODULES
5283 5284 5285

#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)

5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298
static int referenced_filters(struct dyn_ftrace *rec)
{
	struct ftrace_ops *ops;
	int cnt = 0;

	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
		if (ops_references_rec(ops, rec))
		    cnt++;
	}

	return cnt;
}

5299
void ftrace_release_mod(struct module *mod)
5300 5301
{
	struct dyn_ftrace *rec;
5302
	struct ftrace_page **last_pg;
5303
	struct ftrace_page *pg;
5304
	int order;
5305

5306 5307
	mutex_lock(&ftrace_lock);

5308
	if (ftrace_disabled)
5309
		goto out_unlock;
5310

5311 5312 5313 5314 5315 5316 5317
	/*
	 * Each module has its own ftrace_pages, remove
	 * them from the list.
	 */
	last_pg = &ftrace_pages_start;
	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
		rec = &pg->records[0];
5318
		if (within_module_core(rec->ip, mod)) {
5319
			/*
5320 5321
			 * As core pages are first, the first
			 * page should never be a module page.
5322
			 */
5323 5324 5325 5326 5327 5328 5329 5330
			if (WARN_ON(pg == ftrace_pages_start))
				goto out_unlock;

			/* Check if we are deleting the last page */
			if (pg == ftrace_pages)
				ftrace_pages = next_to_ftrace_page(last_pg);

			*last_pg = pg->next;
5331 5332 5333
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
5334 5335 5336
		} else
			last_pg = &pg->next;
	}
5337
 out_unlock:
5338 5339 5340
	mutex_unlock(&ftrace_lock);
}

5341
void ftrace_module_enable(struct module *mod)
5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409
{
	struct dyn_ftrace *rec;
	struct ftrace_page *pg;

	mutex_lock(&ftrace_lock);

	if (ftrace_disabled)
		goto out_unlock;

	/*
	 * If the tracing is enabled, go ahead and enable the record.
	 *
	 * The reason not to enable the record immediatelly is the
	 * inherent check of ftrace_make_nop/ftrace_make_call for
	 * correct previous instructions.  Making first the NOP
	 * conversion puts the module to the correct state, thus
	 * passing the ftrace_make_call check.
	 *
	 * We also delay this to after the module code already set the
	 * text to read-only, as we now need to set it back to read-write
	 * so that we can modify the text.
	 */
	if (ftrace_start_up)
		ftrace_arch_code_modify_prepare();

	do_for_each_ftrace_rec(pg, rec) {
		int cnt;
		/*
		 * do_for_each_ftrace_rec() is a double loop.
		 * module text shares the pg. If a record is
		 * not part of this module, then skip this pg,
		 * which the "break" will do.
		 */
		if (!within_module_core(rec->ip, mod))
			break;

		cnt = 0;

		/*
		 * When adding a module, we need to check if tracers are
		 * currently enabled and if they are, and can trace this record,
		 * we need to enable the module functions as well as update the
		 * reference counts for those function records.
		 */
		if (ftrace_start_up)
			cnt += referenced_filters(rec);

		/* This clears FTRACE_FL_DISABLED */
		rec->flags = cnt;

		if (ftrace_start_up && cnt) {
			int failed = __ftrace_replace_code(rec, 1);
			if (failed) {
				ftrace_bug(failed, rec);
				goto out_loop;
			}
		}

	} while_for_each_ftrace_rec();

 out_loop:
	if (ftrace_start_up)
		ftrace_arch_code_modify_post_process();

 out_unlock:
	mutex_unlock(&ftrace_lock);
}

5410
void ftrace_module_init(struct module *mod)
5411
{
5412
	if (ftrace_disabled || !mod->num_ftrace_callsites)
5413
		return;
5414

5415 5416
	ftrace_process_locs(mod, mod->ftrace_callsites,
			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5417
}
5418 5419
#endif /* CONFIG_MODULES */

5420
void __init ftrace_free_init_mem(void)
5421
{
5422 5423
	unsigned long start = (unsigned long)(&__init_begin);
	unsigned long end = (unsigned long)(&__init_end);
5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463
	struct ftrace_page **last_pg = &ftrace_pages_start;
	struct ftrace_page *pg;
	struct dyn_ftrace *rec;
	struct dyn_ftrace key;
	int order;

	key.ip = start;
	key.flags = end;	/* overload flags, as it is unsigned long */

	mutex_lock(&ftrace_lock);

	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
		if (end < pg->records[0].ip ||
		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
			continue;
 again:
		rec = bsearch(&key, pg->records, pg->index,
			      sizeof(struct dyn_ftrace),
			      ftrace_cmp_recs);
		if (!rec)
			continue;
		pg->index--;
		if (!pg->index) {
			*last_pg = pg->next;
			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
			free_pages((unsigned long)pg->records, order);
			kfree(pg);
			pg = container_of(last_pg, struct ftrace_page, next);
			if (!(*last_pg))
				ftrace_pages = pg;
			continue;
		}
		memmove(rec, rec + 1,
			(pg->index - (rec - pg->records)) * sizeof(*rec));
		/* More than one function may be in this block */
		goto again;
	}
	mutex_unlock(&ftrace_lock);
}

5464 5465
void __init ftrace_init(void)
{
5466 5467
	extern unsigned long __start_mcount_loc[];
	extern unsigned long __stop_mcount_loc[];
5468
	unsigned long count, flags;
5469 5470 5471
	int ret;

	local_irq_save(flags);
5472
	ret = ftrace_dyn_arch_init();
5473
	local_irq_restore(flags);
5474
	if (ret)
5475 5476 5477
		goto failed;

	count = __stop_mcount_loc - __start_mcount_loc;
5478 5479
	if (!count) {
		pr_info("ftrace: No functions to be traced?\n");
5480
		goto failed;
5481 5482 5483 5484
	}

	pr_info("ftrace: allocating %ld entries in %ld pages\n",
		count, count / ENTRIES_PER_PAGE + 1);
5485 5486 5487

	last_ftrace_enabled = ftrace_enabled = 1;

5488
	ret = ftrace_process_locs(NULL,
5489
				  __start_mcount_loc,
5490 5491
				  __stop_mcount_loc);

5492 5493
	set_ftrace_early_filters();

5494 5495 5496 5497 5498
	return;
 failed:
	ftrace_disabled = 1;
}

5499 5500 5501 5502 5503 5504 5505 5506 5507 5508
/* Do nothing if arch does not support this */
void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
	arch_ftrace_update_trampoline(ops);
}

5509
#else
5510

5511
static struct ftrace_ops global_ops = {
5512
	.func			= ftrace_stub,
5513 5514 5515
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				  FTRACE_OPS_FL_INITIALIZED |
				  FTRACE_OPS_FL_PID,
5516 5517
};

5518 5519 5520 5521 5522
static int __init ftrace_nodyn_init(void)
{
	ftrace_enabled = 1;
	return 0;
}
5523
core_initcall(ftrace_nodyn_init);
5524

5525
static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
5526
static inline void ftrace_startup_enable(int command) { }
5527
static inline void ftrace_startup_all(int command) { }
5528
/* Keep as macros so we do not need to define the commands */
5529 5530 5531 5532 5533 5534
# define ftrace_startup(ops, command)					\
	({								\
		int ___ret = __register_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags |= FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
5535
	})
5536 5537 5538 5539 5540 5541 5542
# define ftrace_shutdown(ops, command)					\
	({								\
		int ___ret = __unregister_ftrace_function(ops);		\
		if (!___ret)						\
			(ops)->flags &= ~FTRACE_OPS_FL_ENABLED;		\
		___ret;							\
	})
5543

I
Ingo Molnar 已提交
5544 5545
# define ftrace_startup_sysctl()	do { } while (0)
# define ftrace_shutdown_sysctl()	do { } while (0)
5546 5547

static inline int
5548
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
5549 5550 5551 5552
{
	return 1;
}

5553 5554 5555 5556
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
}

5557 5558
#endif /* CONFIG_DYNAMIC_FTRACE */

5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581
__init void ftrace_init_global_array_ops(struct trace_array *tr)
{
	tr->ops = &global_ops;
	tr->ops->private = tr;
}

void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
{
	/* If we filter on pids, update to use the pid function */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		if (WARN_ON(tr->ops->func != ftrace_stub))
			printk("ftrace ops had %pS for function\n",
			       tr->ops->func);
	}
	tr->ops->func = func;
	tr->ops->private = tr;
}

void ftrace_reset_array_ops(struct trace_array *tr)
{
	tr->ops->func = ftrace_stub;
}

5582 5583
static inline void
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5584
		       struct ftrace_ops *ignored, struct pt_regs *regs)
5585
{
5586
	struct ftrace_ops *op;
5587
	int bit;
5588

5589 5590 5591
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;
5592

5593 5594 5595 5596 5597
	/*
	 * Some of the ops may be dynamically allocated,
	 * they must be freed after a synchronize_sched().
	 */
	preempt_disable_notrace();
5598

5599
	do_for_each_ftrace_op(op, ftrace_ops_list) {
5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613
		/*
		 * Check the following for each ops before calling their func:
		 *  if RCU flag is set, then rcu_is_watching() must be true
		 *  if PER_CPU is set, then ftrace_function_local_disable()
		 *                          must be false
		 *  Otherwise test if the ip matches the ops filter
		 *
		 * If any of the above fails then the op->func() is not executed.
		 */
		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
		    (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
		     !ftrace_function_local_disabled(op)) &&
		    ftrace_ops_test(op, ip, regs)) {
		    
5614 5615
			if (FTRACE_WARN_ON(!op->func)) {
				pr_warn("op=%p %pS\n", op, op);
5616 5617
				goto out;
			}
5618
			op->func(ip, parent_ip, op, regs);
5619
		}
5620
	} while_for_each_ftrace_op(op);
5621
out:
5622
	preempt_enable_notrace();
5623
	trace_clear_recursion(bit);
5624 5625
}

5626 5627 5628 5629 5630
/*
 * Some archs only support passing ip and parent_ip. Even though
 * the list function ignores the op parameter, we do not want any
 * C side effects, where a function is called without the caller
 * sending a third parameter.
5631 5632 5633
 * Archs are to support both the regs and ftrace_ops at the same time.
 * If they support ftrace_ops, it is assumed they support regs.
 * If call backs want to use regs, they must either check for regs
5634 5635
 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
5636
 * An architecture can pass partial regs with ftrace_ops and still
L
Li Bin 已提交
5637
 * set the ARCH_SUPPORTS_FTRACE_OPS.
5638 5639 5640
 */
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
5641
				 struct ftrace_ops *op, struct pt_regs *regs)
5642
{
5643
	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
5644 5645 5646 5647
}
#else
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
{
5648
	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
5649 5650 5651
}
#endif

5652 5653
/*
 * If there's only one function registered but it does not support
5654 5655
 * recursion, needs RCU protection and/or requires per cpu handling, then
 * this function will be called by the mcount trampoline.
5656
 */
5657
static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
5658 5659 5660 5661
				   struct ftrace_ops *op, struct pt_regs *regs)
{
	int bit;

5662 5663 5664
	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
		return;

5665 5666 5667 5668
	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
	if (bit < 0)
		return;

5669
	preempt_disable_notrace();
5670

5671 5672 5673 5674 5675 5676
	if (!(op->flags & FTRACE_OPS_FL_PER_CPU) ||
	    !ftrace_function_local_disabled(op)) {
		op->func(ip, parent_ip, op, regs);
	}

	preempt_enable_notrace();
5677 5678 5679
	trace_clear_recursion(bit);
}

5680 5681 5682 5683 5684 5685 5686
/**
 * ftrace_ops_get_func - get the function a trampoline should call
 * @ops: the ops to get the function for
 *
 * Normally the mcount trampoline will call the ops->func, but there
 * are times that it should not. For example, if the ops does not
 * have its own recursion protection, then it should call the
5687
 * ftrace_ops_assist_func() instead.
5688 5689 5690 5691 5692 5693
 *
 * Returns the function that the trampoline should call for @ops.
 */
ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
{
	/*
5694 5695
	 * If the function does not handle recursion, needs to be RCU safe,
	 * or does per cpu logic, then we need to call the assist handler.
5696
	 */
5697 5698 5699
	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
	    ops->flags & (FTRACE_OPS_FL_RCU | FTRACE_OPS_FL_PER_CPU))
		return ftrace_ops_assist_func;
5700 5701 5702 5703

	return ops->func;
}

5704 5705 5706
static void
ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
		    struct task_struct *prev, struct task_struct *next)
S
Steven Rostedt 已提交
5707
{
5708 5709
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;
S
Steven Rostedt 已提交
5710

5711
	pid_list = rcu_dereference_sched(tr->function_pids);
5712

5713 5714
	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, next));
S
Steven Rostedt 已提交
5715 5716
}

5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753
static void
ftrace_pid_follow_sched_process_fork(void *data,
				     struct task_struct *self,
				     struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, self, task);
}

static void
ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
{
	struct trace_pid_list *pid_list;
	struct trace_array *tr = data;

	pid_list = rcu_dereference_sched(tr->function_pids);
	trace_filter_add_remove_task(pid_list, NULL, task);
}

void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
{
	if (enable) {
		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						  tr);
		register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						  tr);
	} else {
		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
						    tr);
		unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
						    tr);
	}
}

5754
static void clear_ftrace_pids(struct trace_array *tr)
5755
{
5756 5757
	struct trace_pid_list *pid_list;
	int cpu;
5758

5759 5760 5761 5762
	pid_list = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));
	if (!pid_list)
		return;
5763

5764
	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
5765

5766 5767
	for_each_possible_cpu(cpu)
		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
S
Steven Rostedt 已提交
5768

5769
	rcu_assign_pointer(tr->function_pids, NULL);
S
Steven Rostedt 已提交
5770

5771 5772
	/* Wait till all users are no longer using pid filtering */
	synchronize_sched();
5773

5774
	trace_free_pid_list(pid_list);
5775 5776
}

5777
static void ftrace_pid_reset(struct trace_array *tr)
5778
{
5779
	mutex_lock(&ftrace_lock);
5780
	clear_ftrace_pids(tr);
S
Steven Rostedt 已提交
5781

5782
	ftrace_update_pid_func();
5783
	ftrace_startup_all(0);
5784 5785 5786 5787

	mutex_unlock(&ftrace_lock);
}

5788 5789
/* Greater than any max PID */
#define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
5790

5791
static void *fpid_start(struct seq_file *m, loff_t *pos)
5792
	__acquires(RCU)
5793
{
5794 5795 5796
	struct trace_pid_list *pid_list;
	struct trace_array *tr = m->private;

5797
	mutex_lock(&ftrace_lock);
5798 5799 5800
	rcu_read_lock_sched();

	pid_list = rcu_dereference_sched(tr->function_pids);
5801

5802 5803
	if (!pid_list)
		return !(*pos) ? FTRACE_NO_PIDS : NULL;
5804

5805
	return trace_pid_start(pid_list, pos);
5806 5807 5808 5809
}

static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
5810 5811 5812 5813
	struct trace_array *tr = m->private;
	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);

	if (v == FTRACE_NO_PIDS)
5814 5815
		return NULL;

5816
	return trace_pid_next(pid_list, v, pos);
5817 5818 5819
}

static void fpid_stop(struct seq_file *m, void *p)
5820
	__releases(RCU)
5821
{
5822
	rcu_read_unlock_sched();
5823 5824 5825 5826 5827
	mutex_unlock(&ftrace_lock);
}

static int fpid_show(struct seq_file *m, void *v)
{
5828
	if (v == FTRACE_NO_PIDS) {
5829
		seq_puts(m, "no pid\n");
5830 5831 5832
		return 0;
	}

5833
	return trace_pid_show(m, v);
5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845
}

static const struct seq_operations ftrace_pid_sops = {
	.start = fpid_start,
	.next = fpid_next,
	.stop = fpid_stop,
	.show = fpid_show,
};

static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
5846 5847
	struct trace_array *tr = inode->i_private;
	struct seq_file *m;
5848 5849
	int ret = 0;

5850 5851 5852
	if (trace_array_get(tr) < 0)
		return -ENODEV;

5853 5854
	if ((file->f_mode & FMODE_WRITE) &&
	    (file->f_flags & O_TRUNC))
5855
		ftrace_pid_reset(tr);
5856

5857 5858 5859 5860 5861 5862 5863 5864
	ret = seq_open(file, &ftrace_pid_sops);
	if (ret < 0) {
		trace_array_put(tr);
	} else {
		m = file->private_data;
		/* copy tr over to seq ops */
		m->private = tr;
	}
5865 5866 5867 5868

	return ret;
}

5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884
static void ignore_task_cpu(void *data)
{
	struct trace_array *tr = data;
	struct trace_pid_list *pid_list;

	/*
	 * This function is called by on_each_cpu() while the
	 * event_mutex is held.
	 */
	pid_list = rcu_dereference_protected(tr->function_pids,
					     mutex_is_locked(&ftrace_lock));

	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
		       trace_ignore_this_task(pid_list, current));
}

5885 5886 5887 5888
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
5889 5890 5891 5892 5893
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
	struct trace_pid_list *filtered_pids = NULL;
	struct trace_pid_list *pid_list;
	ssize_t ret;
5894

5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905
	if (!cnt)
		return 0;

	mutex_lock(&ftrace_lock);

	filtered_pids = rcu_dereference_protected(tr->function_pids,
					     lockdep_is_held(&ftrace_lock));

	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
	if (ret < 0)
		goto out;
5906

5907
	rcu_assign_pointer(tr->function_pids, pid_list);
5908

5909 5910 5911 5912 5913 5914 5915
	if (filtered_pids) {
		synchronize_sched();
		trace_free_pid_list(filtered_pids);
	} else if (pid_list) {
		/* Register a probe to set whether to ignore the tracing of a task */
		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
	}
5916

5917
	/*
5918 5919 5920
	 * Ignoring of pids is done at task switch. But we have to
	 * check for those tasks that are currently running.
	 * Always do this in case a pid was appended or removed.
5921
	 */
5922
	on_each_cpu(ignore_task_cpu, tr, 1);
5923

5924 5925 5926 5927
	ftrace_update_pid_func();
	ftrace_startup_all(0);
 out:
	mutex_unlock(&ftrace_lock);
5928

5929 5930
	if (ret > 0)
		*ppos += ret;
5931

5932
	return ret;
5933
}
5934

5935 5936 5937
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
5938
	struct trace_array *tr = inode->i_private;
5939

5940 5941 5942
	trace_array_put(tr);

	return seq_release(inode, file);
5943 5944
}

5945
static const struct file_operations ftrace_pid_fops = {
5946 5947 5948
	.open		= ftrace_pid_open,
	.write		= ftrace_pid_write,
	.read		= seq_read,
5949
	.llseek		= tracing_lseek,
5950
	.release	= ftrace_pid_release,
5951 5952
};

5953
void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
5954
{
5955
	trace_create_file("set_ftrace_pid", 0644, d_tracer,
5956
			    tr, &ftrace_pid_fops);
5957 5958
}

5959 5960 5961 5962 5963 5964 5965 5966 5967 5968
void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
					 struct dentry *d_tracer)
{
	/* Only the top level directory has the dyn_tracefs and profile */
	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));

	ftrace_init_dyn_tracefs(d_tracer);
	ftrace_profile_tracefs(d_tracer);
}

S
Steven Rostedt 已提交
5969
/**
5970
 * ftrace_kill - kill ftrace
S
Steven Rostedt 已提交
5971 5972 5973 5974 5975
 *
 * This function should be used by panic code. It stops ftrace
 * but in a not so nice way. If you need to simply kill ftrace
 * from a non-atomic section, use ftrace_kill.
 */
5976
void ftrace_kill(void)
S
Steven Rostedt 已提交
5977 5978 5979 5980 5981 5982
{
	ftrace_disabled = 1;
	ftrace_enabled = 0;
	clear_ftrace_function();
}

5983 5984 5985 5986 5987 5988 5989 5990
/**
 * Test if ftrace is dead or not.
 */
int ftrace_is_dead(void)
{
	return ftrace_disabled;
}

5991
/**
5992 5993
 * register_ftrace_function - register a function for profiling
 * @ops - ops structure that holds the function for profiling.
5994
 *
5995 5996 5997 5998 5999 6000
 * Register a function to be called by all functions in the
 * kernel.
 *
 * Note: @ops->func and all the functions it calls must be labeled
 *       with "notrace", otherwise it will go into a
 *       recursive loop.
6001
 */
6002
int register_ftrace_function(struct ftrace_ops *ops)
6003
{
6004
	int ret = -1;
6005

6006 6007
	ftrace_ops_init(ops);

S
Steven Rostedt 已提交
6008
	mutex_lock(&ftrace_lock);
6009

6010
	ret = ftrace_startup(ops, 0);
6011

S
Steven Rostedt 已提交
6012
	mutex_unlock(&ftrace_lock);
6013

6014
	return ret;
6015
}
6016
EXPORT_SYMBOL_GPL(register_ftrace_function);
6017 6018

/**
6019
 * unregister_ftrace_function - unregister a function for profiling.
6020 6021 6022 6023 6024 6025 6026 6027
 * @ops - ops structure that holds the function to unregister
 *
 * Unregister a function that was added to be called by ftrace profiling.
 */
int unregister_ftrace_function(struct ftrace_ops *ops)
{
	int ret;

S
Steven Rostedt 已提交
6028
	mutex_lock(&ftrace_lock);
6029
	ret = ftrace_shutdown(ops, 0);
S
Steven Rostedt 已提交
6030
	mutex_unlock(&ftrace_lock);
6031 6032 6033

	return ret;
}
6034
EXPORT_SYMBOL_GPL(unregister_ftrace_function);
6035

I
Ingo Molnar 已提交
6036
int
6037
ftrace_enable_sysctl(struct ctl_table *table, int write,
6038
		     void __user *buffer, size_t *lenp,
6039 6040
		     loff_t *ppos)
{
6041
	int ret = -ENODEV;
6042

S
Steven Rostedt 已提交
6043
	mutex_lock(&ftrace_lock);
6044

6045 6046 6047 6048
	if (unlikely(ftrace_disabled))
		goto out;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);
6049

6050
	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
6051 6052
		goto out;

6053
	last_ftrace_enabled = !!ftrace_enabled;
6054 6055 6056 6057

	if (ftrace_enabled) {

		/* we are starting ftrace again */
6058 6059
		if (ftrace_ops_list != &ftrace_list_end)
			update_ftrace_function();
6060

6061 6062
		ftrace_startup_sysctl();

6063 6064 6065 6066 6067 6068 6069 6070
	} else {
		/* stopping ftrace calls (just send to ftrace_stub) */
		ftrace_trace_function = ftrace_stub;

		ftrace_shutdown_sysctl();
	}

 out:
S
Steven Rostedt 已提交
6071
	mutex_unlock(&ftrace_lock);
6072
	return ret;
6073
}
6074

6075
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
6076

6077 6078 6079 6080
static struct ftrace_ops graph_ops = {
	.func			= ftrace_stub,
	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
				   FTRACE_OPS_FL_INITIALIZED |
6081
				   FTRACE_OPS_FL_PID |
6082 6083 6084
				   FTRACE_OPS_FL_STUB,
#ifdef FTRACE_GRAPH_TRAMP_ADDR
	.trampoline		= FTRACE_GRAPH_TRAMP_ADDR,
6085
	/* trampoline_size is only needed for dynamically allocated tramps */
6086 6087 6088 6089
#endif
	ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
};

6090 6091 6092 6093 6094 6095 6096 6097 6098 6099
void ftrace_graph_sleep_time_control(bool enable)
{
	fgraph_sleep_time = enable;
}

void ftrace_graph_graph_time_control(bool enable)
{
	fgraph_graph_time = enable;
}

6100 6101 6102 6103 6104
int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
{
	return 0;
}

6105 6106 6107
/* The callbacks that hook a function */
trace_func_graph_ret_t ftrace_graph_return =
			(trace_func_graph_ret_t)ftrace_stub;
6108
trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
6109
static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130

/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
{
	int i;
	int ret = 0;
	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
	struct task_struct *g, *t;

	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
					* sizeof(struct ftrace_ret_stack),
					GFP_KERNEL);
		if (!ret_stack_list[i]) {
			start = 0;
			end = i;
			ret = -ENOMEM;
			goto free;
		}
	}

6131
	read_lock(&tasklist_lock);
6132 6133 6134 6135 6136 6137 6138
	do_each_thread(g, t) {
		if (start == end) {
			ret = -EAGAIN;
			goto unlock;
		}

		if (t->ret_stack == NULL) {
6139
			atomic_set(&t->tracing_graph_pause, 0);
6140
			atomic_set(&t->trace_overrun, 0);
6141 6142 6143 6144
			t->curr_ret_stack = -1;
			/* Make sure the tasks see the -1 first: */
			smp_wmb();
			t->ret_stack = ret_stack_list[start++];
6145 6146 6147 6148
		}
	} while_each_thread(g, t);

unlock:
6149
	read_unlock(&tasklist_lock);
6150 6151 6152 6153 6154 6155
free:
	for (i = start; i < end; i++)
		kfree(ret_stack_list[i]);
	return ret;
}

6156
static void
6157
ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
6158
			struct task_struct *prev, struct task_struct *next)
6159 6160 6161 6162
{
	unsigned long long timestamp;
	int index;

6163 6164 6165 6166
	/*
	 * Does the user want to count the time a function was asleep.
	 * If so, do not update the time stamps.
	 */
6167
	if (fgraph_sleep_time)
6168 6169
		return;

6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187
	timestamp = trace_clock_local();

	prev->ftrace_timestamp = timestamp;

	/* only process tasks that we timestamped */
	if (!next->ftrace_timestamp)
		return;

	/*
	 * Update all the counters in next to make up for the
	 * time next was sleeping.
	 */
	timestamp -= next->ftrace_timestamp;

	for (index = next->curr_ret_stack; index >= 0; index--)
		next->ret_stack[index].calltime += timestamp;
}

6188
/* Allocate a return stack for each task */
6189
static int start_graph_tracing(void)
6190 6191
{
	struct ftrace_ret_stack **ret_stack_list;
6192
	int ret, cpu;
6193 6194 6195 6196 6197 6198 6199 6200

	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
				sizeof(struct ftrace_ret_stack *),
				GFP_KERNEL);

	if (!ret_stack_list)
		return -ENOMEM;

6201
	/* The cpu_boot init_task->ret_stack will never be freed */
6202 6203
	for_each_online_cpu(cpu) {
		if (!idle_task(cpu)->ret_stack)
6204
			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
6205
	}
6206

6207 6208 6209 6210
	do {
		ret = alloc_retstack_tasklist(ret_stack_list);
	} while (ret == -EAGAIN);

6211
	if (!ret) {
6212
		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6213 6214 6215 6216 6217
		if (ret)
			pr_info("ftrace_graph: Couldn't activate tracepoint"
				" probe to kernel_sched_switch\n");
	}

6218 6219 6220 6221
	kfree(ret_stack_list);
	return ret;
}

6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242
/*
 * Hibernation protection.
 * The state of the current task is too much unstable during
 * suspend/restore to disk. We want to protect against that.
 */
static int
ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
							void *unused)
{
	switch (state) {
	case PM_HIBERNATION_PREPARE:
		pause_graph_tracing();
		break;

	case PM_POST_HIBERNATION:
		unpause_graph_tracing();
		break;
	}
	return NOTIFY_DONE;
}

6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258
static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
{
	if (!ftrace_ops_test(&global_ops, trace->func, NULL))
		return 0;
	return __ftrace_graph_entry(trace);
}

/*
 * The function graph tracer should only trace the functions defined
 * by set_ftrace_filter and set_ftrace_notrace. If another function
 * tracer ops is registered, the graph tracer requires testing the
 * function against the global ops, and not just trace any function
 * that any ftrace_ops registered.
 */
static void update_function_graph_func(void)
{
6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277
	struct ftrace_ops *op;
	bool do_test = false;

	/*
	 * The graph and global ops share the same set of functions
	 * to test. If any other ops is on the list, then
	 * the graph tracing needs to test if its the function
	 * it should call.
	 */
	do_for_each_ftrace_op(op, ftrace_ops_list) {
		if (op != &global_ops && op != &graph_ops &&
		    op != &ftrace_list_end) {
			do_test = true;
			/* in double loop, break out with goto */
			goto out;
		}
	} while_for_each_ftrace_op(op);
 out:
	if (do_test)
6278
		ftrace_graph_entry = ftrace_graph_entry_test;
6279 6280
	else
		ftrace_graph_entry = __ftrace_graph_entry;
6281 6282
}

6283 6284 6285 6286
static struct notifier_block ftrace_suspend_notifier = {
	.notifier_call = ftrace_suspend_notifier_call,
};

6287 6288
int register_ftrace_graph(trace_func_graph_ret_t retfunc,
			trace_func_graph_ent_t entryfunc)
6289
{
6290 6291
	int ret = 0;

S
Steven Rostedt 已提交
6292
	mutex_lock(&ftrace_lock);
6293

6294
	/* we currently allow only one tracer registered at a time */
6295
	if (ftrace_graph_active) {
6296 6297 6298 6299
		ret = -EBUSY;
		goto out;
	}

6300 6301
	register_pm_notifier(&ftrace_suspend_notifier);

6302
	ftrace_graph_active++;
6303
	ret = start_graph_tracing();
6304
	if (ret) {
6305
		ftrace_graph_active--;
6306 6307
		goto out;
	}
6308

6309
	ftrace_graph_return = retfunc;
6310 6311 6312 6313 6314 6315 6316 6317 6318 6319

	/*
	 * Update the indirect function to the entryfunc, and the
	 * function that gets called to the entry_test first. Then
	 * call the update fgraph entry function to determine if
	 * the entryfunc should be called directly or not.
	 */
	__ftrace_graph_entry = entryfunc;
	ftrace_graph_entry = ftrace_graph_entry_test;
	update_function_graph_func();
6320

6321
	ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
6322
out:
S
Steven Rostedt 已提交
6323
	mutex_unlock(&ftrace_lock);
6324
	return ret;
6325 6326
}

6327
void unregister_ftrace_graph(void)
6328
{
S
Steven Rostedt 已提交
6329
	mutex_lock(&ftrace_lock);
6330

6331
	if (unlikely(!ftrace_graph_active))
6332 6333
		goto out;

6334
	ftrace_graph_active--;
6335
	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
6336
	ftrace_graph_entry = ftrace_graph_entry_stub;
6337
	__ftrace_graph_entry = ftrace_graph_entry_stub;
6338
	ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
6339
	unregister_pm_notifier(&ftrace_suspend_notifier);
6340
	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6341

6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352
#ifdef CONFIG_DYNAMIC_FTRACE
	/*
	 * Function graph does not allocate the trampoline, but
	 * other global_ops do. We need to reset the ALLOC_TRAMP flag
	 * if one was used.
	 */
	global_ops.trampoline = save_global_trampoline;
	if (save_global_flags & FTRACE_OPS_FL_ALLOC_TRAMP)
		global_ops.flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
#endif

6353
 out:
S
Steven Rostedt 已提交
6354
	mutex_unlock(&ftrace_lock);
6355
}
6356

6357 6358 6359 6360 6361 6362 6363 6364
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);

static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
	atomic_set(&t->tracing_graph_pause, 0);
	atomic_set(&t->trace_overrun, 0);
	t->ftrace_timestamp = 0;
L
Lucas De Marchi 已提交
6365
	/* make curr_ret_stack visible before we add the ret_stack */
6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399
	smp_wmb();
	t->ret_stack = ret_stack;
}

/*
 * Allocate a return stack for the idle task. May be the first
 * time through, or it may be done by CPU hotplug online.
 */
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
{
	t->curr_ret_stack = -1;
	/*
	 * The idle task has no parent, it either has its own
	 * stack or no stack at all.
	 */
	if (t->ret_stack)
		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));

	if (ftrace_graph_active) {
		struct ftrace_ret_stack *ret_stack;

		ret_stack = per_cpu(idle_ret_stack, cpu);
		if (!ret_stack) {
			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
					    * sizeof(struct ftrace_ret_stack),
					    GFP_KERNEL);
			if (!ret_stack)
				return;
			per_cpu(idle_ret_stack, cpu) = ret_stack;
		}
		graph_init_task(t, ret_stack);
	}
}

6400
/* Allocate a return stack for newly created task */
6401
void ftrace_graph_init_task(struct task_struct *t)
6402
{
6403 6404
	/* Make sure we do not use the parent ret_stack */
	t->ret_stack = NULL;
6405
	t->curr_ret_stack = -1;
6406

6407
	if (ftrace_graph_active) {
6408 6409 6410
		struct ftrace_ret_stack *ret_stack;

		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
6411 6412
				* sizeof(struct ftrace_ret_stack),
				GFP_KERNEL);
6413
		if (!ret_stack)
6414
			return;
6415
		graph_init_task(t, ret_stack);
6416
	}
6417 6418
}

6419
void ftrace_graph_exit_task(struct task_struct *t)
6420
{
6421 6422
	struct ftrace_ret_stack	*ret_stack = t->ret_stack;

6423
	t->ret_stack = NULL;
6424 6425 6426 6427
	/* NULL must become visible to IRQs before we free it: */
	barrier();

	kfree(ret_stack);
6428
}
6429
#endif