trace.c 132.8 KB
Newer Older
1 2 3
/*
 * ring buffer based function tracer
 *
4
 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 6 7 8 9 10 11
 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally taken from the RT patch by:
 *    Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code from the latency_tracer, that is:
 *  Copyright (C) 2004-2006 Ingo Molnar
12
 *  Copyright (C) 2004 Nadia Yvette Chambers
13
 */
14
#include <linux/ring_buffer.h>
15
#include <generated/utsrelease.h>
16 17
#include <linux/stacktrace.h>
#include <linux/writeback.h>
18 19
#include <linux/kallsyms.h>
#include <linux/seq_file.h>
20
#include <linux/notifier.h>
21
#include <linux/irqflags.h>
22
#include <linux/irq_work.h>
23
#include <linux/debugfs.h>
24
#include <linux/pagemap.h>
25 26 27
#include <linux/hardirq.h>
#include <linux/linkage.h>
#include <linux/uaccess.h>
28
#include <linux/kprobes.h>
29 30 31
#include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/percpu.h>
32
#include <linux/splice.h>
33
#include <linux/kdebug.h>
34
#include <linux/string.h>
35
#include <linux/rwsem.h>
36
#include <linux/slab.h>
37 38
#include <linux/ctype.h>
#include <linux/init.h>
39
#include <linux/poll.h>
40
#include <linux/nmi.h>
41
#include <linux/fs.h>
42
#include <linux/sched/rt.h>
I
Ingo Molnar 已提交
43

44
#include "trace.h"
45
#include "trace_output.h"
46

47 48 49 50
/*
 * On boot up, the ring buffer is set to the minimum size, so that
 * we do not waste memory on systems that are not using tracing.
 */
51
int ring_buffer_expanded;
52

53 54
/*
 * We need to change this state when a selftest is running.
55 56
 * A selftest will lurk into the ring-buffer to count the
 * entries inserted during the selftest although some concurrent
57
 * insertions into the ring-buffer such as trace_printk could occurred
58 59
 * at the same time, giving false positive or negative results.
 */
60
static bool __read_mostly tracing_selftest_running;
61

62 63 64
/*
 * If a tracer is running, we do not want to run SELFTEST.
 */
65
bool __read_mostly tracing_selftest_disabled;
66

67 68 69 70 71 72 73 74 75 76 77 78 79 80
/* For tracers that don't implement custom flags */
static struct tracer_opt dummy_tracer_opt[] = {
	{ }
};

static struct tracer_flags dummy_tracer_flags = {
	.val = 0,
	.opts = dummy_tracer_opt
};

static int dummy_set_flag(u32 old_flags, u32 bit, int set)
{
	return 0;
}
81

82 83 84 85 86 87 88
/*
 * To prevent the comm cache from being overwritten when no
 * tracing is active, only save the comm when a trace event
 * occurred.
 */
static DEFINE_PER_CPU(bool, trace_cmdline_save);

89 90 91 92 93 94 95 96
/*
 * When a reader is waiting for data, then this variable is
 * set to true.
 */
static bool trace_wakeup_needed;

static struct irq_work trace_work_wakeup;

97 98 99 100 101 102
/*
 * Kill all tracing for good (never come back).
 * It is initialized to 1 but will turn to zero if the initialization
 * of the tracer is successful. But that is the only place that sets
 * this back to zero.
 */
103
static int tracing_disabled = 1;
104

105
DEFINE_PER_CPU(int, ftrace_cpu_disabled);
106

107
cpumask_var_t __read_mostly	tracing_buffer_mask;
108

109 110 111 112 113 114 115 116 117 118 119
/*
 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
 *
 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
 * is set, then ftrace_dump is called. This will output the contents
 * of the ftrace buffers to the console.  This is very useful for
 * capturing traces that lead to crashes and outputing it to a
 * serial console.
 *
 * It is default off, but you can enable it with either specifying
 * "ftrace_dump_on_oops" in the kernel command line, or setting
120 121 122
 * /proc/sys/kernel/ftrace_dump_on_oops
 * Set 1 if you want to dump buffers of all CPUs
 * Set 2 if you want to dump the buffer of the CPU that triggered oops
123
 */
124 125

enum ftrace_dump_mode ftrace_dump_on_oops;
126

127 128
static int tracing_set_tracer(const char *buf);

L
Li Zefan 已提交
129 130
#define MAX_TRACER_SIZE		100
static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
131
static char *default_bootup_tracer;
132

133
static int __init set_cmdline_ftrace(char *str)
134
{
L
Li Zefan 已提交
135
	strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
136
	default_bootup_tracer = bootup_tracer_buf;
137 138
	/* We are using ftrace early, expand it */
	ring_buffer_expanded = 1;
139 140
	return 1;
}
141
__setup("ftrace=", set_cmdline_ftrace);
142

143 144
static int __init set_ftrace_dump_on_oops(char *str)
{
145 146 147 148 149 150 151 152 153 154 155
	if (*str++ != '=' || !*str) {
		ftrace_dump_on_oops = DUMP_ALL;
		return 1;
	}

	if (!strcmp("orig_cpu", str)) {
		ftrace_dump_on_oops = DUMP_ORIG;
                return 1;
        }

        return 0;
156 157
}
__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
S
Steven Rostedt 已提交
158

159 160 161 162 163 164 165 166 167 168 169 170

static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
static char *trace_boot_options __initdata;

static int __init set_trace_boot_options(char *str)
{
	strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
	trace_boot_options = trace_boot_options_buf;
	return 0;
}
__setup("trace_options=", set_trace_boot_options);

171
unsigned long long ns2usecs(cycle_t nsec)
172 173 174 175 176 177
{
	nsec += 500;
	do_div(nsec, 1000);
	return nsec;
}

S
Steven Rostedt 已提交
178 179 180 181 182 183 184 185 186 187 188 189
/*
 * The global_trace is the descriptor that holds the tracing
 * buffers for the live tracing. For each CPU, it contains
 * a link list of pages that will store trace entries. The
 * page descriptor of the pages in the memory is used to hold
 * the link list by linking the lru item in the page descriptor
 * to each of the pages in the buffer per CPU.
 *
 * For each active CPU there is a data field that holds the
 * pages for the buffer for that CPU. Each CPU has the same number
 * of pages allocated for its buffer.
 */
190 191
static struct trace_array	global_trace;

192 193
LIST_HEAD(ftrace_trace_arrays);

194 195
int filter_current_check_discard(struct ring_buffer *buffer,
				 struct ftrace_event_call *call, void *rec,
196 197
				 struct ring_buffer_event *event)
{
198
	return filter_check_discard(call, rec, buffer, event);
199
}
200
EXPORT_SYMBOL_GPL(filter_current_check_discard);
201

202 203 204 205 206 207 208 209 210 211 212 213 214
cycle_t ftrace_now(int cpu)
{
	u64 ts;

	/* Early boot up does not have a buffer yet */
	if (!global_trace.buffer)
		return trace_clock_local();

	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);

	return ts;
}
215

S
Steven Rostedt 已提交
216 217 218 219 220 221 222 223 224 225
/*
 * The max_tr is used to snapshot the global_trace when a maximum
 * latency is reached. Some tracers will use this to store a maximum
 * trace while it continues examining live traces.
 *
 * The buffers for the max_tr are set up the same as the global_trace.
 * When a snapshot is taken, the link list of the max_tr is swapped
 * with the link list of the global_trace and the buffers are reset for
 * the global_trace so the tracing can continue.
 */
226 227
static struct trace_array	max_tr;

228 229
int tracing_is_enabled(void)
{
230
	return tracing_is_on();
231 232
}

S
Steven Rostedt 已提交
233
/*
234 235 236
 * trace_buf_size is the size in bytes that is allocated
 * for a buffer. Note, the number of bytes is always rounded
 * to page size.
237 238 239 240 241
 *
 * This number is purposely set to a low number of 16384.
 * If the dump on oops happens, it will be much appreciated
 * to not have to wait for all that output. Anyway this can be
 * boot time and run time configurable.
S
Steven Rostedt 已提交
242
 */
243
#define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */
244

245
static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
246

S
Steven Rostedt 已提交
247
/* trace_types holds a link list of available tracers. */
248
static struct tracer		*trace_types __read_mostly;
S
Steven Rostedt 已提交
249 250 251 252

/*
 * trace_types_lock is used to protect the trace_types list.
 */
253
static DEFINE_MUTEX(trace_types_lock);
S
Steven Rostedt 已提交
254

255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
/*
 * serialize the access of the ring buffer
 *
 * ring buffer serializes readers, but it is low level protection.
 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 * are not protected by ring buffer.
 *
 * The content of events may become garbage if we allow other process consumes
 * these events concurrently:
 *   A) the page of the consumed events may become a normal page
 *      (not reader page) in ring buffer, and this page will be rewrited
 *      by events producer.
 *   B) The page of the consumed events may become a page for splice_read,
 *      and this page will be returned to system.
 *
 * These primitives allow multi process access to different cpu ring buffer
 * concurrently.
 *
 * These primitives don't distinguish read-only and read-consume access.
 * Multi read-only access are also serialized.
 */

#ifdef CONFIG_SMP
static DECLARE_RWSEM(all_cpu_access_lock);
static DEFINE_PER_CPU(struct mutex, cpu_access_lock);

static inline void trace_access_lock(int cpu)
{
283
	if (cpu == RING_BUFFER_ALL_CPUS) {
284 285 286 287 288
		/* gain it for accessing the whole ring buffer. */
		down_write(&all_cpu_access_lock);
	} else {
		/* gain it for accessing a cpu ring buffer. */

289
		/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
290 291 292 293 294 295 296 297 298
		down_read(&all_cpu_access_lock);

		/* Secondly block other access to this @cpu ring buffer. */
		mutex_lock(&per_cpu(cpu_access_lock, cpu));
	}
}

static inline void trace_access_unlock(int cpu)
{
299
	if (cpu == RING_BUFFER_ALL_CPUS) {
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
		up_write(&all_cpu_access_lock);
	} else {
		mutex_unlock(&per_cpu(cpu_access_lock, cpu));
		up_read(&all_cpu_access_lock);
	}
}

static inline void trace_access_lock_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu)
		mutex_init(&per_cpu(cpu_access_lock, cpu));
}

#else

static DEFINE_MUTEX(access_lock);

static inline void trace_access_lock(int cpu)
{
	(void)cpu;
	mutex_lock(&access_lock);
}

static inline void trace_access_unlock(int cpu)
{
	(void)cpu;
	mutex_unlock(&access_lock);
}

static inline void trace_access_lock_init(void)
{
}

#endif

S
Steven Rostedt 已提交
337
/* trace_wait is a waitqueue for tasks blocked on trace_poll */
338 339
static DECLARE_WAIT_QUEUE_HEAD(trace_wait);

340
/* trace_flags holds trace_options default values */
341
unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
342
	TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
343
	TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
344
	TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS;
345

346 347 348 349 350 351 352 353
/**
 * trace_wake_up - wake up tasks waiting for trace input
 *
 * Schedules a delayed work to wake up any task that is blocked on the
 * trace_wait queue. These is used with trace_poll for tasks polling the
 * trace.
 */
static void trace_wake_up(struct irq_work *work)
354
{
355
	wake_up_all(&trace_wait);
356

357
}
358

359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
/**
 * tracing_on - enable tracing buffers
 *
 * This function enables tracing buffers that may have been
 * disabled with tracing_off.
 */
void tracing_on(void)
{
	if (global_trace.buffer)
		ring_buffer_record_on(global_trace.buffer);
	/*
	 * This flag is only looked at when buffers haven't been
	 * allocated yet. We don't really care about the race
	 * between setting this flag and actually turning
	 * on the buffer.
	 */
	global_trace.buffer_disabled = 0;
}
EXPORT_SYMBOL_GPL(tracing_on);

/**
 * tracing_off - turn off tracing buffers
 *
 * This function stops the tracing buffers from recording data.
 * It does not disable any overhead the tracers themselves may
 * be causing. This function simply causes all recording to
 * the ring buffers to fail.
 */
void tracing_off(void)
{
	if (global_trace.buffer)
390
		ring_buffer_record_off(global_trace.buffer);
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
	/*
	 * This flag is only looked at when buffers haven't been
	 * allocated yet. We don't really care about the race
	 * between setting this flag and actually turning
	 * on the buffer.
	 */
	global_trace.buffer_disabled = 1;
}
EXPORT_SYMBOL_GPL(tracing_off);

/**
 * tracing_is_on - show state of ring buffers enabled
 */
int tracing_is_on(void)
{
	if (global_trace.buffer)
		return ring_buffer_record_is_on(global_trace.buffer);
	return !global_trace.buffer_disabled;
}
EXPORT_SYMBOL_GPL(tracing_is_on);

412
static int __init set_buf_size(char *str)
413
{
414
	unsigned long buf_size;
415

416 417
	if (!str)
		return 0;
418
	buf_size = memparse(str, &str);
419
	/* nr_entries can not be zero */
420
	if (buf_size == 0)
421
		return 0;
422
	trace_buf_size = buf_size;
423 424
	return 1;
}
425
__setup("trace_buf_size=", set_buf_size);
426

427 428
static int __init set_tracing_thresh(char *str)
{
429
	unsigned long threshold;
430 431 432 433
	int ret;

	if (!str)
		return 0;
434
	ret = kstrtoul(str, 0, &threshold);
435 436
	if (ret < 0)
		return 0;
437
	tracing_thresh = threshold * 1000;
438 439 440 441
	return 1;
}
__setup("tracing_thresh=", set_tracing_thresh);

S
Steven Rostedt 已提交
442 443 444 445 446
unsigned long nsecs_to_usecs(unsigned long nsecs)
{
	return nsecs / 1000;
}

S
Steven Rostedt 已提交
447
/* These must match the bit postions in trace_iterator_flags */
448 449 450 451 452
static const char *trace_options[] = {
	"print-parent",
	"sym-offset",
	"sym-addr",
	"verbose",
I
Ingo Molnar 已提交
453
	"raw",
454
	"hex",
I
Ingo Molnar 已提交
455
	"bin",
456
	"block",
I
Ingo Molnar 已提交
457
	"stacktrace",
458
	"trace_printk",
459
	"ftrace_preempt",
460
	"branch",
461
	"annotate",
462
	"userstacktrace",
463
	"sym-userobj",
464
	"printk-msg-only",
465
	"context-info",
466
	"latency-format",
467
	"sleep-time",
468
	"graph-time",
469
	"record-cmd",
470
	"overwrite",
471
	"disable_on_free",
472
	"irq-info",
473
	"markers",
474 475 476
	NULL
};

477 478 479
static struct {
	u64 (*func)(void);
	const char *name;
480
	int in_ns;		/* is this clock in nanoseconds? */
481
} trace_clocks[] = {
482 483 484
	{ trace_clock_local,	"local",	1 },
	{ trace_clock_global,	"global",	1 },
	{ trace_clock_counter,	"counter",	0 },
D
David Sharp 已提交
485
	ARCH_TRACE_CLOCKS
486 487 488 489
};

int trace_clock_id;

490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
/*
 * trace_parser_get_init - gets the buffer for trace parser
 */
int trace_parser_get_init(struct trace_parser *parser, int size)
{
	memset(parser, 0, sizeof(*parser));

	parser->buffer = kmalloc(size, GFP_KERNEL);
	if (!parser->buffer)
		return 1;

	parser->size = size;
	return 0;
}

/*
 * trace_parser_put - frees the buffer for trace parser
 */
void trace_parser_put(struct trace_parser *parser)
{
	kfree(parser->buffer);
}

/*
 * trace_get_user - reads the user input string separated by  space
 * (matched by isspace(ch))
 *
 * For each string found the 'struct trace_parser' is updated,
 * and the function returns.
 *
 * Returns number of bytes read.
 *
 * See kernel/trace/trace.h for 'struct trace_parser' details.
 */
int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
	size_t cnt, loff_t *ppos)
{
	char ch;
	size_t read = 0;
	ssize_t ret;

	if (!*ppos)
		trace_parser_clear(parser);

	ret = get_user(ch, ubuf++);
	if (ret)
		goto out;

	read++;
	cnt--;

	/*
	 * The parser is not finished with the last write,
	 * continue reading the user input without skipping spaces.
	 */
	if (!parser->cont) {
		/* skip white space */
		while (cnt && isspace(ch)) {
			ret = get_user(ch, ubuf++);
			if (ret)
				goto out;
			read++;
			cnt--;
		}

		/* only spaces were written */
		if (isspace(ch)) {
			*ppos += read;
			ret = read;
			goto out;
		}

		parser->idx = 0;
	}

	/* read the non-space input */
	while (cnt && !isspace(ch)) {
567
		if (parser->idx < parser->size - 1)
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
			parser->buffer[parser->idx++] = ch;
		else {
			ret = -EINVAL;
			goto out;
		}
		ret = get_user(ch, ubuf++);
		if (ret)
			goto out;
		read++;
		cnt--;
	}

	/* We either got finished input or we have to wait for another call. */
	if (isspace(ch)) {
		parser->buffer[parser->idx] = 0;
		parser->cont = false;
	} else {
		parser->cont = true;
		parser->buffer[parser->idx++] = ch;
	}

	*ppos += read;
	ret = read;

out:
	return ret;
}

596 597 598 599 600
ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
{
	int len;
	int ret;

601 602 603
	if (!cnt)
		return 0;

604 605 606 607 608 609 610
	if (s->len <= s->readpos)
		return -EBUSY;

	len = s->len - s->readpos;
	if (cnt > len)
		cnt = len;
	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
611
	if (ret == cnt)
612 613
		return -EFAULT;

614 615
	cnt -= ret;

616
	s->readpos += cnt;
617
	return cnt;
S
Steven Rostedt 已提交
618 619
}

620
static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
621 622 623 624 625 626 627 628 629
{
	int len;

	if (s->len <= s->readpos)
		return -EBUSY;

	len = s->len - s->readpos;
	if (cnt > len)
		cnt = len;
630
	memcpy(buf, s->buffer + s->readpos, cnt);
631

632
	s->readpos += cnt;
633 634 635
	return cnt;
}

636 637 638 639 640 641
/*
 * ftrace_max_lock is used to protect the swapping of buffers
 * when taking a max snapshot. The buffers themselves are
 * protected by per_cpu spinlocks. But the action of the swap
 * needs its own lock.
 *
642
 * This is defined as a arch_spinlock_t in order to help
643 644 645 646 647 648
 * with performance when lockdep debugging is enabled.
 *
 * It is also used in other places outside the update_max_tr
 * so it needs to be defined outside of the
 * CONFIG_TRACER_MAX_TRACE.
 */
649
static arch_spinlock_t ftrace_max_lock =
650
	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
651

652 653
unsigned long __read_mostly	tracing_thresh;

654 655 656 657 658 659 660 661 662 663 664
#ifdef CONFIG_TRACER_MAX_TRACE
unsigned long __read_mostly	tracing_max_latency;

/*
 * Copy the new maximum trace into the separate maximum-trace
 * structure. (this way the maximum trace is permanently saved,
 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
 */
static void
__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
665
	struct trace_array_cpu *data = per_cpu_ptr(tr->data, cpu);
666
	struct trace_array_cpu *max_data;
667 668 669 670

	max_tr.cpu = cpu;
	max_tr.time_start = data->preempt_timestamp;

671
	max_data = per_cpu_ptr(max_tr.data, cpu);
672 673 674
	max_data->saved_latency = tracing_max_latency;
	max_data->critical_start = data->critical_start;
	max_data->critical_end = data->critical_end;
675

676
	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
677 678 679 680 681
	max_data->pid = tsk->pid;
	max_data->uid = task_uid(tsk);
	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
	max_data->policy = tsk->policy;
	max_data->rt_priority = tsk->rt_priority;
682 683 684 685 686

	/* record this tasks comm */
	tracing_record_cmdline(tsk);
}

S
Steven Rostedt 已提交
687 688 689 690 691 692 693 694 695
/**
 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 * @tr: tracer
 * @tsk: the task with the latency
 * @cpu: The cpu that initiated the trace.
 *
 * Flip the buffers between the @tr and the max_tr and record information
 * about which task was the cause of this latency.
 */
I
Ingo Molnar 已提交
696
void
697 698
update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
699
	struct ring_buffer *buf;
700

701
	if (tr->stop_count)
702 703
		return;

704
	WARN_ON_ONCE(!irqs_disabled());
705

706
	if (!tr->current_trace->allocated_snapshot) {
707
		/* Only the nop tracer should hit this when disabling */
708
		WARN_ON_ONCE(tr->current_trace != &nop_trace);
709
		return;
710
	}
711

712
	arch_spin_lock(&ftrace_max_lock);
713

714
	buf = tr->buffer;
715 716 717
	tr->buffer = max_tr.buffer;
	max_tr.buffer = buf;

718
	__update_max_tr(tr, tsk, cpu);
719
	arch_spin_unlock(&ftrace_max_lock);
720 721 722 723 724 725 726
}

/**
 * update_max_tr_single - only copy one trace over, and reset the rest
 * @tr - tracer
 * @tsk - task with the latency
 * @cpu - the cpu of the buffer to copy.
S
Steven Rostedt 已提交
727 728
 *
 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
729
 */
I
Ingo Molnar 已提交
730
void
731 732
update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
733
	int ret;
734

735
	if (tr->stop_count)
736 737
		return;

738
	WARN_ON_ONCE(!irqs_disabled());
739
	if (WARN_ON_ONCE(!tr->current_trace->allocated_snapshot))
740 741
		return;

742
	arch_spin_lock(&ftrace_max_lock);
743

744 745
	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);

746 747 748 749 750 751 752 753 754 755 756 757
	if (ret == -EBUSY) {
		/*
		 * We failed to swap the buffer due to a commit taking
		 * place on this CPU. We fail to record, but we reset
		 * the max trace buffer (no one writes directly to it)
		 * and flag that it failed.
		 */
		trace_array_printk(&max_tr, _THIS_IP_,
			"Failed to swap buffers due to commit in progress\n");
	}

	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
758 759

	__update_max_tr(tr, tsk, cpu);
760
	arch_spin_unlock(&ftrace_max_lock);
761
}
762
#endif /* CONFIG_TRACER_MAX_TRACE */
763

764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797
static void default_wait_pipe(struct trace_iterator *iter)
{
	DEFINE_WAIT(wait);

	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);

	/*
	 * The events can happen in critical sections where
	 * checking a work queue can cause deadlocks.
	 * After adding a task to the queue, this flag is set
	 * only to notify events to try to wake up the queue
	 * using irq_work.
	 *
	 * We don't clear it even if the buffer is no longer
	 * empty. The flag only causes the next event to run
	 * irq_work to do the work queue wake up. The worse
	 * that can happen if we race with !trace_empty() is that
	 * an event will cause an irq_work to try to wake up
	 * an empty queue.
	 *
	 * There's no reason to protect this flag either, as
	 * the work queue and irq_work logic will do the necessary
	 * synchronization for the wake ups. The only thing
	 * that is necessary is that the wake up happens after
	 * a task has been queued. It's OK for spurious wake ups.
	 */
	trace_wakeup_needed = true;

	if (trace_empty(iter))
		schedule();

	finish_wait(&trace_wait, &wait);
}

S
Steven Rostedt 已提交
798 799 800 801 802 803
/**
 * register_tracer - register a tracer with the ftrace system.
 * @type - the plugin for the tracer
 *
 * Register a new plugin tracer.
 */
804 805 806 807 808 809 810 811 812 813
int register_tracer(struct tracer *type)
{
	struct tracer *t;
	int ret = 0;

	if (!type->name) {
		pr_info("Tracer must have a name\n");
		return -1;
	}

814
	if (strlen(type->name) >= MAX_TRACER_SIZE) {
L
Li Zefan 已提交
815 816 817 818
		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
		return -1;
	}

819
	mutex_lock(&trace_types_lock);
I
Ingo Molnar 已提交
820

821 822
	tracing_selftest_running = true;

823 824 825
	for (t = trace_types; t; t = t->next) {
		if (strcmp(type->name, t->name) == 0) {
			/* already found */
L
Li Zefan 已提交
826
			pr_info("Tracer %s already registered\n",
827 828 829 830 831 832
				type->name);
			ret = -1;
			goto out;
		}
	}

833 834 835 836 837 838 839
	if (!type->set_flag)
		type->set_flag = &dummy_set_flag;
	if (!type->flags)
		type->flags = &dummy_tracer_flags;
	else
		if (!type->flags->opts)
			type->flags->opts = dummy_tracer_opt;
840 841 842
	if (!type->wait_pipe)
		type->wait_pipe = default_wait_pipe;

843

S
Steven Rostedt 已提交
844
#ifdef CONFIG_FTRACE_STARTUP_TEST
845
	if (type->selftest && !tracing_selftest_disabled) {
S
Steven Rostedt 已提交
846
		struct trace_array *tr = &global_trace;
847
		struct tracer *saved_tracer = tr->current_trace;
848

S
Steven Rostedt 已提交
849 850 851 852 853 854 855
		/*
		 * Run a selftest on this tracer.
		 * Here we reset the trace buffer, and set the current
		 * tracer to be this tracer. The tracer can then run some
		 * internal tracing to verify that everything is in order.
		 * If we fail, we do not register this tracer.
		 */
856
		tracing_reset_online_cpus(tr);
I
Ingo Molnar 已提交
857

858
		tr->current_trace = type;
859

860 861 862 863 864 865 866
		if (type->use_max_tr) {
			/* If we expanded the buffers, make sure the max is expanded too */
			if (ring_buffer_expanded)
				ring_buffer_resize(max_tr.buffer, trace_buf_size,
						   RING_BUFFER_ALL_CPUS);
			type->allocated_snapshot = true;
		}
867

S
Steven Rostedt 已提交
868 869 870 871
		/* the test is responsible for initializing and enabling */
		pr_info("Testing tracer %s: ", type->name);
		ret = type->selftest(type, tr);
		/* the test is responsible for resetting too */
872
		tr->current_trace = saved_tracer;
S
Steven Rostedt 已提交
873 874
		if (ret) {
			printk(KERN_CONT "FAILED!\n");
875 876
			/* Add the warning after printing 'FAILED' */
			WARN_ON(1);
S
Steven Rostedt 已提交
877 878
			goto out;
		}
S
Steven Rostedt 已提交
879
		/* Only reset on passing, to avoid touching corrupted buffers */
880
		tracing_reset_online_cpus(tr);
I
Ingo Molnar 已提交
881

882 883 884 885 886 887 888 889
		if (type->use_max_tr) {
			type->allocated_snapshot = false;

			/* Shrink the max buffer again */
			if (ring_buffer_expanded)
				ring_buffer_resize(max_tr.buffer, 1,
						   RING_BUFFER_ALL_CPUS);
		}
890

S
Steven Rostedt 已提交
891 892 893 894
		printk(KERN_CONT "PASSED\n");
	}
#endif

895 896
	type->next = trace_types;
	trace_types = type;
S
Steven Rostedt 已提交
897

898
 out:
899
	tracing_selftest_running = false;
900 901
	mutex_unlock(&trace_types_lock);

S
Steven Rostedt 已提交
902 903 904
	if (ret || !default_bootup_tracer)
		goto out_unlock;

L
Li Zefan 已提交
905
	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
S
Steven Rostedt 已提交
906 907 908 909 910 911 912 913
		goto out_unlock;

	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
	/* Do we want this tracer to start on bootup? */
	tracing_set_tracer(type->name);
	default_bootup_tracer = NULL;
	/* disable other selftests, since this will break it. */
	tracing_selftest_disabled = 1;
914
#ifdef CONFIG_FTRACE_STARTUP_TEST
S
Steven Rostedt 已提交
915 916
	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
	       type->name);
917 918
#endif

S
Steven Rostedt 已提交
919
 out_unlock:
920 921 922
	return ret;
}

923 924 925 926
void tracing_reset(struct trace_array *tr, int cpu)
{
	struct ring_buffer *buffer = tr->buffer;

927 928 929
	if (!buffer)
		return;

930 931 932 933
	ring_buffer_record_disable(buffer);

	/* Make sure all commits have finished */
	synchronize_sched();
934
	ring_buffer_reset_cpu(buffer, cpu);
935 936 937 938

	ring_buffer_record_enable(buffer);
}

939 940
void tracing_reset_online_cpus(struct trace_array *tr)
{
941
	struct ring_buffer *buffer = tr->buffer;
942 943
	int cpu;

944 945 946
	if (!buffer)
		return;

947 948 949 950 951
	ring_buffer_record_disable(buffer);

	/* Make sure all commits have finished */
	synchronize_sched();

952 953 954
	tr->time_start = ftrace_now(tr->cpu);

	for_each_online_cpu(cpu)
955
		ring_buffer_reset_cpu(buffer, cpu);
956 957

	ring_buffer_record_enable(buffer);
958 959
}

960 961 962 963 964 965 966 967 968 969
void tracing_reset_current(int cpu)
{
	tracing_reset(&global_trace, cpu);
}

void tracing_reset_current_online_cpus(void)
{
	tracing_reset_online_cpus(&global_trace);
}

970
#define SAVED_CMDLINES 128
971
#define NO_CMDLINE_MAP UINT_MAX
972 973 974 975
static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
static int cmdline_idx;
976
static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
977 978

/* temporary disable recording */
979
static atomic_t trace_record_cmdline_disabled __read_mostly;
980 981 982

static void trace_init_cmdlines(void)
{
983 984
	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
985 986 987
	cmdline_idx = 0;
}

988 989
int is_tracing_stopped(void)
{
990
	return global_trace.stop_count;
991 992
}

S
Steven Rostedt 已提交
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
/**
 * ftrace_off_permanent - disable all ftrace code permanently
 *
 * This should only be called when a serious anomally has
 * been detected.  This will turn off the function tracing,
 * ring buffers, and other tracing utilites. It takes no
 * locks and can be called from any context.
 */
void ftrace_off_permanent(void)
{
	tracing_disabled = 1;
	ftrace_stop();
	tracing_off_permanent();
}

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
/**
 * tracing_start - quick start of the tracer
 *
 * If tracing is enabled but was stopped by tracing_stop,
 * this will start the tracer back up.
 */
void tracing_start(void)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	if (tracing_disabled)
		return;

1022 1023 1024
	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
	if (--global_trace.stop_count) {
		if (global_trace.stop_count < 0) {
1025 1026
			/* Someone screwed up their debugging */
			WARN_ON_ONCE(1);
1027
			global_trace.stop_count = 0;
1028
		}
1029 1030 1031
		goto out;
	}

1032 1033
	/* Prevent the buffers from switching */
	arch_spin_lock(&ftrace_max_lock);
1034 1035 1036 1037 1038 1039 1040 1041 1042

	buffer = global_trace.buffer;
	if (buffer)
		ring_buffer_record_enable(buffer);

	buffer = max_tr.buffer;
	if (buffer)
		ring_buffer_record_enable(buffer);

1043 1044
	arch_spin_unlock(&ftrace_max_lock);

1045 1046
	ftrace_start();
 out:
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
}

static void tracing_start_tr(struct trace_array *tr)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	if (tracing_disabled)
		return;

	/* If global, we need to also start the max tracer */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
		return tracing_start();

	raw_spin_lock_irqsave(&tr->start_lock, flags);

	if (--tr->stop_count) {
		if (tr->stop_count < 0) {
			/* Someone screwed up their debugging */
			WARN_ON_ONCE(1);
			tr->stop_count = 0;
		}
		goto out;
	}

	buffer = tr->buffer;
	if (buffer)
		ring_buffer_record_enable(buffer);

 out:
	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
}

/**
 * tracing_stop - quick stop of the tracer
 *
 * Light weight way to stop tracing. Use in conjunction with
 * tracing_start.
 */
void tracing_stop(void)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	ftrace_stop();
1093 1094
	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
	if (global_trace.stop_count++)
1095 1096
		goto out;

1097 1098 1099
	/* Prevent the buffers from switching */
	arch_spin_lock(&ftrace_max_lock);

1100 1101 1102 1103 1104 1105 1106 1107
	buffer = global_trace.buffer;
	if (buffer)
		ring_buffer_record_disable(buffer);

	buffer = max_tr.buffer;
	if (buffer)
		ring_buffer_record_disable(buffer);

1108 1109
	arch_spin_unlock(&ftrace_max_lock);

1110
 out:
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
}

static void tracing_stop_tr(struct trace_array *tr)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	/* If global, we need to also stop the max tracer */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
		return tracing_stop();

	raw_spin_lock_irqsave(&tr->start_lock, flags);
	if (tr->stop_count++)
		goto out;

	buffer = tr->buffer;
	if (buffer)
		ring_buffer_record_disable(buffer);

 out:
	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1133 1134
}

I
Ingo Molnar 已提交
1135
void trace_stop_cmdline_recording(void);
1136

I
Ingo Molnar 已提交
1137
static void trace_save_cmdline(struct task_struct *tsk)
1138
{
1139
	unsigned pid, idx;
1140 1141 1142 1143 1144 1145 1146 1147 1148 1149

	if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
		return;

	/*
	 * It's not the end of the world if we don't get
	 * the lock, but we also don't want to spin
	 * nor do we want to disable interrupts,
	 * so if we miss here, then better luck next time.
	 */
1150
	if (!arch_spin_trylock(&trace_cmdline_lock))
1151 1152 1153
		return;

	idx = map_pid_to_cmdline[tsk->pid];
1154
	if (idx == NO_CMDLINE_MAP) {
1155 1156
		idx = (cmdline_idx + 1) % SAVED_CMDLINES;

1157 1158 1159 1160 1161 1162 1163 1164 1165
		/*
		 * Check whether the cmdline buffer at idx has a pid
		 * mapped. We are going to overwrite that entry so we
		 * need to clear the map_pid_to_cmdline. Otherwise we
		 * would read the new comm for the old pid.
		 */
		pid = map_cmdline_to_pid[idx];
		if (pid != NO_CMDLINE_MAP)
			map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1166

1167
		map_cmdline_to_pid[idx] = tsk->pid;
1168 1169 1170 1171 1172 1173 1174
		map_pid_to_cmdline[tsk->pid] = idx;

		cmdline_idx = idx;
	}

	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);

1175
	arch_spin_unlock(&trace_cmdline_lock);
1176 1177
}

1178
void trace_find_cmdline(int pid, char comm[])
1179 1180 1181
{
	unsigned map;

1182 1183 1184 1185
	if (!pid) {
		strcpy(comm, "<idle>");
		return;
	}
1186

1187 1188 1189 1190 1191
	if (WARN_ON_ONCE(pid < 0)) {
		strcpy(comm, "<XXX>");
		return;
	}

1192 1193 1194 1195
	if (pid > PID_MAX_DEFAULT) {
		strcpy(comm, "<...>");
		return;
	}
1196

1197
	preempt_disable();
1198
	arch_spin_lock(&trace_cmdline_lock);
1199
	map = map_pid_to_cmdline[pid];
1200 1201 1202 1203
	if (map != NO_CMDLINE_MAP)
		strcpy(comm, saved_cmdlines[map]);
	else
		strcpy(comm, "<...>");
1204

1205
	arch_spin_unlock(&trace_cmdline_lock);
1206
	preempt_enable();
1207 1208
}

I
Ingo Molnar 已提交
1209
void tracing_record_cmdline(struct task_struct *tsk)
1210
{
1211
	if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1212 1213
		return;

1214 1215 1216 1217 1218
	if (!__this_cpu_read(trace_cmdline_save))
		return;

	__this_cpu_write(trace_cmdline_save, false);

1219 1220 1221
	trace_save_cmdline(tsk);
}

1222
void
1223 1224
tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
			     int pc)
1225 1226 1227
{
	struct task_struct *tsk = current;

1228 1229 1230
	entry->preempt_count		= pc & 0xff;
	entry->pid			= (tsk) ? tsk->pid : 0;
	entry->flags =
1231
#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1232
		(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1233 1234 1235
#else
		TRACE_FLAG_IRQS_NOSUPPORT |
#endif
1236 1237 1238 1239
		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
		((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
		(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
}
1240
EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1241

1242 1243 1244 1245 1246
struct ring_buffer_event *
trace_buffer_lock_reserve(struct ring_buffer *buffer,
			  int type,
			  unsigned long len,
			  unsigned long flags, int pc)
1247 1248 1249
{
	struct ring_buffer_event *event;

1250
	event = ring_buffer_lock_reserve(buffer, len);
1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
	if (event != NULL) {
		struct trace_entry *ent = ring_buffer_event_data(event);

		tracing_generic_entry_update(ent, flags, pc);
		ent->type = type;
	}

	return event;
}

1261 1262 1263 1264
void
__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
{
	__this_cpu_write(trace_cmdline_save, true);
1265 1266 1267 1268 1269
	if (trace_wakeup_needed) {
		trace_wakeup_needed = false;
		/* irq_work_queue() supplies it's own memory barriers */
		irq_work_queue(&trace_work_wakeup);
	}
1270 1271 1272
	ring_buffer_unlock_commit(buffer, event);
}

1273 1274 1275
static inline void
__trace_buffer_unlock_commit(struct ring_buffer *buffer,
			     struct ring_buffer_event *event,
1276
			     unsigned long flags, int pc)
1277
{
1278
	__buffer_unlock_commit(buffer, event);
1279

1280 1281
	ftrace_trace_stack(buffer, flags, 6, pc);
	ftrace_trace_userstack(buffer, flags, pc);
1282 1283
}

1284 1285 1286
void trace_buffer_unlock_commit(struct ring_buffer *buffer,
				struct ring_buffer_event *event,
				unsigned long flags, int pc)
1287
{
1288
	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1289
}
1290
EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1291

1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
struct ring_buffer_event *
trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
			  struct ftrace_event_file *ftrace_file,
			  int type, unsigned long len,
			  unsigned long flags, int pc)
{
	*current_rb = ftrace_file->tr->buffer;
	return trace_buffer_lock_reserve(*current_rb,
					 type, len, flags, pc);
}
EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);

1304
struct ring_buffer_event *
1305 1306
trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
				  int type, unsigned long len,
1307 1308
				  unsigned long flags, int pc)
{
1309 1310
	*current_rb = global_trace.buffer;
	return trace_buffer_lock_reserve(*current_rb,
1311 1312
					 type, len, flags, pc);
}
1313
EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1314

1315 1316
void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
					struct ring_buffer_event *event,
1317 1318
					unsigned long flags, int pc)
{
1319
	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1320
}
1321
EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1322

1323 1324 1325 1326
void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
				     struct ring_buffer_event *event,
				     unsigned long flags, int pc,
				     struct pt_regs *regs)
1327
{
1328
	__buffer_unlock_commit(buffer, event);
1329 1330 1331 1332

	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
	ftrace_trace_userstack(buffer, flags, pc);
}
1333
EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1334

1335 1336
void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
					 struct ring_buffer_event *event)
1337
{
1338
	ring_buffer_discard_commit(buffer, event);
1339
}
1340
EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1341

I
Ingo Molnar 已提交
1342
void
1343
trace_function(struct trace_array *tr,
1344 1345
	       unsigned long ip, unsigned long parent_ip, unsigned long flags,
	       int pc)
1346
{
1347
	struct ftrace_event_call *call = &event_function;
1348
	struct ring_buffer *buffer = tr->buffer;
1349
	struct ring_buffer_event *event;
1350
	struct ftrace_entry *entry;
1351

1352
	/* If we are reading the ring buffer, don't trace */
R
Rusty Russell 已提交
1353
	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1354 1355
		return;

1356
	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1357
					  flags, pc);
1358 1359 1360
	if (!event)
		return;
	entry	= ring_buffer_event_data(event);
1361 1362
	entry->ip			= ip;
	entry->parent_ip		= parent_ip;
1363

1364
	if (!filter_check_discard(call, entry, buffer, event))
1365
		__buffer_unlock_commit(buffer, event);
1366 1367
}

I
Ingo Molnar 已提交
1368
void
I
Ingo Molnar 已提交
1369
ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1370 1371
       unsigned long ip, unsigned long parent_ip, unsigned long flags,
       int pc)
I
Ingo Molnar 已提交
1372 1373
{
	if (likely(!atomic_read(&data->disabled)))
1374
		trace_function(tr, ip, parent_ip, flags, pc);
I
Ingo Molnar 已提交
1375 1376
}

1377
#ifdef CONFIG_STACKTRACE
1378 1379 1380 1381 1382 1383 1384 1385 1386

#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
struct ftrace_stack {
	unsigned long		calls[FTRACE_STACK_MAX_ENTRIES];
};

static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
static DEFINE_PER_CPU(int, ftrace_stack_reserve);

1387
static void __ftrace_trace_stack(struct ring_buffer *buffer,
1388
				 unsigned long flags,
1389
				 int skip, int pc, struct pt_regs *regs)
I
Ingo Molnar 已提交
1390
{
1391
	struct ftrace_event_call *call = &event_kernel_stack;
1392
	struct ring_buffer_event *event;
1393
	struct stack_entry *entry;
I
Ingo Molnar 已提交
1394
	struct stack_trace trace;
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
	int use_stack;
	int size = FTRACE_STACK_ENTRIES;

	trace.nr_entries	= 0;
	trace.skip		= skip;

	/*
	 * Since events can happen in NMIs there's no safe way to
	 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
	 * or NMI comes in, it will just have to use the default
	 * FTRACE_STACK_SIZE.
	 */
	preempt_disable_notrace();

1409
	use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
	/*
	 * We don't need any atomic variables, just a barrier.
	 * If an interrupt comes in, we don't care, because it would
	 * have exited and put the counter back to what we want.
	 * We just need a barrier to keep gcc from moving things
	 * around.
	 */
	barrier();
	if (use_stack == 1) {
		trace.entries		= &__get_cpu_var(ftrace_stack).calls[0];
		trace.max_entries	= FTRACE_STACK_MAX_ENTRIES;

		if (regs)
			save_stack_trace_regs(regs, &trace);
		else
			save_stack_trace(&trace);

		if (trace.nr_entries > size)
			size = trace.nr_entries;
	} else
		/* From now on, use_stack is a boolean */
		use_stack = 0;

	size *= sizeof(unsigned long);
I
Ingo Molnar 已提交
1434

1435
	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1436
					  sizeof(*entry) + size, flags, pc);
1437
	if (!event)
1438 1439
		goto out;
	entry = ring_buffer_event_data(event);
I
Ingo Molnar 已提交
1440

1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
	memset(&entry->caller, 0, size);

	if (use_stack)
		memcpy(&entry->caller, trace.entries,
		       trace.nr_entries * sizeof(unsigned long));
	else {
		trace.max_entries	= FTRACE_STACK_ENTRIES;
		trace.entries		= entry->caller;
		if (regs)
			save_stack_trace_regs(regs, &trace);
		else
			save_stack_trace(&trace);
	}

	entry->size = trace.nr_entries;
I
Ingo Molnar 已提交
1456

1457
	if (!filter_check_discard(call, entry, buffer, event))
1458
		__buffer_unlock_commit(buffer, event);
1459 1460 1461 1462

 out:
	/* Again, don't let gcc optimize things here */
	barrier();
1463
	__this_cpu_dec(ftrace_stack_reserve);
1464 1465
	preempt_enable_notrace();

I
Ingo Molnar 已提交
1466 1467
}

1468 1469 1470 1471 1472 1473 1474 1475 1476
void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
			     int skip, int pc, struct pt_regs *regs)
{
	if (!(trace_flags & TRACE_ITER_STACKTRACE))
		return;

	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
}

1477 1478
void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
			int skip, int pc)
1479 1480 1481 1482
{
	if (!(trace_flags & TRACE_ITER_STACKTRACE))
		return;

1483
	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1484 1485
}

1486 1487
void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
		   int pc)
1488
{
1489
	__ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1490 1491
}

S
Steven Rostedt 已提交
1492 1493 1494 1495 1496 1497 1498 1499
/**
 * trace_dump_stack - record a stack back trace in the trace buffer
 */
void trace_dump_stack(void)
{
	unsigned long flags;

	if (tracing_disabled || tracing_selftest_running)
1500
		return;
S
Steven Rostedt 已提交
1501 1502 1503 1504

	local_save_flags(flags);

	/* skipping 3 traces, seems to get us at the caller of this function */
1505
	__ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
S
Steven Rostedt 已提交
1506 1507
}

1508 1509
static DEFINE_PER_CPU(int, user_stack_count);

1510 1511
void
ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1512
{
1513
	struct ftrace_event_call *call = &event_user_stack;
1514
	struct ring_buffer_event *event;
1515 1516 1517 1518 1519 1520
	struct userstack_entry *entry;
	struct stack_trace trace;

	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
		return;

1521 1522 1523 1524 1525 1526
	/*
	 * NMIs can not handle page faults, even with fix ups.
	 * The save user stack can (and often does) fault.
	 */
	if (unlikely(in_nmi()))
		return;
1527

1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
	/*
	 * prevent recursion, since the user stack tracing may
	 * trigger other kernel events.
	 */
	preempt_disable();
	if (__this_cpu_read(user_stack_count))
		goto out;

	__this_cpu_inc(user_stack_count);

1538
	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1539
					  sizeof(*entry), flags, pc);
1540
	if (!event)
L
Li Zefan 已提交
1541
		goto out_drop_count;
1542 1543
	entry	= ring_buffer_event_data(event);

1544
	entry->tgid		= current->tgid;
1545 1546 1547 1548 1549 1550 1551 1552
	memset(&entry->caller, 0, sizeof(entry->caller));

	trace.nr_entries	= 0;
	trace.max_entries	= FTRACE_STACK_ENTRIES;
	trace.skip		= 0;
	trace.entries		= entry->caller;

	save_stack_trace_user(&trace);
1553
	if (!filter_check_discard(call, entry, buffer, event))
1554
		__buffer_unlock_commit(buffer, event);
1555

L
Li Zefan 已提交
1556
 out_drop_count:
1557 1558 1559
	__this_cpu_dec(user_stack_count);
 out:
	preempt_enable();
1560 1561
}

1562 1563
#ifdef UNUSED
static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1564
{
1565
	ftrace_trace_userstack(tr, flags, preempt_count());
1566
}
1567
#endif /* UNUSED */
1568

1569 1570
#endif /* CONFIG_STACKTRACE */

1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
/* created for use with alloc_percpu */
struct trace_buffer_struct {
	char buffer[TRACE_BUF_SIZE];
};

static struct trace_buffer_struct *trace_percpu_buffer;
static struct trace_buffer_struct *trace_percpu_sirq_buffer;
static struct trace_buffer_struct *trace_percpu_irq_buffer;
static struct trace_buffer_struct *trace_percpu_nmi_buffer;

/*
 * The buffer used is dependent on the context. There is a per cpu
 * buffer for normal context, softirq contex, hard irq context and
 * for NMI context. Thise allows for lockless recording.
 *
 * Note, if the buffers failed to be allocated, then this returns NULL
 */
static char *get_trace_buf(void)
{
	struct trace_buffer_struct *percpu_buffer;

	/*
	 * If we have allocated per cpu buffers, then we do not
	 * need to do any locking.
	 */
	if (in_nmi())
		percpu_buffer = trace_percpu_nmi_buffer;
	else if (in_irq())
		percpu_buffer = trace_percpu_irq_buffer;
	else if (in_softirq())
		percpu_buffer = trace_percpu_sirq_buffer;
	else
		percpu_buffer = trace_percpu_buffer;

	if (!percpu_buffer)
		return NULL;

1608
	return this_cpu_ptr(&percpu_buffer->buffer[0]);
1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651
}

static int alloc_percpu_trace_buffer(void)
{
	struct trace_buffer_struct *buffers;
	struct trace_buffer_struct *sirq_buffers;
	struct trace_buffer_struct *irq_buffers;
	struct trace_buffer_struct *nmi_buffers;

	buffers = alloc_percpu(struct trace_buffer_struct);
	if (!buffers)
		goto err_warn;

	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!sirq_buffers)
		goto err_sirq;

	irq_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!irq_buffers)
		goto err_irq;

	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!nmi_buffers)
		goto err_nmi;

	trace_percpu_buffer = buffers;
	trace_percpu_sirq_buffer = sirq_buffers;
	trace_percpu_irq_buffer = irq_buffers;
	trace_percpu_nmi_buffer = nmi_buffers;

	return 0;

 err_nmi:
	free_percpu(irq_buffers);
 err_irq:
	free_percpu(sirq_buffers);
 err_sirq:
	free_percpu(buffers);
 err_warn:
	WARN(1, "Could not allocate percpu trace_printk buffer");
	return -ENOMEM;
}

1652 1653
static int buffers_allocated;

1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
void trace_printk_init_buffers(void)
{
	if (buffers_allocated)
		return;

	if (alloc_percpu_trace_buffer())
		return;

	pr_info("ftrace: Allocated trace_printk buffers\n");

1664 1665 1666
	/* Expand the buffers to set size */
	tracing_update_buffers();

1667
	buffers_allocated = 1;
1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695

	/*
	 * trace_printk_init_buffers() can be called by modules.
	 * If that happens, then we need to start cmdline recording
	 * directly here. If the global_trace.buffer is already
	 * allocated here, then this was called by module code.
	 */
	if (global_trace.buffer)
		tracing_start_cmdline_record();
}

void trace_printk_start_comm(void)
{
	/* Start tracing comms if trace printk is set */
	if (!buffers_allocated)
		return;
	tracing_start_cmdline_record();
}

static void trace_printk_start_stop_comm(int enabled)
{
	if (!buffers_allocated)
		return;

	if (enabled)
		tracing_start_cmdline_record();
	else
		tracing_stop_cmdline_record();
1696 1697
}

1698
/**
1699
 * trace_vbprintk - write binary msg to tracing buffer
1700 1701
 *
 */
1702
int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1703
{
1704
	struct ftrace_event_call *call = &event_bprint;
1705
	struct ring_buffer_event *event;
1706
	struct ring_buffer *buffer;
1707
	struct trace_array *tr = &global_trace;
1708
	struct bprint_entry *entry;
1709
	unsigned long flags;
1710 1711
	char *tbuffer;
	int len = 0, size, pc;
1712 1713 1714 1715 1716 1717 1718 1719

	if (unlikely(tracing_selftest_running || tracing_disabled))
		return 0;

	/* Don't pollute graph traces with trace_vprintk internals */
	pause_graph_tracing();

	pc = preempt_count();
1720
	preempt_disable_notrace();
1721

1722 1723 1724
	tbuffer = get_trace_buf();
	if (!tbuffer) {
		len = 0;
1725
		goto out;
1726
	}
1727

1728
	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1729

1730 1731
	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
		goto out;
1732

1733
	local_save_flags(flags);
1734
	size = sizeof(*entry) + sizeof(u32) * len;
1735 1736 1737
	buffer = tr->buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
					  flags, pc);
1738
	if (!event)
1739
		goto out;
1740 1741 1742 1743
	entry = ring_buffer_event_data(event);
	entry->ip			= ip;
	entry->fmt			= fmt;

1744
	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1745
	if (!filter_check_discard(call, entry, buffer, event)) {
1746
		__buffer_unlock_commit(buffer, event);
1747 1748
		ftrace_trace_stack(buffer, flags, 6, pc);
	}
1749 1750

out:
1751
	preempt_enable_notrace();
1752 1753 1754 1755
	unpause_graph_tracing();

	return len;
}
1756 1757
EXPORT_SYMBOL_GPL(trace_vbprintk);

1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
int trace_array_printk(struct trace_array *tr,
		       unsigned long ip, const char *fmt, ...)
{
	int ret;
	va_list ap;

	if (!(trace_flags & TRACE_ITER_PRINTK))
		return 0;

	va_start(ap, fmt);
	ret = trace_array_vprintk(tr, ip, fmt, ap);
	va_end(ap);
	return ret;
}

int trace_array_vprintk(struct trace_array *tr,
			unsigned long ip, const char *fmt, va_list args)
1775
{
1776
	struct ftrace_event_call *call = &event_print;
1777
	struct ring_buffer_event *event;
1778
	struct ring_buffer *buffer;
1779
	int len = 0, size, pc;
1780
	struct print_entry *entry;
1781 1782
	unsigned long flags;
	char *tbuffer;
1783 1784 1785 1786

	if (tracing_disabled || tracing_selftest_running)
		return 0;

1787 1788 1789
	/* Don't pollute graph traces with trace_vprintk internals */
	pause_graph_tracing();

1790 1791 1792
	pc = preempt_count();
	preempt_disable_notrace();

1793 1794 1795 1796

	tbuffer = get_trace_buf();
	if (!tbuffer) {
		len = 0;
1797
		goto out;
1798
	}
1799

1800 1801 1802
	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
	if (len > TRACE_BUF_SIZE)
		goto out;
1803

1804
	local_save_flags(flags);
1805
	size = sizeof(*entry) + len + 1;
1806 1807
	buffer = tr->buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1808
					  flags, pc);
1809
	if (!event)
1810
		goto out;
1811
	entry = ring_buffer_event_data(event);
C
Carsten Emde 已提交
1812
	entry->ip = ip;
1813

1814
	memcpy(&entry->buf, tbuffer, len);
C
Carsten Emde 已提交
1815
	entry->buf[len] = '\0';
1816
	if (!filter_check_discard(call, entry, buffer, event)) {
1817
		__buffer_unlock_commit(buffer, event);
1818
		ftrace_trace_stack(buffer, flags, 6, pc);
1819
	}
1820 1821
 out:
	preempt_enable_notrace();
1822
	unpause_graph_tracing();
1823 1824 1825

	return len;
}
1826 1827 1828

int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
{
S
Steven Rostedt 已提交
1829
	return trace_array_vprintk(&global_trace, ip, fmt, args);
1830
}
1831 1832
EXPORT_SYMBOL_GPL(trace_vprintk);

1833
static void trace_iterator_increment(struct trace_iterator *iter)
S
Steven Rostedt 已提交
1834
{
1835 1836
	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);

S
Steven Rostedt 已提交
1837
	iter->idx++;
1838 1839
	if (buf_iter)
		ring_buffer_read(buf_iter, NULL);
S
Steven Rostedt 已提交
1840 1841
}

I
Ingo Molnar 已提交
1842
static struct trace_entry *
1843 1844
peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
		unsigned long *lost_events)
1845
{
1846
	struct ring_buffer_event *event;
1847
	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
1848

1849 1850 1851
	if (buf_iter)
		event = ring_buffer_iter_peek(buf_iter, ts);
	else
1852 1853
		event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
					 lost_events);
1854

1855 1856 1857 1858 1859 1860
	if (event) {
		iter->ent_size = ring_buffer_event_length(event);
		return ring_buffer_event_data(event);
	}
	iter->ent_size = 0;
	return NULL;
1861
}
1862

1863
static struct trace_entry *
1864 1865
__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
		  unsigned long *missing_events, u64 *ent_ts)
1866
{
1867
	struct ring_buffer *buffer = iter->tr->buffer;
1868
	struct trace_entry *ent, *next = NULL;
1869
	unsigned long lost_events = 0, next_lost = 0;
1870
	int cpu_file = iter->cpu_file;
1871
	u64 next_ts = 0, ts;
1872
	int next_cpu = -1;
1873
	int next_size = 0;
1874 1875
	int cpu;

1876 1877 1878 1879
	/*
	 * If we are in a per_cpu trace file, don't bother by iterating over
	 * all cpu and peek directly.
	 */
1880
	if (cpu_file > RING_BUFFER_ALL_CPUS) {
1881 1882
		if (ring_buffer_empty_cpu(buffer, cpu_file))
			return NULL;
1883
		ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1884 1885 1886 1887 1888 1889
		if (ent_cpu)
			*ent_cpu = cpu_file;

		return ent;
	}

1890
	for_each_tracing_cpu(cpu) {
1891

1892 1893
		if (ring_buffer_empty_cpu(buffer, cpu))
			continue;
1894

1895
		ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1896

I
Ingo Molnar 已提交
1897 1898 1899
		/*
		 * Pick the entry with the smallest timestamp:
		 */
1900
		if (ent && (!next || ts < next_ts)) {
1901 1902
			next = ent;
			next_cpu = cpu;
1903
			next_ts = ts;
1904
			next_lost = lost_events;
1905
			next_size = iter->ent_size;
1906 1907 1908
		}
	}

1909 1910
	iter->ent_size = next_size;

1911 1912 1913
	if (ent_cpu)
		*ent_cpu = next_cpu;

1914 1915 1916
	if (ent_ts)
		*ent_ts = next_ts;

1917 1918 1919
	if (missing_events)
		*missing_events = next_lost;

1920 1921 1922
	return next;
}

1923
/* Find the next real entry, without updating the iterator itself */
1924 1925
struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
					  int *ent_cpu, u64 *ent_ts)
1926
{
1927
	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1928 1929 1930
}

/* Find the next real entry, and increment the iterator to the next entry */
1931
void *trace_find_next_entry_inc(struct trace_iterator *iter)
1932
{
1933 1934
	iter->ent = __find_next_entry(iter, &iter->cpu,
				      &iter->lost_events, &iter->ts);
1935

1936
	if (iter->ent)
1937
		trace_iterator_increment(iter);
1938

1939
	return iter->ent ? iter : NULL;
1940
}
1941

I
Ingo Molnar 已提交
1942
static void trace_consume(struct trace_iterator *iter)
1943
{
1944 1945
	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
			    &iter->lost_events);
1946 1947
}

I
Ingo Molnar 已提交
1948
static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1949 1950 1951
{
	struct trace_iterator *iter = m->private;
	int i = (int)*pos;
I
Ingo Molnar 已提交
1952
	void *ent;
1953

1954 1955
	WARN_ON_ONCE(iter->leftover);

1956 1957 1958 1959 1960 1961 1962
	(*pos)++;

	/* can't go backwards */
	if (iter->idx > i)
		return NULL;

	if (iter->idx < 0)
1963
		ent = trace_find_next_entry_inc(iter);
1964 1965 1966 1967
	else
		ent = iter;

	while (ent && iter->idx < i)
1968
		ent = trace_find_next_entry_inc(iter);
1969 1970 1971 1972 1973 1974

	iter->pos = *pos;

	return ent;
}

1975
void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1976 1977 1978 1979 1980 1981 1982
{
	struct trace_array *tr = iter->tr;
	struct ring_buffer_event *event;
	struct ring_buffer_iter *buf_iter;
	unsigned long entries = 0;
	u64 ts;

1983
	per_cpu_ptr(tr->data, cpu)->skipped_entries = 0;
1984

1985 1986
	buf_iter = trace_buffer_iter(iter, cpu);
	if (!buf_iter)
1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002
		return;

	ring_buffer_iter_reset(buf_iter);

	/*
	 * We could have the case with the max latency tracers
	 * that a reset never took place on a cpu. This is evident
	 * by the timestamp being before the start of the buffer.
	 */
	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
		if (ts >= iter->tr->time_start)
			break;
		entries++;
		ring_buffer_read(buf_iter, NULL);
	}

2003
	per_cpu_ptr(tr->data, cpu)->skipped_entries = entries;
2004 2005
}

2006 2007 2008 2009
/*
 * The current tracer is copied to avoid a global locking
 * all around.
 */
2010 2011 2012
static void *s_start(struct seq_file *m, loff_t *pos)
{
	struct trace_iterator *iter = m->private;
2013
	struct trace_array *tr = iter->tr;
2014
	int cpu_file = iter->cpu_file;
2015 2016
	void *p = NULL;
	loff_t l = 0;
2017
	int cpu;
2018

2019 2020 2021 2022 2023 2024
	/*
	 * copy the tracer to avoid using a global lock all around.
	 * iter->trace is a copy of current_trace, the pointer to the
	 * name may be used instead of a strcmp(), as iter->trace->name
	 * will point to the same string as current_trace->name.
	 */
2025
	mutex_lock(&trace_types_lock);
2026 2027
	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
		*iter->trace = *tr->current_trace;
2028
	mutex_unlock(&trace_types_lock);
2029

2030 2031 2032 2033 2034
	if (iter->snapshot && iter->trace->use_max_tr)
		return ERR_PTR(-EBUSY);

	if (!iter->snapshot)
		atomic_inc(&trace_record_cmdline_disabled);
2035 2036 2037 2038 2039 2040

	if (*pos != iter->pos) {
		iter->ent = NULL;
		iter->cpu = 0;
		iter->idx = -1;

2041
		if (cpu_file == RING_BUFFER_ALL_CPUS) {
2042
			for_each_tracing_cpu(cpu)
2043
				tracing_iter_reset(iter, cpu);
2044
		} else
2045
			tracing_iter_reset(iter, cpu_file);
2046

2047
		iter->leftover = 0;
2048 2049 2050 2051
		for (p = iter; p && l < *pos; p = s_next(m, p, &l))
			;

	} else {
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061
		/*
		 * If we overflowed the seq_file before, then we want
		 * to just reuse the trace_seq buffer again.
		 */
		if (iter->leftover)
			p = iter;
		else {
			l = *pos - 1;
			p = s_next(m, p, &l);
		}
2062 2063
	}

2064
	trace_event_read_lock();
2065
	trace_access_lock(cpu_file);
2066 2067 2068 2069 2070
	return p;
}

static void s_stop(struct seq_file *m, void *p)
{
2071 2072
	struct trace_iterator *iter = m->private;

2073 2074 2075 2076 2077
	if (iter->snapshot && iter->trace->use_max_tr)
		return;

	if (!iter->snapshot)
		atomic_dec(&trace_record_cmdline_disabled);
2078
	trace_access_unlock(iter->cpu_file);
2079
	trace_event_read_unlock();
2080 2081
}

2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
static void
get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
{
	unsigned long count;
	int cpu;

	*total = 0;
	*entries = 0;

	for_each_tracing_cpu(cpu) {
		count = ring_buffer_entries_cpu(tr->buffer, cpu);
		/*
		 * If this buffer has skipped entries, then we hold all
		 * entries for the trace and we need to ignore the
		 * ones before the time stamp.
		 */
2098 2099
		if (per_cpu_ptr(tr->data, cpu)->skipped_entries) {
			count -= per_cpu_ptr(tr->data, cpu)->skipped_entries;
2100 2101 2102 2103 2104 2105 2106 2107 2108
			/* total is the same as the entries */
			*total += count;
		} else
			*total += count +
				ring_buffer_overrun_cpu(tr->buffer, cpu);
		*entries += count;
	}
}

I
Ingo Molnar 已提交
2109
static void print_lat_help_header(struct seq_file *m)
2110
{
2111 2112 2113 2114 2115
	seq_puts(m, "#                  _------=> CPU#            \n");
	seq_puts(m, "#                 / _-----=> irqs-off        \n");
	seq_puts(m, "#                | / _----=> need-resched    \n");
	seq_puts(m, "#                || / _---=> hardirq/softirq \n");
	seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2116 2117 2118
	seq_puts(m, "#                |||| /     delay             \n");
	seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
	seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2119 2120
}

2121
static void print_event_info(struct trace_array *tr, struct seq_file *m)
2122
{
2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
	unsigned long total;
	unsigned long entries;

	get_total_entries(tr, &total, &entries);
	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
		   entries, total, num_online_cpus());
	seq_puts(m, "#\n");
}

static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
{
	print_event_info(tr, m);
2135
	seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2136
	seq_puts(m, "#              | |       |          |         |\n");
2137 2138
}

2139
static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2140
{
2141
	print_event_info(tr, m);
2142 2143 2144 2145 2146 2147 2148 2149
	seq_puts(m, "#                              _-----=> irqs-off\n");
	seq_puts(m, "#                             / _----=> need-resched\n");
	seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
	seq_puts(m, "#                            || / _--=> preempt-depth\n");
	seq_puts(m, "#                            ||| /     delay\n");
	seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
	seq_puts(m, "#              | |       |   ||||       |         |\n");
}
2150

2151
void
2152 2153 2154 2155
print_trace_header(struct seq_file *m, struct trace_iterator *iter)
{
	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
	struct trace_array *tr = iter->tr;
2156
	struct trace_array_cpu *data = per_cpu_ptr(tr->data, tr->cpu);
2157
	struct tracer *type = iter->trace;
2158 2159
	unsigned long entries;
	unsigned long total;
2160 2161
	const char *name = "preemption";

2162
	name = type->name;
2163

2164
	get_total_entries(tr, &total, &entries);
2165

2166
	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2167
		   name, UTS_RELEASE);
2168
	seq_puts(m, "# -----------------------------------"
2169
		 "---------------------------------\n");
2170
	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2171
		   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
S
Steven Rostedt 已提交
2172
		   nsecs_to_usecs(data->saved_latency),
2173
		   entries,
2174
		   total,
2175 2176 2177 2178 2179
		   tr->cpu,
#if defined(CONFIG_PREEMPT_NONE)
		   "server",
#elif defined(CONFIG_PREEMPT_VOLUNTARY)
		   "desktop",
2180
#elif defined(CONFIG_PREEMPT)
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
		   "preempt",
#else
		   "unknown",
#endif
		   /* These are reserved for later use */
		   0, 0, 0, 0);
#ifdef CONFIG_SMP
	seq_printf(m, " #P:%d)\n", num_online_cpus());
#else
	seq_puts(m, ")\n");
#endif
2192 2193
	seq_puts(m, "#    -----------------\n");
	seq_printf(m, "#    | task: %.16s-%d "
2194
		   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2195 2196
		   data->comm, data->pid,
		   from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2197
		   data->policy, data->rt_priority);
2198
	seq_puts(m, "#    -----------------\n");
2199 2200

	if (data->critical_start) {
2201
		seq_puts(m, "#  => started at: ");
S
Steven Rostedt 已提交
2202 2203
		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
		trace_print_seq(m, &iter->seq);
2204
		seq_puts(m, "\n#  => ended at:   ");
S
Steven Rostedt 已提交
2205 2206
		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
		trace_print_seq(m, &iter->seq);
2207
		seq_puts(m, "\n#\n");
2208 2209
	}

2210
	seq_puts(m, "#\n");
2211 2212
}

2213 2214 2215 2216
static void test_cpu_buff_start(struct trace_iterator *iter)
{
	struct trace_seq *s = &iter->seq;

2217 2218 2219 2220 2221 2222
	if (!(trace_flags & TRACE_ITER_ANNOTATE))
		return;

	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
		return;

2223
	if (cpumask_test_cpu(iter->cpu, iter->started))
2224 2225
		return;

2226
	if (per_cpu_ptr(iter->tr->data, iter->cpu)->skipped_entries)
2227 2228
		return;

2229
	cpumask_set_cpu(iter->cpu, iter->started);
2230 2231 2232 2233 2234

	/* Don't print started cpu buffer for the first entry of the trace */
	if (iter->idx > 1)
		trace_seq_printf(s, "##### CPU %u buffer started ####\n",
				iter->cpu);
2235 2236
}

2237
static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2238
{
S
Steven Rostedt 已提交
2239
	struct trace_seq *s = &iter->seq;
2240
	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
I
Ingo Molnar 已提交
2241
	struct trace_entry *entry;
2242
	struct trace_event *event;
2243

I
Ingo Molnar 已提交
2244
	entry = iter->ent;
2245

2246 2247
	test_cpu_buff_start(iter);

2248
	event = ftrace_find_event(entry->type);
2249

2250
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2251 2252 2253 2254 2255 2256 2257
		if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
			if (!trace_print_lat_context(iter))
				goto partial;
		} else {
			if (!trace_print_context(iter))
				goto partial;
		}
2258
	}
2259

2260
	if (event)
2261
		return event->funcs->trace(iter, sym_flags, event);
2262 2263 2264

	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
		goto partial;
2265

2266
	return TRACE_TYPE_HANDLED;
2267 2268
partial:
	return TRACE_TYPE_PARTIAL_LINE;
2269 2270
}

2271
static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2272 2273 2274
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *entry;
2275
	struct trace_event *event;
I
Ingo Molnar 已提交
2276 2277

	entry = iter->ent;
2278

2279
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2280 2281 2282
		if (!trace_seq_printf(s, "%d %d %llu ",
				      entry->pid, iter->cpu, iter->ts))
			goto partial;
2283
	}
I
Ingo Molnar 已提交
2284

2285
	event = ftrace_find_event(entry->type);
2286
	if (event)
2287
		return event->funcs->raw(iter, 0, event);
2288 2289 2290

	if (!trace_seq_printf(s, "%d ?\n", entry->type))
		goto partial;
2291

2292
	return TRACE_TYPE_HANDLED;
2293 2294
partial:
	return TRACE_TYPE_PARTIAL_LINE;
I
Ingo Molnar 已提交
2295 2296
}

2297
static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2298 2299 2300 2301
{
	struct trace_seq *s = &iter->seq;
	unsigned char newline = '\n';
	struct trace_entry *entry;
2302
	struct trace_event *event;
2303 2304

	entry = iter->ent;
2305

2306 2307 2308 2309 2310
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
		SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
		SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
		SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
	}
2311

2312
	event = ftrace_find_event(entry->type);
2313
	if (event) {
2314
		enum print_line_t ret = event->funcs->hex(iter, 0, event);
2315 2316 2317
		if (ret != TRACE_TYPE_HANDLED)
			return ret;
	}
S
Steven Rostedt 已提交
2318

2319 2320
	SEQ_PUT_FIELD_RET(s, newline);

2321
	return TRACE_TYPE_HANDLED;
2322 2323
}

2324
static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2325 2326 2327
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *entry;
2328
	struct trace_event *event;
I
Ingo Molnar 已提交
2329 2330

	entry = iter->ent;
2331

2332 2333
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
		SEQ_PUT_FIELD_RET(s, entry->pid);
2334
		SEQ_PUT_FIELD_RET(s, iter->cpu);
2335 2336
		SEQ_PUT_FIELD_RET(s, iter->ts);
	}
I
Ingo Molnar 已提交
2337

2338
	event = ftrace_find_event(entry->type);
2339 2340
	return event ? event->funcs->binary(iter, 0, event) :
		TRACE_TYPE_HANDLED;
I
Ingo Molnar 已提交
2341 2342
}

2343
int trace_empty(struct trace_iterator *iter)
2344
{
2345
	struct ring_buffer_iter *buf_iter;
2346 2347
	int cpu;

2348
	/* If we are looking at one CPU buffer, only check that one */
2349
	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2350
		cpu = iter->cpu_file;
2351 2352 2353
		buf_iter = trace_buffer_iter(iter, cpu);
		if (buf_iter) {
			if (!ring_buffer_iter_empty(buf_iter))
2354 2355 2356 2357 2358 2359 2360 2361
				return 0;
		} else {
			if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
				return 0;
		}
		return 1;
	}

2362
	for_each_tracing_cpu(cpu) {
2363 2364 2365
		buf_iter = trace_buffer_iter(iter, cpu);
		if (buf_iter) {
			if (!ring_buffer_iter_empty(buf_iter))
2366 2367 2368 2369 2370
				return 0;
		} else {
			if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
				return 0;
		}
2371
	}
2372

2373
	return 1;
2374 2375
}

2376
/*  Called with trace_event_read_lock() held. */
2377
enum print_line_t print_trace_line(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2378
{
2379 2380
	enum print_line_t ret;

2381 2382 2383 2384
	if (iter->lost_events &&
	    !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
				 iter->cpu, iter->lost_events))
		return TRACE_TYPE_PARTIAL_LINE;
2385

2386 2387 2388 2389 2390
	if (iter->trace && iter->trace->print_line) {
		ret = iter->trace->print_line(iter);
		if (ret != TRACE_TYPE_UNHANDLED)
			return ret;
	}
2391

2392 2393 2394
	if (iter->ent->type == TRACE_BPRINT &&
			trace_flags & TRACE_ITER_PRINTK &&
			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2395
		return trace_print_bprintk_msg_only(iter);
2396

2397 2398 2399
	if (iter->ent->type == TRACE_PRINT &&
			trace_flags & TRACE_ITER_PRINTK &&
			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2400
		return trace_print_printk_msg_only(iter);
2401

I
Ingo Molnar 已提交
2402 2403 2404
	if (trace_flags & TRACE_ITER_BIN)
		return print_bin_fmt(iter);

2405 2406 2407
	if (trace_flags & TRACE_ITER_HEX)
		return print_hex_fmt(iter);

I
Ingo Molnar 已提交
2408 2409 2410 2411 2412 2413
	if (trace_flags & TRACE_ITER_RAW)
		return print_raw_fmt(iter);

	return print_trace_fmt(iter);
}

2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
void trace_latency_header(struct seq_file *m)
{
	struct trace_iterator *iter = m->private;

	/* print nothing if the buffers are empty */
	if (trace_empty(iter))
		return;

	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
		print_trace_header(m, iter);

	if (!(trace_flags & TRACE_ITER_VERBOSE))
		print_lat_help_header(m);
}

2429 2430 2431 2432
void trace_default_header(struct seq_file *m)
{
	struct trace_iterator *iter = m->private;

2433 2434 2435
	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
		return;

2436 2437 2438 2439 2440 2441 2442 2443
	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
		/* print nothing if the buffers are empty */
		if (trace_empty(iter))
			return;
		print_trace_header(m, iter);
		if (!(trace_flags & TRACE_ITER_VERBOSE))
			print_lat_help_header(m);
	} else {
2444 2445
		if (!(trace_flags & TRACE_ITER_VERBOSE)) {
			if (trace_flags & TRACE_ITER_IRQ_INFO)
2446
				print_func_help_header_irq(iter->tr, m);
2447
			else
2448
				print_func_help_header(iter->tr, m);
2449
		}
2450 2451 2452
	}
}

2453 2454 2455 2456 2457 2458 2459 2460
static void test_ftrace_alive(struct seq_file *m)
{
	if (!ftrace_is_dead())
		return;
	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
	seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
}

2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
#ifdef CONFIG_TRACER_MAX_TRACE
static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
{
	if (iter->trace->allocated_snapshot)
		seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
	else
		seq_printf(m, "#\n# * Snapshot is freed *\n#\n");

	seq_printf(m, "# Snapshot commands:\n");
	seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
	seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
	seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
	seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
	seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
	seq_printf(m, "#                       is not a '0' or '1')\n");
}
#else
/* Should never be called */
static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
#endif

2482 2483 2484
static int s_show(struct seq_file *m, void *v)
{
	struct trace_iterator *iter = v;
2485
	int ret;
2486 2487 2488 2489 2490

	if (iter->ent == NULL) {
		if (iter->tr) {
			seq_printf(m, "# tracer: %s\n", iter->trace->name);
			seq_puts(m, "#\n");
2491
			test_ftrace_alive(m);
2492
		}
2493 2494 2495
		if (iter->snapshot && trace_empty(iter))
			print_snapshot_help(m, iter);
		else if (iter->trace && iter->trace->print_header)
2496
			iter->trace->print_header(m);
2497 2498 2499
		else
			trace_default_header(m);

2500 2501 2502 2503 2504 2505 2506 2507 2508 2509
	} else if (iter->leftover) {
		/*
		 * If we filled the seq_file buffer earlier, we
		 * want to just show it now.
		 */
		ret = trace_print_seq(m, &iter->seq);

		/* ret should this time be zero, but you never know */
		iter->leftover = ret;

2510
	} else {
I
Ingo Molnar 已提交
2511
		print_trace_line(iter);
2512 2513 2514 2515 2516 2517 2518 2519 2520
		ret = trace_print_seq(m, &iter->seq);
		/*
		 * If we overflow the seq_file buffer, then it will
		 * ask us for this data again at start up.
		 * Use that instead.
		 *  ret is 0 if seq_file write succeeded.
		 *        -1 otherwise.
		 */
		iter->leftover = ret;
2521 2522 2523 2524 2525
	}

	return 0;
}

J
James Morris 已提交
2526
static const struct seq_operations tracer_seq_ops = {
I
Ingo Molnar 已提交
2527 2528 2529 2530
	.start		= s_start,
	.next		= s_next,
	.stop		= s_stop,
	.show		= s_show,
2531 2532
};

I
Ingo Molnar 已提交
2533
static struct trace_iterator *
2534
__tracing_open(struct inode *inode, struct file *file, bool snapshot)
2535
{
2536 2537
	struct trace_cpu *tc = inode->i_private;
	struct trace_array *tr = tc->tr;
2538
	struct trace_iterator *iter;
2539
	int cpu;
2540

2541 2542
	if (tracing_disabled)
		return ERR_PTR(-ENODEV);
S
Steven Rostedt 已提交
2543

2544
	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2545 2546
	if (!iter)
		return ERR_PTR(-ENOMEM);
2547

2548 2549
	iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
				    GFP_KERNEL);
2550 2551 2552
	if (!iter->buffer_iter)
		goto release;

2553 2554 2555 2556
	/*
	 * We make a copy of the current tracer to avoid concurrent
	 * changes on it while we are reading.
	 */
2557
	mutex_lock(&trace_types_lock);
2558
	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2559
	if (!iter->trace)
2560
		goto fail;
2561

2562
	*iter->trace = *tr->current_trace;
2563

2564
	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2565 2566
		goto fail;

2567 2568
	/* Currently only the top directory has a snapshot */
	if (tr->current_trace->print_max || snapshot)
2569 2570
		iter->tr = &max_tr;
	else
2571
		iter->tr = tr;
2572
	iter->snapshot = snapshot;
2573
	iter->pos = -1;
2574
	mutex_init(&iter->mutex);
2575
	iter->cpu_file = tc->cpu;
2576

2577 2578
	/* Notify the tracer early; before we stop tracing. */
	if (iter->trace && iter->trace->open)
2579
		iter->trace->open(iter);
2580

2581 2582 2583 2584
	/* Annotate start of buffers if we had overruns */
	if (ring_buffer_overruns(iter->tr->buffer))
		iter->iter_flags |= TRACE_FILE_ANNOTATE;

2585 2586 2587 2588
	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
	if (trace_clocks[trace_clock_id].in_ns)
		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;

2589 2590
	/* stop the trace while dumping if we are not opening "snapshot" */
	if (!iter->snapshot)
2591
		tracing_stop_tr(tr);
2592

2593
	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2594 2595
		for_each_tracing_cpu(cpu) {
			iter->buffer_iter[cpu] =
2596 2597 2598 2599 2600
				ring_buffer_read_prepare(iter->tr->buffer, cpu);
		}
		ring_buffer_read_prepare_sync();
		for_each_tracing_cpu(cpu) {
			ring_buffer_read_start(iter->buffer_iter[cpu]);
2601
			tracing_iter_reset(iter, cpu);
2602 2603 2604
		}
	} else {
		cpu = iter->cpu_file;
2605
		iter->buffer_iter[cpu] =
2606 2607 2608
			ring_buffer_read_prepare(iter->tr->buffer, cpu);
		ring_buffer_read_prepare_sync();
		ring_buffer_read_start(iter->buffer_iter[cpu]);
2609
		tracing_iter_reset(iter, cpu);
2610 2611
	}

2612 2613 2614
	mutex_unlock(&trace_types_lock);

	return iter;
2615

2616
 fail:
2617
	mutex_unlock(&trace_types_lock);
2618
	kfree(iter->trace);
2619
	kfree(iter->buffer_iter);
2620
release:
2621 2622
	seq_release_private(inode, file);
	return ERR_PTR(-ENOMEM);
2623 2624 2625 2626
}

int tracing_open_generic(struct inode *inode, struct file *filp)
{
S
Steven Rostedt 已提交
2627 2628 2629
	if (tracing_disabled)
		return -ENODEV;

2630 2631 2632 2633
	filp->private_data = inode->i_private;
	return 0;
}

2634
static int tracing_release(struct inode *inode, struct file *file)
2635
{
2636
	struct seq_file *m = file->private_data;
2637
	struct trace_iterator *iter;
2638
	struct trace_array *tr;
2639
	int cpu;
2640

2641 2642 2643 2644 2645
	if (!(file->f_mode & FMODE_READ))
		return 0;

	iter = m->private;

2646 2647 2648 2649 2650 2651
	/* Only the global tracer has a matching max_tr */
	if (iter->tr == &max_tr)
		tr = &global_trace;
	else
		tr = iter->tr;

2652
	mutex_lock(&trace_types_lock);
2653 2654 2655 2656 2657
	for_each_tracing_cpu(cpu) {
		if (iter->buffer_iter[cpu])
			ring_buffer_read_finish(iter->buffer_iter[cpu]);
	}

2658 2659 2660
	if (iter->trace && iter->trace->close)
		iter->trace->close(iter);

2661 2662
	if (!iter->snapshot)
		/* reenable tracing if it was previously enabled */
2663
		tracing_start_tr(tr);
2664 2665
	mutex_unlock(&trace_types_lock);

2666
	mutex_destroy(&iter->mutex);
2667
	free_cpumask_var(iter->started);
2668
	kfree(iter->trace);
2669
	kfree(iter->buffer_iter);
2670
	seq_release_private(inode, file);
2671 2672 2673 2674 2675
	return 0;
}

static int tracing_open(struct inode *inode, struct file *file)
{
2676 2677
	struct trace_iterator *iter;
	int ret = 0;
2678

2679 2680
	/* If this file was open for write, then erase contents */
	if ((file->f_mode & FMODE_WRITE) &&
2681
	    (file->f_flags & O_TRUNC)) {
2682 2683
		struct trace_cpu *tc = inode->i_private;
		struct trace_array *tr = tc->tr;
2684

2685 2686
		if (tc->cpu == RING_BUFFER_ALL_CPUS)
			tracing_reset_online_cpus(tr);
2687
		else
2688
			tracing_reset(tr, tc->cpu);
2689
	}
2690

2691
	if (file->f_mode & FMODE_READ) {
2692
		iter = __tracing_open(inode, file, false);
2693 2694 2695 2696 2697
		if (IS_ERR(iter))
			ret = PTR_ERR(iter);
		else if (trace_flags & TRACE_ITER_LATENCY_FMT)
			iter->iter_flags |= TRACE_FILE_LAT_FMT;
	}
2698 2699 2700
	return ret;
}

I
Ingo Molnar 已提交
2701
static void *
2702 2703
t_next(struct seq_file *m, void *v, loff_t *pos)
{
L
Li Zefan 已提交
2704
	struct tracer *t = v;
2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715

	(*pos)++;

	if (t)
		t = t->next;

	return t;
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
L
Li Zefan 已提交
2716
	struct tracer *t;
2717 2718 2719
	loff_t l = 0;

	mutex_lock(&trace_types_lock);
L
Li Zefan 已提交
2720
	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746
		;

	return t;
}

static void t_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&trace_types_lock);
}

static int t_show(struct seq_file *m, void *v)
{
	struct tracer *t = v;

	if (!t)
		return 0;

	seq_printf(m, "%s", t->name);
	if (t->next)
		seq_putc(m, ' ');
	else
		seq_putc(m, '\n');

	return 0;
}

J
James Morris 已提交
2747
static const struct seq_operations show_traces_seq_ops = {
I
Ingo Molnar 已提交
2748 2749 2750 2751
	.start		= t_start,
	.next		= t_next,
	.stop		= t_stop,
	.show		= t_show,
2752 2753 2754 2755
};

static int show_traces_open(struct inode *inode, struct file *file)
{
S
Steven Rostedt 已提交
2756 2757 2758
	if (tracing_disabled)
		return -ENODEV;

L
Li Zefan 已提交
2759
	return seq_open(file, &show_traces_seq_ops);
2760 2761
}

2762 2763 2764 2765 2766 2767 2768
static ssize_t
tracing_write_stub(struct file *filp, const char __user *ubuf,
		   size_t count, loff_t *ppos)
{
	return count;
}

2769 2770 2771 2772 2773 2774 2775 2776
static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
{
	if (file->f_mode & FMODE_READ)
		return seq_lseek(file, offset, origin);
	else
		return 0;
}

2777
static const struct file_operations tracing_fops = {
I
Ingo Molnar 已提交
2778 2779
	.open		= tracing_open,
	.read		= seq_read,
2780
	.write		= tracing_write_stub,
2781
	.llseek		= tracing_seek,
I
Ingo Molnar 已提交
2782
	.release	= tracing_release,
2783 2784
};

2785
static const struct file_operations show_traces_fops = {
I
Ingo Molnar 已提交
2786 2787 2788
	.open		= show_traces_open,
	.read		= seq_read,
	.release	= seq_release,
2789
	.llseek		= seq_lseek,
I
Ingo Molnar 已提交
2790 2791
};

2792 2793 2794
/*
 * Only trace on a CPU if the bitmask is set:
 */
2795
static cpumask_var_t tracing_cpumask;
2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808

/*
 * The tracer itself will not take this lock, but still we want
 * to provide a consistent cpumask to user-space:
 */
static DEFINE_MUTEX(tracing_cpumask_update_lock);

/*
 * Temporary storage for the character representation of the
 * CPU bitmask (and one more byte for the newline):
 */
static char mask_str[NR_CPUS + 1];

I
Ingo Molnar 已提交
2809 2810 2811 2812
static ssize_t
tracing_cpumask_read(struct file *filp, char __user *ubuf,
		     size_t count, loff_t *ppos)
{
2813
	int len;
I
Ingo Molnar 已提交
2814 2815

	mutex_lock(&tracing_cpumask_update_lock);
2816

2817
	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2818 2819 2820 2821 2822 2823 2824 2825
	if (count - len < 2) {
		count = -EINVAL;
		goto out_err;
	}
	len += sprintf(mask_str + len, "\n");
	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);

out_err:
I
Ingo Molnar 已提交
2826 2827 2828 2829 2830 2831 2832 2833 2834
	mutex_unlock(&tracing_cpumask_update_lock);

	return count;
}

static ssize_t
tracing_cpumask_write(struct file *filp, const char __user *ubuf,
		      size_t count, loff_t *ppos)
{
2835
	struct trace_array *tr = filp->private_data;
2836
	cpumask_var_t tracing_cpumask_new;
2837
	int err, cpu;
2838 2839 2840

	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
		return -ENOMEM;
I
Ingo Molnar 已提交
2841

2842
	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
I
Ingo Molnar 已提交
2843
	if (err)
2844 2845
		goto err_unlock;

2846 2847
	mutex_lock(&tracing_cpumask_update_lock);

2848
	local_irq_disable();
2849
	arch_spin_lock(&ftrace_max_lock);
2850
	for_each_tracing_cpu(cpu) {
2851 2852 2853 2854
		/*
		 * Increase/decrease the disabled counter if we are
		 * about to flip a bit in the cpumask:
		 */
2855 2856
		if (cpumask_test_cpu(cpu, tracing_cpumask) &&
				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2857
			atomic_inc(&per_cpu_ptr(tr->data, cpu)->disabled);
2858
			ring_buffer_record_disable_cpu(tr->buffer, cpu);
2859
		}
2860 2861
		if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2862
			atomic_dec(&per_cpu_ptr(tr->data, cpu)->disabled);
2863
			ring_buffer_record_enable_cpu(tr->buffer, cpu);
2864 2865
		}
	}
2866
	arch_spin_unlock(&ftrace_max_lock);
2867
	local_irq_enable();
2868

2869
	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2870 2871

	mutex_unlock(&tracing_cpumask_update_lock);
2872
	free_cpumask_var(tracing_cpumask_new);
I
Ingo Molnar 已提交
2873 2874

	return count;
2875 2876

err_unlock:
2877
	free_cpumask_var(tracing_cpumask_new);
2878 2879

	return err;
I
Ingo Molnar 已提交
2880 2881
}

2882
static const struct file_operations tracing_cpumask_fops = {
I
Ingo Molnar 已提交
2883 2884 2885
	.open		= tracing_open_generic,
	.read		= tracing_cpumask_read,
	.write		= tracing_cpumask_write,
2886
	.llseek		= generic_file_llseek,
2887 2888
};

L
Li Zefan 已提交
2889
static int tracing_trace_options_show(struct seq_file *m, void *v)
2890
{
2891
	struct tracer_opt *trace_opts;
2892
	struct trace_array *tr = m->private;
2893 2894
	u32 tracer_flags;
	int i;
2895

2896
	mutex_lock(&trace_types_lock);
2897 2898
	tracer_flags = tr->current_trace->flags->val;
	trace_opts = tr->current_trace->flags->opts;
2899

2900 2901
	for (i = 0; trace_options[i]; i++) {
		if (trace_flags & (1 << i))
L
Li Zefan 已提交
2902
			seq_printf(m, "%s\n", trace_options[i]);
2903
		else
L
Li Zefan 已提交
2904
			seq_printf(m, "no%s\n", trace_options[i]);
2905 2906
	}

2907 2908
	for (i = 0; trace_opts[i].name; i++) {
		if (tracer_flags & trace_opts[i].bit)
L
Li Zefan 已提交
2909
			seq_printf(m, "%s\n", trace_opts[i].name);
2910
		else
L
Li Zefan 已提交
2911
			seq_printf(m, "no%s\n", trace_opts[i].name);
2912
	}
2913
	mutex_unlock(&trace_types_lock);
2914

L
Li Zefan 已提交
2915
	return 0;
2916 2917
}

L
Li Zefan 已提交
2918 2919 2920 2921 2922
static int __set_tracer_option(struct tracer *trace,
			       struct tracer_flags *tracer_flags,
			       struct tracer_opt *opts, int neg)
{
	int ret;
2923

L
Li Zefan 已提交
2924 2925 2926 2927 2928 2929 2930 2931 2932
	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
	if (ret)
		return ret;

	if (neg)
		tracer_flags->val &= ~opts->bit;
	else
		tracer_flags->val |= opts->bit;
	return 0;
2933 2934
}

2935 2936 2937
/* Try to assign a tracer specific option */
static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
{
2938
	struct tracer_flags *tracer_flags = trace->flags;
2939
	struct tracer_opt *opts = NULL;
L
Li Zefan 已提交
2940
	int i;
2941

2942 2943
	for (i = 0; tracer_flags->opts[i].name; i++) {
		opts = &tracer_flags->opts[i];
2944

L
Li Zefan 已提交
2945 2946 2947
		if (strcmp(cmp, opts->name) == 0)
			return __set_tracer_option(trace, trace->flags,
						   opts, neg);
2948 2949
	}

L
Li Zefan 已提交
2950
	return -EINVAL;
2951 2952
}

2953 2954 2955 2956 2957 2958 2959 2960 2961
/* Some tracers require overwrite to stay enabled */
int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
{
	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
		return -1;

	return 0;
}

2962
int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
2963 2964 2965
{
	/* do nothing if flag is already set */
	if (!!(trace_flags & mask) == !!enabled)
2966 2967 2968
		return 0;

	/* Give the tracer a chance to approve the change */
2969 2970
	if (tr->current_trace->flag_changed)
		if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
2971
			return -EINVAL;
2972 2973 2974 2975 2976

	if (enabled)
		trace_flags |= mask;
	else
		trace_flags &= ~mask;
2977 2978 2979

	if (mask == TRACE_ITER_RECORD_CMD)
		trace_event_enable_cmd_record(enabled);
2980

2981
	if (mask == TRACE_ITER_OVERWRITE) {
2982
		ring_buffer_change_overwrite(global_trace.buffer, enabled);
2983 2984 2985 2986
#ifdef CONFIG_TRACER_MAX_TRACE
		ring_buffer_change_overwrite(max_tr.buffer, enabled);
#endif
	}
2987 2988 2989

	if (mask == TRACE_ITER_PRINTK)
		trace_printk_start_stop_comm(enabled);
2990 2991

	return 0;
2992 2993
}

2994
static int trace_set_options(struct trace_array *tr, char *option)
2995
{
L
Li Zefan 已提交
2996
	char *cmp;
2997
	int neg = 0;
2998
	int ret = -ENODEV;
2999 3000
	int i;

3001
	cmp = strstrip(option);
3002

L
Li Zefan 已提交
3003
	if (strncmp(cmp, "no", 2) == 0) {
3004 3005 3006 3007
		neg = 1;
		cmp += 2;
	}

3008 3009
	mutex_lock(&trace_types_lock);

3010
	for (i = 0; trace_options[i]; i++) {
L
Li Zefan 已提交
3011
		if (strcmp(cmp, trace_options[i]) == 0) {
3012
			ret = set_tracer_flag(tr, 1 << i, !neg);
3013 3014 3015
			break;
		}
	}
3016 3017

	/* If no option could be set, test the specific tracer options */
3018
	if (!trace_options[i])
3019
		ret = set_tracer_option(tr->current_trace, cmp, neg);
3020 3021

	mutex_unlock(&trace_types_lock);
3022

3023 3024 3025 3026 3027 3028 3029
	return ret;
}

static ssize_t
tracing_trace_options_write(struct file *filp, const char __user *ubuf,
			size_t cnt, loff_t *ppos)
{
3030 3031
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
3032
	char buf[64];
3033
	int ret;
3034 3035 3036 3037 3038 3039 3040

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

3041 3042
	buf[cnt] = 0;

3043
	ret = trace_set_options(tr, buf);
3044 3045
	if (ret < 0)
		return ret;
3046

3047
	*ppos += cnt;
3048 3049 3050 3051

	return cnt;
}

L
Li Zefan 已提交
3052 3053 3054 3055
static int tracing_trace_options_open(struct inode *inode, struct file *file)
{
	if (tracing_disabled)
		return -ENODEV;
3056 3057

	return single_open(file, tracing_trace_options_show, inode->i_private);
L
Li Zefan 已提交
3058 3059
}

3060
static const struct file_operations tracing_iter_fops = {
L
Li Zefan 已提交
3061 3062 3063 3064
	.open		= tracing_trace_options_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
3065
	.write		= tracing_trace_options_write,
3066 3067
};

I
Ingo Molnar 已提交
3068 3069
static const char readme_msg[] =
	"tracing mini-HOWTO:\n\n"
3070 3071
	"# mount -t debugfs nodev /sys/kernel/debug\n\n"
	"# cat /sys/kernel/debug/tracing/available_tracers\n"
3072
	"wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
3073
	"# cat /sys/kernel/debug/tracing/current_tracer\n"
3074
	"nop\n"
3075
	"# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
3076
	"# cat /sys/kernel/debug/tracing/current_tracer\n"
3077
	"wakeup\n"
3078
	"# cat /sys/kernel/debug/tracing/trace_options\n"
I
Ingo Molnar 已提交
3079
	"noprint-parent nosym-offset nosym-addr noverbose\n"
3080
	"# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
3081
	"# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
3082
	"# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
3083
	"# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
I
Ingo Molnar 已提交
3084 3085 3086 3087 3088 3089 3090 3091 3092 3093
;

static ssize_t
tracing_readme_read(struct file *filp, char __user *ubuf,
		       size_t cnt, loff_t *ppos)
{
	return simple_read_from_buffer(ubuf, cnt, ppos,
					readme_msg, strlen(readme_msg));
}

3094
static const struct file_operations tracing_readme_fops = {
I
Ingo Molnar 已提交
3095 3096
	.open		= tracing_open_generic,
	.read		= tracing_readme_read,
3097
	.llseek		= generic_file_llseek,
I
Ingo Molnar 已提交
3098 3099
};

3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147
static ssize_t
tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
				size_t cnt, loff_t *ppos)
{
	char *buf_comm;
	char *file_buf;
	char *buf;
	int len = 0;
	int pid;
	int i;

	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
	if (!file_buf)
		return -ENOMEM;

	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
	if (!buf_comm) {
		kfree(file_buf);
		return -ENOMEM;
	}

	buf = file_buf;

	for (i = 0; i < SAVED_CMDLINES; i++) {
		int r;

		pid = map_cmdline_to_pid[i];
		if (pid == -1 || pid == NO_CMDLINE_MAP)
			continue;

		trace_find_cmdline(pid, buf_comm);
		r = sprintf(buf, "%d %s\n", pid, buf_comm);
		buf += r;
		len += r;
	}

	len = simple_read_from_buffer(ubuf, cnt, ppos,
				      file_buf, len);

	kfree(file_buf);
	kfree(buf_comm);

	return len;
}

static const struct file_operations tracing_saved_cmdlines_fops = {
    .open       = tracing_open_generic,
    .read       = tracing_saved_cmdlines_read,
3148
    .llseek	= generic_file_llseek,
3149 3150
};

3151 3152 3153 3154
static ssize_t
tracing_set_trace_read(struct file *filp, char __user *ubuf,
		       size_t cnt, loff_t *ppos)
{
3155
	struct trace_array *tr = filp->private_data;
L
Li Zefan 已提交
3156
	char buf[MAX_TRACER_SIZE+2];
3157 3158 3159
	int r;

	mutex_lock(&trace_types_lock);
3160
	r = sprintf(buf, "%s\n", tr->current_trace->name);
3161 3162
	mutex_unlock(&trace_types_lock);

I
Ingo Molnar 已提交
3163
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3164 3165
}

3166 3167 3168 3169 3170 3171
int tracer_init(struct tracer *t, struct trace_array *tr)
{
	tracing_reset_online_cpus(tr);
	return t->init(tr);
}

3172 3173 3174 3175
static void set_buffer_entries(struct trace_array *tr, unsigned long val)
{
	int cpu;
	for_each_tracing_cpu(cpu)
3176
		per_cpu_ptr(tr->data, cpu)->entries = val;
3177 3178
}

3179 3180 3181 3182 3183 3184 3185 3186 3187
/* resize @tr's buffer to the size of @size_tr's entries */
static int resize_buffer_duplicate_size(struct trace_array *tr,
					struct trace_array *size_tr, int cpu_id)
{
	int cpu, ret = 0;

	if (cpu_id == RING_BUFFER_ALL_CPUS) {
		for_each_tracing_cpu(cpu) {
			ret = ring_buffer_resize(tr->buffer,
3188
				 per_cpu_ptr(size_tr->data, cpu)->entries, cpu);
3189 3190
			if (ret < 0)
				break;
3191 3192
			per_cpu_ptr(tr->data, cpu)->entries =
				per_cpu_ptr(size_tr->data, cpu)->entries;
3193 3194 3195
		}
	} else {
		ret = ring_buffer_resize(tr->buffer,
3196
				 per_cpu_ptr(size_tr->data, cpu_id)->entries, cpu_id);
3197
		if (ret == 0)
3198 3199
			per_cpu_ptr(tr->data, cpu_id)->entries =
				per_cpu_ptr(size_tr->data, cpu_id)->entries;
3200 3201 3202 3203 3204
	}

	return ret;
}

3205 3206
static int __tracing_resize_ring_buffer(struct trace_array *tr,
					unsigned long size, int cpu)
3207 3208 3209 3210 3211
{
	int ret;

	/*
	 * If kernel or user changes the size of the ring buffer
3212 3213
	 * we use the size that was given, and we can forget about
	 * expanding it later.
3214 3215 3216
	 */
	ring_buffer_expanded = 1;

3217
	/* May be called before buffers are initialized */
3218
	if (!tr->buffer)
3219 3220
		return 0;

3221
	ret = ring_buffer_resize(tr->buffer, size, cpu);
3222 3223 3224
	if (ret < 0)
		return ret;

3225 3226
	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
	    !tr->current_trace->use_max_tr)
3227 3228
		goto out;

3229
	ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3230
	if (ret < 0) {
3231
		int r = resize_buffer_duplicate_size(tr, tr, cpu);
3232
		if (r < 0) {
3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246
			/*
			 * AARGH! We are left with different
			 * size max buffer!!!!
			 * The max buffer is our "snapshot" buffer.
			 * When a tracer needs a snapshot (one of the
			 * latency tracers), it swaps the max buffer
			 * with the saved snap shot. We succeeded to
			 * update the size of the main buffer, but failed to
			 * update the size of the max buffer. But when we tried
			 * to reset the main buffer to the original size, we
			 * failed there too. This is very unlikely to
			 * happen, but if it does, warn and kill all
			 * tracing.
			 */
3247 3248 3249 3250 3251 3252
			WARN_ON(1);
			tracing_disabled = 1;
		}
		return ret;
	}

3253 3254 3255
	if (cpu == RING_BUFFER_ALL_CPUS)
		set_buffer_entries(&max_tr, size);
	else
3256
		per_cpu_ptr(max_tr.data, cpu)->entries = size;
3257

3258
 out:
3259
	if (cpu == RING_BUFFER_ALL_CPUS)
3260
		set_buffer_entries(tr, size);
3261
	else
3262
		per_cpu_ptr(tr->data, cpu)->entries = size;
3263 3264 3265 3266

	return ret;
}

3267 3268
static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
					  unsigned long size, int cpu_id)
3269
{
3270
	int ret = size;
3271 3272 3273

	mutex_lock(&trace_types_lock);

3274 3275 3276 3277 3278 3279 3280
	if (cpu_id != RING_BUFFER_ALL_CPUS) {
		/* make sure, this cpu is enabled in the mask */
		if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
			ret = -EINVAL;
			goto out;
		}
	}
3281

3282
	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3283 3284 3285
	if (ret < 0)
		ret = -ENOMEM;

3286
out:
3287 3288 3289 3290 3291
	mutex_unlock(&trace_types_lock);

	return ret;
}

3292

3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
/**
 * tracing_update_buffers - used by tracing facility to expand ring buffers
 *
 * To save on memory when the tracing is never used on a system with it
 * configured in. The ring buffers are set to a minimum size. But once
 * a user starts to use the tracing facility, then they need to grow
 * to their default size.
 *
 * This function is to be called when a tracer is about to be used.
 */
int tracing_update_buffers(void)
{
	int ret = 0;

3307
	mutex_lock(&trace_types_lock);
3308
	if (!ring_buffer_expanded)
3309
		ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3310
						RING_BUFFER_ALL_CPUS);
3311
	mutex_unlock(&trace_types_lock);
3312 3313 3314 3315

	return ret;
}

3316 3317 3318
struct trace_option_dentry;

static struct trace_option_dentry *
3319
create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3320 3321 3322 3323

static void
destroy_trace_option_files(struct trace_option_dentry *topts);

3324
static int tracing_set_tracer(const char *buf)
3325
{
3326
	static struct trace_option_dentry *topts;
3327 3328
	struct trace_array *tr = &global_trace;
	struct tracer *t;
3329
	bool had_max_tr;
3330
	int ret = 0;
3331

3332 3333
	mutex_lock(&trace_types_lock);

3334
	if (!ring_buffer_expanded) {
3335
		ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3336
						RING_BUFFER_ALL_CPUS);
3337
		if (ret < 0)
3338
			goto out;
3339 3340 3341
		ret = 0;
	}

3342 3343 3344 3345
	for (t = trace_types; t; t = t->next) {
		if (strcmp(t->name, buf) == 0)
			break;
	}
3346 3347 3348 3349
	if (!t) {
		ret = -EINVAL;
		goto out;
	}
3350
	if (t == tr->current_trace)
3351 3352
		goto out;

3353
	trace_branch_disable();
3354

3355
	tr->current_trace->enabled = false;
3356

3357 3358
	if (tr->current_trace->reset)
		tr->current_trace->reset(tr);
3359

3360 3361
	had_max_tr = tr->current_trace->allocated_snapshot;
	tr->current_trace = &nop_trace;
3362 3363 3364 3365 3366 3367 3368 3369 3370 3371

	if (had_max_tr && !t->use_max_tr) {
		/*
		 * We need to make sure that the update_max_tr sees that
		 * current_trace changed to nop_trace to keep it from
		 * swapping the buffers after we resize it.
		 * The update_max_tr is called from interrupts disabled
		 * so a synchronized_sched() is sufficient.
		 */
		synchronize_sched();
3372 3373 3374 3375 3376
		/*
		 * We don't free the ring buffer. instead, resize it because
		 * The max_tr ring buffer has some state (e.g. ring->clock) and
		 * we want preserve it.
		 */
3377 3378
		ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
		set_buffer_entries(&max_tr, 1);
3379
		tracing_reset_online_cpus(&max_tr);
3380
		tr->current_trace->allocated_snapshot = false;
3381
	}
3382 3383
	destroy_trace_option_files(topts);

3384
	topts = create_trace_option_files(tr, t);
3385
	if (t->use_max_tr && !had_max_tr) {
3386
		/* we need to make per cpu buffer sizes equivalent */
3387 3388 3389 3390
		ret = resize_buffer_duplicate_size(&max_tr, &global_trace,
						   RING_BUFFER_ALL_CPUS);
		if (ret < 0)
			goto out;
3391
		t->allocated_snapshot = true;
3392
	}
3393

3394
	if (t->init) {
3395
		ret = tracer_init(t, tr);
3396 3397 3398
		if (ret)
			goto out;
	}
3399

3400 3401
	tr->current_trace = t;
	tr->current_trace->enabled = true;
3402
	trace_branch_enable(tr);
3403 3404 3405
 out:
	mutex_unlock(&trace_types_lock);

3406 3407 3408 3409 3410 3411 3412
	return ret;
}

static ssize_t
tracing_set_trace_write(struct file *filp, const char __user *ubuf,
			size_t cnt, loff_t *ppos)
{
L
Li Zefan 已提交
3413
	char buf[MAX_TRACER_SIZE+1];
3414 3415
	int i;
	size_t ret;
3416 3417 3418
	int err;

	ret = cnt;
3419

L
Li Zefan 已提交
3420 3421
	if (cnt > MAX_TRACER_SIZE)
		cnt = MAX_TRACER_SIZE;
3422 3423 3424 3425 3426 3427 3428 3429 3430 3431

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	/* strip ending whitespace. */
	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
		buf[i] = 0;

3432 3433 3434
	err = tracing_set_tracer(buf);
	if (err)
		return err;
3435

3436
	*ppos += ret;
3437

3438
	return ret;
3439 3440 3441 3442 3443 3444 3445 3446 3447 3448
}

static ssize_t
tracing_max_lat_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long *ptr = filp->private_data;
	char buf[64];
	int r;

S
Steven Rostedt 已提交
3449
	r = snprintf(buf, sizeof(buf), "%ld\n",
3450
		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
S
Steven Rostedt 已提交
3451 3452
	if (r > sizeof(buf))
		r = sizeof(buf);
I
Ingo Molnar 已提交
3453
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3454 3455 3456 3457 3458 3459
}

static ssize_t
tracing_max_lat_write(struct file *filp, const char __user *ubuf,
		      size_t cnt, loff_t *ppos)
{
3460 3461
	unsigned long *ptr = filp->private_data;
	unsigned long val;
3462
	int ret;
3463

3464 3465
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
3466
		return ret;
3467 3468 3469 3470 3471 3472

	*ptr = val * 1000;

	return cnt;
}

3473 3474
static int tracing_open_pipe(struct inode *inode, struct file *filp)
{
3475 3476
	struct trace_cpu *tc = inode->i_private;
	struct trace_array *tr = tc->tr;
3477
	struct trace_iterator *iter;
3478
	int ret = 0;
3479 3480 3481 3482

	if (tracing_disabled)
		return -ENODEV;

3483 3484
	mutex_lock(&trace_types_lock);

3485 3486
	/* create a buffer to store the information to pass to userspace */
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3487 3488 3489 3490
	if (!iter) {
		ret = -ENOMEM;
		goto out;
	}
3491

3492 3493 3494 3495 3496 3497 3498 3499 3500
	/*
	 * We make a copy of the current tracer to avoid concurrent
	 * changes on it while we are reading.
	 */
	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
	if (!iter->trace) {
		ret = -ENOMEM;
		goto fail;
	}
3501
	*iter->trace = *tr->current_trace;
3502

3503
	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3504
		ret = -ENOMEM;
3505
		goto fail;
3506 3507
	}

3508
	/* trace pipe does not show start of buffer */
3509
	cpumask_setall(iter->started);
3510

3511 3512 3513
	if (trace_flags & TRACE_ITER_LATENCY_FMT)
		iter->iter_flags |= TRACE_FILE_LAT_FMT;

3514 3515 3516 3517
	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
	if (trace_clocks[trace_clock_id].in_ns)
		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;

3518 3519
	iter->cpu_file = tc->cpu;
	iter->tr = tc->tr;
3520
	mutex_init(&iter->mutex);
3521 3522
	filp->private_data = iter;

3523 3524 3525
	if (iter->trace->pipe_open)
		iter->trace->pipe_open(iter);

3526
	nonseekable_open(inode, filp);
3527 3528 3529
out:
	mutex_unlock(&trace_types_lock);
	return ret;
3530 3531 3532 3533 3534 3535

fail:
	kfree(iter->trace);
	kfree(iter);
	mutex_unlock(&trace_types_lock);
	return ret;
3536 3537 3538 3539 3540 3541
}

static int tracing_release_pipe(struct inode *inode, struct file *file)
{
	struct trace_iterator *iter = file->private_data;

3542 3543
	mutex_lock(&trace_types_lock);

3544
	if (iter->trace->pipe_close)
S
Steven Rostedt 已提交
3545 3546
		iter->trace->pipe_close(iter);

3547 3548
	mutex_unlock(&trace_types_lock);

3549
	free_cpumask_var(iter->started);
3550 3551
	mutex_destroy(&iter->mutex);
	kfree(iter->trace);
3552 3553 3554 3555 3556
	kfree(iter);

	return 0;
}

3557 3558 3559 3560 3561 3562 3563 3564 3565 3566
static unsigned int
tracing_poll_pipe(struct file *filp, poll_table *poll_table)
{
	struct trace_iterator *iter = filp->private_data;

	if (trace_flags & TRACE_ITER_BLOCK) {
		/*
		 * Always select as readable when in blocking mode
		 */
		return POLLIN | POLLRDNORM;
I
Ingo Molnar 已提交
3567
	} else {
3568 3569 3570 3571 3572 3573 3574 3575 3576 3577
		if (!trace_empty(iter))
			return POLLIN | POLLRDNORM;
		poll_wait(filp, &trace_wait, poll_table);
		if (!trace_empty(iter))
			return POLLIN | POLLRDNORM;

		return 0;
	}
}

3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596
/*
 * This is a make-shift waitqueue.
 * A tracer might use this callback on some rare cases:
 *
 *  1) the current tracer might hold the runqueue lock when it wakes up
 *     a reader, hence a deadlock (sched, function, and function graph tracers)
 *  2) the function tracers, trace all functions, we don't want
 *     the overhead of calling wake_up and friends
 *     (and tracing them too)
 *
 *     Anyway, this is really very primitive wakeup.
 */
void poll_wait_pipe(struct trace_iterator *iter)
{
	set_current_state(TASK_INTERRUPTIBLE);
	/* sleep for 100 msecs, and try again. */
	schedule_timeout(HZ / 10);
}

3597 3598
/* Must be called with trace_types_lock mutex held. */
static int tracing_wait_pipe(struct file *filp)
3599 3600 3601 3602
{
	struct trace_iterator *iter = filp->private_data;

	while (trace_empty(iter)) {
3603

3604
		if ((filp->f_flags & O_NONBLOCK)) {
3605
			return -EAGAIN;
3606
		}
3607

3608
		mutex_unlock(&iter->mutex);
3609

3610
		iter->trace->wait_pipe(iter);
3611

3612
		mutex_lock(&iter->mutex);
3613

3614
		if (signal_pending(current))
3615
			return -EINTR;
3616 3617

		/*
L
Liu Bo 已提交
3618
		 * We block until we read something and tracing is disabled.
3619 3620 3621 3622 3623 3624 3625
		 * We still block if tracing is disabled, but we have never
		 * read anything. This allows a user to cat this file, and
		 * then enable tracing. But after we have read something,
		 * we give an EOF when tracing is again disabled.
		 *
		 * iter->pos will be 0 if we haven't read anything.
		 */
L
Liu Bo 已提交
3626
		if (!tracing_is_enabled() && iter->pos)
3627 3628 3629
			break;
	}

3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640
	return 1;
}

/*
 * Consumer reader.
 */
static ssize_t
tracing_read_pipe(struct file *filp, char __user *ubuf,
		  size_t cnt, loff_t *ppos)
{
	struct trace_iterator *iter = filp->private_data;
3641
	struct trace_array *tr = iter->tr;
3642 3643 3644 3645 3646 3647 3648
	ssize_t sret;

	/* return any leftover data */
	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
	if (sret != -EBUSY)
		return sret;

3649
	trace_seq_init(&iter->seq);
3650

3651
	/* copy the tracer to avoid using a global lock all around */
3652
	mutex_lock(&trace_types_lock);
3653 3654
	if (unlikely(iter->trace->name != tr->current_trace->name))
		*iter->trace = *tr->current_trace;
3655 3656 3657 3658 3659 3660 3661 3662
	mutex_unlock(&trace_types_lock);

	/*
	 * Avoid more than one consumer on a single file descriptor
	 * This is just a matter of traces coherency, the ring buffer itself
	 * is protected.
	 */
	mutex_lock(&iter->mutex);
3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673
	if (iter->trace->read) {
		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
		if (sret)
			goto out;
	}

waitagain:
	sret = tracing_wait_pipe(filp);
	if (sret <= 0)
		goto out;

3674
	/* stop when tracing is finished */
3675 3676
	if (trace_empty(iter)) {
		sret = 0;
3677
		goto out;
3678
	}
3679 3680 3681 3682

	if (cnt >= PAGE_SIZE)
		cnt = PAGE_SIZE - 1;

3683 3684 3685 3686
	/* reset all but tr, trace, and overruns */
	memset(&iter->seq, 0,
	       sizeof(struct trace_iterator) -
	       offsetof(struct trace_iterator, seq));
3687
	iter->pos = -1;
3688

3689
	trace_event_read_lock();
3690
	trace_access_lock(iter->cpu_file);
3691
	while (trace_find_next_entry_inc(iter) != NULL) {
3692
		enum print_line_t ret;
S
Steven Rostedt 已提交
3693 3694
		int len = iter->seq.len;

I
Ingo Molnar 已提交
3695
		ret = print_trace_line(iter);
3696
		if (ret == TRACE_TYPE_PARTIAL_LINE) {
S
Steven Rostedt 已提交
3697 3698
			/* don't print partial lines */
			iter->seq.len = len;
3699
			break;
S
Steven Rostedt 已提交
3700
		}
3701 3702
		if (ret != TRACE_TYPE_NO_CONSUME)
			trace_consume(iter);
3703 3704 3705

		if (iter->seq.len >= cnt)
			break;
3706 3707 3708 3709 3710 3711 3712 3713

		/*
		 * Setting the full flag means we reached the trace_seq buffer
		 * size and we should leave by partial output condition above.
		 * One of the trace_seq_* functions is not used properly.
		 */
		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
			  iter->ent->type);
3714
	}
3715
	trace_access_unlock(iter->cpu_file);
3716
	trace_event_read_unlock();
3717 3718

	/* Now copy what we have to the user */
3719 3720
	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
	if (iter->seq.readpos >= iter->seq.len)
3721
		trace_seq_init(&iter->seq);
P
Pekka Paalanen 已提交
3722 3723

	/*
L
Lucas De Marchi 已提交
3724
	 * If there was nothing to send to user, in spite of consuming trace
P
Pekka Paalanen 已提交
3725 3726
	 * entries, go back to wait for more entries.
	 */
3727
	if (sret == -EBUSY)
P
Pekka Paalanen 已提交
3728
		goto waitagain;
3729

3730
out:
3731
	mutex_unlock(&iter->mutex);
3732

3733
	return sret;
3734 3735
}

3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747
static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
				     struct pipe_buffer *buf)
{
	__free_page(buf->page);
}

static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
				     unsigned int idx)
{
	__free_page(spd->pages[idx]);
}

3748
static const struct pipe_buf_operations tracing_pipe_buf_ops = {
S
Steven Rostedt 已提交
3749 3750 3751 3752 3753 3754 3755
	.can_merge		= 0,
	.map			= generic_pipe_buf_map,
	.unmap			= generic_pipe_buf_unmap,
	.confirm		= generic_pipe_buf_confirm,
	.release		= tracing_pipe_buf_release,
	.steal			= generic_pipe_buf_steal,
	.get			= generic_pipe_buf_get,
3756 3757
};

S
Steven Rostedt 已提交
3758
static size_t
3759
tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
S
Steven Rostedt 已提交
3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778
{
	size_t count;
	int ret;

	/* Seq buffer is page-sized, exactly what we need. */
	for (;;) {
		count = iter->seq.len;
		ret = print_trace_line(iter);
		count = iter->seq.len - count;
		if (rem < count) {
			rem = 0;
			iter->seq.len -= count;
			break;
		}
		if (ret == TRACE_TYPE_PARTIAL_LINE) {
			iter->seq.len -= count;
			break;
		}

3779 3780
		if (ret != TRACE_TYPE_NO_CONSUME)
			trace_consume(iter);
S
Steven Rostedt 已提交
3781
		rem -= count;
3782
		if (!trace_find_next_entry_inc(iter))	{
S
Steven Rostedt 已提交
3783 3784 3785 3786 3787 3788 3789 3790 3791
			rem = 0;
			iter->ent = NULL;
			break;
		}
	}

	return rem;
}

3792 3793 3794 3795 3796 3797
static ssize_t tracing_splice_read_pipe(struct file *filp,
					loff_t *ppos,
					struct pipe_inode_info *pipe,
					size_t len,
					unsigned int flags)
{
3798 3799
	struct page *pages_def[PIPE_DEF_BUFFERS];
	struct partial_page partial_def[PIPE_DEF_BUFFERS];
3800 3801
	struct trace_iterator *iter = filp->private_data;
	struct splice_pipe_desc spd = {
3802 3803
		.pages		= pages_def,
		.partial	= partial_def,
S
Steven Rostedt 已提交
3804
		.nr_pages	= 0, /* This gets updated below. */
3805
		.nr_pages_max	= PIPE_DEF_BUFFERS,
S
Steven Rostedt 已提交
3806 3807 3808
		.flags		= flags,
		.ops		= &tracing_pipe_buf_ops,
		.spd_release	= tracing_spd_release_pipe,
3809
	};
3810
	struct trace_array *tr = iter->tr;
3811
	ssize_t ret;
S
Steven Rostedt 已提交
3812
	size_t rem;
3813 3814
	unsigned int i;

3815 3816 3817
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

3818
	/* copy the tracer to avoid using a global lock all around */
3819
	mutex_lock(&trace_types_lock);
3820 3821
	if (unlikely(iter->trace->name != tr->current_trace->name))
		*iter->trace = *tr->current_trace;
3822 3823 3824
	mutex_unlock(&trace_types_lock);

	mutex_lock(&iter->mutex);
3825 3826 3827 3828 3829

	if (iter->trace->splice_read) {
		ret = iter->trace->splice_read(iter, filp,
					       ppos, pipe, len, flags);
		if (ret)
S
Steven Rostedt 已提交
3830
			goto out_err;
3831 3832 3833 3834
	}

	ret = tracing_wait_pipe(filp);
	if (ret <= 0)
S
Steven Rostedt 已提交
3835
		goto out_err;
3836

3837
	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3838
		ret = -EFAULT;
S
Steven Rostedt 已提交
3839
		goto out_err;
3840 3841
	}

3842
	trace_event_read_lock();
3843
	trace_access_lock(iter->cpu_file);
3844

3845
	/* Fill as many pages as possible. */
3846 3847 3848
	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
		spd.pages[i] = alloc_page(GFP_KERNEL);
		if (!spd.pages[i])
S
Steven Rostedt 已提交
3849
			break;
3850

3851
		rem = tracing_fill_pipe_page(rem, iter);
3852 3853 3854

		/* Copy the data into the page, so we can start over. */
		ret = trace_seq_to_buffer(&iter->seq,
3855
					  page_address(spd.pages[i]),
3856 3857
					  iter->seq.len);
		if (ret < 0) {
3858
			__free_page(spd.pages[i]);
3859 3860
			break;
		}
3861 3862
		spd.partial[i].offset = 0;
		spd.partial[i].len = iter->seq.len;
3863

3864
		trace_seq_init(&iter->seq);
3865 3866
	}

3867
	trace_access_unlock(iter->cpu_file);
3868
	trace_event_read_unlock();
3869
	mutex_unlock(&iter->mutex);
3870 3871 3872

	spd.nr_pages = i;

3873 3874
	ret = splice_to_pipe(pipe, &spd);
out:
3875
	splice_shrink_spd(&spd);
3876
	return ret;
3877

S
Steven Rostedt 已提交
3878
out_err:
3879
	mutex_unlock(&iter->mutex);
3880
	goto out;
3881 3882
}

3883 3884 3885 3886
static ssize_t
tracing_entries_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
3887 3888
	struct trace_cpu *tc = filp->private_data;
	struct trace_array *tr = tc->tr;
3889 3890 3891
	char buf[64];
	int r = 0;
	ssize_t ret;
3892

3893
	mutex_lock(&trace_types_lock);
3894

3895
	if (tc->cpu == RING_BUFFER_ALL_CPUS) {
3896 3897 3898 3899 3900 3901 3902 3903 3904
		int cpu, buf_size_same;
		unsigned long size;

		size = 0;
		buf_size_same = 1;
		/* check if all cpu sizes are same */
		for_each_tracing_cpu(cpu) {
			/* fill in the size from first enabled cpu */
			if (size == 0)
3905 3906
				size = per_cpu_ptr(tr->data, cpu)->entries;
			if (size != per_cpu_ptr(tr->data, cpu)->entries) {
3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921
				buf_size_same = 0;
				break;
			}
		}

		if (buf_size_same) {
			if (!ring_buffer_expanded)
				r = sprintf(buf, "%lu (expanded: %lu)\n",
					    size >> 10,
					    trace_buf_size >> 10);
			else
				r = sprintf(buf, "%lu\n", size >> 10);
		} else
			r = sprintf(buf, "X\n");
	} else
3922
		r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->data, tc->cpu)->entries >> 10);
3923

3924 3925
	mutex_unlock(&trace_types_lock);

3926 3927
	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	return ret;
3928 3929 3930 3931 3932 3933
}

static ssize_t
tracing_entries_write(struct file *filp, const char __user *ubuf,
		      size_t cnt, loff_t *ppos)
{
3934
	struct trace_cpu *tc = filp->private_data;
3935
	unsigned long val;
3936
	int ret;
3937

3938 3939
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
3940
		return ret;
3941 3942 3943 3944 3945

	/* must have at least 1 entry */
	if (!val)
		return -EINVAL;

3946 3947 3948
	/* value is in KB */
	val <<= 10;

3949
	ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
3950 3951
	if (ret < 0)
		return ret;
3952

3953
	*ppos += cnt;
3954

3955 3956
	return cnt;
}
S
Steven Rostedt 已提交
3957

3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968
static ssize_t
tracing_total_entries_read(struct file *filp, char __user *ubuf,
				size_t cnt, loff_t *ppos)
{
	struct trace_array *tr = filp->private_data;
	char buf[64];
	int r, cpu;
	unsigned long size = 0, expanded_size = 0;

	mutex_lock(&trace_types_lock);
	for_each_tracing_cpu(cpu) {
3969
		size += per_cpu_ptr(tr->data, cpu)->entries >> 10;
3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981
		if (!ring_buffer_expanded)
			expanded_size += trace_buf_size >> 10;
	}
	if (ring_buffer_expanded)
		r = sprintf(buf, "%lu\n", size);
	else
		r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
	mutex_unlock(&trace_types_lock);

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

3982 3983 3984 3985 3986 3987 3988 3989 3990 3991
static ssize_t
tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
			  size_t cnt, loff_t *ppos)
{
	/*
	 * There is no need to read what the user has written, this function
	 * is just to make sure that there is no error when "echo" is used
	 */

	*ppos += cnt;
3992 3993 3994 3995

	return cnt;
}

3996 3997 3998
static int
tracing_free_buffer_release(struct inode *inode, struct file *filp)
{
3999 4000
	struct trace_array *tr = inode->i_private;

4001 4002 4003
	/* disable tracing ? */
	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
		tracing_off();
4004
	/* resize the ring buffer to 0 */
4005
	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4006 4007 4008 4009

	return 0;
}

4010 4011 4012 4013
static ssize_t
tracing_mark_write(struct file *filp, const char __user *ubuf,
					size_t cnt, loff_t *fpos)
{
4014 4015 4016 4017 4018 4019
	unsigned long addr = (unsigned long)ubuf;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	struct print_entry *entry;
	unsigned long irq_flags;
	struct page *pages[2];
4020
	void *map_page[2];
4021 4022 4023 4024 4025 4026
	int nr_pages = 1;
	ssize_t written;
	int offset;
	int size;
	int len;
	int ret;
4027
	int i;
4028

S
Steven Rostedt 已提交
4029
	if (tracing_disabled)
4030 4031
		return -EINVAL;

4032 4033 4034
	if (!(trace_flags & TRACE_ITER_MARKERS))
		return -EINVAL;

4035 4036 4037
	if (cnt > TRACE_BUF_SIZE)
		cnt = TRACE_BUF_SIZE;

4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052
	/*
	 * Userspace is injecting traces into the kernel trace buffer.
	 * We want to be as non intrusive as possible.
	 * To do so, we do not want to allocate any special buffers
	 * or take any locks, but instead write the userspace data
	 * straight into the ring buffer.
	 *
	 * First we need to pin the userspace buffer into memory,
	 * which, most likely it is, because it just referenced it.
	 * But there's no guarantee that it is. By using get_user_pages_fast()
	 * and kmap_atomic/kunmap_atomic() we can get access to the
	 * pages directly. We then write the data directly into the
	 * ring buffer.
	 */
	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4053

4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066
	/* check if we cross pages */
	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
		nr_pages = 2;

	offset = addr & (PAGE_SIZE - 1);
	addr &= PAGE_MASK;

	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
	if (ret < nr_pages) {
		while (--ret >= 0)
			put_page(pages[ret]);
		written = -EFAULT;
		goto out;
4067
	}
4068

4069 4070
	for (i = 0; i < nr_pages; i++)
		map_page[i] = kmap_atomic(pages[i]);
4071 4072 4073 4074 4075 4076 4077 4078 4079 4080

	local_save_flags(irq_flags);
	size = sizeof(*entry) + cnt + 2; /* possible \n added */
	buffer = global_trace.buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
					  irq_flags, preempt_count());
	if (!event) {
		/* Ring buffer disabled, return as if not open for write */
		written = -EBADF;
		goto out_unlock;
4081
	}
4082 4083 4084 4085 4086 4087

	entry = ring_buffer_event_data(event);
	entry->ip = _THIS_IP_;

	if (nr_pages == 2) {
		len = PAGE_SIZE - offset;
4088 4089
		memcpy(&entry->buf, map_page[0] + offset, len);
		memcpy(&entry->buf[len], map_page[1], cnt - len);
C
Carsten Emde 已提交
4090
	} else
4091
		memcpy(&entry->buf, map_page[0] + offset, cnt);
4092

4093 4094 4095 4096 4097 4098
	if (entry->buf[cnt - 1] != '\n') {
		entry->buf[cnt] = '\n';
		entry->buf[cnt + 1] = '\0';
	} else
		entry->buf[cnt] = '\0';

4099
	__buffer_unlock_commit(buffer, event);
4100

4101
	written = cnt;
4102

4103
	*fpos += written;
4104

4105
 out_unlock:
4106 4107 4108 4109
	for (i = 0; i < nr_pages; i++){
		kunmap_atomic(map_page[i]);
		put_page(pages[i]);
	}
4110
 out:
4111
	return written;
4112 4113
}

L
Li Zefan 已提交
4114
static int tracing_clock_show(struct seq_file *m, void *v)
4115
{
4116
	struct trace_array *tr = m->private;
4117 4118 4119
	int i;

	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
L
Li Zefan 已提交
4120
		seq_printf(m,
4121
			"%s%s%s%s", i ? " " : "",
4122 4123
			i == tr->clock_id ? "[" : "", trace_clocks[i].name,
			i == tr->clock_id ? "]" : "");
L
Li Zefan 已提交
4124
	seq_putc(m, '\n');
4125

L
Li Zefan 已提交
4126
	return 0;
4127 4128 4129 4130 4131
}

static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
				   size_t cnt, loff_t *fpos)
{
4132 4133
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156
	char buf[64];
	const char *clockstr;
	int i;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	clockstr = strstrip(buf);

	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
		if (strcmp(trace_clocks[i].name, clockstr) == 0)
			break;
	}
	if (i == ARRAY_SIZE(trace_clocks))
		return -EINVAL;

	mutex_lock(&trace_types_lock);

4157 4158 4159 4160
	tr->clock_id = i;

	ring_buffer_set_clock(tr->buffer, trace_clocks[i].func);
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL && max_tr.buffer)
4161 4162
		ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);

4163 4164 4165 4166 4167
	/*
	 * New clock may not be consistent with the previous clock.
	 * Reset the buffer so that it doesn't have incomparable timestamps.
	 */
	tracing_reset_online_cpus(&global_trace);
4168
	tracing_reset_online_cpus(&max_tr);
4169

4170 4171 4172 4173 4174 4175 4176
	mutex_unlock(&trace_types_lock);

	*fpos += cnt;

	return cnt;
}

L
Li Zefan 已提交
4177 4178 4179 4180
static int tracing_clock_open(struct inode *inode, struct file *file)
{
	if (tracing_disabled)
		return -ENODEV;
4181 4182

	return single_open(file, tracing_clock_show, inode->i_private);
L
Li Zefan 已提交
4183 4184
}

4185 4186 4187
#ifdef CONFIG_TRACER_SNAPSHOT
static int tracing_snapshot_open(struct inode *inode, struct file *file)
{
4188
	struct trace_cpu *tc = inode->i_private;
4189
	struct trace_iterator *iter;
4190
	struct seq_file *m;
4191 4192 4193 4194 4195 4196
	int ret = 0;

	if (file->f_mode & FMODE_READ) {
		iter = __tracing_open(inode, file, true);
		if (IS_ERR(iter))
			ret = PTR_ERR(iter);
4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209
	} else {
		/* Writes still need the seq_file to hold the private data */
		m = kzalloc(sizeof(*m), GFP_KERNEL);
		if (!m)
			return -ENOMEM;
		iter = kzalloc(sizeof(*iter), GFP_KERNEL);
		if (!iter) {
			kfree(m);
			return -ENOMEM;
		}
		iter->tr = tc->tr;
		m->private = iter;
		file->private_data = m;
4210
	}
4211

4212 4213 4214 4215 4216 4217 4218
	return ret;
}

static ssize_t
tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
		       loff_t *ppos)
{
4219 4220 4221
	struct seq_file *m = filp->private_data;
	struct trace_iterator *iter = m->private;
	struct trace_array *tr = iter->tr;
4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234
	unsigned long val;
	int ret;

	ret = tracing_update_buffers();
	if (ret < 0)
		return ret;

	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
		return ret;

	mutex_lock(&trace_types_lock);

4235
	if (tr->current_trace->use_max_tr) {
4236 4237 4238 4239 4240 4241
		ret = -EBUSY;
		goto out;
	}

	switch (val) {
	case 0:
4242
		if (tr->current_trace->allocated_snapshot) {
4243 4244 4245 4246 4247
			/* free spare buffer */
			ring_buffer_resize(max_tr.buffer, 1,
					   RING_BUFFER_ALL_CPUS);
			set_buffer_entries(&max_tr, 1);
			tracing_reset_online_cpus(&max_tr);
4248
			tr->current_trace->allocated_snapshot = false;
4249 4250 4251
		}
		break;
	case 1:
4252
		if (!tr->current_trace->allocated_snapshot) {
4253 4254 4255 4256 4257
			/* allocate spare buffer */
			ret = resize_buffer_duplicate_size(&max_tr,
					&global_trace, RING_BUFFER_ALL_CPUS);
			if (ret < 0)
				break;
4258
			tr->current_trace->allocated_snapshot = true;
4259 4260 4261 4262 4263 4264 4265 4266
		}

		local_irq_disable();
		/* Now, we're going to swap */
		update_max_tr(&global_trace, current, smp_processor_id());
		local_irq_enable();
		break;
	default:
4267
		if (tr->current_trace->allocated_snapshot)
4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279
			tracing_reset_online_cpus(&max_tr);
		break;
	}

	if (ret >= 0) {
		*ppos += cnt;
		ret = cnt;
	}
out:
	mutex_unlock(&trace_types_lock);
	return ret;
}
4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295

static int tracing_snapshot_release(struct inode *inode, struct file *file)
{
	struct seq_file *m = file->private_data;

	if (file->f_mode & FMODE_READ)
		return tracing_release(inode, file);

	/* If write only, the seq_file is just a stub */
	if (m)
		kfree(m->private);
	kfree(m);

	return 0;
}

4296 4297 4298
#endif /* CONFIG_TRACER_SNAPSHOT */


4299
static const struct file_operations tracing_max_lat_fops = {
I
Ingo Molnar 已提交
4300 4301 4302
	.open		= tracing_open_generic,
	.read		= tracing_max_lat_read,
	.write		= tracing_max_lat_write,
4303
	.llseek		= generic_file_llseek,
4304 4305
};

4306
static const struct file_operations set_tracer_fops = {
I
Ingo Molnar 已提交
4307 4308 4309
	.open		= tracing_open_generic,
	.read		= tracing_set_trace_read,
	.write		= tracing_set_trace_write,
4310
	.llseek		= generic_file_llseek,
4311 4312
};

4313
static const struct file_operations tracing_pipe_fops = {
I
Ingo Molnar 已提交
4314
	.open		= tracing_open_pipe,
4315
	.poll		= tracing_poll_pipe,
I
Ingo Molnar 已提交
4316
	.read		= tracing_read_pipe,
4317
	.splice_read	= tracing_splice_read_pipe,
I
Ingo Molnar 已提交
4318
	.release	= tracing_release_pipe,
4319
	.llseek		= no_llseek,
4320 4321
};

4322
static const struct file_operations tracing_entries_fops = {
4323
	.open		= tracing_open_generic,
4324 4325
	.read		= tracing_entries_read,
	.write		= tracing_entries_write,
4326
	.llseek		= generic_file_llseek,
4327 4328
};

4329 4330 4331 4332 4333 4334
static const struct file_operations tracing_total_entries_fops = {
	.open		= tracing_open_generic,
	.read		= tracing_total_entries_read,
	.llseek		= generic_file_llseek,
};

4335 4336 4337 4338 4339
static const struct file_operations tracing_free_buffer_fops = {
	.write		= tracing_free_buffer_write,
	.release	= tracing_free_buffer_release,
};

4340
static const struct file_operations tracing_mark_fops = {
4341
	.open		= tracing_open_generic,
4342
	.write		= tracing_mark_write,
4343
	.llseek		= generic_file_llseek,
4344 4345
};

4346
static const struct file_operations trace_clock_fops = {
L
Li Zefan 已提交
4347 4348 4349 4350
	.open		= tracing_clock_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
4351 4352 4353
	.write		= tracing_clock_write,
};

4354 4355 4356 4357 4358 4359
#ifdef CONFIG_TRACER_SNAPSHOT
static const struct file_operations snapshot_fops = {
	.open		= tracing_snapshot_open,
	.read		= seq_read,
	.write		= tracing_snapshot_write,
	.llseek		= tracing_seek,
4360
	.release	= tracing_snapshot_release,
4361 4362 4363
};
#endif /* CONFIG_TRACER_SNAPSHOT */

4364 4365 4366 4367 4368 4369 4370 4371 4372
struct ftrace_buffer_info {
	struct trace_array	*tr;
	void			*spare;
	int			cpu;
	unsigned int		read;
};

static int tracing_buffers_open(struct inode *inode, struct file *filp)
{
4373 4374
	struct trace_cpu *tc = inode->i_private;
	struct trace_array *tr = tc->tr;
4375 4376 4377 4378 4379 4380 4381 4382 4383
	struct ftrace_buffer_info *info;

	if (tracing_disabled)
		return -ENODEV;

	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		return -ENOMEM;

4384 4385
	info->tr	= tr;
	info->cpu	= tc->cpu;
4386
	info->spare	= NULL;
4387 4388 4389 4390 4391
	/* Force reading ring buffer for first read */
	info->read	= (unsigned int)-1;

	filp->private_data = info;

4392
	return nonseekable_open(inode, filp);
4393 4394 4395 4396 4397 4398 4399 4400 4401 4402
}

static ssize_t
tracing_buffers_read(struct file *filp, char __user *ubuf,
		     size_t count, loff_t *ppos)
{
	struct ftrace_buffer_info *info = filp->private_data;
	ssize_t ret;
	size_t size;

4403 4404 4405
	if (!count)
		return 0;

4406
	if (!info->spare)
4407
		info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4408 4409 4410
	if (!info->spare)
		return -ENOMEM;

4411 4412 4413 4414
	/* Do we have previous read data to read? */
	if (info->read < PAGE_SIZE)
		goto read;

4415
	trace_access_lock(info->cpu);
4416 4417 4418 4419
	ret = ring_buffer_read_page(info->tr->buffer,
				    &info->spare,
				    count,
				    info->cpu, 0);
4420
	trace_access_unlock(info->cpu);
4421 4422 4423
	if (ret < 0)
		return 0;

4424 4425
	info->read = 0;

4426 4427 4428 4429 4430 4431
read:
	size = PAGE_SIZE - info->read;
	if (size > count)
		size = count;

	ret = copy_to_user(ubuf, info->spare + info->read, size);
4432
	if (ret == size)
4433
		return -EFAULT;
4434 4435
	size -= ret;

4436 4437 4438 4439 4440 4441 4442 4443 4444 4445
	*ppos += size;
	info->read += size;

	return size;
}

static int tracing_buffers_release(struct inode *inode, struct file *file)
{
	struct ftrace_buffer_info *info = file->private_data;

4446 4447
	if (info->spare)
		ring_buffer_free_read_page(info->tr->buffer, info->spare);
4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480
	kfree(info);

	return 0;
}

struct buffer_ref {
	struct ring_buffer	*buffer;
	void			*page;
	int			ref;
};

static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	struct buffer_ref *ref = (struct buffer_ref *)buf->private;

	if (--ref->ref)
		return;

	ring_buffer_free_read_page(ref->buffer, ref->page);
	kfree(ref);
	buf->private = 0;
}

static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
				struct pipe_buffer *buf)
{
	struct buffer_ref *ref = (struct buffer_ref *)buf->private;

	ref->ref++;
}

/* Pipe buffer operations for a buffer. */
4481
static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4482 4483 4484 4485 4486
	.can_merge		= 0,
	.map			= generic_pipe_buf_map,
	.unmap			= generic_pipe_buf_unmap,
	.confirm		= generic_pipe_buf_confirm,
	.release		= buffer_pipe_buf_release,
4487
	.steal			= generic_pipe_buf_steal,
4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513
	.get			= buffer_pipe_buf_get,
};

/*
 * Callback from splice_to_pipe(), if we need to release some pages
 * at the end of the spd in case we error'ed out in filling the pipe.
 */
static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
	struct buffer_ref *ref =
		(struct buffer_ref *)spd->partial[i].private;

	if (--ref->ref)
		return;

	ring_buffer_free_read_page(ref->buffer, ref->page);
	kfree(ref);
	spd->partial[i].private = 0;
}

static ssize_t
tracing_buffers_splice_read(struct file *file, loff_t *ppos,
			    struct pipe_inode_info *pipe, size_t len,
			    unsigned int flags)
{
	struct ftrace_buffer_info *info = file->private_data;
4514 4515
	struct partial_page partial_def[PIPE_DEF_BUFFERS];
	struct page *pages_def[PIPE_DEF_BUFFERS];
4516
	struct splice_pipe_desc spd = {
4517 4518
		.pages		= pages_def,
		.partial	= partial_def,
4519
		.nr_pages_max	= PIPE_DEF_BUFFERS,
4520 4521 4522 4523 4524
		.flags		= flags,
		.ops		= &buffer_pipe_buf_ops,
		.spd_release	= buffer_spd_release,
	};
	struct buffer_ref *ref;
4525
	int entries, size, i;
4526 4527
	size_t ret;

4528 4529 4530
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

4531
	if (*ppos & (PAGE_SIZE - 1)) {
4532 4533
		ret = -EINVAL;
		goto out;
4534 4535 4536
	}

	if (len & (PAGE_SIZE - 1)) {
4537 4538 4539 4540
		if (len < PAGE_SIZE) {
			ret = -EINVAL;
			goto out;
		}
4541 4542 4543
		len &= PAGE_MASK;
	}

4544
	trace_access_lock(info->cpu);
4545 4546
	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);

4547
	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4548 4549 4550 4551 4552 4553 4554
		struct page *page;
		int r;

		ref = kzalloc(sizeof(*ref), GFP_KERNEL);
		if (!ref)
			break;

4555
		ref->ref = 1;
4556
		ref->buffer = info->tr->buffer;
4557
		ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4558 4559 4560 4561 4562 4563
		if (!ref->page) {
			kfree(ref);
			break;
		}

		r = ring_buffer_read_page(ref->buffer, &ref->page,
4564
					  len, info->cpu, 1);
4565
		if (r < 0) {
4566
			ring_buffer_free_read_page(ref->buffer, ref->page);
4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585
			kfree(ref);
			break;
		}

		/*
		 * zero out any left over data, this is going to
		 * user land.
		 */
		size = ring_buffer_page_len(ref->page);
		if (size < PAGE_SIZE)
			memset(ref->page + size, 0, PAGE_SIZE - size);

		page = virt_to_page(ref->page);

		spd.pages[i] = page;
		spd.partial[i].len = PAGE_SIZE;
		spd.partial[i].offset = 0;
		spd.partial[i].private = (unsigned long)ref;
		spd.nr_pages++;
4586
		*ppos += PAGE_SIZE;
4587 4588

		entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4589 4590
	}

4591
	trace_access_unlock(info->cpu);
4592 4593 4594 4595
	spd.nr_pages = i;

	/* did we read anything? */
	if (!spd.nr_pages) {
4596
		if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
4597 4598 4599 4600
			ret = -EAGAIN;
		else
			ret = 0;
		/* TODO: block */
4601
		goto out;
4602 4603 4604
	}

	ret = splice_to_pipe(pipe, &spd);
4605
	splice_shrink_spd(&spd);
4606
out:
4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617
	return ret;
}

static const struct file_operations tracing_buffers_fops = {
	.open		= tracing_buffers_open,
	.read		= tracing_buffers_read,
	.release	= tracing_buffers_release,
	.splice_read	= tracing_buffers_splice_read,
	.llseek		= no_llseek,
};

4618 4619 4620 4621
static ssize_t
tracing_stats_read(struct file *filp, char __user *ubuf,
		   size_t count, loff_t *ppos)
{
4622 4623
	struct trace_cpu *tc = filp->private_data;
	struct trace_array *tr = tc->tr;
4624 4625
	struct trace_seq *s;
	unsigned long cnt;
4626 4627
	unsigned long long t;
	unsigned long usec_rem;
4628
	int cpu = tc->cpu;
4629

4630
	s = kmalloc(sizeof(*s), GFP_KERNEL);
4631
	if (!s)
4632
		return -ENOMEM;
4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644

	trace_seq_init(s);

	cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "entries: %ld\n", cnt);

	cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "overrun: %ld\n", cnt);

	cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "commit overrun: %ld\n", cnt);

4645 4646 4647
	cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "bytes: %ld\n", cnt);

4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661
	if (trace_clocks[trace_clock_id].in_ns) {
		/* local or global for trace_clock */
		t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
		usec_rem = do_div(t, USEC_PER_SEC);
		trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
								t, usec_rem);

		t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
		usec_rem = do_div(t, USEC_PER_SEC);
		trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
	} else {
		/* counter or tsc mode for trace_clock */
		trace_seq_printf(s, "oldest event ts: %llu\n",
				ring_buffer_oldest_event_ts(tr->buffer, cpu));
4662

4663 4664 4665
		trace_seq_printf(s, "now ts: %llu\n",
				ring_buffer_time_stamp(tr->buffer, cpu));
	}
4666

4667 4668 4669
	cnt = ring_buffer_dropped_events_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "dropped events: %ld\n", cnt);

4670 4671 4672
	cnt = ring_buffer_read_events_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "read events: %ld\n", cnt);

4673 4674 4675 4676 4677 4678 4679 4680 4681 4682
	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);

	kfree(s);

	return count;
}

static const struct file_operations tracing_stats_fops = {
	.open		= tracing_open_generic,
	.read		= tracing_stats_read,
4683
	.llseek		= generic_file_llseek,
4684 4685
};

4686 4687
#ifdef CONFIG_DYNAMIC_FTRACE

S
Steven Rostedt 已提交
4688 4689 4690 4691 4692
int __weak ftrace_arch_read_dyn_info(char *buf, int size)
{
	return 0;
}

4693
static ssize_t
S
Steven Rostedt 已提交
4694
tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4695 4696
		  size_t cnt, loff_t *ppos)
{
S
Steven Rostedt 已提交
4697 4698
	static char ftrace_dyn_info_buffer[1024];
	static DEFINE_MUTEX(dyn_info_mutex);
4699
	unsigned long *p = filp->private_data;
S
Steven Rostedt 已提交
4700
	char *buf = ftrace_dyn_info_buffer;
S
Steven Rostedt 已提交
4701
	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4702 4703
	int r;

S
Steven Rostedt 已提交
4704 4705
	mutex_lock(&dyn_info_mutex);
	r = sprintf(buf, "%ld ", *p);
I
Ingo Molnar 已提交
4706

S
Steven Rostedt 已提交
4707
	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
S
Steven Rostedt 已提交
4708 4709 4710 4711 4712 4713 4714
	buf[r++] = '\n';

	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);

	mutex_unlock(&dyn_info_mutex);

	return r;
4715 4716
}

4717
static const struct file_operations tracing_dyn_info_fops = {
I
Ingo Molnar 已提交
4718
	.open		= tracing_open_generic,
S
Steven Rostedt 已提交
4719
	.read		= tracing_read_dyn_info,
4720
	.llseek		= generic_file_llseek,
4721 4722 4723
};
#endif

4724
struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
4725 4726 4727
{
	static int once;

4728 4729
	if (tr->dir)
		return tr->dir;
4730

4731 4732 4733
	if (!debugfs_initialized())
		return NULL;

4734 4735
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
		tr->dir = debugfs_create_dir("tracing", NULL);
4736

4737
	if (!tr->dir && !once) {
4738 4739 4740 4741 4742
		once = 1;
		pr_warning("Could not create debugfs directory 'tracing'\n");
		return NULL;
	}

4743
	return tr->dir;
4744 4745
}

4746 4747 4748 4749
struct dentry *tracing_init_dentry(void)
{
	return tracing_init_dentry_tr(&global_trace);
}
4750

4751
static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
4752 4753 4754
{
	struct dentry *d_tracer;

4755 4756
	if (tr->percpu_dir)
		return tr->percpu_dir;
4757

4758
	d_tracer = tracing_init_dentry_tr(tr);
4759 4760 4761
	if (!d_tracer)
		return NULL;

4762
	tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
4763

4764 4765
	WARN_ONCE(!tr->percpu_dir,
		  "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
4766

4767
	return tr->percpu_dir;
4768 4769
}

4770 4771
static void
tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
4772
{
4773
	struct trace_array_cpu *data = per_cpu_ptr(tr->data, cpu);
4774
	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
4775
	struct dentry *d_cpu;
4776
	char cpu_dir[30]; /* 30 characters should be more than enough */
4777

4778 4779 4780
	if (!d_percpu)
		return;

4781
	snprintf(cpu_dir, 30, "cpu%ld", cpu);
4782 4783 4784 4785 4786
	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
	if (!d_cpu) {
		pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
		return;
	}
4787

4788
	/* per cpu trace_pipe */
4789
	trace_create_file("trace_pipe", 0444, d_cpu,
4790
			(void *)&data->trace_cpu, &tracing_pipe_fops);
4791 4792

	/* per cpu trace */
4793
	trace_create_file("trace", 0644, d_cpu,
4794
			(void *)&data->trace_cpu, &tracing_fops);
4795

4796
	trace_create_file("trace_pipe_raw", 0444, d_cpu,
4797
			(void *)&data->trace_cpu, &tracing_buffers_fops);
4798

4799
	trace_create_file("stats", 0444, d_cpu,
4800
			(void *)&data->trace_cpu, &tracing_stats_fops);
4801 4802

	trace_create_file("buffer_size_kb", 0444, d_cpu,
4803
			(void *)&data->trace_cpu, &tracing_entries_fops);
4804 4805
}

S
Steven Rostedt 已提交
4806 4807 4808 4809 4810
#ifdef CONFIG_FTRACE_SELFTEST
/* Let selftest have access to static functions in this file */
#include "trace_selftest.c"
#endif

4811 4812 4813
struct trace_option_dentry {
	struct tracer_opt		*opt;
	struct tracer_flags		*flags;
4814
	struct trace_array		*tr;
4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840
	struct dentry			*entry;
};

static ssize_t
trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
			loff_t *ppos)
{
	struct trace_option_dentry *topt = filp->private_data;
	char *buf;

	if (topt->flags->val & topt->opt->bit)
		buf = "1\n";
	else
		buf = "0\n";

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
}

static ssize_t
trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
			 loff_t *ppos)
{
	struct trace_option_dentry *topt = filp->private_data;
	unsigned long val;
	int ret;

4841 4842
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
4843 4844
		return ret;

L
Li Zefan 已提交
4845 4846
	if (val != 0 && val != 1)
		return -EINVAL;
4847

L
Li Zefan 已提交
4848
	if (!!(topt->flags->val & topt->opt->bit) != val) {
4849
		mutex_lock(&trace_types_lock);
4850
		ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
4851
					  topt->opt, !val);
4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866
		mutex_unlock(&trace_types_lock);
		if (ret)
			return ret;
	}

	*ppos += cnt;

	return cnt;
}


static const struct file_operations trace_options_fops = {
	.open = tracing_open_generic,
	.read = trace_options_read,
	.write = trace_options_write,
4867
	.llseek	= generic_file_llseek,
4868 4869
};

4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888
static ssize_t
trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
			loff_t *ppos)
{
	long index = (long)filp->private_data;
	char *buf;

	if (trace_flags & (1 << index))
		buf = "1\n";
	else
		buf = "0\n";

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
}

static ssize_t
trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
			 loff_t *ppos)
{
4889
	struct trace_array *tr = &global_trace;
4890 4891 4892 4893
	long index = (long)filp->private_data;
	unsigned long val;
	int ret;

4894 4895
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
4896 4897
		return ret;

4898
	if (val != 0 && val != 1)
4899
		return -EINVAL;
4900 4901

	mutex_lock(&trace_types_lock);
4902
	ret = set_tracer_flag(tr, 1 << index, val);
4903
	mutex_unlock(&trace_types_lock);
4904

4905 4906 4907
	if (ret < 0)
		return ret;

4908 4909 4910 4911 4912 4913 4914 4915 4916
	*ppos += cnt;

	return cnt;
}

static const struct file_operations trace_options_core_fops = {
	.open = tracing_open_generic,
	.read = trace_options_core_read,
	.write = trace_options_core_write,
4917
	.llseek = generic_file_llseek,
4918 4919
};

4920
struct dentry *trace_create_file(const char *name,
A
Al Viro 已提交
4921
				 umode_t mode,
4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935
				 struct dentry *parent,
				 void *data,
				 const struct file_operations *fops)
{
	struct dentry *ret;

	ret = debugfs_create_file(name, mode, parent, data, fops);
	if (!ret)
		pr_warning("Could not create debugfs '%s' entry\n", name);

	return ret;
}


4936
static struct dentry *trace_options_init_dentry(struct trace_array *tr)
4937 4938 4939
{
	struct dentry *d_tracer;

4940 4941
	if (tr->options)
		return tr->options;
4942

4943
	d_tracer = tracing_init_dentry_tr(tr);
4944 4945 4946
	if (!d_tracer)
		return NULL;

4947 4948
	tr->options = debugfs_create_dir("options", d_tracer);
	if (!tr->options) {
4949 4950 4951 4952
		pr_warning("Could not create debugfs directory 'options'\n");
		return NULL;
	}

4953
	return tr->options;
4954 4955
}

4956
static void
4957 4958
create_trace_option_file(struct trace_array *tr,
			 struct trace_option_dentry *topt,
4959 4960 4961 4962 4963
			 struct tracer_flags *flags,
			 struct tracer_opt *opt)
{
	struct dentry *t_options;

4964
	t_options = trace_options_init_dentry(tr);
4965 4966 4967 4968 4969
	if (!t_options)
		return;

	topt->flags = flags;
	topt->opt = opt;
4970
	topt->tr = tr;
4971

4972
	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4973 4974 4975 4976 4977
				    &trace_options_fops);

}

static struct trace_option_dentry *
4978
create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997
{
	struct trace_option_dentry *topts;
	struct tracer_flags *flags;
	struct tracer_opt *opts;
	int cnt;

	if (!tracer)
		return NULL;

	flags = tracer->flags;

	if (!flags || !flags->opts)
		return NULL;

	opts = flags->opts;

	for (cnt = 0; opts[cnt].name; cnt++)
		;

4998
	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4999 5000 5001 5002
	if (!topts)
		return NULL;

	for (cnt = 0; opts[cnt].name; cnt++)
5003
		create_trace_option_file(tr, &topts[cnt], flags,
5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024
					 &opts[cnt]);

	return topts;
}

static void
destroy_trace_option_files(struct trace_option_dentry *topts)
{
	int cnt;

	if (!topts)
		return;

	for (cnt = 0; topts[cnt].opt; cnt++) {
		if (topts[cnt].entry)
			debugfs_remove(topts[cnt].entry);
	}

	kfree(topts);
}

5025
static struct dentry *
5026 5027
create_trace_option_core_file(struct trace_array *tr,
			      const char *option, long index)
5028 5029 5030
{
	struct dentry *t_options;

5031
	t_options = trace_options_init_dentry(tr);
5032 5033 5034
	if (!t_options)
		return NULL;

5035
	return trace_create_file(option, 0644, t_options, (void *)index,
5036 5037 5038
				    &trace_options_core_fops);
}

5039
static __init void create_trace_options_dir(struct trace_array *tr)
5040 5041 5042 5043
{
	struct dentry *t_options;
	int i;

5044
	t_options = trace_options_init_dentry(tr);
5045 5046 5047
	if (!t_options)
		return;

5048
	for (i = 0; trace_options[i]; i++)
5049
		create_trace_option_core_file(tr, trace_options[i], i);
5050 5051
}

5052 5053 5054 5055
static ssize_t
rb_simple_read(struct file *filp, char __user *ubuf,
	       size_t cnt, loff_t *ppos)
{
5056 5057
	struct trace_array *tr = filp->private_data;
	struct ring_buffer *buffer = tr->buffer;
5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074
	char buf[64];
	int r;

	if (buffer)
		r = ring_buffer_record_is_on(buffer);
	else
		r = 0;

	r = sprintf(buf, "%d\n", r);

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

static ssize_t
rb_simple_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
5075 5076
	struct trace_array *tr = filp->private_data;
	struct ring_buffer *buffer = tr->buffer;
5077 5078 5079 5080 5081 5082 5083 5084
	unsigned long val;
	int ret;

	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
		return ret;

	if (buffer) {
5085 5086
		mutex_lock(&trace_types_lock);
		if (val) {
5087
			ring_buffer_record_on(buffer);
5088 5089
			if (tr->current_trace->start)
				tr->current_trace->start(tr);
5090
		} else {
5091
			ring_buffer_record_off(buffer);
5092 5093
			if (tr->current_trace->stop)
				tr->current_trace->stop(tr);
5094 5095
		}
		mutex_unlock(&trace_types_lock);
5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109
	}

	(*ppos)++;

	return cnt;
}

static const struct file_operations rb_simple_fops = {
	.open		= tracing_open_generic,
	.read		= rb_simple_read,
	.write		= rb_simple_write,
	.llseek		= default_llseek,
};

5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194
struct dentry *trace_instance_dir;

static void
init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);

static int new_instance_create(const char *name)
{
	enum ring_buffer_flags rb_flags;
	struct trace_array *tr;
	int ret;
	int i;

	mutex_lock(&trace_types_lock);

	ret = -EEXIST;
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (tr->name && strcmp(tr->name, name) == 0)
			goto out_unlock;
	}

	ret = -ENOMEM;
	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
	if (!tr)
		goto out_unlock;

	tr->name = kstrdup(name, GFP_KERNEL);
	if (!tr->name)
		goto out_free_tr;

	raw_spin_lock_init(&tr->start_lock);

	tr->current_trace = &nop_trace;

	INIT_LIST_HEAD(&tr->systems);
	INIT_LIST_HEAD(&tr->events);

	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;

	tr->buffer = ring_buffer_alloc(trace_buf_size, rb_flags);
	if (!tr->buffer)
		goto out_free_tr;

	tr->data = alloc_percpu(struct trace_array_cpu);
	if (!tr->data)
		goto out_free_tr;

	for_each_tracing_cpu(i) {
		memset(per_cpu_ptr(tr->data, i), 0, sizeof(struct trace_array_cpu));
		per_cpu_ptr(tr->data, i)->trace_cpu.cpu = i;
		per_cpu_ptr(tr->data, i)->trace_cpu.tr = tr;
	}

	/* Holder for file callbacks */
	tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
	tr->trace_cpu.tr = tr;

	tr->dir = debugfs_create_dir(name, trace_instance_dir);
	if (!tr->dir)
		goto out_free_tr;

	ret = event_trace_add_tracer(tr->dir, tr);
	if (ret)
		goto out_free_tr;

	init_tracer_debugfs(tr, tr->dir);

	list_add(&tr->list, &ftrace_trace_arrays);

	mutex_unlock(&trace_types_lock);

	return 0;

 out_free_tr:
	if (tr->buffer)
		ring_buffer_free(tr->buffer);
	kfree(tr->name);
	kfree(tr);

 out_unlock:
	mutex_unlock(&trace_types_lock);

	return ret;

}

5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230
static int instance_delete(const char *name)
{
	struct trace_array *tr;
	int found = 0;
	int ret;

	mutex_lock(&trace_types_lock);

	ret = -ENODEV;
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (tr->name && strcmp(tr->name, name) == 0) {
			found = 1;
			break;
		}
	}
	if (!found)
		goto out_unlock;

	list_del(&tr->list);

	event_trace_del_tracer(tr);
	debugfs_remove_recursive(tr->dir);
	free_percpu(tr->data);
	ring_buffer_free(tr->buffer);

	kfree(tr->name);
	kfree(tr);

	ret = 0;

 out_unlock:
	mutex_unlock(&trace_types_lock);

	return ret;
}

5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257
static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
{
	struct dentry *parent;
	int ret;

	/* Paranoid: Make sure the parent is the "instances" directory */
	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
	if (WARN_ON_ONCE(parent != trace_instance_dir))
		return -ENOENT;

	/*
	 * The inode mutex is locked, but debugfs_create_dir() will also
	 * take the mutex. As the instances directory can not be destroyed
	 * or changed in any other way, it is safe to unlock it, and
	 * let the dentry try. If two users try to make the same dir at
	 * the same time, then the new_instance_create() will determine the
	 * winner.
	 */
	mutex_unlock(&inode->i_mutex);

	ret = new_instance_create(dentry->d_iname);

	mutex_lock(&inode->i_mutex);

	return ret;
}

5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288
static int instance_rmdir(struct inode *inode, struct dentry *dentry)
{
	struct dentry *parent;
	int ret;

	/* Paranoid: Make sure the parent is the "instances" directory */
	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
	if (WARN_ON_ONCE(parent != trace_instance_dir))
		return -ENOENT;

	/* The caller did a dget() on dentry */
	mutex_unlock(&dentry->d_inode->i_mutex);

	/*
	 * The inode mutex is locked, but debugfs_create_dir() will also
	 * take the mutex. As the instances directory can not be destroyed
	 * or changed in any other way, it is safe to unlock it, and
	 * let the dentry try. If two users try to make the same dir at
	 * the same time, then the instance_delete() will determine the
	 * winner.
	 */
	mutex_unlock(&inode->i_mutex);

	ret = instance_delete(dentry->d_iname);

	mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
	mutex_lock(&dentry->d_inode->i_mutex);

	return ret;
}

5289 5290 5291
static const struct inode_operations instance_dir_inode_operations = {
	.lookup		= simple_lookup,
	.mkdir		= instance_mkdir,
5292
	.rmdir		= instance_rmdir,
5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304
};

static __init void create_trace_instances(struct dentry *d_tracer)
{
	trace_instance_dir = debugfs_create_dir("instances", d_tracer);
	if (WARN_ON(!trace_instance_dir))
		return;

	/* Hijack the dir inode operations, to allow mkdir */
	trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
}

5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336
static void
init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
{

	trace_create_file("trace_options", 0644, d_tracer,
			  tr, &tracing_iter_fops);

	trace_create_file("trace", 0644, d_tracer,
			(void *)&tr->trace_cpu, &tracing_fops);

	trace_create_file("trace_pipe", 0444, d_tracer,
			(void *)&tr->trace_cpu, &tracing_pipe_fops);

	trace_create_file("buffer_size_kb", 0644, d_tracer,
			(void *)&tr->trace_cpu, &tracing_entries_fops);

	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
			  tr, &tracing_total_entries_fops);

	trace_create_file("free_buffer", 0644, d_tracer,
			  tr, &tracing_free_buffer_fops);

	trace_create_file("trace_marker", 0220, d_tracer,
			  tr, &tracing_mark_fops);

	trace_create_file("trace_clock", 0644, d_tracer, tr,
			  &trace_clock_fops);

	trace_create_file("tracing_on", 0644, d_tracer,
			    tr, &rb_simple_fops);
}

5337
static __init int tracer_init_debugfs(void)
5338 5339
{
	struct dentry *d_tracer;
5340
	int cpu;
5341

5342 5343
	trace_access_lock_init();

5344 5345
	d_tracer = tracing_init_dentry();

5346
	init_tracer_debugfs(&global_trace, d_tracer);
5347

5348
	trace_create_file("tracing_cpumask", 0644, d_tracer,
5349
			&global_trace, &tracing_cpumask_fops);
5350

5351 5352 5353
	trace_create_file("available_tracers", 0444, d_tracer,
			&global_trace, &show_traces_fops);

5354
	trace_create_file("current_tracer", 0644, d_tracer,
5355 5356
			&global_trace, &set_tracer_fops);

5357
#ifdef CONFIG_TRACER_MAX_TRACE
5358 5359
	trace_create_file("tracing_max_latency", 0644, d_tracer,
			&tracing_max_latency, &tracing_max_lat_fops);
5360
#endif
5361 5362 5363

	trace_create_file("tracing_thresh", 0644, d_tracer,
			&tracing_thresh, &tracing_max_lat_fops);
5364

5365
	trace_create_file("README", 0444, d_tracer,
5366 5367
			NULL, &tracing_readme_fops);

5368 5369
	trace_create_file("saved_cmdlines", 0444, d_tracer,
			NULL, &tracing_saved_cmdlines_fops);
5370

5371
#ifdef CONFIG_DYNAMIC_FTRACE
5372 5373
	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
			&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
5374
#endif
5375

5376 5377
#ifdef CONFIG_TRACER_SNAPSHOT
	trace_create_file("snapshot", 0644, d_tracer,
5378
			  (void *)&global_trace.trace_cpu, &snapshot_fops);
5379 5380
#endif

5381 5382
	create_trace_instances(d_tracer);

5383
	create_trace_options_dir(&global_trace);
5384

5385
	for_each_tracing_cpu(cpu)
5386
		tracing_init_debugfs_percpu(&global_trace, cpu);
5387

5388
	return 0;
5389 5390
}

5391 5392 5393
static int trace_panic_handler(struct notifier_block *this,
			       unsigned long event, void *unused)
{
5394
	if (ftrace_dump_on_oops)
5395
		ftrace_dump(ftrace_dump_on_oops);
5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410
	return NOTIFY_OK;
}

static struct notifier_block trace_panic_notifier = {
	.notifier_call  = trace_panic_handler,
	.next           = NULL,
	.priority       = 150   /* priority: INT_MAX >= x >= 0 */
};

static int trace_die_handler(struct notifier_block *self,
			     unsigned long val,
			     void *data)
{
	switch (val) {
	case DIE_OOPS:
5411
		if (ftrace_dump_on_oops)
5412
			ftrace_dump(ftrace_dump_on_oops);
5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block trace_die_notifier = {
	.notifier_call = trace_die_handler,
	.priority = 200
};

/*
 * printk is set to max of 1024, we really don't need it that big.
 * Nothing should be printing 1000 characters anyway.
 */
#define TRACE_MAX_PRINT		1000

/*
 * Define here KERN_TRACE so that we have one place to modify
 * it if we decide to change what log level the ftrace dump
 * should be at.
 */
5436
#define KERN_TRACE		KERN_EMERG
5437

5438
void
5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449
trace_printk_seq(struct trace_seq *s)
{
	/* Probably should print a warning here. */
	if (s->len >= 1000)
		s->len = 1000;

	/* should be zero ended, but we are paranoid. */
	s->buffer[s->len] = 0;

	printk(KERN_TRACE "%s", s->buffer);

5450
	trace_seq_init(s);
5451 5452
}

5453 5454 5455
void trace_init_global_iter(struct trace_iterator *iter)
{
	iter->tr = &global_trace;
5456
	iter->trace = iter->tr->current_trace;
5457
	iter->cpu_file = RING_BUFFER_ALL_CPUS;
5458 5459
}

5460 5461
static void
__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
5462
{
5463
	static arch_spinlock_t ftrace_dump_lock =
5464
		(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
5465 5466
	/* use static because iter can be a bit big for the stack */
	static struct trace_iterator iter;
5467
	unsigned int old_userobj;
5468
	static int dump_ran;
5469 5470
	unsigned long flags;
	int cnt = 0, cpu;
5471 5472

	/* only one dump */
5473
	local_irq_save(flags);
5474
	arch_spin_lock(&ftrace_dump_lock);
5475 5476 5477 5478 5479
	if (dump_ran)
		goto out;

	dump_ran = 1;

5480
	tracing_off();
5481

5482 5483 5484 5485 5486 5487
	/* Did function tracer already get disabled? */
	if (ftrace_is_dead()) {
		printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
		printk("#          MAY BE MISSING FUNCTION EVENTS\n");
	}

5488 5489
	if (disable_tracing)
		ftrace_kill();
5490

5491
	/* Simulate the iterator */
5492 5493
	trace_init_global_iter(&iter);

5494
	for_each_tracing_cpu(cpu) {
5495
		atomic_inc(&per_cpu_ptr(iter.tr->data, cpu)->disabled);
5496 5497
	}

5498 5499
	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;

5500 5501 5502
	/* don't look at user memory in panic mode */
	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;

5503 5504
	switch (oops_dump_mode) {
	case DUMP_ALL:
5505
		iter.cpu_file = RING_BUFFER_ALL_CPUS;
5506 5507 5508 5509 5510 5511 5512 5513
		break;
	case DUMP_ORIG:
		iter.cpu_file = raw_smp_processor_id();
		break;
	case DUMP_NONE:
		goto out_enable;
	default:
		printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5514
		iter.cpu_file = RING_BUFFER_ALL_CPUS;
5515 5516 5517
	}

	printk(KERN_TRACE "Dumping ftrace buffer:\n");
5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539

	/*
	 * We need to stop all tracing on all CPUS to read the
	 * the next buffer. This is a bit expensive, but is
	 * not done often. We fill all what we can read,
	 * and then release the locks again.
	 */

	while (!trace_empty(&iter)) {

		if (!cnt)
			printk(KERN_TRACE "---------------------------------\n");

		cnt++;

		/* reset all but tr, trace, and overruns */
		memset(&iter.seq, 0,
		       sizeof(struct trace_iterator) -
		       offsetof(struct trace_iterator, seq));
		iter.iter_flags |= TRACE_FILE_LAT_FMT;
		iter.pos = -1;

5540
		if (trace_find_next_entry_inc(&iter) != NULL) {
5541 5542 5543 5544 5545
			int ret;

			ret = print_trace_line(&iter);
			if (ret != TRACE_TYPE_NO_CONSUME)
				trace_consume(&iter);
5546
		}
5547
		touch_nmi_watchdog();
5548 5549 5550 5551 5552 5553 5554 5555 5556

		trace_printk_seq(&iter.seq);
	}

	if (!cnt)
		printk(KERN_TRACE "   (ftrace buffer empty)\n");
	else
		printk(KERN_TRACE "---------------------------------\n");

5557
 out_enable:
5558 5559 5560 5561 5562
	/* Re-enable tracing if requested */
	if (!disable_tracing) {
		trace_flags |= old_userobj;

		for_each_tracing_cpu(cpu) {
5563
			atomic_dec(&per_cpu_ptr(iter.tr->data, cpu)->disabled);
5564 5565 5566 5567
		}
		tracing_on();
	}

5568
 out:
5569
	arch_spin_unlock(&ftrace_dump_lock);
5570
	local_irq_restore(flags);
5571 5572
}

5573
/* By default: disable tracing after the dump */
5574
void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5575
{
5576
	__ftrace_dump(true, oops_dump_mode);
5577
}
5578
EXPORT_SYMBOL_GPL(ftrace_dump);
5579

5580
__init static int tracer_alloc_buffers(void)
5581
{
5582
	int ring_buf_size;
5583
	enum ring_buffer_flags rb_flags;
5584
	int i;
5585
	int ret = -ENOMEM;
5586

5587

5588 5589 5590 5591 5592
	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
		goto out;

	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
		goto out_free_buffer_mask;
5593

5594 5595
	/* Only allocate trace_printk buffers if a trace_printk exists */
	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5596
		/* Must be called before global_trace.buffer is allocated */
5597 5598
		trace_printk_init_buffers();

5599 5600 5601 5602 5603 5604
	/* To save memory, keep the ring buffer size to its minimum */
	if (ring_buffer_expanded)
		ring_buf_size = trace_buf_size;
	else
		ring_buf_size = 1;

5605 5606
	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;

5607 5608 5609
	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
	cpumask_copy(tracing_cpumask, cpu_all_mask);

5610 5611
	raw_spin_lock_init(&global_trace.start_lock);

5612
	/* TODO: make the number of buffers hot pluggable with CPUS */
5613
	global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5614 5615 5616
	if (!global_trace.buffer) {
		printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
		WARN_ON(1);
5617
		goto out_free_cpumask;
5618
	}
5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633

	global_trace.data = alloc_percpu(struct trace_array_cpu);

	if (!global_trace.data) {
		printk(KERN_ERR "tracer: failed to allocate percpu memory!\n");
		WARN_ON(1);
		goto out_free_cpumask;
	}

	for_each_tracing_cpu(i) {
		memset(per_cpu_ptr(global_trace.data, i), 0, sizeof(struct trace_array_cpu));
		per_cpu_ptr(global_trace.data, i)->trace_cpu.cpu = i;
		per_cpu_ptr(global_trace.data, i)->trace_cpu.tr = &global_trace;
	}

5634 5635
	if (global_trace.buffer_disabled)
		tracing_off();
5636 5637

#ifdef CONFIG_TRACER_MAX_TRACE
5638 5639 5640 5641 5642 5643
	max_tr.data = alloc_percpu(struct trace_array_cpu);
	if (!max_tr.data) {
		printk(KERN_ERR "tracer: failed to allocate percpu memory!\n");
		WARN_ON(1);
		goto out_free_cpumask;
	}
5644
	max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5645
	raw_spin_lock_init(&max_tr.start_lock);
5646 5647 5648 5649
	if (!max_tr.buffer) {
		printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
		WARN_ON(1);
		ring_buffer_free(global_trace.buffer);
5650
		goto out_free_cpumask;
5651
	}
5652 5653

	for_each_tracing_cpu(i) {
5654 5655 5656
		memset(per_cpu_ptr(max_tr.data, i), 0, sizeof(struct trace_array_cpu));
		per_cpu_ptr(max_tr.data, i)->trace_cpu.cpu = i;
		per_cpu_ptr(max_tr.data, i)->trace_cpu.tr = &max_tr;
5657
	}
5658
#endif
5659

5660
	/* Allocate the first page for all buffers */
5661 5662
	set_buffer_entries(&global_trace,
			   ring_buffer_size(global_trace.buffer, 0));
5663 5664 5665
#ifdef CONFIG_TRACER_MAX_TRACE
	set_buffer_entries(&max_tr, 1);
#endif
5666 5667

	trace_init_cmdlines();
5668
	init_irq_work(&trace_work_wakeup, trace_wake_up);
5669

5670
	register_tracer(&nop_trace);
5671

5672 5673
	global_trace.current_trace = &nop_trace;

S
Steven Rostedt 已提交
5674 5675
	/* All seems OK, enable tracing */
	tracing_disabled = 0;
5676

5677 5678 5679 5680
	atomic_notifier_chain_register(&panic_notifier_list,
				       &trace_panic_notifier);

	register_die_notifier(&trace_die_notifier);
5681

5682 5683
	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;

5684 5685 5686 5687
	/* Holder for file callbacks */
	global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
	global_trace.trace_cpu.tr = &global_trace;

5688 5689 5690 5691
	INIT_LIST_HEAD(&global_trace.systems);
	INIT_LIST_HEAD(&global_trace.events);
	list_add(&global_trace.list, &ftrace_trace_arrays);

5692 5693 5694 5695
	while (trace_boot_options) {
		char *option;

		option = strsep(&trace_boot_options, ",");
5696
		trace_set_options(&global_trace, option);
5697 5698
	}

5699
	return 0;
5700

5701
out_free_cpumask:
5702 5703
	free_percpu(global_trace.data);
	free_percpu(max_tr.data);
5704 5705 5706 5707 5708
	free_cpumask_var(tracing_cpumask);
out_free_buffer_mask:
	free_cpumask_var(tracing_buffer_mask);
out:
	return ret;
5709
}
5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729

__init static int clear_boot_tracer(void)
{
	/*
	 * The default tracer at boot buffer is an init section.
	 * This function is called in lateinit. If we did not
	 * find the boot tracer, then clear it out, to prevent
	 * later registration from accessing the buffer that is
	 * about to be freed.
	 */
	if (!default_bootup_tracer)
		return 0;

	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
	       default_bootup_tracer);
	default_bootup_tracer = NULL;

	return 0;
}

5730 5731
early_initcall(tracer_alloc_buffers);
fs_initcall(tracer_init_debugfs);
5732
late_initcall(clear_boot_tracer);