trace.c 117.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * ring buffer based function tracer
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally taken from the RT patch by:
 *    Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code from the latency_tracer, that is:
 *  Copyright (C) 2004-2006 Ingo Molnar
 *  Copyright (C) 2004 William Lee Irwin III
 */
14
#include <linux/ring_buffer.h>
15
#include <generated/utsrelease.h>
16 17
#include <linux/stacktrace.h>
#include <linux/writeback.h>
18 19
#include <linux/kallsyms.h>
#include <linux/seq_file.h>
20
#include <linux/notifier.h>
21
#include <linux/irqflags.h>
22
#include <linux/debugfs.h>
23
#include <linux/pagemap.h>
24 25 26
#include <linux/hardirq.h>
#include <linux/linkage.h>
#include <linux/uaccess.h>
27
#include <linux/kprobes.h>
28 29 30
#include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/percpu.h>
31
#include <linux/splice.h>
32
#include <linux/kdebug.h>
33
#include <linux/string.h>
34
#include <linux/rwsem.h>
35
#include <linux/slab.h>
36 37
#include <linux/ctype.h>
#include <linux/init.h>
38
#include <linux/poll.h>
39
#include <linux/nmi.h>
40
#include <linux/fs.h>
I
Ingo Molnar 已提交
41

42
#include "trace.h"
43
#include "trace_output.h"
44

45 46 47 48
/*
 * On boot up, the ring buffer is set to the minimum size, so that
 * we do not waste memory on systems that are not using tracing.
 */
49
int ring_buffer_expanded;
50

51 52
/*
 * We need to change this state when a selftest is running.
53 54
 * A selftest will lurk into the ring-buffer to count the
 * entries inserted during the selftest although some concurrent
55
 * insertions into the ring-buffer such as trace_printk could occurred
56 57
 * at the same time, giving false positive or negative results.
 */
58
static bool __read_mostly tracing_selftest_running;
59

60 61 62
/*
 * If a tracer is running, we do not want to run SELFTEST.
 */
63
bool __read_mostly tracing_selftest_disabled;
64

65 66 67 68 69 70 71 72 73 74 75 76 77 78
/* For tracers that don't implement custom flags */
static struct tracer_opt dummy_tracer_opt[] = {
	{ }
};

static struct tracer_flags dummy_tracer_flags = {
	.val = 0,
	.opts = dummy_tracer_opt
};

static int dummy_set_flag(u32 old_flags, u32 bit, int set)
{
	return 0;
}
79 80 81 82 83 84 85

/*
 * Kill all tracing for good (never come back).
 * It is initialized to 1 but will turn to zero if the initialization
 * of the tracer is successful. But that is the only place that sets
 * this back to zero.
 */
86
static int tracing_disabled = 1;
87

88
DEFINE_PER_CPU(int, ftrace_cpu_disabled);
89

90
cpumask_var_t __read_mostly	tracing_buffer_mask;
91

92 93 94 95 96 97 98 99 100 101 102
/*
 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
 *
 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
 * is set, then ftrace_dump is called. This will output the contents
 * of the ftrace buffers to the console.  This is very useful for
 * capturing traces that lead to crashes and outputing it to a
 * serial console.
 *
 * It is default off, but you can enable it with either specifying
 * "ftrace_dump_on_oops" in the kernel command line, or setting
103 104 105
 * /proc/sys/kernel/ftrace_dump_on_oops
 * Set 1 if you want to dump buffers of all CPUs
 * Set 2 if you want to dump the buffer of the CPU that triggered oops
106
 */
107 108

enum ftrace_dump_mode ftrace_dump_on_oops;
109

110 111
static int tracing_set_tracer(const char *buf);

L
Li Zefan 已提交
112 113
#define MAX_TRACER_SIZE		100
static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
114
static char *default_bootup_tracer;
115

116
static int __init set_cmdline_ftrace(char *str)
117
{
L
Li Zefan 已提交
118
	strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
119
	default_bootup_tracer = bootup_tracer_buf;
120 121
	/* We are using ftrace early, expand it */
	ring_buffer_expanded = 1;
122 123
	return 1;
}
124
__setup("ftrace=", set_cmdline_ftrace);
125

126 127
static int __init set_ftrace_dump_on_oops(char *str)
{
128 129 130 131 132 133 134 135 136 137 138
	if (*str++ != '=' || !*str) {
		ftrace_dump_on_oops = DUMP_ALL;
		return 1;
	}

	if (!strcmp("orig_cpu", str)) {
		ftrace_dump_on_oops = DUMP_ORIG;
                return 1;
        }

        return 0;
139 140
}
__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
S
Steven Rostedt 已提交
141

142
unsigned long long ns2usecs(cycle_t nsec)
143 144 145 146 147 148
{
	nsec += 500;
	do_div(nsec, 1000);
	return nsec;
}

S
Steven Rostedt 已提交
149 150 151 152 153 154 155 156 157 158 159 160
/*
 * The global_trace is the descriptor that holds the tracing
 * buffers for the live tracing. For each CPU, it contains
 * a link list of pages that will store trace entries. The
 * page descriptor of the pages in the memory is used to hold
 * the link list by linking the lru item in the page descriptor
 * to each of the pages in the buffer per CPU.
 *
 * For each active CPU there is a data field that holds the
 * pages for the buffer for that CPU. Each CPU has the same number
 * of pages allocated for its buffer.
 */
161 162 163 164
static struct trace_array	global_trace;

static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);

165 166
int filter_current_check_discard(struct ring_buffer *buffer,
				 struct ftrace_event_call *call, void *rec,
167 168
				 struct ring_buffer_event *event)
{
169
	return filter_check_discard(call, rec, buffer, event);
170
}
171
EXPORT_SYMBOL_GPL(filter_current_check_discard);
172

173 174 175 176 177 178 179 180 181 182 183 184 185
cycle_t ftrace_now(int cpu)
{
	u64 ts;

	/* Early boot up does not have a buffer yet */
	if (!global_trace.buffer)
		return trace_clock_local();

	ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
	ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);

	return ts;
}
186

S
Steven Rostedt 已提交
187 188 189 190 191 192 193 194 195 196
/*
 * The max_tr is used to snapshot the global_trace when a maximum
 * latency is reached. Some tracers will use this to store a maximum
 * trace while it continues examining live traces.
 *
 * The buffers for the max_tr are set up the same as the global_trace.
 * When a snapshot is taken, the link list of the max_tr is swapped
 * with the link list of the global_trace and the buffers are reset for
 * the global_trace so the tracing can continue.
 */
197 198
static struct trace_array	max_tr;

199
static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
200

S
Steven Rostedt 已提交
201
/* tracer_enabled is used to toggle activation of a tracer */
202
static int			tracer_enabled = 1;
S
Steven Rostedt 已提交
203

204 205 206 207 208 209 210 211 212 213 214 215 216
/**
 * tracing_is_enabled - return tracer_enabled status
 *
 * This function is used by other tracers to know the status
 * of the tracer_enabled flag.  Tracers may use this function
 * to know if it should enable their features when starting
 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
 */
int tracing_is_enabled(void)
{
	return tracer_enabled;
}

S
Steven Rostedt 已提交
217
/*
218 219 220
 * trace_buf_size is the size in bytes that is allocated
 * for a buffer. Note, the number of bytes is always rounded
 * to page size.
221 222 223 224 225
 *
 * This number is purposely set to a low number of 16384.
 * If the dump on oops happens, it will be much appreciated
 * to not have to wait for all that output. Anyway this can be
 * boot time and run time configurable.
S
Steven Rostedt 已提交
226
 */
227
#define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */
228

229
static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
230

S
Steven Rostedt 已提交
231
/* trace_types holds a link list of available tracers. */
232
static struct tracer		*trace_types __read_mostly;
S
Steven Rostedt 已提交
233 234

/* current_trace points to the tracer that is currently active */
235
static struct tracer		*current_trace __read_mostly;
S
Steven Rostedt 已提交
236 237 238 239

/*
 * trace_types_lock is used to protect the trace_types list.
 */
240
static DEFINE_MUTEX(trace_types_lock);
S
Steven Rostedt 已提交
241

242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
/*
 * serialize the access of the ring buffer
 *
 * ring buffer serializes readers, but it is low level protection.
 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 * are not protected by ring buffer.
 *
 * The content of events may become garbage if we allow other process consumes
 * these events concurrently:
 *   A) the page of the consumed events may become a normal page
 *      (not reader page) in ring buffer, and this page will be rewrited
 *      by events producer.
 *   B) The page of the consumed events may become a page for splice_read,
 *      and this page will be returned to system.
 *
 * These primitives allow multi process access to different cpu ring buffer
 * concurrently.
 *
 * These primitives don't distinguish read-only and read-consume access.
 * Multi read-only access are also serialized.
 */

#ifdef CONFIG_SMP
static DECLARE_RWSEM(all_cpu_access_lock);
static DEFINE_PER_CPU(struct mutex, cpu_access_lock);

static inline void trace_access_lock(int cpu)
{
	if (cpu == TRACE_PIPE_ALL_CPU) {
		/* gain it for accessing the whole ring buffer. */
		down_write(&all_cpu_access_lock);
	} else {
		/* gain it for accessing a cpu ring buffer. */

		/* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
		down_read(&all_cpu_access_lock);

		/* Secondly block other access to this @cpu ring buffer. */
		mutex_lock(&per_cpu(cpu_access_lock, cpu));
	}
}

static inline void trace_access_unlock(int cpu)
{
	if (cpu == TRACE_PIPE_ALL_CPU) {
		up_write(&all_cpu_access_lock);
	} else {
		mutex_unlock(&per_cpu(cpu_access_lock, cpu));
		up_read(&all_cpu_access_lock);
	}
}

static inline void trace_access_lock_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu)
		mutex_init(&per_cpu(cpu_access_lock, cpu));
}

#else

static DEFINE_MUTEX(access_lock);

static inline void trace_access_lock(int cpu)
{
	(void)cpu;
	mutex_lock(&access_lock);
}

static inline void trace_access_unlock(int cpu)
{
	(void)cpu;
	mutex_unlock(&access_lock);
}

static inline void trace_access_lock_init(void)
{
}

#endif

S
Steven Rostedt 已提交
324
/* trace_wait is a waitqueue for tasks blocked on trace_poll */
325 326
static DECLARE_WAIT_QUEUE_HEAD(trace_wait);

327
/* trace_flags holds trace_options default values */
328
unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
329
	TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
330 331
	TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
	TRACE_ITER_IRQ_INFO;
332

333
static int trace_stop_count;
334
static DEFINE_RAW_SPINLOCK(tracing_start_lock);
335

336 337 338 339 340 341 342
static void wakeup_work_handler(struct work_struct *work)
{
	wake_up(&trace_wait);
}

static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);

343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
/**
 * tracing_on - enable tracing buffers
 *
 * This function enables tracing buffers that may have been
 * disabled with tracing_off.
 */
void tracing_on(void)
{
	if (global_trace.buffer)
		ring_buffer_record_on(global_trace.buffer);
	/*
	 * This flag is only looked at when buffers haven't been
	 * allocated yet. We don't really care about the race
	 * between setting this flag and actually turning
	 * on the buffer.
	 */
	global_trace.buffer_disabled = 0;
}
EXPORT_SYMBOL_GPL(tracing_on);

/**
 * tracing_off - turn off tracing buffers
 *
 * This function stops the tracing buffers from recording data.
 * It does not disable any overhead the tracers themselves may
 * be causing. This function simply causes all recording to
 * the ring buffers to fail.
 */
void tracing_off(void)
{
	if (global_trace.buffer)
374
		ring_buffer_record_off(global_trace.buffer);
375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
	/*
	 * This flag is only looked at when buffers haven't been
	 * allocated yet. We don't really care about the race
	 * between setting this flag and actually turning
	 * on the buffer.
	 */
	global_trace.buffer_disabled = 1;
}
EXPORT_SYMBOL_GPL(tracing_off);

/**
 * tracing_is_on - show state of ring buffers enabled
 */
int tracing_is_on(void)
{
	if (global_trace.buffer)
		return ring_buffer_record_is_on(global_trace.buffer);
	return !global_trace.buffer_disabled;
}
EXPORT_SYMBOL_GPL(tracing_is_on);

S
Steven Rostedt 已提交
396 397 398
/**
 * trace_wake_up - wake up tasks waiting for trace input
 *
399 400 401
 * Schedules a delayed work to wake up any task that is blocked on the
 * trace_wait queue. These is used with trace_poll for tasks polling the
 * trace.
S
Steven Rostedt 已提交
402
 */
403 404
void trace_wake_up(void)
{
405
	const unsigned long delay = msecs_to_jiffies(2);
406 407 408

	if (trace_flags & TRACE_ITER_BLOCK)
		return;
409
	schedule_delayed_work(&wakeup_work, delay);
410
}
411

412
static int __init set_buf_size(char *str)
413
{
414
	unsigned long buf_size;
415

416 417
	if (!str)
		return 0;
418
	buf_size = memparse(str, &str);
419
	/* nr_entries can not be zero */
420
	if (buf_size == 0)
421
		return 0;
422
	trace_buf_size = buf_size;
423 424
	return 1;
}
425
__setup("trace_buf_size=", set_buf_size);
426

427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
static int __init set_tracing_thresh(char *str)
{
	unsigned long threshhold;
	int ret;

	if (!str)
		return 0;
	ret = strict_strtoul(str, 0, &threshhold);
	if (ret < 0)
		return 0;
	tracing_thresh = threshhold * 1000;
	return 1;
}
__setup("tracing_thresh=", set_tracing_thresh);

S
Steven Rostedt 已提交
442 443 444 445 446
unsigned long nsecs_to_usecs(unsigned long nsecs)
{
	return nsecs / 1000;
}

S
Steven Rostedt 已提交
447
/* These must match the bit postions in trace_iterator_flags */
448 449 450 451 452
static const char *trace_options[] = {
	"print-parent",
	"sym-offset",
	"sym-addr",
	"verbose",
I
Ingo Molnar 已提交
453
	"raw",
454
	"hex",
I
Ingo Molnar 已提交
455
	"bin",
456
	"block",
I
Ingo Molnar 已提交
457
	"stacktrace",
458
	"trace_printk",
459
	"ftrace_preempt",
460
	"branch",
461
	"annotate",
462
	"userstacktrace",
463
	"sym-userobj",
464
	"printk-msg-only",
465
	"context-info",
466
	"latency-format",
467
	"sleep-time",
468
	"graph-time",
469
	"record-cmd",
470
	"overwrite",
471
	"disable_on_free",
472
	"irq-info",
473 474 475
	NULL
};

476 477 478 479 480 481
static struct {
	u64 (*func)(void);
	const char *name;
} trace_clocks[] = {
	{ trace_clock_local,	"local" },
	{ trace_clock_global,	"global" },
482
	{ trace_clock_counter,	"counter" },
483 484 485 486
};

int trace_clock_id;

487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
/*
 * trace_parser_get_init - gets the buffer for trace parser
 */
int trace_parser_get_init(struct trace_parser *parser, int size)
{
	memset(parser, 0, sizeof(*parser));

	parser->buffer = kmalloc(size, GFP_KERNEL);
	if (!parser->buffer)
		return 1;

	parser->size = size;
	return 0;
}

/*
 * trace_parser_put - frees the buffer for trace parser
 */
void trace_parser_put(struct trace_parser *parser)
{
	kfree(parser->buffer);
}

/*
 * trace_get_user - reads the user input string separated by  space
 * (matched by isspace(ch))
 *
 * For each string found the 'struct trace_parser' is updated,
 * and the function returns.
 *
 * Returns number of bytes read.
 *
 * See kernel/trace/trace.h for 'struct trace_parser' details.
 */
int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
	size_t cnt, loff_t *ppos)
{
	char ch;
	size_t read = 0;
	ssize_t ret;

	if (!*ppos)
		trace_parser_clear(parser);

	ret = get_user(ch, ubuf++);
	if (ret)
		goto out;

	read++;
	cnt--;

	/*
	 * The parser is not finished with the last write,
	 * continue reading the user input without skipping spaces.
	 */
	if (!parser->cont) {
		/* skip white space */
		while (cnt && isspace(ch)) {
			ret = get_user(ch, ubuf++);
			if (ret)
				goto out;
			read++;
			cnt--;
		}

		/* only spaces were written */
		if (isspace(ch)) {
			*ppos += read;
			ret = read;
			goto out;
		}

		parser->idx = 0;
	}

	/* read the non-space input */
	while (cnt && !isspace(ch)) {
564
		if (parser->idx < parser->size - 1)
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
			parser->buffer[parser->idx++] = ch;
		else {
			ret = -EINVAL;
			goto out;
		}
		ret = get_user(ch, ubuf++);
		if (ret)
			goto out;
		read++;
		cnt--;
	}

	/* We either got finished input or we have to wait for another call. */
	if (isspace(ch)) {
		parser->buffer[parser->idx] = 0;
		parser->cont = false;
	} else {
		parser->cont = true;
		parser->buffer[parser->idx++] = ch;
	}

	*ppos += read;
	ret = read;

out:
	return ret;
}

593 594 595 596 597
ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
{
	int len;
	int ret;

598 599 600
	if (!cnt)
		return 0;

601 602 603 604 605 606 607
	if (s->len <= s->readpos)
		return -EBUSY;

	len = s->len - s->readpos;
	if (cnt > len)
		cnt = len;
	ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
608
	if (ret == cnt)
609 610
		return -EFAULT;

611 612
	cnt -= ret;

613
	s->readpos += cnt;
614
	return cnt;
S
Steven Rostedt 已提交
615 616
}

617
static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
618 619 620 621 622 623 624 625 626
{
	int len;

	if (s->len <= s->readpos)
		return -EBUSY;

	len = s->len - s->readpos;
	if (cnt > len)
		cnt = len;
627
	memcpy(buf, s->buffer + s->readpos, cnt);
628

629
	s->readpos += cnt;
630 631 632
	return cnt;
}

633 634 635 636 637 638
/*
 * ftrace_max_lock is used to protect the swapping of buffers
 * when taking a max snapshot. The buffers themselves are
 * protected by per_cpu spinlocks. But the action of the swap
 * needs its own lock.
 *
639
 * This is defined as a arch_spinlock_t in order to help
640 641 642 643 644 645
 * with performance when lockdep debugging is enabled.
 *
 * It is also used in other places outside the update_max_tr
 * so it needs to be defined outside of the
 * CONFIG_TRACER_MAX_TRACE.
 */
646
static arch_spinlock_t ftrace_max_lock =
647
	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
648

649 650
unsigned long __read_mostly	tracing_thresh;

651 652 653 654 655 656 657 658 659 660 661 662
#ifdef CONFIG_TRACER_MAX_TRACE
unsigned long __read_mostly	tracing_max_latency;

/*
 * Copy the new maximum trace into the separate maximum-trace
 * structure. (this way the maximum trace is permanently saved,
 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
 */
static void
__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
	struct trace_array_cpu *data = tr->data[cpu];
663
	struct trace_array_cpu *max_data;
664 665 666 667

	max_tr.cpu = cpu;
	max_tr.time_start = data->preempt_timestamp;

668 669 670 671
	max_data = max_tr.data[cpu];
	max_data->saved_latency = tracing_max_latency;
	max_data->critical_start = data->critical_start;
	max_data->critical_end = data->critical_end;
672

673
	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
674 675 676 677 678
	max_data->pid = tsk->pid;
	max_data->uid = task_uid(tsk);
	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
	max_data->policy = tsk->policy;
	max_data->rt_priority = tsk->rt_priority;
679 680 681 682 683

	/* record this tasks comm */
	tracing_record_cmdline(tsk);
}

S
Steven Rostedt 已提交
684 685 686 687 688 689 690 691 692
/**
 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 * @tr: tracer
 * @tsk: the task with the latency
 * @cpu: The cpu that initiated the trace.
 *
 * Flip the buffers between the @tr and the max_tr and record information
 * about which task was the cause of this latency.
 */
I
Ingo Molnar 已提交
693
void
694 695
update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
696
	struct ring_buffer *buf = tr->buffer;
697

698 699 700
	if (trace_stop_count)
		return;

701
	WARN_ON_ONCE(!irqs_disabled());
702 703 704 705
	if (!current_trace->use_max_tr) {
		WARN_ON_ONCE(1);
		return;
	}
706
	arch_spin_lock(&ftrace_max_lock);
707 708 709 710

	tr->buffer = max_tr.buffer;
	max_tr.buffer = buf;

711
	__update_max_tr(tr, tsk, cpu);
712
	arch_spin_unlock(&ftrace_max_lock);
713 714 715 716 717 718 719
}

/**
 * update_max_tr_single - only copy one trace over, and reset the rest
 * @tr - tracer
 * @tsk - task with the latency
 * @cpu - the cpu of the buffer to copy.
S
Steven Rostedt 已提交
720 721
 *
 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
722
 */
I
Ingo Molnar 已提交
723
void
724 725
update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
726
	int ret;
727

728 729 730
	if (trace_stop_count)
		return;

731
	WARN_ON_ONCE(!irqs_disabled());
732 733 734 735 736
	if (!current_trace->use_max_tr) {
		WARN_ON_ONCE(1);
		return;
	}

737
	arch_spin_lock(&ftrace_max_lock);
738

739 740
	ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);

741 742 743 744 745 746 747 748 749 750 751 752
	if (ret == -EBUSY) {
		/*
		 * We failed to swap the buffer due to a commit taking
		 * place on this CPU. We fail to record, but we reset
		 * the max trace buffer (no one writes directly to it)
		 * and flag that it failed.
		 */
		trace_array_printk(&max_tr, _THIS_IP_,
			"Failed to swap buffers due to commit in progress\n");
	}

	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
753 754

	__update_max_tr(tr, tsk, cpu);
755
	arch_spin_unlock(&ftrace_max_lock);
756
}
757
#endif /* CONFIG_TRACER_MAX_TRACE */
758

S
Steven Rostedt 已提交
759 760 761 762 763 764
/**
 * register_tracer - register a tracer with the ftrace system.
 * @type - the plugin for the tracer
 *
 * Register a new plugin tracer.
 */
765 766 767 768 769 770 771 772 773 774
int register_tracer(struct tracer *type)
{
	struct tracer *t;
	int ret = 0;

	if (!type->name) {
		pr_info("Tracer must have a name\n");
		return -1;
	}

775
	if (strlen(type->name) >= MAX_TRACER_SIZE) {
L
Li Zefan 已提交
776 777 778 779
		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
		return -1;
	}

780
	mutex_lock(&trace_types_lock);
I
Ingo Molnar 已提交
781

782 783
	tracing_selftest_running = true;

784 785 786
	for (t = trace_types; t; t = t->next) {
		if (strcmp(type->name, t->name) == 0) {
			/* already found */
L
Li Zefan 已提交
787
			pr_info("Tracer %s already registered\n",
788 789 790 791 792 793
				type->name);
			ret = -1;
			goto out;
		}
	}

794 795 796 797 798 799 800
	if (!type->set_flag)
		type->set_flag = &dummy_set_flag;
	if (!type->flags)
		type->flags = &dummy_tracer_flags;
	else
		if (!type->flags->opts)
			type->flags->opts = dummy_tracer_opt;
801 802 803
	if (!type->wait_pipe)
		type->wait_pipe = default_wait_pipe;

804

S
Steven Rostedt 已提交
805
#ifdef CONFIG_FTRACE_STARTUP_TEST
806
	if (type->selftest && !tracing_selftest_disabled) {
S
Steven Rostedt 已提交
807 808
		struct tracer *saved_tracer = current_trace;
		struct trace_array *tr = &global_trace;
809

S
Steven Rostedt 已提交
810 811 812 813 814 815 816
		/*
		 * Run a selftest on this tracer.
		 * Here we reset the trace buffer, and set the current
		 * tracer to be this tracer. The tracer can then run some
		 * internal tracing to verify that everything is in order.
		 * If we fail, we do not register this tracer.
		 */
817
		tracing_reset_online_cpus(tr);
I
Ingo Molnar 已提交
818

S
Steven Rostedt 已提交
819
		current_trace = type;
820 821 822

		/* If we expanded the buffers, make sure the max is expanded too */
		if (ring_buffer_expanded && type->use_max_tr)
823 824
			ring_buffer_resize(max_tr.buffer, trace_buf_size,
						RING_BUFFER_ALL_CPUS);
825

S
Steven Rostedt 已提交
826 827 828 829 830 831 832 833 834
		/* the test is responsible for initializing and enabling */
		pr_info("Testing tracer %s: ", type->name);
		ret = type->selftest(type, tr);
		/* the test is responsible for resetting too */
		current_trace = saved_tracer;
		if (ret) {
			printk(KERN_CONT "FAILED!\n");
			goto out;
		}
S
Steven Rostedt 已提交
835
		/* Only reset on passing, to avoid touching corrupted buffers */
836
		tracing_reset_online_cpus(tr);
I
Ingo Molnar 已提交
837

838 839
		/* Shrink the max buffer again */
		if (ring_buffer_expanded && type->use_max_tr)
840 841
			ring_buffer_resize(max_tr.buffer, 1,
						RING_BUFFER_ALL_CPUS);
842

S
Steven Rostedt 已提交
843 844 845 846
		printk(KERN_CONT "PASSED\n");
	}
#endif

847 848
	type->next = trace_types;
	trace_types = type;
S
Steven Rostedt 已提交
849

850
 out:
851
	tracing_selftest_running = false;
852 853
	mutex_unlock(&trace_types_lock);

S
Steven Rostedt 已提交
854 855 856
	if (ret || !default_bootup_tracer)
		goto out_unlock;

L
Li Zefan 已提交
857
	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
S
Steven Rostedt 已提交
858 859 860 861 862 863 864 865
		goto out_unlock;

	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
	/* Do we want this tracer to start on bootup? */
	tracing_set_tracer(type->name);
	default_bootup_tracer = NULL;
	/* disable other selftests, since this will break it. */
	tracing_selftest_disabled = 1;
866
#ifdef CONFIG_FTRACE_STARTUP_TEST
S
Steven Rostedt 已提交
867 868
	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
	       type->name);
869 870
#endif

S
Steven Rostedt 已提交
871
 out_unlock:
872 873 874 875 876 877 878 879 880 881 882 883
	return ret;
}

void unregister_tracer(struct tracer *type)
{
	struct tracer **t;

	mutex_lock(&trace_types_lock);
	for (t = &trace_types; *t; t = &(*t)->next) {
		if (*t == type)
			goto found;
	}
L
Li Zefan 已提交
884
	pr_info("Tracer %s not registered\n", type->name);
885 886 887 888
	goto out;

 found:
	*t = (*t)->next;
889 890 891 892 893 894 895 896

	if (type == current_trace && tracer_enabled) {
		tracer_enabled = 0;
		tracing_stop();
		if (current_trace->stop)
			current_trace->stop(&global_trace);
		current_trace = &nop_trace;
	}
L
Li Zefan 已提交
897
out:
898 899 900
	mutex_unlock(&trace_types_lock);
}

901 902 903 904 905 906 907 908
void tracing_reset(struct trace_array *tr, int cpu)
{
	struct ring_buffer *buffer = tr->buffer;

	ring_buffer_record_disable(buffer);

	/* Make sure all commits have finished */
	synchronize_sched();
909
	ring_buffer_reset_cpu(buffer, cpu);
910 911 912 913

	ring_buffer_record_enable(buffer);
}

914 915
void tracing_reset_online_cpus(struct trace_array *tr)
{
916
	struct ring_buffer *buffer = tr->buffer;
917 918
	int cpu;

919 920 921 922 923
	ring_buffer_record_disable(buffer);

	/* Make sure all commits have finished */
	synchronize_sched();

924 925 926
	tr->time_start = ftrace_now(tr->cpu);

	for_each_online_cpu(cpu)
927
		ring_buffer_reset_cpu(buffer, cpu);
928 929

	ring_buffer_record_enable(buffer);
930 931
}

932 933 934 935 936 937 938 939 940 941
void tracing_reset_current(int cpu)
{
	tracing_reset(&global_trace, cpu);
}

void tracing_reset_current_online_cpus(void)
{
	tracing_reset_online_cpus(&global_trace);
}

942
#define SAVED_CMDLINES 128
943
#define NO_CMDLINE_MAP UINT_MAX
944 945 946 947
static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
static int cmdline_idx;
948
static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
949 950

/* temporary disable recording */
951
static atomic_t trace_record_cmdline_disabled __read_mostly;
952 953 954

static void trace_init_cmdlines(void)
{
955 956
	memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
	memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
957 958 959
	cmdline_idx = 0;
}

960 961 962 963 964
int is_tracing_stopped(void)
{
	return trace_stop_count;
}

S
Steven Rostedt 已提交
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979
/**
 * ftrace_off_permanent - disable all ftrace code permanently
 *
 * This should only be called when a serious anomally has
 * been detected.  This will turn off the function tracing,
 * ring buffers, and other tracing utilites. It takes no
 * locks and can be called from any context.
 */
void ftrace_off_permanent(void)
{
	tracing_disabled = 1;
	ftrace_stop();
	tracing_off_permanent();
}

980 981 982 983 984 985 986 987 988 989 990 991 992 993
/**
 * tracing_start - quick start of the tracer
 *
 * If tracing is enabled but was stopped by tracing_stop,
 * this will start the tracer back up.
 */
void tracing_start(void)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	if (tracing_disabled)
		return;

994
	raw_spin_lock_irqsave(&tracing_start_lock, flags);
995 996 997 998 999 1000
	if (--trace_stop_count) {
		if (trace_stop_count < 0) {
			/* Someone screwed up their debugging */
			WARN_ON_ONCE(1);
			trace_stop_count = 0;
		}
1001 1002 1003
		goto out;
	}

1004 1005
	/* Prevent the buffers from switching */
	arch_spin_lock(&ftrace_max_lock);
1006 1007 1008 1009 1010 1011 1012 1013 1014

	buffer = global_trace.buffer;
	if (buffer)
		ring_buffer_record_enable(buffer);

	buffer = max_tr.buffer;
	if (buffer)
		ring_buffer_record_enable(buffer);

1015 1016
	arch_spin_unlock(&ftrace_max_lock);

1017 1018
	ftrace_start();
 out:
1019
	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
}

/**
 * tracing_stop - quick stop of the tracer
 *
 * Light weight way to stop tracing. Use in conjunction with
 * tracing_start.
 */
void tracing_stop(void)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	ftrace_stop();
1034
	raw_spin_lock_irqsave(&tracing_start_lock, flags);
1035 1036 1037
	if (trace_stop_count++)
		goto out;

1038 1039 1040
	/* Prevent the buffers from switching */
	arch_spin_lock(&ftrace_max_lock);

1041 1042 1043 1044 1045 1046 1047 1048
	buffer = global_trace.buffer;
	if (buffer)
		ring_buffer_record_disable(buffer);

	buffer = max_tr.buffer;
	if (buffer)
		ring_buffer_record_disable(buffer);

1049 1050
	arch_spin_unlock(&ftrace_max_lock);

1051
 out:
1052
	raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1053 1054
}

I
Ingo Molnar 已提交
1055
void trace_stop_cmdline_recording(void);
1056

I
Ingo Molnar 已提交
1057
static void trace_save_cmdline(struct task_struct *tsk)
1058
{
1059
	unsigned pid, idx;
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069

	if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
		return;

	/*
	 * It's not the end of the world if we don't get
	 * the lock, but we also don't want to spin
	 * nor do we want to disable interrupts,
	 * so if we miss here, then better luck next time.
	 */
1070
	if (!arch_spin_trylock(&trace_cmdline_lock))
1071 1072 1073
		return;

	idx = map_pid_to_cmdline[tsk->pid];
1074
	if (idx == NO_CMDLINE_MAP) {
1075 1076
		idx = (cmdline_idx + 1) % SAVED_CMDLINES;

1077 1078 1079 1080 1081 1082 1083 1084 1085
		/*
		 * Check whether the cmdline buffer at idx has a pid
		 * mapped. We are going to overwrite that entry so we
		 * need to clear the map_pid_to_cmdline. Otherwise we
		 * would read the new comm for the old pid.
		 */
		pid = map_cmdline_to_pid[idx];
		if (pid != NO_CMDLINE_MAP)
			map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1086

1087
		map_cmdline_to_pid[idx] = tsk->pid;
1088 1089 1090 1091 1092 1093 1094
		map_pid_to_cmdline[tsk->pid] = idx;

		cmdline_idx = idx;
	}

	memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);

1095
	arch_spin_unlock(&trace_cmdline_lock);
1096 1097
}

1098
void trace_find_cmdline(int pid, char comm[])
1099 1100 1101
{
	unsigned map;

1102 1103 1104 1105
	if (!pid) {
		strcpy(comm, "<idle>");
		return;
	}
1106

1107 1108 1109 1110 1111
	if (WARN_ON_ONCE(pid < 0)) {
		strcpy(comm, "<XXX>");
		return;
	}

1112 1113 1114 1115
	if (pid > PID_MAX_DEFAULT) {
		strcpy(comm, "<...>");
		return;
	}
1116

1117
	preempt_disable();
1118
	arch_spin_lock(&trace_cmdline_lock);
1119
	map = map_pid_to_cmdline[pid];
1120 1121 1122 1123
	if (map != NO_CMDLINE_MAP)
		strcpy(comm, saved_cmdlines[map]);
	else
		strcpy(comm, "<...>");
1124

1125
	arch_spin_unlock(&trace_cmdline_lock);
1126
	preempt_enable();
1127 1128
}

I
Ingo Molnar 已提交
1129
void tracing_record_cmdline(struct task_struct *tsk)
1130
{
1131 1132
	if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
	    !tracing_is_on())
1133 1134 1135 1136 1137
		return;

	trace_save_cmdline(tsk);
}

1138
void
1139 1140
tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
			     int pc)
1141 1142 1143
{
	struct task_struct *tsk = current;

1144 1145
	entry->preempt_count		= pc & 0xff;
	entry->pid			= (tsk) ? tsk->pid : 0;
1146
	entry->padding			= 0;
1147
	entry->flags =
1148
#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1149
		(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1150 1151 1152
#else
		TRACE_FLAG_IRQS_NOSUPPORT |
#endif
1153 1154 1155 1156
		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
		((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
		(need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
}
1157
EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1158

1159 1160 1161 1162 1163
struct ring_buffer_event *
trace_buffer_lock_reserve(struct ring_buffer *buffer,
			  int type,
			  unsigned long len,
			  unsigned long flags, int pc)
1164 1165 1166
{
	struct ring_buffer_event *event;

1167
	event = ring_buffer_lock_reserve(buffer, len);
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
	if (event != NULL) {
		struct trace_entry *ent = ring_buffer_event_data(event);

		tracing_generic_entry_update(ent, flags, pc);
		ent->type = type;
	}

	return event;
}

1178 1179 1180 1181 1182
static inline void
__trace_buffer_unlock_commit(struct ring_buffer *buffer,
			     struct ring_buffer_event *event,
			     unsigned long flags, int pc,
			     int wake)
1183
{
1184
	ring_buffer_unlock_commit(buffer, event);
1185

1186 1187
	ftrace_trace_stack(buffer, flags, 6, pc);
	ftrace_trace_userstack(buffer, flags, pc);
1188 1189 1190 1191 1192

	if (wake)
		trace_wake_up();
}

1193 1194 1195
void trace_buffer_unlock_commit(struct ring_buffer *buffer,
				struct ring_buffer_event *event,
				unsigned long flags, int pc)
1196
{
1197
	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1198 1199
}

1200
struct ring_buffer_event *
1201 1202
trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
				  int type, unsigned long len,
1203 1204
				  unsigned long flags, int pc)
{
1205 1206
	*current_rb = global_trace.buffer;
	return trace_buffer_lock_reserve(*current_rb,
1207 1208
					 type, len, flags, pc);
}
1209
EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1210

1211 1212
void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
					struct ring_buffer_event *event,
1213 1214
					unsigned long flags, int pc)
{
1215
	__trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1216
}
1217
EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1218

1219 1220 1221
void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
				       struct ring_buffer_event *event,
				       unsigned long flags, int pc)
1222
{
1223
	__trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1224
}
1225
EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1226

1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
					    struct ring_buffer_event *event,
					    unsigned long flags, int pc,
					    struct pt_regs *regs)
{
	ring_buffer_unlock_commit(buffer, event);

	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
	ftrace_trace_userstack(buffer, flags, pc);
}
EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);

1239 1240
void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
					 struct ring_buffer_event *event)
1241
{
1242
	ring_buffer_discard_commit(buffer, event);
1243
}
1244
EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1245

I
Ingo Molnar 已提交
1246
void
1247
trace_function(struct trace_array *tr,
1248 1249
	       unsigned long ip, unsigned long parent_ip, unsigned long flags,
	       int pc)
1250
{
1251
	struct ftrace_event_call *call = &event_function;
1252
	struct ring_buffer *buffer = tr->buffer;
1253
	struct ring_buffer_event *event;
1254
	struct ftrace_entry *entry;
1255

1256
	/* If we are reading the ring buffer, don't trace */
R
Rusty Russell 已提交
1257
	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1258 1259
		return;

1260
	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1261
					  flags, pc);
1262 1263 1264
	if (!event)
		return;
	entry	= ring_buffer_event_data(event);
1265 1266
	entry->ip			= ip;
	entry->parent_ip		= parent_ip;
1267

1268 1269
	if (!filter_check_discard(call, entry, buffer, event))
		ring_buffer_unlock_commit(buffer, event);
1270 1271
}

I
Ingo Molnar 已提交
1272
void
I
Ingo Molnar 已提交
1273
ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1274 1275
       unsigned long ip, unsigned long parent_ip, unsigned long flags,
       int pc)
I
Ingo Molnar 已提交
1276 1277
{
	if (likely(!atomic_read(&data->disabled)))
1278
		trace_function(tr, ip, parent_ip, flags, pc);
I
Ingo Molnar 已提交
1279 1280
}

1281
#ifdef CONFIG_STACKTRACE
1282 1283 1284 1285 1286 1287 1288 1289 1290

#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
struct ftrace_stack {
	unsigned long		calls[FTRACE_STACK_MAX_ENTRIES];
};

static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
static DEFINE_PER_CPU(int, ftrace_stack_reserve);

1291
static void __ftrace_trace_stack(struct ring_buffer *buffer,
1292
				 unsigned long flags,
1293
				 int skip, int pc, struct pt_regs *regs)
I
Ingo Molnar 已提交
1294
{
1295
	struct ftrace_event_call *call = &event_kernel_stack;
1296
	struct ring_buffer_event *event;
1297
	struct stack_entry *entry;
I
Ingo Molnar 已提交
1298
	struct stack_trace trace;
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
	int use_stack;
	int size = FTRACE_STACK_ENTRIES;

	trace.nr_entries	= 0;
	trace.skip		= skip;

	/*
	 * Since events can happen in NMIs there's no safe way to
	 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
	 * or NMI comes in, it will just have to use the default
	 * FTRACE_STACK_SIZE.
	 */
	preempt_disable_notrace();

	use_stack = ++__get_cpu_var(ftrace_stack_reserve);
	/*
	 * We don't need any atomic variables, just a barrier.
	 * If an interrupt comes in, we don't care, because it would
	 * have exited and put the counter back to what we want.
	 * We just need a barrier to keep gcc from moving things
	 * around.
	 */
	barrier();
	if (use_stack == 1) {
		trace.entries		= &__get_cpu_var(ftrace_stack).calls[0];
		trace.max_entries	= FTRACE_STACK_MAX_ENTRIES;

		if (regs)
			save_stack_trace_regs(regs, &trace);
		else
			save_stack_trace(&trace);

		if (trace.nr_entries > size)
			size = trace.nr_entries;
	} else
		/* From now on, use_stack is a boolean */
		use_stack = 0;

	size *= sizeof(unsigned long);
I
Ingo Molnar 已提交
1338

1339
	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1340
					  sizeof(*entry) + size, flags, pc);
1341
	if (!event)
1342 1343
		goto out;
	entry = ring_buffer_event_data(event);
I
Ingo Molnar 已提交
1344

1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
	memset(&entry->caller, 0, size);

	if (use_stack)
		memcpy(&entry->caller, trace.entries,
		       trace.nr_entries * sizeof(unsigned long));
	else {
		trace.max_entries	= FTRACE_STACK_ENTRIES;
		trace.entries		= entry->caller;
		if (regs)
			save_stack_trace_regs(regs, &trace);
		else
			save_stack_trace(&trace);
	}

	entry->size = trace.nr_entries;
I
Ingo Molnar 已提交
1360

1361 1362
	if (!filter_check_discard(call, entry, buffer, event))
		ring_buffer_unlock_commit(buffer, event);
1363 1364 1365 1366 1367 1368 1369

 out:
	/* Again, don't let gcc optimize things here */
	barrier();
	__get_cpu_var(ftrace_stack_reserve)--;
	preempt_enable_notrace();

I
Ingo Molnar 已提交
1370 1371
}

1372 1373 1374 1375 1376 1377 1378 1379 1380
void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
			     int skip, int pc, struct pt_regs *regs)
{
	if (!(trace_flags & TRACE_ITER_STACKTRACE))
		return;

	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
}

1381 1382
void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
			int skip, int pc)
1383 1384 1385 1386
{
	if (!(trace_flags & TRACE_ITER_STACKTRACE))
		return;

1387
	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1388 1389
}

1390 1391
void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
		   int pc)
1392
{
1393
	__ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1394 1395
}

S
Steven Rostedt 已提交
1396 1397 1398 1399 1400 1401 1402 1403
/**
 * trace_dump_stack - record a stack back trace in the trace buffer
 */
void trace_dump_stack(void)
{
	unsigned long flags;

	if (tracing_disabled || tracing_selftest_running)
1404
		return;
S
Steven Rostedt 已提交
1405 1406 1407 1408

	local_save_flags(flags);

	/* skipping 3 traces, seems to get us at the caller of this function */
1409
	__ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
S
Steven Rostedt 已提交
1410 1411
}

1412 1413
static DEFINE_PER_CPU(int, user_stack_count);

1414 1415
void
ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1416
{
1417
	struct ftrace_event_call *call = &event_user_stack;
1418
	struct ring_buffer_event *event;
1419 1420 1421 1422 1423 1424
	struct userstack_entry *entry;
	struct stack_trace trace;

	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
		return;

1425 1426 1427 1428 1429 1430
	/*
	 * NMIs can not handle page faults, even with fix ups.
	 * The save user stack can (and often does) fault.
	 */
	if (unlikely(in_nmi()))
		return;
1431

1432 1433 1434 1435 1436 1437 1438 1439 1440 1441
	/*
	 * prevent recursion, since the user stack tracing may
	 * trigger other kernel events.
	 */
	preempt_disable();
	if (__this_cpu_read(user_stack_count))
		goto out;

	__this_cpu_inc(user_stack_count);

1442
	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1443
					  sizeof(*entry), flags, pc);
1444
	if (!event)
L
Li Zefan 已提交
1445
		goto out_drop_count;
1446 1447
	entry	= ring_buffer_event_data(event);

1448
	entry->tgid		= current->tgid;
1449 1450 1451 1452 1453 1454 1455 1456
	memset(&entry->caller, 0, sizeof(entry->caller));

	trace.nr_entries	= 0;
	trace.max_entries	= FTRACE_STACK_ENTRIES;
	trace.skip		= 0;
	trace.entries		= entry->caller;

	save_stack_trace_user(&trace);
1457 1458
	if (!filter_check_discard(call, entry, buffer, event))
		ring_buffer_unlock_commit(buffer, event);
1459

L
Li Zefan 已提交
1460
 out_drop_count:
1461 1462 1463
	__this_cpu_dec(user_stack_count);
 out:
	preempt_enable();
1464 1465
}

1466 1467
#ifdef UNUSED
static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1468
{
1469
	ftrace_trace_userstack(tr, flags, preempt_count());
1470
}
1471
#endif /* UNUSED */
1472

1473 1474
#endif /* CONFIG_STACKTRACE */

1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
/* created for use with alloc_percpu */
struct trace_buffer_struct {
	char buffer[TRACE_BUF_SIZE];
};

static struct trace_buffer_struct *trace_percpu_buffer;
static struct trace_buffer_struct *trace_percpu_sirq_buffer;
static struct trace_buffer_struct *trace_percpu_irq_buffer;
static struct trace_buffer_struct *trace_percpu_nmi_buffer;

/*
 * The buffer used is dependent on the context. There is a per cpu
 * buffer for normal context, softirq contex, hard irq context and
 * for NMI context. Thise allows for lockless recording.
 *
 * Note, if the buffers failed to be allocated, then this returns NULL
 */
static char *get_trace_buf(void)
{
	struct trace_buffer_struct *percpu_buffer;
	struct trace_buffer_struct *buffer;

	/*
	 * If we have allocated per cpu buffers, then we do not
	 * need to do any locking.
	 */
	if (in_nmi())
		percpu_buffer = trace_percpu_nmi_buffer;
	else if (in_irq())
		percpu_buffer = trace_percpu_irq_buffer;
	else if (in_softirq())
		percpu_buffer = trace_percpu_sirq_buffer;
	else
		percpu_buffer = trace_percpu_buffer;

	if (!percpu_buffer)
		return NULL;

	buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());

	return buffer->buffer;
}

static int alloc_percpu_trace_buffer(void)
{
	struct trace_buffer_struct *buffers;
	struct trace_buffer_struct *sirq_buffers;
	struct trace_buffer_struct *irq_buffers;
	struct trace_buffer_struct *nmi_buffers;

	buffers = alloc_percpu(struct trace_buffer_struct);
	if (!buffers)
		goto err_warn;

	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!sirq_buffers)
		goto err_sirq;

	irq_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!irq_buffers)
		goto err_irq;

	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!nmi_buffers)
		goto err_nmi;

	trace_percpu_buffer = buffers;
	trace_percpu_sirq_buffer = sirq_buffers;
	trace_percpu_irq_buffer = irq_buffers;
	trace_percpu_nmi_buffer = nmi_buffers;

	return 0;

 err_nmi:
	free_percpu(irq_buffers);
 err_irq:
	free_percpu(sirq_buffers);
 err_sirq:
	free_percpu(buffers);
 err_warn:
	WARN(1, "Could not allocate percpu trace_printk buffer");
	return -ENOMEM;
}

void trace_printk_init_buffers(void)
{
	static int buffers_allocated;

	if (buffers_allocated)
		return;

	if (alloc_percpu_trace_buffer())
		return;

	pr_info("ftrace: Allocated trace_printk buffers\n");

	buffers_allocated = 1;
}

1574
/**
1575
 * trace_vbprintk - write binary msg to tracing buffer
1576 1577
 *
 */
1578
int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1579
{
1580
	struct ftrace_event_call *call = &event_bprint;
1581
	struct ring_buffer_event *event;
1582
	struct ring_buffer *buffer;
1583
	struct trace_array *tr = &global_trace;
1584
	struct bprint_entry *entry;
1585
	unsigned long flags;
1586 1587
	char *tbuffer;
	int len = 0, size, pc;
1588 1589 1590 1591 1592 1593 1594 1595

	if (unlikely(tracing_selftest_running || tracing_disabled))
		return 0;

	/* Don't pollute graph traces with trace_vprintk internals */
	pause_graph_tracing();

	pc = preempt_count();
1596
	preempt_disable_notrace();
1597

1598 1599 1600
	tbuffer = get_trace_buf();
	if (!tbuffer) {
		len = 0;
1601
		goto out;
1602
	}
1603

1604
	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1605

1606 1607
	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
		goto out;
1608

1609
	local_save_flags(flags);
1610
	size = sizeof(*entry) + sizeof(u32) * len;
1611 1612 1613
	buffer = tr->buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
					  flags, pc);
1614
	if (!event)
1615
		goto out;
1616 1617 1618 1619
	entry = ring_buffer_event_data(event);
	entry->ip			= ip;
	entry->fmt			= fmt;

1620
	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1621
	if (!filter_check_discard(call, entry, buffer, event)) {
1622
		ring_buffer_unlock_commit(buffer, event);
1623 1624
		ftrace_trace_stack(buffer, flags, 6, pc);
	}
1625 1626

out:
1627
	preempt_enable_notrace();
1628 1629 1630 1631
	unpause_graph_tracing();

	return len;
}
1632 1633
EXPORT_SYMBOL_GPL(trace_vbprintk);

1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
int trace_array_printk(struct trace_array *tr,
		       unsigned long ip, const char *fmt, ...)
{
	int ret;
	va_list ap;

	if (!(trace_flags & TRACE_ITER_PRINTK))
		return 0;

	va_start(ap, fmt);
	ret = trace_array_vprintk(tr, ip, fmt, ap);
	va_end(ap);
	return ret;
}

int trace_array_vprintk(struct trace_array *tr,
			unsigned long ip, const char *fmt, va_list args)
1651
{
1652
	struct ftrace_event_call *call = &event_print;
1653
	struct ring_buffer_event *event;
1654
	struct ring_buffer *buffer;
1655
	int len = 0, size, pc;
1656
	struct print_entry *entry;
1657 1658
	unsigned long flags;
	char *tbuffer;
1659 1660 1661 1662

	if (tracing_disabled || tracing_selftest_running)
		return 0;

1663 1664 1665
	/* Don't pollute graph traces with trace_vprintk internals */
	pause_graph_tracing();

1666 1667 1668
	pc = preempt_count();
	preempt_disable_notrace();

1669 1670 1671 1672

	tbuffer = get_trace_buf();
	if (!tbuffer) {
		len = 0;
1673
		goto out;
1674
	}
1675

1676 1677 1678
	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
	if (len > TRACE_BUF_SIZE)
		goto out;
1679

1680
	local_save_flags(flags);
1681
	size = sizeof(*entry) + len + 1;
1682 1683
	buffer = tr->buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1684
					  flags, pc);
1685
	if (!event)
1686
		goto out;
1687
	entry = ring_buffer_event_data(event);
C
Carsten Emde 已提交
1688
	entry->ip = ip;
1689

1690
	memcpy(&entry->buf, tbuffer, len);
C
Carsten Emde 已提交
1691
	entry->buf[len] = '\0';
1692
	if (!filter_check_discard(call, entry, buffer, event)) {
1693
		ring_buffer_unlock_commit(buffer, event);
1694
		ftrace_trace_stack(buffer, flags, 6, pc);
1695
	}
1696 1697
 out:
	preempt_enable_notrace();
1698
	unpause_graph_tracing();
1699 1700 1701

	return len;
}
1702 1703 1704

int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
{
S
Steven Rostedt 已提交
1705
	return trace_array_vprintk(&global_trace, ip, fmt, args);
1706
}
1707 1708
EXPORT_SYMBOL_GPL(trace_vprintk);

1709
static void trace_iterator_increment(struct trace_iterator *iter)
S
Steven Rostedt 已提交
1710 1711
{
	iter->idx++;
1712 1713
	if (iter->buffer_iter[iter->cpu])
		ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
S
Steven Rostedt 已提交
1714 1715
}

I
Ingo Molnar 已提交
1716
static struct trace_entry *
1717 1718
peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
		unsigned long *lost_events)
1719
{
1720 1721
	struct ring_buffer_event *event;
	struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1722

1723 1724 1725
	if (buf_iter)
		event = ring_buffer_iter_peek(buf_iter, ts);
	else
1726 1727
		event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
					 lost_events);
1728

1729 1730 1731 1732 1733 1734
	if (event) {
		iter->ent_size = ring_buffer_event_length(event);
		return ring_buffer_event_data(event);
	}
	iter->ent_size = 0;
	return NULL;
1735
}
1736

1737
static struct trace_entry *
1738 1739
__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
		  unsigned long *missing_events, u64 *ent_ts)
1740
{
1741
	struct ring_buffer *buffer = iter->tr->buffer;
1742
	struct trace_entry *ent, *next = NULL;
1743
	unsigned long lost_events = 0, next_lost = 0;
1744
	int cpu_file = iter->cpu_file;
1745
	u64 next_ts = 0, ts;
1746
	int next_cpu = -1;
1747
	int next_size = 0;
1748 1749
	int cpu;

1750 1751 1752 1753 1754 1755 1756
	/*
	 * If we are in a per_cpu trace file, don't bother by iterating over
	 * all cpu and peek directly.
	 */
	if (cpu_file > TRACE_PIPE_ALL_CPU) {
		if (ring_buffer_empty_cpu(buffer, cpu_file))
			return NULL;
1757
		ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1758 1759 1760 1761 1762 1763
		if (ent_cpu)
			*ent_cpu = cpu_file;

		return ent;
	}

1764
	for_each_tracing_cpu(cpu) {
1765

1766 1767
		if (ring_buffer_empty_cpu(buffer, cpu))
			continue;
1768

1769
		ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1770

I
Ingo Molnar 已提交
1771 1772 1773
		/*
		 * Pick the entry with the smallest timestamp:
		 */
1774
		if (ent && (!next || ts < next_ts)) {
1775 1776
			next = ent;
			next_cpu = cpu;
1777
			next_ts = ts;
1778
			next_lost = lost_events;
1779
			next_size = iter->ent_size;
1780 1781 1782
		}
	}

1783 1784
	iter->ent_size = next_size;

1785 1786 1787
	if (ent_cpu)
		*ent_cpu = next_cpu;

1788 1789 1790
	if (ent_ts)
		*ent_ts = next_ts;

1791 1792 1793
	if (missing_events)
		*missing_events = next_lost;

1794 1795 1796
	return next;
}

1797
/* Find the next real entry, without updating the iterator itself */
1798 1799
struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
					  int *ent_cpu, u64 *ent_ts)
1800
{
1801
	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1802 1803 1804
}

/* Find the next real entry, and increment the iterator to the next entry */
1805
void *trace_find_next_entry_inc(struct trace_iterator *iter)
1806
{
1807 1808
	iter->ent = __find_next_entry(iter, &iter->cpu,
				      &iter->lost_events, &iter->ts);
1809

1810
	if (iter->ent)
1811
		trace_iterator_increment(iter);
1812

1813
	return iter->ent ? iter : NULL;
1814
}
1815

I
Ingo Molnar 已提交
1816
static void trace_consume(struct trace_iterator *iter)
1817
{
1818 1819
	ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
			    &iter->lost_events);
1820 1821
}

I
Ingo Molnar 已提交
1822
static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1823 1824 1825
{
	struct trace_iterator *iter = m->private;
	int i = (int)*pos;
I
Ingo Molnar 已提交
1826
	void *ent;
1827

1828 1829
	WARN_ON_ONCE(iter->leftover);

1830 1831 1832 1833 1834 1835 1836
	(*pos)++;

	/* can't go backwards */
	if (iter->idx > i)
		return NULL;

	if (iter->idx < 0)
1837
		ent = trace_find_next_entry_inc(iter);
1838 1839 1840 1841
	else
		ent = iter;

	while (ent && iter->idx < i)
1842
		ent = trace_find_next_entry_inc(iter);
1843 1844 1845 1846 1847 1848

	iter->pos = *pos;

	return ent;
}

1849
void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
{
	struct trace_array *tr = iter->tr;
	struct ring_buffer_event *event;
	struct ring_buffer_iter *buf_iter;
	unsigned long entries = 0;
	u64 ts;

	tr->data[cpu]->skipped_entries = 0;

	if (!iter->buffer_iter[cpu])
		return;

	buf_iter = iter->buffer_iter[cpu];
	ring_buffer_iter_reset(buf_iter);

	/*
	 * We could have the case with the max latency tracers
	 * that a reset never took place on a cpu. This is evident
	 * by the timestamp being before the start of the buffer.
	 */
	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
		if (ts >= iter->tr->time_start)
			break;
		entries++;
		ring_buffer_read(buf_iter, NULL);
	}

	tr->data[cpu]->skipped_entries = entries;
}

1880 1881 1882 1883
/*
 * The current tracer is copied to avoid a global locking
 * all around.
 */
1884 1885 1886
static void *s_start(struct seq_file *m, loff_t *pos)
{
	struct trace_iterator *iter = m->private;
1887
	static struct tracer *old_tracer;
1888
	int cpu_file = iter->cpu_file;
1889 1890
	void *p = NULL;
	loff_t l = 0;
1891
	int cpu;
1892

1893
	/* copy the tracer to avoid using a global lock all around */
1894
	mutex_lock(&trace_types_lock);
1895 1896 1897
	if (unlikely(old_tracer != current_trace && current_trace)) {
		old_tracer = current_trace;
		*iter->trace = *current_trace;
1898
	}
1899
	mutex_unlock(&trace_types_lock);
1900 1901 1902 1903 1904 1905 1906 1907

	atomic_inc(&trace_record_cmdline_disabled);

	if (*pos != iter->pos) {
		iter->ent = NULL;
		iter->cpu = 0;
		iter->idx = -1;

1908 1909
		if (cpu_file == TRACE_PIPE_ALL_CPU) {
			for_each_tracing_cpu(cpu)
1910
				tracing_iter_reset(iter, cpu);
1911
		} else
1912
			tracing_iter_reset(iter, cpu_file);
1913

1914
		iter->leftover = 0;
1915 1916 1917 1918
		for (p = iter; p && l < *pos; p = s_next(m, p, &l))
			;

	} else {
1919 1920 1921 1922 1923 1924 1925 1926 1927 1928
		/*
		 * If we overflowed the seq_file before, then we want
		 * to just reuse the trace_seq buffer again.
		 */
		if (iter->leftover)
			p = iter;
		else {
			l = *pos - 1;
			p = s_next(m, p, &l);
		}
1929 1930
	}

1931
	trace_event_read_lock();
1932
	trace_access_lock(cpu_file);
1933 1934 1935 1936 1937
	return p;
}

static void s_stop(struct seq_file *m, void *p)
{
1938 1939
	struct trace_iterator *iter = m->private;

1940
	atomic_dec(&trace_record_cmdline_disabled);
1941
	trace_access_unlock(iter->cpu_file);
1942
	trace_event_read_unlock();
1943 1944
}

1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
static void
get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
{
	unsigned long count;
	int cpu;

	*total = 0;
	*entries = 0;

	for_each_tracing_cpu(cpu) {
		count = ring_buffer_entries_cpu(tr->buffer, cpu);
		/*
		 * If this buffer has skipped entries, then we hold all
		 * entries for the trace and we need to ignore the
		 * ones before the time stamp.
		 */
		if (tr->data[cpu]->skipped_entries) {
			count -= tr->data[cpu]->skipped_entries;
			/* total is the same as the entries */
			*total += count;
		} else
			*total += count +
				ring_buffer_overrun_cpu(tr->buffer, cpu);
		*entries += count;
	}
}

I
Ingo Molnar 已提交
1972
static void print_lat_help_header(struct seq_file *m)
1973
{
1974 1975 1976 1977 1978
	seq_puts(m, "#                  _------=> CPU#            \n");
	seq_puts(m, "#                 / _-----=> irqs-off        \n");
	seq_puts(m, "#                | / _----=> need-resched    \n");
	seq_puts(m, "#                || / _---=> hardirq/softirq \n");
	seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1979 1980 1981
	seq_puts(m, "#                |||| /     delay             \n");
	seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
	seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
1982 1983
}

1984
static void print_event_info(struct trace_array *tr, struct seq_file *m)
1985
{
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
	unsigned long total;
	unsigned long entries;

	get_total_entries(tr, &total, &entries);
	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
		   entries, total, num_online_cpus());
	seq_puts(m, "#\n");
}

static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
{
	print_event_info(tr, m);
1998
	seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
1999
	seq_puts(m, "#              | |       |          |         |\n");
2000 2001
}

2002
static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2003
{
2004
	print_event_info(tr, m);
2005 2006 2007 2008 2009 2010 2011 2012
	seq_puts(m, "#                              _-----=> irqs-off\n");
	seq_puts(m, "#                             / _----=> need-resched\n");
	seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
	seq_puts(m, "#                            || / _--=> preempt-depth\n");
	seq_puts(m, "#                            ||| /     delay\n");
	seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
	seq_puts(m, "#              | |       |   ||||       |         |\n");
}
2013

2014
void
2015 2016 2017 2018 2019 2020
print_trace_header(struct seq_file *m, struct trace_iterator *iter)
{
	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
	struct trace_array *tr = iter->tr;
	struct trace_array_cpu *data = tr->data[tr->cpu];
	struct tracer *type = current_trace;
2021 2022
	unsigned long entries;
	unsigned long total;
2023 2024 2025 2026 2027
	const char *name = "preemption";

	if (type)
		name = type->name;

2028
	get_total_entries(tr, &total, &entries);
2029

2030
	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2031
		   name, UTS_RELEASE);
2032
	seq_puts(m, "# -----------------------------------"
2033
		 "---------------------------------\n");
2034
	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2035
		   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
S
Steven Rostedt 已提交
2036
		   nsecs_to_usecs(data->saved_latency),
2037
		   entries,
2038
		   total,
2039 2040 2041 2042 2043
		   tr->cpu,
#if defined(CONFIG_PREEMPT_NONE)
		   "server",
#elif defined(CONFIG_PREEMPT_VOLUNTARY)
		   "desktop",
2044
#elif defined(CONFIG_PREEMPT)
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055
		   "preempt",
#else
		   "unknown",
#endif
		   /* These are reserved for later use */
		   0, 0, 0, 0);
#ifdef CONFIG_SMP
	seq_printf(m, " #P:%d)\n", num_online_cpus());
#else
	seq_puts(m, ")\n");
#endif
2056 2057
	seq_puts(m, "#    -----------------\n");
	seq_printf(m, "#    | task: %.16s-%d "
2058 2059 2060
		   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
		   data->comm, data->pid, data->uid, data->nice,
		   data->policy, data->rt_priority);
2061
	seq_puts(m, "#    -----------------\n");
2062 2063

	if (data->critical_start) {
2064
		seq_puts(m, "#  => started at: ");
S
Steven Rostedt 已提交
2065 2066
		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
		trace_print_seq(m, &iter->seq);
2067
		seq_puts(m, "\n#  => ended at:   ");
S
Steven Rostedt 已提交
2068 2069
		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
		trace_print_seq(m, &iter->seq);
2070
		seq_puts(m, "\n#\n");
2071 2072
	}

2073
	seq_puts(m, "#\n");
2074 2075
}

2076 2077 2078 2079
static void test_cpu_buff_start(struct trace_iterator *iter)
{
	struct trace_seq *s = &iter->seq;

2080 2081 2082 2083 2084 2085
	if (!(trace_flags & TRACE_ITER_ANNOTATE))
		return;

	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
		return;

2086
	if (cpumask_test_cpu(iter->cpu, iter->started))
2087 2088
		return;

2089 2090 2091
	if (iter->tr->data[iter->cpu]->skipped_entries)
		return;

2092
	cpumask_set_cpu(iter->cpu, iter->started);
2093 2094 2095 2096 2097

	/* Don't print started cpu buffer for the first entry of the trace */
	if (iter->idx > 1)
		trace_seq_printf(s, "##### CPU %u buffer started ####\n",
				iter->cpu);
2098 2099
}

2100
static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2101
{
S
Steven Rostedt 已提交
2102
	struct trace_seq *s = &iter->seq;
2103
	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
I
Ingo Molnar 已提交
2104
	struct trace_entry *entry;
2105
	struct trace_event *event;
2106

I
Ingo Molnar 已提交
2107
	entry = iter->ent;
2108

2109 2110
	test_cpu_buff_start(iter);

2111
	event = ftrace_find_event(entry->type);
2112

2113
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2114 2115 2116 2117 2118 2119 2120
		if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
			if (!trace_print_lat_context(iter))
				goto partial;
		} else {
			if (!trace_print_context(iter))
				goto partial;
		}
2121
	}
2122

2123
	if (event)
2124
		return event->funcs->trace(iter, sym_flags, event);
2125 2126 2127

	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
		goto partial;
2128

2129
	return TRACE_TYPE_HANDLED;
2130 2131
partial:
	return TRACE_TYPE_PARTIAL_LINE;
2132 2133
}

2134
static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2135 2136 2137
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *entry;
2138
	struct trace_event *event;
I
Ingo Molnar 已提交
2139 2140

	entry = iter->ent;
2141

2142
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2143 2144 2145
		if (!trace_seq_printf(s, "%d %d %llu ",
				      entry->pid, iter->cpu, iter->ts))
			goto partial;
2146
	}
I
Ingo Molnar 已提交
2147

2148
	event = ftrace_find_event(entry->type);
2149
	if (event)
2150
		return event->funcs->raw(iter, 0, event);
2151 2152 2153

	if (!trace_seq_printf(s, "%d ?\n", entry->type))
		goto partial;
2154

2155
	return TRACE_TYPE_HANDLED;
2156 2157
partial:
	return TRACE_TYPE_PARTIAL_LINE;
I
Ingo Molnar 已提交
2158 2159
}

2160
static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2161 2162 2163 2164
{
	struct trace_seq *s = &iter->seq;
	unsigned char newline = '\n';
	struct trace_entry *entry;
2165
	struct trace_event *event;
2166 2167

	entry = iter->ent;
2168

2169 2170 2171 2172 2173
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
		SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
		SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
		SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
	}
2174

2175
	event = ftrace_find_event(entry->type);
2176
	if (event) {
2177
		enum print_line_t ret = event->funcs->hex(iter, 0, event);
2178 2179 2180
		if (ret != TRACE_TYPE_HANDLED)
			return ret;
	}
S
Steven Rostedt 已提交
2181

2182 2183
	SEQ_PUT_FIELD_RET(s, newline);

2184
	return TRACE_TYPE_HANDLED;
2185 2186
}

2187
static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2188 2189 2190
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *entry;
2191
	struct trace_event *event;
I
Ingo Molnar 已提交
2192 2193

	entry = iter->ent;
2194

2195 2196
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
		SEQ_PUT_FIELD_RET(s, entry->pid);
2197
		SEQ_PUT_FIELD_RET(s, iter->cpu);
2198 2199
		SEQ_PUT_FIELD_RET(s, iter->ts);
	}
I
Ingo Molnar 已提交
2200

2201
	event = ftrace_find_event(entry->type);
2202 2203
	return event ? event->funcs->binary(iter, 0, event) :
		TRACE_TYPE_HANDLED;
I
Ingo Molnar 已提交
2204 2205
}

2206
int trace_empty(struct trace_iterator *iter)
2207 2208 2209
{
	int cpu;

2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222
	/* If we are looking at one CPU buffer, only check that one */
	if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
		cpu = iter->cpu_file;
		if (iter->buffer_iter[cpu]) {
			if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
				return 0;
		} else {
			if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
				return 0;
		}
		return 1;
	}

2223
	for_each_tracing_cpu(cpu) {
2224 2225 2226 2227 2228 2229 2230
		if (iter->buffer_iter[cpu]) {
			if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
				return 0;
		} else {
			if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
				return 0;
		}
2231
	}
2232

2233
	return 1;
2234 2235
}

2236
/*  Called with trace_event_read_lock() held. */
2237
enum print_line_t print_trace_line(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2238
{
2239 2240
	enum print_line_t ret;

2241 2242 2243 2244
	if (iter->lost_events &&
	    !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
				 iter->cpu, iter->lost_events))
		return TRACE_TYPE_PARTIAL_LINE;
2245

2246 2247 2248 2249 2250
	if (iter->trace && iter->trace->print_line) {
		ret = iter->trace->print_line(iter);
		if (ret != TRACE_TYPE_UNHANDLED)
			return ret;
	}
2251

2252 2253 2254
	if (iter->ent->type == TRACE_BPRINT &&
			trace_flags & TRACE_ITER_PRINTK &&
			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2255
		return trace_print_bprintk_msg_only(iter);
2256

2257 2258 2259
	if (iter->ent->type == TRACE_PRINT &&
			trace_flags & TRACE_ITER_PRINTK &&
			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2260
		return trace_print_printk_msg_only(iter);
2261

I
Ingo Molnar 已提交
2262 2263 2264
	if (trace_flags & TRACE_ITER_BIN)
		return print_bin_fmt(iter);

2265 2266 2267
	if (trace_flags & TRACE_ITER_HEX)
		return print_hex_fmt(iter);

I
Ingo Molnar 已提交
2268 2269 2270 2271 2272 2273
	if (trace_flags & TRACE_ITER_RAW)
		return print_raw_fmt(iter);

	return print_trace_fmt(iter);
}

2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288
void trace_latency_header(struct seq_file *m)
{
	struct trace_iterator *iter = m->private;

	/* print nothing if the buffers are empty */
	if (trace_empty(iter))
		return;

	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
		print_trace_header(m, iter);

	if (!(trace_flags & TRACE_ITER_VERBOSE))
		print_lat_help_header(m);
}

2289 2290 2291 2292
void trace_default_header(struct seq_file *m)
{
	struct trace_iterator *iter = m->private;

2293 2294 2295
	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
		return;

2296 2297 2298 2299 2300 2301 2302 2303
	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
		/* print nothing if the buffers are empty */
		if (trace_empty(iter))
			return;
		print_trace_header(m, iter);
		if (!(trace_flags & TRACE_ITER_VERBOSE))
			print_lat_help_header(m);
	} else {
2304 2305
		if (!(trace_flags & TRACE_ITER_VERBOSE)) {
			if (trace_flags & TRACE_ITER_IRQ_INFO)
2306
				print_func_help_header_irq(iter->tr, m);
2307
			else
2308
				print_func_help_header(iter->tr, m);
2309
		}
2310 2311 2312
	}
}

2313 2314 2315 2316 2317 2318 2319 2320
static void test_ftrace_alive(struct seq_file *m)
{
	if (!ftrace_is_dead())
		return;
	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
	seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
}

2321 2322 2323
static int s_show(struct seq_file *m, void *v)
{
	struct trace_iterator *iter = v;
2324
	int ret;
2325 2326 2327 2328 2329

	if (iter->ent == NULL) {
		if (iter->tr) {
			seq_printf(m, "# tracer: %s\n", iter->trace->name);
			seq_puts(m, "#\n");
2330
			test_ftrace_alive(m);
2331
		}
2332 2333
		if (iter->trace && iter->trace->print_header)
			iter->trace->print_header(m);
2334 2335 2336
		else
			trace_default_header(m);

2337 2338 2339 2340 2341 2342 2343 2344 2345 2346
	} else if (iter->leftover) {
		/*
		 * If we filled the seq_file buffer earlier, we
		 * want to just show it now.
		 */
		ret = trace_print_seq(m, &iter->seq);

		/* ret should this time be zero, but you never know */
		iter->leftover = ret;

2347
	} else {
I
Ingo Molnar 已提交
2348
		print_trace_line(iter);
2349 2350 2351 2352 2353 2354 2355 2356 2357
		ret = trace_print_seq(m, &iter->seq);
		/*
		 * If we overflow the seq_file buffer, then it will
		 * ask us for this data again at start up.
		 * Use that instead.
		 *  ret is 0 if seq_file write succeeded.
		 *        -1 otherwise.
		 */
		iter->leftover = ret;
2358 2359 2360 2361 2362
	}

	return 0;
}

J
James Morris 已提交
2363
static const struct seq_operations tracer_seq_ops = {
I
Ingo Molnar 已提交
2364 2365 2366 2367
	.start		= s_start,
	.next		= s_next,
	.stop		= s_stop,
	.show		= s_show,
2368 2369
};

I
Ingo Molnar 已提交
2370
static struct trace_iterator *
2371
__tracing_open(struct inode *inode, struct file *file)
2372
{
2373
	long cpu_file = (long) inode->i_private;
2374
	struct trace_iterator *iter;
2375
	int cpu;
2376

2377 2378
	if (tracing_disabled)
		return ERR_PTR(-ENODEV);
S
Steven Rostedt 已提交
2379

2380
	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2381 2382
	if (!iter)
		return ERR_PTR(-ENOMEM);
2383

2384 2385 2386 2387
	/*
	 * We make a copy of the current tracer to avoid concurrent
	 * changes on it while we are reading.
	 */
2388
	mutex_lock(&trace_types_lock);
2389
	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2390
	if (!iter->trace)
2391
		goto fail;
2392

2393 2394 2395
	if (current_trace)
		*iter->trace = *current_trace;

2396
	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2397 2398
		goto fail;

2399 2400 2401
	if (current_trace && current_trace->print_max)
		iter->tr = &max_tr;
	else
2402
		iter->tr = &global_trace;
2403
	iter->pos = -1;
2404
	mutex_init(&iter->mutex);
2405
	iter->cpu_file = cpu_file;
2406

2407 2408
	/* Notify the tracer early; before we stop tracing. */
	if (iter->trace && iter->trace->open)
2409
		iter->trace->open(iter);
2410

2411 2412 2413 2414
	/* Annotate start of buffers if we had overruns */
	if (ring_buffer_overruns(iter->tr->buffer))
		iter->iter_flags |= TRACE_FILE_ANNOTATE;

2415 2416 2417
	/* stop the trace while dumping */
	tracing_stop();

2418 2419 2420
	if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
		for_each_tracing_cpu(cpu) {
			iter->buffer_iter[cpu] =
2421 2422 2423 2424 2425
				ring_buffer_read_prepare(iter->tr->buffer, cpu);
		}
		ring_buffer_read_prepare_sync();
		for_each_tracing_cpu(cpu) {
			ring_buffer_read_start(iter->buffer_iter[cpu]);
2426
			tracing_iter_reset(iter, cpu);
2427 2428 2429
		}
	} else {
		cpu = iter->cpu_file;
2430
		iter->buffer_iter[cpu] =
2431 2432 2433
			ring_buffer_read_prepare(iter->tr->buffer, cpu);
		ring_buffer_read_prepare_sync();
		ring_buffer_read_start(iter->buffer_iter[cpu]);
2434
		tracing_iter_reset(iter, cpu);
2435 2436
	}

2437 2438 2439
	mutex_unlock(&trace_types_lock);

	return iter;
2440

2441
 fail:
2442
	mutex_unlock(&trace_types_lock);
2443
	kfree(iter->trace);
2444 2445
	seq_release_private(inode, file);
	return ERR_PTR(-ENOMEM);
2446 2447 2448 2449
}

int tracing_open_generic(struct inode *inode, struct file *filp)
{
S
Steven Rostedt 已提交
2450 2451 2452
	if (tracing_disabled)
		return -ENODEV;

2453 2454 2455 2456
	filp->private_data = inode->i_private;
	return 0;
}

2457
static int tracing_release(struct inode *inode, struct file *file)
2458
{
2459
	struct seq_file *m = file->private_data;
2460
	struct trace_iterator *iter;
2461
	int cpu;
2462

2463 2464 2465 2466 2467
	if (!(file->f_mode & FMODE_READ))
		return 0;

	iter = m->private;

2468
	mutex_lock(&trace_types_lock);
2469 2470 2471 2472 2473
	for_each_tracing_cpu(cpu) {
		if (iter->buffer_iter[cpu])
			ring_buffer_read_finish(iter->buffer_iter[cpu]);
	}

2474 2475 2476 2477
	if (iter->trace && iter->trace->close)
		iter->trace->close(iter);

	/* reenable tracing if it was previously enabled */
2478
	tracing_start();
2479 2480
	mutex_unlock(&trace_types_lock);

2481
	mutex_destroy(&iter->mutex);
2482
	free_cpumask_var(iter->started);
2483
	kfree(iter->trace);
2484
	seq_release_private(inode, file);
2485 2486 2487 2488 2489
	return 0;
}

static int tracing_open(struct inode *inode, struct file *file)
{
2490 2491
	struct trace_iterator *iter;
	int ret = 0;
2492

2493 2494
	/* If this file was open for write, then erase contents */
	if ((file->f_mode & FMODE_WRITE) &&
2495
	    (file->f_flags & O_TRUNC)) {
2496
		long cpu = (long) inode->i_private;
2497

2498 2499 2500 2501 2502
		if (cpu == TRACE_PIPE_ALL_CPU)
			tracing_reset_online_cpus(&global_trace);
		else
			tracing_reset(&global_trace, cpu);
	}
2503

2504 2505 2506 2507 2508 2509 2510
	if (file->f_mode & FMODE_READ) {
		iter = __tracing_open(inode, file);
		if (IS_ERR(iter))
			ret = PTR_ERR(iter);
		else if (trace_flags & TRACE_ITER_LATENCY_FMT)
			iter->iter_flags |= TRACE_FILE_LAT_FMT;
	}
2511 2512 2513
	return ret;
}

I
Ingo Molnar 已提交
2514
static void *
2515 2516
t_next(struct seq_file *m, void *v, loff_t *pos)
{
L
Li Zefan 已提交
2517
	struct tracer *t = v;
2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528

	(*pos)++;

	if (t)
		t = t->next;

	return t;
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
L
Li Zefan 已提交
2529
	struct tracer *t;
2530 2531 2532
	loff_t l = 0;

	mutex_lock(&trace_types_lock);
L
Li Zefan 已提交
2533
	for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
		;

	return t;
}

static void t_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&trace_types_lock);
}

static int t_show(struct seq_file *m, void *v)
{
	struct tracer *t = v;

	if (!t)
		return 0;

	seq_printf(m, "%s", t->name);
	if (t->next)
		seq_putc(m, ' ');
	else
		seq_putc(m, '\n');

	return 0;
}

J
James Morris 已提交
2560
static const struct seq_operations show_traces_seq_ops = {
I
Ingo Molnar 已提交
2561 2562 2563 2564
	.start		= t_start,
	.next		= t_next,
	.stop		= t_stop,
	.show		= t_show,
2565 2566 2567 2568
};

static int show_traces_open(struct inode *inode, struct file *file)
{
S
Steven Rostedt 已提交
2569 2570 2571
	if (tracing_disabled)
		return -ENODEV;

L
Li Zefan 已提交
2572
	return seq_open(file, &show_traces_seq_ops);
2573 2574
}

2575 2576 2577 2578 2579 2580 2581
static ssize_t
tracing_write_stub(struct file *filp, const char __user *ubuf,
		   size_t count, loff_t *ppos)
{
	return count;
}

2582 2583 2584 2585 2586 2587 2588 2589
static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
{
	if (file->f_mode & FMODE_READ)
		return seq_lseek(file, offset, origin);
	else
		return 0;
}

2590
static const struct file_operations tracing_fops = {
I
Ingo Molnar 已提交
2591 2592
	.open		= tracing_open,
	.read		= seq_read,
2593
	.write		= tracing_write_stub,
2594
	.llseek		= tracing_seek,
I
Ingo Molnar 已提交
2595
	.release	= tracing_release,
2596 2597
};

2598
static const struct file_operations show_traces_fops = {
I
Ingo Molnar 已提交
2599 2600 2601
	.open		= show_traces_open,
	.read		= seq_read,
	.release	= seq_release,
2602
	.llseek		= seq_lseek,
I
Ingo Molnar 已提交
2603 2604
};

2605 2606 2607
/*
 * Only trace on a CPU if the bitmask is set:
 */
2608
static cpumask_var_t tracing_cpumask;
2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621

/*
 * The tracer itself will not take this lock, but still we want
 * to provide a consistent cpumask to user-space:
 */
static DEFINE_MUTEX(tracing_cpumask_update_lock);

/*
 * Temporary storage for the character representation of the
 * CPU bitmask (and one more byte for the newline):
 */
static char mask_str[NR_CPUS + 1];

I
Ingo Molnar 已提交
2622 2623 2624 2625
static ssize_t
tracing_cpumask_read(struct file *filp, char __user *ubuf,
		     size_t count, loff_t *ppos)
{
2626
	int len;
I
Ingo Molnar 已提交
2627 2628

	mutex_lock(&tracing_cpumask_update_lock);
2629

2630
	len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2631 2632 2633 2634 2635 2636 2637 2638
	if (count - len < 2) {
		count = -EINVAL;
		goto out_err;
	}
	len += sprintf(mask_str + len, "\n");
	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);

out_err:
I
Ingo Molnar 已提交
2639 2640 2641 2642 2643 2644 2645 2646 2647
	mutex_unlock(&tracing_cpumask_update_lock);

	return count;
}

static ssize_t
tracing_cpumask_write(struct file *filp, const char __user *ubuf,
		      size_t count, loff_t *ppos)
{
2648
	int err, cpu;
2649 2650 2651 2652
	cpumask_var_t tracing_cpumask_new;

	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
		return -ENOMEM;
I
Ingo Molnar 已提交
2653

2654
	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
I
Ingo Molnar 已提交
2655
	if (err)
2656 2657
		goto err_unlock;

2658 2659
	mutex_lock(&tracing_cpumask_update_lock);

2660
	local_irq_disable();
2661
	arch_spin_lock(&ftrace_max_lock);
2662
	for_each_tracing_cpu(cpu) {
2663 2664 2665 2666
		/*
		 * Increase/decrease the disabled counter if we are
		 * about to flip a bit in the cpumask:
		 */
2667 2668
		if (cpumask_test_cpu(cpu, tracing_cpumask) &&
				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2669
			atomic_inc(&global_trace.data[cpu]->disabled);
2670
			ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2671
		}
2672 2673
		if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2674
			atomic_dec(&global_trace.data[cpu]->disabled);
2675
			ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2676 2677
		}
	}
2678
	arch_spin_unlock(&ftrace_max_lock);
2679
	local_irq_enable();
2680

2681
	cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2682 2683

	mutex_unlock(&tracing_cpumask_update_lock);
2684
	free_cpumask_var(tracing_cpumask_new);
I
Ingo Molnar 已提交
2685 2686

	return count;
2687 2688

err_unlock:
2689
	free_cpumask_var(tracing_cpumask_new);
2690 2691

	return err;
I
Ingo Molnar 已提交
2692 2693
}

2694
static const struct file_operations tracing_cpumask_fops = {
I
Ingo Molnar 已提交
2695 2696 2697
	.open		= tracing_open_generic,
	.read		= tracing_cpumask_read,
	.write		= tracing_cpumask_write,
2698
	.llseek		= generic_file_llseek,
2699 2700
};

L
Li Zefan 已提交
2701
static int tracing_trace_options_show(struct seq_file *m, void *v)
2702
{
2703 2704 2705
	struct tracer_opt *trace_opts;
	u32 tracer_flags;
	int i;
2706

2707 2708 2709 2710
	mutex_lock(&trace_types_lock);
	tracer_flags = current_trace->flags->val;
	trace_opts = current_trace->flags->opts;

2711 2712
	for (i = 0; trace_options[i]; i++) {
		if (trace_flags & (1 << i))
L
Li Zefan 已提交
2713
			seq_printf(m, "%s\n", trace_options[i]);
2714
		else
L
Li Zefan 已提交
2715
			seq_printf(m, "no%s\n", trace_options[i]);
2716 2717
	}

2718 2719
	for (i = 0; trace_opts[i].name; i++) {
		if (tracer_flags & trace_opts[i].bit)
L
Li Zefan 已提交
2720
			seq_printf(m, "%s\n", trace_opts[i].name);
2721
		else
L
Li Zefan 已提交
2722
			seq_printf(m, "no%s\n", trace_opts[i].name);
2723
	}
2724
	mutex_unlock(&trace_types_lock);
2725

L
Li Zefan 已提交
2726
	return 0;
2727 2728
}

L
Li Zefan 已提交
2729 2730 2731 2732 2733
static int __set_tracer_option(struct tracer *trace,
			       struct tracer_flags *tracer_flags,
			       struct tracer_opt *opts, int neg)
{
	int ret;
2734

L
Li Zefan 已提交
2735 2736 2737 2738 2739 2740 2741 2742 2743
	ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
	if (ret)
		return ret;

	if (neg)
		tracer_flags->val &= ~opts->bit;
	else
		tracer_flags->val |= opts->bit;
	return 0;
2744 2745
}

2746 2747 2748
/* Try to assign a tracer specific option */
static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
{
2749
	struct tracer_flags *tracer_flags = trace->flags;
2750
	struct tracer_opt *opts = NULL;
L
Li Zefan 已提交
2751
	int i;
2752

2753 2754
	for (i = 0; tracer_flags->opts[i].name; i++) {
		opts = &tracer_flags->opts[i];
2755

L
Li Zefan 已提交
2756 2757 2758
		if (strcmp(cmp, opts->name) == 0)
			return __set_tracer_option(trace, trace->flags,
						   opts, neg);
2759 2760
	}

L
Li Zefan 已提交
2761
	return -EINVAL;
2762 2763
}

2764 2765 2766 2767 2768 2769 2770 2771 2772 2773
static void set_tracer_flags(unsigned int mask, int enabled)
{
	/* do nothing if flag is already set */
	if (!!(trace_flags & mask) == !!enabled)
		return;

	if (enabled)
		trace_flags |= mask;
	else
		trace_flags &= ~mask;
2774 2775 2776

	if (mask == TRACE_ITER_RECORD_CMD)
		trace_event_enable_cmd_record(enabled);
2777 2778 2779

	if (mask == TRACE_ITER_OVERWRITE)
		ring_buffer_change_overwrite(global_trace.buffer, enabled);
2780 2781
}

2782
static ssize_t
2783
tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2784 2785 2786
			size_t cnt, loff_t *ppos)
{
	char buf[64];
L
Li Zefan 已提交
2787
	char *cmp;
2788
	int neg = 0;
2789
	int ret;
2790 2791
	int i;

S
Steven Rostedt 已提交
2792 2793
	if (cnt >= sizeof(buf))
		return -EINVAL;
2794 2795 2796 2797 2798

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;
L
Li Zefan 已提交
2799
	cmp = strstrip(buf);
2800

L
Li Zefan 已提交
2801
	if (strncmp(cmp, "no", 2) == 0) {
2802 2803 2804 2805 2806
		neg = 1;
		cmp += 2;
	}

	for (i = 0; trace_options[i]; i++) {
L
Li Zefan 已提交
2807
		if (strcmp(cmp, trace_options[i]) == 0) {
2808
			set_tracer_flags(1 << i, !neg);
2809 2810 2811
			break;
		}
	}
2812 2813 2814

	/* If no option could be set, test the specific tracer options */
	if (!trace_options[i]) {
2815
		mutex_lock(&trace_types_lock);
2816
		ret = set_tracer_option(current_trace, cmp, neg);
2817
		mutex_unlock(&trace_types_lock);
2818 2819 2820
		if (ret)
			return ret;
	}
2821

2822
	*ppos += cnt;
2823 2824 2825 2826

	return cnt;
}

L
Li Zefan 已提交
2827 2828 2829 2830 2831 2832 2833
static int tracing_trace_options_open(struct inode *inode, struct file *file)
{
	if (tracing_disabled)
		return -ENODEV;
	return single_open(file, tracing_trace_options_show, NULL);
}

2834
static const struct file_operations tracing_iter_fops = {
L
Li Zefan 已提交
2835 2836 2837 2838
	.open		= tracing_trace_options_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
2839
	.write		= tracing_trace_options_write,
2840 2841
};

I
Ingo Molnar 已提交
2842 2843
static const char readme_msg[] =
	"tracing mini-HOWTO:\n\n"
2844 2845
	"# mount -t debugfs nodev /sys/kernel/debug\n\n"
	"# cat /sys/kernel/debug/tracing/available_tracers\n"
2846
	"wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
2847
	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2848
	"nop\n"
2849
	"# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
2850
	"# cat /sys/kernel/debug/tracing/current_tracer\n"
2851
	"wakeup\n"
2852
	"# cat /sys/kernel/debug/tracing/trace_options\n"
I
Ingo Molnar 已提交
2853
	"noprint-parent nosym-offset nosym-addr noverbose\n"
2854
	"# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2855
	"# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
2856
	"# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2857
	"# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
I
Ingo Molnar 已提交
2858 2859 2860 2861 2862 2863 2864 2865 2866 2867
;

static ssize_t
tracing_readme_read(struct file *filp, char __user *ubuf,
		       size_t cnt, loff_t *ppos)
{
	return simple_read_from_buffer(ubuf, cnt, ppos,
					readme_msg, strlen(readme_msg));
}

2868
static const struct file_operations tracing_readme_fops = {
I
Ingo Molnar 已提交
2869 2870
	.open		= tracing_open_generic,
	.read		= tracing_readme_read,
2871
	.llseek		= generic_file_llseek,
I
Ingo Molnar 已提交
2872 2873
};

2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921
static ssize_t
tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
				size_t cnt, loff_t *ppos)
{
	char *buf_comm;
	char *file_buf;
	char *buf;
	int len = 0;
	int pid;
	int i;

	file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
	if (!file_buf)
		return -ENOMEM;

	buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
	if (!buf_comm) {
		kfree(file_buf);
		return -ENOMEM;
	}

	buf = file_buf;

	for (i = 0; i < SAVED_CMDLINES; i++) {
		int r;

		pid = map_cmdline_to_pid[i];
		if (pid == -1 || pid == NO_CMDLINE_MAP)
			continue;

		trace_find_cmdline(pid, buf_comm);
		r = sprintf(buf, "%d %s\n", pid, buf_comm);
		buf += r;
		len += r;
	}

	len = simple_read_from_buffer(ubuf, cnt, ppos,
				      file_buf, len);

	kfree(file_buf);
	kfree(buf_comm);

	return len;
}

static const struct file_operations tracing_saved_cmdlines_fops = {
    .open       = tracing_open_generic,
    .read       = tracing_saved_cmdlines_read,
2922
    .llseek	= generic_file_llseek,
2923 2924
};

2925 2926 2927 2928 2929 2930 2931
static ssize_t
tracing_ctrl_read(struct file *filp, char __user *ubuf,
		  size_t cnt, loff_t *ppos)
{
	char buf[64];
	int r;

2932
	r = sprintf(buf, "%u\n", tracer_enabled);
I
Ingo Molnar 已提交
2933
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2934 2935 2936 2937 2938 2939 2940
}

static ssize_t
tracing_ctrl_write(struct file *filp, const char __user *ubuf,
		   size_t cnt, loff_t *ppos)
{
	struct trace_array *tr = filp->private_data;
2941
	unsigned long val;
2942
	int ret;
2943

2944 2945
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
2946
		return ret;
2947 2948 2949 2950

	val = !!val;

	mutex_lock(&trace_types_lock);
2951
	if (tracer_enabled ^ val) {
2952 2953 2954 2955

		/* Only need to warn if this is used to change the state */
		WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");

2956
		if (val) {
2957
			tracer_enabled = 1;
2958 2959 2960 2961
			if (current_trace->start)
				current_trace->start(tr);
			tracing_start();
		} else {
2962
			tracer_enabled = 0;
2963 2964 2965 2966
			tracing_stop();
			if (current_trace->stop)
				current_trace->stop(tr);
		}
2967 2968 2969
	}
	mutex_unlock(&trace_types_lock);

2970
	*ppos += cnt;
2971 2972 2973 2974 2975 2976 2977 2978

	return cnt;
}

static ssize_t
tracing_set_trace_read(struct file *filp, char __user *ubuf,
		       size_t cnt, loff_t *ppos)
{
L
Li Zefan 已提交
2979
	char buf[MAX_TRACER_SIZE+2];
2980 2981 2982 2983 2984 2985 2986 2987 2988
	int r;

	mutex_lock(&trace_types_lock);
	if (current_trace)
		r = sprintf(buf, "%s\n", current_trace->name);
	else
		r = sprintf(buf, "\n");
	mutex_unlock(&trace_types_lock);

I
Ingo Molnar 已提交
2989
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2990 2991
}

2992 2993 2994 2995 2996 2997
int tracer_init(struct tracer *t, struct trace_array *tr)
{
	tracing_reset_online_cpus(tr);
	return t->init(tr);
}

2998 2999 3000 3001 3002 3003 3004 3005
static void set_buffer_entries(struct trace_array *tr, unsigned long val)
{
	int cpu;
	for_each_tracing_cpu(cpu)
		tr->data[cpu]->entries = val;
}

static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3006 3007 3008 3009 3010
{
	int ret;

	/*
	 * If kernel or user changes the size of the ring buffer
3011 3012
	 * we use the size that was given, and we can forget about
	 * expanding it later.
3013 3014 3015
	 */
	ring_buffer_expanded = 1;

3016
	ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3017 3018 3019
	if (ret < 0)
		return ret;

3020 3021 3022
	if (!current_trace->use_max_tr)
		goto out;

3023
	ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3024
	if (ret < 0) {
3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040
		int r = 0;

		if (cpu == RING_BUFFER_ALL_CPUS) {
			int i;
			for_each_tracing_cpu(i) {
				r = ring_buffer_resize(global_trace.buffer,
						global_trace.data[i]->entries,
						i);
				if (r < 0)
					break;
			}
		} else {
			r = ring_buffer_resize(global_trace.buffer,
						global_trace.data[cpu]->entries,
						cpu);
		}
3041 3042

		if (r < 0) {
3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056
			/*
			 * AARGH! We are left with different
			 * size max buffer!!!!
			 * The max buffer is our "snapshot" buffer.
			 * When a tracer needs a snapshot (one of the
			 * latency tracers), it swaps the max buffer
			 * with the saved snap shot. We succeeded to
			 * update the size of the main buffer, but failed to
			 * update the size of the max buffer. But when we tried
			 * to reset the main buffer to the original size, we
			 * failed there too. This is very unlikely to
			 * happen, but if it does, warn and kill all
			 * tracing.
			 */
3057 3058 3059 3060 3061 3062
			WARN_ON(1);
			tracing_disabled = 1;
		}
		return ret;
	}

3063 3064 3065 3066 3067
	if (cpu == RING_BUFFER_ALL_CPUS)
		set_buffer_entries(&max_tr, size);
	else
		max_tr.data[cpu]->entries = size;

3068
 out:
3069 3070 3071 3072
	if (cpu == RING_BUFFER_ALL_CPUS)
		set_buffer_entries(&global_trace, size);
	else
		global_trace.data[cpu]->entries = size;
3073 3074 3075 3076

	return ret;
}

3077
static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3078
{
3079
	int ret = size;
3080 3081 3082

	mutex_lock(&trace_types_lock);

3083 3084 3085 3086 3087 3088 3089
	if (cpu_id != RING_BUFFER_ALL_CPUS) {
		/* make sure, this cpu is enabled in the mask */
		if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
			ret = -EINVAL;
			goto out;
		}
	}
3090

3091
	ret = __tracing_resize_ring_buffer(size, cpu_id);
3092 3093 3094
	if (ret < 0)
		ret = -ENOMEM;

3095
out:
3096 3097 3098 3099 3100
	mutex_unlock(&trace_types_lock);

	return ret;
}

3101

3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115
/**
 * tracing_update_buffers - used by tracing facility to expand ring buffers
 *
 * To save on memory when the tracing is never used on a system with it
 * configured in. The ring buffers are set to a minimum size. But once
 * a user starts to use the tracing facility, then they need to grow
 * to their default size.
 *
 * This function is to be called when a tracer is about to be used.
 */
int tracing_update_buffers(void)
{
	int ret = 0;

3116
	mutex_lock(&trace_types_lock);
3117
	if (!ring_buffer_expanded)
3118 3119
		ret = __tracing_resize_ring_buffer(trace_buf_size,
						RING_BUFFER_ALL_CPUS);
3120
	mutex_unlock(&trace_types_lock);
3121 3122 3123 3124

	return ret;
}

3125 3126 3127 3128 3129 3130 3131 3132
struct trace_option_dentry;

static struct trace_option_dentry *
create_trace_option_files(struct tracer *tracer);

static void
destroy_trace_option_files(struct trace_option_dentry *topts);

3133
static int tracing_set_tracer(const char *buf)
3134
{
3135
	static struct trace_option_dentry *topts;
3136 3137
	struct trace_array *tr = &global_trace;
	struct tracer *t;
3138
	int ret = 0;
3139

3140 3141
	mutex_lock(&trace_types_lock);

3142
	if (!ring_buffer_expanded) {
3143 3144
		ret = __tracing_resize_ring_buffer(trace_buf_size,
						RING_BUFFER_ALL_CPUS);
3145
		if (ret < 0)
3146
			goto out;
3147 3148 3149
		ret = 0;
	}

3150 3151 3152 3153
	for (t = trace_types; t; t = t->next) {
		if (strcmp(t->name, buf) == 0)
			break;
	}
3154 3155 3156 3157 3158
	if (!t) {
		ret = -EINVAL;
		goto out;
	}
	if (t == current_trace)
3159 3160
		goto out;

3161
	trace_branch_disable();
3162 3163
	if (current_trace && current_trace->reset)
		current_trace->reset(tr);
3164 3165 3166 3167 3168 3169
	if (current_trace && current_trace->use_max_tr) {
		/*
		 * We don't free the ring buffer. instead, resize it because
		 * The max_tr ring buffer has some state (e.g. ring->clock) and
		 * we want preserve it.
		 */
3170 3171
		ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
		set_buffer_entries(&max_tr, 1);
3172
	}
3173 3174
	destroy_trace_option_files(topts);

3175
	current_trace = t;
3176 3177

	topts = create_trace_option_files(current_trace);
3178
	if (current_trace->use_max_tr) {
3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189
		int cpu;
		/* we need to make per cpu buffer sizes equivalent */
		for_each_tracing_cpu(cpu) {
			ret = ring_buffer_resize(max_tr.buffer,
						global_trace.data[cpu]->entries,
						cpu);
			if (ret < 0)
				goto out;
			max_tr.data[cpu]->entries =
					global_trace.data[cpu]->entries;
		}
3190
	}
3191

3192
	if (t->init) {
3193
		ret = tracer_init(t, tr);
3194 3195 3196
		if (ret)
			goto out;
	}
3197

3198
	trace_branch_enable(tr);
3199 3200 3201
 out:
	mutex_unlock(&trace_types_lock);

3202 3203 3204 3205 3206 3207 3208
	return ret;
}

static ssize_t
tracing_set_trace_write(struct file *filp, const char __user *ubuf,
			size_t cnt, loff_t *ppos)
{
L
Li Zefan 已提交
3209
	char buf[MAX_TRACER_SIZE+1];
3210 3211
	int i;
	size_t ret;
3212 3213 3214
	int err;

	ret = cnt;
3215

L
Li Zefan 已提交
3216 3217
	if (cnt > MAX_TRACER_SIZE)
		cnt = MAX_TRACER_SIZE;
3218 3219 3220 3221 3222 3223 3224 3225 3226 3227

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	/* strip ending whitespace. */
	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
		buf[i] = 0;

3228 3229 3230
	err = tracing_set_tracer(buf);
	if (err)
		return err;
3231

3232
	*ppos += ret;
3233

3234
	return ret;
3235 3236 3237 3238 3239 3240 3241 3242 3243 3244
}

static ssize_t
tracing_max_lat_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	unsigned long *ptr = filp->private_data;
	char buf[64];
	int r;

S
Steven Rostedt 已提交
3245
	r = snprintf(buf, sizeof(buf), "%ld\n",
3246
		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
S
Steven Rostedt 已提交
3247 3248
	if (r > sizeof(buf))
		r = sizeof(buf);
I
Ingo Molnar 已提交
3249
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3250 3251 3252 3253 3254 3255
}

static ssize_t
tracing_max_lat_write(struct file *filp, const char __user *ubuf,
		      size_t cnt, loff_t *ppos)
{
3256 3257
	unsigned long *ptr = filp->private_data;
	unsigned long val;
3258
	int ret;
3259

3260 3261
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
3262
		return ret;
3263 3264 3265 3266 3267 3268

	*ptr = val * 1000;

	return cnt;
}

3269 3270
static int tracing_open_pipe(struct inode *inode, struct file *filp)
{
3271
	long cpu_file = (long) inode->i_private;
3272
	struct trace_iterator *iter;
3273
	int ret = 0;
3274 3275 3276 3277

	if (tracing_disabled)
		return -ENODEV;

3278 3279
	mutex_lock(&trace_types_lock);

3280 3281
	/* create a buffer to store the information to pass to userspace */
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3282 3283 3284 3285
	if (!iter) {
		ret = -ENOMEM;
		goto out;
	}
3286

3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298
	/*
	 * We make a copy of the current tracer to avoid concurrent
	 * changes on it while we are reading.
	 */
	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
	if (!iter->trace) {
		ret = -ENOMEM;
		goto fail;
	}
	if (current_trace)
		*iter->trace = *current_trace;

3299
	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3300
		ret = -ENOMEM;
3301
		goto fail;
3302 3303
	}

3304
	/* trace pipe does not show start of buffer */
3305
	cpumask_setall(iter->started);
3306

3307 3308 3309
	if (trace_flags & TRACE_ITER_LATENCY_FMT)
		iter->iter_flags |= TRACE_FILE_LAT_FMT;

3310
	iter->cpu_file = cpu_file;
3311
	iter->tr = &global_trace;
3312
	mutex_init(&iter->mutex);
3313 3314
	filp->private_data = iter;

3315 3316 3317
	if (iter->trace->pipe_open)
		iter->trace->pipe_open(iter);

3318
	nonseekable_open(inode, filp);
3319 3320 3321
out:
	mutex_unlock(&trace_types_lock);
	return ret;
3322 3323 3324 3325 3326 3327

fail:
	kfree(iter->trace);
	kfree(iter);
	mutex_unlock(&trace_types_lock);
	return ret;
3328 3329 3330 3331 3332 3333
}

static int tracing_release_pipe(struct inode *inode, struct file *file)
{
	struct trace_iterator *iter = file->private_data;

3334 3335
	mutex_lock(&trace_types_lock);

3336
	if (iter->trace->pipe_close)
S
Steven Rostedt 已提交
3337 3338
		iter->trace->pipe_close(iter);

3339 3340
	mutex_unlock(&trace_types_lock);

3341
	free_cpumask_var(iter->started);
3342 3343
	mutex_destroy(&iter->mutex);
	kfree(iter->trace);
3344 3345 3346 3347 3348
	kfree(iter);

	return 0;
}

3349 3350 3351 3352 3353 3354 3355 3356 3357 3358
static unsigned int
tracing_poll_pipe(struct file *filp, poll_table *poll_table)
{
	struct trace_iterator *iter = filp->private_data;

	if (trace_flags & TRACE_ITER_BLOCK) {
		/*
		 * Always select as readable when in blocking mode
		 */
		return POLLIN | POLLRDNORM;
I
Ingo Molnar 已提交
3359
	} else {
3360 3361 3362 3363 3364 3365 3366 3367 3368 3369
		if (!trace_empty(iter))
			return POLLIN | POLLRDNORM;
		poll_wait(filp, &trace_wait, poll_table);
		if (!trace_empty(iter))
			return POLLIN | POLLRDNORM;

		return 0;
	}
}

3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401

void default_wait_pipe(struct trace_iterator *iter)
{
	DEFINE_WAIT(wait);

	prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);

	if (trace_empty(iter))
		schedule();

	finish_wait(&trace_wait, &wait);
}

/*
 * This is a make-shift waitqueue.
 * A tracer might use this callback on some rare cases:
 *
 *  1) the current tracer might hold the runqueue lock when it wakes up
 *     a reader, hence a deadlock (sched, function, and function graph tracers)
 *  2) the function tracers, trace all functions, we don't want
 *     the overhead of calling wake_up and friends
 *     (and tracing them too)
 *
 *     Anyway, this is really very primitive wakeup.
 */
void poll_wait_pipe(struct trace_iterator *iter)
{
	set_current_state(TASK_INTERRUPTIBLE);
	/* sleep for 100 msecs, and try again. */
	schedule_timeout(HZ / 10);
}

3402 3403
/* Must be called with trace_types_lock mutex held. */
static int tracing_wait_pipe(struct file *filp)
3404 3405 3406 3407
{
	struct trace_iterator *iter = filp->private_data;

	while (trace_empty(iter)) {
3408

3409
		if ((filp->f_flags & O_NONBLOCK)) {
3410
			return -EAGAIN;
3411
		}
3412

3413
		mutex_unlock(&iter->mutex);
3414

3415
		iter->trace->wait_pipe(iter);
3416

3417
		mutex_lock(&iter->mutex);
3418

3419
		if (signal_pending(current))
3420
			return -EINTR;
3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434

		/*
		 * We block until we read something and tracing is disabled.
		 * We still block if tracing is disabled, but we have never
		 * read anything. This allows a user to cat this file, and
		 * then enable tracing. But after we have read something,
		 * we give an EOF when tracing is again disabled.
		 *
		 * iter->pos will be 0 if we haven't read anything.
		 */
		if (!tracer_enabled && iter->pos)
			break;
	}

3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445
	return 1;
}

/*
 * Consumer reader.
 */
static ssize_t
tracing_read_pipe(struct file *filp, char __user *ubuf,
		  size_t cnt, loff_t *ppos)
{
	struct trace_iterator *iter = filp->private_data;
3446
	static struct tracer *old_tracer;
3447 3448 3449 3450 3451 3452 3453
	ssize_t sret;

	/* return any leftover data */
	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
	if (sret != -EBUSY)
		return sret;

3454
	trace_seq_init(&iter->seq);
3455

3456
	/* copy the tracer to avoid using a global lock all around */
3457
	mutex_lock(&trace_types_lock);
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469
	if (unlikely(old_tracer != current_trace && current_trace)) {
		old_tracer = current_trace;
		*iter->trace = *current_trace;
	}
	mutex_unlock(&trace_types_lock);

	/*
	 * Avoid more than one consumer on a single file descriptor
	 * This is just a matter of traces coherency, the ring buffer itself
	 * is protected.
	 */
	mutex_lock(&iter->mutex);
3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480
	if (iter->trace->read) {
		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
		if (sret)
			goto out;
	}

waitagain:
	sret = tracing_wait_pipe(filp);
	if (sret <= 0)
		goto out;

3481
	/* stop when tracing is finished */
3482 3483
	if (trace_empty(iter)) {
		sret = 0;
3484
		goto out;
3485
	}
3486 3487 3488 3489

	if (cnt >= PAGE_SIZE)
		cnt = PAGE_SIZE - 1;

3490 3491 3492 3493
	/* reset all but tr, trace, and overruns */
	memset(&iter->seq, 0,
	       sizeof(struct trace_iterator) -
	       offsetof(struct trace_iterator, seq));
3494
	iter->pos = -1;
3495

3496
	trace_event_read_lock();
3497
	trace_access_lock(iter->cpu_file);
3498
	while (trace_find_next_entry_inc(iter) != NULL) {
3499
		enum print_line_t ret;
S
Steven Rostedt 已提交
3500 3501
		int len = iter->seq.len;

I
Ingo Molnar 已提交
3502
		ret = print_trace_line(iter);
3503
		if (ret == TRACE_TYPE_PARTIAL_LINE) {
S
Steven Rostedt 已提交
3504 3505
			/* don't print partial lines */
			iter->seq.len = len;
3506
			break;
S
Steven Rostedt 已提交
3507
		}
3508 3509
		if (ret != TRACE_TYPE_NO_CONSUME)
			trace_consume(iter);
3510 3511 3512

		if (iter->seq.len >= cnt)
			break;
3513 3514 3515 3516 3517 3518 3519 3520

		/*
		 * Setting the full flag means we reached the trace_seq buffer
		 * size and we should leave by partial output condition above.
		 * One of the trace_seq_* functions is not used properly.
		 */
		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
			  iter->ent->type);
3521
	}
3522
	trace_access_unlock(iter->cpu_file);
3523
	trace_event_read_unlock();
3524 3525

	/* Now copy what we have to the user */
3526 3527
	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
	if (iter->seq.readpos >= iter->seq.len)
3528
		trace_seq_init(&iter->seq);
P
Pekka Paalanen 已提交
3529 3530

	/*
L
Lucas De Marchi 已提交
3531
	 * If there was nothing to send to user, in spite of consuming trace
P
Pekka Paalanen 已提交
3532 3533
	 * entries, go back to wait for more entries.
	 */
3534
	if (sret == -EBUSY)
P
Pekka Paalanen 已提交
3535
		goto waitagain;
3536

3537
out:
3538
	mutex_unlock(&iter->mutex);
3539

3540
	return sret;
3541 3542
}

3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554
static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
				     struct pipe_buffer *buf)
{
	__free_page(buf->page);
}

static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
				     unsigned int idx)
{
	__free_page(spd->pages[idx]);
}

3555
static const struct pipe_buf_operations tracing_pipe_buf_ops = {
S
Steven Rostedt 已提交
3556 3557 3558 3559 3560 3561 3562
	.can_merge		= 0,
	.map			= generic_pipe_buf_map,
	.unmap			= generic_pipe_buf_unmap,
	.confirm		= generic_pipe_buf_confirm,
	.release		= tracing_pipe_buf_release,
	.steal			= generic_pipe_buf_steal,
	.get			= generic_pipe_buf_get,
3563 3564
};

S
Steven Rostedt 已提交
3565
static size_t
3566
tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
S
Steven Rostedt 已提交
3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585
{
	size_t count;
	int ret;

	/* Seq buffer is page-sized, exactly what we need. */
	for (;;) {
		count = iter->seq.len;
		ret = print_trace_line(iter);
		count = iter->seq.len - count;
		if (rem < count) {
			rem = 0;
			iter->seq.len -= count;
			break;
		}
		if (ret == TRACE_TYPE_PARTIAL_LINE) {
			iter->seq.len -= count;
			break;
		}

3586 3587
		if (ret != TRACE_TYPE_NO_CONSUME)
			trace_consume(iter);
S
Steven Rostedt 已提交
3588
		rem -= count;
3589
		if (!trace_find_next_entry_inc(iter))	{
S
Steven Rostedt 已提交
3590 3591 3592 3593 3594 3595 3596 3597 3598
			rem = 0;
			iter->ent = NULL;
			break;
		}
	}

	return rem;
}

3599 3600 3601 3602 3603 3604
static ssize_t tracing_splice_read_pipe(struct file *filp,
					loff_t *ppos,
					struct pipe_inode_info *pipe,
					size_t len,
					unsigned int flags)
{
3605 3606
	struct page *pages_def[PIPE_DEF_BUFFERS];
	struct partial_page partial_def[PIPE_DEF_BUFFERS];
3607 3608
	struct trace_iterator *iter = filp->private_data;
	struct splice_pipe_desc spd = {
3609 3610
		.pages		= pages_def,
		.partial	= partial_def,
S
Steven Rostedt 已提交
3611 3612 3613 3614
		.nr_pages	= 0, /* This gets updated below. */
		.flags		= flags,
		.ops		= &tracing_pipe_buf_ops,
		.spd_release	= tracing_spd_release_pipe,
3615
	};
3616
	static struct tracer *old_tracer;
3617
	ssize_t ret;
S
Steven Rostedt 已提交
3618
	size_t rem;
3619 3620
	unsigned int i;

3621 3622 3623
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

3624
	/* copy the tracer to avoid using a global lock all around */
3625
	mutex_lock(&trace_types_lock);
3626 3627 3628 3629 3630 3631 3632
	if (unlikely(old_tracer != current_trace && current_trace)) {
		old_tracer = current_trace;
		*iter->trace = *current_trace;
	}
	mutex_unlock(&trace_types_lock);

	mutex_lock(&iter->mutex);
3633 3634 3635 3636 3637

	if (iter->trace->splice_read) {
		ret = iter->trace->splice_read(iter, filp,
					       ppos, pipe, len, flags);
		if (ret)
S
Steven Rostedt 已提交
3638
			goto out_err;
3639 3640 3641 3642
	}

	ret = tracing_wait_pipe(filp);
	if (ret <= 0)
S
Steven Rostedt 已提交
3643
		goto out_err;
3644

3645
	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3646
		ret = -EFAULT;
S
Steven Rostedt 已提交
3647
		goto out_err;
3648 3649
	}

3650
	trace_event_read_lock();
3651
	trace_access_lock(iter->cpu_file);
3652

3653
	/* Fill as many pages as possible. */
3654 3655 3656
	for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
		spd.pages[i] = alloc_page(GFP_KERNEL);
		if (!spd.pages[i])
S
Steven Rostedt 已提交
3657
			break;
3658

3659
		rem = tracing_fill_pipe_page(rem, iter);
3660 3661 3662

		/* Copy the data into the page, so we can start over. */
		ret = trace_seq_to_buffer(&iter->seq,
3663
					  page_address(spd.pages[i]),
3664 3665
					  iter->seq.len);
		if (ret < 0) {
3666
			__free_page(spd.pages[i]);
3667 3668
			break;
		}
3669 3670
		spd.partial[i].offset = 0;
		spd.partial[i].len = iter->seq.len;
3671

3672
		trace_seq_init(&iter->seq);
3673 3674
	}

3675
	trace_access_unlock(iter->cpu_file);
3676
	trace_event_read_unlock();
3677
	mutex_unlock(&iter->mutex);
3678 3679 3680

	spd.nr_pages = i;

3681 3682 3683 3684
	ret = splice_to_pipe(pipe, &spd);
out:
	splice_shrink_spd(pipe, &spd);
	return ret;
3685

S
Steven Rostedt 已提交
3686
out_err:
3687
	mutex_unlock(&iter->mutex);
3688
	goto out;
3689 3690
}

3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714
struct ftrace_entries_info {
	struct trace_array	*tr;
	int			cpu;
};

static int tracing_entries_open(struct inode *inode, struct file *filp)
{
	struct ftrace_entries_info *info;

	if (tracing_disabled)
		return -ENODEV;

	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		return -ENOMEM;

	info->tr = &global_trace;
	info->cpu = (unsigned long)inode->i_private;

	filp->private_data = info;

	return 0;
}

3715 3716 3717 3718
static ssize_t
tracing_entries_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
3719 3720 3721 3722 3723
	struct ftrace_entries_info *info = filp->private_data;
	struct trace_array *tr = info->tr;
	char buf[64];
	int r = 0;
	ssize_t ret;
3724

3725
	mutex_lock(&trace_types_lock);
3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755

	if (info->cpu == RING_BUFFER_ALL_CPUS) {
		int cpu, buf_size_same;
		unsigned long size;

		size = 0;
		buf_size_same = 1;
		/* check if all cpu sizes are same */
		for_each_tracing_cpu(cpu) {
			/* fill in the size from first enabled cpu */
			if (size == 0)
				size = tr->data[cpu]->entries;
			if (size != tr->data[cpu]->entries) {
				buf_size_same = 0;
				break;
			}
		}

		if (buf_size_same) {
			if (!ring_buffer_expanded)
				r = sprintf(buf, "%lu (expanded: %lu)\n",
					    size >> 10,
					    trace_buf_size >> 10);
			else
				r = sprintf(buf, "%lu\n", size >> 10);
		} else
			r = sprintf(buf, "X\n");
	} else
		r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);

3756 3757
	mutex_unlock(&trace_types_lock);

3758 3759
	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	return ret;
3760 3761 3762 3763 3764 3765
}

static ssize_t
tracing_entries_write(struct file *filp, const char __user *ubuf,
		      size_t cnt, loff_t *ppos)
{
3766
	struct ftrace_entries_info *info = filp->private_data;
3767
	unsigned long val;
3768
	int ret;
3769

3770 3771
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
3772
		return ret;
3773 3774 3775 3776 3777

	/* must have at least 1 entry */
	if (!val)
		return -EINVAL;

3778 3779 3780
	/* value is in KB */
	val <<= 10;

3781
	ret = tracing_resize_ring_buffer(val, info->cpu);
3782 3783
	if (ret < 0)
		return ret;
3784

3785
	*ppos += cnt;
3786

3787 3788
	return cnt;
}
S
Steven Rostedt 已提交
3789

3790 3791 3792 3793 3794 3795 3796 3797 3798 3799
static int
tracing_entries_release(struct inode *inode, struct file *filp)
{
	struct ftrace_entries_info *info = filp->private_data;

	kfree(info);

	return 0;
}

3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810
static ssize_t
tracing_total_entries_read(struct file *filp, char __user *ubuf,
				size_t cnt, loff_t *ppos)
{
	struct trace_array *tr = filp->private_data;
	char buf[64];
	int r, cpu;
	unsigned long size = 0, expanded_size = 0;

	mutex_lock(&trace_types_lock);
	for_each_tracing_cpu(cpu) {
3811
		size += tr->data[cpu]->entries >> 10;
3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823
		if (!ring_buffer_expanded)
			expanded_size += trace_buf_size >> 10;
	}
	if (ring_buffer_expanded)
		r = sprintf(buf, "%lu\n", size);
	else
		r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
	mutex_unlock(&trace_types_lock);

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

3824 3825 3826 3827 3828 3829 3830 3831 3832 3833
static ssize_t
tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
			  size_t cnt, loff_t *ppos)
{
	/*
	 * There is no need to read what the user has written, this function
	 * is just to make sure that there is no error when "echo" is used
	 */

	*ppos += cnt;
3834 3835 3836 3837

	return cnt;
}

3838 3839 3840
static int
tracing_free_buffer_release(struct inode *inode, struct file *filp)
{
3841 3842 3843
	/* disable tracing ? */
	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
		tracing_off();
3844
	/* resize the ring buffer to 0 */
3845
	tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3846 3847 3848 3849

	return 0;
}

3850 3851 3852 3853
static ssize_t
tracing_mark_write(struct file *filp, const char __user *ubuf,
					size_t cnt, loff_t *fpos)
{
3854 3855 3856 3857 3858 3859
	unsigned long addr = (unsigned long)ubuf;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	struct print_entry *entry;
	unsigned long irq_flags;
	struct page *pages[2];
3860
	void *map_page[2];
3861 3862 3863 3864 3865 3866
	int nr_pages = 1;
	ssize_t written;
	int offset;
	int size;
	int len;
	int ret;
3867
	int i;
3868

S
Steven Rostedt 已提交
3869
	if (tracing_disabled)
3870 3871 3872 3873 3874
		return -EINVAL;

	if (cnt > TRACE_BUF_SIZE)
		cnt = TRACE_BUF_SIZE;

3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889
	/*
	 * Userspace is injecting traces into the kernel trace buffer.
	 * We want to be as non intrusive as possible.
	 * To do so, we do not want to allocate any special buffers
	 * or take any locks, but instead write the userspace data
	 * straight into the ring buffer.
	 *
	 * First we need to pin the userspace buffer into memory,
	 * which, most likely it is, because it just referenced it.
	 * But there's no guarantee that it is. By using get_user_pages_fast()
	 * and kmap_atomic/kunmap_atomic() we can get access to the
	 * pages directly. We then write the data directly into the
	 * ring buffer.
	 */
	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
3890

3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903
	/* check if we cross pages */
	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
		nr_pages = 2;

	offset = addr & (PAGE_SIZE - 1);
	addr &= PAGE_MASK;

	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
	if (ret < nr_pages) {
		while (--ret >= 0)
			put_page(pages[ret]);
		written = -EFAULT;
		goto out;
3904
	}
3905

3906 3907
	for (i = 0; i < nr_pages; i++)
		map_page[i] = kmap_atomic(pages[i]);
3908 3909 3910 3911 3912 3913 3914 3915 3916 3917

	local_save_flags(irq_flags);
	size = sizeof(*entry) + cnt + 2; /* possible \n added */
	buffer = global_trace.buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
					  irq_flags, preempt_count());
	if (!event) {
		/* Ring buffer disabled, return as if not open for write */
		written = -EBADF;
		goto out_unlock;
3918
	}
3919 3920 3921 3922 3923 3924

	entry = ring_buffer_event_data(event);
	entry->ip = _THIS_IP_;

	if (nr_pages == 2) {
		len = PAGE_SIZE - offset;
3925 3926
		memcpy(&entry->buf, map_page[0] + offset, len);
		memcpy(&entry->buf[len], map_page[1], cnt - len);
C
Carsten Emde 已提交
3927
	} else
3928
		memcpy(&entry->buf, map_page[0] + offset, cnt);
3929

3930 3931 3932 3933 3934 3935 3936
	if (entry->buf[cnt - 1] != '\n') {
		entry->buf[cnt] = '\n';
		entry->buf[cnt + 1] = '\0';
	} else
		entry->buf[cnt] = '\0';

	ring_buffer_unlock_commit(buffer, event);
3937

3938
	written = cnt;
3939

3940
	*fpos += written;
3941

3942
 out_unlock:
3943 3944 3945 3946
	for (i = 0; i < nr_pages; i++){
		kunmap_atomic(map_page[i]);
		put_page(pages[i]);
	}
3947
 out:
3948
	return written;
3949 3950
}

L
Li Zefan 已提交
3951
static int tracing_clock_show(struct seq_file *m, void *v)
3952 3953 3954 3955
{
	int i;

	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
L
Li Zefan 已提交
3956
		seq_printf(m,
3957 3958 3959
			"%s%s%s%s", i ? " " : "",
			i == trace_clock_id ? "[" : "", trace_clocks[i].name,
			i == trace_clock_id ? "]" : "");
L
Li Zefan 已提交
3960
	seq_putc(m, '\n');
3961

L
Li Zefan 已提交
3962
	return 0;
3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003
}

static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
				   size_t cnt, loff_t *fpos)
{
	char buf[64];
	const char *clockstr;
	int i;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	clockstr = strstrip(buf);

	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
		if (strcmp(trace_clocks[i].name, clockstr) == 0)
			break;
	}
	if (i == ARRAY_SIZE(trace_clocks))
		return -EINVAL;

	trace_clock_id = i;

	mutex_lock(&trace_types_lock);

	ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
	if (max_tr.buffer)
		ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);

	mutex_unlock(&trace_types_lock);

	*fpos += cnt;

	return cnt;
}

L
Li Zefan 已提交
4004 4005 4006 4007 4008 4009 4010
static int tracing_clock_open(struct inode *inode, struct file *file)
{
	if (tracing_disabled)
		return -ENODEV;
	return single_open(file, tracing_clock_show, NULL);
}

4011
static const struct file_operations tracing_max_lat_fops = {
I
Ingo Molnar 已提交
4012 4013 4014
	.open		= tracing_open_generic,
	.read		= tracing_max_lat_read,
	.write		= tracing_max_lat_write,
4015
	.llseek		= generic_file_llseek,
4016 4017
};

4018
static const struct file_operations tracing_ctrl_fops = {
I
Ingo Molnar 已提交
4019 4020 4021
	.open		= tracing_open_generic,
	.read		= tracing_ctrl_read,
	.write		= tracing_ctrl_write,
4022
	.llseek		= generic_file_llseek,
4023 4024
};

4025
static const struct file_operations set_tracer_fops = {
I
Ingo Molnar 已提交
4026 4027 4028
	.open		= tracing_open_generic,
	.read		= tracing_set_trace_read,
	.write		= tracing_set_trace_write,
4029
	.llseek		= generic_file_llseek,
4030 4031
};

4032
static const struct file_operations tracing_pipe_fops = {
I
Ingo Molnar 已提交
4033
	.open		= tracing_open_pipe,
4034
	.poll		= tracing_poll_pipe,
I
Ingo Molnar 已提交
4035
	.read		= tracing_read_pipe,
4036
	.splice_read	= tracing_splice_read_pipe,
I
Ingo Molnar 已提交
4037
	.release	= tracing_release_pipe,
4038
	.llseek		= no_llseek,
4039 4040
};

4041
static const struct file_operations tracing_entries_fops = {
4042
	.open		= tracing_entries_open,
4043 4044
	.read		= tracing_entries_read,
	.write		= tracing_entries_write,
4045
	.release	= tracing_entries_release,
4046
	.llseek		= generic_file_llseek,
4047 4048
};

4049 4050 4051 4052 4053 4054
static const struct file_operations tracing_total_entries_fops = {
	.open		= tracing_open_generic,
	.read		= tracing_total_entries_read,
	.llseek		= generic_file_llseek,
};

4055 4056 4057 4058 4059
static const struct file_operations tracing_free_buffer_fops = {
	.write		= tracing_free_buffer_write,
	.release	= tracing_free_buffer_release,
};

4060
static const struct file_operations tracing_mark_fops = {
4061
	.open		= tracing_open_generic,
4062
	.write		= tracing_mark_write,
4063
	.llseek		= generic_file_llseek,
4064 4065
};

4066
static const struct file_operations trace_clock_fops = {
L
Li Zefan 已提交
4067 4068 4069 4070
	.open		= tracing_clock_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
4071 4072 4073
	.write		= tracing_clock_write,
};

4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094
struct ftrace_buffer_info {
	struct trace_array	*tr;
	void			*spare;
	int			cpu;
	unsigned int		read;
};

static int tracing_buffers_open(struct inode *inode, struct file *filp)
{
	int cpu = (int)(long)inode->i_private;
	struct ftrace_buffer_info *info;

	if (tracing_disabled)
		return -ENODEV;

	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		return -ENOMEM;

	info->tr	= &global_trace;
	info->cpu	= cpu;
4095
	info->spare	= NULL;
4096 4097 4098 4099 4100
	/* Force reading ring buffer for first read */
	info->read	= (unsigned int)-1;

	filp->private_data = info;

4101
	return nonseekable_open(inode, filp);
4102 4103 4104 4105 4106 4107 4108 4109 4110 4111
}

static ssize_t
tracing_buffers_read(struct file *filp, char __user *ubuf,
		     size_t count, loff_t *ppos)
{
	struct ftrace_buffer_info *info = filp->private_data;
	ssize_t ret;
	size_t size;

4112 4113 4114
	if (!count)
		return 0;

4115
	if (!info->spare)
4116
		info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4117 4118 4119
	if (!info->spare)
		return -ENOMEM;

4120 4121 4122 4123
	/* Do we have previous read data to read? */
	if (info->read < PAGE_SIZE)
		goto read;

4124
	trace_access_lock(info->cpu);
4125 4126 4127 4128
	ret = ring_buffer_read_page(info->tr->buffer,
				    &info->spare,
				    count,
				    info->cpu, 0);
4129
	trace_access_unlock(info->cpu);
4130 4131 4132
	if (ret < 0)
		return 0;

4133 4134
	info->read = 0;

4135 4136 4137 4138 4139 4140
read:
	size = PAGE_SIZE - info->read;
	if (size > count)
		size = count;

	ret = copy_to_user(ubuf, info->spare + info->read, size);
4141
	if (ret == size)
4142
		return -EFAULT;
4143 4144
	size -= ret;

4145 4146 4147 4148 4149 4150 4151 4152 4153 4154
	*ppos += size;
	info->read += size;

	return size;
}

static int tracing_buffers_release(struct inode *inode, struct file *file)
{
	struct ftrace_buffer_info *info = file->private_data;

4155 4156
	if (info->spare)
		ring_buffer_free_read_page(info->tr->buffer, info->spare);
4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195
	kfree(info);

	return 0;
}

struct buffer_ref {
	struct ring_buffer	*buffer;
	void			*page;
	int			ref;
};

static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	struct buffer_ref *ref = (struct buffer_ref *)buf->private;

	if (--ref->ref)
		return;

	ring_buffer_free_read_page(ref->buffer, ref->page);
	kfree(ref);
	buf->private = 0;
}

static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
				 struct pipe_buffer *buf)
{
	return 1;
}

static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
				struct pipe_buffer *buf)
{
	struct buffer_ref *ref = (struct buffer_ref *)buf->private;

	ref->ref++;
}

/* Pipe buffer operations for a buffer. */
4196
static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228
	.can_merge		= 0,
	.map			= generic_pipe_buf_map,
	.unmap			= generic_pipe_buf_unmap,
	.confirm		= generic_pipe_buf_confirm,
	.release		= buffer_pipe_buf_release,
	.steal			= buffer_pipe_buf_steal,
	.get			= buffer_pipe_buf_get,
};

/*
 * Callback from splice_to_pipe(), if we need to release some pages
 * at the end of the spd in case we error'ed out in filling the pipe.
 */
static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
	struct buffer_ref *ref =
		(struct buffer_ref *)spd->partial[i].private;

	if (--ref->ref)
		return;

	ring_buffer_free_read_page(ref->buffer, ref->page);
	kfree(ref);
	spd->partial[i].private = 0;
}

static ssize_t
tracing_buffers_splice_read(struct file *file, loff_t *ppos,
			    struct pipe_inode_info *pipe, size_t len,
			    unsigned int flags)
{
	struct ftrace_buffer_info *info = file->private_data;
4229 4230
	struct partial_page partial_def[PIPE_DEF_BUFFERS];
	struct page *pages_def[PIPE_DEF_BUFFERS];
4231
	struct splice_pipe_desc spd = {
4232 4233
		.pages		= pages_def,
		.partial	= partial_def,
4234 4235 4236 4237 4238
		.flags		= flags,
		.ops		= &buffer_pipe_buf_ops,
		.spd_release	= buffer_spd_release,
	};
	struct buffer_ref *ref;
4239
	int entries, size, i;
4240 4241
	size_t ret;

4242 4243 4244
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

4245 4246
	if (*ppos & (PAGE_SIZE - 1)) {
		WARN_ONCE(1, "Ftrace: previous read must page-align\n");
4247 4248
		ret = -EINVAL;
		goto out;
4249 4250 4251 4252
	}

	if (len & (PAGE_SIZE - 1)) {
		WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
4253 4254 4255 4256
		if (len < PAGE_SIZE) {
			ret = -EINVAL;
			goto out;
		}
4257 4258 4259
		len &= PAGE_MASK;
	}

4260
	trace_access_lock(info->cpu);
4261 4262
	entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);

4263
	for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4264 4265 4266 4267 4268 4269 4270
		struct page *page;
		int r;

		ref = kzalloc(sizeof(*ref), GFP_KERNEL);
		if (!ref)
			break;

4271
		ref->ref = 1;
4272
		ref->buffer = info->tr->buffer;
4273
		ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4274 4275 4276 4277 4278 4279
		if (!ref->page) {
			kfree(ref);
			break;
		}

		r = ring_buffer_read_page(ref->buffer, &ref->page,
4280
					  len, info->cpu, 1);
4281
		if (r < 0) {
4282
			ring_buffer_free_read_page(ref->buffer, ref->page);
4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301
			kfree(ref);
			break;
		}

		/*
		 * zero out any left over data, this is going to
		 * user land.
		 */
		size = ring_buffer_page_len(ref->page);
		if (size < PAGE_SIZE)
			memset(ref->page + size, 0, PAGE_SIZE - size);

		page = virt_to_page(ref->page);

		spd.pages[i] = page;
		spd.partial[i].len = PAGE_SIZE;
		spd.partial[i].offset = 0;
		spd.partial[i].private = (unsigned long)ref;
		spd.nr_pages++;
4302
		*ppos += PAGE_SIZE;
4303 4304

		entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4305 4306
	}

4307
	trace_access_unlock(info->cpu);
4308 4309 4310 4311 4312 4313 4314 4315 4316
	spd.nr_pages = i;

	/* did we read anything? */
	if (!spd.nr_pages) {
		if (flags & SPLICE_F_NONBLOCK)
			ret = -EAGAIN;
		else
			ret = 0;
		/* TODO: block */
4317
		goto out;
4318 4319 4320
	}

	ret = splice_to_pipe(pipe, &spd);
4321 4322
	splice_shrink_spd(pipe, &spd);
out:
4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333
	return ret;
}

static const struct file_operations tracing_buffers_fops = {
	.open		= tracing_buffers_open,
	.read		= tracing_buffers_read,
	.release	= tracing_buffers_release,
	.splice_read	= tracing_buffers_splice_read,
	.llseek		= no_llseek,
};

4334 4335 4336 4337 4338 4339 4340 4341
static ssize_t
tracing_stats_read(struct file *filp, char __user *ubuf,
		   size_t count, loff_t *ppos)
{
	unsigned long cpu = (unsigned long)filp->private_data;
	struct trace_array *tr = &global_trace;
	struct trace_seq *s;
	unsigned long cnt;
4342 4343
	unsigned long long t;
	unsigned long usec_rem;
4344

4345
	s = kmalloc(sizeof(*s), GFP_KERNEL);
4346
	if (!s)
4347
		return -ENOMEM;
4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359

	trace_seq_init(s);

	cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "entries: %ld\n", cnt);

	cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "overrun: %ld\n", cnt);

	cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "commit overrun: %ld\n", cnt);

4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370
	cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
	trace_seq_printf(s, "bytes: %ld\n", cnt);

	t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
	usec_rem = do_div(t, USEC_PER_SEC);
	trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);

	t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
	usec_rem = do_div(t, USEC_PER_SEC);
	trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);

4371 4372 4373 4374 4375 4376 4377 4378 4379 4380
	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);

	kfree(s);

	return count;
}

static const struct file_operations tracing_stats_fops = {
	.open		= tracing_open_generic,
	.read		= tracing_stats_read,
4381
	.llseek		= generic_file_llseek,
4382 4383
};

4384 4385
#ifdef CONFIG_DYNAMIC_FTRACE

S
Steven Rostedt 已提交
4386 4387 4388 4389 4390
int __weak ftrace_arch_read_dyn_info(char *buf, int size)
{
	return 0;
}

4391
static ssize_t
S
Steven Rostedt 已提交
4392
tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4393 4394
		  size_t cnt, loff_t *ppos)
{
S
Steven Rostedt 已提交
4395 4396
	static char ftrace_dyn_info_buffer[1024];
	static DEFINE_MUTEX(dyn_info_mutex);
4397
	unsigned long *p = filp->private_data;
S
Steven Rostedt 已提交
4398
	char *buf = ftrace_dyn_info_buffer;
S
Steven Rostedt 已提交
4399
	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4400 4401
	int r;

S
Steven Rostedt 已提交
4402 4403
	mutex_lock(&dyn_info_mutex);
	r = sprintf(buf, "%ld ", *p);
I
Ingo Molnar 已提交
4404

S
Steven Rostedt 已提交
4405
	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
S
Steven Rostedt 已提交
4406 4407 4408 4409 4410 4411 4412
	buf[r++] = '\n';

	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);

	mutex_unlock(&dyn_info_mutex);

	return r;
4413 4414
}

4415
static const struct file_operations tracing_dyn_info_fops = {
I
Ingo Molnar 已提交
4416
	.open		= tracing_open_generic,
S
Steven Rostedt 已提交
4417
	.read		= tracing_read_dyn_info,
4418
	.llseek		= generic_file_llseek,
4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430
};
#endif

static struct dentry *d_tracer;

struct dentry *tracing_init_dentry(void)
{
	static int once;

	if (d_tracer)
		return d_tracer;

4431 4432 4433
	if (!debugfs_initialized())
		return NULL;

4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444
	d_tracer = debugfs_create_dir("tracing", NULL);

	if (!d_tracer && !once) {
		once = 1;
		pr_warning("Could not create debugfs directory 'tracing'\n");
		return NULL;
	}

	return d_tracer;
}

4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473
static struct dentry *d_percpu;

struct dentry *tracing_dentry_percpu(void)
{
	static int once;
	struct dentry *d_tracer;

	if (d_percpu)
		return d_percpu;

	d_tracer = tracing_init_dentry();

	if (!d_tracer)
		return NULL;

	d_percpu = debugfs_create_dir("per_cpu", d_tracer);

	if (!d_percpu && !once) {
		once = 1;
		pr_warning("Could not create debugfs directory 'per_cpu'\n");
		return NULL;
	}

	return d_percpu;
}

static void tracing_init_debugfs_percpu(long cpu)
{
	struct dentry *d_percpu = tracing_dentry_percpu();
4474
	struct dentry *d_cpu;
4475
	char cpu_dir[30]; /* 30 characters should be more than enough */
4476

4477 4478 4479
	if (!d_percpu)
		return;

4480
	snprintf(cpu_dir, 30, "cpu%ld", cpu);
4481 4482 4483 4484 4485
	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
	if (!d_cpu) {
		pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
		return;
	}
4486

4487
	/* per cpu trace_pipe */
4488 4489
	trace_create_file("trace_pipe", 0444, d_cpu,
			(void *) cpu, &tracing_pipe_fops);
4490 4491

	/* per cpu trace */
4492 4493
	trace_create_file("trace", 0644, d_cpu,
			(void *) cpu, &tracing_fops);
4494

4495 4496
	trace_create_file("trace_pipe_raw", 0444, d_cpu,
			(void *) cpu, &tracing_buffers_fops);
4497

4498 4499
	trace_create_file("stats", 0444, d_cpu,
			(void *) cpu, &tracing_stats_fops);
4500 4501 4502

	trace_create_file("buffer_size_kb", 0444, d_cpu,
			(void *) cpu, &tracing_entries_fops);
4503 4504
}

S
Steven Rostedt 已提交
4505 4506 4507 4508 4509
#ifdef CONFIG_FTRACE_SELFTEST
/* Let selftest have access to static functions in this file */
#include "trace_selftest.c"
#endif

4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538
struct trace_option_dentry {
	struct tracer_opt		*opt;
	struct tracer_flags		*flags;
	struct dentry			*entry;
};

static ssize_t
trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
			loff_t *ppos)
{
	struct trace_option_dentry *topt = filp->private_data;
	char *buf;

	if (topt->flags->val & topt->opt->bit)
		buf = "1\n";
	else
		buf = "0\n";

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
}

static ssize_t
trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
			 loff_t *ppos)
{
	struct trace_option_dentry *topt = filp->private_data;
	unsigned long val;
	int ret;

4539 4540
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
4541 4542
		return ret;

L
Li Zefan 已提交
4543 4544
	if (val != 0 && val != 1)
		return -EINVAL;
4545

L
Li Zefan 已提交
4546
	if (!!(topt->flags->val & topt->opt->bit) != val) {
4547
		mutex_lock(&trace_types_lock);
L
Li Zefan 已提交
4548
		ret = __set_tracer_option(current_trace, topt->flags,
4549
					  topt->opt, !val);
4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564
		mutex_unlock(&trace_types_lock);
		if (ret)
			return ret;
	}

	*ppos += cnt;

	return cnt;
}


static const struct file_operations trace_options_fops = {
	.open = tracing_open_generic,
	.read = trace_options_read,
	.write = trace_options_write,
4565
	.llseek	= generic_file_llseek,
4566 4567
};

4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590
static ssize_t
trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
			loff_t *ppos)
{
	long index = (long)filp->private_data;
	char *buf;

	if (trace_flags & (1 << index))
		buf = "1\n";
	else
		buf = "0\n";

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
}

static ssize_t
trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
			 loff_t *ppos)
{
	long index = (long)filp->private_data;
	unsigned long val;
	int ret;

4591 4592
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
4593 4594
		return ret;

4595
	if (val != 0 && val != 1)
4596
		return -EINVAL;
4597
	set_tracer_flags(1 << index, val);
4598 4599 4600 4601 4602 4603 4604 4605 4606 4607

	*ppos += cnt;

	return cnt;
}

static const struct file_operations trace_options_core_fops = {
	.open = tracing_open_generic,
	.read = trace_options_core_read,
	.write = trace_options_core_write,
4608
	.llseek = generic_file_llseek,
4609 4610
};

4611
struct dentry *trace_create_file(const char *name,
A
Al Viro 已提交
4612
				 umode_t mode,
4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626
				 struct dentry *parent,
				 void *data,
				 const struct file_operations *fops)
{
	struct dentry *ret;

	ret = debugfs_create_file(name, mode, parent, data, fops);
	if (!ret)
		pr_warning("Could not create debugfs '%s' entry\n", name);

	return ret;
}


4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647
static struct dentry *trace_options_init_dentry(void)
{
	struct dentry *d_tracer;
	static struct dentry *t_options;

	if (t_options)
		return t_options;

	d_tracer = tracing_init_dentry();
	if (!d_tracer)
		return NULL;

	t_options = debugfs_create_dir("options", d_tracer);
	if (!t_options) {
		pr_warning("Could not create debugfs directory 'options'\n");
		return NULL;
	}

	return t_options;
}

4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661
static void
create_trace_option_file(struct trace_option_dentry *topt,
			 struct tracer_flags *flags,
			 struct tracer_opt *opt)
{
	struct dentry *t_options;

	t_options = trace_options_init_dentry();
	if (!t_options)
		return;

	topt->flags = flags;
	topt->opt = opt;

4662
	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687
				    &trace_options_fops);

}

static struct trace_option_dentry *
create_trace_option_files(struct tracer *tracer)
{
	struct trace_option_dentry *topts;
	struct tracer_flags *flags;
	struct tracer_opt *opts;
	int cnt;

	if (!tracer)
		return NULL;

	flags = tracer->flags;

	if (!flags || !flags->opts)
		return NULL;

	opts = flags->opts;

	for (cnt = 0; opts[cnt].name; cnt++)
		;

4688
	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714
	if (!topts)
		return NULL;

	for (cnt = 0; opts[cnt].name; cnt++)
		create_trace_option_file(&topts[cnt], flags,
					 &opts[cnt]);

	return topts;
}

static void
destroy_trace_option_files(struct trace_option_dentry *topts)
{
	int cnt;

	if (!topts)
		return;

	for (cnt = 0; topts[cnt].opt; cnt++) {
		if (topts[cnt].entry)
			debugfs_remove(topts[cnt].entry);
	}

	kfree(topts);
}

4715 4716 4717 4718 4719 4720 4721 4722 4723
static struct dentry *
create_trace_option_core_file(const char *option, long index)
{
	struct dentry *t_options;

	t_options = trace_options_init_dentry();
	if (!t_options)
		return NULL;

4724
	return trace_create_file(option, 0644, t_options, (void *)index,
4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736
				    &trace_options_core_fops);
}

static __init void create_trace_options_dir(void)
{
	struct dentry *t_options;
	int i;

	t_options = trace_options_init_dentry();
	if (!t_options)
		return;

4737 4738
	for (i = 0; trace_options[i]; i++)
		create_trace_option_core_file(trace_options[i], i);
4739 4740
}

4741 4742 4743 4744
static ssize_t
rb_simple_read(struct file *filp, char __user *ubuf,
	       size_t cnt, loff_t *ppos)
{
4745 4746
	struct trace_array *tr = filp->private_data;
	struct ring_buffer *buffer = tr->buffer;
4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763
	char buf[64];
	int r;

	if (buffer)
		r = ring_buffer_record_is_on(buffer);
	else
		r = 0;

	r = sprintf(buf, "%d\n", r);

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

static ssize_t
rb_simple_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
4764 4765
	struct trace_array *tr = filp->private_data;
	struct ring_buffer *buffer = tr->buffer;
4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791
	unsigned long val;
	int ret;

	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
		return ret;

	if (buffer) {
		if (val)
			ring_buffer_record_on(buffer);
		else
			ring_buffer_record_off(buffer);
	}

	(*ppos)++;

	return cnt;
}

static const struct file_operations rb_simple_fops = {
	.open		= tracing_open_generic,
	.read		= rb_simple_read,
	.write		= rb_simple_write,
	.llseek		= default_llseek,
};

4792
static __init int tracer_init_debugfs(void)
4793 4794
{
	struct dentry *d_tracer;
4795
	int cpu;
4796

4797 4798
	trace_access_lock_init();

4799 4800
	d_tracer = tracing_init_dentry();

4801 4802
	trace_create_file("tracing_enabled", 0644, d_tracer,
			&global_trace, &tracing_ctrl_fops);
4803

4804 4805
	trace_create_file("trace_options", 0644, d_tracer,
			NULL, &tracing_iter_fops);
4806

4807 4808 4809 4810 4811
	trace_create_file("tracing_cpumask", 0644, d_tracer,
			NULL, &tracing_cpumask_fops);

	trace_create_file("trace", 0644, d_tracer,
			(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4812

4813 4814 4815
	trace_create_file("available_tracers", 0444, d_tracer,
			&global_trace, &show_traces_fops);

4816
	trace_create_file("current_tracer", 0644, d_tracer,
4817 4818
			&global_trace, &set_tracer_fops);

4819
#ifdef CONFIG_TRACER_MAX_TRACE
4820 4821
	trace_create_file("tracing_max_latency", 0644, d_tracer,
			&tracing_max_latency, &tracing_max_lat_fops);
4822
#endif
4823 4824 4825

	trace_create_file("tracing_thresh", 0644, d_tracer,
			&tracing_thresh, &tracing_max_lat_fops);
4826

4827
	trace_create_file("README", 0444, d_tracer,
4828 4829 4830
			NULL, &tracing_readme_fops);

	trace_create_file("trace_pipe", 0444, d_tracer,
4831
			(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4832 4833

	trace_create_file("buffer_size_kb", 0644, d_tracer,
4834
			(void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4835

4836 4837 4838
	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
			&global_trace, &tracing_total_entries_fops);

4839 4840 4841
	trace_create_file("free_buffer", 0644, d_tracer,
			&global_trace, &tracing_free_buffer_fops);

4842 4843
	trace_create_file("trace_marker", 0220, d_tracer,
			NULL, &tracing_mark_fops);
4844

4845 4846
	trace_create_file("saved_cmdlines", 0444, d_tracer,
			NULL, &tracing_saved_cmdlines_fops);
4847

4848 4849 4850
	trace_create_file("trace_clock", 0644, d_tracer, NULL,
			  &trace_clock_fops);

4851
	trace_create_file("tracing_on", 0644, d_tracer,
4852
			    &global_trace, &rb_simple_fops);
4853

4854
#ifdef CONFIG_DYNAMIC_FTRACE
4855 4856
	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
			&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4857
#endif
4858

4859 4860
	create_trace_options_dir();

4861 4862 4863
	for_each_tracing_cpu(cpu)
		tracing_init_debugfs_percpu(cpu);

4864
	return 0;
4865 4866
}

4867 4868 4869
static int trace_panic_handler(struct notifier_block *this,
			       unsigned long event, void *unused)
{
4870
	if (ftrace_dump_on_oops)
4871
		ftrace_dump(ftrace_dump_on_oops);
4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886
	return NOTIFY_OK;
}

static struct notifier_block trace_panic_notifier = {
	.notifier_call  = trace_panic_handler,
	.next           = NULL,
	.priority       = 150   /* priority: INT_MAX >= x >= 0 */
};

static int trace_die_handler(struct notifier_block *self,
			     unsigned long val,
			     void *data)
{
	switch (val) {
	case DIE_OOPS:
4887
		if (ftrace_dump_on_oops)
4888
			ftrace_dump(ftrace_dump_on_oops);
4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block trace_die_notifier = {
	.notifier_call = trace_die_handler,
	.priority = 200
};

/*
 * printk is set to max of 1024, we really don't need it that big.
 * Nothing should be printing 1000 characters anyway.
 */
#define TRACE_MAX_PRINT		1000

/*
 * Define here KERN_TRACE so that we have one place to modify
 * it if we decide to change what log level the ftrace dump
 * should be at.
 */
4912
#define KERN_TRACE		KERN_EMERG
4913

4914
void
4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925
trace_printk_seq(struct trace_seq *s)
{
	/* Probably should print a warning here. */
	if (s->len >= 1000)
		s->len = 1000;

	/* should be zero ended, but we are paranoid. */
	s->buffer[s->len] = 0;

	printk(KERN_TRACE "%s", s->buffer);

4926
	trace_seq_init(s);
4927 4928
}

4929 4930 4931 4932 4933 4934 4935
void trace_init_global_iter(struct trace_iterator *iter)
{
	iter->tr = &global_trace;
	iter->trace = current_trace;
	iter->cpu_file = TRACE_PIPE_ALL_CPU;
}

4936 4937
static void
__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
4938
{
4939
	static arch_spinlock_t ftrace_dump_lock =
4940
		(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4941 4942
	/* use static because iter can be a bit big for the stack */
	static struct trace_iterator iter;
4943
	unsigned int old_userobj;
4944
	static int dump_ran;
4945 4946
	unsigned long flags;
	int cnt = 0, cpu;
4947 4948

	/* only one dump */
4949
	local_irq_save(flags);
4950
	arch_spin_lock(&ftrace_dump_lock);
4951 4952 4953 4954 4955
	if (dump_ran)
		goto out;

	dump_ran = 1;

4956
	tracing_off();
4957

4958 4959 4960 4961 4962 4963
	/* Did function tracer already get disabled? */
	if (ftrace_is_dead()) {
		printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
		printk("#          MAY BE MISSING FUNCTION EVENTS\n");
	}

4964 4965
	if (disable_tracing)
		ftrace_kill();
4966

4967 4968
	trace_init_global_iter(&iter);

4969
	for_each_tracing_cpu(cpu) {
4970
		atomic_inc(&iter.tr->data[cpu]->disabled);
4971 4972
	}

4973 4974
	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;

4975 4976 4977
	/* don't look at user memory in panic mode */
	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;

4978
	/* Simulate the iterator */
4979 4980
	iter.tr = &global_trace;
	iter.trace = current_trace;
4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996

	switch (oops_dump_mode) {
	case DUMP_ALL:
		iter.cpu_file = TRACE_PIPE_ALL_CPU;
		break;
	case DUMP_ORIG:
		iter.cpu_file = raw_smp_processor_id();
		break;
	case DUMP_NONE:
		goto out_enable;
	default:
		printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
		iter.cpu_file = TRACE_PIPE_ALL_CPU;
	}

	printk(KERN_TRACE "Dumping ftrace buffer:\n");
4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018

	/*
	 * We need to stop all tracing on all CPUS to read the
	 * the next buffer. This is a bit expensive, but is
	 * not done often. We fill all what we can read,
	 * and then release the locks again.
	 */

	while (!trace_empty(&iter)) {

		if (!cnt)
			printk(KERN_TRACE "---------------------------------\n");

		cnt++;

		/* reset all but tr, trace, and overruns */
		memset(&iter.seq, 0,
		       sizeof(struct trace_iterator) -
		       offsetof(struct trace_iterator, seq));
		iter.iter_flags |= TRACE_FILE_LAT_FMT;
		iter.pos = -1;

5019
		if (trace_find_next_entry_inc(&iter) != NULL) {
5020 5021 5022 5023 5024
			int ret;

			ret = print_trace_line(&iter);
			if (ret != TRACE_TYPE_NO_CONSUME)
				trace_consume(&iter);
5025
		}
5026
		touch_nmi_watchdog();
5027 5028 5029 5030 5031 5032 5033 5034 5035

		trace_printk_seq(&iter.seq);
	}

	if (!cnt)
		printk(KERN_TRACE "   (ftrace buffer empty)\n");
	else
		printk(KERN_TRACE "---------------------------------\n");

5036
 out_enable:
5037 5038 5039 5040 5041
	/* Re-enable tracing if requested */
	if (!disable_tracing) {
		trace_flags |= old_userobj;

		for_each_tracing_cpu(cpu) {
5042
			atomic_dec(&iter.tr->data[cpu]->disabled);
5043 5044 5045 5046
		}
		tracing_on();
	}

5047
 out:
5048
	arch_spin_unlock(&ftrace_dump_lock);
5049
	local_irq_restore(flags);
5050 5051
}

5052
/* By default: disable tracing after the dump */
5053
void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5054
{
5055
	__ftrace_dump(true, oops_dump_mode);
5056
}
5057
EXPORT_SYMBOL_GPL(ftrace_dump);
5058

5059
__init static int tracer_alloc_buffers(void)
5060
{
5061
	int ring_buf_size;
5062
	enum ring_buffer_flags rb_flags;
5063
	int i;
5064
	int ret = -ENOMEM;
5065

5066

5067 5068 5069 5070 5071
	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
		goto out;

	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
		goto out_free_buffer_mask;
5072

5073 5074 5075 5076
	/* Only allocate trace_printk buffers if a trace_printk exists */
	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
		trace_printk_init_buffers();

5077 5078 5079 5080 5081 5082
	/* To save memory, keep the ring buffer size to its minimum */
	if (ring_buffer_expanded)
		ring_buf_size = trace_buf_size;
	else
		ring_buf_size = 1;

5083 5084
	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;

5085 5086 5087 5088
	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
	cpumask_copy(tracing_cpumask, cpu_all_mask);

	/* TODO: make the number of buffers hot pluggable with CPUS */
5089
	global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5090 5091 5092
	if (!global_trace.buffer) {
		printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
		WARN_ON(1);
5093
		goto out_free_cpumask;
5094
	}
5095 5096
	if (global_trace.buffer_disabled)
		tracing_off();
5097

5098

5099
#ifdef CONFIG_TRACER_MAX_TRACE
5100
	max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5101 5102 5103 5104
	if (!max_tr.buffer) {
		printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
		WARN_ON(1);
		ring_buffer_free(global_trace.buffer);
5105
		goto out_free_cpumask;
5106
	}
5107
#endif
5108

5109
	/* Allocate the first page for all buffers */
5110
	for_each_tracing_cpu(i) {
5111
		global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5112
		max_tr.data[i] = &per_cpu(max_tr_data, i);
5113
	}
5114

5115 5116
	set_buffer_entries(&global_trace,
			   ring_buffer_size(global_trace.buffer, 0));
5117 5118 5119
#ifdef CONFIG_TRACER_MAX_TRACE
	set_buffer_entries(&max_tr, 1);
#endif
5120 5121 5122

	trace_init_cmdlines();

5123
	register_tracer(&nop_trace);
5124
	current_trace = &nop_trace;
S
Steven Rostedt 已提交
5125 5126
	/* All seems OK, enable tracing */
	tracing_disabled = 0;
5127

5128 5129 5130 5131
	atomic_notifier_chain_register(&panic_notifier_list,
				       &trace_panic_notifier);

	register_die_notifier(&trace_die_notifier);
5132 5133

	return 0;
5134

5135 5136 5137 5138 5139 5140
out_free_cpumask:
	free_cpumask_var(tracing_cpumask);
out_free_buffer_mask:
	free_cpumask_var(tracing_buffer_mask);
out:
	return ret;
5141
}
5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161

__init static int clear_boot_tracer(void)
{
	/*
	 * The default tracer at boot buffer is an init section.
	 * This function is called in lateinit. If we did not
	 * find the boot tracer, then clear it out, to prevent
	 * later registration from accessing the buffer that is
	 * about to be freed.
	 */
	if (!default_bootup_tracer)
		return 0;

	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
	       default_bootup_tracer);
	default_bootup_tracer = NULL;

	return 0;
}

5162 5163
early_initcall(tracer_alloc_buffers);
fs_initcall(tracer_init_debugfs);
5164
late_initcall(clear_boot_tracer);