trace.c 163.3 KB
Newer Older
1 2 3
/*
 * ring buffer based function tracer
 *
4
 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 6 7 8 9 10 11
 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
 *
 * Originally taken from the RT patch by:
 *    Arnaldo Carvalho de Melo <acme@redhat.com>
 *
 * Based on code from the latency_tracer, that is:
 *  Copyright (C) 2004-2006 Ingo Molnar
12
 *  Copyright (C) 2004 Nadia Yvette Chambers
13
 */
14
#include <linux/ring_buffer.h>
15
#include <generated/utsrelease.h>
16 17
#include <linux/stacktrace.h>
#include <linux/writeback.h>
18 19
#include <linux/kallsyms.h>
#include <linux/seq_file.h>
20
#include <linux/notifier.h>
21
#include <linux/irqflags.h>
22
#include <linux/debugfs.h>
23
#include <linux/pagemap.h>
24 25 26
#include <linux/hardirq.h>
#include <linux/linkage.h>
#include <linux/uaccess.h>
27
#include <linux/kprobes.h>
28 29 30
#include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/percpu.h>
31
#include <linux/splice.h>
32
#include <linux/kdebug.h>
33
#include <linux/string.h>
34
#include <linux/rwsem.h>
35
#include <linux/slab.h>
36 37
#include <linux/ctype.h>
#include <linux/init.h>
38
#include <linux/poll.h>
39
#include <linux/nmi.h>
40
#include <linux/fs.h>
41
#include <linux/sched/rt.h>
I
Ingo Molnar 已提交
42

43
#include "trace.h"
44
#include "trace_output.h"
45

46 47 48 49
/*
 * On boot up, the ring buffer is set to the minimum size, so that
 * we do not waste memory on systems that are not using tracing.
 */
50
bool ring_buffer_expanded;
51

52 53
/*
 * We need to change this state when a selftest is running.
54 55
 * A selftest will lurk into the ring-buffer to count the
 * entries inserted during the selftest although some concurrent
56
 * insertions into the ring-buffer such as trace_printk could occurred
57 58
 * at the same time, giving false positive or negative results.
 */
59
static bool __read_mostly tracing_selftest_running;
60

61 62 63
/*
 * If a tracer is running, we do not want to run SELFTEST.
 */
64
bool __read_mostly tracing_selftest_disabled;
65

66 67 68 69 70 71 72 73 74 75
/* For tracers that don't implement custom flags */
static struct tracer_opt dummy_tracer_opt[] = {
	{ }
};

static struct tracer_flags dummy_tracer_flags = {
	.val = 0,
	.opts = dummy_tracer_opt
};

76 77
static int
dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
78 79 80
{
	return 0;
}
81

82 83 84 85 86 87 88
/*
 * To prevent the comm cache from being overwritten when no
 * tracing is active, only save the comm when a trace event
 * occurred.
 */
static DEFINE_PER_CPU(bool, trace_cmdline_save);

89 90 91 92 93 94
/*
 * Kill all tracing for good (never come back).
 * It is initialized to 1 but will turn to zero if the initialization
 * of the tracer is successful. But that is the only place that sets
 * this back to zero.
 */
95
static int tracing_disabled = 1;
96

97
DEFINE_PER_CPU(int, ftrace_cpu_disabled);
98

99
cpumask_var_t __read_mostly	tracing_buffer_mask;
100

101 102 103 104 105 106 107 108 109 110 111
/*
 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
 *
 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
 * is set, then ftrace_dump is called. This will output the contents
 * of the ftrace buffers to the console.  This is very useful for
 * capturing traces that lead to crashes and outputing it to a
 * serial console.
 *
 * It is default off, but you can enable it with either specifying
 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 113 114
 * /proc/sys/kernel/ftrace_dump_on_oops
 * Set 1 if you want to dump buffers of all CPUs
 * Set 2 if you want to dump the buffer of the CPU that triggered oops
115
 */
116 117

enum ftrace_dump_mode ftrace_dump_on_oops;
118

119 120 121
/* When set, tracing will stop when a WARN*() is hit */
int __disable_trace_on_warning;

122
static int tracing_set_tracer(struct trace_array *tr, const char *buf);
123

L
Li Zefan 已提交
124 125
#define MAX_TRACER_SIZE		100
static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126
static char *default_bootup_tracer;
127

128 129
static bool allocate_snapshot;

130
static int __init set_cmdline_ftrace(char *str)
131
{
132
	strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133
	default_bootup_tracer = bootup_tracer_buf;
134
	/* We are using ftrace early, expand it */
135
	ring_buffer_expanded = true;
136 137
	return 1;
}
138
__setup("ftrace=", set_cmdline_ftrace);
139

140 141
static int __init set_ftrace_dump_on_oops(char *str)
{
142 143 144 145 146 147 148 149 150 151 152
	if (*str++ != '=' || !*str) {
		ftrace_dump_on_oops = DUMP_ALL;
		return 1;
	}

	if (!strcmp("orig_cpu", str)) {
		ftrace_dump_on_oops = DUMP_ORIG;
                return 1;
        }

        return 0;
153 154
}
__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
S
Steven Rostedt 已提交
155

156 157 158 159 160 161 162
static int __init stop_trace_on_warning(char *str)
{
	__disable_trace_on_warning = 1;
	return 1;
}
__setup("traceoff_on_warning=", stop_trace_on_warning);

163
static int __init boot_alloc_snapshot(char *str)
164 165 166 167 168 169
{
	allocate_snapshot = true;
	/* We also need the main ring buffer expanded */
	ring_buffer_expanded = true;
	return 1;
}
170
__setup("alloc_snapshot", boot_alloc_snapshot);
171

172 173 174 175 176 177

static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
static char *trace_boot_options __initdata;

static int __init set_trace_boot_options(char *str)
{
178
	strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
179 180 181 182 183
	trace_boot_options = trace_boot_options_buf;
	return 0;
}
__setup("trace_options=", set_trace_boot_options);

184 185 186 187 188 189 190 191 192 193 194
static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
static char *trace_boot_clock __initdata;

static int __init set_trace_boot_clock(char *str)
{
	strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
	trace_boot_clock = trace_boot_clock_buf;
	return 0;
}
__setup("trace_clock=", set_trace_boot_clock);

195

196
unsigned long long ns2usecs(cycle_t nsec)
197 198 199 200 201 202
{
	nsec += 500;
	do_div(nsec, 1000);
	return nsec;
}

S
Steven Rostedt 已提交
203 204 205 206 207 208 209 210 211 212 213 214
/*
 * The global_trace is the descriptor that holds the tracing
 * buffers for the live tracing. For each CPU, it contains
 * a link list of pages that will store trace entries. The
 * page descriptor of the pages in the memory is used to hold
 * the link list by linking the lru item in the page descriptor
 * to each of the pages in the buffer per CPU.
 *
 * For each active CPU there is a data field that holds the
 * pages for the buffer for that CPU. Each CPU has the same number
 * of pages allocated for its buffer.
 */
215 216
static struct trace_array	global_trace;

217
LIST_HEAD(ftrace_trace_arrays);
218

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
int trace_array_get(struct trace_array *this_tr)
{
	struct trace_array *tr;
	int ret = -ENODEV;

	mutex_lock(&trace_types_lock);
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (tr == this_tr) {
			tr->ref++;
			ret = 0;
			break;
		}
	}
	mutex_unlock(&trace_types_lock);

	return ret;
}

static void __trace_array_put(struct trace_array *this_tr)
{
	WARN_ON(!this_tr->ref);
	this_tr->ref--;
}

void trace_array_put(struct trace_array *this_tr)
{
	mutex_lock(&trace_types_lock);
	__trace_array_put(this_tr);
	mutex_unlock(&trace_types_lock);
}

250 251 252
int filter_check_discard(struct ftrace_event_file *file, void *rec,
			 struct ring_buffer *buffer,
			 struct ring_buffer_event *event)
253
{
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
	if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
	    !filter_match_preds(file->filter, rec)) {
		ring_buffer_discard_commit(buffer, event);
		return 1;
	}

	return 0;
}
EXPORT_SYMBOL_GPL(filter_check_discard);

int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
			      struct ring_buffer *buffer,
			      struct ring_buffer_event *event)
{
	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
	    !filter_match_preds(call->filter, rec)) {
		ring_buffer_discard_commit(buffer, event);
		return 1;
	}

	return 0;
275
}
276
EXPORT_SYMBOL_GPL(call_filter_check_discard);
277

278
static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
279 280 281 282
{
	u64 ts;

	/* Early boot up does not have a buffer yet */
283
	if (!buf->buffer)
284 285
		return trace_clock_local();

286 287
	ts = ring_buffer_time_stamp(buf->buffer, cpu);
	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
288 289 290

	return ts;
}
291

292 293 294 295 296
cycle_t ftrace_now(int cpu)
{
	return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
}

297 298 299 300 301 302 303 304 305
/**
 * tracing_is_enabled - Show if global_trace has been disabled
 *
 * Shows if the global trace has been enabled or not. It uses the
 * mirror flag "buffer_disabled" to be used in fast paths such as for
 * the irqsoff tracer. But it may be inaccurate due to races. If you
 * need to know the accurate state, use tracing_is_on() which is a little
 * slower, but accurate.
 */
306 307
int tracing_is_enabled(void)
{
308 309 310 311 312 313 314
	/*
	 * For quick access (irqsoff uses this in fast path), just
	 * return the mirror variable of the state of the ring buffer.
	 * It's a little racy, but we don't really care.
	 */
	smp_rmb();
	return !global_trace.buffer_disabled;
315 316
}

S
Steven Rostedt 已提交
317
/*
318 319 320
 * trace_buf_size is the size in bytes that is allocated
 * for a buffer. Note, the number of bytes is always rounded
 * to page size.
321 322 323 324 325
 *
 * This number is purposely set to a low number of 16384.
 * If the dump on oops happens, it will be much appreciated
 * to not have to wait for all that output. Anyway this can be
 * boot time and run time configurable.
S
Steven Rostedt 已提交
326
 */
327
#define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */
328

329
static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
330

S
Steven Rostedt 已提交
331
/* trace_types holds a link list of available tracers. */
332
static struct tracer		*trace_types __read_mostly;
S
Steven Rostedt 已提交
333 334 335 336

/*
 * trace_types_lock is used to protect the trace_types list.
 */
337
DEFINE_MUTEX(trace_types_lock);
S
Steven Rostedt 已提交
338

339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
/*
 * serialize the access of the ring buffer
 *
 * ring buffer serializes readers, but it is low level protection.
 * The validity of the events (which returns by ring_buffer_peek() ..etc)
 * are not protected by ring buffer.
 *
 * The content of events may become garbage if we allow other process consumes
 * these events concurrently:
 *   A) the page of the consumed events may become a normal page
 *      (not reader page) in ring buffer, and this page will be rewrited
 *      by events producer.
 *   B) The page of the consumed events may become a page for splice_read,
 *      and this page will be returned to system.
 *
 * These primitives allow multi process access to different cpu ring buffer
 * concurrently.
 *
 * These primitives don't distinguish read-only and read-consume access.
 * Multi read-only access are also serialized.
 */

#ifdef CONFIG_SMP
static DECLARE_RWSEM(all_cpu_access_lock);
static DEFINE_PER_CPU(struct mutex, cpu_access_lock);

static inline void trace_access_lock(int cpu)
{
367
	if (cpu == RING_BUFFER_ALL_CPUS) {
368 369 370 371 372
		/* gain it for accessing the whole ring buffer. */
		down_write(&all_cpu_access_lock);
	} else {
		/* gain it for accessing a cpu ring buffer. */

373
		/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
374 375 376 377 378 379 380 381 382
		down_read(&all_cpu_access_lock);

		/* Secondly block other access to this @cpu ring buffer. */
		mutex_lock(&per_cpu(cpu_access_lock, cpu));
	}
}

static inline void trace_access_unlock(int cpu)
{
383
	if (cpu == RING_BUFFER_ALL_CPUS) {
384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
		up_write(&all_cpu_access_lock);
	} else {
		mutex_unlock(&per_cpu(cpu_access_lock, cpu));
		up_read(&all_cpu_access_lock);
	}
}

static inline void trace_access_lock_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu)
		mutex_init(&per_cpu(cpu_access_lock, cpu));
}

#else

static DEFINE_MUTEX(access_lock);

static inline void trace_access_lock(int cpu)
{
	(void)cpu;
	mutex_lock(&access_lock);
}

static inline void trace_access_unlock(int cpu)
{
	(void)cpu;
	mutex_unlock(&access_lock);
}

static inline void trace_access_lock_init(void)
{
}

#endif

421
/* trace_flags holds trace_options default values */
422
unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
423
	TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
424
	TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
425
	TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
426

427
static void tracer_tracing_on(struct trace_array *tr)
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
{
	if (tr->trace_buffer.buffer)
		ring_buffer_record_on(tr->trace_buffer.buffer);
	/*
	 * This flag is looked at when buffers haven't been allocated
	 * yet, or by some tracers (like irqsoff), that just want to
	 * know if the ring buffer has been disabled, but it can handle
	 * races of where it gets disabled but we still do a record.
	 * As the check is in the fast path of the tracers, it is more
	 * important to be fast than accurate.
	 */
	tr->buffer_disabled = 0;
	/* Make the flag seen by readers */
	smp_wmb();
}

444 445 446 447 448 449 450 451
/**
 * tracing_on - enable tracing buffers
 *
 * This function enables tracing buffers that may have been
 * disabled with tracing_off.
 */
void tracing_on(void)
{
452
	tracer_tracing_on(&global_trace);
453 454 455
}
EXPORT_SYMBOL_GPL(tracing_on);

456 457 458 459 460 461 462 463 464 465 466 467 468
/**
 * __trace_puts - write a constant string into the trace buffer.
 * @ip:	   The address of the caller
 * @str:   The constant string to write
 * @size:  The size of the string.
 */
int __trace_puts(unsigned long ip, const char *str, int size)
{
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	struct print_entry *entry;
	unsigned long irq_flags;
	int alloc;
469 470
	int pc;

471 472 473
	if (!(trace_flags & TRACE_ITER_PRINTK))
		return 0;

474
	pc = preempt_count();
475

476 477 478
	if (unlikely(tracing_selftest_running || tracing_disabled))
		return 0;

479 480 481 482 483
	alloc = sizeof(*entry) + size + 2; /* possible \n added */

	local_save_flags(irq_flags);
	buffer = global_trace.trace_buffer.buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
484
					  irq_flags, pc);
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
	if (!event)
		return 0;

	entry = ring_buffer_event_data(event);
	entry->ip = ip;

	memcpy(&entry->buf, str, size);

	/* Add a newline if necessary */
	if (entry->buf[size - 1] != '\n') {
		entry->buf[size] = '\n';
		entry->buf[size + 1] = '\0';
	} else
		entry->buf[size] = '\0';

	__buffer_unlock_commit(buffer, event);
501
	ftrace_trace_stack(buffer, irq_flags, 4, pc);
502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518

	return size;
}
EXPORT_SYMBOL_GPL(__trace_puts);

/**
 * __trace_bputs - write the pointer to a constant string into trace buffer
 * @ip:	   The address of the caller
 * @str:   The constant string to write to the buffer to
 */
int __trace_bputs(unsigned long ip, const char *str)
{
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	struct bputs_entry *entry;
	unsigned long irq_flags;
	int size = sizeof(struct bputs_entry);
519 520
	int pc;

521 522 523
	if (!(trace_flags & TRACE_ITER_PRINTK))
		return 0;

524
	pc = preempt_count();
525

526 527 528
	if (unlikely(tracing_selftest_running || tracing_disabled))
		return 0;

529 530 531
	local_save_flags(irq_flags);
	buffer = global_trace.trace_buffer.buffer;
	event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
532
					  irq_flags, pc);
533 534 535 536 537 538 539 540
	if (!event)
		return 0;

	entry = ring_buffer_event_data(event);
	entry->ip			= ip;
	entry->str			= str;

	__buffer_unlock_commit(buffer, event);
541
	ftrace_trace_stack(buffer, irq_flags, 4, pc);
542 543 544 545 546

	return 1;
}
EXPORT_SYMBOL_GPL(__trace_bputs);

547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567
#ifdef CONFIG_TRACER_SNAPSHOT
/**
 * trace_snapshot - take a snapshot of the current buffer.
 *
 * This causes a swap between the snapshot buffer and the current live
 * tracing buffer. You can use this to take snapshots of the live
 * trace when some condition is triggered, but continue to trace.
 *
 * Note, make sure to allocate the snapshot with either
 * a tracing_snapshot_alloc(), or by doing it manually
 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
 *
 * If the snapshot buffer is not allocated, it will stop tracing.
 * Basically making a permanent snapshot.
 */
void tracing_snapshot(void)
{
	struct trace_array *tr = &global_trace;
	struct tracer *tracer = tr->current_trace;
	unsigned long flags;

568 569 570 571 572 573
	if (in_nmi()) {
		internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
		internal_trace_puts("*** snapshot is being ignored        ***\n");
		return;
	}

574
	if (!tr->allocated_snapshot) {
575 576
		internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
		internal_trace_puts("*** stopping trace here!   ***\n");
577 578 579 580 581 582
		tracing_off();
		return;
	}

	/* Note, snapshot can not be used when the tracer uses it */
	if (tracer->use_max_tr) {
583 584
		internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
		internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
585 586 587 588 589 590 591
		return;
	}

	local_irq_save(flags);
	update_max_tr(tr, current, smp_processor_id());
	local_irq_restore(flags);
}
592
EXPORT_SYMBOL_GPL(tracing_snapshot);
593 594 595

static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
					struct trace_buffer *size_buf, int cpu_id);
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);

static int alloc_snapshot(struct trace_array *tr)
{
	int ret;

	if (!tr->allocated_snapshot) {

		/* allocate spare buffer */
		ret = resize_buffer_duplicate_size(&tr->max_buffer,
				   &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
		if (ret < 0)
			return ret;

		tr->allocated_snapshot = true;
	}

	return 0;
}

616
static void free_snapshot(struct trace_array *tr)
617 618 619 620 621 622 623 624 625 626 627
{
	/*
	 * We don't free the ring buffer. instead, resize it because
	 * The max_tr ring buffer has some state (e.g. ring->clock) and
	 * we want preserve it.
	 */
	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
	set_buffer_entries(&tr->max_buffer, 1);
	tracing_reset_online_cpus(&tr->max_buffer);
	tr->allocated_snapshot = false;
}
628

629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
/**
 * tracing_alloc_snapshot - allocate snapshot buffer.
 *
 * This only allocates the snapshot buffer if it isn't already
 * allocated - it doesn't also take a snapshot.
 *
 * This is meant to be used in cases where the snapshot buffer needs
 * to be set up for events that can't sleep but need to be able to
 * trigger a snapshot.
 */
int tracing_alloc_snapshot(void)
{
	struct trace_array *tr = &global_trace;
	int ret;

	ret = alloc_snapshot(tr);
	WARN_ON(ret < 0);

	return ret;
}
EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);

651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
/**
 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
 *
 * This is similar to trace_snapshot(), but it will allocate the
 * snapshot buffer if it isn't already allocated. Use this only
 * where it is safe to sleep, as the allocation may sleep.
 *
 * This causes a swap between the snapshot buffer and the current live
 * tracing buffer. You can use this to take snapshots of the live
 * trace when some condition is triggered, but continue to trace.
 */
void tracing_snapshot_alloc(void)
{
	int ret;

666 667
	ret = tracing_alloc_snapshot();
	if (ret < 0)
668
		return;
669 670 671

	tracing_snapshot();
}
672
EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
673 674 675 676 677
#else
void tracing_snapshot(void)
{
	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
}
678
EXPORT_SYMBOL_GPL(tracing_snapshot);
679 680 681 682 683 684
int tracing_alloc_snapshot(void)
{
	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
	return -ENODEV;
}
EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
685 686 687 688 689
void tracing_snapshot_alloc(void)
{
	/* Give warning */
	tracing_snapshot();
}
690
EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
691 692
#endif /* CONFIG_TRACER_SNAPSHOT */

693
static void tracer_tracing_off(struct trace_array *tr)
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
{
	if (tr->trace_buffer.buffer)
		ring_buffer_record_off(tr->trace_buffer.buffer);
	/*
	 * This flag is looked at when buffers haven't been allocated
	 * yet, or by some tracers (like irqsoff), that just want to
	 * know if the ring buffer has been disabled, but it can handle
	 * races of where it gets disabled but we still do a record.
	 * As the check is in the fast path of the tracers, it is more
	 * important to be fast than accurate.
	 */
	tr->buffer_disabled = 1;
	/* Make the flag seen by readers */
	smp_wmb();
}

710 711 712 713 714 715 716 717 718 719
/**
 * tracing_off - turn off tracing buffers
 *
 * This function stops the tracing buffers from recording data.
 * It does not disable any overhead the tracers themselves may
 * be causing. This function simply causes all recording to
 * the ring buffers to fail.
 */
void tracing_off(void)
{
720
	tracer_tracing_off(&global_trace);
721 722 723
}
EXPORT_SYMBOL_GPL(tracing_off);

724 725 726 727 728 729
void disable_trace_on_warning(void)
{
	if (__disable_trace_on_warning)
		tracing_off();
}

730 731 732 733 734 735
/**
 * tracer_tracing_is_on - show real state of ring buffer enabled
 * @tr : the trace array to know if ring buffer is enabled
 *
 * Shows real state of the ring buffer if it is enabled or not.
 */
736
static int tracer_tracing_is_on(struct trace_array *tr)
737 738 739 740 741 742
{
	if (tr->trace_buffer.buffer)
		return ring_buffer_record_is_on(tr->trace_buffer.buffer);
	return !tr->buffer_disabled;
}

743 744 745 746 747
/**
 * tracing_is_on - show state of ring buffers enabled
 */
int tracing_is_on(void)
{
748
	return tracer_tracing_is_on(&global_trace);
749 750 751
}
EXPORT_SYMBOL_GPL(tracing_is_on);

752
static int __init set_buf_size(char *str)
753
{
754
	unsigned long buf_size;
755

756 757
	if (!str)
		return 0;
758
	buf_size = memparse(str, &str);
759
	/* nr_entries can not be zero */
760
	if (buf_size == 0)
761
		return 0;
762
	trace_buf_size = buf_size;
763 764
	return 1;
}
765
__setup("trace_buf_size=", set_buf_size);
766

767 768
static int __init set_tracing_thresh(char *str)
{
769
	unsigned long threshold;
770 771 772 773
	int ret;

	if (!str)
		return 0;
774
	ret = kstrtoul(str, 0, &threshold);
775 776
	if (ret < 0)
		return 0;
777
	tracing_thresh = threshold * 1000;
778 779 780 781
	return 1;
}
__setup("tracing_thresh=", set_tracing_thresh);

S
Steven Rostedt 已提交
782 783 784 785 786
unsigned long nsecs_to_usecs(unsigned long nsecs)
{
	return nsecs / 1000;
}

S
Steven Rostedt 已提交
787
/* These must match the bit postions in trace_iterator_flags */
788 789 790 791 792
static const char *trace_options[] = {
	"print-parent",
	"sym-offset",
	"sym-addr",
	"verbose",
I
Ingo Molnar 已提交
793
	"raw",
794
	"hex",
I
Ingo Molnar 已提交
795
	"bin",
796
	"block",
I
Ingo Molnar 已提交
797
	"stacktrace",
798
	"trace_printk",
799
	"ftrace_preempt",
800
	"branch",
801
	"annotate",
802
	"userstacktrace",
803
	"sym-userobj",
804
	"printk-msg-only",
805
	"context-info",
806
	"latency-format",
807
	"sleep-time",
808
	"graph-time",
809
	"record-cmd",
810
	"overwrite",
811
	"disable_on_free",
812
	"irq-info",
813
	"markers",
814
	"function-trace",
815 816 817
	NULL
};

818 819 820
static struct {
	u64 (*func)(void);
	const char *name;
821
	int in_ns;		/* is this clock in nanoseconds? */
822
} trace_clocks[] = {
823 824 825
	{ trace_clock_local,	"local",	1 },
	{ trace_clock_global,	"global",	1 },
	{ trace_clock_counter,	"counter",	0 },
826
	{ trace_clock_jiffies,	"uptime",	0 },
827
	{ trace_clock,		"perf",		1 },
D
David Sharp 已提交
828
	ARCH_TRACE_CLOCKS
829 830
};

831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
/*
 * trace_parser_get_init - gets the buffer for trace parser
 */
int trace_parser_get_init(struct trace_parser *parser, int size)
{
	memset(parser, 0, sizeof(*parser));

	parser->buffer = kmalloc(size, GFP_KERNEL);
	if (!parser->buffer)
		return 1;

	parser->size = size;
	return 0;
}

/*
 * trace_parser_put - frees the buffer for trace parser
 */
void trace_parser_put(struct trace_parser *parser)
{
	kfree(parser->buffer);
}

/*
 * trace_get_user - reads the user input string separated by  space
 * (matched by isspace(ch))
 *
 * For each string found the 'struct trace_parser' is updated,
 * and the function returns.
 *
 * Returns number of bytes read.
 *
 * See kernel/trace/trace.h for 'struct trace_parser' details.
 */
int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
	size_t cnt, loff_t *ppos)
{
	char ch;
	size_t read = 0;
	ssize_t ret;

	if (!*ppos)
		trace_parser_clear(parser);

	ret = get_user(ch, ubuf++);
	if (ret)
		goto out;

	read++;
	cnt--;

	/*
	 * The parser is not finished with the last write,
	 * continue reading the user input without skipping spaces.
	 */
	if (!parser->cont) {
		/* skip white space */
		while (cnt && isspace(ch)) {
			ret = get_user(ch, ubuf++);
			if (ret)
				goto out;
			read++;
			cnt--;
		}

		/* only spaces were written */
		if (isspace(ch)) {
			*ppos += read;
			ret = read;
			goto out;
		}

		parser->idx = 0;
	}

	/* read the non-space input */
	while (cnt && !isspace(ch)) {
908
		if (parser->idx < parser->size - 1)
909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
			parser->buffer[parser->idx++] = ch;
		else {
			ret = -EINVAL;
			goto out;
		}
		ret = get_user(ch, ubuf++);
		if (ret)
			goto out;
		read++;
		cnt--;
	}

	/* We either got finished input or we have to wait for another call. */
	if (isspace(ch)) {
		parser->buffer[parser->idx] = 0;
		parser->cont = false;
925
	} else if (parser->idx < parser->size - 1) {
926 927
		parser->cont = true;
		parser->buffer[parser->idx++] = ch;
928 929 930
	} else {
		ret = -EINVAL;
		goto out;
931 932 933 934 935 936 937 938 939
	}

	*ppos += read;
	ret = read;

out:
	return ret;
}

940
static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
941 942 943 944 945 946 947 948 949
{
	int len;

	if (s->len <= s->readpos)
		return -EBUSY;

	len = s->len - s->readpos;
	if (cnt > len)
		cnt = len;
950
	memcpy(buf, s->buffer + s->readpos, cnt);
951

952
	s->readpos += cnt;
953 954 955
	return cnt;
}

956 957
unsigned long __read_mostly	tracing_thresh;

958 959 960 961 962 963 964 965 966
#ifdef CONFIG_TRACER_MAX_TRACE
/*
 * Copy the new maximum trace into the separate maximum-trace
 * structure. (this way the maximum trace is permanently saved,
 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
 */
static void
__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
967 968 969 970
	struct trace_buffer *trace_buf = &tr->trace_buffer;
	struct trace_buffer *max_buf = &tr->max_buffer;
	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
971

972 973
	max_buf->cpu = cpu;
	max_buf->time_start = data->preempt_timestamp;
974

975
	max_data->saved_latency = tr->max_latency;
976 977
	max_data->critical_start = data->critical_start;
	max_data->critical_end = data->critical_end;
978

979
	memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
980
	max_data->pid = tsk->pid;
981 982 983 984 985 986 987 988 989
	/*
	 * If tsk == current, then use current_uid(), as that does not use
	 * RCU. The irq tracer can be called out of RCU scope.
	 */
	if (tsk == current)
		max_data->uid = current_uid();
	else
		max_data->uid = task_uid(tsk);

990 991 992
	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
	max_data->policy = tsk->policy;
	max_data->rt_priority = tsk->rt_priority;
993 994 995 996 997

	/* record this tasks comm */
	tracing_record_cmdline(tsk);
}

S
Steven Rostedt 已提交
998 999 1000 1001 1002 1003 1004 1005 1006
/**
 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 * @tr: tracer
 * @tsk: the task with the latency
 * @cpu: The cpu that initiated the trace.
 *
 * Flip the buffers between the @tr and the max_tr and record information
 * about which task was the cause of this latency.
 */
I
Ingo Molnar 已提交
1007
void
1008 1009
update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
1010
	struct ring_buffer *buf;
1011

1012
	if (tr->stop_count)
1013 1014
		return;

1015
	WARN_ON_ONCE(!irqs_disabled());
1016

1017
	if (!tr->allocated_snapshot) {
1018
		/* Only the nop tracer should hit this when disabling */
1019
		WARN_ON_ONCE(tr->current_trace != &nop_trace);
1020
		return;
1021
	}
1022

1023
	arch_spin_lock(&tr->max_lock);
1024

1025 1026 1027
	buf = tr->trace_buffer.buffer;
	tr->trace_buffer.buffer = tr->max_buffer.buffer;
	tr->max_buffer.buffer = buf;
1028

1029
	__update_max_tr(tr, tsk, cpu);
1030
	arch_spin_unlock(&tr->max_lock);
1031 1032 1033 1034 1035 1036 1037
}

/**
 * update_max_tr_single - only copy one trace over, and reset the rest
 * @tr - tracer
 * @tsk - task with the latency
 * @cpu - the cpu of the buffer to copy.
S
Steven Rostedt 已提交
1038 1039
 *
 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1040
 */
I
Ingo Molnar 已提交
1041
void
1042 1043
update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
{
1044
	int ret;
1045

1046
	if (tr->stop_count)
1047 1048
		return;

1049
	WARN_ON_ONCE(!irqs_disabled());
S
Steven Rostedt 已提交
1050
	if (!tr->allocated_snapshot) {
1051
		/* Only the nop tracer should hit this when disabling */
1052
		WARN_ON_ONCE(tr->current_trace != &nop_trace);
1053
		return;
1054
	}
1055

1056
	arch_spin_lock(&tr->max_lock);
1057

1058
	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1059

1060 1061 1062 1063 1064 1065 1066
	if (ret == -EBUSY) {
		/*
		 * We failed to swap the buffer due to a commit taking
		 * place on this CPU. We fail to record, but we reset
		 * the max trace buffer (no one writes directly to it)
		 * and flag that it failed.
		 */
1067
		trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1068 1069 1070 1071
			"Failed to swap buffers due to commit in progress\n");
	}

	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1072 1073

	__update_max_tr(tr, tsk, cpu);
1074
	arch_spin_unlock(&tr->max_lock);
1075
}
1076
#endif /* CONFIG_TRACER_MAX_TRACE */
1077

1078
static int wait_on_pipe(struct trace_iterator *iter)
1079
{
1080 1081
	/* Iterators are static, they should be filled or empty */
	if (trace_buffer_iter(iter, iter->cpu_file))
1082
		return 0;
1083

1084
	return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1085 1086
}

1087 1088 1089 1090 1091 1092
#ifdef CONFIG_FTRACE_STARTUP_TEST
static int run_tracer_selftest(struct tracer *type)
{
	struct trace_array *tr = &global_trace;
	struct tracer *saved_tracer = tr->current_trace;
	int ret;
1093

1094 1095
	if (!type->selftest || tracing_selftest_disabled)
		return 0;
1096 1097

	/*
1098 1099 1100 1101 1102
	 * Run a selftest on this tracer.
	 * Here we reset the trace buffer, and set the current
	 * tracer to be this tracer. The tracer can then run some
	 * internal tracing to verify that everything is in order.
	 * If we fail, we do not register this tracer.
1103
	 */
1104
	tracing_reset_online_cpus(&tr->trace_buffer);
1105

1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
	tr->current_trace = type;

#ifdef CONFIG_TRACER_MAX_TRACE
	if (type->use_max_tr) {
		/* If we expanded the buffers, make sure the max is expanded too */
		if (ring_buffer_expanded)
			ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
					   RING_BUFFER_ALL_CPUS);
		tr->allocated_snapshot = true;
	}
#endif

	/* the test is responsible for initializing and enabling */
	pr_info("Testing tracer %s: ", type->name);
	ret = type->selftest(type, tr);
	/* the test is responsible for resetting too */
	tr->current_trace = saved_tracer;
	if (ret) {
		printk(KERN_CONT "FAILED!\n");
		/* Add the warning after printing 'FAILED' */
		WARN_ON(1);
		return -1;
	}
	/* Only reset on passing, to avoid touching corrupted buffers */
	tracing_reset_online_cpus(&tr->trace_buffer);

#ifdef CONFIG_TRACER_MAX_TRACE
	if (type->use_max_tr) {
		tr->allocated_snapshot = false;
1135

1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
		/* Shrink the max buffer again */
		if (ring_buffer_expanded)
			ring_buffer_resize(tr->max_buffer.buffer, 1,
					   RING_BUFFER_ALL_CPUS);
	}
#endif

	printk(KERN_CONT "PASSED\n");
	return 0;
}
#else
static inline int run_tracer_selftest(struct tracer *type)
{
	return 0;
1150
}
1151
#endif /* CONFIG_FTRACE_STARTUP_TEST */
1152

S
Steven Rostedt 已提交
1153 1154 1155 1156 1157 1158
/**
 * register_tracer - register a tracer with the ftrace system.
 * @type - the plugin for the tracer
 *
 * Register a new plugin tracer.
 */
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
int register_tracer(struct tracer *type)
{
	struct tracer *t;
	int ret = 0;

	if (!type->name) {
		pr_info("Tracer must have a name\n");
		return -1;
	}

1169
	if (strlen(type->name) >= MAX_TRACER_SIZE) {
L
Li Zefan 已提交
1170 1171 1172 1173
		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
		return -1;
	}

1174
	mutex_lock(&trace_types_lock);
I
Ingo Molnar 已提交
1175

1176 1177
	tracing_selftest_running = true;

1178 1179 1180
	for (t = trace_types; t; t = t->next) {
		if (strcmp(type->name, t->name) == 0) {
			/* already found */
L
Li Zefan 已提交
1181
			pr_info("Tracer %s already registered\n",
1182 1183 1184 1185 1186 1187
				type->name);
			ret = -1;
			goto out;
		}
	}

1188 1189 1190 1191 1192 1193 1194
	if (!type->set_flag)
		type->set_flag = &dummy_set_flag;
	if (!type->flags)
		type->flags = &dummy_tracer_flags;
	else
		if (!type->flags->opts)
			type->flags->opts = dummy_tracer_opt;
1195

1196 1197 1198
	ret = run_tracer_selftest(type);
	if (ret < 0)
		goto out;
S
Steven Rostedt 已提交
1199

1200 1201
	type->next = trace_types;
	trace_types = type;
S
Steven Rostedt 已提交
1202

1203
 out:
1204
	tracing_selftest_running = false;
1205 1206
	mutex_unlock(&trace_types_lock);

S
Steven Rostedt 已提交
1207 1208 1209
	if (ret || !default_bootup_tracer)
		goto out_unlock;

L
Li Zefan 已提交
1210
	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
S
Steven Rostedt 已提交
1211 1212 1213 1214
		goto out_unlock;

	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
	/* Do we want this tracer to start on bootup? */
1215
	tracing_set_tracer(&global_trace, type->name);
S
Steven Rostedt 已提交
1216 1217
	default_bootup_tracer = NULL;
	/* disable other selftests, since this will break it. */
1218
	tracing_selftest_disabled = true;
1219
#ifdef CONFIG_FTRACE_STARTUP_TEST
S
Steven Rostedt 已提交
1220 1221
	printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
	       type->name);
1222 1223
#endif

S
Steven Rostedt 已提交
1224
 out_unlock:
1225 1226 1227
	return ret;
}

1228
void tracing_reset(struct trace_buffer *buf, int cpu)
1229
{
1230
	struct ring_buffer *buffer = buf->buffer;
1231

1232 1233 1234
	if (!buffer)
		return;

1235 1236 1237 1238
	ring_buffer_record_disable(buffer);

	/* Make sure all commits have finished */
	synchronize_sched();
1239
	ring_buffer_reset_cpu(buffer, cpu);
1240 1241 1242 1243

	ring_buffer_record_enable(buffer);
}

1244
void tracing_reset_online_cpus(struct trace_buffer *buf)
1245
{
1246
	struct ring_buffer *buffer = buf->buffer;
1247 1248
	int cpu;

1249 1250 1251
	if (!buffer)
		return;

1252 1253 1254 1255 1256
	ring_buffer_record_disable(buffer);

	/* Make sure all commits have finished */
	synchronize_sched();

1257
	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1258 1259

	for_each_online_cpu(cpu)
1260
		ring_buffer_reset_cpu(buffer, cpu);
1261 1262

	ring_buffer_record_enable(buffer);
1263 1264
}

1265
/* Must have trace_types_lock held */
1266
void tracing_reset_all_online_cpus(void)
1267
{
1268 1269 1270
	struct trace_array *tr;

	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1271 1272 1273 1274
		tracing_reset_online_cpus(&tr->trace_buffer);
#ifdef CONFIG_TRACER_MAX_TRACE
		tracing_reset_online_cpus(&tr->max_buffer);
#endif
1275
	}
1276 1277
}

1278
#define SAVED_CMDLINES_DEFAULT 128
1279
#define NO_CMDLINE_MAP UINT_MAX
1280
static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1281 1282 1283 1284 1285 1286 1287 1288
struct saved_cmdlines_buffer {
	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
	unsigned *map_cmdline_to_pid;
	unsigned cmdline_num;
	int cmdline_idx;
	char *saved_cmdlines;
};
static struct saved_cmdlines_buffer *savedcmd;
1289 1290

/* temporary disable recording */
1291
static atomic_t trace_record_cmdline_disabled __read_mostly;
1292

1293 1294 1295 1296 1297 1298
static inline char *get_saved_cmdlines(int idx)
{
	return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
}

static inline void set_cmdline(int idx, const char *cmdline)
1299
{
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
	memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
}

static int allocate_cmdlines_buffer(unsigned int val,
				    struct saved_cmdlines_buffer *s)
{
	s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
					GFP_KERNEL);
	if (!s->map_cmdline_to_pid)
		return -ENOMEM;

	s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
	if (!s->saved_cmdlines) {
		kfree(s->map_cmdline_to_pid);
		return -ENOMEM;
	}

	s->cmdline_idx = 0;
	s->cmdline_num = val;
	memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
	       sizeof(s->map_pid_to_cmdline));
	memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
	       val * sizeof(*s->map_cmdline_to_pid));

	return 0;
}

static int trace_create_savedcmd(void)
{
	int ret;

1331
	savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
	if (!savedcmd)
		return -ENOMEM;

	ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
	if (ret < 0) {
		kfree(savedcmd);
		savedcmd = NULL;
		return -ENOMEM;
	}

	return 0;
1343 1344
}

1345 1346
int is_tracing_stopped(void)
{
1347
	return global_trace.stop_count;
1348 1349
}

1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
/**
 * tracing_start - quick start of the tracer
 *
 * If tracing is enabled but was stopped by tracing_stop,
 * this will start the tracer back up.
 */
void tracing_start(void)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	if (tracing_disabled)
		return;

1364 1365 1366
	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
	if (--global_trace.stop_count) {
		if (global_trace.stop_count < 0) {
1367 1368
			/* Someone screwed up their debugging */
			WARN_ON_ONCE(1);
1369
			global_trace.stop_count = 0;
1370
		}
1371 1372 1373
		goto out;
	}

1374
	/* Prevent the buffers from switching */
1375
	arch_spin_lock(&global_trace.max_lock);
1376

1377
	buffer = global_trace.trace_buffer.buffer;
1378 1379 1380
	if (buffer)
		ring_buffer_record_enable(buffer);

1381 1382
#ifdef CONFIG_TRACER_MAX_TRACE
	buffer = global_trace.max_buffer.buffer;
1383 1384
	if (buffer)
		ring_buffer_record_enable(buffer);
1385
#endif
1386

1387
	arch_spin_unlock(&global_trace.max_lock);
1388

1389
 out:
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
}

static void tracing_start_tr(struct trace_array *tr)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	if (tracing_disabled)
		return;

	/* If global, we need to also start the max tracer */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
		return tracing_start();

	raw_spin_lock_irqsave(&tr->start_lock, flags);

	if (--tr->stop_count) {
		if (tr->stop_count < 0) {
			/* Someone screwed up their debugging */
			WARN_ON_ONCE(1);
			tr->stop_count = 0;
		}
		goto out;
	}

1416
	buffer = tr->trace_buffer.buffer;
1417 1418 1419 1420 1421
	if (buffer)
		ring_buffer_record_enable(buffer);

 out:
	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434
}

/**
 * tracing_stop - quick stop of the tracer
 *
 * Light weight way to stop tracing. Use in conjunction with
 * tracing_start.
 */
void tracing_stop(void)
{
	struct ring_buffer *buffer;
	unsigned long flags;

1435 1436
	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
	if (global_trace.stop_count++)
1437 1438
		goto out;

1439
	/* Prevent the buffers from switching */
1440
	arch_spin_lock(&global_trace.max_lock);
1441

1442
	buffer = global_trace.trace_buffer.buffer;
1443 1444 1445
	if (buffer)
		ring_buffer_record_disable(buffer);

1446 1447
#ifdef CONFIG_TRACER_MAX_TRACE
	buffer = global_trace.max_buffer.buffer;
1448 1449
	if (buffer)
		ring_buffer_record_disable(buffer);
1450
#endif
1451

1452
	arch_spin_unlock(&global_trace.max_lock);
1453

1454
 out:
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
	raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
}

static void tracing_stop_tr(struct trace_array *tr)
{
	struct ring_buffer *buffer;
	unsigned long flags;

	/* If global, we need to also stop the max tracer */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
		return tracing_stop();

	raw_spin_lock_irqsave(&tr->start_lock, flags);
	if (tr->stop_count++)
		goto out;

1471
	buffer = tr->trace_buffer.buffer;
1472 1473 1474 1475 1476
	if (buffer)
		ring_buffer_record_disable(buffer);

 out:
	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1477 1478
}

I
Ingo Molnar 已提交
1479
void trace_stop_cmdline_recording(void);
1480

1481
static int trace_save_cmdline(struct task_struct *tsk)
1482
{
1483
	unsigned pid, idx;
1484 1485

	if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1486
		return 0;
1487 1488 1489 1490 1491 1492 1493

	/*
	 * It's not the end of the world if we don't get
	 * the lock, but we also don't want to spin
	 * nor do we want to disable interrupts,
	 * so if we miss here, then better luck next time.
	 */
1494
	if (!arch_spin_trylock(&trace_cmdline_lock))
1495
		return 0;
1496

1497
	idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1498
	if (idx == NO_CMDLINE_MAP) {
1499
		idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1500

1501 1502 1503 1504 1505 1506
		/*
		 * Check whether the cmdline buffer at idx has a pid
		 * mapped. We are going to overwrite that entry so we
		 * need to clear the map_pid_to_cmdline. Otherwise we
		 * would read the new comm for the old pid.
		 */
1507
		pid = savedcmd->map_cmdline_to_pid[idx];
1508
		if (pid != NO_CMDLINE_MAP)
1509
			savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1510

1511 1512
		savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
		savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1513

1514
		savedcmd->cmdline_idx = idx;
1515 1516
	}

1517
	set_cmdline(idx, tsk->comm);
1518

1519
	arch_spin_unlock(&trace_cmdline_lock);
1520 1521

	return 1;
1522 1523
}

1524
static void __trace_find_cmdline(int pid, char comm[])
1525 1526 1527
{
	unsigned map;

1528 1529 1530 1531
	if (!pid) {
		strcpy(comm, "<idle>");
		return;
	}
1532

1533 1534 1535 1536 1537
	if (WARN_ON_ONCE(pid < 0)) {
		strcpy(comm, "<XXX>");
		return;
	}

1538 1539 1540 1541
	if (pid > PID_MAX_DEFAULT) {
		strcpy(comm, "<...>");
		return;
	}
1542

1543
	map = savedcmd->map_pid_to_cmdline[pid];
1544
	if (map != NO_CMDLINE_MAP)
1545
		strcpy(comm, get_saved_cmdlines(map));
1546 1547
	else
		strcpy(comm, "<...>");
1548 1549 1550 1551 1552 1553 1554 1555
}

void trace_find_cmdline(int pid, char comm[])
{
	preempt_disable();
	arch_spin_lock(&trace_cmdline_lock);

	__trace_find_cmdline(pid, comm);
1556

1557
	arch_spin_unlock(&trace_cmdline_lock);
1558
	preempt_enable();
1559 1560
}

I
Ingo Molnar 已提交
1561
void tracing_record_cmdline(struct task_struct *tsk)
1562
{
1563
	if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1564 1565
		return;

1566 1567 1568
	if (!__this_cpu_read(trace_cmdline_save))
		return;

1569 1570
	if (trace_save_cmdline(tsk))
		__this_cpu_write(trace_cmdline_save, false);
1571 1572
}

1573
void
1574 1575
tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
			     int pc)
1576 1577 1578
{
	struct task_struct *tsk = current;

1579 1580 1581
	entry->preempt_count		= pc & 0xff;
	entry->pid			= (tsk) ? tsk->pid : 0;
	entry->flags =
1582
#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1583
		(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1584 1585 1586
#else
		TRACE_FLAG_IRQS_NOSUPPORT |
#endif
1587 1588
		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
		((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1589 1590
		(tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
		(test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1591
}
1592
EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1593

1594 1595 1596 1597 1598
struct ring_buffer_event *
trace_buffer_lock_reserve(struct ring_buffer *buffer,
			  int type,
			  unsigned long len,
			  unsigned long flags, int pc)
1599 1600 1601
{
	struct ring_buffer_event *event;

1602
	event = ring_buffer_lock_reserve(buffer, len);
1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
	if (event != NULL) {
		struct trace_entry *ent = ring_buffer_event_data(event);

		tracing_generic_entry_update(ent, flags, pc);
		ent->type = type;
	}

	return event;
}

1613 1614 1615 1616 1617 1618 1619
void
__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
{
	__this_cpu_write(trace_cmdline_save, true);
	ring_buffer_unlock_commit(buffer, event);
}

1620 1621 1622
static inline void
__trace_buffer_unlock_commit(struct ring_buffer *buffer,
			     struct ring_buffer_event *event,
1623
			     unsigned long flags, int pc)
1624
{
1625
	__buffer_unlock_commit(buffer, event);
1626

1627 1628
	ftrace_trace_stack(buffer, flags, 6, pc);
	ftrace_trace_userstack(buffer, flags, pc);
1629 1630
}

1631 1632 1633
void trace_buffer_unlock_commit(struct ring_buffer *buffer,
				struct ring_buffer_event *event,
				unsigned long flags, int pc)
1634
{
1635
	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1636
}
1637
EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1638

1639 1640
static struct ring_buffer *temp_buffer;

1641 1642 1643 1644 1645 1646
struct ring_buffer_event *
trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
			  struct ftrace_event_file *ftrace_file,
			  int type, unsigned long len,
			  unsigned long flags, int pc)
{
1647 1648
	struct ring_buffer_event *entry;

1649
	*current_rb = ftrace_file->tr->trace_buffer.buffer;
1650
	entry = trace_buffer_lock_reserve(*current_rb,
1651
					 type, len, flags, pc);
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
	/*
	 * If tracing is off, but we have triggers enabled
	 * we still need to look at the event data. Use the temp_buffer
	 * to store the trace event for the tigger to use. It's recusive
	 * safe and will not be recorded anywhere.
	 */
	if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
		*current_rb = temp_buffer;
		entry = trace_buffer_lock_reserve(*current_rb,
						  type, len, flags, pc);
	}
	return entry;
1664 1665 1666
}
EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);

1667
struct ring_buffer_event *
1668 1669
trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
				  int type, unsigned long len,
1670 1671
				  unsigned long flags, int pc)
{
1672
	*current_rb = global_trace.trace_buffer.buffer;
1673
	return trace_buffer_lock_reserve(*current_rb,
1674 1675
					 type, len, flags, pc);
}
1676
EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1677

1678 1679
void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
					struct ring_buffer_event *event,
1680 1681
					unsigned long flags, int pc)
{
1682
	__trace_buffer_unlock_commit(buffer, event, flags, pc);
1683
}
1684
EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1685

1686 1687 1688 1689
void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
				     struct ring_buffer_event *event,
				     unsigned long flags, int pc,
				     struct pt_regs *regs)
1690
{
1691
	__buffer_unlock_commit(buffer, event);
1692 1693 1694 1695

	ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
	ftrace_trace_userstack(buffer, flags, pc);
}
1696
EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1697

1698 1699
void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
					 struct ring_buffer_event *event)
1700
{
1701
	ring_buffer_discard_commit(buffer, event);
1702
}
1703
EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1704

I
Ingo Molnar 已提交
1705
void
1706
trace_function(struct trace_array *tr,
1707 1708
	       unsigned long ip, unsigned long parent_ip, unsigned long flags,
	       int pc)
1709
{
1710
	struct ftrace_event_call *call = &event_function;
1711
	struct ring_buffer *buffer = tr->trace_buffer.buffer;
1712
	struct ring_buffer_event *event;
1713
	struct ftrace_entry *entry;
1714

1715
	/* If we are reading the ring buffer, don't trace */
R
Rusty Russell 已提交
1716
	if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1717 1718
		return;

1719
	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1720
					  flags, pc);
1721 1722 1723
	if (!event)
		return;
	entry	= ring_buffer_event_data(event);
1724 1725
	entry->ip			= ip;
	entry->parent_ip		= parent_ip;
1726

1727
	if (!call_filter_check_discard(call, entry, buffer, event))
1728
		__buffer_unlock_commit(buffer, event);
1729 1730
}

1731
#ifdef CONFIG_STACKTRACE
1732 1733 1734 1735 1736 1737 1738 1739 1740

#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
struct ftrace_stack {
	unsigned long		calls[FTRACE_STACK_MAX_ENTRIES];
};

static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
static DEFINE_PER_CPU(int, ftrace_stack_reserve);

1741
static void __ftrace_trace_stack(struct ring_buffer *buffer,
1742
				 unsigned long flags,
1743
				 int skip, int pc, struct pt_regs *regs)
I
Ingo Molnar 已提交
1744
{
1745
	struct ftrace_event_call *call = &event_kernel_stack;
1746
	struct ring_buffer_event *event;
1747
	struct stack_entry *entry;
I
Ingo Molnar 已提交
1748
	struct stack_trace trace;
1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762
	int use_stack;
	int size = FTRACE_STACK_ENTRIES;

	trace.nr_entries	= 0;
	trace.skip		= skip;

	/*
	 * Since events can happen in NMIs there's no safe way to
	 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
	 * or NMI comes in, it will just have to use the default
	 * FTRACE_STACK_SIZE.
	 */
	preempt_disable_notrace();

1763
	use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1764 1765 1766 1767 1768 1769 1770 1771 1772
	/*
	 * We don't need any atomic variables, just a barrier.
	 * If an interrupt comes in, we don't care, because it would
	 * have exited and put the counter back to what we want.
	 * We just need a barrier to keep gcc from moving things
	 * around.
	 */
	barrier();
	if (use_stack == 1) {
1773
		trace.entries		= this_cpu_ptr(ftrace_stack.calls);
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
		trace.max_entries	= FTRACE_STACK_MAX_ENTRIES;

		if (regs)
			save_stack_trace_regs(regs, &trace);
		else
			save_stack_trace(&trace);

		if (trace.nr_entries > size)
			size = trace.nr_entries;
	} else
		/* From now on, use_stack is a boolean */
		use_stack = 0;

	size *= sizeof(unsigned long);
I
Ingo Molnar 已提交
1788

1789
	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1790
					  sizeof(*entry) + size, flags, pc);
1791
	if (!event)
1792 1793
		goto out;
	entry = ring_buffer_event_data(event);
I
Ingo Molnar 已提交
1794

1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809
	memset(&entry->caller, 0, size);

	if (use_stack)
		memcpy(&entry->caller, trace.entries,
		       trace.nr_entries * sizeof(unsigned long));
	else {
		trace.max_entries	= FTRACE_STACK_ENTRIES;
		trace.entries		= entry->caller;
		if (regs)
			save_stack_trace_regs(regs, &trace);
		else
			save_stack_trace(&trace);
	}

	entry->size = trace.nr_entries;
I
Ingo Molnar 已提交
1810

1811
	if (!call_filter_check_discard(call, entry, buffer, event))
1812
		__buffer_unlock_commit(buffer, event);
1813 1814 1815 1816

 out:
	/* Again, don't let gcc optimize things here */
	barrier();
1817
	__this_cpu_dec(ftrace_stack_reserve);
1818 1819
	preempt_enable_notrace();

I
Ingo Molnar 已提交
1820 1821
}

1822 1823 1824 1825 1826 1827 1828 1829 1830
void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
			     int skip, int pc, struct pt_regs *regs)
{
	if (!(trace_flags & TRACE_ITER_STACKTRACE))
		return;

	__ftrace_trace_stack(buffer, flags, skip, pc, regs);
}

1831 1832
void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
			int skip, int pc)
1833 1834 1835 1836
{
	if (!(trace_flags & TRACE_ITER_STACKTRACE))
		return;

1837
	__ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1838 1839
}

1840 1841
void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
		   int pc)
1842
{
1843
	__ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1844 1845
}

S
Steven Rostedt 已提交
1846 1847
/**
 * trace_dump_stack - record a stack back trace in the trace buffer
1848
 * @skip: Number of functions to skip (helper handlers)
S
Steven Rostedt 已提交
1849
 */
1850
void trace_dump_stack(int skip)
S
Steven Rostedt 已提交
1851 1852 1853 1854
{
	unsigned long flags;

	if (tracing_disabled || tracing_selftest_running)
1855
		return;
S
Steven Rostedt 已提交
1856 1857 1858

	local_save_flags(flags);

1859 1860 1861 1862 1863 1864 1865
	/*
	 * Skip 3 more, seems to get us at the caller of
	 * this function.
	 */
	skip += 3;
	__ftrace_trace_stack(global_trace.trace_buffer.buffer,
			     flags, skip, preempt_count(), NULL);
S
Steven Rostedt 已提交
1866 1867
}

1868 1869
static DEFINE_PER_CPU(int, user_stack_count);

1870 1871
void
ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1872
{
1873
	struct ftrace_event_call *call = &event_user_stack;
1874
	struct ring_buffer_event *event;
1875 1876 1877 1878 1879 1880
	struct userstack_entry *entry;
	struct stack_trace trace;

	if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
		return;

1881 1882 1883 1884 1885 1886
	/*
	 * NMIs can not handle page faults, even with fix ups.
	 * The save user stack can (and often does) fault.
	 */
	if (unlikely(in_nmi()))
		return;
1887

1888 1889 1890 1891 1892 1893 1894 1895 1896 1897
	/*
	 * prevent recursion, since the user stack tracing may
	 * trigger other kernel events.
	 */
	preempt_disable();
	if (__this_cpu_read(user_stack_count))
		goto out;

	__this_cpu_inc(user_stack_count);

1898
	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1899
					  sizeof(*entry), flags, pc);
1900
	if (!event)
L
Li Zefan 已提交
1901
		goto out_drop_count;
1902 1903
	entry	= ring_buffer_event_data(event);

1904
	entry->tgid		= current->tgid;
1905 1906 1907 1908 1909 1910 1911 1912
	memset(&entry->caller, 0, sizeof(entry->caller));

	trace.nr_entries	= 0;
	trace.max_entries	= FTRACE_STACK_ENTRIES;
	trace.skip		= 0;
	trace.entries		= entry->caller;

	save_stack_trace_user(&trace);
1913
	if (!call_filter_check_discard(call, entry, buffer, event))
1914
		__buffer_unlock_commit(buffer, event);
1915

L
Li Zefan 已提交
1916
 out_drop_count:
1917 1918 1919
	__this_cpu_dec(user_stack_count);
 out:
	preempt_enable();
1920 1921
}

1922 1923
#ifdef UNUSED
static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1924
{
1925
	ftrace_trace_userstack(tr, flags, preempt_count());
1926
}
1927
#endif /* UNUSED */
1928

1929 1930
#endif /* CONFIG_STACKTRACE */

1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
/* created for use with alloc_percpu */
struct trace_buffer_struct {
	char buffer[TRACE_BUF_SIZE];
};

static struct trace_buffer_struct *trace_percpu_buffer;
static struct trace_buffer_struct *trace_percpu_sirq_buffer;
static struct trace_buffer_struct *trace_percpu_irq_buffer;
static struct trace_buffer_struct *trace_percpu_nmi_buffer;

/*
 * The buffer used is dependent on the context. There is a per cpu
 * buffer for normal context, softirq contex, hard irq context and
 * for NMI context. Thise allows for lockless recording.
 *
 * Note, if the buffers failed to be allocated, then this returns NULL
 */
static char *get_trace_buf(void)
{
	struct trace_buffer_struct *percpu_buffer;

	/*
	 * If we have allocated per cpu buffers, then we do not
	 * need to do any locking.
	 */
	if (in_nmi())
		percpu_buffer = trace_percpu_nmi_buffer;
	else if (in_irq())
		percpu_buffer = trace_percpu_irq_buffer;
	else if (in_softirq())
		percpu_buffer = trace_percpu_sirq_buffer;
	else
		percpu_buffer = trace_percpu_buffer;

	if (!percpu_buffer)
		return NULL;

1968
	return this_cpu_ptr(&percpu_buffer->buffer[0]);
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
}

static int alloc_percpu_trace_buffer(void)
{
	struct trace_buffer_struct *buffers;
	struct trace_buffer_struct *sirq_buffers;
	struct trace_buffer_struct *irq_buffers;
	struct trace_buffer_struct *nmi_buffers;

	buffers = alloc_percpu(struct trace_buffer_struct);
	if (!buffers)
		goto err_warn;

	sirq_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!sirq_buffers)
		goto err_sirq;

	irq_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!irq_buffers)
		goto err_irq;

	nmi_buffers = alloc_percpu(struct trace_buffer_struct);
	if (!nmi_buffers)
		goto err_nmi;

	trace_percpu_buffer = buffers;
	trace_percpu_sirq_buffer = sirq_buffers;
	trace_percpu_irq_buffer = irq_buffers;
	trace_percpu_nmi_buffer = nmi_buffers;

	return 0;

 err_nmi:
	free_percpu(irq_buffers);
 err_irq:
	free_percpu(sirq_buffers);
 err_sirq:
	free_percpu(buffers);
 err_warn:
	WARN(1, "Could not allocate percpu trace_printk buffer");
	return -ENOMEM;
}

2012 2013
static int buffers_allocated;

2014 2015 2016 2017 2018 2019 2020 2021
void trace_printk_init_buffers(void)
{
	if (buffers_allocated)
		return;

	if (alloc_percpu_trace_buffer())
		return;

2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
	/* trace_printk() is for debug use only. Don't use it in production. */

	pr_warning("\n**********************************************************\n");
	pr_warning("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
	pr_warning("**                                                      **\n");
	pr_warning("** trace_printk() being used. Allocating extra memory.  **\n");
	pr_warning("**                                                      **\n");
	pr_warning("** This means that this is a DEBUG kernel and it is     **\n");
	pr_warning("** unsafe for produciton use.                           **\n");
	pr_warning("**                                                      **\n");
	pr_warning("** If you see this message and you are not debugging    **\n");
	pr_warning("** the kernel, report this immediately to your vendor!  **\n");
	pr_warning("**                                                      **\n");
	pr_warning("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
	pr_warning("**********************************************************\n");
2037

2038 2039 2040
	/* Expand the buffers to set size */
	tracing_update_buffers();

2041
	buffers_allocated = 1;
2042 2043 2044 2045 2046 2047 2048

	/*
	 * trace_printk_init_buffers() can be called by modules.
	 * If that happens, then we need to start cmdline recording
	 * directly here. If the global_trace.buffer is already
	 * allocated here, then this was called by module code.
	 */
2049
	if (global_trace.trace_buffer.buffer)
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069
		tracing_start_cmdline_record();
}

void trace_printk_start_comm(void)
{
	/* Start tracing comms if trace printk is set */
	if (!buffers_allocated)
		return;
	tracing_start_cmdline_record();
}

static void trace_printk_start_stop_comm(int enabled)
{
	if (!buffers_allocated)
		return;

	if (enabled)
		tracing_start_cmdline_record();
	else
		tracing_stop_cmdline_record();
2070 2071
}

2072
/**
2073
 * trace_vbprintk - write binary msg to tracing buffer
2074 2075
 *
 */
2076
int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2077
{
2078
	struct ftrace_event_call *call = &event_bprint;
2079
	struct ring_buffer_event *event;
2080
	struct ring_buffer *buffer;
2081
	struct trace_array *tr = &global_trace;
2082
	struct bprint_entry *entry;
2083
	unsigned long flags;
2084 2085
	char *tbuffer;
	int len = 0, size, pc;
2086 2087 2088 2089 2090 2091 2092 2093

	if (unlikely(tracing_selftest_running || tracing_disabled))
		return 0;

	/* Don't pollute graph traces with trace_vprintk internals */
	pause_graph_tracing();

	pc = preempt_count();
2094
	preempt_disable_notrace();
2095

2096 2097 2098
	tbuffer = get_trace_buf();
	if (!tbuffer) {
		len = 0;
2099
		goto out;
2100
	}
2101

2102
	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2103

2104 2105
	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
		goto out;
2106

2107
	local_save_flags(flags);
2108
	size = sizeof(*entry) + sizeof(u32) * len;
2109
	buffer = tr->trace_buffer.buffer;
2110 2111
	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
					  flags, pc);
2112
	if (!event)
2113
		goto out;
2114 2115 2116 2117
	entry = ring_buffer_event_data(event);
	entry->ip			= ip;
	entry->fmt			= fmt;

2118
	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2119
	if (!call_filter_check_discard(call, entry, buffer, event)) {
2120
		__buffer_unlock_commit(buffer, event);
2121 2122
		ftrace_trace_stack(buffer, flags, 6, pc);
	}
2123 2124

out:
2125
	preempt_enable_notrace();
2126 2127 2128 2129
	unpause_graph_tracing();

	return len;
}
2130 2131
EXPORT_SYMBOL_GPL(trace_vbprintk);

2132 2133 2134
static int
__trace_array_vprintk(struct ring_buffer *buffer,
		      unsigned long ip, const char *fmt, va_list args)
2135
{
2136
	struct ftrace_event_call *call = &event_print;
2137
	struct ring_buffer_event *event;
2138
	int len = 0, size, pc;
2139
	struct print_entry *entry;
2140 2141
	unsigned long flags;
	char *tbuffer;
2142 2143 2144 2145

	if (tracing_disabled || tracing_selftest_running)
		return 0;

2146 2147 2148
	/* Don't pollute graph traces with trace_vprintk internals */
	pause_graph_tracing();

2149 2150 2151
	pc = preempt_count();
	preempt_disable_notrace();

2152 2153 2154 2155

	tbuffer = get_trace_buf();
	if (!tbuffer) {
		len = 0;
2156
		goto out;
2157
	}
2158

2159 2160 2161
	len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
	if (len > TRACE_BUF_SIZE)
		goto out;
2162

2163
	local_save_flags(flags);
2164
	size = sizeof(*entry) + len + 1;
2165
	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2166
					  flags, pc);
2167
	if (!event)
2168
		goto out;
2169
	entry = ring_buffer_event_data(event);
C
Carsten Emde 已提交
2170
	entry->ip = ip;
2171

2172
	memcpy(&entry->buf, tbuffer, len);
C
Carsten Emde 已提交
2173
	entry->buf[len] = '\0';
2174
	if (!call_filter_check_discard(call, entry, buffer, event)) {
2175
		__buffer_unlock_commit(buffer, event);
2176
		ftrace_trace_stack(buffer, flags, 6, pc);
2177
	}
2178 2179
 out:
	preempt_enable_notrace();
2180
	unpause_graph_tracing();
2181 2182 2183

	return len;
}
2184

2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220
int trace_array_vprintk(struct trace_array *tr,
			unsigned long ip, const char *fmt, va_list args)
{
	return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
}

int trace_array_printk(struct trace_array *tr,
		       unsigned long ip, const char *fmt, ...)
{
	int ret;
	va_list ap;

	if (!(trace_flags & TRACE_ITER_PRINTK))
		return 0;

	va_start(ap, fmt);
	ret = trace_array_vprintk(tr, ip, fmt, ap);
	va_end(ap);
	return ret;
}

int trace_array_printk_buf(struct ring_buffer *buffer,
			   unsigned long ip, const char *fmt, ...)
{
	int ret;
	va_list ap;

	if (!(trace_flags & TRACE_ITER_PRINTK))
		return 0;

	va_start(ap, fmt);
	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
	va_end(ap);
	return ret;
}

2221 2222
int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
{
S
Steven Rostedt 已提交
2223
	return trace_array_vprintk(&global_trace, ip, fmt, args);
2224
}
2225 2226
EXPORT_SYMBOL_GPL(trace_vprintk);

2227
static void trace_iterator_increment(struct trace_iterator *iter)
S
Steven Rostedt 已提交
2228
{
2229 2230
	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);

S
Steven Rostedt 已提交
2231
	iter->idx++;
2232 2233
	if (buf_iter)
		ring_buffer_read(buf_iter, NULL);
S
Steven Rostedt 已提交
2234 2235
}

I
Ingo Molnar 已提交
2236
static struct trace_entry *
2237 2238
peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
		unsigned long *lost_events)
2239
{
2240
	struct ring_buffer_event *event;
2241
	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2242

2243 2244 2245
	if (buf_iter)
		event = ring_buffer_iter_peek(buf_iter, ts);
	else
2246
		event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2247
					 lost_events);
2248

2249 2250 2251 2252 2253 2254
	if (event) {
		iter->ent_size = ring_buffer_event_length(event);
		return ring_buffer_event_data(event);
	}
	iter->ent_size = 0;
	return NULL;
2255
}
2256

2257
static struct trace_entry *
2258 2259
__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
		  unsigned long *missing_events, u64 *ent_ts)
2260
{
2261
	struct ring_buffer *buffer = iter->trace_buffer->buffer;
2262
	struct trace_entry *ent, *next = NULL;
2263
	unsigned long lost_events = 0, next_lost = 0;
2264
	int cpu_file = iter->cpu_file;
2265
	u64 next_ts = 0, ts;
2266
	int next_cpu = -1;
2267
	int next_size = 0;
2268 2269
	int cpu;

2270 2271 2272 2273
	/*
	 * If we are in a per_cpu trace file, don't bother by iterating over
	 * all cpu and peek directly.
	 */
2274
	if (cpu_file > RING_BUFFER_ALL_CPUS) {
2275 2276
		if (ring_buffer_empty_cpu(buffer, cpu_file))
			return NULL;
2277
		ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2278 2279 2280 2281 2282 2283
		if (ent_cpu)
			*ent_cpu = cpu_file;

		return ent;
	}

2284
	for_each_tracing_cpu(cpu) {
2285

2286 2287
		if (ring_buffer_empty_cpu(buffer, cpu))
			continue;
2288

2289
		ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2290

I
Ingo Molnar 已提交
2291 2292 2293
		/*
		 * Pick the entry with the smallest timestamp:
		 */
2294
		if (ent && (!next || ts < next_ts)) {
2295 2296
			next = ent;
			next_cpu = cpu;
2297
			next_ts = ts;
2298
			next_lost = lost_events;
2299
			next_size = iter->ent_size;
2300 2301 2302
		}
	}

2303 2304
	iter->ent_size = next_size;

2305 2306 2307
	if (ent_cpu)
		*ent_cpu = next_cpu;

2308 2309 2310
	if (ent_ts)
		*ent_ts = next_ts;

2311 2312 2313
	if (missing_events)
		*missing_events = next_lost;

2314 2315 2316
	return next;
}

2317
/* Find the next real entry, without updating the iterator itself */
2318 2319
struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
					  int *ent_cpu, u64 *ent_ts)
2320
{
2321
	return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2322 2323 2324
}

/* Find the next real entry, and increment the iterator to the next entry */
2325
void *trace_find_next_entry_inc(struct trace_iterator *iter)
2326
{
2327 2328
	iter->ent = __find_next_entry(iter, &iter->cpu,
				      &iter->lost_events, &iter->ts);
2329

2330
	if (iter->ent)
2331
		trace_iterator_increment(iter);
2332

2333
	return iter->ent ? iter : NULL;
2334
}
2335

I
Ingo Molnar 已提交
2336
static void trace_consume(struct trace_iterator *iter)
2337
{
2338
	ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2339
			    &iter->lost_events);
2340 2341
}

I
Ingo Molnar 已提交
2342
static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2343 2344 2345
{
	struct trace_iterator *iter = m->private;
	int i = (int)*pos;
I
Ingo Molnar 已提交
2346
	void *ent;
2347

2348 2349
	WARN_ON_ONCE(iter->leftover);

2350 2351 2352 2353 2354 2355 2356
	(*pos)++;

	/* can't go backwards */
	if (iter->idx > i)
		return NULL;

	if (iter->idx < 0)
2357
		ent = trace_find_next_entry_inc(iter);
2358 2359 2360 2361
	else
		ent = iter;

	while (ent && iter->idx < i)
2362
		ent = trace_find_next_entry_inc(iter);
2363 2364 2365 2366 2367 2368

	iter->pos = *pos;

	return ent;
}

2369
void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2370 2371 2372 2373 2374 2375
{
	struct ring_buffer_event *event;
	struct ring_buffer_iter *buf_iter;
	unsigned long entries = 0;
	u64 ts;

2376
	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2377

2378 2379
	buf_iter = trace_buffer_iter(iter, cpu);
	if (!buf_iter)
2380 2381 2382 2383 2384 2385 2386 2387 2388 2389
		return;

	ring_buffer_iter_reset(buf_iter);

	/*
	 * We could have the case with the max latency tracers
	 * that a reset never took place on a cpu. This is evident
	 * by the timestamp being before the start of the buffer.
	 */
	while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2390
		if (ts >= iter->trace_buffer->time_start)
2391 2392 2393 2394 2395
			break;
		entries++;
		ring_buffer_read(buf_iter, NULL);
	}

2396
	per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2397 2398
}

2399 2400 2401 2402
/*
 * The current tracer is copied to avoid a global locking
 * all around.
 */
2403 2404 2405
static void *s_start(struct seq_file *m, loff_t *pos)
{
	struct trace_iterator *iter = m->private;
2406
	struct trace_array *tr = iter->tr;
2407
	int cpu_file = iter->cpu_file;
2408 2409
	void *p = NULL;
	loff_t l = 0;
2410
	int cpu;
2411

2412 2413 2414 2415 2416 2417
	/*
	 * copy the tracer to avoid using a global lock all around.
	 * iter->trace is a copy of current_trace, the pointer to the
	 * name may be used instead of a strcmp(), as iter->trace->name
	 * will point to the same string as current_trace->name.
	 */
2418
	mutex_lock(&trace_types_lock);
2419 2420
	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
		*iter->trace = *tr->current_trace;
2421
	mutex_unlock(&trace_types_lock);
2422

2423
#ifdef CONFIG_TRACER_MAX_TRACE
2424 2425
	if (iter->snapshot && iter->trace->use_max_tr)
		return ERR_PTR(-EBUSY);
2426
#endif
2427 2428 2429

	if (!iter->snapshot)
		atomic_inc(&trace_record_cmdline_disabled);
2430 2431 2432 2433 2434 2435

	if (*pos != iter->pos) {
		iter->ent = NULL;
		iter->cpu = 0;
		iter->idx = -1;

2436
		if (cpu_file == RING_BUFFER_ALL_CPUS) {
2437
			for_each_tracing_cpu(cpu)
2438
				tracing_iter_reset(iter, cpu);
2439
		} else
2440
			tracing_iter_reset(iter, cpu_file);
2441

2442
		iter->leftover = 0;
2443 2444 2445 2446
		for (p = iter; p && l < *pos; p = s_next(m, p, &l))
			;

	} else {
2447 2448 2449 2450 2451 2452 2453 2454 2455 2456
		/*
		 * If we overflowed the seq_file before, then we want
		 * to just reuse the trace_seq buffer again.
		 */
		if (iter->leftover)
			p = iter;
		else {
			l = *pos - 1;
			p = s_next(m, p, &l);
		}
2457 2458
	}

2459
	trace_event_read_lock();
2460
	trace_access_lock(cpu_file);
2461 2462 2463 2464 2465
	return p;
}

static void s_stop(struct seq_file *m, void *p)
{
2466 2467
	struct trace_iterator *iter = m->private;

2468
#ifdef CONFIG_TRACER_MAX_TRACE
2469 2470
	if (iter->snapshot && iter->trace->use_max_tr)
		return;
2471
#endif
2472 2473 2474

	if (!iter->snapshot)
		atomic_dec(&trace_record_cmdline_disabled);
2475

2476
	trace_access_unlock(iter->cpu_file);
2477
	trace_event_read_unlock();
2478 2479
}

2480
static void
2481 2482
get_total_entries(struct trace_buffer *buf,
		  unsigned long *total, unsigned long *entries)
2483 2484 2485 2486 2487 2488 2489 2490
{
	unsigned long count;
	int cpu;

	*total = 0;
	*entries = 0;

	for_each_tracing_cpu(cpu) {
2491
		count = ring_buffer_entries_cpu(buf->buffer, cpu);
2492 2493 2494 2495 2496
		/*
		 * If this buffer has skipped entries, then we hold all
		 * entries for the trace and we need to ignore the
		 * ones before the time stamp.
		 */
2497 2498
		if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
			count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2499 2500 2501 2502
			/* total is the same as the entries */
			*total += count;
		} else
			*total += count +
2503
				ring_buffer_overrun_cpu(buf->buffer, cpu);
2504 2505 2506 2507
		*entries += count;
	}
}

I
Ingo Molnar 已提交
2508
static void print_lat_help_header(struct seq_file *m)
2509
{
2510 2511 2512 2513 2514
	seq_puts(m, "#                  _------=> CPU#            \n");
	seq_puts(m, "#                 / _-----=> irqs-off        \n");
	seq_puts(m, "#                | / _----=> need-resched    \n");
	seq_puts(m, "#                || / _---=> hardirq/softirq \n");
	seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2515 2516 2517
	seq_puts(m, "#                |||| /     delay             \n");
	seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
	seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2518 2519
}

2520
static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2521
{
2522 2523 2524
	unsigned long total;
	unsigned long entries;

2525
	get_total_entries(buf, &total, &entries);
2526 2527 2528 2529 2530
	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
		   entries, total, num_online_cpus());
	seq_puts(m, "#\n");
}

2531
static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2532
{
2533
	print_event_info(buf, m);
2534
	seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2535
	seq_puts(m, "#              | |       |          |         |\n");
2536 2537
}

2538
static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2539
{
2540
	print_event_info(buf, m);
2541 2542 2543 2544 2545 2546 2547 2548
	seq_puts(m, "#                              _-----=> irqs-off\n");
	seq_puts(m, "#                             / _----=> need-resched\n");
	seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
	seq_puts(m, "#                            || / _--=> preempt-depth\n");
	seq_puts(m, "#                            ||| /     delay\n");
	seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
	seq_puts(m, "#              | |       |   ||||       |         |\n");
}
2549

2550
void
2551 2552 2553
print_trace_header(struct seq_file *m, struct trace_iterator *iter)
{
	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2554 2555
	struct trace_buffer *buf = iter->trace_buffer;
	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2556
	struct tracer *type = iter->trace;
2557 2558
	unsigned long entries;
	unsigned long total;
2559 2560
	const char *name = "preemption";

2561
	name = type->name;
2562

2563
	get_total_entries(buf, &total, &entries);
2564

2565
	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2566
		   name, UTS_RELEASE);
2567
	seq_puts(m, "# -----------------------------------"
2568
		 "---------------------------------\n");
2569
	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2570
		   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
S
Steven Rostedt 已提交
2571
		   nsecs_to_usecs(data->saved_latency),
2572
		   entries,
2573
		   total,
2574
		   buf->cpu,
2575 2576 2577 2578
#if defined(CONFIG_PREEMPT_NONE)
		   "server",
#elif defined(CONFIG_PREEMPT_VOLUNTARY)
		   "desktop",
2579
#elif defined(CONFIG_PREEMPT)
2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590
		   "preempt",
#else
		   "unknown",
#endif
		   /* These are reserved for later use */
		   0, 0, 0, 0);
#ifdef CONFIG_SMP
	seq_printf(m, " #P:%d)\n", num_online_cpus());
#else
	seq_puts(m, ")\n");
#endif
2591 2592
	seq_puts(m, "#    -----------------\n");
	seq_printf(m, "#    | task: %.16s-%d "
2593
		   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2594 2595
		   data->comm, data->pid,
		   from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2596
		   data->policy, data->rt_priority);
2597
	seq_puts(m, "#    -----------------\n");
2598 2599

	if (data->critical_start) {
2600
		seq_puts(m, "#  => started at: ");
S
Steven Rostedt 已提交
2601 2602
		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
		trace_print_seq(m, &iter->seq);
2603
		seq_puts(m, "\n#  => ended at:   ");
S
Steven Rostedt 已提交
2604 2605
		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
		trace_print_seq(m, &iter->seq);
2606
		seq_puts(m, "\n#\n");
2607 2608
	}

2609
	seq_puts(m, "#\n");
2610 2611
}

2612 2613 2614 2615
static void test_cpu_buff_start(struct trace_iterator *iter)
{
	struct trace_seq *s = &iter->seq;

2616 2617 2618 2619 2620 2621
	if (!(trace_flags & TRACE_ITER_ANNOTATE))
		return;

	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
		return;

2622
	if (cpumask_test_cpu(iter->cpu, iter->started))
2623 2624
		return;

2625
	if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2626 2627
		return;

2628
	cpumask_set_cpu(iter->cpu, iter->started);
2629 2630 2631 2632 2633

	/* Don't print started cpu buffer for the first entry of the trace */
	if (iter->idx > 1)
		trace_seq_printf(s, "##### CPU %u buffer started ####\n",
				iter->cpu);
2634 2635
}

2636
static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2637
{
S
Steven Rostedt 已提交
2638
	struct trace_seq *s = &iter->seq;
2639
	unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
I
Ingo Molnar 已提交
2640
	struct trace_entry *entry;
2641
	struct trace_event *event;
2642

I
Ingo Molnar 已提交
2643
	entry = iter->ent;
2644

2645 2646
	test_cpu_buff_start(iter);

2647
	event = ftrace_find_event(entry->type);
2648

2649
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2650 2651 2652 2653 2654 2655 2656
		if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
			if (!trace_print_lat_context(iter))
				goto partial;
		} else {
			if (!trace_print_context(iter))
				goto partial;
		}
2657
	}
2658

2659
	if (event)
2660
		return event->funcs->trace(iter, sym_flags, event);
2661 2662 2663

	if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
		goto partial;
2664

2665
	return TRACE_TYPE_HANDLED;
2666 2667
partial:
	return TRACE_TYPE_PARTIAL_LINE;
2668 2669
}

2670
static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2671 2672 2673
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *entry;
2674
	struct trace_event *event;
I
Ingo Molnar 已提交
2675 2676

	entry = iter->ent;
2677

2678
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2679 2680 2681
		if (!trace_seq_printf(s, "%d %d %llu ",
				      entry->pid, iter->cpu, iter->ts))
			goto partial;
2682
	}
I
Ingo Molnar 已提交
2683

2684
	event = ftrace_find_event(entry->type);
2685
	if (event)
2686
		return event->funcs->raw(iter, 0, event);
2687 2688 2689

	if (!trace_seq_printf(s, "%d ?\n", entry->type))
		goto partial;
2690

2691
	return TRACE_TYPE_HANDLED;
2692 2693
partial:
	return TRACE_TYPE_PARTIAL_LINE;
I
Ingo Molnar 已提交
2694 2695
}

2696
static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2697 2698 2699 2700
{
	struct trace_seq *s = &iter->seq;
	unsigned char newline = '\n';
	struct trace_entry *entry;
2701
	struct trace_event *event;
2702 2703

	entry = iter->ent;
2704

2705 2706 2707 2708 2709
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
		SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
		SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
		SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
	}
2710

2711
	event = ftrace_find_event(entry->type);
2712
	if (event) {
2713
		enum print_line_t ret = event->funcs->hex(iter, 0, event);
2714 2715 2716
		if (ret != TRACE_TYPE_HANDLED)
			return ret;
	}
S
Steven Rostedt 已提交
2717

2718 2719
	SEQ_PUT_FIELD_RET(s, newline);

2720
	return TRACE_TYPE_HANDLED;
2721 2722
}

2723
static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2724 2725 2726
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *entry;
2727
	struct trace_event *event;
I
Ingo Molnar 已提交
2728 2729

	entry = iter->ent;
2730

2731 2732
	if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
		SEQ_PUT_FIELD_RET(s, entry->pid);
2733
		SEQ_PUT_FIELD_RET(s, iter->cpu);
2734 2735
		SEQ_PUT_FIELD_RET(s, iter->ts);
	}
I
Ingo Molnar 已提交
2736

2737
	event = ftrace_find_event(entry->type);
2738 2739
	return event ? event->funcs->binary(iter, 0, event) :
		TRACE_TYPE_HANDLED;
I
Ingo Molnar 已提交
2740 2741
}

2742
int trace_empty(struct trace_iterator *iter)
2743
{
2744
	struct ring_buffer_iter *buf_iter;
2745 2746
	int cpu;

2747
	/* If we are looking at one CPU buffer, only check that one */
2748
	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2749
		cpu = iter->cpu_file;
2750 2751 2752
		buf_iter = trace_buffer_iter(iter, cpu);
		if (buf_iter) {
			if (!ring_buffer_iter_empty(buf_iter))
2753 2754
				return 0;
		} else {
2755
			if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2756 2757 2758 2759 2760
				return 0;
		}
		return 1;
	}

2761
	for_each_tracing_cpu(cpu) {
2762 2763 2764
		buf_iter = trace_buffer_iter(iter, cpu);
		if (buf_iter) {
			if (!ring_buffer_iter_empty(buf_iter))
2765 2766
				return 0;
		} else {
2767
			if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2768 2769
				return 0;
		}
2770
	}
2771

2772
	return 1;
2773 2774
}

2775
/*  Called with trace_event_read_lock() held. */
2776
enum print_line_t print_trace_line(struct trace_iterator *iter)
I
Ingo Molnar 已提交
2777
{
2778 2779
	enum print_line_t ret;

2780 2781 2782 2783
	if (iter->lost_events &&
	    !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
				 iter->cpu, iter->lost_events))
		return TRACE_TYPE_PARTIAL_LINE;
2784

2785 2786 2787 2788 2789
	if (iter->trace && iter->trace->print_line) {
		ret = iter->trace->print_line(iter);
		if (ret != TRACE_TYPE_UNHANDLED)
			return ret;
	}
2790

2791 2792 2793 2794 2795
	if (iter->ent->type == TRACE_BPUTS &&
			trace_flags & TRACE_ITER_PRINTK &&
			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
		return trace_print_bputs_msg_only(iter);

2796 2797 2798
	if (iter->ent->type == TRACE_BPRINT &&
			trace_flags & TRACE_ITER_PRINTK &&
			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2799
		return trace_print_bprintk_msg_only(iter);
2800

2801 2802 2803
	if (iter->ent->type == TRACE_PRINT &&
			trace_flags & TRACE_ITER_PRINTK &&
			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2804
		return trace_print_printk_msg_only(iter);
2805

I
Ingo Molnar 已提交
2806 2807 2808
	if (trace_flags & TRACE_ITER_BIN)
		return print_bin_fmt(iter);

2809 2810 2811
	if (trace_flags & TRACE_ITER_HEX)
		return print_hex_fmt(iter);

I
Ingo Molnar 已提交
2812 2813 2814 2815 2816 2817
	if (trace_flags & TRACE_ITER_RAW)
		return print_raw_fmt(iter);

	return print_trace_fmt(iter);
}

2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832
void trace_latency_header(struct seq_file *m)
{
	struct trace_iterator *iter = m->private;

	/* print nothing if the buffers are empty */
	if (trace_empty(iter))
		return;

	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
		print_trace_header(m, iter);

	if (!(trace_flags & TRACE_ITER_VERBOSE))
		print_lat_help_header(m);
}

2833 2834 2835 2836
void trace_default_header(struct seq_file *m)
{
	struct trace_iterator *iter = m->private;

2837 2838 2839
	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
		return;

2840 2841 2842 2843 2844 2845 2846 2847
	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
		/* print nothing if the buffers are empty */
		if (trace_empty(iter))
			return;
		print_trace_header(m, iter);
		if (!(trace_flags & TRACE_ITER_VERBOSE))
			print_lat_help_header(m);
	} else {
2848 2849
		if (!(trace_flags & TRACE_ITER_VERBOSE)) {
			if (trace_flags & TRACE_ITER_IRQ_INFO)
2850
				print_func_help_header_irq(iter->trace_buffer, m);
2851
			else
2852
				print_func_help_header(iter->trace_buffer, m);
2853
		}
2854 2855 2856
	}
}

2857 2858 2859 2860 2861 2862 2863 2864
static void test_ftrace_alive(struct seq_file *m)
{
	if (!ftrace_is_dead())
		return;
	seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
	seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
}

2865
#ifdef CONFIG_TRACER_MAX_TRACE
2866
static void show_snapshot_main_help(struct seq_file *m)
2867 2868 2869 2870
{
	seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
	seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
	seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
2871
	seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2872 2873 2874
	seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
	seq_printf(m, "#                       is not a '0' or '1')\n");
}
2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890

static void show_snapshot_percpu_help(struct seq_file *m)
{
	seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
	seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
	seq_printf(m, "#                      Takes a snapshot of the main buffer for this cpu.\n");
#else
	seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
	seq_printf(m, "#                     Must use main snapshot file to allocate.\n");
#endif
	seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
	seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
	seq_printf(m, "#                       is not a '0' or '1')\n");
}

2891 2892
static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
{
2893
	if (iter->tr->allocated_snapshot)
2894 2895 2896 2897 2898
		seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
	else
		seq_printf(m, "#\n# * Snapshot is freed *\n#\n");

	seq_printf(m, "# Snapshot commands:\n");
2899 2900 2901 2902
	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
		show_snapshot_main_help(m);
	else
		show_snapshot_percpu_help(m);
2903 2904 2905 2906 2907 2908
}
#else
/* Should never be called */
static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
#endif

2909 2910 2911
static int s_show(struct seq_file *m, void *v)
{
	struct trace_iterator *iter = v;
2912
	int ret;
2913 2914 2915 2916 2917

	if (iter->ent == NULL) {
		if (iter->tr) {
			seq_printf(m, "# tracer: %s\n", iter->trace->name);
			seq_puts(m, "#\n");
2918
			test_ftrace_alive(m);
2919
		}
2920 2921 2922
		if (iter->snapshot && trace_empty(iter))
			print_snapshot_help(m, iter);
		else if (iter->trace && iter->trace->print_header)
2923
			iter->trace->print_header(m);
2924 2925 2926
		else
			trace_default_header(m);

2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
	} else if (iter->leftover) {
		/*
		 * If we filled the seq_file buffer earlier, we
		 * want to just show it now.
		 */
		ret = trace_print_seq(m, &iter->seq);

		/* ret should this time be zero, but you never know */
		iter->leftover = ret;

2937
	} else {
I
Ingo Molnar 已提交
2938
		print_trace_line(iter);
2939 2940 2941 2942 2943 2944 2945 2946 2947
		ret = trace_print_seq(m, &iter->seq);
		/*
		 * If we overflow the seq_file buffer, then it will
		 * ask us for this data again at start up.
		 * Use that instead.
		 *  ret is 0 if seq_file write succeeded.
		 *        -1 otherwise.
		 */
		iter->leftover = ret;
2948 2949 2950 2951 2952
	}

	return 0;
}

2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963
/*
 * Should be used after trace_array_get(), trace_types_lock
 * ensures that i_cdev was already initialized.
 */
static inline int tracing_get_cpu(struct inode *inode)
{
	if (inode->i_cdev) /* See trace_create_cpu_file() */
		return (long)inode->i_cdev - 1;
	return RING_BUFFER_ALL_CPUS;
}

J
James Morris 已提交
2964
static const struct seq_operations tracer_seq_ops = {
I
Ingo Molnar 已提交
2965 2966 2967 2968
	.start		= s_start,
	.next		= s_next,
	.stop		= s_stop,
	.show		= s_show,
2969 2970
};

I
Ingo Molnar 已提交
2971
static struct trace_iterator *
2972
__tracing_open(struct inode *inode, struct file *file, bool snapshot)
2973
{
2974
	struct trace_array *tr = inode->i_private;
2975
	struct trace_iterator *iter;
2976
	int cpu;
2977

2978 2979
	if (tracing_disabled)
		return ERR_PTR(-ENODEV);
S
Steven Rostedt 已提交
2980

2981
	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2982 2983
	if (!iter)
		return ERR_PTR(-ENOMEM);
2984

2985 2986
	iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
				    GFP_KERNEL);
2987 2988 2989
	if (!iter->buffer_iter)
		goto release;

2990 2991 2992 2993
	/*
	 * We make a copy of the current tracer to avoid concurrent
	 * changes on it while we are reading.
	 */
2994
	mutex_lock(&trace_types_lock);
2995
	iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2996
	if (!iter->trace)
2997
		goto fail;
2998

2999
	*iter->trace = *tr->current_trace;
3000

3001
	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3002 3003
		goto fail;

3004 3005 3006
	iter->tr = tr;

#ifdef CONFIG_TRACER_MAX_TRACE
3007 3008
	/* Currently only the top directory has a snapshot */
	if (tr->current_trace->print_max || snapshot)
3009
		iter->trace_buffer = &tr->max_buffer;
3010
	else
3011 3012
#endif
		iter->trace_buffer = &tr->trace_buffer;
3013
	iter->snapshot = snapshot;
3014
	iter->pos = -1;
3015
	iter->cpu_file = tracing_get_cpu(inode);
3016
	mutex_init(&iter->mutex);
3017

3018 3019
	/* Notify the tracer early; before we stop tracing. */
	if (iter->trace && iter->trace->open)
3020
		iter->trace->open(iter);
3021

3022
	/* Annotate start of buffers if we had overruns */
3023
	if (ring_buffer_overruns(iter->trace_buffer->buffer))
3024 3025
		iter->iter_flags |= TRACE_FILE_ANNOTATE;

3026
	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
3027
	if (trace_clocks[tr->clock_id].in_ns)
3028 3029
		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;

3030 3031
	/* stop the trace while dumping if we are not opening "snapshot" */
	if (!iter->snapshot)
3032
		tracing_stop_tr(tr);
3033

3034
	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3035 3036
		for_each_tracing_cpu(cpu) {
			iter->buffer_iter[cpu] =
3037
				ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3038 3039 3040 3041
		}
		ring_buffer_read_prepare_sync();
		for_each_tracing_cpu(cpu) {
			ring_buffer_read_start(iter->buffer_iter[cpu]);
3042
			tracing_iter_reset(iter, cpu);
3043 3044 3045
		}
	} else {
		cpu = iter->cpu_file;
3046
		iter->buffer_iter[cpu] =
3047
			ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3048 3049
		ring_buffer_read_prepare_sync();
		ring_buffer_read_start(iter->buffer_iter[cpu]);
3050
		tracing_iter_reset(iter, cpu);
3051 3052
	}

3053 3054 3055
	mutex_unlock(&trace_types_lock);

	return iter;
3056

3057
 fail:
3058
	mutex_unlock(&trace_types_lock);
3059
	kfree(iter->trace);
3060
	kfree(iter->buffer_iter);
3061
release:
3062 3063
	seq_release_private(inode, file);
	return ERR_PTR(-ENOMEM);
3064 3065 3066 3067
}

int tracing_open_generic(struct inode *inode, struct file *filp)
{
S
Steven Rostedt 已提交
3068 3069 3070
	if (tracing_disabled)
		return -ENODEV;

3071 3072 3073 3074
	filp->private_data = inode->i_private;
	return 0;
}

3075 3076 3077 3078 3079
bool tracing_is_disabled(void)
{
	return (tracing_disabled) ? true: false;
}

3080 3081 3082 3083
/*
 * Open and update trace_array ref count.
 * Must have the current trace_array passed to it.
 */
3084
static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098
{
	struct trace_array *tr = inode->i_private;

	if (tracing_disabled)
		return -ENODEV;

	if (trace_array_get(tr) < 0)
		return -ENODEV;

	filp->private_data = inode->i_private;

	return 0;
}

3099
static int tracing_release(struct inode *inode, struct file *file)
3100
{
3101
	struct trace_array *tr = inode->i_private;
3102
	struct seq_file *m = file->private_data;
3103
	struct trace_iterator *iter;
3104
	int cpu;
3105

3106
	if (!(file->f_mode & FMODE_READ)) {
3107
		trace_array_put(tr);
3108
		return 0;
3109
	}
3110

3111
	/* Writes do not use seq_file */
3112
	iter = m->private;
3113
	mutex_lock(&trace_types_lock);
3114

3115 3116 3117 3118 3119
	for_each_tracing_cpu(cpu) {
		if (iter->buffer_iter[cpu])
			ring_buffer_read_finish(iter->buffer_iter[cpu]);
	}

3120 3121 3122
	if (iter->trace && iter->trace->close)
		iter->trace->close(iter);

3123 3124
	if (!iter->snapshot)
		/* reenable tracing if it was previously enabled */
3125
		tracing_start_tr(tr);
3126 3127 3128

	__trace_array_put(tr);

3129 3130
	mutex_unlock(&trace_types_lock);

3131
	mutex_destroy(&iter->mutex);
3132
	free_cpumask_var(iter->started);
3133
	kfree(iter->trace);
3134
	kfree(iter->buffer_iter);
3135
	seq_release_private(inode, file);
3136

3137 3138 3139
	return 0;
}

3140 3141 3142 3143 3144
static int tracing_release_generic_tr(struct inode *inode, struct file *file)
{
	struct trace_array *tr = inode->i_private;

	trace_array_put(tr);
3145 3146 3147
	return 0;
}

3148 3149 3150 3151 3152 3153 3154 3155 3156
static int tracing_single_release_tr(struct inode *inode, struct file *file)
{
	struct trace_array *tr = inode->i_private;

	trace_array_put(tr);

	return single_release(inode, file);
}

3157 3158
static int tracing_open(struct inode *inode, struct file *file)
{
3159
	struct trace_array *tr = inode->i_private;
3160 3161
	struct trace_iterator *iter;
	int ret = 0;
3162

3163 3164 3165
	if (trace_array_get(tr) < 0)
		return -ENODEV;

3166
	/* If this file was open for write, then erase contents */
3167 3168 3169 3170
	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
		int cpu = tracing_get_cpu(inode);

		if (cpu == RING_BUFFER_ALL_CPUS)
3171
			tracing_reset_online_cpus(&tr->trace_buffer);
3172
		else
3173
			tracing_reset(&tr->trace_buffer, cpu);
3174
	}
3175

3176
	if (file->f_mode & FMODE_READ) {
3177
		iter = __tracing_open(inode, file, false);
3178 3179 3180 3181 3182
		if (IS_ERR(iter))
			ret = PTR_ERR(iter);
		else if (trace_flags & TRACE_ITER_LATENCY_FMT)
			iter->iter_flags |= TRACE_FILE_LAT_FMT;
	}
3183 3184 3185 3186

	if (ret < 0)
		trace_array_put(tr);

3187 3188 3189
	return ret;
}

3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210
/*
 * Some tracers are not suitable for instance buffers.
 * A tracer is always available for the global array (toplevel)
 * or if it explicitly states that it is.
 */
static bool
trace_ok_for_array(struct tracer *t, struct trace_array *tr)
{
	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
}

/* Find the next tracer that this trace array may use */
static struct tracer *
get_tracer_for_array(struct trace_array *tr, struct tracer *t)
{
	while (t && !trace_ok_for_array(t, tr))
		t = t->next;

	return t;
}

I
Ingo Molnar 已提交
3211
static void *
3212 3213
t_next(struct seq_file *m, void *v, loff_t *pos)
{
3214
	struct trace_array *tr = m->private;
L
Li Zefan 已提交
3215
	struct tracer *t = v;
3216 3217 3218 3219

	(*pos)++;

	if (t)
3220
		t = get_tracer_for_array(tr, t->next);
3221 3222 3223 3224 3225 3226

	return t;
}

static void *t_start(struct seq_file *m, loff_t *pos)
{
3227
	struct trace_array *tr = m->private;
L
Li Zefan 已提交
3228
	struct tracer *t;
3229 3230 3231
	loff_t l = 0;

	mutex_lock(&trace_types_lock);
3232 3233 3234 3235

	t = get_tracer_for_array(tr, trace_types);
	for (; t && l < *pos; t = t_next(m, t, &l))
			;
3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260

	return t;
}

static void t_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&trace_types_lock);
}

static int t_show(struct seq_file *m, void *v)
{
	struct tracer *t = v;

	if (!t)
		return 0;

	seq_printf(m, "%s", t->name);
	if (t->next)
		seq_putc(m, ' ');
	else
		seq_putc(m, '\n');

	return 0;
}

J
James Morris 已提交
3261
static const struct seq_operations show_traces_seq_ops = {
I
Ingo Molnar 已提交
3262 3263 3264 3265
	.start		= t_start,
	.next		= t_next,
	.stop		= t_stop,
	.show		= t_show,
3266 3267 3268 3269
};

static int show_traces_open(struct inode *inode, struct file *file)
{
3270 3271 3272 3273
	struct trace_array *tr = inode->i_private;
	struct seq_file *m;
	int ret;

S
Steven Rostedt 已提交
3274 3275 3276
	if (tracing_disabled)
		return -ENODEV;

3277 3278 3279 3280 3281 3282 3283 3284
	ret = seq_open(file, &show_traces_seq_ops);
	if (ret)
		return ret;

	m = file->private_data;
	m->private = tr;

	return 0;
3285 3286
}

3287 3288 3289 3290 3291 3292 3293
static ssize_t
tracing_write_stub(struct file *filp, const char __user *ubuf,
		   size_t count, loff_t *ppos)
{
	return count;
}

3294
loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3295
{
3296 3297
	int ret;

3298
	if (file->f_mode & FMODE_READ)
3299
		ret = seq_lseek(file, offset, whence);
3300
	else
3301 3302 3303
		file->f_pos = ret = 0;

	return ret;
3304 3305
}

3306
static const struct file_operations tracing_fops = {
I
Ingo Molnar 已提交
3307 3308
	.open		= tracing_open,
	.read		= seq_read,
3309
	.write		= tracing_write_stub,
3310
	.llseek		= tracing_lseek,
I
Ingo Molnar 已提交
3311
	.release	= tracing_release,
3312 3313
};

3314
static const struct file_operations show_traces_fops = {
I
Ingo Molnar 已提交
3315 3316 3317
	.open		= show_traces_open,
	.read		= seq_read,
	.release	= seq_release,
3318
	.llseek		= seq_lseek,
I
Ingo Molnar 已提交
3319 3320
};

3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332
/*
 * The tracer itself will not take this lock, but still we want
 * to provide a consistent cpumask to user-space:
 */
static DEFINE_MUTEX(tracing_cpumask_update_lock);

/*
 * Temporary storage for the character representation of the
 * CPU bitmask (and one more byte for the newline):
 */
static char mask_str[NR_CPUS + 1];

I
Ingo Molnar 已提交
3333 3334 3335 3336
static ssize_t
tracing_cpumask_read(struct file *filp, char __user *ubuf,
		     size_t count, loff_t *ppos)
{
3337
	struct trace_array *tr = file_inode(filp)->i_private;
3338
	int len;
I
Ingo Molnar 已提交
3339 3340

	mutex_lock(&tracing_cpumask_update_lock);
3341

3342
	len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3343 3344 3345 3346 3347 3348 3349 3350
	if (count - len < 2) {
		count = -EINVAL;
		goto out_err;
	}
	len += sprintf(mask_str + len, "\n");
	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);

out_err:
I
Ingo Molnar 已提交
3351 3352 3353 3354 3355 3356 3357 3358 3359
	mutex_unlock(&tracing_cpumask_update_lock);

	return count;
}

static ssize_t
tracing_cpumask_write(struct file *filp, const char __user *ubuf,
		      size_t count, loff_t *ppos)
{
3360
	struct trace_array *tr = file_inode(filp)->i_private;
3361
	cpumask_var_t tracing_cpumask_new;
3362
	int err, cpu;
3363 3364 3365

	if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
		return -ENOMEM;
I
Ingo Molnar 已提交
3366

3367
	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
I
Ingo Molnar 已提交
3368
	if (err)
3369 3370
		goto err_unlock;

3371 3372
	mutex_lock(&tracing_cpumask_update_lock);

3373
	local_irq_disable();
3374
	arch_spin_lock(&tr->max_lock);
3375
	for_each_tracing_cpu(cpu) {
3376 3377 3378 3379
		/*
		 * Increase/decrease the disabled counter if we are
		 * about to flip a bit in the cpumask:
		 */
3380
		if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3381
				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3382 3383
			atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
			ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3384
		}
3385
		if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3386
				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3387 3388
			atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
			ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3389 3390
		}
	}
3391
	arch_spin_unlock(&tr->max_lock);
3392
	local_irq_enable();
3393

3394
	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3395 3396

	mutex_unlock(&tracing_cpumask_update_lock);
3397
	free_cpumask_var(tracing_cpumask_new);
I
Ingo Molnar 已提交
3398 3399

	return count;
3400 3401

err_unlock:
3402
	free_cpumask_var(tracing_cpumask_new);
3403 3404

	return err;
I
Ingo Molnar 已提交
3405 3406
}

3407
static const struct file_operations tracing_cpumask_fops = {
3408
	.open		= tracing_open_generic_tr,
I
Ingo Molnar 已提交
3409 3410
	.read		= tracing_cpumask_read,
	.write		= tracing_cpumask_write,
3411
	.release	= tracing_release_generic_tr,
3412
	.llseek		= generic_file_llseek,
3413 3414
};

L
Li Zefan 已提交
3415
static int tracing_trace_options_show(struct seq_file *m, void *v)
3416
{
3417
	struct tracer_opt *trace_opts;
3418
	struct trace_array *tr = m->private;
3419 3420
	u32 tracer_flags;
	int i;
3421

3422
	mutex_lock(&trace_types_lock);
3423 3424
	tracer_flags = tr->current_trace->flags->val;
	trace_opts = tr->current_trace->flags->opts;
3425

3426 3427
	for (i = 0; trace_options[i]; i++) {
		if (trace_flags & (1 << i))
L
Li Zefan 已提交
3428
			seq_printf(m, "%s\n", trace_options[i]);
3429
		else
L
Li Zefan 已提交
3430
			seq_printf(m, "no%s\n", trace_options[i]);
3431 3432
	}

3433 3434
	for (i = 0; trace_opts[i].name; i++) {
		if (tracer_flags & trace_opts[i].bit)
L
Li Zefan 已提交
3435
			seq_printf(m, "%s\n", trace_opts[i].name);
3436
		else
L
Li Zefan 已提交
3437
			seq_printf(m, "no%s\n", trace_opts[i].name);
3438
	}
3439
	mutex_unlock(&trace_types_lock);
3440

L
Li Zefan 已提交
3441
	return 0;
3442 3443
}

3444
static int __set_tracer_option(struct trace_array *tr,
L
Li Zefan 已提交
3445 3446 3447
			       struct tracer_flags *tracer_flags,
			       struct tracer_opt *opts, int neg)
{
3448
	struct tracer *trace = tr->current_trace;
L
Li Zefan 已提交
3449
	int ret;
3450

3451
	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
L
Li Zefan 已提交
3452 3453 3454 3455 3456 3457 3458 3459
	if (ret)
		return ret;

	if (neg)
		tracer_flags->val &= ~opts->bit;
	else
		tracer_flags->val |= opts->bit;
	return 0;
3460 3461
}

3462
/* Try to assign a tracer specific option */
3463
static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3464
{
3465
	struct tracer *trace = tr->current_trace;
3466
	struct tracer_flags *tracer_flags = trace->flags;
3467
	struct tracer_opt *opts = NULL;
L
Li Zefan 已提交
3468
	int i;
3469

3470 3471
	for (i = 0; tracer_flags->opts[i].name; i++) {
		opts = &tracer_flags->opts[i];
3472

L
Li Zefan 已提交
3473
		if (strcmp(cmp, opts->name) == 0)
3474
			return __set_tracer_option(tr, trace->flags, opts, neg);
3475 3476
	}

L
Li Zefan 已提交
3477
	return -EINVAL;
3478 3479
}

3480 3481 3482 3483 3484 3485 3486 3487 3488
/* Some tracers require overwrite to stay enabled */
int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
{
	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
		return -1;

	return 0;
}

3489
int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3490 3491 3492
{
	/* do nothing if flag is already set */
	if (!!(trace_flags & mask) == !!enabled)
3493 3494 3495
		return 0;

	/* Give the tracer a chance to approve the change */
3496
	if (tr->current_trace->flag_changed)
3497
		if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3498
			return -EINVAL;
3499 3500 3501 3502 3503

	if (enabled)
		trace_flags |= mask;
	else
		trace_flags &= ~mask;
3504 3505 3506

	if (mask == TRACE_ITER_RECORD_CMD)
		trace_event_enable_cmd_record(enabled);
3507

3508
	if (mask == TRACE_ITER_OVERWRITE) {
3509
		ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3510
#ifdef CONFIG_TRACER_MAX_TRACE
3511
		ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3512 3513
#endif
	}
3514 3515 3516

	if (mask == TRACE_ITER_PRINTK)
		trace_printk_start_stop_comm(enabled);
3517 3518

	return 0;
3519 3520
}

3521
static int trace_set_options(struct trace_array *tr, char *option)
3522
{
L
Li Zefan 已提交
3523
	char *cmp;
3524
	int neg = 0;
3525
	int ret = -ENODEV;
3526 3527
	int i;

3528
	cmp = strstrip(option);
3529

L
Li Zefan 已提交
3530
	if (strncmp(cmp, "no", 2) == 0) {
3531 3532 3533 3534
		neg = 1;
		cmp += 2;
	}

3535 3536
	mutex_lock(&trace_types_lock);

3537
	for (i = 0; trace_options[i]; i++) {
L
Li Zefan 已提交
3538
		if (strcmp(cmp, trace_options[i]) == 0) {
3539
			ret = set_tracer_flag(tr, 1 << i, !neg);
3540 3541 3542
			break;
		}
	}
3543 3544

	/* If no option could be set, test the specific tracer options */
3545
	if (!trace_options[i])
3546
		ret = set_tracer_option(tr, cmp, neg);
3547 3548

	mutex_unlock(&trace_types_lock);
3549

3550 3551 3552 3553 3554 3555 3556
	return ret;
}

static ssize_t
tracing_trace_options_write(struct file *filp, const char __user *ubuf,
			size_t cnt, loff_t *ppos)
{
3557 3558
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
3559
	char buf[64];
3560
	int ret;
3561 3562 3563 3564 3565 3566 3567

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

3568 3569
	buf[cnt] = 0;

3570
	ret = trace_set_options(tr, buf);
3571 3572
	if (ret < 0)
		return ret;
3573

3574
	*ppos += cnt;
3575 3576 3577 3578

	return cnt;
}

L
Li Zefan 已提交
3579 3580
static int tracing_trace_options_open(struct inode *inode, struct file *file)
{
3581
	struct trace_array *tr = inode->i_private;
3582
	int ret;
3583

L
Li Zefan 已提交
3584 3585
	if (tracing_disabled)
		return -ENODEV;
3586

3587 3588 3589
	if (trace_array_get(tr) < 0)
		return -ENODEV;

3590 3591 3592 3593 3594
	ret = single_open(file, tracing_trace_options_show, inode->i_private);
	if (ret < 0)
		trace_array_put(tr);

	return ret;
L
Li Zefan 已提交
3595 3596
}

3597
static const struct file_operations tracing_iter_fops = {
L
Li Zefan 已提交
3598 3599 3600
	.open		= tracing_trace_options_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
3601
	.release	= tracing_single_release_tr,
3602
	.write		= tracing_trace_options_write,
3603 3604
};

I
Ingo Molnar 已提交
3605 3606
static const char readme_msg[] =
	"tracing mini-HOWTO:\n\n"
3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630
	"# echo 0 > tracing_on : quick way to disable tracing\n"
	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
	" Important files:\n"
	"  trace\t\t\t- The static contents of the buffer\n"
	"\t\t\t  To clear the buffer write into this file: echo > trace\n"
	"  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
	"  current_tracer\t- function and latency tracers\n"
	"  available_tracers\t- list of configured tracers for current_tracer\n"
	"  buffer_size_kb\t- view and modify size of per cpu buffer\n"
	"  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
	"  trace_clock\t\t-change the clock used to order events\n"
	"       local:   Per cpu clock but may not be synced across CPUs\n"
	"      global:   Synced across CPUs but slows tracing down.\n"
	"     counter:   Not a clock, but just an increment\n"
	"      uptime:   Jiffy counter from time of boot\n"
	"        perf:   Same clock that perf events use\n"
#ifdef CONFIG_X86_64
	"     x86-tsc:   TSC cycle counter\n"
#endif
	"\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
	"  tracing_cpumask\t- Limit which CPUs to trace\n"
	"  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
	"\t\t\t  Remove sub-buffer with rmdir\n"
	"  trace_options\t\t- Set format or modify how tracing happens\n"
3631 3632
	"\t\t\t  Disable an option by adding a suffix 'no' to the\n"
	"\t\t\t  option name\n"
3633
	"  saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3634 3635
#ifdef CONFIG_DYNAMIC_FTRACE
	"\n  available_filter_functions - list of functions that can be filtered on\n"
3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646
	"  set_ftrace_filter\t- echo function name in here to only trace these\n"
	"\t\t\t  functions\n"
	"\t     accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
	"\t     modules: Can select a group via module\n"
	"\t      Format: :mod:<module-name>\n"
	"\t     example: echo :mod:ext3 > set_ftrace_filter\n"
	"\t    triggers: a command to perform when function is hit\n"
	"\t      Format: <function>:<trigger>[:count]\n"
	"\t     trigger: traceon, traceoff\n"
	"\t\t      enable_event:<system>:<event>\n"
	"\t\t      disable_event:<system>:<event>\n"
3647
#ifdef CONFIG_STACKTRACE
3648
	"\t\t      stacktrace\n"
3649 3650
#endif
#ifdef CONFIG_TRACER_SNAPSHOT
3651
	"\t\t      snapshot\n"
3652
#endif
3653 3654
	"\t\t      dump\n"
	"\t\t      cpudump\n"
3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666
	"\t     example: echo do_fault:traceoff > set_ftrace_filter\n"
	"\t              echo do_trap:traceoff:3 > set_ftrace_filter\n"
	"\t     The first one will disable tracing every time do_fault is hit\n"
	"\t     The second will disable tracing at most 3 times when do_trap is hit\n"
	"\t       The first time do trap is hit and it disables tracing, the\n"
	"\t       counter will decrement to 2. If tracing is already disabled,\n"
	"\t       the counter will not decrement. It only decrements when the\n"
	"\t       trigger did work\n"
	"\t     To remove trigger without count:\n"
	"\t       echo '!<function>:<trigger> > set_ftrace_filter\n"
	"\t     To remove trigger with a count:\n"
	"\t       echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3667
	"  set_ftrace_notrace\t- echo function name in here to never trace.\n"
3668 3669 3670
	"\t    accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
	"\t    modules: Can select a group via module command :mod:\n"
	"\t    Does not accept triggers\n"
3671 3672
#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_FUNCTION_TRACER
3673 3674
	"  set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
	"\t\t    (function)\n"
3675 3676 3677
#endif
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	"  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3678
	"  set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3679 3680 3681
	"  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
#endif
#ifdef CONFIG_TRACER_SNAPSHOT
3682 3683 3684
	"\n  snapshot\t\t- Like 'trace' but shows the content of the static\n"
	"\t\t\t  snapshot buffer. Read the contents for more\n"
	"\t\t\t  information\n"
3685
#endif
3686
#ifdef CONFIG_STACK_TRACER
3687 3688
	"  stack_trace\t\t- Shows the max stack trace when active\n"
	"  stack_max_size\t- Shows current max stack size that was traced\n"
3689 3690
	"\t\t\t  Write into this file to reset the max size (trigger a\n"
	"\t\t\t  new trace)\n"
3691
#ifdef CONFIG_DYNAMIC_FTRACE
3692 3693
	"  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
	"\t\t\t  traces\n"
3694
#endif
3695
#endif /* CONFIG_STACK_TRACER */
3696 3697 3698
	"  events/\t\t- Directory containing all trace event subsystems:\n"
	"      enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
	"  events/<system>/\t- Directory containing all trace events for <system>:\n"
3699 3700
	"      enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
	"\t\t\t  events\n"
3701
	"      filter\t\t- If set, only events passing filter are traced\n"
3702 3703
	"  events/<system>/<event>/\t- Directory containing control files for\n"
	"\t\t\t  <event>:\n"
3704 3705 3706
	"      enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
	"      filter\t\t- If set, only events passing filter are traced\n"
	"      trigger\t\t- If set, a command to perform when event is hit\n"
3707 3708 3709 3710
	"\t    Format: <trigger>[:count][if <filter>]\n"
	"\t   trigger: traceon, traceoff\n"
	"\t            enable_event:<system>:<event>\n"
	"\t            disable_event:<system>:<event>\n"
3711
#ifdef CONFIG_STACKTRACE
3712
	"\t\t    stacktrace\n"
3713 3714
#endif
#ifdef CONFIG_TRACER_SNAPSHOT
3715
	"\t\t    snapshot\n"
3716
#endif
3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731
	"\t   example: echo traceoff > events/block/block_unplug/trigger\n"
	"\t            echo traceoff:3 > events/block/block_unplug/trigger\n"
	"\t            echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
	"\t                  events/block/block_unplug/trigger\n"
	"\t   The first disables tracing every time block_unplug is hit.\n"
	"\t   The second disables tracing the first 3 times block_unplug is hit.\n"
	"\t   The third enables the kmalloc event the first 3 times block_unplug\n"
	"\t     is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
	"\t   Like function triggers, the counter is only decremented if it\n"
	"\t    enabled or disabled tracing.\n"
	"\t   To remove a trigger without a count:\n"
	"\t     echo '!<trigger> > <system>/<event>/trigger\n"
	"\t   To remove a trigger with a count:\n"
	"\t     echo '!<trigger>:0 > <system>/<event>/trigger\n"
	"\t   Filters can be ignored when removing a trigger.\n"
I
Ingo Molnar 已提交
3732 3733 3734 3735 3736 3737 3738 3739 3740 3741
;

static ssize_t
tracing_readme_read(struct file *filp, char __user *ubuf,
		       size_t cnt, loff_t *ppos)
{
	return simple_read_from_buffer(ubuf, cnt, ppos,
					readme_msg, strlen(readme_msg));
}

3742
static const struct file_operations tracing_readme_fops = {
I
Ingo Molnar 已提交
3743 3744
	.open		= tracing_open_generic,
	.read		= tracing_readme_read,
3745
	.llseek		= generic_file_llseek,
I
Ingo Molnar 已提交
3746 3747
};

3748 3749 3750
static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
{
	unsigned int *ptr = v;
3751

3752 3753
	if (*pos || m->count)
		ptr++;
3754

3755
	(*pos)++;
3756

3757 3758
	for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
	     ptr++) {
3759 3760
		if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
			continue;
3761

3762 3763
		return ptr;
	}
3764

3765 3766 3767 3768 3769 3770 3771
	return NULL;
}

static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
{
	void *v;
	loff_t l = 0;
3772

3773 3774 3775
	preempt_disable();
	arch_spin_lock(&trace_cmdline_lock);

3776
	v = &savedcmd->map_cmdline_to_pid[0];
3777 3778 3779 3780
	while (l <= *pos) {
		v = saved_cmdlines_next(m, v, &l);
		if (!v)
			return NULL;
3781 3782
	}

3783 3784 3785 3786 3787
	return v;
}

static void saved_cmdlines_stop(struct seq_file *m, void *v)
{
3788 3789
	arch_spin_unlock(&trace_cmdline_lock);
	preempt_enable();
3790
}
3791

3792 3793 3794 3795
static int saved_cmdlines_show(struct seq_file *m, void *v)
{
	char buf[TASK_COMM_LEN];
	unsigned int *pid = v;
3796

3797
	__trace_find_cmdline(*pid, buf);
3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814
	seq_printf(m, "%d %s\n", *pid, buf);
	return 0;
}

static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
	.start		= saved_cmdlines_start,
	.next		= saved_cmdlines_next,
	.stop		= saved_cmdlines_stop,
	.show		= saved_cmdlines_show,
};

static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
{
	if (tracing_disabled)
		return -ENODEV;

	return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3815 3816 3817
}

static const struct file_operations tracing_saved_cmdlines_fops = {
3818 3819 3820 3821
	.open		= tracing_saved_cmdlines_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release,
3822 3823
};

3824 3825 3826 3827 3828 3829 3830 3831
static ssize_t
tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
				 size_t cnt, loff_t *ppos)
{
	char buf[64];
	int r;

	arch_spin_lock(&trace_cmdline_lock);
3832
	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848
	arch_spin_unlock(&trace_cmdline_lock);

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
{
	kfree(s->saved_cmdlines);
	kfree(s->map_cmdline_to_pid);
	kfree(s);
}

static int tracing_resize_saved_cmdlines(unsigned int val)
{
	struct saved_cmdlines_buffer *s, *savedcmd_temp;

3849
	s = kmalloc(sizeof(*s), GFP_KERNEL);
3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896
	if (!s)
		return -ENOMEM;

	if (allocate_cmdlines_buffer(val, s) < 0) {
		kfree(s);
		return -ENOMEM;
	}

	arch_spin_lock(&trace_cmdline_lock);
	savedcmd_temp = savedcmd;
	savedcmd = s;
	arch_spin_unlock(&trace_cmdline_lock);
	free_saved_cmdlines_buffer(savedcmd_temp);

	return 0;
}

static ssize_t
tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
				  size_t cnt, loff_t *ppos)
{
	unsigned long val;
	int ret;

	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
		return ret;

	/* must have at least 1 entry or less than PID_MAX_DEFAULT */
	if (!val || val > PID_MAX_DEFAULT)
		return -EINVAL;

	ret = tracing_resize_saved_cmdlines((unsigned int)val);
	if (ret < 0)
		return ret;

	*ppos += cnt;

	return cnt;
}

static const struct file_operations tracing_saved_cmdlines_size_fops = {
	.open		= tracing_open_generic,
	.read		= tracing_saved_cmdlines_size_read,
	.write		= tracing_saved_cmdlines_size_write,
};

3897 3898 3899 3900
static ssize_t
tracing_set_trace_read(struct file *filp, char __user *ubuf,
		       size_t cnt, loff_t *ppos)
{
3901
	struct trace_array *tr = filp->private_data;
L
Li Zefan 已提交
3902
	char buf[MAX_TRACER_SIZE+2];
3903 3904 3905
	int r;

	mutex_lock(&trace_types_lock);
3906
	r = sprintf(buf, "%s\n", tr->current_trace->name);
3907 3908
	mutex_unlock(&trace_types_lock);

I
Ingo Molnar 已提交
3909
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3910 3911
}

3912 3913
int tracer_init(struct tracer *t, struct trace_array *tr)
{
3914
	tracing_reset_online_cpus(&tr->trace_buffer);
3915 3916 3917
	return t->init(tr);
}

3918
static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3919 3920
{
	int cpu;
3921

3922
	for_each_tracing_cpu(cpu)
3923
		per_cpu_ptr(buf->data, cpu)->entries = val;
3924 3925
}

3926
#ifdef CONFIG_TRACER_MAX_TRACE
3927
/* resize @tr's buffer to the size of @size_tr's entries */
3928 3929
static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
					struct trace_buffer *size_buf, int cpu_id)
3930 3931 3932 3933 3934
{
	int cpu, ret = 0;

	if (cpu_id == RING_BUFFER_ALL_CPUS) {
		for_each_tracing_cpu(cpu) {
3935 3936
			ret = ring_buffer_resize(trace_buf->buffer,
				 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3937 3938
			if (ret < 0)
				break;
3939 3940
			per_cpu_ptr(trace_buf->data, cpu)->entries =
				per_cpu_ptr(size_buf->data, cpu)->entries;
3941 3942
		}
	} else {
3943 3944
		ret = ring_buffer_resize(trace_buf->buffer,
				 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3945
		if (ret == 0)
3946 3947
			per_cpu_ptr(trace_buf->data, cpu_id)->entries =
				per_cpu_ptr(size_buf->data, cpu_id)->entries;
3948 3949 3950 3951
	}

	return ret;
}
3952
#endif /* CONFIG_TRACER_MAX_TRACE */
3953

3954 3955
static int __tracing_resize_ring_buffer(struct trace_array *tr,
					unsigned long size, int cpu)
3956 3957 3958 3959 3960
{
	int ret;

	/*
	 * If kernel or user changes the size of the ring buffer
3961 3962
	 * we use the size that was given, and we can forget about
	 * expanding it later.
3963
	 */
3964
	ring_buffer_expanded = true;
3965

3966
	/* May be called before buffers are initialized */
3967
	if (!tr->trace_buffer.buffer)
3968 3969
		return 0;

3970
	ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3971 3972 3973
	if (ret < 0)
		return ret;

3974
#ifdef CONFIG_TRACER_MAX_TRACE
3975 3976
	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
	    !tr->current_trace->use_max_tr)
3977 3978
		goto out;

3979
	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3980
	if (ret < 0) {
3981 3982
		int r = resize_buffer_duplicate_size(&tr->trace_buffer,
						     &tr->trace_buffer, cpu);
3983
		if (r < 0) {
3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997
			/*
			 * AARGH! We are left with different
			 * size max buffer!!!!
			 * The max buffer is our "snapshot" buffer.
			 * When a tracer needs a snapshot (one of the
			 * latency tracers), it swaps the max buffer
			 * with the saved snap shot. We succeeded to
			 * update the size of the main buffer, but failed to
			 * update the size of the max buffer. But when we tried
			 * to reset the main buffer to the original size, we
			 * failed there too. This is very unlikely to
			 * happen, but if it does, warn and kill all
			 * tracing.
			 */
3998 3999 4000 4001 4002 4003
			WARN_ON(1);
			tracing_disabled = 1;
		}
		return ret;
	}

4004
	if (cpu == RING_BUFFER_ALL_CPUS)
4005
		set_buffer_entries(&tr->max_buffer, size);
4006
	else
4007
		per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4008

4009
 out:
4010 4011
#endif /* CONFIG_TRACER_MAX_TRACE */

4012
	if (cpu == RING_BUFFER_ALL_CPUS)
4013
		set_buffer_entries(&tr->trace_buffer, size);
4014
	else
4015
		per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4016 4017 4018 4019

	return ret;
}

4020 4021
static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
					  unsigned long size, int cpu_id)
4022
{
4023
	int ret = size;
4024 4025 4026

	mutex_lock(&trace_types_lock);

4027 4028 4029 4030 4031 4032 4033
	if (cpu_id != RING_BUFFER_ALL_CPUS) {
		/* make sure, this cpu is enabled in the mask */
		if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
			ret = -EINVAL;
			goto out;
		}
	}
4034

4035
	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4036 4037 4038
	if (ret < 0)
		ret = -ENOMEM;

4039
out:
4040 4041 4042 4043 4044
	mutex_unlock(&trace_types_lock);

	return ret;
}

4045

4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
/**
 * tracing_update_buffers - used by tracing facility to expand ring buffers
 *
 * To save on memory when the tracing is never used on a system with it
 * configured in. The ring buffers are set to a minimum size. But once
 * a user starts to use the tracing facility, then they need to grow
 * to their default size.
 *
 * This function is to be called when a tracer is about to be used.
 */
int tracing_update_buffers(void)
{
	int ret = 0;

4060
	mutex_lock(&trace_types_lock);
4061
	if (!ring_buffer_expanded)
4062
		ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4063
						RING_BUFFER_ALL_CPUS);
4064
	mutex_unlock(&trace_types_lock);
4065 4066 4067 4068

	return ret;
}

4069 4070 4071
struct trace_option_dentry;

static struct trace_option_dentry *
4072
create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4073 4074 4075 4076

static void
destroy_trace_option_files(struct trace_option_dentry *topts);

4077 4078 4079 4080 4081 4082 4083 4084 4085
/*
 * Used to clear out the tracer before deletion of an instance.
 * Must have trace_types_lock held.
 */
static void tracing_set_nop(struct trace_array *tr)
{
	if (tr->current_trace == &nop_trace)
		return;
	
4086
	tr->current_trace->enabled--;
4087 4088 4089 4090 4091 4092 4093

	if (tr->current_trace->reset)
		tr->current_trace->reset(tr);

	tr->current_trace = &nop_trace;
}

4094
static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4095
{
4096
	static struct trace_option_dentry *topts;
4097
	struct tracer *t;
4098
#ifdef CONFIG_TRACER_MAX_TRACE
4099
	bool had_max_tr;
4100
#endif
4101
	int ret = 0;
4102

4103 4104
	mutex_lock(&trace_types_lock);

4105
	if (!ring_buffer_expanded) {
4106
		ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4107
						RING_BUFFER_ALL_CPUS);
4108
		if (ret < 0)
4109
			goto out;
4110 4111 4112
		ret = 0;
	}

4113 4114 4115 4116
	for (t = trace_types; t; t = t->next) {
		if (strcmp(t->name, buf) == 0)
			break;
	}
4117 4118 4119 4120
	if (!t) {
		ret = -EINVAL;
		goto out;
	}
4121
	if (t == tr->current_trace)
4122 4123
		goto out;

4124 4125 4126 4127 4128 4129
	/* Some tracers are only allowed for the top level buffer */
	if (!trace_ok_for_array(t, tr)) {
		ret = -EINVAL;
		goto out;
	}

4130
	trace_branch_disable();
4131

4132
	tr->current_trace->enabled--;
4133

4134 4135
	if (tr->current_trace->reset)
		tr->current_trace->reset(tr);
4136

4137
	/* Current trace needs to be nop_trace before synchronize_sched */
4138
	tr->current_trace = &nop_trace;
4139

4140 4141
#ifdef CONFIG_TRACER_MAX_TRACE
	had_max_tr = tr->allocated_snapshot;
4142 4143 4144 4145 4146 4147 4148 4149 4150 4151

	if (had_max_tr && !t->use_max_tr) {
		/*
		 * We need to make sure that the update_max_tr sees that
		 * current_trace changed to nop_trace to keep it from
		 * swapping the buffers after we resize it.
		 * The update_max_tr is called from interrupts disabled
		 * so a synchronized_sched() is sufficient.
		 */
		synchronize_sched();
4152
		free_snapshot(tr);
4153
	}
4154
#endif
4155 4156 4157 4158 4159
	/* Currently, only the top instance has options */
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		destroy_trace_option_files(topts);
		topts = create_trace_option_files(tr, t);
	}
4160 4161

#ifdef CONFIG_TRACER_MAX_TRACE
4162
	if (t->use_max_tr && !had_max_tr) {
4163
		ret = alloc_snapshot(tr);
4164 4165
		if (ret < 0)
			goto out;
4166
	}
4167
#endif
4168

4169
	if (t->init) {
4170
		ret = tracer_init(t, tr);
4171 4172 4173
		if (ret)
			goto out;
	}
4174

4175
	tr->current_trace = t;
4176
	tr->current_trace->enabled++;
4177
	trace_branch_enable(tr);
4178 4179 4180
 out:
	mutex_unlock(&trace_types_lock);

4181 4182 4183 4184 4185 4186 4187
	return ret;
}

static ssize_t
tracing_set_trace_write(struct file *filp, const char __user *ubuf,
			size_t cnt, loff_t *ppos)
{
4188
	struct trace_array *tr = filp->private_data;
L
Li Zefan 已提交
4189
	char buf[MAX_TRACER_SIZE+1];
4190 4191
	int i;
	size_t ret;
4192 4193 4194
	int err;

	ret = cnt;
4195

L
Li Zefan 已提交
4196 4197
	if (cnt > MAX_TRACER_SIZE)
		cnt = MAX_TRACER_SIZE;
4198 4199 4200 4201 4202 4203 4204 4205 4206 4207

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	/* strip ending whitespace. */
	for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
		buf[i] = 0;

4208
	err = tracing_set_tracer(tr, buf);
4209 4210
	if (err)
		return err;
4211

4212
	*ppos += ret;
4213

4214
	return ret;
4215 4216 4217
}

static ssize_t
4218 4219
tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
		   size_t cnt, loff_t *ppos)
4220 4221 4222 4223
{
	char buf[64];
	int r;

S
Steven Rostedt 已提交
4224
	r = snprintf(buf, sizeof(buf), "%ld\n",
4225
		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
S
Steven Rostedt 已提交
4226 4227
	if (r > sizeof(buf))
		r = sizeof(buf);
I
Ingo Molnar 已提交
4228
	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4229 4230 4231
}

static ssize_t
4232 4233
tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
		    size_t cnt, loff_t *ppos)
4234
{
4235
	unsigned long val;
4236
	int ret;
4237

4238 4239
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
4240
		return ret;
4241 4242 4243 4244 4245 4246

	*ptr = val * 1000;

	return cnt;
}

4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292
static ssize_t
tracing_thresh_read(struct file *filp, char __user *ubuf,
		    size_t cnt, loff_t *ppos)
{
	return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
}

static ssize_t
tracing_thresh_write(struct file *filp, const char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	struct trace_array *tr = filp->private_data;
	int ret;

	mutex_lock(&trace_types_lock);
	ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
	if (ret < 0)
		goto out;

	if (tr->current_trace->update_thresh) {
		ret = tr->current_trace->update_thresh(tr);
		if (ret < 0)
			goto out;
	}

	ret = cnt;
out:
	mutex_unlock(&trace_types_lock);

	return ret;
}

static ssize_t
tracing_max_lat_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
	return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
}

static ssize_t
tracing_max_lat_write(struct file *filp, const char __user *ubuf,
		      size_t cnt, loff_t *ppos)
{
	return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
}

4293 4294
static int tracing_open_pipe(struct inode *inode, struct file *filp)
{
4295
	struct trace_array *tr = inode->i_private;
4296
	struct trace_iterator *iter;
4297
	int ret = 0;
4298 4299 4300 4301

	if (tracing_disabled)
		return -ENODEV;

4302 4303 4304
	if (trace_array_get(tr) < 0)
		return -ENODEV;

4305 4306
	mutex_lock(&trace_types_lock);

4307 4308
	/* create a buffer to store the information to pass to userspace */
	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4309 4310
	if (!iter) {
		ret = -ENOMEM;
4311
		__trace_array_put(tr);
4312 4313
		goto out;
	}
4314

4315 4316 4317 4318 4319 4320 4321 4322 4323
	/*
	 * We make a copy of the current tracer to avoid concurrent
	 * changes on it while we are reading.
	 */
	iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
	if (!iter->trace) {
		ret = -ENOMEM;
		goto fail;
	}
4324
	*iter->trace = *tr->current_trace;
4325

4326
	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4327
		ret = -ENOMEM;
4328
		goto fail;
4329 4330
	}

4331
	/* trace pipe does not show start of buffer */
4332
	cpumask_setall(iter->started);
4333

4334 4335 4336
	if (trace_flags & TRACE_ITER_LATENCY_FMT)
		iter->iter_flags |= TRACE_FILE_LAT_FMT;

4337
	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
4338
	if (trace_clocks[tr->clock_id].in_ns)
4339 4340
		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;

4341 4342 4343
	iter->tr = tr;
	iter->trace_buffer = &tr->trace_buffer;
	iter->cpu_file = tracing_get_cpu(inode);
4344
	mutex_init(&iter->mutex);
4345 4346
	filp->private_data = iter;

4347 4348 4349
	if (iter->trace->pipe_open)
		iter->trace->pipe_open(iter);

4350
	nonseekable_open(inode, filp);
4351 4352 4353
out:
	mutex_unlock(&trace_types_lock);
	return ret;
4354 4355 4356 4357

fail:
	kfree(iter->trace);
	kfree(iter);
4358
	__trace_array_put(tr);
4359 4360
	mutex_unlock(&trace_types_lock);
	return ret;
4361 4362 4363 4364 4365
}

static int tracing_release_pipe(struct inode *inode, struct file *file)
{
	struct trace_iterator *iter = file->private_data;
4366
	struct trace_array *tr = inode->i_private;
4367

4368 4369
	mutex_lock(&trace_types_lock);

4370
	if (iter->trace->pipe_close)
S
Steven Rostedt 已提交
4371 4372
		iter->trace->pipe_close(iter);

4373 4374
	mutex_unlock(&trace_types_lock);

4375
	free_cpumask_var(iter->started);
4376 4377
	mutex_destroy(&iter->mutex);
	kfree(iter->trace);
4378 4379
	kfree(iter);

4380 4381
	trace_array_put(tr);

4382 4383 4384
	return 0;
}

4385
static unsigned int
4386
trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4387
{
4388 4389 4390
	/* Iterators are static, they should be filled or empty */
	if (trace_buffer_iter(iter, iter->cpu_file))
		return POLLIN | POLLRDNORM;
4391

4392
	if (trace_flags & TRACE_ITER_BLOCK)
4393 4394 4395 4396
		/*
		 * Always select as readable when in blocking mode
		 */
		return POLLIN | POLLRDNORM;
4397
	else
4398
		return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4399
					     filp, poll_table);
4400 4401
}

4402 4403 4404 4405 4406 4407
static unsigned int
tracing_poll_pipe(struct file *filp, poll_table *poll_table)
{
	struct trace_iterator *iter = filp->private_data;

	return trace_poll(iter, filp, poll_table);
4408 4409
}

4410 4411
/* Must be called with trace_types_lock mutex held. */
static int tracing_wait_pipe(struct file *filp)
4412 4413
{
	struct trace_iterator *iter = filp->private_data;
4414
	int ret;
4415 4416

	while (trace_empty(iter)) {
4417

4418
		if ((filp->f_flags & O_NONBLOCK)) {
4419
			return -EAGAIN;
4420
		}
4421

4422
		/*
L
Liu Bo 已提交
4423
		 * We block until we read something and tracing is disabled.
4424 4425 4426 4427 4428 4429 4430
		 * We still block if tracing is disabled, but we have never
		 * read anything. This allows a user to cat this file, and
		 * then enable tracing. But after we have read something,
		 * we give an EOF when tracing is again disabled.
		 *
		 * iter->pos will be 0 if we haven't read anything.
		 */
4431
		if (!tracing_is_on() && iter->pos)
4432
			break;
4433 4434 4435

		mutex_unlock(&iter->mutex);

4436
		ret = wait_on_pipe(iter);
4437 4438 4439

		mutex_lock(&iter->mutex);

4440 4441 4442
		if (ret)
			return ret;

4443 4444
		if (signal_pending(current))
			return -EINTR;
4445 4446
	}

4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457
	return 1;
}

/*
 * Consumer reader.
 */
static ssize_t
tracing_read_pipe(struct file *filp, char __user *ubuf,
		  size_t cnt, loff_t *ppos)
{
	struct trace_iterator *iter = filp->private_data;
4458
	struct trace_array *tr = iter->tr;
4459 4460 4461 4462 4463 4464 4465
	ssize_t sret;

	/* return any leftover data */
	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
	if (sret != -EBUSY)
		return sret;

4466
	trace_seq_init(&iter->seq);
4467

4468
	/* copy the tracer to avoid using a global lock all around */
4469
	mutex_lock(&trace_types_lock);
4470 4471
	if (unlikely(iter->trace->name != tr->current_trace->name))
		*iter->trace = *tr->current_trace;
4472 4473 4474 4475 4476 4477 4478 4479
	mutex_unlock(&trace_types_lock);

	/*
	 * Avoid more than one consumer on a single file descriptor
	 * This is just a matter of traces coherency, the ring buffer itself
	 * is protected.
	 */
	mutex_lock(&iter->mutex);
4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490
	if (iter->trace->read) {
		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
		if (sret)
			goto out;
	}

waitagain:
	sret = tracing_wait_pipe(filp);
	if (sret <= 0)
		goto out;

4491
	/* stop when tracing is finished */
4492 4493
	if (trace_empty(iter)) {
		sret = 0;
4494
		goto out;
4495
	}
4496 4497 4498 4499

	if (cnt >= PAGE_SIZE)
		cnt = PAGE_SIZE - 1;

4500 4501 4502 4503
	/* reset all but tr, trace, and overruns */
	memset(&iter->seq, 0,
	       sizeof(struct trace_iterator) -
	       offsetof(struct trace_iterator, seq));
4504
	cpumask_clear(iter->started);
4505
	iter->pos = -1;
4506

4507
	trace_event_read_lock();
4508
	trace_access_lock(iter->cpu_file);
4509
	while (trace_find_next_entry_inc(iter) != NULL) {
4510
		enum print_line_t ret;
S
Steven Rostedt 已提交
4511 4512
		int len = iter->seq.len;

I
Ingo Molnar 已提交
4513
		ret = print_trace_line(iter);
4514
		if (ret == TRACE_TYPE_PARTIAL_LINE) {
S
Steven Rostedt 已提交
4515 4516
			/* don't print partial lines */
			iter->seq.len = len;
4517
			break;
S
Steven Rostedt 已提交
4518
		}
4519 4520
		if (ret != TRACE_TYPE_NO_CONSUME)
			trace_consume(iter);
4521 4522 4523

		if (iter->seq.len >= cnt)
			break;
4524 4525 4526 4527 4528 4529 4530 4531

		/*
		 * Setting the full flag means we reached the trace_seq buffer
		 * size and we should leave by partial output condition above.
		 * One of the trace_seq_* functions is not used properly.
		 */
		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
			  iter->ent->type);
4532
	}
4533
	trace_access_unlock(iter->cpu_file);
4534
	trace_event_read_unlock();
4535 4536

	/* Now copy what we have to the user */
4537 4538
	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
	if (iter->seq.readpos >= iter->seq.len)
4539
		trace_seq_init(&iter->seq);
P
Pekka Paalanen 已提交
4540 4541

	/*
L
Lucas De Marchi 已提交
4542
	 * If there was nothing to send to user, in spite of consuming trace
P
Pekka Paalanen 已提交
4543 4544
	 * entries, go back to wait for more entries.
	 */
4545
	if (sret == -EBUSY)
P
Pekka Paalanen 已提交
4546
		goto waitagain;
4547

4548
out:
4549
	mutex_unlock(&iter->mutex);
4550

4551
	return sret;
4552 4553
}

4554 4555 4556 4557 4558 4559
static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
				     unsigned int idx)
{
	__free_page(spd->pages[idx]);
}

4560
static const struct pipe_buf_operations tracing_pipe_buf_ops = {
S
Steven Rostedt 已提交
4561 4562
	.can_merge		= 0,
	.confirm		= generic_pipe_buf_confirm,
4563
	.release		= generic_pipe_buf_release,
S
Steven Rostedt 已提交
4564 4565
	.steal			= generic_pipe_buf_steal,
	.get			= generic_pipe_buf_get,
4566 4567
};

S
Steven Rostedt 已提交
4568
static size_t
4569
tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
S
Steven Rostedt 已提交
4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588
{
	size_t count;
	int ret;

	/* Seq buffer is page-sized, exactly what we need. */
	for (;;) {
		count = iter->seq.len;
		ret = print_trace_line(iter);
		count = iter->seq.len - count;
		if (rem < count) {
			rem = 0;
			iter->seq.len -= count;
			break;
		}
		if (ret == TRACE_TYPE_PARTIAL_LINE) {
			iter->seq.len -= count;
			break;
		}

4589 4590
		if (ret != TRACE_TYPE_NO_CONSUME)
			trace_consume(iter);
S
Steven Rostedt 已提交
4591
		rem -= count;
4592
		if (!trace_find_next_entry_inc(iter))	{
S
Steven Rostedt 已提交
4593 4594 4595 4596 4597 4598 4599 4600 4601
			rem = 0;
			iter->ent = NULL;
			break;
		}
	}

	return rem;
}

4602 4603 4604 4605 4606 4607
static ssize_t tracing_splice_read_pipe(struct file *filp,
					loff_t *ppos,
					struct pipe_inode_info *pipe,
					size_t len,
					unsigned int flags)
{
4608 4609
	struct page *pages_def[PIPE_DEF_BUFFERS];
	struct partial_page partial_def[PIPE_DEF_BUFFERS];
4610 4611
	struct trace_iterator *iter = filp->private_data;
	struct splice_pipe_desc spd = {
4612 4613
		.pages		= pages_def,
		.partial	= partial_def,
S
Steven Rostedt 已提交
4614
		.nr_pages	= 0, /* This gets updated below. */
4615
		.nr_pages_max	= PIPE_DEF_BUFFERS,
S
Steven Rostedt 已提交
4616 4617 4618
		.flags		= flags,
		.ops		= &tracing_pipe_buf_ops,
		.spd_release	= tracing_spd_release_pipe,
4619
	};
4620
	struct trace_array *tr = iter->tr;
4621
	ssize_t ret;
S
Steven Rostedt 已提交
4622
	size_t rem;
4623 4624
	unsigned int i;

4625 4626 4627
	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

4628
	/* copy the tracer to avoid using a global lock all around */
4629
	mutex_lock(&trace_types_lock);
4630 4631
	if (unlikely(iter->trace->name != tr->current_trace->name))
		*iter->trace = *tr->current_trace;
4632 4633 4634
	mutex_unlock(&trace_types_lock);

	mutex_lock(&iter->mutex);
4635 4636 4637 4638 4639

	if (iter->trace->splice_read) {
		ret = iter->trace->splice_read(iter, filp,
					       ppos, pipe, len, flags);
		if (ret)
S
Steven Rostedt 已提交
4640
			goto out_err;
4641 4642 4643 4644
	}

	ret = tracing_wait_pipe(filp);
	if (ret <= 0)
S
Steven Rostedt 已提交
4645
		goto out_err;
4646

4647
	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4648
		ret = -EFAULT;
S
Steven Rostedt 已提交
4649
		goto out_err;
4650 4651
	}

4652
	trace_event_read_lock();
4653
	trace_access_lock(iter->cpu_file);
4654

4655
	/* Fill as many pages as possible. */
4656
	for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4657 4658
		spd.pages[i] = alloc_page(GFP_KERNEL);
		if (!spd.pages[i])
S
Steven Rostedt 已提交
4659
			break;
4660

4661
		rem = tracing_fill_pipe_page(rem, iter);
4662 4663 4664

		/* Copy the data into the page, so we can start over. */
		ret = trace_seq_to_buffer(&iter->seq,
4665
					  page_address(spd.pages[i]),
4666 4667
					  iter->seq.len);
		if (ret < 0) {
4668
			__free_page(spd.pages[i]);
4669 4670
			break;
		}
4671 4672
		spd.partial[i].offset = 0;
		spd.partial[i].len = iter->seq.len;
4673

4674
		trace_seq_init(&iter->seq);
4675 4676
	}

4677
	trace_access_unlock(iter->cpu_file);
4678
	trace_event_read_unlock();
4679
	mutex_unlock(&iter->mutex);
4680 4681 4682

	spd.nr_pages = i;

4683 4684
	ret = splice_to_pipe(pipe, &spd);
out:
4685
	splice_shrink_spd(&spd);
4686
	return ret;
4687

S
Steven Rostedt 已提交
4688
out_err:
4689
	mutex_unlock(&iter->mutex);
4690
	goto out;
4691 4692
}

4693 4694 4695 4696
static ssize_t
tracing_entries_read(struct file *filp, char __user *ubuf,
		     size_t cnt, loff_t *ppos)
{
4697 4698 4699
	struct inode *inode = file_inode(filp);
	struct trace_array *tr = inode->i_private;
	int cpu = tracing_get_cpu(inode);
4700 4701 4702
	char buf[64];
	int r = 0;
	ssize_t ret;
4703

4704
	mutex_lock(&trace_types_lock);
4705

4706
	if (cpu == RING_BUFFER_ALL_CPUS) {
4707 4708 4709 4710 4711 4712 4713 4714 4715
		int cpu, buf_size_same;
		unsigned long size;

		size = 0;
		buf_size_same = 1;
		/* check if all cpu sizes are same */
		for_each_tracing_cpu(cpu) {
			/* fill in the size from first enabled cpu */
			if (size == 0)
4716 4717
				size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
			if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732
				buf_size_same = 0;
				break;
			}
		}

		if (buf_size_same) {
			if (!ring_buffer_expanded)
				r = sprintf(buf, "%lu (expanded: %lu)\n",
					    size >> 10,
					    trace_buf_size >> 10);
			else
				r = sprintf(buf, "%lu\n", size >> 10);
		} else
			r = sprintf(buf, "X\n");
	} else
4733
		r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4734

4735 4736
	mutex_unlock(&trace_types_lock);

4737 4738
	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	return ret;
4739 4740 4741 4742 4743 4744
}

static ssize_t
tracing_entries_write(struct file *filp, const char __user *ubuf,
		      size_t cnt, loff_t *ppos)
{
4745 4746
	struct inode *inode = file_inode(filp);
	struct trace_array *tr = inode->i_private;
4747
	unsigned long val;
4748
	int ret;
4749

4750 4751
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
4752
		return ret;
4753 4754 4755 4756 4757

	/* must have at least 1 entry */
	if (!val)
		return -EINVAL;

4758 4759
	/* value is in KB */
	val <<= 10;
4760
	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4761 4762
	if (ret < 0)
		return ret;
4763

4764
	*ppos += cnt;
4765

4766 4767
	return cnt;
}
S
Steven Rostedt 已提交
4768

4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779
static ssize_t
tracing_total_entries_read(struct file *filp, char __user *ubuf,
				size_t cnt, loff_t *ppos)
{
	struct trace_array *tr = filp->private_data;
	char buf[64];
	int r, cpu;
	unsigned long size = 0, expanded_size = 0;

	mutex_lock(&trace_types_lock);
	for_each_tracing_cpu(cpu) {
4780
		size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792
		if (!ring_buffer_expanded)
			expanded_size += trace_buf_size >> 10;
	}
	if (ring_buffer_expanded)
		r = sprintf(buf, "%lu\n", size);
	else
		r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
	mutex_unlock(&trace_types_lock);

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

4793 4794 4795 4796 4797 4798 4799 4800 4801 4802
static ssize_t
tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
			  size_t cnt, loff_t *ppos)
{
	/*
	 * There is no need to read what the user has written, this function
	 * is just to make sure that there is no error when "echo" is used
	 */

	*ppos += cnt;
4803 4804 4805 4806

	return cnt;
}

4807 4808 4809
static int
tracing_free_buffer_release(struct inode *inode, struct file *filp)
{
4810 4811
	struct trace_array *tr = inode->i_private;

4812 4813
	/* disable tracing ? */
	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4814
		tracer_tracing_off(tr);
4815
	/* resize the ring buffer to 0 */
4816
	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4817

4818 4819
	trace_array_put(tr);

4820 4821 4822
	return 0;
}

4823 4824 4825 4826
static ssize_t
tracing_mark_write(struct file *filp, const char __user *ubuf,
					size_t cnt, loff_t *fpos)
{
4827
	unsigned long addr = (unsigned long)ubuf;
4828
	struct trace_array *tr = filp->private_data;
4829 4830 4831 4832 4833
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	struct print_entry *entry;
	unsigned long irq_flags;
	struct page *pages[2];
4834
	void *map_page[2];
4835 4836 4837 4838 4839 4840
	int nr_pages = 1;
	ssize_t written;
	int offset;
	int size;
	int len;
	int ret;
4841
	int i;
4842

S
Steven Rostedt 已提交
4843
	if (tracing_disabled)
4844 4845
		return -EINVAL;

4846 4847 4848
	if (!(trace_flags & TRACE_ITER_MARKERS))
		return -EINVAL;

4849 4850 4851
	if (cnt > TRACE_BUF_SIZE)
		cnt = TRACE_BUF_SIZE;

4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866
	/*
	 * Userspace is injecting traces into the kernel trace buffer.
	 * We want to be as non intrusive as possible.
	 * To do so, we do not want to allocate any special buffers
	 * or take any locks, but instead write the userspace data
	 * straight into the ring buffer.
	 *
	 * First we need to pin the userspace buffer into memory,
	 * which, most likely it is, because it just referenced it.
	 * But there's no guarantee that it is. By using get_user_pages_fast()
	 * and kmap_atomic/kunmap_atomic() we can get access to the
	 * pages directly. We then write the data directly into the
	 * ring buffer.
	 */
	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4867

4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880
	/* check if we cross pages */
	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
		nr_pages = 2;

	offset = addr & (PAGE_SIZE - 1);
	addr &= PAGE_MASK;

	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
	if (ret < nr_pages) {
		while (--ret >= 0)
			put_page(pages[ret]);
		written = -EFAULT;
		goto out;
4881
	}
4882

4883 4884
	for (i = 0; i < nr_pages; i++)
		map_page[i] = kmap_atomic(pages[i]);
4885 4886 4887

	local_save_flags(irq_flags);
	size = sizeof(*entry) + cnt + 2; /* possible \n added */
4888
	buffer = tr->trace_buffer.buffer;
4889 4890 4891 4892 4893 4894
	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
					  irq_flags, preempt_count());
	if (!event) {
		/* Ring buffer disabled, return as if not open for write */
		written = -EBADF;
		goto out_unlock;
4895
	}
4896 4897 4898 4899 4900 4901

	entry = ring_buffer_event_data(event);
	entry->ip = _THIS_IP_;

	if (nr_pages == 2) {
		len = PAGE_SIZE - offset;
4902 4903
		memcpy(&entry->buf, map_page[0] + offset, len);
		memcpy(&entry->buf[len], map_page[1], cnt - len);
C
Carsten Emde 已提交
4904
	} else
4905
		memcpy(&entry->buf, map_page[0] + offset, cnt);
4906

4907 4908 4909 4910 4911 4912
	if (entry->buf[cnt - 1] != '\n') {
		entry->buf[cnt] = '\n';
		entry->buf[cnt + 1] = '\0';
	} else
		entry->buf[cnt] = '\0';

4913
	__buffer_unlock_commit(buffer, event);
4914

4915
	written = cnt;
4916

4917
	*fpos += written;
4918

4919
 out_unlock:
4920 4921 4922 4923
	for (i = 0; i < nr_pages; i++){
		kunmap_atomic(map_page[i]);
		put_page(pages[i]);
	}
4924
 out:
4925
	return written;
4926 4927
}

L
Li Zefan 已提交
4928
static int tracing_clock_show(struct seq_file *m, void *v)
4929
{
4930
	struct trace_array *tr = m->private;
4931 4932 4933
	int i;

	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
L
Li Zefan 已提交
4934
		seq_printf(m,
4935
			"%s%s%s%s", i ? " " : "",
4936 4937
			i == tr->clock_id ? "[" : "", trace_clocks[i].name,
			i == tr->clock_id ? "]" : "");
L
Li Zefan 已提交
4938
	seq_putc(m, '\n');
4939

L
Li Zefan 已提交
4940
	return 0;
4941 4942
}

4943
static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955
{
	int i;

	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
		if (strcmp(trace_clocks[i].name, clockstr) == 0)
			break;
	}
	if (i == ARRAY_SIZE(trace_clocks))
		return -EINVAL;

	mutex_lock(&trace_types_lock);

4956 4957
	tr->clock_id = i;

4958
	ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4959

4960 4961 4962 4963
	/*
	 * New clock may not be consistent with the previous clock.
	 * Reset the buffer so that it doesn't have incomparable timestamps.
	 */
4964
	tracing_reset_online_cpus(&tr->trace_buffer);
4965 4966 4967 4968

#ifdef CONFIG_TRACER_MAX_TRACE
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
		ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4969
	tracing_reset_online_cpus(&tr->max_buffer);
4970
#endif
4971

4972 4973
	mutex_unlock(&trace_types_lock);

4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999
	return 0;
}

static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
				   size_t cnt, loff_t *fpos)
{
	struct seq_file *m = filp->private_data;
	struct trace_array *tr = m->private;
	char buf[64];
	const char *clockstr;
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	clockstr = strstrip(buf);

	ret = tracing_set_clock(tr, clockstr);
	if (ret)
		return ret;

5000 5001 5002 5003 5004
	*fpos += cnt;

	return cnt;
}

L
Li Zefan 已提交
5005 5006
static int tracing_clock_open(struct inode *inode, struct file *file)
{
5007 5008 5009
	struct trace_array *tr = inode->i_private;
	int ret;

L
Li Zefan 已提交
5010 5011
	if (tracing_disabled)
		return -ENODEV;
5012

5013 5014 5015 5016 5017 5018 5019 5020
	if (trace_array_get(tr))
		return -ENODEV;

	ret = single_open(file, tracing_clock_show, inode->i_private);
	if (ret < 0)
		trace_array_put(tr);

	return ret;
L
Li Zefan 已提交
5021 5022
}

5023 5024 5025 5026 5027 5028
struct ftrace_buffer_info {
	struct trace_iterator	iter;
	void			*spare;
	unsigned int		read;
};

5029 5030 5031
#ifdef CONFIG_TRACER_SNAPSHOT
static int tracing_snapshot_open(struct inode *inode, struct file *file)
{
5032
	struct trace_array *tr = inode->i_private;
5033
	struct trace_iterator *iter;
5034
	struct seq_file *m;
5035 5036
	int ret = 0;

5037 5038 5039
	if (trace_array_get(tr) < 0)
		return -ENODEV;

5040
	if (file->f_mode & FMODE_READ) {
5041
		iter = __tracing_open(inode, file, true);
5042 5043
		if (IS_ERR(iter))
			ret = PTR_ERR(iter);
5044 5045
	} else {
		/* Writes still need the seq_file to hold the private data */
5046
		ret = -ENOMEM;
5047 5048
		m = kzalloc(sizeof(*m), GFP_KERNEL);
		if (!m)
5049
			goto out;
5050 5051 5052
		iter = kzalloc(sizeof(*iter), GFP_KERNEL);
		if (!iter) {
			kfree(m);
5053
			goto out;
5054
		}
5055 5056
		ret = 0;

5057
		iter->tr = tr;
5058 5059
		iter->trace_buffer = &tr->max_buffer;
		iter->cpu_file = tracing_get_cpu(inode);
5060 5061
		m->private = iter;
		file->private_data = m;
5062
	}
5063
out:
5064 5065 5066
	if (ret < 0)
		trace_array_put(tr);

5067 5068 5069 5070 5071 5072 5073
	return ret;
}

static ssize_t
tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
		       loff_t *ppos)
{
5074 5075 5076
	struct seq_file *m = filp->private_data;
	struct trace_iterator *iter = m->private;
	struct trace_array *tr = iter->tr;
5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089
	unsigned long val;
	int ret;

	ret = tracing_update_buffers();
	if (ret < 0)
		return ret;

	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
		return ret;

	mutex_lock(&trace_types_lock);

5090
	if (tr->current_trace->use_max_tr) {
5091 5092 5093 5094 5095 5096
		ret = -EBUSY;
		goto out;
	}

	switch (val) {
	case 0:
5097 5098 5099
		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
			ret = -EINVAL;
			break;
5100
		}
5101 5102
		if (tr->allocated_snapshot)
			free_snapshot(tr);
5103 5104
		break;
	case 1:
5105 5106 5107 5108 5109 5110 5111
/* Only allow per-cpu swap if the ring buffer supports it */
#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
			ret = -EINVAL;
			break;
		}
#endif
5112
		if (!tr->allocated_snapshot) {
5113
			ret = alloc_snapshot(tr);
5114 5115 5116 5117 5118
			if (ret < 0)
				break;
		}
		local_irq_disable();
		/* Now, we're going to swap */
5119
		if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5120
			update_max_tr(tr, current, smp_processor_id());
5121
		else
5122
			update_max_tr_single(tr, current, iter->cpu_file);
5123 5124 5125
		local_irq_enable();
		break;
	default:
5126
		if (tr->allocated_snapshot) {
5127 5128 5129 5130 5131
			if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
				tracing_reset_online_cpus(&tr->max_buffer);
			else
				tracing_reset(&tr->max_buffer, iter->cpu_file);
		}
5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142
		break;
	}

	if (ret >= 0) {
		*ppos += cnt;
		ret = cnt;
	}
out:
	mutex_unlock(&trace_types_lock);
	return ret;
}
5143 5144 5145 5146

static int tracing_snapshot_release(struct inode *inode, struct file *file)
{
	struct seq_file *m = file->private_data;
5147 5148 5149
	int ret;

	ret = tracing_release(inode, file);
5150 5151

	if (file->f_mode & FMODE_READ)
5152
		return ret;
5153 5154 5155 5156 5157 5158 5159 5160 5161

	/* If write only, the seq_file is just a stub */
	if (m)
		kfree(m->private);
	kfree(m);

	return 0;
}

5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190
static int tracing_buffers_open(struct inode *inode, struct file *filp);
static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
				    size_t count, loff_t *ppos);
static int tracing_buffers_release(struct inode *inode, struct file *file);
static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
		   struct pipe_inode_info *pipe, size_t len, unsigned int flags);

static int snapshot_raw_open(struct inode *inode, struct file *filp)
{
	struct ftrace_buffer_info *info;
	int ret;

	ret = tracing_buffers_open(inode, filp);
	if (ret < 0)
		return ret;

	info = filp->private_data;

	if (info->iter.trace->use_max_tr) {
		tracing_buffers_release(inode, filp);
		return -EBUSY;
	}

	info->iter.snapshot = true;
	info->iter.trace_buffer = &info->iter.tr->max_buffer;

	return ret;
}

5191 5192 5193
#endif /* CONFIG_TRACER_SNAPSHOT */


5194 5195 5196 5197 5198 5199 5200
static const struct file_operations tracing_thresh_fops = {
	.open		= tracing_open_generic,
	.read		= tracing_thresh_read,
	.write		= tracing_thresh_write,
	.llseek		= generic_file_llseek,
};

5201
static const struct file_operations tracing_max_lat_fops = {
I
Ingo Molnar 已提交
5202 5203 5204
	.open		= tracing_open_generic,
	.read		= tracing_max_lat_read,
	.write		= tracing_max_lat_write,
5205
	.llseek		= generic_file_llseek,
5206 5207
};

5208
static const struct file_operations set_tracer_fops = {
I
Ingo Molnar 已提交
5209 5210 5211
	.open		= tracing_open_generic,
	.read		= tracing_set_trace_read,
	.write		= tracing_set_trace_write,
5212
	.llseek		= generic_file_llseek,
5213 5214
};

5215
static const struct file_operations tracing_pipe_fops = {
I
Ingo Molnar 已提交
5216
	.open		= tracing_open_pipe,
5217
	.poll		= tracing_poll_pipe,
I
Ingo Molnar 已提交
5218
	.read		= tracing_read_pipe,
5219
	.splice_read	= tracing_splice_read_pipe,
I
Ingo Molnar 已提交
5220
	.release	= tracing_release_pipe,
5221
	.llseek		= no_llseek,
5222 5223
};

5224
static const struct file_operations tracing_entries_fops = {
5225
	.open		= tracing_open_generic_tr,
5226 5227
	.read		= tracing_entries_read,
	.write		= tracing_entries_write,
5228
	.llseek		= generic_file_llseek,
5229
	.release	= tracing_release_generic_tr,
5230 5231
};

5232
static const struct file_operations tracing_total_entries_fops = {
5233
	.open		= tracing_open_generic_tr,
5234 5235
	.read		= tracing_total_entries_read,
	.llseek		= generic_file_llseek,
5236
	.release	= tracing_release_generic_tr,
5237 5238
};

5239
static const struct file_operations tracing_free_buffer_fops = {
5240
	.open		= tracing_open_generic_tr,
5241 5242 5243 5244
	.write		= tracing_free_buffer_write,
	.release	= tracing_free_buffer_release,
};

5245
static const struct file_operations tracing_mark_fops = {
5246
	.open		= tracing_open_generic_tr,
5247
	.write		= tracing_mark_write,
5248
	.llseek		= generic_file_llseek,
5249
	.release	= tracing_release_generic_tr,
5250 5251
};

5252
static const struct file_operations trace_clock_fops = {
L
Li Zefan 已提交
5253 5254 5255
	.open		= tracing_clock_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
5256
	.release	= tracing_single_release_tr,
5257 5258 5259
	.write		= tracing_clock_write,
};

5260 5261 5262 5263 5264
#ifdef CONFIG_TRACER_SNAPSHOT
static const struct file_operations snapshot_fops = {
	.open		= tracing_snapshot_open,
	.read		= seq_read,
	.write		= tracing_snapshot_write,
5265
	.llseek		= tracing_lseek,
5266
	.release	= tracing_snapshot_release,
5267 5268
};

5269 5270 5271 5272 5273 5274
static const struct file_operations snapshot_raw_fops = {
	.open		= snapshot_raw_open,
	.read		= tracing_buffers_read,
	.release	= tracing_buffers_release,
	.splice_read	= tracing_buffers_splice_read,
	.llseek		= no_llseek,
5275 5276
};

5277 5278
#endif /* CONFIG_TRACER_SNAPSHOT */

5279 5280
static int tracing_buffers_open(struct inode *inode, struct file *filp)
{
5281
	struct trace_array *tr = inode->i_private;
5282
	struct ftrace_buffer_info *info;
5283
	int ret;
5284 5285 5286 5287

	if (tracing_disabled)
		return -ENODEV;

5288 5289 5290
	if (trace_array_get(tr) < 0)
		return -ENODEV;

5291
	info = kzalloc(sizeof(*info), GFP_KERNEL);
5292 5293
	if (!info) {
		trace_array_put(tr);
5294
		return -ENOMEM;
5295
	}
5296

5297 5298
	mutex_lock(&trace_types_lock);

5299
	info->iter.tr		= tr;
5300
	info->iter.cpu_file	= tracing_get_cpu(inode);
5301
	info->iter.trace	= tr->current_trace;
5302
	info->iter.trace_buffer = &tr->trace_buffer;
5303
	info->spare		= NULL;
5304
	/* Force reading ring buffer for first read */
5305
	info->read		= (unsigned int)-1;
5306 5307 5308

	filp->private_data = info;

5309 5310
	mutex_unlock(&trace_types_lock);

5311 5312 5313 5314 5315
	ret = nonseekable_open(inode, filp);
	if (ret < 0)
		trace_array_put(tr);

	return ret;
5316 5317
}

5318 5319 5320 5321 5322 5323 5324 5325 5326
static unsigned int
tracing_buffers_poll(struct file *filp, poll_table *poll_table)
{
	struct ftrace_buffer_info *info = filp->private_data;
	struct trace_iterator *iter = &info->iter;

	return trace_poll(iter, filp, poll_table);
}

5327 5328 5329 5330 5331
static ssize_t
tracing_buffers_read(struct file *filp, char __user *ubuf,
		     size_t count, loff_t *ppos)
{
	struct ftrace_buffer_info *info = filp->private_data;
5332
	struct trace_iterator *iter = &info->iter;
5333
	ssize_t ret;
5334
	ssize_t size;
5335

5336 5337 5338
	if (!count)
		return 0;

5339 5340 5341 5342 5343 5344 5345 5346 5347
	mutex_lock(&trace_types_lock);

#ifdef CONFIG_TRACER_MAX_TRACE
	if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
		size = -EBUSY;
		goto out_unlock;
	}
#endif

5348
	if (!info->spare)
5349 5350
		info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
							  iter->cpu_file);
5351
	size = -ENOMEM;
5352
	if (!info->spare)
5353
		goto out_unlock;
5354

5355 5356 5357 5358
	/* Do we have previous read data to read? */
	if (info->read < PAGE_SIZE)
		goto read;

5359
 again:
5360
	trace_access_lock(iter->cpu_file);
5361
	ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5362 5363
				    &info->spare,
				    count,
5364 5365
				    iter->cpu_file, 0);
	trace_access_unlock(iter->cpu_file);
5366

5367 5368
	if (ret < 0) {
		if (trace_empty(iter)) {
5369 5370 5371 5372 5373
			if ((filp->f_flags & O_NONBLOCK)) {
				size = -EAGAIN;
				goto out_unlock;
			}
			mutex_unlock(&trace_types_lock);
5374
			ret = wait_on_pipe(iter);
5375
			mutex_lock(&trace_types_lock);
5376 5377 5378 5379
			if (ret) {
				size = ret;
				goto out_unlock;
			}
5380 5381 5382 5383
			if (signal_pending(current)) {
				size = -EINTR;
				goto out_unlock;
			}
5384 5385
			goto again;
		}
5386 5387
		size = 0;
		goto out_unlock;
5388
	}
5389 5390

	info->read = 0;
5391
 read:
5392 5393 5394 5395 5396
	size = PAGE_SIZE - info->read;
	if (size > count)
		size = count;

	ret = copy_to_user(ubuf, info->spare + info->read, size);
5397 5398 5399 5400
	if (ret == size) {
		size = -EFAULT;
		goto out_unlock;
	}
5401 5402
	size -= ret;

5403 5404 5405
	*ppos += size;
	info->read += size;

5406 5407 5408
 out_unlock:
	mutex_unlock(&trace_types_lock);

5409 5410 5411 5412 5413 5414
	return size;
}

static int tracing_buffers_release(struct inode *inode, struct file *file)
{
	struct ftrace_buffer_info *info = file->private_data;
5415
	struct trace_iterator *iter = &info->iter;
5416

5417 5418
	mutex_lock(&trace_types_lock);

5419
	__trace_array_put(iter->tr);
5420

5421
	if (info->spare)
5422
		ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5423 5424
	kfree(info);

5425 5426
	mutex_unlock(&trace_types_lock);

5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457
	return 0;
}

struct buffer_ref {
	struct ring_buffer	*buffer;
	void			*page;
	int			ref;
};

static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	struct buffer_ref *ref = (struct buffer_ref *)buf->private;

	if (--ref->ref)
		return;

	ring_buffer_free_read_page(ref->buffer, ref->page);
	kfree(ref);
	buf->private = 0;
}

static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
				struct pipe_buffer *buf)
{
	struct buffer_ref *ref = (struct buffer_ref *)buf->private;

	ref->ref++;
}

/* Pipe buffer operations for a buffer. */
5458
static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5459 5460 5461
	.can_merge		= 0,
	.confirm		= generic_pipe_buf_confirm,
	.release		= buffer_pipe_buf_release,
5462
	.steal			= generic_pipe_buf_steal,
5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488
	.get			= buffer_pipe_buf_get,
};

/*
 * Callback from splice_to_pipe(), if we need to release some pages
 * at the end of the spd in case we error'ed out in filling the pipe.
 */
static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
	struct buffer_ref *ref =
		(struct buffer_ref *)spd->partial[i].private;

	if (--ref->ref)
		return;

	ring_buffer_free_read_page(ref->buffer, ref->page);
	kfree(ref);
	spd->partial[i].private = 0;
}

static ssize_t
tracing_buffers_splice_read(struct file *file, loff_t *ppos,
			    struct pipe_inode_info *pipe, size_t len,
			    unsigned int flags)
{
	struct ftrace_buffer_info *info = file->private_data;
5489
	struct trace_iterator *iter = &info->iter;
5490 5491
	struct partial_page partial_def[PIPE_DEF_BUFFERS];
	struct page *pages_def[PIPE_DEF_BUFFERS];
5492
	struct splice_pipe_desc spd = {
5493 5494
		.pages		= pages_def,
		.partial	= partial_def,
5495
		.nr_pages_max	= PIPE_DEF_BUFFERS,
5496 5497 5498 5499 5500
		.flags		= flags,
		.ops		= &buffer_pipe_buf_ops,
		.spd_release	= buffer_spd_release,
	};
	struct buffer_ref *ref;
5501
	int entries, size, i;
5502
	ssize_t ret;
5503

5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516
	mutex_lock(&trace_types_lock);

#ifdef CONFIG_TRACER_MAX_TRACE
	if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
		ret = -EBUSY;
		goto out;
	}
#endif

	if (splice_grow_spd(pipe, &spd)) {
		ret = -ENOMEM;
		goto out;
	}
5517

5518
	if (*ppos & (PAGE_SIZE - 1)) {
5519 5520
		ret = -EINVAL;
		goto out;
5521 5522 5523
	}

	if (len & (PAGE_SIZE - 1)) {
5524 5525 5526 5527
		if (len < PAGE_SIZE) {
			ret = -EINVAL;
			goto out;
		}
5528 5529 5530
		len &= PAGE_MASK;
	}

5531 5532
 again:
	trace_access_lock(iter->cpu_file);
5533
	entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5534

5535
	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5536 5537 5538 5539 5540 5541 5542
		struct page *page;
		int r;

		ref = kzalloc(sizeof(*ref), GFP_KERNEL);
		if (!ref)
			break;

5543
		ref->ref = 1;
5544
		ref->buffer = iter->trace_buffer->buffer;
5545
		ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5546 5547 5548 5549 5550 5551
		if (!ref->page) {
			kfree(ref);
			break;
		}

		r = ring_buffer_read_page(ref->buffer, &ref->page,
5552
					  len, iter->cpu_file, 1);
5553
		if (r < 0) {
5554
			ring_buffer_free_read_page(ref->buffer, ref->page);
5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573
			kfree(ref);
			break;
		}

		/*
		 * zero out any left over data, this is going to
		 * user land.
		 */
		size = ring_buffer_page_len(ref->page);
		if (size < PAGE_SIZE)
			memset(ref->page + size, 0, PAGE_SIZE - size);

		page = virt_to_page(ref->page);

		spd.pages[i] = page;
		spd.partial[i].len = PAGE_SIZE;
		spd.partial[i].offset = 0;
		spd.partial[i].private = (unsigned long)ref;
		spd.nr_pages++;
5574
		*ppos += PAGE_SIZE;
5575

5576
		entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5577 5578
	}

5579
	trace_access_unlock(iter->cpu_file);
5580 5581 5582 5583
	spd.nr_pages = i;

	/* did we read anything? */
	if (!spd.nr_pages) {
5584
		if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5585
			ret = -EAGAIN;
5586 5587
			goto out;
		}
5588
		mutex_unlock(&trace_types_lock);
5589
		ret = wait_on_pipe(iter);
5590
		mutex_lock(&trace_types_lock);
5591 5592
		if (ret)
			goto out;
5593 5594 5595 5596 5597
		if (signal_pending(current)) {
			ret = -EINTR;
			goto out;
		}
		goto again;
5598 5599 5600
	}

	ret = splice_to_pipe(pipe, &spd);
5601
	splice_shrink_spd(&spd);
5602
out:
5603 5604
	mutex_unlock(&trace_types_lock);

5605 5606 5607 5608 5609 5610
	return ret;
}

static const struct file_operations tracing_buffers_fops = {
	.open		= tracing_buffers_open,
	.read		= tracing_buffers_read,
5611
	.poll		= tracing_buffers_poll,
5612 5613 5614 5615 5616
	.release	= tracing_buffers_release,
	.splice_read	= tracing_buffers_splice_read,
	.llseek		= no_llseek,
};

5617 5618 5619 5620
static ssize_t
tracing_stats_read(struct file *filp, char __user *ubuf,
		   size_t count, loff_t *ppos)
{
5621 5622
	struct inode *inode = file_inode(filp);
	struct trace_array *tr = inode->i_private;
5623
	struct trace_buffer *trace_buf = &tr->trace_buffer;
5624
	int cpu = tracing_get_cpu(inode);
5625 5626
	struct trace_seq *s;
	unsigned long cnt;
5627 5628
	unsigned long long t;
	unsigned long usec_rem;
5629

5630
	s = kmalloc(sizeof(*s), GFP_KERNEL);
5631
	if (!s)
5632
		return -ENOMEM;
5633 5634 5635

	trace_seq_init(s);

5636
	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5637 5638
	trace_seq_printf(s, "entries: %ld\n", cnt);

5639
	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5640 5641
	trace_seq_printf(s, "overrun: %ld\n", cnt);

5642
	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5643 5644
	trace_seq_printf(s, "commit overrun: %ld\n", cnt);

5645
	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5646 5647
	trace_seq_printf(s, "bytes: %ld\n", cnt);

5648
	if (trace_clocks[tr->clock_id].in_ns) {
5649
		/* local or global for trace_clock */
5650
		t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5651 5652 5653 5654
		usec_rem = do_div(t, USEC_PER_SEC);
		trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
								t, usec_rem);

5655
		t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5656 5657 5658 5659 5660
		usec_rem = do_div(t, USEC_PER_SEC);
		trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
	} else {
		/* counter or tsc mode for trace_clock */
		trace_seq_printf(s, "oldest event ts: %llu\n",
5661
				ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5662

5663
		trace_seq_printf(s, "now ts: %llu\n",
5664
				ring_buffer_time_stamp(trace_buf->buffer, cpu));
5665
	}
5666

5667
	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5668 5669
	trace_seq_printf(s, "dropped events: %ld\n", cnt);

5670
	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5671 5672
	trace_seq_printf(s, "read events: %ld\n", cnt);

5673 5674 5675 5676 5677 5678 5679 5680
	count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);

	kfree(s);

	return count;
}

static const struct file_operations tracing_stats_fops = {
5681
	.open		= tracing_open_generic_tr,
5682
	.read		= tracing_stats_read,
5683
	.llseek		= generic_file_llseek,
5684
	.release	= tracing_release_generic_tr,
5685 5686
};

5687 5688
#ifdef CONFIG_DYNAMIC_FTRACE

S
Steven Rostedt 已提交
5689 5690 5691 5692 5693
int __weak ftrace_arch_read_dyn_info(char *buf, int size)
{
	return 0;
}

5694
static ssize_t
S
Steven Rostedt 已提交
5695
tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5696 5697
		  size_t cnt, loff_t *ppos)
{
S
Steven Rostedt 已提交
5698 5699
	static char ftrace_dyn_info_buffer[1024];
	static DEFINE_MUTEX(dyn_info_mutex);
5700
	unsigned long *p = filp->private_data;
S
Steven Rostedt 已提交
5701
	char *buf = ftrace_dyn_info_buffer;
S
Steven Rostedt 已提交
5702
	int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5703 5704
	int r;

S
Steven Rostedt 已提交
5705 5706
	mutex_lock(&dyn_info_mutex);
	r = sprintf(buf, "%ld ", *p);
I
Ingo Molnar 已提交
5707

S
Steven Rostedt 已提交
5708
	r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
S
Steven Rostedt 已提交
5709 5710 5711 5712 5713 5714 5715
	buf[r++] = '\n';

	r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);

	mutex_unlock(&dyn_info_mutex);

	return r;
5716 5717
}

5718
static const struct file_operations tracing_dyn_info_fops = {
I
Ingo Molnar 已提交
5719
	.open		= tracing_open_generic,
S
Steven Rostedt 已提交
5720
	.read		= tracing_read_dyn_info,
5721
	.llseek		= generic_file_llseek,
5722
};
5723
#endif /* CONFIG_DYNAMIC_FTRACE */
5724

5725 5726 5727 5728 5729 5730
#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
static void
ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
{
	tracing_snapshot();
}
5731

5732 5733
static void
ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5734
{
5735 5736 5737 5738
	unsigned long *count = (long *)data;

	if (!*count)
		return;
5739

5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823
	if (*count != -1)
		(*count)--;

	tracing_snapshot();
}

static int
ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
		      struct ftrace_probe_ops *ops, void *data)
{
	long count = (long)data;

	seq_printf(m, "%ps:", (void *)ip);

	seq_printf(m, "snapshot");

	if (count == -1)
		seq_printf(m, ":unlimited\n");
	else
		seq_printf(m, ":count=%ld\n", count);

	return 0;
}

static struct ftrace_probe_ops snapshot_probe_ops = {
	.func			= ftrace_snapshot,
	.print			= ftrace_snapshot_print,
};

static struct ftrace_probe_ops snapshot_count_probe_ops = {
	.func			= ftrace_count_snapshot,
	.print			= ftrace_snapshot_print,
};

static int
ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
			       char *glob, char *cmd, char *param, int enable)
{
	struct ftrace_probe_ops *ops;
	void *count = (void *)-1;
	char *number;
	int ret;

	/* hash funcs only work with set_ftrace_filter */
	if (!enable)
		return -EINVAL;

	ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;

	if (glob[0] == '!') {
		unregister_ftrace_function_probe_func(glob+1, ops);
		return 0;
	}

	if (!param)
		goto out_reg;

	number = strsep(&param, ":");

	if (!strlen(number))
		goto out_reg;

	/*
	 * We use the callback data field (which is a pointer)
	 * as our counter.
	 */
	ret = kstrtoul(number, 0, (unsigned long *)&count);
	if (ret)
		return ret;

 out_reg:
	ret = register_ftrace_function_probe(glob, ops, count);

	if (ret >= 0)
		alloc_snapshot(&global_trace);

	return ret < 0 ? ret : 0;
}

static struct ftrace_func_command ftrace_snapshot_cmd = {
	.name			= "snapshot",
	.func			= ftrace_trace_snapshot_callback,
};

5824
static __init int register_snapshot_cmd(void)
5825 5826 5827 5828
{
	return register_ftrace_command(&ftrace_snapshot_cmd);
}
#else
5829
static inline __init int register_snapshot_cmd(void) { return 0; }
5830
#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5831

5832
struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5833
{
5834 5835
	if (tr->dir)
		return tr->dir;
5836

5837 5838 5839
	if (!debugfs_initialized())
		return NULL;

5840 5841
	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
		tr->dir = debugfs_create_dir("tracing", NULL);
5842

5843 5844
	if (!tr->dir)
		pr_warn_once("Could not create debugfs directory 'tracing'\n");
5845

5846
	return tr->dir;
5847 5848
}

5849 5850 5851 5852
struct dentry *tracing_init_dentry(void)
{
	return tracing_init_dentry_tr(&global_trace);
}
5853

5854
static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5855 5856 5857
{
	struct dentry *d_tracer;

5858 5859
	if (tr->percpu_dir)
		return tr->percpu_dir;
5860

5861
	d_tracer = tracing_init_dentry_tr(tr);
5862 5863 5864
	if (!d_tracer)
		return NULL;

5865
	tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5866

5867 5868
	WARN_ONCE(!tr->percpu_dir,
		  "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5869

5870
	return tr->percpu_dir;
5871 5872
}

5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883
static struct dentry *
trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
		      void *data, long cpu, const struct file_operations *fops)
{
	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);

	if (ret) /* See tracing_get_cpu() */
		ret->d_inode->i_cdev = (void *)(cpu + 1);
	return ret;
}

5884 5885
static void
tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5886
{
5887
	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5888
	struct dentry *d_cpu;
5889
	char cpu_dir[30]; /* 30 characters should be more than enough */
5890

5891 5892 5893
	if (!d_percpu)
		return;

5894
	snprintf(cpu_dir, 30, "cpu%ld", cpu);
5895 5896 5897 5898 5899
	d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
	if (!d_cpu) {
		pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
		return;
	}
5900

5901
	/* per cpu trace_pipe */
5902
	trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5903
				tr, cpu, &tracing_pipe_fops);
5904 5905

	/* per cpu trace */
5906
	trace_create_cpu_file("trace", 0644, d_cpu,
5907
				tr, cpu, &tracing_fops);
5908

5909
	trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5910
				tr, cpu, &tracing_buffers_fops);
5911

5912
	trace_create_cpu_file("stats", 0444, d_cpu,
5913
				tr, cpu, &tracing_stats_fops);
5914

5915
	trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5916
				tr, cpu, &tracing_entries_fops);
5917 5918

#ifdef CONFIG_TRACER_SNAPSHOT
5919
	trace_create_cpu_file("snapshot", 0644, d_cpu,
5920
				tr, cpu, &snapshot_fops);
5921

5922
	trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5923
				tr, cpu, &snapshot_raw_fops);
5924
#endif
5925 5926
}

S
Steven Rostedt 已提交
5927 5928 5929 5930 5931
#ifdef CONFIG_FTRACE_SELFTEST
/* Let selftest have access to static functions in this file */
#include "trace_selftest.c"
#endif

5932 5933 5934
struct trace_option_dentry {
	struct tracer_opt		*opt;
	struct tracer_flags		*flags;
5935
	struct trace_array		*tr;
5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961
	struct dentry			*entry;
};

static ssize_t
trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
			loff_t *ppos)
{
	struct trace_option_dentry *topt = filp->private_data;
	char *buf;

	if (topt->flags->val & topt->opt->bit)
		buf = "1\n";
	else
		buf = "0\n";

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
}

static ssize_t
trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
			 loff_t *ppos)
{
	struct trace_option_dentry *topt = filp->private_data;
	unsigned long val;
	int ret;

5962 5963
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
5964 5965
		return ret;

L
Li Zefan 已提交
5966 5967
	if (val != 0 && val != 1)
		return -EINVAL;
5968

L
Li Zefan 已提交
5969
	if (!!(topt->flags->val & topt->opt->bit) != val) {
5970
		mutex_lock(&trace_types_lock);
5971
		ret = __set_tracer_option(topt->tr, topt->flags,
5972
					  topt->opt, !val);
5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987
		mutex_unlock(&trace_types_lock);
		if (ret)
			return ret;
	}

	*ppos += cnt;

	return cnt;
}


static const struct file_operations trace_options_fops = {
	.open = tracing_open_generic,
	.read = trace_options_read,
	.write = trace_options_write,
5988
	.llseek	= generic_file_llseek,
5989 5990
};

5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009
static ssize_t
trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
			loff_t *ppos)
{
	long index = (long)filp->private_data;
	char *buf;

	if (trace_flags & (1 << index))
		buf = "1\n";
	else
		buf = "0\n";

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
}

static ssize_t
trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
			 loff_t *ppos)
{
6010
	struct trace_array *tr = &global_trace;
6011 6012 6013 6014
	long index = (long)filp->private_data;
	unsigned long val;
	int ret;

6015 6016
	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
6017 6018
		return ret;

6019
	if (val != 0 && val != 1)
6020
		return -EINVAL;
6021 6022

	mutex_lock(&trace_types_lock);
6023
	ret = set_tracer_flag(tr, 1 << index, val);
6024
	mutex_unlock(&trace_types_lock);
6025

6026 6027 6028
	if (ret < 0)
		return ret;

6029 6030 6031 6032 6033 6034 6035 6036 6037
	*ppos += cnt;

	return cnt;
}

static const struct file_operations trace_options_core_fops = {
	.open = tracing_open_generic,
	.read = trace_options_core_read,
	.write = trace_options_core_write,
6038
	.llseek = generic_file_llseek,
6039 6040
};

6041
struct dentry *trace_create_file(const char *name,
A
Al Viro 已提交
6042
				 umode_t mode,
6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056
				 struct dentry *parent,
				 void *data,
				 const struct file_operations *fops)
{
	struct dentry *ret;

	ret = debugfs_create_file(name, mode, parent, data, fops);
	if (!ret)
		pr_warning("Could not create debugfs '%s' entry\n", name);

	return ret;
}


6057
static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6058 6059 6060
{
	struct dentry *d_tracer;

6061 6062
	if (tr->options)
		return tr->options;
6063

6064
	d_tracer = tracing_init_dentry_tr(tr);
6065 6066 6067
	if (!d_tracer)
		return NULL;

6068 6069
	tr->options = debugfs_create_dir("options", d_tracer);
	if (!tr->options) {
6070 6071 6072 6073
		pr_warning("Could not create debugfs directory 'options'\n");
		return NULL;
	}

6074
	return tr->options;
6075 6076
}

6077
static void
6078 6079
create_trace_option_file(struct trace_array *tr,
			 struct trace_option_dentry *topt,
6080 6081 6082 6083 6084
			 struct tracer_flags *flags,
			 struct tracer_opt *opt)
{
	struct dentry *t_options;

6085
	t_options = trace_options_init_dentry(tr);
6086 6087 6088 6089 6090
	if (!t_options)
		return;

	topt->flags = flags;
	topt->opt = opt;
6091
	topt->tr = tr;
6092

6093
	topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6094 6095 6096 6097 6098
				    &trace_options_fops);

}

static struct trace_option_dentry *
6099
create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118
{
	struct trace_option_dentry *topts;
	struct tracer_flags *flags;
	struct tracer_opt *opts;
	int cnt;

	if (!tracer)
		return NULL;

	flags = tracer->flags;

	if (!flags || !flags->opts)
		return NULL;

	opts = flags->opts;

	for (cnt = 0; opts[cnt].name; cnt++)
		;

6119
	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6120 6121 6122 6123
	if (!topts)
		return NULL;

	for (cnt = 0; opts[cnt].name; cnt++)
6124
		create_trace_option_file(tr, &topts[cnt], flags,
6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137
					 &opts[cnt]);

	return topts;
}

static void
destroy_trace_option_files(struct trace_option_dentry *topts)
{
	int cnt;

	if (!topts)
		return;

6138 6139
	for (cnt = 0; topts[cnt].opt; cnt++)
		debugfs_remove(topts[cnt].entry);
6140 6141 6142 6143

	kfree(topts);
}

6144
static struct dentry *
6145 6146
create_trace_option_core_file(struct trace_array *tr,
			      const char *option, long index)
6147 6148 6149
{
	struct dentry *t_options;

6150
	t_options = trace_options_init_dentry(tr);
6151 6152 6153
	if (!t_options)
		return NULL;

6154
	return trace_create_file(option, 0644, t_options, (void *)index,
6155 6156 6157
				    &trace_options_core_fops);
}

6158
static __init void create_trace_options_dir(struct trace_array *tr)
6159 6160 6161 6162
{
	struct dentry *t_options;
	int i;

6163
	t_options = trace_options_init_dentry(tr);
6164 6165 6166
	if (!t_options)
		return;

6167
	for (i = 0; trace_options[i]; i++)
6168
		create_trace_option_core_file(tr, trace_options[i], i);
6169 6170
}

6171 6172 6173 6174
static ssize_t
rb_simple_read(struct file *filp, char __user *ubuf,
	       size_t cnt, loff_t *ppos)
{
6175
	struct trace_array *tr = filp->private_data;
6176 6177 6178
	char buf[64];
	int r;

6179
	r = tracer_tracing_is_on(tr);
6180 6181 6182 6183 6184 6185 6186 6187 6188
	r = sprintf(buf, "%d\n", r);

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

static ssize_t
rb_simple_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
6189
	struct trace_array *tr = filp->private_data;
6190
	struct ring_buffer *buffer = tr->trace_buffer.buffer;
6191 6192 6193 6194 6195 6196 6197 6198
	unsigned long val;
	int ret;

	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	if (ret)
		return ret;

	if (buffer) {
6199 6200
		mutex_lock(&trace_types_lock);
		if (val) {
6201
			tracer_tracing_on(tr);
6202 6203
			if (tr->current_trace->start)
				tr->current_trace->start(tr);
6204
		} else {
6205
			tracer_tracing_off(tr);
6206 6207
			if (tr->current_trace->stop)
				tr->current_trace->stop(tr);
6208 6209
		}
		mutex_unlock(&trace_types_lock);
6210 6211 6212 6213 6214 6215 6216 6217
	}

	(*ppos)++;

	return cnt;
}

static const struct file_operations rb_simple_fops = {
6218
	.open		= tracing_open_generic_tr,
6219 6220
	.read		= rb_simple_read,
	.write		= rb_simple_write,
6221
	.release	= tracing_release_generic_tr,
6222 6223 6224
	.llseek		= default_llseek,
};

6225 6226 6227 6228 6229
struct dentry *trace_instance_dir;

static void
init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);

6230 6231
static int
allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6232 6233
{
	enum ring_buffer_flags rb_flags;
6234 6235 6236

	rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;

6237 6238
	buf->tr = tr;

6239 6240 6241
	buf->buffer = ring_buffer_alloc(size, rb_flags);
	if (!buf->buffer)
		return -ENOMEM;
6242

6243 6244 6245 6246 6247
	buf->data = alloc_percpu(struct trace_array_cpu);
	if (!buf->data) {
		ring_buffer_free(buf->buffer);
		return -ENOMEM;
	}
6248 6249 6250 6251 6252

	/* Allocate the first page for all buffers */
	set_buffer_entries(&tr->trace_buffer,
			   ring_buffer_size(tr->trace_buffer.buffer, 0));

6253 6254
	return 0;
}
6255

6256 6257 6258
static int allocate_trace_buffers(struct trace_array *tr, int size)
{
	int ret;
6259

6260 6261 6262
	ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
	if (ret)
		return ret;
6263

6264 6265 6266 6267
#ifdef CONFIG_TRACER_MAX_TRACE
	ret = allocate_trace_buffer(tr, &tr->max_buffer,
				    allocate_snapshot ? size : 1);
	if (WARN_ON(ret)) {
6268
		ring_buffer_free(tr->trace_buffer.buffer);
6269 6270 6271 6272
		free_percpu(tr->trace_buffer.data);
		return -ENOMEM;
	}
	tr->allocated_snapshot = allocate_snapshot;
6273

6274 6275 6276 6277 6278
	/*
	 * Only the top level trace array gets its snapshot allocated
	 * from the kernel command line.
	 */
	allocate_snapshot = false;
6279
#endif
6280
	return 0;
6281 6282
}

6283 6284 6285 6286 6287 6288 6289 6290 6291 6292
static void free_trace_buffer(struct trace_buffer *buf)
{
	if (buf->buffer) {
		ring_buffer_free(buf->buffer);
		buf->buffer = NULL;
		free_percpu(buf->data);
		buf->data = NULL;
	}
}

6293 6294 6295 6296 6297
static void free_trace_buffers(struct trace_array *tr)
{
	if (!tr)
		return;

6298
	free_trace_buffer(&tr->trace_buffer);
6299 6300

#ifdef CONFIG_TRACER_MAX_TRACE
6301
	free_trace_buffer(&tr->max_buffer);
6302 6303 6304
#endif
}

6305 6306
static int new_instance_create(const char *name)
{
6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326
	struct trace_array *tr;
	int ret;

	mutex_lock(&trace_types_lock);

	ret = -EEXIST;
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (tr->name && strcmp(tr->name, name) == 0)
			goto out_unlock;
	}

	ret = -ENOMEM;
	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
	if (!tr)
		goto out_unlock;

	tr->name = kstrdup(name, GFP_KERNEL);
	if (!tr->name)
		goto out_free_tr;

6327 6328 6329 6330 6331
	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
		goto out_free_tr;

	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);

6332 6333
	raw_spin_lock_init(&tr->start_lock);

6334 6335
	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;

6336 6337 6338 6339 6340
	tr->current_trace = &nop_trace;

	INIT_LIST_HEAD(&tr->systems);
	INIT_LIST_HEAD(&tr->events);

6341
	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6342 6343 6344 6345 6346 6347 6348
		goto out_free_tr;

	tr->dir = debugfs_create_dir(name, trace_instance_dir);
	if (!tr->dir)
		goto out_free_tr;

	ret = event_trace_add_tracer(tr->dir, tr);
6349 6350
	if (ret) {
		debugfs_remove_recursive(tr->dir);
6351
		goto out_free_tr;
6352
	}
6353 6354 6355 6356 6357 6358 6359 6360 6361 6362

	init_tracer_debugfs(tr, tr->dir);

	list_add(&tr->list, &ftrace_trace_arrays);

	mutex_unlock(&trace_types_lock);

	return 0;

 out_free_tr:
6363
	free_trace_buffers(tr);
6364
	free_cpumask_var(tr->tracing_cpumask);
6365 6366 6367 6368 6369 6370 6371 6372 6373 6374
	kfree(tr->name);
	kfree(tr);

 out_unlock:
	mutex_unlock(&trace_types_lock);

	return ret;

}

6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392
static int instance_delete(const char *name)
{
	struct trace_array *tr;
	int found = 0;
	int ret;

	mutex_lock(&trace_types_lock);

	ret = -ENODEV;
	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
		if (tr->name && strcmp(tr->name, name) == 0) {
			found = 1;
			break;
		}
	}
	if (!found)
		goto out_unlock;

6393 6394 6395 6396
	ret = -EBUSY;
	if (tr->ref)
		goto out_unlock;

6397 6398
	list_del(&tr->list);

6399
	tracing_set_nop(tr);
6400
	event_trace_del_tracer(tr);
6401
	ftrace_destroy_function_files(tr);
6402
	debugfs_remove_recursive(tr->dir);
6403
	free_trace_buffers(tr);
6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415

	kfree(tr->name);
	kfree(tr);

	ret = 0;

 out_unlock:
	mutex_unlock(&trace_types_lock);

	return ret;
}

6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442
static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
{
	struct dentry *parent;
	int ret;

	/* Paranoid: Make sure the parent is the "instances" directory */
	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
	if (WARN_ON_ONCE(parent != trace_instance_dir))
		return -ENOENT;

	/*
	 * The inode mutex is locked, but debugfs_create_dir() will also
	 * take the mutex. As the instances directory can not be destroyed
	 * or changed in any other way, it is safe to unlock it, and
	 * let the dentry try. If two users try to make the same dir at
	 * the same time, then the new_instance_create() will determine the
	 * winner.
	 */
	mutex_unlock(&inode->i_mutex);

	ret = new_instance_create(dentry->d_iname);

	mutex_lock(&inode->i_mutex);

	return ret;
}

6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473
static int instance_rmdir(struct inode *inode, struct dentry *dentry)
{
	struct dentry *parent;
	int ret;

	/* Paranoid: Make sure the parent is the "instances" directory */
	parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
	if (WARN_ON_ONCE(parent != trace_instance_dir))
		return -ENOENT;

	/* The caller did a dget() on dentry */
	mutex_unlock(&dentry->d_inode->i_mutex);

	/*
	 * The inode mutex is locked, but debugfs_create_dir() will also
	 * take the mutex. As the instances directory can not be destroyed
	 * or changed in any other way, it is safe to unlock it, and
	 * let the dentry try. If two users try to make the same dir at
	 * the same time, then the instance_delete() will determine the
	 * winner.
	 */
	mutex_unlock(&inode->i_mutex);

	ret = instance_delete(dentry->d_iname);

	mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
	mutex_lock(&dentry->d_inode->i_mutex);

	return ret;
}

6474 6475 6476
static const struct inode_operations instance_dir_inode_operations = {
	.lookup		= simple_lookup,
	.mkdir		= instance_mkdir,
6477
	.rmdir		= instance_rmdir,
6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489
};

static __init void create_trace_instances(struct dentry *d_tracer)
{
	trace_instance_dir = debugfs_create_dir("instances", d_tracer);
	if (WARN_ON(!trace_instance_dir))
		return;

	/* Hijack the dir inode operations, to allow mkdir */
	trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
}

6490 6491 6492
static void
init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
{
6493
	int cpu;
6494

6495 6496 6497 6498 6499 6500
	trace_create_file("available_tracers", 0444, d_tracer,
			tr, &show_traces_fops);

	trace_create_file("current_tracer", 0644, d_tracer,
			tr, &set_tracer_fops);

6501 6502 6503
	trace_create_file("tracing_cpumask", 0644, d_tracer,
			  tr, &tracing_cpumask_fops);

6504 6505 6506 6507
	trace_create_file("trace_options", 0644, d_tracer,
			  tr, &tracing_iter_fops);

	trace_create_file("trace", 0644, d_tracer,
6508
			  tr, &tracing_fops);
6509 6510

	trace_create_file("trace_pipe", 0444, d_tracer,
6511
			  tr, &tracing_pipe_fops);
6512 6513

	trace_create_file("buffer_size_kb", 0644, d_tracer,
6514
			  tr, &tracing_entries_fops);
6515 6516 6517 6518

	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
			  tr, &tracing_total_entries_fops);

6519
	trace_create_file("free_buffer", 0200, d_tracer,
6520 6521 6522 6523 6524 6525 6526 6527 6528
			  tr, &tracing_free_buffer_fops);

	trace_create_file("trace_marker", 0220, d_tracer,
			  tr, &tracing_mark_fops);

	trace_create_file("trace_clock", 0644, d_tracer, tr,
			  &trace_clock_fops);

	trace_create_file("tracing_on", 0644, d_tracer,
6529
			  tr, &rb_simple_fops);
6530

6531 6532 6533 6534 6535
#ifdef CONFIG_TRACER_MAX_TRACE
	trace_create_file("tracing_max_latency", 0644, d_tracer,
			&tr->max_latency, &tracing_max_lat_fops);
#endif

6536 6537 6538
	if (ftrace_create_function_files(tr, d_tracer))
		WARN(1, "Could not allocate function filter files");

6539 6540
#ifdef CONFIG_TRACER_SNAPSHOT
	trace_create_file("snapshot", 0644, d_tracer,
6541
			  tr, &snapshot_fops);
6542
#endif
6543 6544 6545 6546

	for_each_tracing_cpu(cpu)
		tracing_init_debugfs_percpu(tr, cpu);

6547 6548
}

6549
static __init int tracer_init_debugfs(void)
6550 6551 6552
{
	struct dentry *d_tracer;

6553 6554
	trace_access_lock_init();

6555
	d_tracer = tracing_init_dentry();
6556 6557
	if (!d_tracer)
		return 0;
6558

6559
	init_tracer_debugfs(&global_trace, d_tracer);
6560

6561
	trace_create_file("tracing_thresh", 0644, d_tracer,
6562
			&global_trace, &tracing_thresh_fops);
6563

6564
	trace_create_file("README", 0444, d_tracer,
6565 6566
			NULL, &tracing_readme_fops);

6567 6568
	trace_create_file("saved_cmdlines", 0444, d_tracer,
			NULL, &tracing_saved_cmdlines_fops);
6569

6570 6571 6572
	trace_create_file("saved_cmdlines_size", 0644, d_tracer,
			  NULL, &tracing_saved_cmdlines_size_fops);

6573
#ifdef CONFIG_DYNAMIC_FTRACE
6574 6575
	trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
			&ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6576
#endif
6577

6578
	create_trace_instances(d_tracer);
6579

6580
	create_trace_options_dir(&global_trace);
6581

6582
	return 0;
6583 6584
}

6585 6586 6587
static int trace_panic_handler(struct notifier_block *this,
			       unsigned long event, void *unused)
{
6588
	if (ftrace_dump_on_oops)
6589
		ftrace_dump(ftrace_dump_on_oops);
6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604
	return NOTIFY_OK;
}

static struct notifier_block trace_panic_notifier = {
	.notifier_call  = trace_panic_handler,
	.next           = NULL,
	.priority       = 150   /* priority: INT_MAX >= x >= 0 */
};

static int trace_die_handler(struct notifier_block *self,
			     unsigned long val,
			     void *data)
{
	switch (val) {
	case DIE_OOPS:
6605
		if (ftrace_dump_on_oops)
6606
			ftrace_dump(ftrace_dump_on_oops);
6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block trace_die_notifier = {
	.notifier_call = trace_die_handler,
	.priority = 200
};

/*
 * printk is set to max of 1024, we really don't need it that big.
 * Nothing should be printing 1000 characters anyway.
 */
#define TRACE_MAX_PRINT		1000

/*
 * Define here KERN_TRACE so that we have one place to modify
 * it if we decide to change what log level the ftrace dump
 * should be at.
 */
6630
#define KERN_TRACE		KERN_EMERG
6631

6632
void
6633 6634 6635
trace_printk_seq(struct trace_seq *s)
{
	/* Probably should print a warning here. */
6636 6637
	if (s->len >= TRACE_MAX_PRINT)
		s->len = TRACE_MAX_PRINT;
6638 6639 6640 6641 6642 6643

	/* should be zero ended, but we are paranoid. */
	s->buffer[s->len] = 0;

	printk(KERN_TRACE "%s", s->buffer);

6644
	trace_seq_init(s);
6645 6646
}

6647 6648 6649
void trace_init_global_iter(struct trace_iterator *iter)
{
	iter->tr = &global_trace;
6650
	iter->trace = iter->tr->current_trace;
6651
	iter->cpu_file = RING_BUFFER_ALL_CPUS;
6652
	iter->trace_buffer = &global_trace.trace_buffer;
6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663

	if (iter->trace && iter->trace->open)
		iter->trace->open(iter);

	/* Annotate start of buffers if we had overruns */
	if (ring_buffer_overruns(iter->trace_buffer->buffer))
		iter->iter_flags |= TRACE_FILE_ANNOTATE;

	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
	if (trace_clocks[iter->tr->clock_id].in_ns)
		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6664 6665
}

6666
void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6667 6668 6669
{
	/* use static because iter can be a bit big for the stack */
	static struct trace_iterator iter;
6670
	static atomic_t dump_running;
6671
	unsigned int old_userobj;
6672 6673
	unsigned long flags;
	int cnt = 0, cpu;
6674

6675 6676 6677 6678 6679
	/* Only allow one dump user at a time. */
	if (atomic_inc_return(&dump_running) != 1) {
		atomic_dec(&dump_running);
		return;
	}
6680

6681 6682 6683 6684 6685 6686 6687 6688
	/*
	 * Always turn off tracing when we dump.
	 * We don't need to show trace output of what happens
	 * between multiple crashes.
	 *
	 * If the user does a sysrq-z, then they can re-enable
	 * tracing with echo 1 > tracing_on.
	 */
6689
	tracing_off();
6690

6691
	local_irq_save(flags);
6692

6693
	/* Simulate the iterator */
6694 6695
	trace_init_global_iter(&iter);

6696
	for_each_tracing_cpu(cpu) {
6697
		atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6698 6699
	}

6700 6701
	old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;

6702 6703 6704
	/* don't look at user memory in panic mode */
	trace_flags &= ~TRACE_ITER_SYM_USEROBJ;

6705 6706
	switch (oops_dump_mode) {
	case DUMP_ALL:
6707
		iter.cpu_file = RING_BUFFER_ALL_CPUS;
6708 6709 6710 6711 6712 6713 6714 6715
		break;
	case DUMP_ORIG:
		iter.cpu_file = raw_smp_processor_id();
		break;
	case DUMP_NONE:
		goto out_enable;
	default:
		printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6716
		iter.cpu_file = RING_BUFFER_ALL_CPUS;
6717 6718 6719
	}

	printk(KERN_TRACE "Dumping ftrace buffer:\n");
6720

6721 6722 6723 6724 6725 6726
	/* Did function tracer already get disabled? */
	if (ftrace_is_dead()) {
		printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
		printk("#          MAY BE MISSING FUNCTION EVENTS\n");
	}

6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747
	/*
	 * We need to stop all tracing on all CPUS to read the
	 * the next buffer. This is a bit expensive, but is
	 * not done often. We fill all what we can read,
	 * and then release the locks again.
	 */

	while (!trace_empty(&iter)) {

		if (!cnt)
			printk(KERN_TRACE "---------------------------------\n");

		cnt++;

		/* reset all but tr, trace, and overruns */
		memset(&iter.seq, 0,
		       sizeof(struct trace_iterator) -
		       offsetof(struct trace_iterator, seq));
		iter.iter_flags |= TRACE_FILE_LAT_FMT;
		iter.pos = -1;

6748
		if (trace_find_next_entry_inc(&iter) != NULL) {
6749 6750 6751 6752 6753
			int ret;

			ret = print_trace_line(&iter);
			if (ret != TRACE_TYPE_NO_CONSUME)
				trace_consume(&iter);
6754
		}
6755
		touch_nmi_watchdog();
6756 6757 6758 6759 6760 6761 6762 6763 6764

		trace_printk_seq(&iter.seq);
	}

	if (!cnt)
		printk(KERN_TRACE "   (ftrace buffer empty)\n");
	else
		printk(KERN_TRACE "---------------------------------\n");

6765
 out_enable:
6766
	trace_flags |= old_userobj;
6767

6768 6769
	for_each_tracing_cpu(cpu) {
		atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6770
	}
6771
 	atomic_dec(&dump_running);
6772
	local_irq_restore(flags);
6773
}
6774
EXPORT_SYMBOL_GPL(ftrace_dump);
6775

6776
__init static int tracer_alloc_buffers(void)
6777
{
6778
	int ring_buf_size;
6779
	int ret = -ENOMEM;
6780

6781

6782 6783 6784
	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
		goto out;

6785
	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6786
		goto out_free_buffer_mask;
6787

6788 6789
	/* Only allocate trace_printk buffers if a trace_printk exists */
	if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6790
		/* Must be called before global_trace.buffer is allocated */
6791 6792
		trace_printk_init_buffers();

6793 6794 6795 6796 6797 6798
	/* To save memory, keep the ring buffer size to its minimum */
	if (ring_buffer_expanded)
		ring_buf_size = trace_buf_size;
	else
		ring_buf_size = 1;

6799
	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6800
	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6801

6802 6803
	raw_spin_lock_init(&global_trace.start_lock);

6804 6805 6806 6807 6808
	/* Used for event triggers */
	temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
	if (!temp_buffer)
		goto out_free_cpumask;

6809 6810 6811
	if (trace_create_savedcmd() < 0)
		goto out_free_temp_buffer;

6812
	/* TODO: make the number of buffers hot pluggable with CPUS */
6813
	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6814 6815
		printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
		WARN_ON(1);
6816
		goto out_free_savedcmd;
6817
	}
6818

6819 6820
	if (global_trace.buffer_disabled)
		tracing_off();
6821

6822 6823 6824 6825 6826 6827 6828
	if (trace_boot_clock) {
		ret = tracing_set_clock(&global_trace, trace_boot_clock);
		if (ret < 0)
			pr_warning("Trace clock %s not defined, going back to default\n",
				   trace_boot_clock);
	}

6829 6830 6831 6832 6833
	/*
	 * register_tracer() might reference current_trace, so it
	 * needs to be set before we register anything. This is
	 * just a bootstrap of current_trace anyway.
	 */
6834 6835
	global_trace.current_trace = &nop_trace;

6836 6837
	global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;

6838 6839
	ftrace_init_global_array_ops(&global_trace);

6840 6841
	register_tracer(&nop_trace);

S
Steven Rostedt 已提交
6842 6843
	/* All seems OK, enable tracing */
	tracing_disabled = 0;
6844

6845 6846 6847 6848
	atomic_notifier_chain_register(&panic_notifier_list,
				       &trace_panic_notifier);

	register_die_notifier(&trace_die_notifier);
6849

6850 6851 6852 6853 6854 6855
	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;

	INIT_LIST_HEAD(&global_trace.systems);
	INIT_LIST_HEAD(&global_trace.events);
	list_add(&global_trace.list, &ftrace_trace_arrays);

6856 6857 6858 6859
	while (trace_boot_options) {
		char *option;

		option = strsep(&trace_boot_options, ",");
6860
		trace_set_options(&global_trace, option);
6861 6862
	}

6863 6864
	register_snapshot_cmd();

6865
	return 0;
6866

6867 6868
out_free_savedcmd:
	free_saved_cmdlines_buffer(savedcmd);
6869 6870
out_free_temp_buffer:
	ring_buffer_free(temp_buffer);
6871
out_free_cpumask:
6872
	free_cpumask_var(global_trace.tracing_cpumask);
6873 6874 6875 6876
out_free_buffer_mask:
	free_cpumask_var(tracing_buffer_mask);
out:
	return ret;
6877
}
6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897

__init static int clear_boot_tracer(void)
{
	/*
	 * The default tracer at boot buffer is an init section.
	 * This function is called in lateinit. If we did not
	 * find the boot tracer, then clear it out, to prevent
	 * later registration from accessing the buffer that is
	 * about to be freed.
	 */
	if (!default_bootup_tracer)
		return 0;

	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
	       default_bootup_tracer);
	default_bootup_tracer = NULL;

	return 0;
}

6898 6899
early_initcall(tracer_alloc_buffers);
fs_initcall(tracer_init_debugfs);
6900
late_initcall(clear_boot_tracer);