ring_buffer.c 78.0 KB
Newer Older
S
Steven Rostedt 已提交
1 2 3 4 5 6
/*
 * Generic ring buffer
 *
 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
 */
#include <linux/ring_buffer.h>
7
#include <linux/trace_clock.h>
8
#include <linux/ftrace_irq.h>
S
Steven Rostedt 已提交
9 10 11
#include <linux/spinlock.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
12
#include <linux/hardirq.h>
S
Steven Rostedt 已提交
13 14 15 16 17 18
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/mutex.h>
#include <linux/init.h>
#include <linux/hash.h>
#include <linux/list.h>
19
#include <linux/cpu.h>
S
Steven Rostedt 已提交
20 21
#include <linux/fs.h>

22 23
#include "trace.h"

24 25 26 27 28 29 30
/*
 * The ring buffer header is special. We must manually up keep it.
 */
int ring_buffer_print_entry_header(struct trace_seq *s)
{
	int ret;

31 32
	ret = trace_seq_printf(s, "# compressed entry header\n");
	ret = trace_seq_printf(s, "\ttype_len    :    5 bits\n");
33 34 35 36 37 38 39
	ret = trace_seq_printf(s, "\ttime_delta  :   27 bits\n");
	ret = trace_seq_printf(s, "\tarray       :   32 bits\n");
	ret = trace_seq_printf(s, "\n");
	ret = trace_seq_printf(s, "\tpadding     : type == %d\n",
			       RINGBUF_TYPE_PADDING);
	ret = trace_seq_printf(s, "\ttime_extend : type == %d\n",
			       RINGBUF_TYPE_TIME_EXTEND);
40 41
	ret = trace_seq_printf(s, "\tdata max type_len  == %d\n",
			       RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
42 43 44 45

	return ret;
}

46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
/*
 * The ring buffer is made up of a list of pages. A separate list of pages is
 * allocated for each CPU. A writer may only write to a buffer that is
 * associated with the CPU it is currently executing on.  A reader may read
 * from any per cpu buffer.
 *
 * The reader is special. For each per cpu buffer, the reader has its own
 * reader page. When a reader has read the entire reader page, this reader
 * page is swapped with another page in the ring buffer.
 *
 * Now, as long as the writer is off the reader page, the reader can do what
 * ever it wants with that page. The writer will never write to that page
 * again (as long as it is out of the ring buffer).
 *
 * Here's some silly ASCII art.
 *
 *   +------+
 *   |reader|          RING BUFFER
 *   |page  |
 *   +------+        +---+   +---+   +---+
 *                   |   |-->|   |-->|   |
 *                   +---+   +---+   +---+
 *                     ^               |
 *                     |               |
 *                     +---------------+
 *
 *
 *   +------+
 *   |reader|          RING BUFFER
 *   |page  |------------------v
 *   +------+        +---+   +---+   +---+
 *                   |   |-->|   |-->|   |
 *                   +---+   +---+   +---+
 *                     ^               |
 *                     |               |
 *                     +---------------+
 *
 *
 *   +------+
 *   |reader|          RING BUFFER
 *   |page  |------------------v
 *   +------+        +---+   +---+   +---+
 *      ^            |   |-->|   |-->|   |
 *      |            +---+   +---+   +---+
 *      |                              |
 *      |                              |
 *      +------------------------------+
 *
 *
 *   +------+
 *   |buffer|          RING BUFFER
 *   |page  |------------------v
 *   +------+        +---+   +---+   +---+
 *      ^            |   |   |   |-->|   |
 *      |   New      +---+   +---+   +---+
 *      |  Reader------^               |
 *      |   page                       |
 *      +------------------------------+
 *
 *
 * After we make this swap, the reader can hand this page off to the splice
 * code and be done with it. It can even allocate a new page if it needs to
 * and swap that into the ring buffer.
 *
 * We will be using cmpxchg soon to make all this lockless.
 *
 */

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
/*
 * A fast way to enable or disable all ring buffers is to
 * call tracing_on or tracing_off. Turning off the ring buffers
 * prevents all ring buffers from being recorded to.
 * Turning this switch on, makes it OK to write to the
 * ring buffer, if the ring buffer is enabled itself.
 *
 * There's three layers that must be on in order to write
 * to the ring buffer.
 *
 * 1) This global flag must be set.
 * 2) The ring buffer must be enabled for recording.
 * 3) The per cpu buffer must be enabled for recording.
 *
 * In case of an anomaly, this global flag has a bit set that
 * will permantly disable all ring buffers.
 */

/*
 * Global flag to disable all recording to ring buffers
 *  This has two bits: ON, DISABLED
 *
 *  ON   DISABLED
 * ---- ----------
 *   0      0        : ring buffers are off
 *   1      0        : ring buffers are on
 *   X      1        : ring buffers are permanently disabled
 */

enum {
	RB_BUFFERS_ON_BIT	= 0,
	RB_BUFFERS_DISABLED_BIT	= 1,
};

enum {
	RB_BUFFERS_ON		= 1 << RB_BUFFERS_ON_BIT,
	RB_BUFFERS_DISABLED	= 1 << RB_BUFFERS_DISABLED_BIT,
};

153
static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
154

155 156
#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)

157 158 159 160 161 162 163 164
/**
 * tracing_on - enable all tracing buffers
 *
 * This function enables all tracing buffers that may have been
 * disabled with tracing_off.
 */
void tracing_on(void)
{
165
	set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
166
}
167
EXPORT_SYMBOL_GPL(tracing_on);
168 169 170 171 172 173 174 175 176 177 178

/**
 * tracing_off - turn off all tracing buffers
 *
 * This function stops all tracing buffers from recording data.
 * It does not disable any overhead the tracers themselves may
 * be causing. This function simply causes all recording to
 * the ring buffers to fail.
 */
void tracing_off(void)
{
179 180
	clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
}
181
EXPORT_SYMBOL_GPL(tracing_off);
182 183 184 185 186

/**
 * tracing_off_permanent - permanently disable ring buffers
 *
 * This function, once called, will disable all ring buffers
W
Wenji Huang 已提交
187
 * permanently.
188 189 190 191
 */
void tracing_off_permanent(void)
{
	set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
192 193
}

194 195 196 197 198 199 200 201 202
/**
 * tracing_is_on - show state of ring buffers enabled
 */
int tracing_is_on(void)
{
	return ring_buffer_flags == RB_BUFFERS_ON;
}
EXPORT_SYMBOL_GPL(tracing_is_on);

203 204
#include "trace.h"

205
#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
206
#define RB_ALIGNMENT		4U
207
#define RB_MAX_SMALL_DATA	(RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
208
#define RB_EVNT_MIN_SIZE	8U	/* two 32bit words */
209 210 211

/* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */
#define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX
S
Steven Rostedt 已提交
212 213 214 215 216 217

enum {
	RB_LEN_TIME_EXTEND = 8,
	RB_LEN_TIME_STAMP = 16,
};

218 219
static inline int rb_null_event(struct ring_buffer_event *event)
{
220 221
	return event->type_len == RINGBUF_TYPE_PADDING
			&& event->time_delta == 0;
222 223 224 225
}

static inline int rb_discarded_event(struct ring_buffer_event *event)
{
226
	return event->type_len == RINGBUF_TYPE_PADDING && event->time_delta;
227 228 229 230
}

static void rb_event_set_padding(struct ring_buffer_event *event)
{
231
	event->type_len = RINGBUF_TYPE_PADDING;
232 233 234
	event->time_delta = 0;
}

235
static unsigned
236
rb_event_data_length(struct ring_buffer_event *event)
S
Steven Rostedt 已提交
237 238 239
{
	unsigned length;

240 241
	if (event->type_len)
		length = event->type_len * RB_ALIGNMENT;
242 243 244 245 246 247 248 249 250
	else
		length = event->array[0];
	return length + RB_EVNT_HDR_SIZE;
}

/* inline for ring buffer fast paths */
static unsigned
rb_event_length(struct ring_buffer_event *event)
{
251
	switch (event->type_len) {
S
Steven Rostedt 已提交
252
	case RINGBUF_TYPE_PADDING:
253 254 255
		if (rb_null_event(event))
			/* undefined */
			return -1;
256
		return  event->array[0] + RB_EVNT_HDR_SIZE;
S
Steven Rostedt 已提交
257 258 259 260 261 262 263 264

	case RINGBUF_TYPE_TIME_EXTEND:
		return RB_LEN_TIME_EXTEND;

	case RINGBUF_TYPE_TIME_STAMP:
		return RB_LEN_TIME_STAMP;

	case RINGBUF_TYPE_DATA:
265
		return rb_event_data_length(event);
S
Steven Rostedt 已提交
266 267 268 269 270 271 272 273 274 275 276 277 278
	default:
		BUG();
	}
	/* not hit */
	return 0;
}

/**
 * ring_buffer_event_length - return the length of the event
 * @event: the event to get the length of
 */
unsigned ring_buffer_event_length(struct ring_buffer_event *event)
{
279
	unsigned length = rb_event_length(event);
280
	if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
281 282 283 284 285
		return length;
	length -= RB_EVNT_HDR_SIZE;
	if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
                length -= sizeof(event->array[0]);
	return length;
S
Steven Rostedt 已提交
286
}
287
EXPORT_SYMBOL_GPL(ring_buffer_event_length);
S
Steven Rostedt 已提交
288 289

/* inline for ring buffer fast paths */
290
static void *
S
Steven Rostedt 已提交
291 292
rb_event_data(struct ring_buffer_event *event)
{
293
	BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
S
Steven Rostedt 已提交
294
	/* If length is in len field, then array[0] has the data */
295
	if (event->type_len)
S
Steven Rostedt 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308
		return (void *)&event->array[0];
	/* Otherwise length is in array[0] and array[1] has the data */
	return (void *)&event->array[1];
}

/**
 * ring_buffer_event_data - return the data of the event
 * @event: the event to get the data from
 */
void *ring_buffer_event_data(struct ring_buffer_event *event)
{
	return rb_event_data(event);
}
309
EXPORT_SYMBOL_GPL(ring_buffer_event_data);
S
Steven Rostedt 已提交
310 311

#define for_each_buffer_cpu(buffer, cpu)		\
312
	for_each_cpu(cpu, buffer->cpumask)
S
Steven Rostedt 已提交
313 314 315 316 317

#define TS_SHIFT	27
#define TS_MASK		((1ULL << TS_SHIFT) - 1)
#define TS_DELTA_TEST	(~TS_MASK)

318
struct buffer_data_page {
319
	u64		 time_stamp;	/* page time stamp */
W
Wenji Huang 已提交
320
	local_t		 commit;	/* write committed index */
321 322 323 324
	unsigned char	 data[];	/* data of buffer page */
};

struct buffer_page {
325
	struct list_head list;		/* list of buffer pages */
326
	local_t		 write;		/* index for next write */
327
	unsigned	 read;		/* index for next read */
328
	local_t		 entries;	/* entries on this page */
329
	struct buffer_data_page *page;	/* Actual data page */
S
Steven Rostedt 已提交
330 331
};

332
static void rb_init_page(struct buffer_data_page *bpage)
333
{
334
	local_set(&bpage->commit, 0);
335 336
}

337 338 339 340 341 342
/**
 * ring_buffer_page_len - the size of data on the page.
 * @page: The page to read
 *
 * Returns the amount of data on the page, including buffer page header.
 */
343 344
size_t ring_buffer_page_len(void *page)
{
345 346
	return local_read(&((struct buffer_data_page *)page)->commit)
		+ BUF_PAGE_HDR_SIZE;
347 348
}

349 350 351 352
/*
 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
 * this issue out.
 */
353
static void free_buffer_page(struct buffer_page *bpage)
354
{
355
	free_page((unsigned long)bpage->page);
356
	kfree(bpage);
357 358
}

S
Steven Rostedt 已提交
359 360 361 362 363 364 365 366 367 368
/*
 * We need to fit the time_stamp delta into 27 bits.
 */
static inline int test_time_stamp(u64 delta)
{
	if (delta & TS_DELTA_TEST)
		return 1;
	return 0;
}

369
#define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
S
Steven Rostedt 已提交
370

371 372 373
/* Max payload is BUF_PAGE_SIZE - header (8bytes) */
#define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))

374 375 376
/* Max number of timestamps that can fit on a page */
#define RB_TIMESTAMPS_PER_PAGE	(BUF_PAGE_SIZE / RB_LEN_TIME_STAMP)

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
int ring_buffer_print_page_header(struct trace_seq *s)
{
	struct buffer_data_page field;
	int ret;

	ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t"
			       "offset:0;\tsize:%u;\n",
			       (unsigned int)sizeof(field.time_stamp));

	ret = trace_seq_printf(s, "\tfield: local_t commit;\t"
			       "offset:%u;\tsize:%u;\n",
			       (unsigned int)offsetof(typeof(field), commit),
			       (unsigned int)sizeof(field.commit));

	ret = trace_seq_printf(s, "\tfield: char data;\t"
			       "offset:%u;\tsize:%u;\n",
			       (unsigned int)offsetof(typeof(field), data),
			       (unsigned int)BUF_PAGE_SIZE);

	return ret;
}

S
Steven Rostedt 已提交
399 400 401 402 403 404
/*
 * head_page == tail_page && head == tail then buffer is empty.
 */
struct ring_buffer_per_cpu {
	int				cpu;
	struct ring_buffer		*buffer;
S
Steven Rostedt 已提交
405
	spinlock_t			reader_lock; /* serialize readers */
406
	raw_spinlock_t			lock;
S
Steven Rostedt 已提交
407 408
	struct lock_class_key		lock_key;
	struct list_head		pages;
409 410
	struct buffer_page		*head_page;	/* read from head */
	struct buffer_page		*tail_page;	/* write to tail */
W
Wenji Huang 已提交
411
	struct buffer_page		*commit_page;	/* committed pages */
412
	struct buffer_page		*reader_page;
413 414
	unsigned long			nmi_dropped;
	unsigned long			commit_overrun;
S
Steven Rostedt 已提交
415
	unsigned long			overrun;
416 417
	unsigned long			read;
	local_t				entries;
418 419
	local_t				committing;
	local_t				commits;
S
Steven Rostedt 已提交
420 421 422 423 424 425 426 427 428 429
	u64				write_stamp;
	u64				read_stamp;
	atomic_t			record_disabled;
};

struct ring_buffer {
	unsigned			pages;
	unsigned			flags;
	int				cpus;
	atomic_t			record_disabled;
430
	cpumask_var_t			cpumask;
S
Steven Rostedt 已提交
431

432 433
	struct lock_class_key		*reader_lock_key;

S
Steven Rostedt 已提交
434 435 436
	struct mutex			mutex;

	struct ring_buffer_per_cpu	**buffers;
437

438
#ifdef CONFIG_HOTPLUG_CPU
439 440
	struct notifier_block		cpu_notify;
#endif
441
	u64				(*clock)(void);
S
Steven Rostedt 已提交
442 443 444 445 446 447 448 449 450
};

struct ring_buffer_iter {
	struct ring_buffer_per_cpu	*cpu_buffer;
	unsigned long			head;
	struct buffer_page		*head_page;
	u64				read_stamp;
};

451
/* buffer may be either ring_buffer or ring_buffer_per_cpu */
S
Steven Rostedt 已提交
452
#define RB_WARN_ON(buffer, cond)				\
S
Steven Rostedt 已提交
453 454 455
	({							\
		int _____ret = unlikely(cond);			\
		if (_____ret) {					\
S
Steven Rostedt 已提交
456 457 458
			atomic_inc(&buffer->record_disabled);	\
			WARN_ON(1);				\
		}						\
S
Steven Rostedt 已提交
459 460
		_____ret;					\
	})
461

462 463 464
/* Up this if you want to test the TIME_EXTENTS and normalization */
#define DEBUG_SHIFT 0

465 466 467 468 469 470
static inline u64 rb_time_stamp(struct ring_buffer *buffer, int cpu)
{
	/* shift to debug/test normalization and TIME_EXTENTS */
	return buffer->clock() << DEBUG_SHIFT;
}

471 472 473 474 475
u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
{
	u64 time;

	preempt_disable_notrace();
476
	time = rb_time_stamp(buffer, cpu);
477 478 479 480 481 482 483 484 485 486 487 488 489 490
	preempt_enable_no_resched_notrace();

	return time;
}
EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);

void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
				      int cpu, u64 *ts)
{
	/* Just stupid testing the normalize function and deltas */
	*ts >>= DEBUG_SHIFT;
}
EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);

S
Steven Rostedt 已提交
491 492 493 494
/**
 * check_pages - integrity check of buffer pages
 * @cpu_buffer: CPU buffer with pages to test
 *
W
Wenji Huang 已提交
495
 * As a safety measure we check to make sure the data pages have not
S
Steven Rostedt 已提交
496 497 498 499 500
 * been corrupted.
 */
static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
{
	struct list_head *head = &cpu_buffer->pages;
501
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
502

S
Steven Rostedt 已提交
503 504 505 506
	if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
		return -1;
	if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
		return -1;
S
Steven Rostedt 已提交
507

508
	list_for_each_entry_safe(bpage, tmp, head, list) {
S
Steven Rostedt 已提交
509
		if (RB_WARN_ON(cpu_buffer,
510
			       bpage->list.next->prev != &bpage->list))
S
Steven Rostedt 已提交
511 512
			return -1;
		if (RB_WARN_ON(cpu_buffer,
513
			       bpage->list.prev->next != &bpage->list))
S
Steven Rostedt 已提交
514
			return -1;
S
Steven Rostedt 已提交
515 516 517 518 519 520 521 522 523
	}

	return 0;
}

static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
			     unsigned nr_pages)
{
	struct list_head *head = &cpu_buffer->pages;
524
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
525 526 527 528 529
	unsigned long addr;
	LIST_HEAD(pages);
	unsigned i;

	for (i = 0; i < nr_pages; i++) {
530
		bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
S
Steven Rostedt 已提交
531
				    GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));
532
		if (!bpage)
533
			goto free_pages;
534
		list_add(&bpage->list, &pages);
535

S
Steven Rostedt 已提交
536 537 538
		addr = __get_free_page(GFP_KERNEL);
		if (!addr)
			goto free_pages;
539 540
		bpage->page = (void *)addr;
		rb_init_page(bpage->page);
S
Steven Rostedt 已提交
541 542 543 544 545 546 547 548 549
	}

	list_splice(&pages, head);

	rb_check_pages(cpu_buffer);

	return 0;

 free_pages:
550 551 552
	list_for_each_entry_safe(bpage, tmp, &pages, list) {
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
553 554 555 556 557 558 559 560
	}
	return -ENOMEM;
}

static struct ring_buffer_per_cpu *
rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
561
	struct buffer_page *bpage;
562
	unsigned long addr;
S
Steven Rostedt 已提交
563 564 565 566 567 568 569 570 571
	int ret;

	cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
				  GFP_KERNEL, cpu_to_node(cpu));
	if (!cpu_buffer)
		return NULL;

	cpu_buffer->cpu = cpu;
	cpu_buffer->buffer = buffer;
S
Steven Rostedt 已提交
572
	spin_lock_init(&cpu_buffer->reader_lock);
573
	lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
574
	cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
S
Steven Rostedt 已提交
575 576
	INIT_LIST_HEAD(&cpu_buffer->pages);

577
	bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
578
			    GFP_KERNEL, cpu_to_node(cpu));
579
	if (!bpage)
580 581
		goto fail_free_buffer;

582
	cpu_buffer->reader_page = bpage;
583 584
	addr = __get_free_page(GFP_KERNEL);
	if (!addr)
585
		goto fail_free_reader;
586 587
	bpage->page = (void *)addr;
	rb_init_page(bpage->page);
588

589 590
	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);

S
Steven Rostedt 已提交
591 592
	ret = rb_allocate_pages(cpu_buffer, buffer->pages);
	if (ret < 0)
593
		goto fail_free_reader;
S
Steven Rostedt 已提交
594 595 596

	cpu_buffer->head_page
		= list_entry(cpu_buffer->pages.next, struct buffer_page, list);
S
Steven Rostedt 已提交
597
	cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
S
Steven Rostedt 已提交
598 599 600

	return cpu_buffer;

601 602 603
 fail_free_reader:
	free_buffer_page(cpu_buffer->reader_page);

S
Steven Rostedt 已提交
604 605 606 607 608 609 610 611
 fail_free_buffer:
	kfree(cpu_buffer);
	return NULL;
}

static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
{
	struct list_head *head = &cpu_buffer->pages;
612
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
613

614 615
	free_buffer_page(cpu_buffer->reader_page);

616 617 618
	list_for_each_entry_safe(bpage, tmp, head, list) {
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
619 620 621 622
	}
	kfree(cpu_buffer);
}

623
#ifdef CONFIG_HOTPLUG_CPU
624 625
static int rb_cpu_notify(struct notifier_block *self,
			 unsigned long action, void *hcpu);
626 627
#endif

S
Steven Rostedt 已提交
628 629
/**
 * ring_buffer_alloc - allocate a new ring_buffer
630
 * @size: the size in bytes per cpu that is needed.
S
Steven Rostedt 已提交
631 632 633 634 635 636 637
 * @flags: attributes to set for the ring buffer.
 *
 * Currently the only flag that is available is the RB_FL_OVERWRITE
 * flag. This flag means that the buffer will overwrite old data
 * when the buffer wraps. If this flag is not set, the buffer will
 * drop data when the tail hits the head.
 */
638 639
struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
					struct lock_class_key *key)
S
Steven Rostedt 已提交
640 641 642 643 644 645 646 647 648 649 650
{
	struct ring_buffer *buffer;
	int bsize;
	int cpu;

	/* keep it in its own cache line */
	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
			 GFP_KERNEL);
	if (!buffer)
		return NULL;

651 652 653
	if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
		goto fail_free_buffer;

S
Steven Rostedt 已提交
654 655
	buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
	buffer->flags = flags;
656
	buffer->clock = trace_clock_local;
657
	buffer->reader_lock_key = key;
S
Steven Rostedt 已提交
658 659

	/* need at least two pages */
660 661
	if (buffer->pages < 2)
		buffer->pages = 2;
S
Steven Rostedt 已提交
662

663 664 665 666 667 668
	/*
	 * In case of non-hotplug cpu, if the ring-buffer is allocated
	 * in early initcall, it will not be notified of secondary cpus.
	 * In that off case, we need to allocate for all possible cpus.
	 */
#ifdef CONFIG_HOTPLUG_CPU
669 670
	get_online_cpus();
	cpumask_copy(buffer->cpumask, cpu_online_mask);
671 672 673
#else
	cpumask_copy(buffer->cpumask, cpu_possible_mask);
#endif
S
Steven Rostedt 已提交
674 675 676 677 678 679
	buffer->cpus = nr_cpu_ids;

	bsize = sizeof(void *) * nr_cpu_ids;
	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
				  GFP_KERNEL);
	if (!buffer->buffers)
680
		goto fail_free_cpumask;
S
Steven Rostedt 已提交
681 682 683 684 685 686 687 688

	for_each_buffer_cpu(buffer, cpu) {
		buffer->buffers[cpu] =
			rb_allocate_cpu_buffer(buffer, cpu);
		if (!buffer->buffers[cpu])
			goto fail_free_buffers;
	}

689
#ifdef CONFIG_HOTPLUG_CPU
690 691 692 693 694 695
	buffer->cpu_notify.notifier_call = rb_cpu_notify;
	buffer->cpu_notify.priority = 0;
	register_cpu_notifier(&buffer->cpu_notify);
#endif

	put_online_cpus();
S
Steven Rostedt 已提交
696 697 698 699 700 701 702 703 704 705 706
	mutex_init(&buffer->mutex);

	return buffer;

 fail_free_buffers:
	for_each_buffer_cpu(buffer, cpu) {
		if (buffer->buffers[cpu])
			rb_free_cpu_buffer(buffer->buffers[cpu]);
	}
	kfree(buffer->buffers);

707 708
 fail_free_cpumask:
	free_cpumask_var(buffer->cpumask);
709
	put_online_cpus();
710

S
Steven Rostedt 已提交
711 712 713 714
 fail_free_buffer:
	kfree(buffer);
	return NULL;
}
715
EXPORT_SYMBOL_GPL(__ring_buffer_alloc);
S
Steven Rostedt 已提交
716 717 718 719 720 721 722 723 724 725

/**
 * ring_buffer_free - free a ring buffer.
 * @buffer: the buffer to free.
 */
void
ring_buffer_free(struct ring_buffer *buffer)
{
	int cpu;

726 727
	get_online_cpus();

728
#ifdef CONFIG_HOTPLUG_CPU
729 730 731
	unregister_cpu_notifier(&buffer->cpu_notify);
#endif

S
Steven Rostedt 已提交
732 733 734
	for_each_buffer_cpu(buffer, cpu)
		rb_free_cpu_buffer(buffer->buffers[cpu]);

735 736
	put_online_cpus();

737 738
	free_cpumask_var(buffer->cpumask);

S
Steven Rostedt 已提交
739 740
	kfree(buffer);
}
741
EXPORT_SYMBOL_GPL(ring_buffer_free);
S
Steven Rostedt 已提交
742

743 744 745 746 747 748
void ring_buffer_set_clock(struct ring_buffer *buffer,
			   u64 (*clock)(void))
{
	buffer->clock = clock;
}

S
Steven Rostedt 已提交
749 750 751 752 753
static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);

static void
rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
{
754
	struct buffer_page *bpage;
S
Steven Rostedt 已提交
755 756 757 758 759 760 761
	struct list_head *p;
	unsigned i;

	atomic_inc(&cpu_buffer->record_disabled);
	synchronize_sched();

	for (i = 0; i < nr_pages; i++) {
S
Steven Rostedt 已提交
762 763
		if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
			return;
S
Steven Rostedt 已提交
764
		p = cpu_buffer->pages.next;
765 766 767
		bpage = list_entry(p, struct buffer_page, list);
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
768
	}
S
Steven Rostedt 已提交
769 770
	if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
		return;
S
Steven Rostedt 已提交
771 772 773 774 775 776 777 778 779 780 781 782 783

	rb_reset_cpu(cpu_buffer);

	rb_check_pages(cpu_buffer);

	atomic_dec(&cpu_buffer->record_disabled);

}

static void
rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
		struct list_head *pages, unsigned nr_pages)
{
784
	struct buffer_page *bpage;
S
Steven Rostedt 已提交
785 786 787 788 789 790 791
	struct list_head *p;
	unsigned i;

	atomic_inc(&cpu_buffer->record_disabled);
	synchronize_sched();

	for (i = 0; i < nr_pages; i++) {
S
Steven Rostedt 已提交
792 793
		if (RB_WARN_ON(cpu_buffer, list_empty(pages)))
			return;
S
Steven Rostedt 已提交
794
		p = pages->next;
795 796 797
		bpage = list_entry(p, struct buffer_page, list);
		list_del_init(&bpage->list);
		list_add_tail(&bpage->list, &cpu_buffer->pages);
S
Steven Rostedt 已提交
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823
	}
	rb_reset_cpu(cpu_buffer);

	rb_check_pages(cpu_buffer);

	atomic_dec(&cpu_buffer->record_disabled);
}

/**
 * ring_buffer_resize - resize the ring buffer
 * @buffer: the buffer to resize.
 * @size: the new size.
 *
 * The tracer is responsible for making sure that the buffer is
 * not being used while changing the size.
 * Note: We may be able to change the above requirement by using
 *  RCU synchronizations.
 *
 * Minimum size is 2 * BUF_PAGE_SIZE.
 *
 * Returns -1 on failure.
 */
int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned nr_pages, rm_pages, new_pages;
824
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
825 826 827 828 829
	unsigned long buffer_size;
	unsigned long addr;
	LIST_HEAD(pages);
	int i, cpu;

830 831 832 833 834 835
	/*
	 * Always succeed at resizing a non-existent buffer:
	 */
	if (!buffer)
		return size;

S
Steven Rostedt 已提交
836 837 838 839 840 841 842 843 844 845 846 847
	size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
	size *= BUF_PAGE_SIZE;
	buffer_size = buffer->pages * BUF_PAGE_SIZE;

	/* we need a minimum of two pages */
	if (size < BUF_PAGE_SIZE * 2)
		size = BUF_PAGE_SIZE * 2;

	if (size == buffer_size)
		return size;

	mutex_lock(&buffer->mutex);
848
	get_online_cpus();
S
Steven Rostedt 已提交
849 850 851 852 853 854

	nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);

	if (size < buffer_size) {

		/* easy case, just free pages */
855 856
		if (RB_WARN_ON(buffer, nr_pages >= buffer->pages))
			goto out_fail;
S
Steven Rostedt 已提交
857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874

		rm_pages = buffer->pages - nr_pages;

		for_each_buffer_cpu(buffer, cpu) {
			cpu_buffer = buffer->buffers[cpu];
			rb_remove_pages(cpu_buffer, rm_pages);
		}
		goto out;
	}

	/*
	 * This is a bit more difficult. We only want to add pages
	 * when we can allocate enough for all CPUs. We do this
	 * by allocating all the pages and storing them on a local
	 * link list. If we succeed in our allocation, then we
	 * add these pages to the cpu_buffers. Otherwise we just free
	 * them all and return -ENOMEM;
	 */
875 876
	if (RB_WARN_ON(buffer, nr_pages <= buffer->pages))
		goto out_fail;
877

S
Steven Rostedt 已提交
878 879 880 881
	new_pages = nr_pages - buffer->pages;

	for_each_buffer_cpu(buffer, cpu) {
		for (i = 0; i < new_pages; i++) {
882
			bpage = kzalloc_node(ALIGN(sizeof(*bpage),
883 884
						  cache_line_size()),
					    GFP_KERNEL, cpu_to_node(cpu));
885
			if (!bpage)
886
				goto free_pages;
887
			list_add(&bpage->list, &pages);
S
Steven Rostedt 已提交
888 889 890
			addr = __get_free_page(GFP_KERNEL);
			if (!addr)
				goto free_pages;
891 892
			bpage->page = (void *)addr;
			rb_init_page(bpage->page);
S
Steven Rostedt 已提交
893 894 895 896 897 898 899 900
		}
	}

	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
		rb_insert_pages(cpu_buffer, &pages, new_pages);
	}

901 902
	if (RB_WARN_ON(buffer, !list_empty(&pages)))
		goto out_fail;
S
Steven Rostedt 已提交
903 904 905

 out:
	buffer->pages = nr_pages;
906
	put_online_cpus();
S
Steven Rostedt 已提交
907 908 909 910 911
	mutex_unlock(&buffer->mutex);

	return size;

 free_pages:
912 913 914
	list_for_each_entry_safe(bpage, tmp, &pages, list) {
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
915
	}
916
	put_online_cpus();
917
	mutex_unlock(&buffer->mutex);
S
Steven Rostedt 已提交
918
	return -ENOMEM;
919 920 921 922 923 924 925 926 927

	/*
	 * Something went totally wrong, and we are too paranoid
	 * to even clean up the mess.
	 */
 out_fail:
	put_online_cpus();
	mutex_unlock(&buffer->mutex);
	return -1;
S
Steven Rostedt 已提交
928
}
929
EXPORT_SYMBOL_GPL(ring_buffer_resize);
S
Steven Rostedt 已提交
930

S
Steven Rostedt 已提交
931
static inline void *
932
__rb_data_page_index(struct buffer_data_page *bpage, unsigned index)
S
Steven Rostedt 已提交
933
{
934
	return bpage->data + index;
S
Steven Rostedt 已提交
935 936
}

937
static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
S
Steven Rostedt 已提交
938
{
939
	return bpage->page->data + index;
S
Steven Rostedt 已提交
940 941 942
}

static inline struct ring_buffer_event *
943
rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
944
{
945 946 947 948 949 950 951 952 953
	return __rb_page_index(cpu_buffer->reader_page,
			       cpu_buffer->reader_page->read);
}

static inline struct ring_buffer_event *
rb_head_event(struct ring_buffer_per_cpu *cpu_buffer)
{
	return __rb_page_index(cpu_buffer->head_page,
			       cpu_buffer->head_page->read);
S
Steven Rostedt 已提交
954 955 956 957 958
}

static inline struct ring_buffer_event *
rb_iter_head_event(struct ring_buffer_iter *iter)
{
959
	return __rb_page_index(iter->head_page, iter->head);
S
Steven Rostedt 已提交
960 961
}

S
Steven Rostedt 已提交
962 963 964 965 966 967 968
static inline unsigned rb_page_write(struct buffer_page *bpage)
{
	return local_read(&bpage->write);
}

static inline unsigned rb_page_commit(struct buffer_page *bpage)
{
969
	return local_read(&bpage->page->commit);
S
Steven Rostedt 已提交
970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
}

/* Size is determined by what has been commited */
static inline unsigned rb_page_size(struct buffer_page *bpage)
{
	return rb_page_commit(bpage);
}

static inline unsigned
rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
{
	return rb_page_commit(cpu_buffer->commit_page);
}

static inline unsigned rb_head_size(struct ring_buffer_per_cpu *cpu_buffer)
{
	return rb_page_commit(cpu_buffer->head_page);
}

S
Steven Rostedt 已提交
989
static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
990
			       struct buffer_page **bpage)
S
Steven Rostedt 已提交
991
{
992
	struct list_head *p = (*bpage)->list.next;
S
Steven Rostedt 已提交
993 994 995 996

	if (p == &cpu_buffer->pages)
		p = p->next;

997
	*bpage = list_entry(p, struct buffer_page, list);
S
Steven Rostedt 已提交
998 999
}

S
Steven Rostedt 已提交
1000 1001 1002 1003 1004
static inline unsigned
rb_event_index(struct ring_buffer_event *event)
{
	unsigned long addr = (unsigned long)event;

1005
	return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;
S
Steven Rostedt 已提交
1006 1007
}

1008
static inline int
1009 1010
rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
		   struct ring_buffer_event *event)
S
Steven Rostedt 已提交
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
{
	unsigned long addr = (unsigned long)event;
	unsigned long index;

	index = rb_event_index(event);
	addr &= PAGE_MASK;

	return cpu_buffer->commit_page->page == (void *)addr &&
		rb_commit_index(cpu_buffer) == index;
}

1022
static void
S
Steven Rostedt 已提交
1023
rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
1024
{
S
Steven Rostedt 已提交
1025 1026 1027 1028 1029 1030 1031 1032
	/*
	 * We only race with interrupts and NMIs on this CPU.
	 * If we own the commit event, then we can commit
	 * all others that interrupted us, since the interruptions
	 * are in stack format (they finish before they come
	 * back to us). This allows us to do a simple loop to
	 * assign the commit to the tail.
	 */
1033
 again:
S
Steven Rostedt 已提交
1034
	while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
1035
		cpu_buffer->commit_page->page->commit =
S
Steven Rostedt 已提交
1036 1037
			cpu_buffer->commit_page->write;
		rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
1038 1039
		cpu_buffer->write_stamp =
			cpu_buffer->commit_page->page->time_stamp;
S
Steven Rostedt 已提交
1040 1041 1042 1043 1044
		/* add barrier to keep gcc from optimizing too much */
		barrier();
	}
	while (rb_commit_index(cpu_buffer) !=
	       rb_page_write(cpu_buffer->commit_page)) {
1045
		cpu_buffer->commit_page->page->commit =
S
Steven Rostedt 已提交
1046 1047 1048
			cpu_buffer->commit_page->write;
		barrier();
	}
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059

	/* again, keep gcc from optimizing */
	barrier();

	/*
	 * If an interrupt came in just after the first while loop
	 * and pushed the tail page forward, we will be left with
	 * a dangling commit that will never go forward.
	 */
	if (unlikely(cpu_buffer->commit_page != cpu_buffer->tail_page))
		goto again;
S
Steven Rostedt 已提交
1060 1061
}

1062
static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
1063
{
1064
	cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
1065
	cpu_buffer->reader_page->read = 0;
1066 1067
}

1068
static void rb_inc_iter(struct ring_buffer_iter *iter)
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

	/*
	 * The iterator could be on the reader page (it starts there).
	 * But the head could have moved, since the reader was
	 * found. Check for this case and assign the iterator
	 * to the head page instead of next.
	 */
	if (iter->head_page == cpu_buffer->reader_page)
		iter->head_page = cpu_buffer->head_page;
	else
		rb_inc_page(cpu_buffer, &iter->head_page);

1083
	iter->read_stamp = iter->head_page->page->time_stamp;
S
Steven Rostedt 已提交
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
	iter->head = 0;
}

/**
 * ring_buffer_update_event - update event type and data
 * @event: the even to update
 * @type: the type of event
 * @length: the size of the event field in the ring buffer
 *
 * Update the type and data fields of the event. The length
 * is the actual size that is written to the ring buffer,
 * and with this, we can determine what to place into the
 * data field.
 */
1098
static void
S
Steven Rostedt 已提交
1099 1100 1101
rb_update_event(struct ring_buffer_event *event,
			 unsigned type, unsigned length)
{
1102
	event->type_len = type;
S
Steven Rostedt 已提交
1103 1104 1105 1106 1107 1108 1109 1110

	switch (type) {

	case RINGBUF_TYPE_PADDING:
	case RINGBUF_TYPE_TIME_EXTEND:
	case RINGBUF_TYPE_TIME_STAMP:
		break;

1111
	case 0:
S
Steven Rostedt 已提交
1112
		length -= RB_EVNT_HDR_SIZE;
1113
		if (length > RB_MAX_SMALL_DATA)
S
Steven Rostedt 已提交
1114
			event->array[0] = length;
1115 1116
		else
			event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);
S
Steven Rostedt 已提交
1117 1118 1119 1120 1121 1122
		break;
	default:
		BUG();
	}
}

1123
static unsigned rb_calculate_event_length(unsigned length)
S
Steven Rostedt 已提交
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
{
	struct ring_buffer_event event; /* Used only for sizeof array */

	/* zero length can cause confusions */
	if (!length)
		length = 1;

	if (length > RB_MAX_SMALL_DATA)
		length += sizeof(event.array[0]);

	length += RB_EVNT_HDR_SIZE;
	length = ALIGN(length, RB_ALIGNMENT);

	return length;
}

1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
static inline void
rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
	      struct buffer_page *tail_page,
	      unsigned long tail, unsigned long length)
{
	struct ring_buffer_event *event;

	/*
	 * Only the event that crossed the page boundary
	 * must fill the old tail_page with padding.
	 */
	if (tail >= BUF_PAGE_SIZE) {
		local_sub(length, &tail_page->write);
		return;
	}

	event = __rb_page_index(tail_page, tail);

	/*
	 * If this event is bigger than the minimum size, then
	 * we need to be careful that we don't subtract the
	 * write counter enough to allow another writer to slip
	 * in on this page.
	 * We put in a discarded commit instead, to make sure
	 * that this space is not used again.
	 *
	 * If we are less than the minimum size, we don't need to
	 * worry about it.
	 */
	if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {
		/* No room for any events */

		/* Mark the rest of the page with padding */
		rb_event_set_padding(event);

		/* Set the write back to the previous setting */
		local_sub(length, &tail_page->write);
		return;
	}

	/* Put in a discarded event */
	event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;
	event->type_len = RINGBUF_TYPE_PADDING;
	/* time delta must be non zero */
	event->time_delta = 1;
	/* Account for this as an entry */
	local_inc(&tail_page->entries);
	local_inc(&cpu_buffer->entries);

	/* Set write to end of buffer */
	length = (tail + length) - BUF_PAGE_SIZE;
	local_sub(length, &tail_page->write);
}
1193

S
Steven Rostedt 已提交
1194
static struct ring_buffer_event *
1195 1196 1197 1198
rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
	     unsigned long length, unsigned long tail,
	     struct buffer_page *commit_page,
	     struct buffer_page *tail_page, u64 *ts)
S
Steven Rostedt 已提交
1199
{
1200
	struct buffer_page *next_page, *head_page, *reader_page;
S
Steven Rostedt 已提交
1201
	struct ring_buffer *buffer = cpu_buffer->buffer;
1202
	bool lock_taken = false;
1203
	unsigned long flags;
1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281

	next_page = tail_page;

	local_irq_save(flags);
	/*
	 * Since the write to the buffer is still not
	 * fully lockless, we must be careful with NMIs.
	 * The locks in the writers are taken when a write
	 * crosses to a new page. The locks protect against
	 * races with the readers (this will soon be fixed
	 * with a lockless solution).
	 *
	 * Because we can not protect against NMIs, and we
	 * want to keep traces reentrant, we need to manage
	 * what happens when we are in an NMI.
	 *
	 * NMIs can happen after we take the lock.
	 * If we are in an NMI, only take the lock
	 * if it is not already taken. Otherwise
	 * simply fail.
	 */
	if (unlikely(in_nmi())) {
		if (!__raw_spin_trylock(&cpu_buffer->lock)) {
			cpu_buffer->nmi_dropped++;
			goto out_reset;
		}
	} else
		__raw_spin_lock(&cpu_buffer->lock);

	lock_taken = true;

	rb_inc_page(cpu_buffer, &next_page);

	head_page = cpu_buffer->head_page;
	reader_page = cpu_buffer->reader_page;

	/* we grabbed the lock before incrementing */
	if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
		goto out_reset;

	/*
	 * If for some reason, we had an interrupt storm that made
	 * it all the way around the buffer, bail, and warn
	 * about it.
	 */
	if (unlikely(next_page == commit_page)) {
		cpu_buffer->commit_overrun++;
		goto out_reset;
	}

	if (next_page == head_page) {
		if (!(buffer->flags & RB_FL_OVERWRITE))
			goto out_reset;

		/* tail_page has not moved yet? */
		if (tail_page == cpu_buffer->tail_page) {
			/* count overflows */
			cpu_buffer->overrun +=
				local_read(&head_page->entries);

			rb_inc_page(cpu_buffer, &head_page);
			cpu_buffer->head_page = head_page;
			cpu_buffer->head_page->read = 0;
		}
	}

	/*
	 * If the tail page is still the same as what we think
	 * it is, then it is up to us to update the tail
	 * pointer.
	 */
	if (tail_page == cpu_buffer->tail_page) {
		local_set(&next_page->write, 0);
		local_set(&next_page->entries, 0);
		local_set(&next_page->page->commit, 0);
		cpu_buffer->tail_page = next_page;

		/* reread the time stamp */
1282
		*ts = rb_time_stamp(buffer, cpu_buffer->cpu);
1283 1284 1285
		cpu_buffer->tail_page->page->time_stamp = *ts;
	}

1286
	rb_reset_tail(cpu_buffer, tail_page, tail, length);
1287 1288 1289 1290 1291 1292 1293

	__raw_spin_unlock(&cpu_buffer->lock);
	local_irq_restore(flags);

	/* fail and let the caller try again */
	return ERR_PTR(-EAGAIN);

1294
 out_reset:
1295
	/* reset write */
1296
	rb_reset_tail(cpu_buffer, tail_page, tail, length);
1297

1298 1299
	if (likely(lock_taken))
		__raw_spin_unlock(&cpu_buffer->lock);
1300
	local_irq_restore(flags);
S
Steven Rostedt 已提交
1301
	return NULL;
S
Steven Rostedt 已提交
1302 1303
}

1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
static struct ring_buffer_event *
__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
		  unsigned type, unsigned long length, u64 *ts)
{
	struct buffer_page *tail_page, *commit_page;
	struct ring_buffer_event *event;
	unsigned long tail, write;

	commit_page = cpu_buffer->commit_page;
	/* we just need to protect against interrupts */
	barrier();
	tail_page = cpu_buffer->tail_page;
	write = local_add_return(length, &tail_page->write);
	tail = write - length;

	/* See if we shot pass the end of this buffer page */
	if (write > BUF_PAGE_SIZE)
		return rb_move_tail(cpu_buffer, length, tail,
				    commit_page, tail_page, ts);

	/* We reserved something on the buffer */

	event = __rb_page_index(tail_page, tail);
	rb_update_event(event, type, length);

	/* The passed in type is zero for DATA */
	if (likely(!type))
		local_inc(&tail_page->entries);

	/*
1334 1335
	 * If this is the first commit on the page, then update
	 * its timestamp.
1336
	 */
1337 1338
	if (!tail)
		tail_page->page->time_stamp = *ts;
1339 1340 1341 1342

	return event;
}

1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
static inline int
rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
		  struct ring_buffer_event *event)
{
	unsigned long new_index, old_index;
	struct buffer_page *bpage;
	unsigned long index;
	unsigned long addr;

	new_index = rb_event_index(event);
	old_index = new_index + rb_event_length(event);
	addr = (unsigned long)event;
	addr &= PAGE_MASK;

	bpage = cpu_buffer->tail_page;

	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
		/*
		 * This is on the tail page. It is possible that
		 * a write could come in and move the tail page
		 * and write to the next page. That is fine
		 * because we just shorten what is on this page.
		 */
		index = local_cmpxchg(&bpage->write, old_index, new_index);
		if (index == old_index)
			return 1;
	}

	/* could not discard */
	return 0;
}

S
Steven Rostedt 已提交
1375 1376 1377 1378 1379 1380
static int
rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
		  u64 *ts, u64 *delta)
{
	struct ring_buffer_event *event;
	static int once;
S
Steven Rostedt 已提交
1381
	int ret;
S
Steven Rostedt 已提交
1382 1383 1384 1385

	if (unlikely(*delta > (1ULL << 59) && !once++)) {
		printk(KERN_WARNING "Delta way too big! %llu"
		       " ts=%llu write stamp = %llu\n",
1386 1387 1388
		       (unsigned long long)*delta,
		       (unsigned long long)*ts,
		       (unsigned long long)cpu_buffer->write_stamp);
S
Steven Rostedt 已提交
1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
		WARN_ON(1);
	}

	/*
	 * The delta is too big, we to add a
	 * new timestamp.
	 */
	event = __rb_reserve_next(cpu_buffer,
				  RINGBUF_TYPE_TIME_EXTEND,
				  RB_LEN_TIME_EXTEND,
				  ts);
	if (!event)
S
Steven Rostedt 已提交
1401
		return -EBUSY;
S
Steven Rostedt 已提交
1402

S
Steven Rostedt 已提交
1403 1404 1405 1406
	if (PTR_ERR(event) == -EAGAIN)
		return -EAGAIN;

	/* Only a commited time event can update the write stamp */
1407
	if (rb_event_is_commit(cpu_buffer, event)) {
S
Steven Rostedt 已提交
1408
		/*
1409 1410 1411
		 * If this is the first on the page, then it was
		 * updated with the page itself. Try to discard it
		 * and if we can't just make it zero.
S
Steven Rostedt 已提交
1412 1413 1414 1415 1416
		 */
		if (rb_event_index(event)) {
			event->time_delta = *delta & TS_MASK;
			event->array[0] = *delta >> TS_SHIFT;
		} else {
1417 1418 1419 1420 1421 1422
			/* try to discard, since we do not need this */
			if (!rb_try_to_discard(cpu_buffer, event)) {
				/* nope, just zero it */
				event->time_delta = 0;
				event->array[0] = 0;
			}
S
Steven Rostedt 已提交
1423
		}
S
Steven Rostedt 已提交
1424
		cpu_buffer->write_stamp = *ts;
S
Steven Rostedt 已提交
1425 1426 1427
		/* let the caller know this was the commit */
		ret = 1;
	} else {
1428 1429 1430 1431 1432 1433
		/* Try to discard the event */
		if (!rb_try_to_discard(cpu_buffer, event)) {
			/* Darn, this is just wasted space */
			event->time_delta = 0;
			event->array[0] = 0;
		}
1434
		ret = 0;
S
Steven Rostedt 已提交
1435 1436
	}

S
Steven Rostedt 已提交
1437 1438 1439
	*delta = 0;

	return ret;
S
Steven Rostedt 已提交
1440 1441
}

1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479
static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
{
	local_inc(&cpu_buffer->committing);
	local_inc(&cpu_buffer->commits);
}

static void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
{
	unsigned long commits;

	if (RB_WARN_ON(cpu_buffer,
		       !local_read(&cpu_buffer->committing)))
		return;

 again:
	commits = local_read(&cpu_buffer->commits);
	/* synchronize with interrupts */
	barrier();
	if (local_read(&cpu_buffer->committing) == 1)
		rb_set_commit_to_write(cpu_buffer);

	local_dec(&cpu_buffer->committing);

	/* synchronize with interrupts */
	barrier();

	/*
	 * Need to account for interrupts coming in between the
	 * updating of the commit page and the clearing of the
	 * committing counter.
	 */
	if (unlikely(local_read(&cpu_buffer->commits) != commits) &&
	    !local_read(&cpu_buffer->committing)) {
		local_inc(&cpu_buffer->committing);
		goto again;
	}
}

S
Steven Rostedt 已提交
1480 1481
static struct ring_buffer_event *
rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
1482
		      unsigned long length)
S
Steven Rostedt 已提交
1483 1484
{
	struct ring_buffer_event *event;
1485
	u64 ts, delta = 0;
S
Steven Rostedt 已提交
1486
	int commit = 0;
1487
	int nr_loops = 0;
S
Steven Rostedt 已提交
1488

1489 1490
	rb_start_commit(cpu_buffer);

1491
	length = rb_calculate_event_length(length);
S
Steven Rostedt 已提交
1492
 again:
1493 1494 1495 1496 1497 1498 1499 1500 1501
	/*
	 * We allow for interrupts to reenter here and do a trace.
	 * If one does, it will cause this original code to loop
	 * back here. Even with heavy interrupts happening, this
	 * should only happen a few times in a row. If this happens
	 * 1000 times in a row, there must be either an interrupt
	 * storm or we have something buggy.
	 * Bail!
	 */
S
Steven Rostedt 已提交
1502
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
1503
		goto out_fail;
1504

1505
	ts = rb_time_stamp(cpu_buffer->buffer, cpu_buffer->cpu);
S
Steven Rostedt 已提交
1506

S
Steven Rostedt 已提交
1507 1508 1509 1510 1511 1512 1513 1514
	/*
	 * Only the first commit can update the timestamp.
	 * Yes there is a race here. If an interrupt comes in
	 * just after the conditional and it traces too, then it
	 * will also check the deltas. More than one timestamp may
	 * also be made. But only the entry that did the actual
	 * commit will be something other than zero.
	 */
1515 1516 1517
	if (likely(cpu_buffer->tail_page == cpu_buffer->commit_page &&
		   rb_page_write(cpu_buffer->tail_page) ==
		   rb_commit_index(cpu_buffer))) {
1518
		u64 diff;
S
Steven Rostedt 已提交
1519

1520
		diff = ts - cpu_buffer->write_stamp;
S
Steven Rostedt 已提交
1521

1522
		/* make sure this diff is calculated here */
S
Steven Rostedt 已提交
1523 1524 1525 1526
		barrier();

		/* Did the write stamp get updated already? */
		if (unlikely(ts < cpu_buffer->write_stamp))
1527
			goto get_event;
S
Steven Rostedt 已提交
1528

1529 1530
		delta = diff;
		if (unlikely(test_time_stamp(delta))) {
S
Steven Rostedt 已提交
1531

S
Steven Rostedt 已提交
1532 1533
			commit = rb_add_time_stamp(cpu_buffer, &ts, &delta);
			if (commit == -EBUSY)
1534
				goto out_fail;
S
Steven Rostedt 已提交
1535 1536 1537 1538 1539

			if (commit == -EAGAIN)
				goto again;

			RB_WARN_ON(cpu_buffer, commit < 0);
S
Steven Rostedt 已提交
1540
		}
1541
	}
S
Steven Rostedt 已提交
1542

1543
 get_event:
1544
	event = __rb_reserve_next(cpu_buffer, 0, length, &ts);
1545
	if (unlikely(PTR_ERR(event) == -EAGAIN))
S
Steven Rostedt 已提交
1546 1547
		goto again;

1548 1549
	if (!event)
		goto out_fail;
S
Steven Rostedt 已提交
1550

1551
	if (!rb_event_is_commit(cpu_buffer, event))
S
Steven Rostedt 已提交
1552 1553 1554 1555 1556
		delta = 0;

	event->time_delta = delta;

	return event;
1557 1558 1559 1560

 out_fail:
	rb_end_commit(cpu_buffer);
	return NULL;
S
Steven Rostedt 已提交
1561 1562
}

1563
#define TRACE_RECURSIVE_DEPTH 16
1564 1565 1566

static int trace_recursive_lock(void)
{
1567
	current->trace_recursion++;
1568

1569 1570
	if (likely(current->trace_recursion < TRACE_RECURSIVE_DEPTH))
		return 0;
1571

1572 1573
	/* Disable all tracing before we do anything else */
	tracing_off_permanent();
1574

S
Steven Rostedt 已提交
1575
	printk_once(KERN_WARNING "Tracing recursion: depth[%ld]:"
1576 1577 1578 1579 1580
		    "HC[%lu]:SC[%lu]:NMI[%lu]\n",
		    current->trace_recursion,
		    hardirq_count() >> HARDIRQ_SHIFT,
		    softirq_count() >> SOFTIRQ_SHIFT,
		    in_nmi());
1581

1582 1583
	WARN_ON_ONCE(1);
	return -1;
1584 1585 1586 1587
}

static void trace_recursive_unlock(void)
{
1588
	WARN_ON_ONCE(!current->trace_recursion);
1589

1590
	current->trace_recursion--;
1591 1592
}

S
Steven Rostedt 已提交
1593 1594
static DEFINE_PER_CPU(int, rb_need_resched);

S
Steven Rostedt 已提交
1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610
/**
 * ring_buffer_lock_reserve - reserve a part of the buffer
 * @buffer: the ring buffer to reserve from
 * @length: the length of the data to reserve (excluding event header)
 *
 * Returns a reseverd event on the ring buffer to copy directly to.
 * The user of this interface will need to get the body to write into
 * and can use the ring_buffer_event_data() interface.
 *
 * The length is the length of the data needed, not the event length
 * which also includes the event header.
 *
 * Must be paired with ring_buffer_unlock_commit, unless NULL is returned.
 * If NULL is returned, then nothing has been allocated or locked.
 */
struct ring_buffer_event *
1611
ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
S
Steven Rostedt 已提交
1612 1613 1614
{
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
1615
	int cpu, resched;
S
Steven Rostedt 已提交
1616

1617
	if (ring_buffer_flags != RB_BUFFERS_ON)
1618 1619
		return NULL;

S
Steven Rostedt 已提交
1620 1621 1622
	if (atomic_read(&buffer->record_disabled))
		return NULL;

S
Steven Rostedt 已提交
1623
	/* If we are tracing schedule, we don't want to recurse */
1624
	resched = ftrace_preempt_disable();
S
Steven Rostedt 已提交
1625

1626 1627 1628
	if (trace_recursive_lock())
		goto out_nocheck;

S
Steven Rostedt 已提交
1629 1630
	cpu = raw_smp_processor_id();

1631
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1632
		goto out;
S
Steven Rostedt 已提交
1633 1634 1635 1636

	cpu_buffer = buffer->buffers[cpu];

	if (atomic_read(&cpu_buffer->record_disabled))
1637
		goto out;
S
Steven Rostedt 已提交
1638

1639
	if (length > BUF_MAX_DATA_SIZE)
S
Steven Rostedt 已提交
1640
		goto out;
S
Steven Rostedt 已提交
1641

1642
	event = rb_reserve_next_event(cpu_buffer, length);
S
Steven Rostedt 已提交
1643
	if (!event)
1644
		goto out;
S
Steven Rostedt 已提交
1645

S
Steven Rostedt 已提交
1646 1647 1648 1649 1650 1651 1652 1653
	/*
	 * Need to store resched state on this cpu.
	 * Only the first needs to.
	 */

	if (preempt_count() == 1)
		per_cpu(rb_need_resched, cpu) = resched;

S
Steven Rostedt 已提交
1654 1655
	return event;

1656
 out:
1657 1658 1659
	trace_recursive_unlock();

 out_nocheck:
1660
	ftrace_preempt_enable(resched);
S
Steven Rostedt 已提交
1661 1662
	return NULL;
}
1663
EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
S
Steven Rostedt 已提交
1664 1665 1666 1667

static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
		      struct ring_buffer_event *event)
{
1668
	local_inc(&cpu_buffer->entries);
S
Steven Rostedt 已提交
1669

1670 1671 1672 1673 1674 1675
	/*
	 * The event first in the commit queue updates the
	 * time stamp.
	 */
	if (rb_event_is_commit(cpu_buffer, event))
		cpu_buffer->write_stamp += event->time_delta;
S
Steven Rostedt 已提交
1676

1677
	rb_end_commit(cpu_buffer);
S
Steven Rostedt 已提交
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
}

/**
 * ring_buffer_unlock_commit - commit a reserved
 * @buffer: The buffer to commit to
 * @event: The event pointer to commit.
 *
 * This commits the data to the ring buffer, and releases any locks held.
 *
 * Must be paired with ring_buffer_lock_reserve.
 */
int ring_buffer_unlock_commit(struct ring_buffer *buffer,
1690
			      struct ring_buffer_event *event)
S
Steven Rostedt 已提交
1691 1692 1693 1694 1695 1696 1697 1698
{
	struct ring_buffer_per_cpu *cpu_buffer;
	int cpu = raw_smp_processor_id();

	cpu_buffer = buffer->buffers[cpu];

	rb_commit(cpu_buffer, event);

1699 1700
	trace_recursive_unlock();

S
Steven Rostedt 已提交
1701 1702 1703
	/*
	 * Only the last preempt count needs to restore preemption.
	 */
1704 1705 1706
	if (preempt_count() == 1)
		ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
	else
S
Steven Rostedt 已提交
1707
		preempt_enable_no_resched_notrace();
S
Steven Rostedt 已提交
1708 1709 1710

	return 0;
}
1711
EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
S
Steven Rostedt 已提交
1712

1713 1714
static inline void rb_event_discard(struct ring_buffer_event *event)
{
1715 1716 1717
	/* array[0] holds the actual length for the discarded event */
	event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;
	event->type_len = RINGBUF_TYPE_PADDING;
1718 1719 1720 1721 1722
	/* time delta must be non zero */
	if (!event->time_delta)
		event->time_delta = 1;
}

1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
/**
 * ring_buffer_event_discard - discard any event in the ring buffer
 * @event: the event to discard
 *
 * Sometimes a event that is in the ring buffer needs to be ignored.
 * This function lets the user discard an event in the ring buffer
 * and then that event will not be read later.
 *
 * Note, it is up to the user to be careful with this, and protect
 * against races. If the user discards an event that has been consumed
 * it is possible that it could corrupt the ring buffer.
 */
void ring_buffer_event_discard(struct ring_buffer_event *event)
{
1737
	rb_event_discard(event);
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763
}
EXPORT_SYMBOL_GPL(ring_buffer_event_discard);

/**
 * ring_buffer_commit_discard - discard an event that has not been committed
 * @buffer: the ring buffer
 * @event: non committed event to discard
 *
 * This is similar to ring_buffer_event_discard but must only be
 * performed on an event that has not been committed yet. The difference
 * is that this will also try to free the event from the ring buffer
 * if another event has not been added behind it.
 *
 * If another event has been added behind it, it will set the event
 * up as discarded, and perform the commit.
 *
 * If this function is called, do not call ring_buffer_unlock_commit on
 * the event.
 */
void ring_buffer_discard_commit(struct ring_buffer *buffer,
				struct ring_buffer_event *event)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	int cpu;

	/* The event is discarded regardless */
1764
	rb_event_discard(event);
1765

1766 1767 1768
	cpu = smp_processor_id();
	cpu_buffer = buffer->buffers[cpu];

1769 1770 1771 1772 1773
	/*
	 * This must only be called if the event has not been
	 * committed yet. Thus we can assume that preemption
	 * is still disabled.
	 */
1774
	RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));
1775

1776 1777
	if (!rb_try_to_discard(cpu_buffer, event))
		goto out;
1778 1779 1780 1781 1782

	/*
	 * The commit is still visible by the reader, so we
	 * must increment entries.
	 */
1783
	local_inc(&cpu_buffer->entries);
1784
 out:
1785
	rb_end_commit(cpu_buffer);
1786

1787 1788
	trace_recursive_unlock();

1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	/*
	 * Only the last preempt count needs to restore preemption.
	 */
	if (preempt_count() == 1)
		ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
	else
		preempt_enable_no_resched_notrace();

}
EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);

S
Steven Rostedt 已提交
1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820
/**
 * ring_buffer_write - write data to the buffer without reserving
 * @buffer: The ring buffer to write to.
 * @length: The length of the data being written (excluding the event header)
 * @data: The data to write to the buffer.
 *
 * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as
 * one function. If you already have the data to write to the buffer, it
 * may be easier to simply call this function.
 *
 * Note, like ring_buffer_lock_reserve, the length is the length of the data
 * and not the length of the event which would hold the header.
 */
int ring_buffer_write(struct ring_buffer *buffer,
			unsigned long length,
			void *data)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
	void *body;
	int ret = -EBUSY;
S
Steven Rostedt 已提交
1821
	int cpu, resched;
S
Steven Rostedt 已提交
1822

1823
	if (ring_buffer_flags != RB_BUFFERS_ON)
1824 1825
		return -EBUSY;

S
Steven Rostedt 已提交
1826 1827 1828
	if (atomic_read(&buffer->record_disabled))
		return -EBUSY;

1829
	resched = ftrace_preempt_disable();
S
Steven Rostedt 已提交
1830

S
Steven Rostedt 已提交
1831 1832
	cpu = raw_smp_processor_id();

1833
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1834
		goto out;
S
Steven Rostedt 已提交
1835 1836 1837 1838 1839 1840

	cpu_buffer = buffer->buffers[cpu];

	if (atomic_read(&cpu_buffer->record_disabled))
		goto out;

1841 1842 1843 1844
	if (length > BUF_MAX_DATA_SIZE)
		goto out;

	event = rb_reserve_next_event(cpu_buffer, length);
S
Steven Rostedt 已提交
1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855
	if (!event)
		goto out;

	body = rb_event_data(event);

	memcpy(body, data, length);

	rb_commit(cpu_buffer, event);

	ret = 0;
 out:
1856
	ftrace_preempt_enable(resched);
S
Steven Rostedt 已提交
1857 1858 1859

	return ret;
}
1860
EXPORT_SYMBOL_GPL(ring_buffer_write);
S
Steven Rostedt 已提交
1861

1862
static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873
{
	struct buffer_page *reader = cpu_buffer->reader_page;
	struct buffer_page *head = cpu_buffer->head_page;
	struct buffer_page *commit = cpu_buffer->commit_page;

	return reader->read == rb_page_commit(reader) &&
		(commit == reader ||
		 (commit == head &&
		  head->read == rb_page_commit(commit)));
}

S
Steven Rostedt 已提交
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
/**
 * ring_buffer_record_disable - stop all writes into the buffer
 * @buffer: The ring buffer to stop writes to.
 *
 * This prevents all writes to the buffer. Any attempt to write
 * to the buffer after this will fail and return NULL.
 *
 * The caller should call synchronize_sched() after this.
 */
void ring_buffer_record_disable(struct ring_buffer *buffer)
{
	atomic_inc(&buffer->record_disabled);
}
1887
EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
S
Steven Rostedt 已提交
1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899

/**
 * ring_buffer_record_enable - enable writes to the buffer
 * @buffer: The ring buffer to enable writes
 *
 * Note, multiple disables will need the same number of enables
 * to truely enable the writing (much like preempt_disable).
 */
void ring_buffer_record_enable(struct ring_buffer *buffer)
{
	atomic_dec(&buffer->record_disabled);
}
1900
EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
S
Steven Rostedt 已提交
1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915

/**
 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
 * @buffer: The ring buffer to stop writes to.
 * @cpu: The CPU buffer to stop
 *
 * This prevents all writes to the buffer. Any attempt to write
 * to the buffer after this will fail and return NULL.
 *
 * The caller should call synchronize_sched() after this.
 */
void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;

1916
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1917
		return;
S
Steven Rostedt 已提交
1918 1919 1920 1921

	cpu_buffer = buffer->buffers[cpu];
	atomic_inc(&cpu_buffer->record_disabled);
}
1922
EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
S
Steven Rostedt 已提交
1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935

/**
 * ring_buffer_record_enable_cpu - enable writes to the buffer
 * @buffer: The ring buffer to enable writes
 * @cpu: The CPU to enable.
 *
 * Note, multiple disables will need the same number of enables
 * to truely enable the writing (much like preempt_disable).
 */
void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;

1936
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1937
		return;
S
Steven Rostedt 已提交
1938 1939 1940 1941

	cpu_buffer = buffer->buffers[cpu];
	atomic_dec(&cpu_buffer->record_disabled);
}
1942
EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
S
Steven Rostedt 已提交
1943 1944 1945 1946 1947 1948 1949 1950 1951

/**
 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the entries from.
 */
unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
1952
	unsigned long ret;
S
Steven Rostedt 已提交
1953

1954
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1955
		return 0;
S
Steven Rostedt 已提交
1956 1957

	cpu_buffer = buffer->buffers[cpu];
1958 1959
	ret = (local_read(&cpu_buffer->entries) - cpu_buffer->overrun)
		- cpu_buffer->read;
1960 1961

	return ret;
S
Steven Rostedt 已提交
1962
}
1963
EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
S
Steven Rostedt 已提交
1964 1965 1966 1967 1968 1969 1970 1971 1972

/**
 * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
1973
	unsigned long ret;
S
Steven Rostedt 已提交
1974

1975
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1976
		return 0;
S
Steven Rostedt 已提交
1977 1978

	cpu_buffer = buffer->buffers[cpu];
1979 1980 1981
	ret = cpu_buffer->overrun;

	return ret;
S
Steven Rostedt 已提交
1982
}
1983
EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
S
Steven Rostedt 已提交
1984

1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
/**
 * ring_buffer_nmi_dropped_cpu - get the number of nmis that were dropped
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
unsigned long ring_buffer_nmi_dropped_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long ret;

	if (!cpumask_test_cpu(cpu, buffer->cpumask))
		return 0;

	cpu_buffer = buffer->buffers[cpu];
	ret = cpu_buffer->nmi_dropped;

	return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_nmi_dropped_cpu);

/**
 * ring_buffer_commit_overrun_cpu - get the number of overruns caused by commits
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
unsigned long
ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long ret;

	if (!cpumask_test_cpu(cpu, buffer->cpumask))
		return 0;

	cpu_buffer = buffer->buffers[cpu];
	ret = cpu_buffer->commit_overrun;

	return ret;
}
EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu);

S
Steven Rostedt 已提交
2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041
/**
 * ring_buffer_entries - get the number of entries in a buffer
 * @buffer: The ring buffer
 *
 * Returns the total number of entries in the ring buffer
 * (all CPU entries)
 */
unsigned long ring_buffer_entries(struct ring_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long entries = 0;
	int cpu;

	/* if you care about this being correct, lock the buffer */
	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
2042 2043
		entries += (local_read(&cpu_buffer->entries) -
			    cpu_buffer->overrun) - cpu_buffer->read;
S
Steven Rostedt 已提交
2044 2045 2046 2047
	}

	return entries;
}
2048
EXPORT_SYMBOL_GPL(ring_buffer_entries);
S
Steven Rostedt 已提交
2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070

/**
 * ring_buffer_overrun_cpu - get the number of overruns in buffer
 * @buffer: The ring buffer
 *
 * Returns the total number of overruns in the ring buffer
 * (all CPU entries)
 */
unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long overruns = 0;
	int cpu;

	/* if you care about this being correct, lock the buffer */
	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
		overruns += cpu_buffer->overrun;
	}

	return overruns;
}
2071
EXPORT_SYMBOL_GPL(ring_buffer_overruns);
S
Steven Rostedt 已提交
2072

2073
static void rb_iter_reset(struct ring_buffer_iter *iter)
S
Steven Rostedt 已提交
2074 2075 2076
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

2077 2078 2079
	/* Iterator usage is expected to have record disabled */
	if (list_empty(&cpu_buffer->reader_page->list)) {
		iter->head_page = cpu_buffer->head_page;
2080
		iter->head = cpu_buffer->head_page->read;
2081 2082
	} else {
		iter->head_page = cpu_buffer->reader_page;
2083
		iter->head = cpu_buffer->reader_page->read;
2084 2085 2086 2087
	}
	if (iter->head)
		iter->read_stamp = cpu_buffer->read_stamp;
	else
2088
		iter->read_stamp = iter->head_page->page->time_stamp;
2089
}
S
Steven Rostedt 已提交
2090

2091 2092 2093 2094 2095 2096 2097 2098 2099
/**
 * ring_buffer_iter_reset - reset an iterator
 * @iter: The iterator to reset
 *
 * Resets the iterator, so that it will start from the beginning
 * again.
 */
void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
{
2100
	struct ring_buffer_per_cpu *cpu_buffer;
2101 2102
	unsigned long flags;

2103 2104 2105 2106 2107
	if (!iter)
		return;

	cpu_buffer = iter->cpu_buffer;

2108 2109
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	rb_iter_reset(iter);
S
Steven Rostedt 已提交
2110
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
S
Steven Rostedt 已提交
2111
}
2112
EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
S
Steven Rostedt 已提交
2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123

/**
 * ring_buffer_iter_empty - check if an iterator has no more to read
 * @iter: The iterator to check
 */
int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
{
	struct ring_buffer_per_cpu *cpu_buffer;

	cpu_buffer = iter->cpu_buffer;

S
Steven Rostedt 已提交
2124 2125
	return iter->head_page == cpu_buffer->commit_page &&
		iter->head == rb_commit_index(cpu_buffer);
S
Steven Rostedt 已提交
2126
}
2127
EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
S
Steven Rostedt 已提交
2128 2129 2130 2131 2132 2133 2134

static void
rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
		     struct ring_buffer_event *event)
{
	u64 delta;

2135
	switch (event->type_len) {
S
Steven Rostedt 已提交
2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
	case RINGBUF_TYPE_PADDING:
		return;

	case RINGBUF_TYPE_TIME_EXTEND:
		delta = event->array[0];
		delta <<= TS_SHIFT;
		delta += event->time_delta;
		cpu_buffer->read_stamp += delta;
		return;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
		return;

	case RINGBUF_TYPE_DATA:
		cpu_buffer->read_stamp += event->time_delta;
		return;

	default:
		BUG();
	}
	return;
}

static void
rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
			  struct ring_buffer_event *event)
{
	u64 delta;

2166
	switch (event->type_len) {
S
Steven Rostedt 已提交
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
	case RINGBUF_TYPE_PADDING:
		return;

	case RINGBUF_TYPE_TIME_EXTEND:
		delta = event->array[0];
		delta <<= TS_SHIFT;
		delta += event->time_delta;
		iter->read_stamp += delta;
		return;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
		return;

	case RINGBUF_TYPE_DATA:
		iter->read_stamp += event->time_delta;
		return;

	default:
		BUG();
	}
	return;
}

2191 2192
static struct buffer_page *
rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
2193
{
2194 2195
	struct buffer_page *reader = NULL;
	unsigned long flags;
2196
	int nr_loops = 0;
2197

2198 2199
	local_irq_save(flags);
	__raw_spin_lock(&cpu_buffer->lock);
2200 2201

 again:
2202 2203 2204 2205 2206 2207
	/*
	 * This should normally only loop twice. But because the
	 * start of the reader inserts an empty page, it causes
	 * a case where we will loop three times. There should be no
	 * reason to loop four times (that I know of).
	 */
S
Steven Rostedt 已提交
2208
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
2209 2210 2211 2212
		reader = NULL;
		goto out;
	}

2213 2214 2215
	reader = cpu_buffer->reader_page;

	/* If there's more to read, return this page */
S
Steven Rostedt 已提交
2216
	if (cpu_buffer->reader_page->read < rb_page_size(reader))
2217 2218 2219
		goto out;

	/* Never should we have an index greater than the size */
S
Steven Rostedt 已提交
2220 2221 2222
	if (RB_WARN_ON(cpu_buffer,
		       cpu_buffer->reader_page->read > rb_page_size(reader)))
		goto out;
2223 2224 2225

	/* check if we caught up to the tail */
	reader = NULL;
S
Steven Rostedt 已提交
2226
	if (cpu_buffer->commit_page == cpu_buffer->reader_page)
2227
		goto out;
S
Steven Rostedt 已提交
2228 2229

	/*
2230 2231
	 * Splice the empty reader page into the list around the head.
	 * Reset the reader page to size zero.
S
Steven Rostedt 已提交
2232 2233
	 */

2234 2235 2236
	reader = cpu_buffer->head_page;
	cpu_buffer->reader_page->list.next = reader->list.next;
	cpu_buffer->reader_page->list.prev = reader->list.prev;
S
Steven Rostedt 已提交
2237 2238

	local_set(&cpu_buffer->reader_page->write, 0);
2239
	local_set(&cpu_buffer->reader_page->entries, 0);
2240
	local_set(&cpu_buffer->reader_page->page->commit, 0);
S
Steven Rostedt 已提交
2241

2242 2243 2244
	/* Make the reader page now replace the head */
	reader->list.prev->next = &cpu_buffer->reader_page->list;
	reader->list.next->prev = &cpu_buffer->reader_page->list;
S
Steven Rostedt 已提交
2245 2246

	/*
2247 2248
	 * If the tail is on the reader, then we must set the head
	 * to the inserted page, otherwise we set it one before.
S
Steven Rostedt 已提交
2249
	 */
2250
	cpu_buffer->head_page = cpu_buffer->reader_page;
S
Steven Rostedt 已提交
2251

S
Steven Rostedt 已提交
2252
	if (cpu_buffer->commit_page != reader)
2253 2254 2255 2256 2257 2258 2259 2260 2261
		rb_inc_page(cpu_buffer, &cpu_buffer->head_page);

	/* Finally update the reader page to the new head */
	cpu_buffer->reader_page = reader;
	rb_reset_reader_page(cpu_buffer);

	goto again;

 out:
2262 2263
	__raw_spin_unlock(&cpu_buffer->lock);
	local_irq_restore(flags);
2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274

	return reader;
}

static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
{
	struct ring_buffer_event *event;
	struct buffer_page *reader;
	unsigned length;

	reader = rb_get_reader_page(cpu_buffer);
S
Steven Rostedt 已提交
2275

2276
	/* This function should not be called when buffer is empty */
S
Steven Rostedt 已提交
2277 2278
	if (RB_WARN_ON(cpu_buffer, !reader))
		return;
S
Steven Rostedt 已提交
2279

2280 2281
	event = rb_reader_event(cpu_buffer);

2282 2283
	if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX
			|| rb_discarded_event(event))
2284
		cpu_buffer->read++;
2285 2286 2287 2288

	rb_update_read_stamp(cpu_buffer, event);

	length = rb_event_length(event);
2289
	cpu_buffer->reader_page->read += length;
S
Steven Rostedt 已提交
2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304
}

static void rb_advance_iter(struct ring_buffer_iter *iter)
{
	struct ring_buffer *buffer;
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
	unsigned length;

	cpu_buffer = iter->cpu_buffer;
	buffer = cpu_buffer->buffer;

	/*
	 * Check if we are at the end of the buffer.
	 */
S
Steven Rostedt 已提交
2305
	if (iter->head >= rb_page_size(iter->head_page)) {
2306 2307
		/* discarded commits can make the page empty */
		if (iter->head_page == cpu_buffer->commit_page)
S
Steven Rostedt 已提交
2308
			return;
2309
		rb_inc_iter(iter);
S
Steven Rostedt 已提交
2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320
		return;
	}

	event = rb_iter_head_event(iter);

	length = rb_event_length(event);

	/*
	 * This should not be called to advance the header if we are
	 * at the tail of the buffer.
	 */
S
Steven Rostedt 已提交
2321
	if (RB_WARN_ON(cpu_buffer,
2322
		       (iter->head_page == cpu_buffer->commit_page) &&
S
Steven Rostedt 已提交
2323 2324
		       (iter->head + length > rb_commit_index(cpu_buffer))))
		return;
S
Steven Rostedt 已提交
2325 2326 2327 2328 2329 2330

	rb_update_iter_read_stamp(iter, event);

	iter->head += length;

	/* check for end of page padding */
S
Steven Rostedt 已提交
2331 2332
	if ((iter->head >= rb_page_size(iter->head_page)) &&
	    (iter->head_page != cpu_buffer->commit_page))
S
Steven Rostedt 已提交
2333 2334 2335
		rb_advance_iter(iter);
}

S
Steven Rostedt 已提交
2336 2337
static struct ring_buffer_event *
rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
S
Steven Rostedt 已提交
2338 2339 2340
{
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
2341
	struct buffer_page *reader;
2342
	int nr_loops = 0;
S
Steven Rostedt 已提交
2343 2344 2345 2346

	cpu_buffer = buffer->buffers[cpu];

 again:
2347 2348 2349
	/*
	 * We repeat when a timestamp is encountered. It is possible
	 * to get multiple timestamps from an interrupt entering just
2350 2351
	 * as one timestamp is about to be written, or from discarded
	 * commits. The most that we can have is the number on a single page.
2352
	 */
2353
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > RB_TIMESTAMPS_PER_PAGE))
2354 2355
		return NULL;

2356 2357
	reader = rb_get_reader_page(cpu_buffer);
	if (!reader)
S
Steven Rostedt 已提交
2358 2359
		return NULL;

2360
	event = rb_reader_event(cpu_buffer);
S
Steven Rostedt 已提交
2361

2362
	switch (event->type_len) {
S
Steven Rostedt 已提交
2363
	case RINGBUF_TYPE_PADDING:
2364 2365 2366 2367 2368 2369 2370 2371 2372 2373
		if (rb_null_event(event))
			RB_WARN_ON(cpu_buffer, 1);
		/*
		 * Because the writer could be discarding every
		 * event it creates (which would probably be bad)
		 * if we were to go back to "again" then we may never
		 * catch up, and will trigger the warn on, or lock
		 * the box. Return the padding, and we will release
		 * the current locks, and try again.
		 */
2374
		rb_advance_reader(cpu_buffer);
2375
		return event;
S
Steven Rostedt 已提交
2376 2377 2378

	case RINGBUF_TYPE_TIME_EXTEND:
		/* Internal data, OK to advance */
2379
		rb_advance_reader(cpu_buffer);
S
Steven Rostedt 已提交
2380 2381 2382 2383
		goto again;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
2384
		rb_advance_reader(cpu_buffer);
S
Steven Rostedt 已提交
2385 2386 2387 2388 2389
		goto again;

	case RINGBUF_TYPE_DATA:
		if (ts) {
			*ts = cpu_buffer->read_stamp + event->time_delta;
2390 2391
			ring_buffer_normalize_time_stamp(buffer,
							 cpu_buffer->cpu, ts);
S
Steven Rostedt 已提交
2392 2393 2394 2395 2396 2397 2398 2399 2400
		}
		return event;

	default:
		BUG();
	}

	return NULL;
}
2401
EXPORT_SYMBOL_GPL(ring_buffer_peek);
S
Steven Rostedt 已提交
2402

S
Steven Rostedt 已提交
2403 2404
static struct ring_buffer_event *
rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
S
Steven Rostedt 已提交
2405 2406 2407 2408
{
	struct ring_buffer *buffer;
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
2409
	int nr_loops = 0;
S
Steven Rostedt 已提交
2410 2411 2412 2413 2414 2415 2416 2417

	if (ring_buffer_iter_empty(iter))
		return NULL;

	cpu_buffer = iter->cpu_buffer;
	buffer = cpu_buffer->buffer;

 again:
2418
	/*
2419 2420 2421 2422 2423 2424
	 * We repeat when a timestamp is encountered.
	 * We can get multiple timestamps by nested interrupts or also
	 * if filtering is on (discarding commits). Since discarding
	 * commits can be frequent we can get a lot of timestamps.
	 * But we limit them by not adding timestamps if they begin
	 * at the start of a page.
2425
	 */
2426
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > RB_TIMESTAMPS_PER_PAGE))
2427 2428
		return NULL;

S
Steven Rostedt 已提交
2429 2430 2431 2432 2433
	if (rb_per_cpu_empty(cpu_buffer))
		return NULL;

	event = rb_iter_head_event(iter);

2434
	switch (event->type_len) {
S
Steven Rostedt 已提交
2435
	case RINGBUF_TYPE_PADDING:
2436 2437 2438 2439 2440 2441
		if (rb_null_event(event)) {
			rb_inc_iter(iter);
			goto again;
		}
		rb_advance_iter(iter);
		return event;
S
Steven Rostedt 已提交
2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455

	case RINGBUF_TYPE_TIME_EXTEND:
		/* Internal data, OK to advance */
		rb_advance_iter(iter);
		goto again;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
		rb_advance_iter(iter);
		goto again;

	case RINGBUF_TYPE_DATA:
		if (ts) {
			*ts = iter->read_stamp + event->time_delta;
2456 2457
			ring_buffer_normalize_time_stamp(buffer,
							 cpu_buffer->cpu, ts);
S
Steven Rostedt 已提交
2458 2459 2460 2461 2462 2463 2464 2465 2466
		}
		return event;

	default:
		BUG();
	}

	return NULL;
}
2467
EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
S
Steven Rostedt 已提交
2468

S
Steven Rostedt 已提交
2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
/**
 * ring_buffer_peek - peek at the next event to be read
 * @buffer: The ring buffer to read
 * @cpu: The cpu to peak at
 * @ts: The timestamp counter of this event.
 *
 * This will return the event that will be read next, but does
 * not consume the data.
 */
struct ring_buffer_event *
ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
{
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
2482
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
2483 2484
	unsigned long flags;

2485
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
2486
		return NULL;
2487

2488
 again:
S
Steven Rostedt 已提交
2489 2490 2491 2492
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	event = rb_buffer_peek(buffer, cpu, ts);
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

2493
	if (event && event->type_len == RINGBUF_TYPE_PADDING) {
2494 2495 2496 2497
		cpu_relax();
		goto again;
	}

S
Steven Rostedt 已提交
2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
	return event;
}

/**
 * ring_buffer_iter_peek - peek at the next event to be read
 * @iter: The ring buffer iterator
 * @ts: The timestamp counter of this event.
 *
 * This will return the event that will be read next, but does
 * not increment the iterator.
 */
struct ring_buffer_event *
ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
	struct ring_buffer_event *event;
	unsigned long flags;

2516
 again:
S
Steven Rostedt 已提交
2517 2518 2519 2520
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	event = rb_iter_peek(iter, ts);
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

2521
	if (event && event->type_len == RINGBUF_TYPE_PADDING) {
2522 2523 2524 2525
		cpu_relax();
		goto again;
	}

S
Steven Rostedt 已提交
2526 2527 2528
	return event;
}

S
Steven Rostedt 已提交
2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539
/**
 * ring_buffer_consume - return an event and consume it
 * @buffer: The ring buffer to get the next event from
 *
 * Returns the next event in the ring buffer, and that event is consumed.
 * Meaning, that sequential reads will keep returning a different event,
 * and eventually empty the ring buffer if the producer is slower.
 */
struct ring_buffer_event *
ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
{
2540 2541
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event = NULL;
S
Steven Rostedt 已提交
2542
	unsigned long flags;
S
Steven Rostedt 已提交
2543

2544
 again:
2545 2546 2547
	/* might be called in atomic */
	preempt_disable();

2548
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
2549
		goto out;
S
Steven Rostedt 已提交
2550

2551
	cpu_buffer = buffer->buffers[cpu];
S
Steven Rostedt 已提交
2552 2553 2554
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

	event = rb_buffer_peek(buffer, cpu, ts);
S
Steven Rostedt 已提交
2555
	if (!event)
2556
		goto out_unlock;
S
Steven Rostedt 已提交
2557

2558
	rb_advance_reader(cpu_buffer);
S
Steven Rostedt 已提交
2559

2560
 out_unlock:
S
Steven Rostedt 已提交
2561 2562
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

2563 2564 2565
 out:
	preempt_enable();

2566
	if (event && event->type_len == RINGBUF_TYPE_PADDING) {
2567 2568 2569 2570
		cpu_relax();
		goto again;
	}

S
Steven Rostedt 已提交
2571 2572
	return event;
}
2573
EXPORT_SYMBOL_GPL(ring_buffer_consume);
S
Steven Rostedt 已提交
2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590

/**
 * ring_buffer_read_start - start a non consuming read of the buffer
 * @buffer: The ring buffer to read from
 * @cpu: The cpu buffer to iterate over
 *
 * This starts up an iteration through the buffer. It also disables
 * the recording to the buffer until the reading is finished.
 * This prevents the reading from being corrupted. This is not
 * a consuming read, so a producer is not expected.
 *
 * Must be paired with ring_buffer_finish.
 */
struct ring_buffer_iter *
ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
2591
	struct ring_buffer_iter *iter;
2592
	unsigned long flags;
S
Steven Rostedt 已提交
2593

2594
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
2595
		return NULL;
S
Steven Rostedt 已提交
2596 2597 2598

	iter = kmalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
2599
		return NULL;
S
Steven Rostedt 已提交
2600 2601 2602 2603 2604 2605 2606 2607

	cpu_buffer = buffer->buffers[cpu];

	iter->cpu_buffer = cpu_buffer;

	atomic_inc(&cpu_buffer->record_disabled);
	synchronize_sched();

S
Steven Rostedt 已提交
2608
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2609
	__raw_spin_lock(&cpu_buffer->lock);
2610
	rb_iter_reset(iter);
2611
	__raw_spin_unlock(&cpu_buffer->lock);
S
Steven Rostedt 已提交
2612
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
S
Steven Rostedt 已提交
2613 2614 2615

	return iter;
}
2616
EXPORT_SYMBOL_GPL(ring_buffer_read_start);
S
Steven Rostedt 已提交
2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632

/**
 * ring_buffer_finish - finish reading the iterator of the buffer
 * @iter: The iterator retrieved by ring_buffer_start
 *
 * This re-enables the recording to the buffer, and frees the
 * iterator.
 */
void
ring_buffer_read_finish(struct ring_buffer_iter *iter)
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

	atomic_dec(&cpu_buffer->record_disabled);
	kfree(iter);
}
2633
EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
S
Steven Rostedt 已提交
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645

/**
 * ring_buffer_read - read the next item in the ring buffer by the iterator
 * @iter: The ring buffer iterator
 * @ts: The time stamp of the event read.
 *
 * This reads the next event in the ring buffer and increments the iterator.
 */
struct ring_buffer_event *
ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
{
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
2646 2647
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
	unsigned long flags;
S
Steven Rostedt 已提交
2648

2649
 again:
S
Steven Rostedt 已提交
2650 2651
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	event = rb_iter_peek(iter, ts);
S
Steven Rostedt 已提交
2652
	if (!event)
S
Steven Rostedt 已提交
2653
		goto out;
S
Steven Rostedt 已提交
2654 2655

	rb_advance_iter(iter);
S
Steven Rostedt 已提交
2656 2657
 out:
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
S
Steven Rostedt 已提交
2658

2659
	if (event && event->type_len == RINGBUF_TYPE_PADDING) {
2660 2661 2662 2663
		cpu_relax();
		goto again;
	}

S
Steven Rostedt 已提交
2664 2665
	return event;
}
2666
EXPORT_SYMBOL_GPL(ring_buffer_read);
S
Steven Rostedt 已提交
2667 2668 2669 2670 2671 2672 2673 2674 2675

/**
 * ring_buffer_size - return the size of the ring buffer (in bytes)
 * @buffer: The ring buffer.
 */
unsigned long ring_buffer_size(struct ring_buffer *buffer)
{
	return BUF_PAGE_SIZE * buffer->pages;
}
2676
EXPORT_SYMBOL_GPL(ring_buffer_size);
S
Steven Rostedt 已提交
2677 2678 2679 2680 2681 2682

static void
rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
{
	cpu_buffer->head_page
		= list_entry(cpu_buffer->pages.next, struct buffer_page, list);
S
Steven Rostedt 已提交
2683
	local_set(&cpu_buffer->head_page->write, 0);
2684
	local_set(&cpu_buffer->head_page->entries, 0);
2685
	local_set(&cpu_buffer->head_page->page->commit, 0);
2686

2687
	cpu_buffer->head_page->read = 0;
S
Steven Rostedt 已提交
2688 2689 2690 2691 2692 2693

	cpu_buffer->tail_page = cpu_buffer->head_page;
	cpu_buffer->commit_page = cpu_buffer->head_page;

	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
	local_set(&cpu_buffer->reader_page->write, 0);
2694
	local_set(&cpu_buffer->reader_page->entries, 0);
2695
	local_set(&cpu_buffer->reader_page->page->commit, 0);
2696
	cpu_buffer->reader_page->read = 0;
S
Steven Rostedt 已提交
2697

2698 2699
	cpu_buffer->nmi_dropped = 0;
	cpu_buffer->commit_overrun = 0;
S
Steven Rostedt 已提交
2700
	cpu_buffer->overrun = 0;
2701 2702
	cpu_buffer->read = 0;
	local_set(&cpu_buffer->entries, 0);
2703 2704
	local_set(&cpu_buffer->committing, 0);
	local_set(&cpu_buffer->commits, 0);
2705 2706 2707

	cpu_buffer->write_stamp = 0;
	cpu_buffer->read_stamp = 0;
S
Steven Rostedt 已提交
2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719
}

/**
 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
 * @buffer: The ring buffer to reset a per cpu buffer of
 * @cpu: The CPU buffer to be reset
 */
void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	unsigned long flags;

2720
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
2721
		return;
S
Steven Rostedt 已提交
2722

2723 2724
	atomic_inc(&cpu_buffer->record_disabled);

S
Steven Rostedt 已提交
2725 2726
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

2727
	__raw_spin_lock(&cpu_buffer->lock);
S
Steven Rostedt 已提交
2728 2729 2730

	rb_reset_cpu(cpu_buffer);

2731
	__raw_spin_unlock(&cpu_buffer->lock);
S
Steven Rostedt 已提交
2732 2733

	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2734 2735

	atomic_dec(&cpu_buffer->record_disabled);
S
Steven Rostedt 已提交
2736
}
2737
EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
S
Steven Rostedt 已提交
2738 2739 2740 2741 2742 2743 2744 2745 2746 2747

/**
 * ring_buffer_reset - reset a ring buffer
 * @buffer: The ring buffer to reset all cpu buffers
 */
void ring_buffer_reset(struct ring_buffer *buffer)
{
	int cpu;

	for_each_buffer_cpu(buffer, cpu)
2748
		ring_buffer_reset_cpu(buffer, cpu);
S
Steven Rostedt 已提交
2749
}
2750
EXPORT_SYMBOL_GPL(ring_buffer_reset);
S
Steven Rostedt 已提交
2751 2752 2753 2754 2755 2756 2757 2758

/**
 * rind_buffer_empty - is the ring buffer empty?
 * @buffer: The ring buffer to test
 */
int ring_buffer_empty(struct ring_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
2759
	unsigned long flags;
S
Steven Rostedt 已提交
2760
	int cpu;
2761
	int ret;
S
Steven Rostedt 已提交
2762 2763 2764 2765

	/* yes this is racy, but if you don't like the race, lock the buffer */
	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
2766 2767 2768 2769
		spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
		ret = rb_per_cpu_empty(cpu_buffer);
		spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
		if (!ret)
S
Steven Rostedt 已提交
2770 2771
			return 0;
	}
2772

S
Steven Rostedt 已提交
2773 2774
	return 1;
}
2775
EXPORT_SYMBOL_GPL(ring_buffer_empty);
S
Steven Rostedt 已提交
2776 2777 2778 2779 2780 2781 2782 2783 2784

/**
 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
 * @buffer: The ring buffer
 * @cpu: The CPU buffer to test
 */
int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
2785
	unsigned long flags;
2786
	int ret;
S
Steven Rostedt 已提交
2787

2788
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
2789
		return 1;
S
Steven Rostedt 已提交
2790 2791

	cpu_buffer = buffer->buffers[cpu];
2792
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2793
	ret = rb_per_cpu_empty(cpu_buffer);
2794
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2795 2796

	return ret;
S
Steven Rostedt 已提交
2797
}
2798
EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
S
Steven Rostedt 已提交
2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814

/**
 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
 * @buffer_a: One buffer to swap with
 * @buffer_b: The other buffer to swap with
 *
 * This function is useful for tracers that want to take a "snapshot"
 * of a CPU buffer and has another back up buffer lying around.
 * it is expected that the tracer handles the cpu buffer not being
 * used at the moment.
 */
int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
			 struct ring_buffer *buffer_b, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer_a;
	struct ring_buffer_per_cpu *cpu_buffer_b;
2815 2816
	int ret = -EINVAL;

2817 2818
	if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
	    !cpumask_test_cpu(cpu, buffer_b->cpumask))
2819
		goto out;
S
Steven Rostedt 已提交
2820 2821

	/* At least make sure the two buffers are somewhat the same */
2822
	if (buffer_a->pages != buffer_b->pages)
2823 2824 2825
		goto out;

	ret = -EAGAIN;
S
Steven Rostedt 已提交
2826

2827
	if (ring_buffer_flags != RB_BUFFERS_ON)
2828
		goto out;
2829 2830

	if (atomic_read(&buffer_a->record_disabled))
2831
		goto out;
2832 2833

	if (atomic_read(&buffer_b->record_disabled))
2834
		goto out;
2835

S
Steven Rostedt 已提交
2836 2837 2838
	cpu_buffer_a = buffer_a->buffers[cpu];
	cpu_buffer_b = buffer_b->buffers[cpu];

2839
	if (atomic_read(&cpu_buffer_a->record_disabled))
2840
		goto out;
2841 2842

	if (atomic_read(&cpu_buffer_b->record_disabled))
2843
		goto out;
2844

S
Steven Rostedt 已提交
2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862
	/*
	 * We can't do a synchronize_sched here because this
	 * function can be called in atomic context.
	 * Normally this will be called from the same CPU as cpu.
	 * If not it's up to the caller to protect this.
	 */
	atomic_inc(&cpu_buffer_a->record_disabled);
	atomic_inc(&cpu_buffer_b->record_disabled);

	buffer_a->buffers[cpu] = cpu_buffer_b;
	buffer_b->buffers[cpu] = cpu_buffer_a;

	cpu_buffer_b->buffer = buffer_a;
	cpu_buffer_a->buffer = buffer_b;

	atomic_dec(&cpu_buffer_a->record_disabled);
	atomic_dec(&cpu_buffer_b->record_disabled);

2863 2864 2865
	ret = 0;
out:
	return ret;
S
Steven Rostedt 已提交
2866
}
2867
EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
S
Steven Rostedt 已提交
2868

S
Steven Rostedt 已提交
2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
/**
 * ring_buffer_alloc_read_page - allocate a page to read from buffer
 * @buffer: the buffer to allocate for.
 *
 * This function is used in conjunction with ring_buffer_read_page.
 * When reading a full page from the ring buffer, these functions
 * can be used to speed up the process. The calling function should
 * allocate a few pages first with this function. Then when it
 * needs to get pages from the ring buffer, it passes the result
 * of this function into ring_buffer_read_page, which will swap
 * the page that was allocated, with the read page of the buffer.
 *
 * Returns:
 *  The page allocated, or NULL on error.
 */
void *ring_buffer_alloc_read_page(struct ring_buffer *buffer)
{
2886
	struct buffer_data_page *bpage;
2887
	unsigned long addr;
S
Steven Rostedt 已提交
2888 2889 2890 2891 2892

	addr = __get_free_page(GFP_KERNEL);
	if (!addr)
		return NULL;

2893
	bpage = (void *)addr;
S
Steven Rostedt 已提交
2894

2895 2896
	rb_init_page(bpage);

2897
	return bpage;
S
Steven Rostedt 已提交
2898
}
S
Steven Rostedt 已提交
2899
EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page);
S
Steven Rostedt 已提交
2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911

/**
 * ring_buffer_free_read_page - free an allocated read page
 * @buffer: the buffer the page was allocate for
 * @data: the page to free
 *
 * Free a page allocated from ring_buffer_alloc_read_page.
 */
void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
{
	free_page((unsigned long)data);
}
S
Steven Rostedt 已提交
2912
EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
S
Steven Rostedt 已提交
2913 2914 2915 2916 2917

/**
 * ring_buffer_read_page - extract a page from the ring buffer
 * @buffer: buffer to extract from
 * @data_page: the page to use allocated from ring_buffer_alloc_read_page
2918
 * @len: amount to extract
S
Steven Rostedt 已提交
2919 2920 2921 2922 2923 2924 2925 2926 2927
 * @cpu: the cpu of the buffer to extract
 * @full: should the extraction only happen when the page is full.
 *
 * This function will pull out a page from the ring buffer and consume it.
 * @data_page must be the address of the variable that was returned
 * from ring_buffer_alloc_read_page. This is because the page might be used
 * to swap with a page in the ring buffer.
 *
 * for example:
L
Lai Jiangshan 已提交
2928
 *	rpage = ring_buffer_alloc_read_page(buffer);
S
Steven Rostedt 已提交
2929 2930
 *	if (!rpage)
 *		return error;
2931
 *	ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
2932 2933
 *	if (ret >= 0)
 *		process_page(rpage, ret);
S
Steven Rostedt 已提交
2934 2935 2936 2937 2938 2939 2940 2941 2942 2943
 *
 * When @full is set, the function will not return true unless
 * the writer is off the reader page.
 *
 * Note: it is up to the calling functions to handle sleeps and wakeups.
 *  The ring buffer can be used anywhere in the kernel and can not
 *  blindly call wake_up. The layer that uses the ring buffer must be
 *  responsible for that.
 *
 * Returns:
2944 2945
 *  >=0 if data has been transferred, returns the offset of consumed data.
 *  <0 if no data has been transferred.
S
Steven Rostedt 已提交
2946 2947
 */
int ring_buffer_read_page(struct ring_buffer *buffer,
2948
			  void **data_page, size_t len, int cpu, int full)
S
Steven Rostedt 已提交
2949 2950 2951
{
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	struct ring_buffer_event *event;
2952
	struct buffer_data_page *bpage;
2953
	struct buffer_page *reader;
S
Steven Rostedt 已提交
2954
	unsigned long flags;
2955
	unsigned int commit;
2956
	unsigned int read;
2957
	u64 save_timestamp;
2958
	int ret = -1;
S
Steven Rostedt 已提交
2959

2960 2961 2962
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
		goto out;

2963 2964 2965 2966 2967
	/*
	 * If len is not big enough to hold the page header, then
	 * we can not copy anything.
	 */
	if (len <= BUF_PAGE_HDR_SIZE)
2968
		goto out;
2969 2970 2971

	len -= BUF_PAGE_HDR_SIZE;

S
Steven Rostedt 已提交
2972
	if (!data_page)
2973
		goto out;
S
Steven Rostedt 已提交
2974

2975 2976
	bpage = *data_page;
	if (!bpage)
2977
		goto out;
S
Steven Rostedt 已提交
2978 2979 2980

	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

2981 2982
	reader = rb_get_reader_page(cpu_buffer);
	if (!reader)
2983
		goto out_unlock;
S
Steven Rostedt 已提交
2984

2985 2986 2987 2988
	event = rb_reader_event(cpu_buffer);

	read = reader->read;
	commit = rb_page_commit(reader);
2989

S
Steven Rostedt 已提交
2990
	/*
2991 2992 2993 2994 2995
	 * If this page has been partially read or
	 * if len is not big enough to read the rest of the page or
	 * a writer is still on the page, then
	 * we must copy the data from the page to the buffer.
	 * Otherwise, we can simply swap the page with the one passed in.
S
Steven Rostedt 已提交
2996
	 */
2997
	if (read || (len < (commit - read)) ||
2998
	    cpu_buffer->reader_page == cpu_buffer->commit_page) {
2999
		struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
3000 3001
		unsigned int rpos = read;
		unsigned int pos = 0;
3002
		unsigned int size;
S
Steven Rostedt 已提交
3003 3004

		if (full)
3005
			goto out_unlock;
S
Steven Rostedt 已提交
3006

3007 3008 3009 3010 3011 3012
		if (len > (commit - read))
			len = (commit - read);

		size = rb_event_length(event);

		if (len < size)
3013
			goto out_unlock;
3014

3015 3016 3017
		/* save the current timestamp, since the user will need it */
		save_timestamp = cpu_buffer->read_stamp;

3018 3019
		/* Need to copy one event at a time */
		do {
3020
			memcpy(bpage->data + pos, rpage->data + rpos, size);
3021 3022 3023 3024

			len -= size;

			rb_advance_reader(cpu_buffer);
3025 3026
			rpos = reader->read;
			pos += size;
3027 3028 3029 3030

			event = rb_reader_event(cpu_buffer);
			size = rb_event_length(event);
		} while (len > size);
3031 3032

		/* update bpage */
3033
		local_set(&bpage->commit, pos);
3034
		bpage->time_stamp = save_timestamp;
3035

3036 3037
		/* we copied everything to the beginning */
		read = 0;
S
Steven Rostedt 已提交
3038
	} else {
3039 3040 3041
		/* update the entry counter */
		cpu_buffer->read += local_read(&reader->entries);

S
Steven Rostedt 已提交
3042
		/* swap the pages */
3043
		rb_init_page(bpage);
3044 3045 3046
		bpage = reader->page;
		reader->page = *data_page;
		local_set(&reader->write, 0);
3047
		local_set(&reader->entries, 0);
3048
		reader->read = 0;
3049
		*data_page = bpage;
S
Steven Rostedt 已提交
3050
	}
3051
	ret = read;
S
Steven Rostedt 已提交
3052

3053
 out_unlock:
S
Steven Rostedt 已提交
3054 3055
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

3056
 out:
S
Steven Rostedt 已提交
3057 3058
	return ret;
}
S
Steven Rostedt 已提交
3059
EXPORT_SYMBOL_GPL(ring_buffer_read_page);
S
Steven Rostedt 已提交
3060

3061 3062 3063 3064
static ssize_t
rb_simple_read(struct file *filp, char __user *ubuf,
	       size_t cnt, loff_t *ppos)
{
3065
	unsigned long *p = filp->private_data;
3066 3067 3068
	char buf[64];
	int r;

3069 3070 3071 3072
	if (test_bit(RB_BUFFERS_DISABLED_BIT, p))
		r = sprintf(buf, "permanently disabled\n");
	else
		r = sprintf(buf, "%d\n", test_bit(RB_BUFFERS_ON_BIT, p));
3073 3074 3075 3076 3077 3078 3079 3080

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

static ssize_t
rb_simple_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
3081
	unsigned long *p = filp->private_data;
3082
	char buf[64];
3083
	unsigned long val;
3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	ret = strict_strtoul(buf, 10, &val);
	if (ret < 0)
		return ret;

3098 3099 3100 3101
	if (val)
		set_bit(RB_BUFFERS_ON_BIT, p);
	else
		clear_bit(RB_BUFFERS_ON_BIT, p);
3102 3103 3104 3105 3106 3107

	(*ppos)++;

	return cnt;
}

3108
static const struct file_operations rb_simple_fops = {
3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120
	.open		= tracing_open_generic,
	.read		= rb_simple_read,
	.write		= rb_simple_write,
};


static __init int rb_init_debugfs(void)
{
	struct dentry *d_tracer;

	d_tracer = tracing_init_dentry();

3121 3122
	trace_create_file("tracing_on", 0644, d_tracer,
			    &ring_buffer_flags, &rb_simple_fops);
3123 3124 3125 3126 3127

	return 0;
}

fs_initcall(rb_init_debugfs);
3128

3129
#ifdef CONFIG_HOTPLUG_CPU
3130 3131
static int rb_cpu_notify(struct notifier_block *self,
			 unsigned long action, void *hcpu)
3132 3133 3134 3135 3136 3137 3138 3139
{
	struct ring_buffer *buffer =
		container_of(self, struct ring_buffer, cpu_notify);
	long cpu = (long)hcpu;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
3140
		if (cpumask_test_cpu(cpu, buffer->cpumask))
3141 3142 3143 3144 3145 3146 3147 3148 3149 3150
			return NOTIFY_OK;

		buffer->buffers[cpu] =
			rb_allocate_cpu_buffer(buffer, cpu);
		if (!buffer->buffers[cpu]) {
			WARN(1, "failed to allocate ring buffer on CPU %ld\n",
			     cpu);
			return NOTIFY_OK;
		}
		smp_wmb();
3151
		cpumask_set_cpu(cpu, buffer->cpumask);
3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166
		break;
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
		/*
		 * Do nothing.
		 *  If we were to free the buffer, then the user would
		 *  lose any trace that was in the buffer.
		 */
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}
#endif