ring_buffer.c 63.1 KB
Newer Older
S
Steven Rostedt 已提交
1 2 3 4 5 6
/*
 * Generic ring buffer
 *
 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
 */
#include <linux/ring_buffer.h>
7
#include <linux/ftrace_irq.h>
S
Steven Rostedt 已提交
8 9 10
#include <linux/spinlock.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
11
#include <linux/hardirq.h>
S
Steven Rostedt 已提交
12 13 14 15 16 17 18 19 20
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/mutex.h>
#include <linux/sched.h>	/* used for sched_clock() (for now) */
#include <linux/init.h>
#include <linux/hash.h>
#include <linux/list.h>
#include <linux/fs.h>

21 22
#include "trace.h"

23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
/*
 * A fast way to enable or disable all ring buffers is to
 * call tracing_on or tracing_off. Turning off the ring buffers
 * prevents all ring buffers from being recorded to.
 * Turning this switch on, makes it OK to write to the
 * ring buffer, if the ring buffer is enabled itself.
 *
 * There's three layers that must be on in order to write
 * to the ring buffer.
 *
 * 1) This global flag must be set.
 * 2) The ring buffer must be enabled for recording.
 * 3) The per cpu buffer must be enabled for recording.
 *
 * In case of an anomaly, this global flag has a bit set that
 * will permantly disable all ring buffers.
 */

/*
 * Global flag to disable all recording to ring buffers
 *  This has two bits: ON, DISABLED
 *
 *  ON   DISABLED
 * ---- ----------
 *   0      0        : ring buffers are off
 *   1      0        : ring buffers are on
 *   X      1        : ring buffers are permanently disabled
 */

enum {
	RB_BUFFERS_ON_BIT	= 0,
	RB_BUFFERS_DISABLED_BIT	= 1,
};

enum {
	RB_BUFFERS_ON		= 1 << RB_BUFFERS_ON_BIT,
	RB_BUFFERS_DISABLED	= 1 << RB_BUFFERS_DISABLED_BIT,
};

62
static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
63 64 65 66 67 68 69 70 71

/**
 * tracing_on - enable all tracing buffers
 *
 * This function enables all tracing buffers that may have been
 * disabled with tracing_off.
 */
void tracing_on(void)
{
72
	set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
73
}
74
EXPORT_SYMBOL_GPL(tracing_on);
75 76 77 78 79 80 81 82 83 84 85

/**
 * tracing_off - turn off all tracing buffers
 *
 * This function stops all tracing buffers from recording data.
 * It does not disable any overhead the tracers themselves may
 * be causing. This function simply causes all recording to
 * the ring buffers to fail.
 */
void tracing_off(void)
{
86 87
	clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
}
88
EXPORT_SYMBOL_GPL(tracing_off);
89 90 91 92 93

/**
 * tracing_off_permanent - permanently disable ring buffers
 *
 * This function, once called, will disable all ring buffers
W
Wenji Huang 已提交
94
 * permanently.
95 96 97 98
 */
void tracing_off_permanent(void)
{
	set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
99 100
}

101 102
#include "trace.h"

S
Steven Rostedt 已提交
103 104 105 106 107 108
/* Up this if you want to test the TIME_EXTENTS and normalization */
#define DEBUG_SHIFT 0

/* FIXME!!! */
u64 ring_buffer_time_stamp(int cpu)
{
109 110 111
	u64 time;

	preempt_disable_notrace();
S
Steven Rostedt 已提交
112
	/* shift to debug/test normalization and TIME_EXTENTS */
113
	time = sched_clock() << DEBUG_SHIFT;
114
	preempt_enable_no_resched_notrace();
115 116

	return time;
S
Steven Rostedt 已提交
117
}
118
EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
S
Steven Rostedt 已提交
119 120 121 122 123 124

void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
{
	/* Just stupid testing the normalize function and deltas */
	*ts >>= DEBUG_SHIFT;
}
125
EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
S
Steven Rostedt 已提交
126 127

#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))
128
#define RB_ALIGNMENT		4U
S
Steven Rostedt 已提交
129 130 131 132 133 134 135 136
#define RB_MAX_SMALL_DATA	28

enum {
	RB_LEN_TIME_EXTEND = 8,
	RB_LEN_TIME_STAMP = 16,
};

/* inline for ring buffer fast paths */
137
static unsigned
S
Steven Rostedt 已提交
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
rb_event_length(struct ring_buffer_event *event)
{
	unsigned length;

	switch (event->type) {
	case RINGBUF_TYPE_PADDING:
		/* undefined */
		return -1;

	case RINGBUF_TYPE_TIME_EXTEND:
		return RB_LEN_TIME_EXTEND;

	case RINGBUF_TYPE_TIME_STAMP:
		return RB_LEN_TIME_STAMP;

	case RINGBUF_TYPE_DATA:
		if (event->len)
155
			length = event->len * RB_ALIGNMENT;
S
Steven Rostedt 已提交
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
		else
			length = event->array[0];
		return length + RB_EVNT_HDR_SIZE;
	default:
		BUG();
	}
	/* not hit */
	return 0;
}

/**
 * ring_buffer_event_length - return the length of the event
 * @event: the event to get the length of
 */
unsigned ring_buffer_event_length(struct ring_buffer_event *event)
{
172 173 174 175 176 177 178
	unsigned length = rb_event_length(event);
	if (event->type != RINGBUF_TYPE_DATA)
		return length;
	length -= RB_EVNT_HDR_SIZE;
	if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
                length -= sizeof(event->array[0]);
	return length;
S
Steven Rostedt 已提交
179
}
180
EXPORT_SYMBOL_GPL(ring_buffer_event_length);
S
Steven Rostedt 已提交
181 182

/* inline for ring buffer fast paths */
183
static void *
S
Steven Rostedt 已提交
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
rb_event_data(struct ring_buffer_event *event)
{
	BUG_ON(event->type != RINGBUF_TYPE_DATA);
	/* If length is in len field, then array[0] has the data */
	if (event->len)
		return (void *)&event->array[0];
	/* Otherwise length is in array[0] and array[1] has the data */
	return (void *)&event->array[1];
}

/**
 * ring_buffer_event_data - return the data of the event
 * @event: the event to get the data from
 */
void *ring_buffer_event_data(struct ring_buffer_event *event)
{
	return rb_event_data(event);
}
202
EXPORT_SYMBOL_GPL(ring_buffer_event_data);
S
Steven Rostedt 已提交
203 204

#define for_each_buffer_cpu(buffer, cpu)		\
205
	for_each_cpu(cpu, buffer->cpumask)
S
Steven Rostedt 已提交
206 207 208 209 210

#define TS_SHIFT	27
#define TS_MASK		((1ULL << TS_SHIFT) - 1)
#define TS_DELTA_TEST	(~TS_MASK)

211
struct buffer_data_page {
212
	u64		 time_stamp;	/* page time stamp */
W
Wenji Huang 已提交
213
	local_t		 commit;	/* write committed index */
214 215 216 217 218
	unsigned char	 data[];	/* data of buffer page */
};

struct buffer_page {
	local_t		 write;		/* index for next write */
219
	unsigned	 read;		/* index for next read */
220
	struct list_head list;		/* list of free pages */
221
	struct buffer_data_page *page;	/* Actual data page */
S
Steven Rostedt 已提交
222 223
};

224
static void rb_init_page(struct buffer_data_page *bpage)
225
{
226
	local_set(&bpage->commit, 0);
227 228
}

229 230 231 232
/*
 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
 * this issue out.
 */
233
static void free_buffer_page(struct buffer_page *bpage)
234
{
235
	free_page((unsigned long)bpage->page);
236
	kfree(bpage);
237 238
}

S
Steven Rostedt 已提交
239 240 241 242 243 244 245 246 247 248
/*
 * We need to fit the time_stamp delta into 27 bits.
 */
static inline int test_time_stamp(u64 delta)
{
	if (delta & TS_DELTA_TEST)
		return 1;
	return 0;
}

249
#define BUF_PAGE_SIZE (PAGE_SIZE - offsetof(struct buffer_data_page, data))
S
Steven Rostedt 已提交
250 251 252 253 254 255 256

/*
 * head_page == tail_page && head == tail then buffer is empty.
 */
struct ring_buffer_per_cpu {
	int				cpu;
	struct ring_buffer		*buffer;
S
Steven Rostedt 已提交
257
	spinlock_t			reader_lock; /* serialize readers */
258
	raw_spinlock_t			lock;
S
Steven Rostedt 已提交
259 260
	struct lock_class_key		lock_key;
	struct list_head		pages;
261 262
	struct buffer_page		*head_page;	/* read from head */
	struct buffer_page		*tail_page;	/* write to tail */
W
Wenji Huang 已提交
263
	struct buffer_page		*commit_page;	/* committed pages */
264
	struct buffer_page		*reader_page;
S
Steven Rostedt 已提交
265 266 267 268 269 270 271 272 273 274 275 276
	unsigned long			overrun;
	unsigned long			entries;
	u64				write_stamp;
	u64				read_stamp;
	atomic_t			record_disabled;
};

struct ring_buffer {
	unsigned			pages;
	unsigned			flags;
	int				cpus;
	atomic_t			record_disabled;
277
	cpumask_var_t			cpumask;
S
Steven Rostedt 已提交
278 279 280 281 282 283 284 285 286 287 288 289 290

	struct mutex			mutex;

	struct ring_buffer_per_cpu	**buffers;
};

struct ring_buffer_iter {
	struct ring_buffer_per_cpu	*cpu_buffer;
	unsigned long			head;
	struct buffer_page		*head_page;
	u64				read_stamp;
};

291
/* buffer may be either ring_buffer or ring_buffer_per_cpu */
S
Steven Rostedt 已提交
292
#define RB_WARN_ON(buffer, cond)				\
S
Steven Rostedt 已提交
293 294 295
	({							\
		int _____ret = unlikely(cond);			\
		if (_____ret) {					\
S
Steven Rostedt 已提交
296 297 298
			atomic_inc(&buffer->record_disabled);	\
			WARN_ON(1);				\
		}						\
S
Steven Rostedt 已提交
299 300
		_____ret;					\
	})
301

S
Steven Rostedt 已提交
302 303 304 305
/**
 * check_pages - integrity check of buffer pages
 * @cpu_buffer: CPU buffer with pages to test
 *
W
Wenji Huang 已提交
306
 * As a safety measure we check to make sure the data pages have not
S
Steven Rostedt 已提交
307 308 309 310 311
 * been corrupted.
 */
static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
{
	struct list_head *head = &cpu_buffer->pages;
312
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
313

S
Steven Rostedt 已提交
314 315 316 317
	if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
		return -1;
	if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
		return -1;
S
Steven Rostedt 已提交
318

319
	list_for_each_entry_safe(bpage, tmp, head, list) {
S
Steven Rostedt 已提交
320
		if (RB_WARN_ON(cpu_buffer,
321
			       bpage->list.next->prev != &bpage->list))
S
Steven Rostedt 已提交
322 323
			return -1;
		if (RB_WARN_ON(cpu_buffer,
324
			       bpage->list.prev->next != &bpage->list))
S
Steven Rostedt 已提交
325
			return -1;
S
Steven Rostedt 已提交
326 327 328 329 330 331 332 333 334
	}

	return 0;
}

static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
			     unsigned nr_pages)
{
	struct list_head *head = &cpu_buffer->pages;
335
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
336 337 338 339 340
	unsigned long addr;
	LIST_HEAD(pages);
	unsigned i;

	for (i = 0; i < nr_pages; i++) {
341
		bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
S
Steven Rostedt 已提交
342
				    GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));
343
		if (!bpage)
344
			goto free_pages;
345
		list_add(&bpage->list, &pages);
346

S
Steven Rostedt 已提交
347 348 349
		addr = __get_free_page(GFP_KERNEL);
		if (!addr)
			goto free_pages;
350 351
		bpage->page = (void *)addr;
		rb_init_page(bpage->page);
S
Steven Rostedt 已提交
352 353 354 355 356 357 358 359 360
	}

	list_splice(&pages, head);

	rb_check_pages(cpu_buffer);

	return 0;

 free_pages:
361 362 363
	list_for_each_entry_safe(bpage, tmp, &pages, list) {
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
364 365 366 367 368 369 370 371
	}
	return -ENOMEM;
}

static struct ring_buffer_per_cpu *
rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
372
	struct buffer_page *bpage;
373
	unsigned long addr;
S
Steven Rostedt 已提交
374 375 376 377 378 379 380 381 382
	int ret;

	cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
				  GFP_KERNEL, cpu_to_node(cpu));
	if (!cpu_buffer)
		return NULL;

	cpu_buffer->cpu = cpu;
	cpu_buffer->buffer = buffer;
S
Steven Rostedt 已提交
383
	spin_lock_init(&cpu_buffer->reader_lock);
384
	cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
S
Steven Rostedt 已提交
385 386
	INIT_LIST_HEAD(&cpu_buffer->pages);

387
	bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
388
			    GFP_KERNEL, cpu_to_node(cpu));
389
	if (!bpage)
390 391
		goto fail_free_buffer;

392
	cpu_buffer->reader_page = bpage;
393 394
	addr = __get_free_page(GFP_KERNEL);
	if (!addr)
395
		goto fail_free_reader;
396 397
	bpage->page = (void *)addr;
	rb_init_page(bpage->page);
398

399 400
	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);

S
Steven Rostedt 已提交
401 402
	ret = rb_allocate_pages(cpu_buffer, buffer->pages);
	if (ret < 0)
403
		goto fail_free_reader;
S
Steven Rostedt 已提交
404 405 406

	cpu_buffer->head_page
		= list_entry(cpu_buffer->pages.next, struct buffer_page, list);
S
Steven Rostedt 已提交
407
	cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
S
Steven Rostedt 已提交
408 409 410

	return cpu_buffer;

411 412 413
 fail_free_reader:
	free_buffer_page(cpu_buffer->reader_page);

S
Steven Rostedt 已提交
414 415 416 417 418 419 420 421
 fail_free_buffer:
	kfree(cpu_buffer);
	return NULL;
}

static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
{
	struct list_head *head = &cpu_buffer->pages;
422
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
423

424 425 426
	list_del_init(&cpu_buffer->reader_page->list);
	free_buffer_page(cpu_buffer->reader_page);

427 428 429
	list_for_each_entry_safe(bpage, tmp, head, list) {
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
430 431 432 433
	}
	kfree(cpu_buffer);
}

434 435 436 437 438 439
/*
 * Causes compile errors if the struct buffer_page gets bigger
 * than the struct page.
 */
extern int ring_buffer_page_too_big(void);

S
Steven Rostedt 已提交
440 441
/**
 * ring_buffer_alloc - allocate a new ring_buffer
442
 * @size: the size in bytes per cpu that is needed.
S
Steven Rostedt 已提交
443 444 445 446 447 448 449 450 451 452 453 454 455
 * @flags: attributes to set for the ring buffer.
 *
 * Currently the only flag that is available is the RB_FL_OVERWRITE
 * flag. This flag means that the buffer will overwrite old data
 * when the buffer wraps. If this flag is not set, the buffer will
 * drop data when the tail hits the head.
 */
struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags)
{
	struct ring_buffer *buffer;
	int bsize;
	int cpu;

456 457 458 459 460
	/* Paranoid! Optimizes out when all is well */
	if (sizeof(struct buffer_page) > sizeof(struct page))
		ring_buffer_page_too_big();


S
Steven Rostedt 已提交
461 462 463 464 465 466
	/* keep it in its own cache line */
	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
			 GFP_KERNEL);
	if (!buffer)
		return NULL;

467 468 469
	if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
		goto fail_free_buffer;

S
Steven Rostedt 已提交
470 471 472 473 474 475 476
	buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
	buffer->flags = flags;

	/* need at least two pages */
	if (buffer->pages == 1)
		buffer->pages++;

477
	cpumask_copy(buffer->cpumask, cpu_possible_mask);
S
Steven Rostedt 已提交
478 479 480 481 482 483
	buffer->cpus = nr_cpu_ids;

	bsize = sizeof(void *) * nr_cpu_ids;
	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
				  GFP_KERNEL);
	if (!buffer->buffers)
484
		goto fail_free_cpumask;
S
Steven Rostedt 已提交
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503

	for_each_buffer_cpu(buffer, cpu) {
		buffer->buffers[cpu] =
			rb_allocate_cpu_buffer(buffer, cpu);
		if (!buffer->buffers[cpu])
			goto fail_free_buffers;
	}

	mutex_init(&buffer->mutex);

	return buffer;

 fail_free_buffers:
	for_each_buffer_cpu(buffer, cpu) {
		if (buffer->buffers[cpu])
			rb_free_cpu_buffer(buffer->buffers[cpu]);
	}
	kfree(buffer->buffers);

504 505 506
 fail_free_cpumask:
	free_cpumask_var(buffer->cpumask);

S
Steven Rostedt 已提交
507 508 509 510
 fail_free_buffer:
	kfree(buffer);
	return NULL;
}
511
EXPORT_SYMBOL_GPL(ring_buffer_alloc);
S
Steven Rostedt 已提交
512 513 514 515 516 517 518 519 520 521 522 523 524

/**
 * ring_buffer_free - free a ring buffer.
 * @buffer: the buffer to free.
 */
void
ring_buffer_free(struct ring_buffer *buffer)
{
	int cpu;

	for_each_buffer_cpu(buffer, cpu)
		rb_free_cpu_buffer(buffer->buffers[cpu]);

525 526
	free_cpumask_var(buffer->cpumask);

S
Steven Rostedt 已提交
527 528
	kfree(buffer);
}
529
EXPORT_SYMBOL_GPL(ring_buffer_free);
S
Steven Rostedt 已提交
530 531 532 533 534 535

static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);

static void
rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
{
536
	struct buffer_page *bpage;
S
Steven Rostedt 已提交
537 538 539 540 541 542 543
	struct list_head *p;
	unsigned i;

	atomic_inc(&cpu_buffer->record_disabled);
	synchronize_sched();

	for (i = 0; i < nr_pages; i++) {
S
Steven Rostedt 已提交
544 545
		if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
			return;
S
Steven Rostedt 已提交
546
		p = cpu_buffer->pages.next;
547 548 549
		bpage = list_entry(p, struct buffer_page, list);
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
550
	}
S
Steven Rostedt 已提交
551 552
	if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
		return;
S
Steven Rostedt 已提交
553 554 555 556 557 558 559 560 561 562 563 564 565

	rb_reset_cpu(cpu_buffer);

	rb_check_pages(cpu_buffer);

	atomic_dec(&cpu_buffer->record_disabled);

}

static void
rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
		struct list_head *pages, unsigned nr_pages)
{
566
	struct buffer_page *bpage;
S
Steven Rostedt 已提交
567 568 569 570 571 572 573
	struct list_head *p;
	unsigned i;

	atomic_inc(&cpu_buffer->record_disabled);
	synchronize_sched();

	for (i = 0; i < nr_pages; i++) {
S
Steven Rostedt 已提交
574 575
		if (RB_WARN_ON(cpu_buffer, list_empty(pages)))
			return;
S
Steven Rostedt 已提交
576
		p = pages->next;
577 578 579
		bpage = list_entry(p, struct buffer_page, list);
		list_del_init(&bpage->list);
		list_add_tail(&bpage->list, &cpu_buffer->pages);
S
Steven Rostedt 已提交
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
	}
	rb_reset_cpu(cpu_buffer);

	rb_check_pages(cpu_buffer);

	atomic_dec(&cpu_buffer->record_disabled);
}

/**
 * ring_buffer_resize - resize the ring buffer
 * @buffer: the buffer to resize.
 * @size: the new size.
 *
 * The tracer is responsible for making sure that the buffer is
 * not being used while changing the size.
 * Note: We may be able to change the above requirement by using
 *  RCU synchronizations.
 *
 * Minimum size is 2 * BUF_PAGE_SIZE.
 *
 * Returns -1 on failure.
 */
int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned nr_pages, rm_pages, new_pages;
606
	struct buffer_page *bpage, *tmp;
S
Steven Rostedt 已提交
607 608 609 610 611
	unsigned long buffer_size;
	unsigned long addr;
	LIST_HEAD(pages);
	int i, cpu;

612 613 614 615 616 617
	/*
	 * Always succeed at resizing a non-existent buffer:
	 */
	if (!buffer)
		return size;

S
Steven Rostedt 已提交
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635
	size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
	size *= BUF_PAGE_SIZE;
	buffer_size = buffer->pages * BUF_PAGE_SIZE;

	/* we need a minimum of two pages */
	if (size < BUF_PAGE_SIZE * 2)
		size = BUF_PAGE_SIZE * 2;

	if (size == buffer_size)
		return size;

	mutex_lock(&buffer->mutex);

	nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);

	if (size < buffer_size) {

		/* easy case, just free pages */
S
Steven Rostedt 已提交
636 637 638 639
		if (RB_WARN_ON(buffer, nr_pages >= buffer->pages)) {
			mutex_unlock(&buffer->mutex);
			return -1;
		}
S
Steven Rostedt 已提交
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657

		rm_pages = buffer->pages - nr_pages;

		for_each_buffer_cpu(buffer, cpu) {
			cpu_buffer = buffer->buffers[cpu];
			rb_remove_pages(cpu_buffer, rm_pages);
		}
		goto out;
	}

	/*
	 * This is a bit more difficult. We only want to add pages
	 * when we can allocate enough for all CPUs. We do this
	 * by allocating all the pages and storing them on a local
	 * link list. If we succeed in our allocation, then we
	 * add these pages to the cpu_buffers. Otherwise we just free
	 * them all and return -ENOMEM;
	 */
S
Steven Rostedt 已提交
658 659 660 661
	if (RB_WARN_ON(buffer, nr_pages <= buffer->pages)) {
		mutex_unlock(&buffer->mutex);
		return -1;
	}
662

S
Steven Rostedt 已提交
663 664 665 666
	new_pages = nr_pages - buffer->pages;

	for_each_buffer_cpu(buffer, cpu) {
		for (i = 0; i < new_pages; i++) {
667
			bpage = kzalloc_node(ALIGN(sizeof(*bpage),
668 669
						  cache_line_size()),
					    GFP_KERNEL, cpu_to_node(cpu));
670
			if (!bpage)
671
				goto free_pages;
672
			list_add(&bpage->list, &pages);
S
Steven Rostedt 已提交
673 674 675
			addr = __get_free_page(GFP_KERNEL);
			if (!addr)
				goto free_pages;
676 677
			bpage->page = (void *)addr;
			rb_init_page(bpage->page);
S
Steven Rostedt 已提交
678 679 680 681 682 683 684 685
		}
	}

	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
		rb_insert_pages(cpu_buffer, &pages, new_pages);
	}

S
Steven Rostedt 已提交
686 687 688 689
	if (RB_WARN_ON(buffer, !list_empty(&pages))) {
		mutex_unlock(&buffer->mutex);
		return -1;
	}
S
Steven Rostedt 已提交
690 691 692 693 694 695 696 697

 out:
	buffer->pages = nr_pages;
	mutex_unlock(&buffer->mutex);

	return size;

 free_pages:
698 699 700
	list_for_each_entry_safe(bpage, tmp, &pages, list) {
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
S
Steven Rostedt 已提交
701
	}
702
	mutex_unlock(&buffer->mutex);
S
Steven Rostedt 已提交
703 704
	return -ENOMEM;
}
705
EXPORT_SYMBOL_GPL(ring_buffer_resize);
S
Steven Rostedt 已提交
706 707 708 709 710 711

static inline int rb_null_event(struct ring_buffer_event *event)
{
	return event->type == RINGBUF_TYPE_PADDING;
}

S
Steven Rostedt 已提交
712
static inline void *
713
__rb_data_page_index(struct buffer_data_page *bpage, unsigned index)
S
Steven Rostedt 已提交
714
{
715
	return bpage->data + index;
S
Steven Rostedt 已提交
716 717
}

718
static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
S
Steven Rostedt 已提交
719
{
720
	return bpage->page->data + index;
S
Steven Rostedt 已提交
721 722 723
}

static inline struct ring_buffer_event *
724
rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
725
{
726 727 728 729 730 731 732 733 734
	return __rb_page_index(cpu_buffer->reader_page,
			       cpu_buffer->reader_page->read);
}

static inline struct ring_buffer_event *
rb_head_event(struct ring_buffer_per_cpu *cpu_buffer)
{
	return __rb_page_index(cpu_buffer->head_page,
			       cpu_buffer->head_page->read);
S
Steven Rostedt 已提交
735 736 737 738 739
}

static inline struct ring_buffer_event *
rb_iter_head_event(struct ring_buffer_iter *iter)
{
740
	return __rb_page_index(iter->head_page, iter->head);
S
Steven Rostedt 已提交
741 742
}

S
Steven Rostedt 已提交
743 744 745 746 747 748 749
static inline unsigned rb_page_write(struct buffer_page *bpage)
{
	return local_read(&bpage->write);
}

static inline unsigned rb_page_commit(struct buffer_page *bpage)
{
750
	return local_read(&bpage->page->commit);
S
Steven Rostedt 已提交
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
}

/* Size is determined by what has been commited */
static inline unsigned rb_page_size(struct buffer_page *bpage)
{
	return rb_page_commit(bpage);
}

static inline unsigned
rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
{
	return rb_page_commit(cpu_buffer->commit_page);
}

static inline unsigned rb_head_size(struct ring_buffer_per_cpu *cpu_buffer)
{
	return rb_page_commit(cpu_buffer->head_page);
}

S
Steven Rostedt 已提交
770 771 772 773 774 775 776 777 778 779 780 781 782 783
/*
 * When the tail hits the head and the buffer is in overwrite mode,
 * the head jumps to the next page and all content on the previous
 * page is discarded. But before doing so, we update the overrun
 * variable of the buffer.
 */
static void rb_update_overflow(struct ring_buffer_per_cpu *cpu_buffer)
{
	struct ring_buffer_event *event;
	unsigned long head;

	for (head = 0; head < rb_head_size(cpu_buffer);
	     head += rb_event_length(event)) {

784
		event = __rb_page_index(cpu_buffer->head_page, head);
S
Steven Rostedt 已提交
785 786
		if (RB_WARN_ON(cpu_buffer, rb_null_event(event)))
			return;
S
Steven Rostedt 已提交
787 788 789 790 791 792 793 794 795
		/* Only count data entries */
		if (event->type != RINGBUF_TYPE_DATA)
			continue;
		cpu_buffer->overrun++;
		cpu_buffer->entries--;
	}
}

static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
796
			       struct buffer_page **bpage)
S
Steven Rostedt 已提交
797
{
798
	struct list_head *p = (*bpage)->list.next;
S
Steven Rostedt 已提交
799 800 801 802

	if (p == &cpu_buffer->pages)
		p = p->next;

803
	*bpage = list_entry(p, struct buffer_page, list);
S
Steven Rostedt 已提交
804 805
}

S
Steven Rostedt 已提交
806 807 808 809 810 811 812 813
static inline unsigned
rb_event_index(struct ring_buffer_event *event)
{
	unsigned long addr = (unsigned long)event;

	return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
}

814
static int
S
Steven Rostedt 已提交
815 816 817 818 819 820 821 822 823 824 825 826 827
rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
	     struct ring_buffer_event *event)
{
	unsigned long addr = (unsigned long)event;
	unsigned long index;

	index = rb_event_index(event);
	addr &= PAGE_MASK;

	return cpu_buffer->commit_page->page == (void *)addr &&
		rb_commit_index(cpu_buffer) == index;
}

828
static void
S
Steven Rostedt 已提交
829 830
rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
		    struct ring_buffer_event *event)
S
Steven Rostedt 已提交
831
{
S
Steven Rostedt 已提交
832 833 834 835 836 837 838
	unsigned long addr = (unsigned long)event;
	unsigned long index;

	index = rb_event_index(event);
	addr &= PAGE_MASK;

	while (cpu_buffer->commit_page->page != (void *)addr) {
S
Steven Rostedt 已提交
839 840 841
		if (RB_WARN_ON(cpu_buffer,
			  cpu_buffer->commit_page == cpu_buffer->tail_page))
			return;
842
		cpu_buffer->commit_page->page->commit =
S
Steven Rostedt 已提交
843 844
			cpu_buffer->commit_page->write;
		rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
845 846
		cpu_buffer->write_stamp =
			cpu_buffer->commit_page->page->time_stamp;
S
Steven Rostedt 已提交
847 848 849
	}

	/* Now set the commit to the event's index */
850
	local_set(&cpu_buffer->commit_page->page->commit, index);
S
Steven Rostedt 已提交
851 852
}

853
static void
S
Steven Rostedt 已提交
854
rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
855
{
S
Steven Rostedt 已提交
856 857 858 859 860 861 862 863
	/*
	 * We only race with interrupts and NMIs on this CPU.
	 * If we own the commit event, then we can commit
	 * all others that interrupted us, since the interruptions
	 * are in stack format (they finish before they come
	 * back to us). This allows us to do a simple loop to
	 * assign the commit to the tail.
	 */
864
 again:
S
Steven Rostedt 已提交
865
	while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
866
		cpu_buffer->commit_page->page->commit =
S
Steven Rostedt 已提交
867 868
			cpu_buffer->commit_page->write;
		rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
869 870
		cpu_buffer->write_stamp =
			cpu_buffer->commit_page->page->time_stamp;
S
Steven Rostedt 已提交
871 872 873 874 875
		/* add barrier to keep gcc from optimizing too much */
		barrier();
	}
	while (rb_commit_index(cpu_buffer) !=
	       rb_page_write(cpu_buffer->commit_page)) {
876
		cpu_buffer->commit_page->page->commit =
S
Steven Rostedt 已提交
877 878 879
			cpu_buffer->commit_page->write;
		barrier();
	}
880 881 882 883 884 885 886 887 888 889 890

	/* again, keep gcc from optimizing */
	barrier();

	/*
	 * If an interrupt came in just after the first while loop
	 * and pushed the tail page forward, we will be left with
	 * a dangling commit that will never go forward.
	 */
	if (unlikely(cpu_buffer->commit_page != cpu_buffer->tail_page))
		goto again;
S
Steven Rostedt 已提交
891 892
}

893
static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
894
{
895
	cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
896
	cpu_buffer->reader_page->read = 0;
897 898
}

899
static void rb_inc_iter(struct ring_buffer_iter *iter)
900 901 902 903 904 905 906 907 908 909 910 911 912 913
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

	/*
	 * The iterator could be on the reader page (it starts there).
	 * But the head could have moved, since the reader was
	 * found. Check for this case and assign the iterator
	 * to the head page instead of next.
	 */
	if (iter->head_page == cpu_buffer->reader_page)
		iter->head_page = cpu_buffer->head_page;
	else
		rb_inc_page(cpu_buffer, &iter->head_page);

914
	iter->read_stamp = iter->head_page->page->time_stamp;
S
Steven Rostedt 已提交
915 916 917 918 919 920 921 922 923 924 925 926 927 928
	iter->head = 0;
}

/**
 * ring_buffer_update_event - update event type and data
 * @event: the even to update
 * @type: the type of event
 * @length: the size of the event field in the ring buffer
 *
 * Update the type and data fields of the event. The length
 * is the actual size that is written to the ring buffer,
 * and with this, we can determine what to place into the
 * data field.
 */
929
static void
S
Steven Rostedt 已提交
930 931 932 933 934 935 936 937 938 939 940
rb_update_event(struct ring_buffer_event *event,
			 unsigned type, unsigned length)
{
	event->type = type;

	switch (type) {

	case RINGBUF_TYPE_PADDING:
		break;

	case RINGBUF_TYPE_TIME_EXTEND:
941
		event->len = DIV_ROUND_UP(RB_LEN_TIME_EXTEND, RB_ALIGNMENT);
S
Steven Rostedt 已提交
942 943 944
		break;

	case RINGBUF_TYPE_TIME_STAMP:
945
		event->len = DIV_ROUND_UP(RB_LEN_TIME_STAMP, RB_ALIGNMENT);
S
Steven Rostedt 已提交
946 947 948 949 950 951 952 953
		break;

	case RINGBUF_TYPE_DATA:
		length -= RB_EVNT_HDR_SIZE;
		if (length > RB_MAX_SMALL_DATA) {
			event->len = 0;
			event->array[0] = length;
		} else
954
			event->len = DIV_ROUND_UP(length, RB_ALIGNMENT);
S
Steven Rostedt 已提交
955 956 957 958 959 960
		break;
	default:
		BUG();
	}
}

961
static unsigned rb_calculate_event_length(unsigned length)
S
Steven Rostedt 已提交
962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981
{
	struct ring_buffer_event event; /* Used only for sizeof array */

	/* zero length can cause confusions */
	if (!length)
		length = 1;

	if (length > RB_MAX_SMALL_DATA)
		length += sizeof(event.array[0]);

	length += RB_EVNT_HDR_SIZE;
	length = ALIGN(length, RB_ALIGNMENT);

	return length;
}

static struct ring_buffer_event *
__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
		  unsigned type, unsigned long length, u64 *ts)
{
982
	struct buffer_page *tail_page, *head_page, *reader_page, *commit_page;
S
Steven Rostedt 已提交
983
	unsigned long tail, write;
S
Steven Rostedt 已提交
984 985
	struct ring_buffer *buffer = cpu_buffer->buffer;
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
986
	unsigned long flags;
987
	bool lock_taken = false;
S
Steven Rostedt 已提交
988

989 990 991
	commit_page = cpu_buffer->commit_page;
	/* we just need to protect against interrupts */
	barrier();
S
Steven Rostedt 已提交
992
	tail_page = cpu_buffer->tail_page;
S
Steven Rostedt 已提交
993 994
	write = local_add_return(length, &tail_page->write);
	tail = write - length;
S
Steven Rostedt 已提交
995

S
Steven Rostedt 已提交
996 997
	/* See if we shot pass the end of this buffer page */
	if (write > BUF_PAGE_SIZE) {
S
Steven Rostedt 已提交
998 999
		struct buffer_page *next_page = tail_page;

1000
		local_irq_save(flags);
1001
		/*
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
		 * Since the write to the buffer is still not
		 * fully lockless, we must be careful with NMIs.
		 * The locks in the writers are taken when a write
		 * crosses to a new page. The locks protect against
		 * races with the readers (this will soon be fixed
		 * with a lockless solution).
		 *
		 * Because we can not protect against NMIs, and we
		 * want to keep traces reentrant, we need to manage
		 * what happens when we are in an NMI.
		 *
1013 1014 1015 1016 1017
		 * NMIs can happen after we take the lock.
		 * If we are in an NMI, only take the lock
		 * if it is not already taken. Otherwise
		 * simply fail.
		 */
1018
		if (unlikely(in_nmi())) {
1019
			if (!__raw_spin_trylock(&cpu_buffer->lock))
1020
				goto out_reset;
1021 1022 1023 1024
		} else
			__raw_spin_lock(&cpu_buffer->lock);

		lock_taken = true;
S
Steven Rostedt 已提交
1025

S
Steven Rostedt 已提交
1026 1027
		rb_inc_page(cpu_buffer, &next_page);

1028 1029 1030 1031
		head_page = cpu_buffer->head_page;
		reader_page = cpu_buffer->reader_page;

		/* we grabbed the lock before incrementing */
S
Steven Rostedt 已提交
1032
		if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
1033
			goto out_reset;
S
Steven Rostedt 已提交
1034 1035 1036 1037 1038 1039

		/*
		 * If for some reason, we had an interrupt storm that made
		 * it all the way around the buffer, bail, and warn
		 * about it.
		 */
1040
		if (unlikely(next_page == commit_page)) {
S
Steven Rostedt 已提交
1041
			WARN_ON_ONCE(1);
1042
			goto out_reset;
S
Steven Rostedt 已提交
1043
		}
1044

S
Steven Rostedt 已提交
1045
		if (next_page == head_page) {
1046
			if (!(buffer->flags & RB_FL_OVERWRITE))
1047
				goto out_reset;
S
Steven Rostedt 已提交
1048

S
Steven Rostedt 已提交
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
			/* tail_page has not moved yet? */
			if (tail_page == cpu_buffer->tail_page) {
				/* count overflows */
				rb_update_overflow(cpu_buffer);

				rb_inc_page(cpu_buffer, &head_page);
				cpu_buffer->head_page = head_page;
				cpu_buffer->head_page->read = 0;
			}
		}
S
Steven Rostedt 已提交
1059

S
Steven Rostedt 已提交
1060 1061 1062 1063 1064 1065 1066
		/*
		 * If the tail page is still the same as what we think
		 * it is, then it is up to us to update the tail
		 * pointer.
		 */
		if (tail_page == cpu_buffer->tail_page) {
			local_set(&next_page->write, 0);
1067
			local_set(&next_page->page->commit, 0);
S
Steven Rostedt 已提交
1068 1069 1070 1071
			cpu_buffer->tail_page = next_page;

			/* reread the time stamp */
			*ts = ring_buffer_time_stamp(cpu_buffer->cpu);
1072
			cpu_buffer->tail_page->page->time_stamp = *ts;
S
Steven Rostedt 已提交
1073 1074
		}

S
Steven Rostedt 已提交
1075 1076 1077 1078 1079
		/*
		 * The actual tail page has moved forward.
		 */
		if (tail < BUF_PAGE_SIZE) {
			/* Mark the rest of the page with padding */
1080
			event = __rb_page_index(tail_page, tail);
S
Steven Rostedt 已提交
1081 1082 1083
			event->type = RINGBUF_TYPE_PADDING;
		}

S
Steven Rostedt 已提交
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
		if (tail <= BUF_PAGE_SIZE)
			/* Set the write back to the previous setting */
			local_set(&tail_page->write, tail);

		/*
		 * If this was a commit entry that failed,
		 * increment that too
		 */
		if (tail_page == cpu_buffer->commit_page &&
		    tail == rb_commit_index(cpu_buffer)) {
			rb_set_commit_to_write(cpu_buffer);
		}

1097 1098
		__raw_spin_unlock(&cpu_buffer->lock);
		local_irq_restore(flags);
S
Steven Rostedt 已提交
1099 1100 1101

		/* fail and let the caller try again */
		return ERR_PTR(-EAGAIN);
S
Steven Rostedt 已提交
1102 1103
	}

S
Steven Rostedt 已提交
1104 1105
	/* We reserved something on the buffer */

S
Steven Rostedt 已提交
1106 1107
	if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))
		return NULL;
S
Steven Rostedt 已提交
1108

1109
	event = __rb_page_index(tail_page, tail);
S
Steven Rostedt 已提交
1110 1111
	rb_update_event(event, type, length);

S
Steven Rostedt 已提交
1112 1113 1114 1115 1116
	/*
	 * If this is a commit and the tail is zero, then update
	 * this page's time stamp.
	 */
	if (!tail && rb_is_commit(cpu_buffer, event))
1117
		cpu_buffer->commit_page->page->time_stamp = *ts;
S
Steven Rostedt 已提交
1118

S
Steven Rostedt 已提交
1119
	return event;
S
Steven Rostedt 已提交
1120

1121
 out_reset:
1122 1123 1124 1125
	/* reset write */
	if (tail <= BUF_PAGE_SIZE)
		local_set(&tail_page->write, tail);

1126 1127
	if (likely(lock_taken))
		__raw_spin_unlock(&cpu_buffer->lock);
1128
	local_irq_restore(flags);
S
Steven Rostedt 已提交
1129
	return NULL;
S
Steven Rostedt 已提交
1130 1131 1132 1133 1134 1135 1136 1137
}

static int
rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
		  u64 *ts, u64 *delta)
{
	struct ring_buffer_event *event;
	static int once;
S
Steven Rostedt 已提交
1138
	int ret;
S
Steven Rostedt 已提交
1139 1140 1141 1142

	if (unlikely(*delta > (1ULL << 59) && !once++)) {
		printk(KERN_WARNING "Delta way too big! %llu"
		       " ts=%llu write stamp = %llu\n",
1143 1144 1145
		       (unsigned long long)*delta,
		       (unsigned long long)*ts,
		       (unsigned long long)cpu_buffer->write_stamp);
S
Steven Rostedt 已提交
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
		WARN_ON(1);
	}

	/*
	 * The delta is too big, we to add a
	 * new timestamp.
	 */
	event = __rb_reserve_next(cpu_buffer,
				  RINGBUF_TYPE_TIME_EXTEND,
				  RB_LEN_TIME_EXTEND,
				  ts);
	if (!event)
S
Steven Rostedt 已提交
1158
		return -EBUSY;
S
Steven Rostedt 已提交
1159

S
Steven Rostedt 已提交
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
	if (PTR_ERR(event) == -EAGAIN)
		return -EAGAIN;

	/* Only a commited time event can update the write stamp */
	if (rb_is_commit(cpu_buffer, event)) {
		/*
		 * If this is the first on the page, then we need to
		 * update the page itself, and just put in a zero.
		 */
		if (rb_event_index(event)) {
			event->time_delta = *delta & TS_MASK;
			event->array[0] = *delta >> TS_SHIFT;
		} else {
1173
			cpu_buffer->commit_page->page->time_stamp = *ts;
S
Steven Rostedt 已提交
1174 1175 1176
			event->time_delta = 0;
			event->array[0] = 0;
		}
S
Steven Rostedt 已提交
1177
		cpu_buffer->write_stamp = *ts;
S
Steven Rostedt 已提交
1178 1179 1180 1181 1182 1183 1184
		/* let the caller know this was the commit */
		ret = 1;
	} else {
		/* Darn, this is just wasted space */
		event->time_delta = 0;
		event->array[0] = 0;
		ret = 0;
S
Steven Rostedt 已提交
1185 1186
	}

S
Steven Rostedt 已提交
1187 1188 1189
	*delta = 0;

	return ret;
S
Steven Rostedt 已提交
1190 1191 1192 1193 1194 1195 1196 1197
}

static struct ring_buffer_event *
rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
		      unsigned type, unsigned long length)
{
	struct ring_buffer_event *event;
	u64 ts, delta;
S
Steven Rostedt 已提交
1198
	int commit = 0;
1199
	int nr_loops = 0;
S
Steven Rostedt 已提交
1200

S
Steven Rostedt 已提交
1201
 again:
1202 1203 1204 1205 1206 1207 1208 1209 1210
	/*
	 * We allow for interrupts to reenter here and do a trace.
	 * If one does, it will cause this original code to loop
	 * back here. Even with heavy interrupts happening, this
	 * should only happen a few times in a row. If this happens
	 * 1000 times in a row, there must be either an interrupt
	 * storm or we have something buggy.
	 * Bail!
	 */
S
Steven Rostedt 已提交
1211
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
1212 1213
		return NULL;

S
Steven Rostedt 已提交
1214 1215
	ts = ring_buffer_time_stamp(cpu_buffer->cpu);

S
Steven Rostedt 已提交
1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	/*
	 * Only the first commit can update the timestamp.
	 * Yes there is a race here. If an interrupt comes in
	 * just after the conditional and it traces too, then it
	 * will also check the deltas. More than one timestamp may
	 * also be made. But only the entry that did the actual
	 * commit will be something other than zero.
	 */
	if (cpu_buffer->tail_page == cpu_buffer->commit_page &&
	    rb_page_write(cpu_buffer->tail_page) ==
	    rb_commit_index(cpu_buffer)) {

S
Steven Rostedt 已提交
1228 1229
		delta = ts - cpu_buffer->write_stamp;

S
Steven Rostedt 已提交
1230 1231 1232 1233 1234
		/* make sure this delta is calculated here */
		barrier();

		/* Did the write stamp get updated already? */
		if (unlikely(ts < cpu_buffer->write_stamp))
1235
			delta = 0;
S
Steven Rostedt 已提交
1236

S
Steven Rostedt 已提交
1237 1238
		if (test_time_stamp(delta)) {

S
Steven Rostedt 已提交
1239 1240 1241
			commit = rb_add_time_stamp(cpu_buffer, &ts, &delta);

			if (commit == -EBUSY)
S
Steven Rostedt 已提交
1242
				return NULL;
S
Steven Rostedt 已提交
1243 1244 1245 1246 1247

			if (commit == -EAGAIN)
				goto again;

			RB_WARN_ON(cpu_buffer, commit < 0);
S
Steven Rostedt 已提交
1248
		}
S
Steven Rostedt 已提交
1249 1250
	} else
		/* Non commits have zero deltas */
S
Steven Rostedt 已提交
1251 1252 1253
		delta = 0;

	event = __rb_reserve_next(cpu_buffer, type, length, &ts);
S
Steven Rostedt 已提交
1254 1255 1256 1257 1258 1259 1260 1261 1262 1263
	if (PTR_ERR(event) == -EAGAIN)
		goto again;

	if (!event) {
		if (unlikely(commit))
			/*
			 * Ouch! We needed a timestamp and it was commited. But
			 * we didn't get our event reserved.
			 */
			rb_set_commit_to_write(cpu_buffer);
S
Steven Rostedt 已提交
1264
		return NULL;
S
Steven Rostedt 已提交
1265
	}
S
Steven Rostedt 已提交
1266

S
Steven Rostedt 已提交
1267 1268 1269 1270 1271 1272 1273
	/*
	 * If the timestamp was commited, make the commit our entry
	 * now so that we will update it when needed.
	 */
	if (commit)
		rb_set_commit_event(cpu_buffer, event);
	else if (!rb_is_commit(cpu_buffer, event))
S
Steven Rostedt 已提交
1274 1275 1276 1277 1278 1279 1280
		delta = 0;

	event->time_delta = delta;

	return event;
}

S
Steven Rostedt 已提交
1281 1282
static DEFINE_PER_CPU(int, rb_need_resched);

S
Steven Rostedt 已提交
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
/**
 * ring_buffer_lock_reserve - reserve a part of the buffer
 * @buffer: the ring buffer to reserve from
 * @length: the length of the data to reserve (excluding event header)
 *
 * Returns a reseverd event on the ring buffer to copy directly to.
 * The user of this interface will need to get the body to write into
 * and can use the ring_buffer_event_data() interface.
 *
 * The length is the length of the data needed, not the event length
 * which also includes the event header.
 *
 * Must be paired with ring_buffer_unlock_commit, unless NULL is returned.
 * If NULL is returned, then nothing has been allocated or locked.
 */
struct ring_buffer_event *
1299
ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
S
Steven Rostedt 已提交
1300 1301 1302
{
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
1303
	int cpu, resched;
S
Steven Rostedt 已提交
1304

1305
	if (ring_buffer_flags != RB_BUFFERS_ON)
1306 1307
		return NULL;

S
Steven Rostedt 已提交
1308 1309 1310
	if (atomic_read(&buffer->record_disabled))
		return NULL;

S
Steven Rostedt 已提交
1311
	/* If we are tracing schedule, we don't want to recurse */
1312
	resched = ftrace_preempt_disable();
S
Steven Rostedt 已提交
1313

S
Steven Rostedt 已提交
1314 1315
	cpu = raw_smp_processor_id();

1316
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1317
		goto out;
S
Steven Rostedt 已提交
1318 1319 1320 1321

	cpu_buffer = buffer->buffers[cpu];

	if (atomic_read(&cpu_buffer->record_disabled))
1322
		goto out;
S
Steven Rostedt 已提交
1323 1324 1325

	length = rb_calculate_event_length(length);
	if (length > BUF_PAGE_SIZE)
S
Steven Rostedt 已提交
1326
		goto out;
S
Steven Rostedt 已提交
1327 1328 1329

	event = rb_reserve_next_event(cpu_buffer, RINGBUF_TYPE_DATA, length);
	if (!event)
1330
		goto out;
S
Steven Rostedt 已提交
1331

S
Steven Rostedt 已提交
1332 1333 1334 1335 1336 1337 1338 1339
	/*
	 * Need to store resched state on this cpu.
	 * Only the first needs to.
	 */

	if (preempt_count() == 1)
		per_cpu(rb_need_resched, cpu) = resched;

S
Steven Rostedt 已提交
1340 1341
	return event;

1342
 out:
1343
	ftrace_preempt_enable(resched);
S
Steven Rostedt 已提交
1344 1345
	return NULL;
}
1346
EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
S
Steven Rostedt 已提交
1347 1348 1349 1350 1351

static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
		      struct ring_buffer_event *event)
{
	cpu_buffer->entries++;
S
Steven Rostedt 已提交
1352 1353 1354 1355 1356 1357 1358 1359

	/* Only process further if we own the commit */
	if (!rb_is_commit(cpu_buffer, event))
		return;

	cpu_buffer->write_stamp += event->time_delta;

	rb_set_commit_to_write(cpu_buffer);
S
Steven Rostedt 已提交
1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
}

/**
 * ring_buffer_unlock_commit - commit a reserved
 * @buffer: The buffer to commit to
 * @event: The event pointer to commit.
 *
 * This commits the data to the ring buffer, and releases any locks held.
 *
 * Must be paired with ring_buffer_lock_reserve.
 */
int ring_buffer_unlock_commit(struct ring_buffer *buffer,
1372
			      struct ring_buffer_event *event)
S
Steven Rostedt 已提交
1373 1374 1375 1376 1377 1378 1379 1380
{
	struct ring_buffer_per_cpu *cpu_buffer;
	int cpu = raw_smp_processor_id();

	cpu_buffer = buffer->buffers[cpu];

	rb_commit(cpu_buffer, event);

S
Steven Rostedt 已提交
1381 1382 1383
	/*
	 * Only the last preempt count needs to restore preemption.
	 */
1384 1385 1386
	if (preempt_count() == 1)
		ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
	else
S
Steven Rostedt 已提交
1387
		preempt_enable_no_resched_notrace();
S
Steven Rostedt 已提交
1388 1389 1390

	return 0;
}
1391
EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
S
Steven Rostedt 已提交
1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411

/**
 * ring_buffer_write - write data to the buffer without reserving
 * @buffer: The ring buffer to write to.
 * @length: The length of the data being written (excluding the event header)
 * @data: The data to write to the buffer.
 *
 * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as
 * one function. If you already have the data to write to the buffer, it
 * may be easier to simply call this function.
 *
 * Note, like ring_buffer_lock_reserve, the length is the length of the data
 * and not the length of the event which would hold the header.
 */
int ring_buffer_write(struct ring_buffer *buffer,
			unsigned long length,
			void *data)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
1412
	unsigned long event_length;
S
Steven Rostedt 已提交
1413 1414
	void *body;
	int ret = -EBUSY;
S
Steven Rostedt 已提交
1415
	int cpu, resched;
S
Steven Rostedt 已提交
1416

1417
	if (ring_buffer_flags != RB_BUFFERS_ON)
1418 1419
		return -EBUSY;

S
Steven Rostedt 已提交
1420 1421 1422
	if (atomic_read(&buffer->record_disabled))
		return -EBUSY;

1423
	resched = ftrace_preempt_disable();
S
Steven Rostedt 已提交
1424

S
Steven Rostedt 已提交
1425 1426
	cpu = raw_smp_processor_id();

1427
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
1428
		goto out;
S
Steven Rostedt 已提交
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448

	cpu_buffer = buffer->buffers[cpu];

	if (atomic_read(&cpu_buffer->record_disabled))
		goto out;

	event_length = rb_calculate_event_length(length);
	event = rb_reserve_next_event(cpu_buffer,
				      RINGBUF_TYPE_DATA, event_length);
	if (!event)
		goto out;

	body = rb_event_data(event);

	memcpy(body, data, length);

	rb_commit(cpu_buffer, event);

	ret = 0;
 out:
1449
	ftrace_preempt_enable(resched);
S
Steven Rostedt 已提交
1450 1451 1452

	return ret;
}
1453
EXPORT_SYMBOL_GPL(ring_buffer_write);
S
Steven Rostedt 已提交
1454

1455
static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
{
	struct buffer_page *reader = cpu_buffer->reader_page;
	struct buffer_page *head = cpu_buffer->head_page;
	struct buffer_page *commit = cpu_buffer->commit_page;

	return reader->read == rb_page_commit(reader) &&
		(commit == reader ||
		 (commit == head &&
		  head->read == rb_page_commit(commit)));
}

S
Steven Rostedt 已提交
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479
/**
 * ring_buffer_record_disable - stop all writes into the buffer
 * @buffer: The ring buffer to stop writes to.
 *
 * This prevents all writes to the buffer. Any attempt to write
 * to the buffer after this will fail and return NULL.
 *
 * The caller should call synchronize_sched() after this.
 */
void ring_buffer_record_disable(struct ring_buffer *buffer)
{
	atomic_inc(&buffer->record_disabled);
}
1480
EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
S
Steven Rostedt 已提交
1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492

/**
 * ring_buffer_record_enable - enable writes to the buffer
 * @buffer: The ring buffer to enable writes
 *
 * Note, multiple disables will need the same number of enables
 * to truely enable the writing (much like preempt_disable).
 */
void ring_buffer_record_enable(struct ring_buffer *buffer)
{
	atomic_dec(&buffer->record_disabled);
}
1493
EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
S
Steven Rostedt 已提交
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508

/**
 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
 * @buffer: The ring buffer to stop writes to.
 * @cpu: The CPU buffer to stop
 *
 * This prevents all writes to the buffer. Any attempt to write
 * to the buffer after this will fail and return NULL.
 *
 * The caller should call synchronize_sched() after this.
 */
void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;

1509
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
1510 1511 1512 1513 1514
		return;

	cpu_buffer = buffer->buffers[cpu];
	atomic_inc(&cpu_buffer->record_disabled);
}
1515
EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
S
Steven Rostedt 已提交
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528

/**
 * ring_buffer_record_enable_cpu - enable writes to the buffer
 * @buffer: The ring buffer to enable writes
 * @cpu: The CPU to enable.
 *
 * Note, multiple disables will need the same number of enables
 * to truely enable the writing (much like preempt_disable).
 */
void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;

1529
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
1530 1531 1532 1533 1534
		return;

	cpu_buffer = buffer->buffers[cpu];
	atomic_dec(&cpu_buffer->record_disabled);
}
1535
EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
S
Steven Rostedt 已提交
1536 1537 1538 1539 1540 1541 1542 1543 1544 1545

/**
 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the entries from.
 */
unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;

1546
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
1547 1548 1549 1550 1551
		return 0;

	cpu_buffer = buffer->buffers[cpu];
	return cpu_buffer->entries;
}
1552
EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
S
Steven Rostedt 已提交
1553 1554 1555 1556 1557 1558 1559 1560 1561 1562

/**
 * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
 * @buffer: The ring buffer
 * @cpu: The per CPU buffer to get the number of overruns from
 */
unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;

1563
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
1564 1565 1566 1567 1568
		return 0;

	cpu_buffer = buffer->buffers[cpu];
	return cpu_buffer->overrun;
}
1569
EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
S
Steven Rostedt 已提交
1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591

/**
 * ring_buffer_entries - get the number of entries in a buffer
 * @buffer: The ring buffer
 *
 * Returns the total number of entries in the ring buffer
 * (all CPU entries)
 */
unsigned long ring_buffer_entries(struct ring_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long entries = 0;
	int cpu;

	/* if you care about this being correct, lock the buffer */
	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
		entries += cpu_buffer->entries;
	}

	return entries;
}
1592
EXPORT_SYMBOL_GPL(ring_buffer_entries);
S
Steven Rostedt 已提交
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614

/**
 * ring_buffer_overrun_cpu - get the number of overruns in buffer
 * @buffer: The ring buffer
 *
 * Returns the total number of overruns in the ring buffer
 * (all CPU entries)
 */
unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long overruns = 0;
	int cpu;

	/* if you care about this being correct, lock the buffer */
	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
		overruns += cpu_buffer->overrun;
	}

	return overruns;
}
1615
EXPORT_SYMBOL_GPL(ring_buffer_overruns);
S
Steven Rostedt 已提交
1616

1617
static void rb_iter_reset(struct ring_buffer_iter *iter)
S
Steven Rostedt 已提交
1618 1619 1620
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

1621 1622 1623
	/* Iterator usage is expected to have record disabled */
	if (list_empty(&cpu_buffer->reader_page->list)) {
		iter->head_page = cpu_buffer->head_page;
1624
		iter->head = cpu_buffer->head_page->read;
1625 1626
	} else {
		iter->head_page = cpu_buffer->reader_page;
1627
		iter->head = cpu_buffer->reader_page->read;
1628 1629 1630 1631
	}
	if (iter->head)
		iter->read_stamp = cpu_buffer->read_stamp;
	else
1632
		iter->read_stamp = iter->head_page->page->time_stamp;
1633
}
S
Steven Rostedt 已提交
1634

1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648
/**
 * ring_buffer_iter_reset - reset an iterator
 * @iter: The iterator to reset
 *
 * Resets the iterator, so that it will start from the beginning
 * again.
 */
void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
	unsigned long flags;

	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	rb_iter_reset(iter);
S
Steven Rostedt 已提交
1649
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
S
Steven Rostedt 已提交
1650
}
1651
EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
S
Steven Rostedt 已提交
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662

/**
 * ring_buffer_iter_empty - check if an iterator has no more to read
 * @iter: The iterator to check
 */
int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
{
	struct ring_buffer_per_cpu *cpu_buffer;

	cpu_buffer = iter->cpu_buffer;

S
Steven Rostedt 已提交
1663 1664
	return iter->head_page == cpu_buffer->commit_page &&
		iter->head == rb_commit_index(cpu_buffer);
S
Steven Rostedt 已提交
1665
}
1666
EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
S
Steven Rostedt 已提交
1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729

static void
rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
		     struct ring_buffer_event *event)
{
	u64 delta;

	switch (event->type) {
	case RINGBUF_TYPE_PADDING:
		return;

	case RINGBUF_TYPE_TIME_EXTEND:
		delta = event->array[0];
		delta <<= TS_SHIFT;
		delta += event->time_delta;
		cpu_buffer->read_stamp += delta;
		return;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
		return;

	case RINGBUF_TYPE_DATA:
		cpu_buffer->read_stamp += event->time_delta;
		return;

	default:
		BUG();
	}
	return;
}

static void
rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
			  struct ring_buffer_event *event)
{
	u64 delta;

	switch (event->type) {
	case RINGBUF_TYPE_PADDING:
		return;

	case RINGBUF_TYPE_TIME_EXTEND:
		delta = event->array[0];
		delta <<= TS_SHIFT;
		delta += event->time_delta;
		iter->read_stamp += delta;
		return;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
		return;

	case RINGBUF_TYPE_DATA:
		iter->read_stamp += event->time_delta;
		return;

	default:
		BUG();
	}
	return;
}

1730 1731
static struct buffer_page *
rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
S
Steven Rostedt 已提交
1732
{
1733 1734
	struct buffer_page *reader = NULL;
	unsigned long flags;
1735
	int nr_loops = 0;
1736

1737 1738
	local_irq_save(flags);
	__raw_spin_lock(&cpu_buffer->lock);
1739 1740

 again:
1741 1742 1743 1744 1745 1746
	/*
	 * This should normally only loop twice. But because the
	 * start of the reader inserts an empty page, it causes
	 * a case where we will loop three times. There should be no
	 * reason to loop four times (that I know of).
	 */
S
Steven Rostedt 已提交
1747
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
1748 1749 1750 1751
		reader = NULL;
		goto out;
	}

1752 1753 1754
	reader = cpu_buffer->reader_page;

	/* If there's more to read, return this page */
S
Steven Rostedt 已提交
1755
	if (cpu_buffer->reader_page->read < rb_page_size(reader))
1756 1757 1758
		goto out;

	/* Never should we have an index greater than the size */
S
Steven Rostedt 已提交
1759 1760 1761
	if (RB_WARN_ON(cpu_buffer,
		       cpu_buffer->reader_page->read > rb_page_size(reader)))
		goto out;
1762 1763 1764

	/* check if we caught up to the tail */
	reader = NULL;
S
Steven Rostedt 已提交
1765
	if (cpu_buffer->commit_page == cpu_buffer->reader_page)
1766
		goto out;
S
Steven Rostedt 已提交
1767 1768

	/*
1769 1770
	 * Splice the empty reader page into the list around the head.
	 * Reset the reader page to size zero.
S
Steven Rostedt 已提交
1771 1772
	 */

1773 1774 1775
	reader = cpu_buffer->head_page;
	cpu_buffer->reader_page->list.next = reader->list.next;
	cpu_buffer->reader_page->list.prev = reader->list.prev;
S
Steven Rostedt 已提交
1776 1777

	local_set(&cpu_buffer->reader_page->write, 0);
1778
	local_set(&cpu_buffer->reader_page->page->commit, 0);
S
Steven Rostedt 已提交
1779

1780 1781 1782
	/* Make the reader page now replace the head */
	reader->list.prev->next = &cpu_buffer->reader_page->list;
	reader->list.next->prev = &cpu_buffer->reader_page->list;
S
Steven Rostedt 已提交
1783 1784

	/*
1785 1786
	 * If the tail is on the reader, then we must set the head
	 * to the inserted page, otherwise we set it one before.
S
Steven Rostedt 已提交
1787
	 */
1788
	cpu_buffer->head_page = cpu_buffer->reader_page;
S
Steven Rostedt 已提交
1789

S
Steven Rostedt 已提交
1790
	if (cpu_buffer->commit_page != reader)
1791 1792 1793 1794 1795 1796 1797 1798 1799
		rb_inc_page(cpu_buffer, &cpu_buffer->head_page);

	/* Finally update the reader page to the new head */
	cpu_buffer->reader_page = reader;
	rb_reset_reader_page(cpu_buffer);

	goto again;

 out:
1800 1801
	__raw_spin_unlock(&cpu_buffer->lock);
	local_irq_restore(flags);
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812

	return reader;
}

static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
{
	struct ring_buffer_event *event;
	struct buffer_page *reader;
	unsigned length;

	reader = rb_get_reader_page(cpu_buffer);
S
Steven Rostedt 已提交
1813

1814
	/* This function should not be called when buffer is empty */
S
Steven Rostedt 已提交
1815 1816
	if (RB_WARN_ON(cpu_buffer, !reader))
		return;
S
Steven Rostedt 已提交
1817

1818 1819 1820 1821 1822 1823 1824 1825
	event = rb_reader_event(cpu_buffer);

	if (event->type == RINGBUF_TYPE_DATA)
		cpu_buffer->entries--;

	rb_update_read_stamp(cpu_buffer, event);

	length = rb_event_length(event);
1826
	cpu_buffer->reader_page->read += length;
S
Steven Rostedt 已提交
1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841
}

static void rb_advance_iter(struct ring_buffer_iter *iter)
{
	struct ring_buffer *buffer;
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
	unsigned length;

	cpu_buffer = iter->cpu_buffer;
	buffer = cpu_buffer->buffer;

	/*
	 * Check if we are at the end of the buffer.
	 */
S
Steven Rostedt 已提交
1842
	if (iter->head >= rb_page_size(iter->head_page)) {
S
Steven Rostedt 已提交
1843 1844 1845
		if (RB_WARN_ON(buffer,
			       iter->head_page == cpu_buffer->commit_page))
			return;
1846
		rb_inc_iter(iter);
S
Steven Rostedt 已提交
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857
		return;
	}

	event = rb_iter_head_event(iter);

	length = rb_event_length(event);

	/*
	 * This should not be called to advance the header if we are
	 * at the tail of the buffer.
	 */
S
Steven Rostedt 已提交
1858
	if (RB_WARN_ON(cpu_buffer,
1859
		       (iter->head_page == cpu_buffer->commit_page) &&
S
Steven Rostedt 已提交
1860 1861
		       (iter->head + length > rb_commit_index(cpu_buffer))))
		return;
S
Steven Rostedt 已提交
1862 1863 1864 1865 1866 1867

	rb_update_iter_read_stamp(iter, event);

	iter->head += length;

	/* check for end of page padding */
S
Steven Rostedt 已提交
1868 1869
	if ((iter->head >= rb_page_size(iter->head_page)) &&
	    (iter->head_page != cpu_buffer->commit_page))
S
Steven Rostedt 已提交
1870 1871 1872
		rb_advance_iter(iter);
}

S
Steven Rostedt 已提交
1873 1874
static struct ring_buffer_event *
rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
S
Steven Rostedt 已提交
1875 1876 1877
{
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
1878
	struct buffer_page *reader;
1879
	int nr_loops = 0;
S
Steven Rostedt 已提交
1880

1881
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
1882 1883 1884 1885 1886
		return NULL;

	cpu_buffer = buffer->buffers[cpu];

 again:
1887 1888 1889 1890 1891 1892 1893 1894
	/*
	 * We repeat when a timestamp is encountered. It is possible
	 * to get multiple timestamps from an interrupt entering just
	 * as one timestamp is about to be written. The max times
	 * that this can happen is the number of nested interrupts we
	 * can have.  Nesting 10 deep of interrupts is clearly
	 * an anomaly.
	 */
S
Steven Rostedt 已提交
1895
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 10))
1896 1897
		return NULL;

1898 1899
	reader = rb_get_reader_page(cpu_buffer);
	if (!reader)
S
Steven Rostedt 已提交
1900 1901
		return NULL;

1902
	event = rb_reader_event(cpu_buffer);
S
Steven Rostedt 已提交
1903 1904 1905

	switch (event->type) {
	case RINGBUF_TYPE_PADDING:
S
Steven Rostedt 已提交
1906
		RB_WARN_ON(cpu_buffer, 1);
1907 1908
		rb_advance_reader(cpu_buffer);
		return NULL;
S
Steven Rostedt 已提交
1909 1910 1911

	case RINGBUF_TYPE_TIME_EXTEND:
		/* Internal data, OK to advance */
1912
		rb_advance_reader(cpu_buffer);
S
Steven Rostedt 已提交
1913 1914 1915 1916
		goto again;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
1917
		rb_advance_reader(cpu_buffer);
S
Steven Rostedt 已提交
1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932
		goto again;

	case RINGBUF_TYPE_DATA:
		if (ts) {
			*ts = cpu_buffer->read_stamp + event->time_delta;
			ring_buffer_normalize_time_stamp(cpu_buffer->cpu, ts);
		}
		return event;

	default:
		BUG();
	}

	return NULL;
}
1933
EXPORT_SYMBOL_GPL(ring_buffer_peek);
S
Steven Rostedt 已提交
1934

S
Steven Rostedt 已提交
1935 1936
static struct ring_buffer_event *
rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
S
Steven Rostedt 已提交
1937 1938 1939 1940
{
	struct ring_buffer *buffer;
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_event *event;
1941
	int nr_loops = 0;
S
Steven Rostedt 已提交
1942 1943 1944 1945 1946 1947 1948 1949

	if (ring_buffer_iter_empty(iter))
		return NULL;

	cpu_buffer = iter->cpu_buffer;
	buffer = cpu_buffer->buffer;

 again:
1950 1951 1952 1953 1954 1955 1956 1957
	/*
	 * We repeat when a timestamp is encountered. It is possible
	 * to get multiple timestamps from an interrupt entering just
	 * as one timestamp is about to be written. The max times
	 * that this can happen is the number of nested interrupts we
	 * can have. Nesting 10 deep of interrupts is clearly
	 * an anomaly.
	 */
S
Steven Rostedt 已提交
1958
	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 10))
1959 1960
		return NULL;

S
Steven Rostedt 已提交
1961 1962 1963 1964 1965 1966 1967
	if (rb_per_cpu_empty(cpu_buffer))
		return NULL;

	event = rb_iter_head_event(iter);

	switch (event->type) {
	case RINGBUF_TYPE_PADDING:
1968
		rb_inc_iter(iter);
S
Steven Rostedt 已提交
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
		goto again;

	case RINGBUF_TYPE_TIME_EXTEND:
		/* Internal data, OK to advance */
		rb_advance_iter(iter);
		goto again;

	case RINGBUF_TYPE_TIME_STAMP:
		/* FIXME: not implemented */
		rb_advance_iter(iter);
		goto again;

	case RINGBUF_TYPE_DATA:
		if (ts) {
			*ts = iter->read_stamp + event->time_delta;
			ring_buffer_normalize_time_stamp(cpu_buffer->cpu, ts);
		}
		return event;

	default:
		BUG();
	}

	return NULL;
}
1994
EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
S
Steven Rostedt 已提交
1995

S
Steven Rostedt 已提交
1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
/**
 * ring_buffer_peek - peek at the next event to be read
 * @buffer: The ring buffer to read
 * @cpu: The cpu to peak at
 * @ts: The timestamp counter of this event.
 *
 * This will return the event that will be read next, but does
 * not consume the data.
 */
struct ring_buffer_event *
ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
{
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	struct ring_buffer_event *event;
	unsigned long flags;

	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	event = rb_buffer_peek(buffer, cpu, ts);
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

	return event;
}

/**
 * ring_buffer_iter_peek - peek at the next event to be read
 * @iter: The ring buffer iterator
 * @ts: The timestamp counter of this event.
 *
 * This will return the event that will be read next, but does
 * not increment the iterator.
 */
struct ring_buffer_event *
ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
	struct ring_buffer_event *event;
	unsigned long flags;

	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	event = rb_iter_peek(iter, ts);
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

	return event;
}

S
Steven Rostedt 已提交
2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051
/**
 * ring_buffer_consume - return an event and consume it
 * @buffer: The ring buffer to get the next event from
 *
 * Returns the next event in the ring buffer, and that event is consumed.
 * Meaning, that sequential reads will keep returning a different event,
 * and eventually empty the ring buffer if the producer is slower.
 */
struct ring_buffer_event *
ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
{
S
Steven Rostedt 已提交
2052
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
S
Steven Rostedt 已提交
2053
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
2054
	unsigned long flags;
S
Steven Rostedt 已提交
2055

2056
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
2057 2058
		return NULL;

S
Steven Rostedt 已提交
2059 2060 2061
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

	event = rb_buffer_peek(buffer, cpu, ts);
S
Steven Rostedt 已提交
2062
	if (!event)
S
Steven Rostedt 已提交
2063
		goto out;
S
Steven Rostedt 已提交
2064

2065
	rb_advance_reader(cpu_buffer);
S
Steven Rostedt 已提交
2066

S
Steven Rostedt 已提交
2067 2068 2069
 out:
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

S
Steven Rostedt 已提交
2070 2071
	return event;
}
2072
EXPORT_SYMBOL_GPL(ring_buffer_consume);
S
Steven Rostedt 已提交
2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090

/**
 * ring_buffer_read_start - start a non consuming read of the buffer
 * @buffer: The ring buffer to read from
 * @cpu: The cpu buffer to iterate over
 *
 * This starts up an iteration through the buffer. It also disables
 * the recording to the buffer until the reading is finished.
 * This prevents the reading from being corrupted. This is not
 * a consuming read, so a producer is not expected.
 *
 * Must be paired with ring_buffer_finish.
 */
struct ring_buffer_iter *
ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	struct ring_buffer_iter *iter;
2091
	unsigned long flags;
S
Steven Rostedt 已提交
2092

2093
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
		return NULL;

	iter = kmalloc(sizeof(*iter), GFP_KERNEL);
	if (!iter)
		return NULL;

	cpu_buffer = buffer->buffers[cpu];

	iter->cpu_buffer = cpu_buffer;

	atomic_inc(&cpu_buffer->record_disabled);
	synchronize_sched();

S
Steven Rostedt 已提交
2107
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2108
	__raw_spin_lock(&cpu_buffer->lock);
2109
	rb_iter_reset(iter);
2110
	__raw_spin_unlock(&cpu_buffer->lock);
S
Steven Rostedt 已提交
2111
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
S
Steven Rostedt 已提交
2112 2113 2114

	return iter;
}
2115
EXPORT_SYMBOL_GPL(ring_buffer_read_start);
S
Steven Rostedt 已提交
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131

/**
 * ring_buffer_finish - finish reading the iterator of the buffer
 * @iter: The iterator retrieved by ring_buffer_start
 *
 * This re-enables the recording to the buffer, and frees the
 * iterator.
 */
void
ring_buffer_read_finish(struct ring_buffer_iter *iter)
{
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;

	atomic_dec(&cpu_buffer->record_disabled);
	kfree(iter);
}
2132
EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
S
Steven Rostedt 已提交
2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144

/**
 * ring_buffer_read - read the next item in the ring buffer by the iterator
 * @iter: The ring buffer iterator
 * @ts: The time stamp of the event read.
 *
 * This reads the next event in the ring buffer and increments the iterator.
 */
struct ring_buffer_event *
ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
{
	struct ring_buffer_event *event;
S
Steven Rostedt 已提交
2145 2146
	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
	unsigned long flags;
S
Steven Rostedt 已提交
2147

S
Steven Rostedt 已提交
2148 2149
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
	event = rb_iter_peek(iter, ts);
S
Steven Rostedt 已提交
2150
	if (!event)
S
Steven Rostedt 已提交
2151
		goto out;
S
Steven Rostedt 已提交
2152 2153

	rb_advance_iter(iter);
S
Steven Rostedt 已提交
2154 2155
 out:
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
S
Steven Rostedt 已提交
2156 2157 2158

	return event;
}
2159
EXPORT_SYMBOL_GPL(ring_buffer_read);
S
Steven Rostedt 已提交
2160 2161 2162 2163 2164 2165 2166 2167 2168

/**
 * ring_buffer_size - return the size of the ring buffer (in bytes)
 * @buffer: The ring buffer.
 */
unsigned long ring_buffer_size(struct ring_buffer *buffer)
{
	return BUF_PAGE_SIZE * buffer->pages;
}
2169
EXPORT_SYMBOL_GPL(ring_buffer_size);
S
Steven Rostedt 已提交
2170 2171 2172 2173 2174 2175

static void
rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
{
	cpu_buffer->head_page
		= list_entry(cpu_buffer->pages.next, struct buffer_page, list);
S
Steven Rostedt 已提交
2176
	local_set(&cpu_buffer->head_page->write, 0);
2177
	local_set(&cpu_buffer->head_page->page->commit, 0);
2178

2179
	cpu_buffer->head_page->read = 0;
S
Steven Rostedt 已提交
2180 2181 2182 2183 2184 2185

	cpu_buffer->tail_page = cpu_buffer->head_page;
	cpu_buffer->commit_page = cpu_buffer->head_page;

	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
	local_set(&cpu_buffer->reader_page->write, 0);
2186
	local_set(&cpu_buffer->reader_page->page->commit, 0);
2187
	cpu_buffer->reader_page->read = 0;
S
Steven Rostedt 已提交
2188 2189 2190

	cpu_buffer->overrun = 0;
	cpu_buffer->entries = 0;
2191 2192 2193

	cpu_buffer->write_stamp = 0;
	cpu_buffer->read_stamp = 0;
S
Steven Rostedt 已提交
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
}

/**
 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
 * @buffer: The ring buffer to reset a per cpu buffer of
 * @cpu: The CPU buffer to be reset
 */
void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	unsigned long flags;

2206
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
2207 2208
		return;

S
Steven Rostedt 已提交
2209 2210
	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

2211
	__raw_spin_lock(&cpu_buffer->lock);
S
Steven Rostedt 已提交
2212 2213 2214

	rb_reset_cpu(cpu_buffer);

2215
	__raw_spin_unlock(&cpu_buffer->lock);
S
Steven Rostedt 已提交
2216 2217

	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
S
Steven Rostedt 已提交
2218
}
2219
EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
S
Steven Rostedt 已提交
2220 2221 2222 2223 2224 2225 2226 2227 2228 2229

/**
 * ring_buffer_reset - reset a ring buffer
 * @buffer: The ring buffer to reset all cpu buffers
 */
void ring_buffer_reset(struct ring_buffer *buffer)
{
	int cpu;

	for_each_buffer_cpu(buffer, cpu)
2230
		ring_buffer_reset_cpu(buffer, cpu);
S
Steven Rostedt 已提交
2231
}
2232
EXPORT_SYMBOL_GPL(ring_buffer_reset);
S
Steven Rostedt 已提交
2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250

/**
 * rind_buffer_empty - is the ring buffer empty?
 * @buffer: The ring buffer to test
 */
int ring_buffer_empty(struct ring_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	int cpu;

	/* yes this is racy, but if you don't like the race, lock the buffer */
	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
		if (!rb_per_cpu_empty(cpu_buffer))
			return 0;
	}
	return 1;
}
2251
EXPORT_SYMBOL_GPL(ring_buffer_empty);
S
Steven Rostedt 已提交
2252 2253 2254 2255 2256 2257 2258 2259 2260 2261

/**
 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
 * @buffer: The ring buffer
 * @cpu: The CPU buffer to test
 */
int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;

2262
	if (!cpumask_test_cpu(cpu, buffer->cpumask))
S
Steven Rostedt 已提交
2263 2264 2265 2266 2267
		return 1;

	cpu_buffer = buffer->buffers[cpu];
	return rb_per_cpu_empty(cpu_buffer);
}
2268
EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
S
Steven Rostedt 已提交
2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285

/**
 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
 * @buffer_a: One buffer to swap with
 * @buffer_b: The other buffer to swap with
 *
 * This function is useful for tracers that want to take a "snapshot"
 * of a CPU buffer and has another back up buffer lying around.
 * it is expected that the tracer handles the cpu buffer not being
 * used at the moment.
 */
int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
			 struct ring_buffer *buffer_b, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer_a;
	struct ring_buffer_per_cpu *cpu_buffer_b;

2286 2287
	if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
	    !cpumask_test_cpu(cpu, buffer_b->cpumask))
S
Steven Rostedt 已提交
2288 2289 2290
		return -EINVAL;

	/* At least make sure the two buffers are somewhat the same */
2291
	if (buffer_a->pages != buffer_b->pages)
S
Steven Rostedt 已提交
2292 2293
		return -EINVAL;

2294 2295 2296 2297 2298 2299 2300 2301 2302
	if (ring_buffer_flags != RB_BUFFERS_ON)
		return -EAGAIN;

	if (atomic_read(&buffer_a->record_disabled))
		return -EAGAIN;

	if (atomic_read(&buffer_b->record_disabled))
		return -EAGAIN;

S
Steven Rostedt 已提交
2303 2304 2305
	cpu_buffer_a = buffer_a->buffers[cpu];
	cpu_buffer_b = buffer_b->buffers[cpu];

2306 2307 2308 2309 2310 2311
	if (atomic_read(&cpu_buffer_a->record_disabled))
		return -EAGAIN;

	if (atomic_read(&cpu_buffer_b->record_disabled))
		return -EAGAIN;

S
Steven Rostedt 已提交
2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331
	/*
	 * We can't do a synchronize_sched here because this
	 * function can be called in atomic context.
	 * Normally this will be called from the same CPU as cpu.
	 * If not it's up to the caller to protect this.
	 */
	atomic_inc(&cpu_buffer_a->record_disabled);
	atomic_inc(&cpu_buffer_b->record_disabled);

	buffer_a->buffers[cpu] = cpu_buffer_b;
	buffer_b->buffers[cpu] = cpu_buffer_a;

	cpu_buffer_b->buffer = buffer_a;
	cpu_buffer_a->buffer = buffer_b;

	atomic_dec(&cpu_buffer_a->record_disabled);
	atomic_dec(&cpu_buffer_b->record_disabled);

	return 0;
}
2332
EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
S
Steven Rostedt 已提交
2333

S
Steven Rostedt 已提交
2334
static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer,
2335 2336
			      struct buffer_data_page *bpage,
			      unsigned int offset)
S
Steven Rostedt 已提交
2337 2338 2339 2340 2341
{
	struct ring_buffer_event *event;
	unsigned long head;

	__raw_spin_lock(&cpu_buffer->lock);
2342
	for (head = offset; head < local_read(&bpage->commit);
S
Steven Rostedt 已提交
2343 2344
	     head += rb_event_length(event)) {

2345
		event = __rb_data_page_index(bpage, head);
S
Steven Rostedt 已提交
2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373
		if (RB_WARN_ON(cpu_buffer, rb_null_event(event)))
			return;
		/* Only count data entries */
		if (event->type != RINGBUF_TYPE_DATA)
			continue;
		cpu_buffer->entries--;
	}
	__raw_spin_unlock(&cpu_buffer->lock);
}

/**
 * ring_buffer_alloc_read_page - allocate a page to read from buffer
 * @buffer: the buffer to allocate for.
 *
 * This function is used in conjunction with ring_buffer_read_page.
 * When reading a full page from the ring buffer, these functions
 * can be used to speed up the process. The calling function should
 * allocate a few pages first with this function. Then when it
 * needs to get pages from the ring buffer, it passes the result
 * of this function into ring_buffer_read_page, which will swap
 * the page that was allocated, with the read page of the buffer.
 *
 * Returns:
 *  The page allocated, or NULL on error.
 */
void *ring_buffer_alloc_read_page(struct ring_buffer *buffer)
{
	unsigned long addr;
2374
	struct buffer_data_page *bpage;
S
Steven Rostedt 已提交
2375 2376 2377 2378 2379

	addr = __get_free_page(GFP_KERNEL);
	if (!addr)
		return NULL;

2380
	bpage = (void *)addr;
S
Steven Rostedt 已提交
2381

2382
	return bpage;
S
Steven Rostedt 已提交
2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409
}

/**
 * ring_buffer_free_read_page - free an allocated read page
 * @buffer: the buffer the page was allocate for
 * @data: the page to free
 *
 * Free a page allocated from ring_buffer_alloc_read_page.
 */
void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
{
	free_page((unsigned long)data);
}

/**
 * ring_buffer_read_page - extract a page from the ring buffer
 * @buffer: buffer to extract from
 * @data_page: the page to use allocated from ring_buffer_alloc_read_page
 * @cpu: the cpu of the buffer to extract
 * @full: should the extraction only happen when the page is full.
 *
 * This function will pull out a page from the ring buffer and consume it.
 * @data_page must be the address of the variable that was returned
 * from ring_buffer_alloc_read_page. This is because the page might be used
 * to swap with a page in the ring buffer.
 *
 * for example:
L
Lai Jiangshan 已提交
2410
 *	rpage = ring_buffer_alloc_read_page(buffer);
S
Steven Rostedt 已提交
2411 2412 2413
 *	if (!rpage)
 *		return error;
 *	ret = ring_buffer_read_page(buffer, &rpage, cpu, 0);
2414 2415
 *	if (ret >= 0)
 *		process_page(rpage, ret);
S
Steven Rostedt 已提交
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425
 *
 * When @full is set, the function will not return true unless
 * the writer is off the reader page.
 *
 * Note: it is up to the calling functions to handle sleeps and wakeups.
 *  The ring buffer can be used anywhere in the kernel and can not
 *  blindly call wake_up. The layer that uses the ring buffer must be
 *  responsible for that.
 *
 * Returns:
2426 2427
 *  >=0 if data has been transferred, returns the offset of consumed data.
 *  <0 if no data has been transferred.
S
Steven Rostedt 已提交
2428 2429 2430 2431 2432 2433
 */
int ring_buffer_read_page(struct ring_buffer *buffer,
			    void **data_page, int cpu, int full)
{
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	struct ring_buffer_event *event;
2434
	struct buffer_data_page *bpage;
S
Steven Rostedt 已提交
2435
	unsigned long flags;
2436 2437
	unsigned int read;
	int ret = -1;
S
Steven Rostedt 已提交
2438 2439 2440 2441

	if (!data_page)
		return 0;

2442 2443
	bpage = *data_page;
	if (!bpage)
S
Steven Rostedt 已提交
2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
		return 0;

	spin_lock_irqsave(&cpu_buffer->reader_lock, flags);

	/*
	 * rb_buffer_peek will get the next ring buffer if
	 * the current reader page is empty.
	 */
	event = rb_buffer_peek(buffer, cpu, NULL);
	if (!event)
		goto out;

	/* check for data */
	if (!local_read(&cpu_buffer->reader_page->page->commit))
		goto out;
2459 2460

	read = cpu_buffer->reader_page->read;
S
Steven Rostedt 已提交
2461 2462 2463 2464 2465 2466
	/*
	 * If the writer is already off of the read page, then simply
	 * switch the read page with the given page. Otherwise
	 * we need to copy the data from the reader to the writer.
	 */
	if (cpu_buffer->reader_page == cpu_buffer->commit_page) {
L
Lai Jiangshan 已提交
2467
		unsigned int commit = rb_page_commit(cpu_buffer->reader_page);
2468
		struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
S
Steven Rostedt 已提交
2469 2470 2471 2472

		if (full)
			goto out;
		/* The writer is still on the reader page, we must copy */
2473
		memcpy(bpage->data + read, rpage->data + read, commit - read);
S
Steven Rostedt 已提交
2474 2475

		/* consume what was read */
L
Lai Jiangshan 已提交
2476
		cpu_buffer->reader_page->read = commit;
2477 2478 2479 2480 2481

		/* update bpage */
		local_set(&bpage->commit, commit);
		if (!read)
			bpage->time_stamp = rpage->time_stamp;
S
Steven Rostedt 已提交
2482 2483
	} else {
		/* swap the pages */
2484 2485
		rb_init_page(bpage);
		bpage = cpu_buffer->reader_page->page;
S
Steven Rostedt 已提交
2486 2487
		cpu_buffer->reader_page->page = *data_page;
		cpu_buffer->reader_page->read = 0;
2488
		*data_page = bpage;
S
Steven Rostedt 已提交
2489
	}
2490
	ret = read;
S
Steven Rostedt 已提交
2491 2492

	/* update the entry counter */
2493
	rb_remove_entries(cpu_buffer, bpage, read);
S
Steven Rostedt 已提交
2494 2495 2496 2497 2498 2499
 out:
	spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);

	return ret;
}

2500 2501 2502 2503
static ssize_t
rb_simple_read(struct file *filp, char __user *ubuf,
	       size_t cnt, loff_t *ppos)
{
2504
	unsigned long *p = filp->private_data;
2505 2506 2507
	char buf[64];
	int r;

2508 2509 2510 2511
	if (test_bit(RB_BUFFERS_DISABLED_BIT, p))
		r = sprintf(buf, "permanently disabled\n");
	else
		r = sprintf(buf, "%d\n", test_bit(RB_BUFFERS_ON_BIT, p));
2512 2513 2514 2515 2516 2517 2518 2519

	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}

static ssize_t
rb_simple_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
2520
	unsigned long *p = filp->private_data;
2521
	char buf[64];
2522
	unsigned long val;
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536
	int ret;

	if (cnt >= sizeof(buf))
		return -EINVAL;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	ret = strict_strtoul(buf, 10, &val);
	if (ret < 0)
		return ret;

2537 2538 2539 2540
	if (val)
		set_bit(RB_BUFFERS_ON_BIT, p);
	else
		clear_bit(RB_BUFFERS_ON_BIT, p);
2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561

	(*ppos)++;

	return cnt;
}

static struct file_operations rb_simple_fops = {
	.open		= tracing_open_generic,
	.read		= rb_simple_read,
	.write		= rb_simple_write,
};


static __init int rb_init_debugfs(void)
{
	struct dentry *d_tracer;
	struct dentry *entry;

	d_tracer = tracing_init_dentry();

	entry = debugfs_create_file("tracing_on", 0644, d_tracer,
2562
				    &ring_buffer_flags, &rb_simple_fops);
2563 2564 2565 2566 2567 2568 2569
	if (!entry)
		pr_warning("Could not create debugfs 'tracing_on' entry\n");

	return 0;
}

fs_initcall(rb_init_debugfs);