perf_event.c 154.1 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
P
Peter Zijlstra 已提交
16
#include <linux/idr.h>
17
#include <linux/file.h>
T
Thomas Gleixner 已提交
18
#include <linux/poll.h>
19
#include <linux/slab.h>
20
#include <linux/hash.h>
T
Thomas Gleixner 已提交
21
#include <linux/sysfs.h>
22
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
23
#include <linux/percpu.h>
24
#include <linux/ptrace.h>
P
Peter Zijlstra 已提交
25
#include <linux/reboot.h>
26
#include <linux/vmstat.h>
P
Peter Zijlstra 已提交
27
#include <linux/device.h>
28
#include <linux/vmalloc.h>
29 30
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
31 32 33
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
34
#include <linux/kernel_stat.h>
35
#include <linux/perf_event.h>
L
Li Zefan 已提交
36
#include <linux/ftrace_event.h>
37
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
38

39 40
#include <asm/irq_regs.h>

41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
struct remote_function_call {
	struct task_struct *p;
	int (*func)(void *info);
	void *info;
	int ret;
};

static void remote_function(void *data)
{
	struct remote_function_call *tfc = data;
	struct task_struct *p = tfc->p;

	if (p) {
		tfc->ret = -EAGAIN;
		if (task_cpu(p) != smp_processor_id() || !task_curr(p))
			return;
	}

	tfc->ret = tfc->func(tfc->info);
}

/**
 * task_function_call - call a function on the cpu on which a task runs
 * @p:		the task to evaluate
 * @func:	the function to be called
 * @info:	the function call argument
 *
 * Calls the function @func when the task is currently running. This might
 * be on the current CPU, which just calls the function directly
 *
 * returns: @func return value, or
 *	    -ESRCH  - when the process isn't running
 *	    -EAGAIN - when the process moved away
 */
static int
task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
{
	struct remote_function_call data = {
		.p = p,
		.func = func,
		.info = info,
		.ret = -ESRCH, /* No such (running) process */
	};

	if (task_curr(p))
		smp_call_function_single(task_cpu(p), remote_function, &data, 1);

	return data.ret;
}

/**
 * cpu_function_call - call a function on the cpu
 * @func:	the function to be called
 * @info:	the function call argument
 *
 * Calls the function @func on the remote cpu.
 *
 * returns: @func return value or -ENXIO when the cpu is offline
 */
static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
{
	struct remote_function_call data = {
		.p = NULL,
		.func = func,
		.info = info,
		.ret = -ENXIO, /* No such CPU */
	};

	smp_call_function_single(cpu, remote_function, &data, 1);

	return data.ret;
}

114 115 116 117 118 119
enum event_type_t {
	EVENT_FLEXIBLE = 0x1,
	EVENT_PINNED = 0x2,
	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
};

120
atomic_t perf_task_events __read_mostly;
121 122 123
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
124

P
Peter Zijlstra 已提交
125 126 127 128
static LIST_HEAD(pmus);
static DEFINE_MUTEX(pmus_lock);
static struct srcu_struct pmus_srcu;

129
/*
130
 * perf event paranoia level:
131 132
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
133
 *   1 - disallow cpu events for unpriv
134
 *   2 - disallow kernel profiling for unpriv
135
 */
136
int sysctl_perf_event_paranoid __read_mostly = 1;
137

138
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
139 140

/*
141
 * max perf event sample rate
142
 */
143
int sysctl_perf_event_sample_rate __read_mostly = 100000;
144

145
static atomic64_t perf_event_id;
146

147 148 149 150 151 152
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type);

static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
			     enum event_type_t event_type);

153
void __weak perf_event_print_debug(void)	{ }
T
Thomas Gleixner 已提交
154

155
extern __weak const char *perf_pmu_name(void)
T
Thomas Gleixner 已提交
156
{
157
	return "pmu";
T
Thomas Gleixner 已提交
158 159
}

160 161 162 163 164
static inline u64 perf_clock(void)
{
	return local_clock();
}

P
Peter Zijlstra 已提交
165
void perf_pmu_disable(struct pmu *pmu)
166
{
P
Peter Zijlstra 已提交
167 168 169
	int *count = this_cpu_ptr(pmu->pmu_disable_count);
	if (!(*count)++)
		pmu->pmu_disable(pmu);
170 171
}

P
Peter Zijlstra 已提交
172
void perf_pmu_enable(struct pmu *pmu)
173
{
P
Peter Zijlstra 已提交
174 175 176
	int *count = this_cpu_ptr(pmu->pmu_disable_count);
	if (!--(*count))
		pmu->pmu_enable(pmu);
177 178
}

179 180 181 182 183 184 185
static DEFINE_PER_CPU(struct list_head, rotation_list);

/*
 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
 * because they're strictly cpu affine and rotate_start is called with IRQs
 * disabled, while rotate_context is called from IRQ context.
 */
P
Peter Zijlstra 已提交
186
static void perf_pmu_rotate_start(struct pmu *pmu)
187
{
P
Peter Zijlstra 已提交
188
	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
189
	struct list_head *head = &__get_cpu_var(rotation_list);
190

191
	WARN_ON(!irqs_disabled());
192

193 194
	if (list_empty(&cpuctx->rotation_list))
		list_add(&cpuctx->rotation_list, head);
195 196
}

197
static void get_ctx(struct perf_event_context *ctx)
198
{
199
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
200 201
}

202 203
static void free_ctx(struct rcu_head *head)
{
204
	struct perf_event_context *ctx;
205

206
	ctx = container_of(head, struct perf_event_context, rcu_head);
207 208 209
	kfree(ctx);
}

210
static void put_ctx(struct perf_event_context *ctx)
211
{
212 213 214
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
215 216 217
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
218
	}
219 220
}

221
static void unclone_ctx(struct perf_event_context *ctx)
222 223 224 225 226 227 228
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
{
	/*
	 * only top level events have the pid namespace they were created in
	 */
	if (event->parent)
		event = event->parent;

	return task_tgid_nr_ns(p, event->ns);
}

static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
{
	/*
	 * only top level events have the pid namespace they were created in
	 */
	if (event->parent)
		event = event->parent;

	return task_pid_nr_ns(p, event->ns);
}

251
/*
252
 * If we inherit events we want to return the parent event id
253 254
 * to userspace.
 */
255
static u64 primary_event_id(struct perf_event *event)
256
{
257
	u64 id = event->id;
258

259 260
	if (event->parent)
		id = event->parent->id;
261 262 263 264

	return id;
}

265
/*
266
 * Get the perf_event_context for a task and lock it.
267 268 269
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
270
static struct perf_event_context *
P
Peter Zijlstra 已提交
271
perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
272
{
273
	struct perf_event_context *ctx;
274 275

	rcu_read_lock();
P
Peter Zijlstra 已提交
276
retry:
P
Peter Zijlstra 已提交
277
	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
278 279 280 281
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
282
		 * perf_event_task_sched_out, though the
283 284 285 286 287 288
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
289
		raw_spin_lock_irqsave(&ctx->lock, *flags);
P
Peter Zijlstra 已提交
290
		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
291
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
292 293
			goto retry;
		}
294 295

		if (!atomic_inc_not_zero(&ctx->refcount)) {
296
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
297 298
			ctx = NULL;
		}
299 300 301 302 303 304 305 306 307 308
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
P
Peter Zijlstra 已提交
309 310
static struct perf_event_context *
perf_pin_task_context(struct task_struct *task, int ctxn)
311
{
312
	struct perf_event_context *ctx;
313 314
	unsigned long flags;

P
Peter Zijlstra 已提交
315
	ctx = perf_lock_task_context(task, ctxn, &flags);
316 317
	if (ctx) {
		++ctx->pin_count;
318
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
319 320 321 322
	}
	return ctx;
}

323
static void perf_unpin_context(struct perf_event_context *ctx)
324 325 326
{
	unsigned long flags;

327
	raw_spin_lock_irqsave(&ctx->lock, flags);
328
	--ctx->pin_count;
329
	raw_spin_unlock_irqrestore(&ctx->lock, flags);
330 331
}

332 333 334 335 336 337 338 339 340 341 342
/*
 * Update the record of the current time in a context.
 */
static void update_context_time(struct perf_event_context *ctx)
{
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
}

343 344 345 346 347 348
static u64 perf_event_time(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	return ctx ? ctx->time : 0;
}

349 350 351 352 353 354 355 356 357 358 359 360
/*
 * Update the total_time_enabled and total_time_running fields for a event.
 */
static void update_event_times(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	u64 run_end;

	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
		return;

361
	if (ctx->is_active)
362
		run_end = perf_event_time(event);
363 364 365 366
	else
		run_end = event->tstamp_stopped;

	event->total_time_enabled = run_end - event->tstamp_enabled;
367 368 369 370

	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
	else
371
		run_end = perf_event_time(event);
372 373 374 375

	event->total_time_running = run_end - event->tstamp_running;
}

376 377 378 379 380 381 382 383 384 385 386 387
/*
 * Update total_time_enabled and total_time_running for all events in a group.
 */
static void update_group_times(struct perf_event *leader)
{
	struct perf_event *event;

	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
}

388 389 390 391 392 393 394 395 396
static struct list_head *
ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
{
	if (event->attr.pinned)
		return &ctx->pinned_groups;
	else
		return &ctx->flexible_groups;
}

397
/*
398
 * Add a event from the lists for its context.
399 400
 * Must be called with ctx->mutex and ctx->lock held.
 */
401
static void
402
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
403
{
404 405
	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
	event->attach_state |= PERF_ATTACH_CONTEXT;
406 407

	/*
408 409 410
	 * If we're a stand alone event or group leader, we go to the context
	 * list, group events are kept attached to the group so that
	 * perf_group_detach can, at all times, locate all siblings.
411
	 */
412
	if (event->group_leader == event) {
413 414
		struct list_head *list;

415 416 417
		if (is_software_event(event))
			event->group_flags |= PERF_GROUP_SOFTWARE;

418 419
		list = ctx_group_list(event, ctx);
		list_add_tail(&event->group_entry, list);
P
Peter Zijlstra 已提交
420
	}
P
Peter Zijlstra 已提交
421

422
	list_add_rcu(&event->event_entry, &ctx->event_list);
423
	if (!ctx->nr_events)
P
Peter Zijlstra 已提交
424
		perf_pmu_rotate_start(ctx->pmu);
425 426
	ctx->nr_events++;
	if (event->attr.inherit_stat)
427
		ctx->nr_stat++;
428 429
}

430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
/*
 * Called at perf_event creation and when events are attached/detached from a
 * group.
 */
static void perf_event__read_size(struct perf_event *event)
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		size += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		size += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_ID)
		entry += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
		size += sizeof(u64);
	}

	size += entry * nr;
	event->read_size = size;
}

static void perf_event__header_size(struct perf_event *event)
{
	struct perf_sample_data *data;
	u64 sample_type = event->attr.sample_type;
	u16 size = 0;

	perf_event__read_size(event);

	if (sample_type & PERF_SAMPLE_IP)
		size += sizeof(data->ip);

469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
	if (sample_type & PERF_SAMPLE_ADDR)
		size += sizeof(data->addr);

	if (sample_type & PERF_SAMPLE_PERIOD)
		size += sizeof(data->period);

	if (sample_type & PERF_SAMPLE_READ)
		size += event->read_size;

	event->header_size = size;
}

static void perf_event__id_header_size(struct perf_event *event)
{
	struct perf_sample_data *data;
	u64 sample_type = event->attr.sample_type;
	u16 size = 0;

487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
	if (sample_type & PERF_SAMPLE_TID)
		size += sizeof(data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		size += sizeof(data->time);

	if (sample_type & PERF_SAMPLE_ID)
		size += sizeof(data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		size += sizeof(data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		size += sizeof(data->cpu_entry);

502
	event->id_header_size = size;
503 504
}

505 506
static void perf_group_attach(struct perf_event *event)
{
507
	struct perf_event *group_leader = event->group_leader, *pos;
508

P
Peter Zijlstra 已提交
509 510 511 512 513 514
	/*
	 * We can have double attach due to group movement in perf_event_open.
	 */
	if (event->attach_state & PERF_ATTACH_GROUP)
		return;

515 516 517 518 519 520 521 522 523 524 525
	event->attach_state |= PERF_ATTACH_GROUP;

	if (group_leader == event)
		return;

	if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
			!is_software_event(event))
		group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;

	list_add_tail(&event->group_entry, &group_leader->sibling_list);
	group_leader->nr_siblings++;
526 527 528 529 530

	perf_event__header_size(group_leader);

	list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
		perf_event__header_size(pos);
531 532
}

533
/*
534
 * Remove a event from the lists for its context.
535
 * Must be called with ctx->mutex and ctx->lock held.
536
 */
537
static void
538
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
539
{
540 541 542 543
	/*
	 * We can have double detach due to exit/hot-unplug + close.
	 */
	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
544
		return;
545 546 547

	event->attach_state &= ~PERF_ATTACH_CONTEXT;

548 549
	ctx->nr_events--;
	if (event->attr.inherit_stat)
550
		ctx->nr_stat--;
551

552
	list_del_rcu(&event->event_entry);
553

554 555
	if (event->group_leader == event)
		list_del_init(&event->group_entry);
P
Peter Zijlstra 已提交
556

557
	update_group_times(event);
558 559 560 561 562 563 564 565 566 567

	/*
	 * If event was in error state, then keep it
	 * that way, otherwise bogus counts will be
	 * returned on read(). The only way to get out
	 * of error state is by explicit re-enabling
	 * of the event
	 */
	if (event->state > PERF_EVENT_STATE_OFF)
		event->state = PERF_EVENT_STATE_OFF;
568 569
}

570
static void perf_group_detach(struct perf_event *event)
571 572
{
	struct perf_event *sibling, *tmp;
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588
	struct list_head *list = NULL;

	/*
	 * We can have double detach due to exit/hot-unplug + close.
	 */
	if (!(event->attach_state & PERF_ATTACH_GROUP))
		return;

	event->attach_state &= ~PERF_ATTACH_GROUP;

	/*
	 * If this is a sibling, remove it from its group.
	 */
	if (event->group_leader != event) {
		list_del_init(&event->group_entry);
		event->group_leader->nr_siblings--;
589
		goto out;
590 591 592 593
	}

	if (!list_empty(&event->group_entry))
		list = &event->group_entry;
594

595
	/*
596 597
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
598
	 * to whatever list we are on.
599
	 */
600
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
601 602
		if (list)
			list_move_tail(&sibling->group_entry, list);
603
		sibling->group_leader = sibling;
604 605 606

		/* Inherit group flags from the previous leader */
		sibling->group_flags = event->group_flags;
607
	}
608 609 610 611 612 613

out:
	perf_event__header_size(event->group_leader);

	list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
		perf_event__header_size(tmp);
614 615
}

616 617 618 619 620 621
static inline int
event_filter_match(struct perf_event *event)
{
	return event->cpu == -1 || event->cpu == smp_processor_id();
}

622 623
static void
event_sched_out(struct perf_event *event,
624
		  struct perf_cpu_context *cpuctx,
625
		  struct perf_event_context *ctx)
626
{
627
	u64 tstamp = perf_event_time(event);
628 629 630 631 632 633 634 635 636 637 638
	u64 delta;
	/*
	 * An event which could not be activated because of
	 * filter mismatch still needs to have its timings
	 * maintained, otherwise bogus information is return
	 * via read() for time_enabled, time_running:
	 */
	if (event->state == PERF_EVENT_STATE_INACTIVE
	    && !event_filter_match(event)) {
		delta = ctx->time - event->tstamp_stopped;
		event->tstamp_running += delta;
639
		event->tstamp_stopped = tstamp;
640 641
	}

642
	if (event->state != PERF_EVENT_STATE_ACTIVE)
643
		return;
644

645 646 647 648
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
649
	}
650
	event->tstamp_stopped = tstamp;
P
Peter Zijlstra 已提交
651
	event->pmu->del(event, 0);
652
	event->oncpu = -1;
653

654
	if (!is_software_event(event))
655 656
		cpuctx->active_oncpu--;
	ctx->nr_active--;
657
	if (event->attr.exclusive || !cpuctx->active_oncpu)
658 659 660
		cpuctx->exclusive = 0;
}

661
static void
662
group_sched_out(struct perf_event *group_event,
663
		struct perf_cpu_context *cpuctx,
664
		struct perf_event_context *ctx)
665
{
666
	struct perf_event *event;
667
	int state = group_event->state;
668

669
	event_sched_out(group_event, cpuctx, ctx);
670 671 672 673

	/*
	 * Schedule out siblings (if any):
	 */
674 675
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
676

677
	if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
678 679 680
		cpuctx->exclusive = 0;
}

P
Peter Zijlstra 已提交
681 682 683 684 685 686
static inline struct perf_cpu_context *
__get_cpu_context(struct perf_event_context *ctx)
{
	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
}

T
Thomas Gleixner 已提交
687
/*
688
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
689
 *
690
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
691 692
 * remove it from the context list.
 */
693
static int __perf_remove_from_context(void *info)
T
Thomas Gleixner 已提交
694
{
695 696
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
P
Peter Zijlstra 已提交
697
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
T
Thomas Gleixner 已提交
698

699
	raw_spin_lock(&ctx->lock);
700 701
	event_sched_out(event, cpuctx, ctx);
	list_del_event(event, ctx);
702
	raw_spin_unlock(&ctx->lock);
703 704

	return 0;
T
Thomas Gleixner 已提交
705 706 707 708
}


/*
709
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
710
 *
711
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
712
 * call when the task is on a CPU.
713
 *
714 715
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
716 717
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
718
 * When called from perf_event_exit_task, it's OK because the
719
 * context has been detached from its task.
T
Thomas Gleixner 已提交
720
 */
721
static void perf_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
722
{
723
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
724 725
	struct task_struct *task = ctx->task;

726 727
	lockdep_assert_held(&ctx->mutex);

T
Thomas Gleixner 已提交
728 729
	if (!task) {
		/*
730
		 * Per cpu events are removed via an smp call and
731
		 * the removal is always successful.
T
Thomas Gleixner 已提交
732
		 */
733
		cpu_function_call(event->cpu, __perf_remove_from_context, event);
T
Thomas Gleixner 已提交
734 735 736 737
		return;
	}

retry:
738 739
	if (!task_function_call(task, __perf_remove_from_context, event))
		return;
T
Thomas Gleixner 已提交
740

741
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
742
	/*
743 744
	 * If we failed to find a running task, but find the context active now
	 * that we've acquired the ctx->lock, retry.
T
Thomas Gleixner 已提交
745
	 */
746
	if (ctx->is_active) {
747
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
748 749 750 751
		goto retry;
	}

	/*
752 753
	 * Since the task isn't running, its safe to remove the event, us
	 * holding the ctx->lock ensures the task won't get scheduled in.
T
Thomas Gleixner 已提交
754
	 */
755
	list_del_event(event, ctx);
756
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
757 758
}

759
/*
760
 * Cross CPU call to disable a performance event
761
 */
762
static int __perf_event_disable(void *info)
763
{
764 765
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
P
Peter Zijlstra 已提交
766
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
767 768

	/*
769 770
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
771 772 773
	 *
	 * Can trigger due to concurrent perf_event_context_sched_out()
	 * flipping contexts around.
774
	 */
775
	if (ctx->task && cpuctx->task_ctx != ctx)
776
		return -EINVAL;
777

778
	raw_spin_lock(&ctx->lock);
779 780

	/*
781
	 * If the event is on, turn it off.
782 783
	 * If it is in error state, leave it in error state.
	 */
784
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
785
		update_context_time(ctx);
786 787 788
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
789
		else
790 791
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
792 793
	}

794
	raw_spin_unlock(&ctx->lock);
795 796

	return 0;
797 798 799
}

/*
800
 * Disable a event.
801
 *
802 803
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
804
 * remains valid.  This condition is satisifed when called through
805 806 807 808
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
809
 * is the current context on this CPU and preemption is disabled,
810
 * hence we can't get into perf_event_task_sched_out for this context.
811
 */
812
void perf_event_disable(struct perf_event *event)
813
{
814
	struct perf_event_context *ctx = event->ctx;
815 816 817 818
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
819
		 * Disable the event on the cpu that it's on
820
		 */
821
		cpu_function_call(event->cpu, __perf_event_disable, event);
822 823 824
		return;
	}

P
Peter Zijlstra 已提交
825
retry:
826 827
	if (!task_function_call(task, __perf_event_disable, event))
		return;
828

829
	raw_spin_lock_irq(&ctx->lock);
830
	/*
831
	 * If the event is still active, we need to retry the cross-call.
832
	 */
833
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
834
		raw_spin_unlock_irq(&ctx->lock);
835 836 837 838 839
		/*
		 * Reload the task pointer, it might have been changed by
		 * a concurrent perf_event_context_sched_out().
		 */
		task = ctx->task;
840 841 842 843 844 845 846
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
847 848 849
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
850
	}
851
	raw_spin_unlock_irq(&ctx->lock);
852 853
}

P
Peter Zijlstra 已提交
854 855 856 857
#define MAX_INTERRUPTS (~0ULL)

static void perf_log_throttle(struct perf_event *event, int enable);

858
static int
859
event_sched_in(struct perf_event *event,
860
		 struct perf_cpu_context *cpuctx,
861
		 struct perf_event_context *ctx)
862
{
863 864
	u64 tstamp = perf_event_time(event);

865
	if (event->state <= PERF_EVENT_STATE_OFF)
866 867
		return 0;

868
	event->state = PERF_EVENT_STATE_ACTIVE;
869
	event->oncpu = smp_processor_id();
P
Peter Zijlstra 已提交
870 871 872 873 874 875 876 877 878 879 880

	/*
	 * Unthrottle events, since we scheduled we might have missed several
	 * ticks already, also for a heavily scheduling task there is little
	 * guarantee it'll get a tick in a timely manner.
	 */
	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
		perf_log_throttle(event, 1);
		event->hw.interrupts = 0;
	}

881 882 883 884 885
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

P
Peter Zijlstra 已提交
886
	if (event->pmu->add(event, PERF_EF_START)) {
887 888
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
889 890 891
		return -EAGAIN;
	}

892
	event->tstamp_running += tstamp - event->tstamp_stopped;
893

894
	event->shadow_ctx_time = tstamp - ctx->timestamp;
895

896
	if (!is_software_event(event))
897
		cpuctx->active_oncpu++;
898 899
	ctx->nr_active++;

900
	if (event->attr.exclusive)
901 902
		cpuctx->exclusive = 1;

903 904 905
	return 0;
}

906
static int
907
group_sched_in(struct perf_event *group_event,
908
	       struct perf_cpu_context *cpuctx,
909
	       struct perf_event_context *ctx)
910
{
911
	struct perf_event *event, *partial_group = NULL;
P
Peter Zijlstra 已提交
912
	struct pmu *pmu = group_event->pmu;
913 914
	u64 now = ctx->time;
	bool simulate = false;
915

916
	if (group_event->state == PERF_EVENT_STATE_OFF)
917 918
		return 0;

P
Peter Zijlstra 已提交
919
	pmu->start_txn(pmu);
920

921
	if (event_sched_in(group_event, cpuctx, ctx)) {
P
Peter Zijlstra 已提交
922
		pmu->cancel_txn(pmu);
923
		return -EAGAIN;
924
	}
925 926 927 928

	/*
	 * Schedule in siblings as one group (if any):
	 */
929
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
930
		if (event_sched_in(event, cpuctx, ctx)) {
931
			partial_group = event;
932 933 934 935
			goto group_error;
		}
	}

936
	if (!pmu->commit_txn(pmu))
937
		return 0;
938

939 940 941 942
group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
943 944 945 946 947 948 949 950 951 952
	 * The events up to the failed event are scheduled out normally,
	 * tstamp_stopped will be updated.
	 *
	 * The failed events and the remaining siblings need to have
	 * their timings updated as if they had gone thru event_sched_in()
	 * and event_sched_out(). This is required to get consistent timings
	 * across the group. This also takes care of the case where the group
	 * could never be scheduled by ensuring tstamp_stopped is set to mark
	 * the time the event was actually stopped, such that time delta
	 * calculation in update_event_times() is correct.
953
	 */
954 955
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
956 957 958 959 960 961 962 963
			simulate = true;

		if (simulate) {
			event->tstamp_running += now - event->tstamp_stopped;
			event->tstamp_stopped = now;
		} else {
			event_sched_out(event, cpuctx, ctx);
		}
964
	}
965
	event_sched_out(group_event, cpuctx, ctx);
966

P
Peter Zijlstra 已提交
967
	pmu->cancel_txn(pmu);
968

969 970 971
	return -EAGAIN;
}

972
/*
973
 * Work out whether we can put this event group on the CPU now.
974
 */
975
static int group_can_go_on(struct perf_event *event,
976 977 978 979
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
980
	 * Groups consisting entirely of software events can always go on.
981
	 */
982
	if (event->group_flags & PERF_GROUP_SOFTWARE)
983 984 985
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
986
	 * events can go on.
987 988 989 990 991
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
992
	 * events on the CPU, it can't go on.
993
	 */
994
	if (event->attr.exclusive && cpuctx->active_oncpu)
995 996 997 998 999 1000 1001 1002
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

1003 1004
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
1005
{
1006 1007
	u64 tstamp = perf_event_time(event);

1008
	list_add_event(event, ctx);
1009
	perf_group_attach(event);
1010 1011 1012
	event->tstamp_enabled = tstamp;
	event->tstamp_running = tstamp;
	event->tstamp_stopped = tstamp;
1013 1014
}

1015 1016
static void perf_event_context_sched_in(struct perf_event_context *ctx);

T
Thomas Gleixner 已提交
1017
/*
1018
 * Cross CPU call to install and enable a performance event
1019 1020
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
1021
 */
1022
static int  __perf_install_in_context(void *info)
T
Thomas Gleixner 已提交
1023
{
1024 1025 1026
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
P
Peter Zijlstra 已提交
1027
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1028
	int err;
T
Thomas Gleixner 已提交
1029 1030

	/*
1031 1032 1033
	 * In case we're installing a new context to an already running task,
	 * could also happen before perf_event_task_sched_in() on architectures
	 * which do context switches with IRQs enabled.
T
Thomas Gleixner 已提交
1034
	 */
1035 1036
	if (ctx->task && !cpuctx->task_ctx)
		perf_event_context_sched_in(ctx);
T
Thomas Gleixner 已提交
1037

1038
	raw_spin_lock(&ctx->lock);
1039
	ctx->is_active = 1;
1040
	update_context_time(ctx);
T
Thomas Gleixner 已提交
1041

1042
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
1043

1044
	if (!event_filter_match(event))
1045 1046
		goto unlock;

1047
	/*
1048
	 * Don't put the event on if it is disabled or if
1049 1050
	 * it is in a group and the group isn't on.
	 */
1051 1052
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
1053 1054
		goto unlock;

1055
	/*
1056 1057 1058
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
1059
	 */
1060
	if (!group_can_go_on(event, cpuctx, 1))
1061 1062
		err = -EEXIST;
	else
1063
		err = event_sched_in(event, cpuctx, ctx);
1064

1065 1066
	if (err) {
		/*
1067
		 * This event couldn't go on.  If it is in a group
1068
		 * then we have to pull the whole group off.
1069
		 * If the event group is pinned then put it in error state.
1070
		 */
1071
		if (leader != event)
1072
			group_sched_out(leader, cpuctx, ctx);
1073
		if (leader->attr.pinned) {
1074
			update_group_times(leader);
1075
			leader->state = PERF_EVENT_STATE_ERROR;
1076
		}
1077
	}
T
Thomas Gleixner 已提交
1078

P
Peter Zijlstra 已提交
1079
unlock:
1080
	raw_spin_unlock(&ctx->lock);
1081 1082

	return 0;
T
Thomas Gleixner 已提交
1083 1084 1085
}

/*
1086
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
1087
 *
1088 1089
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
1090
 *
1091
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
1092 1093 1094 1095
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
 */
static void
1096 1097
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
1098 1099 1100 1101
			int cpu)
{
	struct task_struct *task = ctx->task;

1102 1103
	lockdep_assert_held(&ctx->mutex);

1104 1105
	event->ctx = ctx;

T
Thomas Gleixner 已提交
1106 1107
	if (!task) {
		/*
1108
		 * Per cpu events are installed via an smp call and
1109
		 * the install is always successful.
T
Thomas Gleixner 已提交
1110
		 */
1111
		cpu_function_call(cpu, __perf_install_in_context, event);
T
Thomas Gleixner 已提交
1112 1113 1114 1115
		return;
	}

retry:
1116 1117
	if (!task_function_call(task, __perf_install_in_context, event))
		return;
T
Thomas Gleixner 已提交
1118

1119
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1120
	/*
1121 1122
	 * If we failed to find a running task, but find the context active now
	 * that we've acquired the ctx->lock, retry.
T
Thomas Gleixner 已提交
1123
	 */
1124
	if (ctx->is_active) {
1125
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1126 1127 1128 1129
		goto retry;
	}

	/*
1130 1131
	 * Since the task isn't running, its safe to add the event, us holding
	 * the ctx->lock ensures the task won't get scheduled in.
T
Thomas Gleixner 已提交
1132
	 */
1133
	add_event_to_ctx(event, ctx);
1134
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1135 1136
}

1137
/*
1138
 * Put a event into inactive state and update time fields.
1139 1140 1141 1142 1143 1144
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
1145 1146
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
1147
{
1148
	struct perf_event *sub;
1149
	u64 tstamp = perf_event_time(event);
1150

1151
	event->state = PERF_EVENT_STATE_INACTIVE;
1152
	event->tstamp_enabled = tstamp - event->total_time_enabled;
P
Peter Zijlstra 已提交
1153
	list_for_each_entry(sub, &event->sibling_list, group_entry) {
1154 1155
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
P
Peter Zijlstra 已提交
1156
	}
1157 1158
}

1159
/*
1160
 * Cross CPU call to enable a performance event
1161
 */
1162
static int __perf_event_enable(void *info)
1163
{
1164 1165 1166
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
P
Peter Zijlstra 已提交
1167
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1168
	int err;
1169

1170 1171
	if (WARN_ON_ONCE(!ctx->is_active))
		return -EINVAL;
1172

1173
	raw_spin_lock(&ctx->lock);
1174
	update_context_time(ctx);
1175

1176
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1177
		goto unlock;
1178
	__perf_event_mark_enabled(event, ctx);
1179

1180
	if (!event_filter_match(event))
1181 1182
		goto unlock;

1183
	/*
1184
	 * If the event is in a group and isn't the group leader,
1185
	 * then don't put it on unless the group is on.
1186
	 */
1187
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1188
		goto unlock;
1189

1190
	if (!group_can_go_on(event, cpuctx, 1)) {
1191
		err = -EEXIST;
1192
	} else {
1193
		if (event == leader)
1194
			err = group_sched_in(event, cpuctx, ctx);
1195
		else
1196
			err = event_sched_in(event, cpuctx, ctx);
1197
	}
1198 1199 1200

	if (err) {
		/*
1201
		 * If this event can't go on and it's part of a
1202 1203
		 * group, then the whole group has to come off.
		 */
1204
		if (leader != event)
1205
			group_sched_out(leader, cpuctx, ctx);
1206
		if (leader->attr.pinned) {
1207
			update_group_times(leader);
1208
			leader->state = PERF_EVENT_STATE_ERROR;
1209
		}
1210 1211
	}

P
Peter Zijlstra 已提交
1212
unlock:
1213
	raw_spin_unlock(&ctx->lock);
1214 1215

	return 0;
1216 1217 1218
}

/*
1219
 * Enable a event.
1220
 *
1221 1222
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
1223
 * remains valid.  This condition is satisfied when called through
1224 1225
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
1226
 */
1227
void perf_event_enable(struct perf_event *event)
1228
{
1229
	struct perf_event_context *ctx = event->ctx;
1230 1231 1232 1233
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
1234
		 * Enable the event on the cpu that it's on
1235
		 */
1236
		cpu_function_call(event->cpu, __perf_event_enable, event);
1237 1238 1239
		return;
	}

1240
	raw_spin_lock_irq(&ctx->lock);
1241
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1242 1243 1244
		goto out;

	/*
1245 1246
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
1247 1248 1249 1250
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
1251 1252
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
1253

P
Peter Zijlstra 已提交
1254
retry:
1255 1256 1257 1258 1259
	if (!ctx->is_active) {
		__perf_event_mark_enabled(event, ctx);
		goto out;
	}

1260
	raw_spin_unlock_irq(&ctx->lock);
1261 1262 1263

	if (!task_function_call(task, __perf_event_enable, event))
		return;
1264

1265
	raw_spin_lock_irq(&ctx->lock);
1266 1267

	/*
1268
	 * If the context is active and the event is still off,
1269 1270
	 * we need to retry the cross-call.
	 */
1271 1272 1273 1274 1275 1276
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
		/*
		 * task could have been flipped by a concurrent
		 * perf_event_context_sched_out()
		 */
		task = ctx->task;
1277
		goto retry;
1278
	}
1279

P
Peter Zijlstra 已提交
1280
out:
1281
	raw_spin_unlock_irq(&ctx->lock);
1282 1283
}

1284
static int perf_event_refresh(struct perf_event *event, int refresh)
1285
{
1286
	/*
1287
	 * not supported on inherited events
1288
	 */
1289
	if (event->attr.inherit || !is_sampling_event(event))
1290 1291
		return -EINVAL;

1292 1293
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1294 1295

	return 0;
1296 1297
}

1298 1299 1300
static void ctx_sched_out(struct perf_event_context *ctx,
			  struct perf_cpu_context *cpuctx,
			  enum event_type_t event_type)
1301
{
1302
	struct perf_event *event;
1303

1304
	raw_spin_lock(&ctx->lock);
P
Peter Zijlstra 已提交
1305
	perf_pmu_disable(ctx->pmu);
1306
	ctx->is_active = 0;
1307
	if (likely(!ctx->nr_events))
1308
		goto out;
1309
	update_context_time(ctx);
1310

1311
	if (!ctx->nr_active)
1312
		goto out;
1313

P
Peter Zijlstra 已提交
1314
	if (event_type & EVENT_PINNED) {
1315 1316
		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
			group_sched_out(event, cpuctx, ctx);
P
Peter Zijlstra 已提交
1317
	}
1318

P
Peter Zijlstra 已提交
1319
	if (event_type & EVENT_FLEXIBLE) {
1320
		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1321
			group_sched_out(event, cpuctx, ctx);
P
Peter Zijlstra 已提交
1322 1323
	}
out:
P
Peter Zijlstra 已提交
1324
	perf_pmu_enable(ctx->pmu);
1325
	raw_spin_unlock(&ctx->lock);
1326 1327
}

1328 1329 1330
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1331 1332 1333 1334
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1335
 * in them directly with an fd; we can only enable/disable all
1336
 * events via prctl, or enable/disable all events in a family
1337 1338
 * via ioctl, which will have the same effect on both contexts.
 */
1339 1340
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1341 1342
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1343
		&& ctx1->parent_gen == ctx2->parent_gen
1344
		&& !ctx1->pin_count && !ctx2->pin_count;
1345 1346
}

1347 1348
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1349 1350 1351
{
	u64 value;

1352
	if (!event->attr.inherit_stat)
1353 1354 1355
		return;

	/*
1356
	 * Update the event value, we cannot use perf_event_read()
1357 1358
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1359
	 * we know the event must be on the current CPU, therefore we
1360 1361
	 * don't need to use it.
	 */
1362 1363
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1364 1365
		event->pmu->read(event);
		/* fall-through */
1366

1367 1368
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1369 1370 1371 1372 1373 1374 1375
		break;

	default:
		break;
	}

	/*
1376
	 * In order to keep per-task stats reliable we need to flip the event
1377 1378
	 * values when we flip the contexts.
	 */
1379 1380 1381
	value = local64_read(&next_event->count);
	value = local64_xchg(&event->count, value);
	local64_set(&next_event->count, value);
1382

1383 1384
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1385

1386
	/*
1387
	 * Since we swizzled the values, update the user visible data too.
1388
	 */
1389 1390
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1391 1392 1393 1394 1395
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1396 1397
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1398
{
1399
	struct perf_event *event, *next_event;
1400 1401 1402 1403

	if (!ctx->nr_stat)
		return;

1404 1405
	update_context_time(ctx);

1406 1407
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1408

1409 1410
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1411

1412 1413
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1414

1415
		__perf_event_sync_stat(event, next_event);
1416

1417 1418
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1419 1420 1421
	}
}

1422 1423
static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
					 struct task_struct *next)
T
Thomas Gleixner 已提交
1424
{
P
Peter Zijlstra 已提交
1425
	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1426 1427
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
P
Peter Zijlstra 已提交
1428
	struct perf_cpu_context *cpuctx;
1429
	int do_switch = 1;
T
Thomas Gleixner 已提交
1430

P
Peter Zijlstra 已提交
1431 1432
	if (likely(!ctx))
		return;
1433

P
Peter Zijlstra 已提交
1434 1435
	cpuctx = __get_cpu_context(ctx);
	if (!cpuctx->task_ctx)
T
Thomas Gleixner 已提交
1436 1437
		return;

1438 1439
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
P
Peter Zijlstra 已提交
1440
	next_ctx = next->perf_event_ctxp[ctxn];
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
1452 1453
		raw_spin_lock(&ctx->lock);
		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1454
		if (context_equiv(ctx, next_ctx)) {
1455 1456
			/*
			 * XXX do we need a memory barrier of sorts
1457
			 * wrt to rcu_dereference() of perf_event_ctxp
1458
			 */
P
Peter Zijlstra 已提交
1459 1460
			task->perf_event_ctxp[ctxn] = next_ctx;
			next->perf_event_ctxp[ctxn] = ctx;
1461 1462 1463
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1464

1465
			perf_event_sync_stat(ctx, next_ctx);
1466
		}
1467 1468
		raw_spin_unlock(&next_ctx->lock);
		raw_spin_unlock(&ctx->lock);
1469
	}
1470
	rcu_read_unlock();
1471

1472
	if (do_switch) {
1473
		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1474 1475
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1476 1477
}

P
Peter Zijlstra 已提交
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
#define for_each_task_context_nr(ctxn)					\
	for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)

/*
 * Called from scheduler to remove the events of the current task,
 * with interrupts disabled.
 *
 * We stop each event and update the event value in event->count.
 *
 * This does not protect us against NMI, but disable()
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
 */
1492 1493
void __perf_event_task_sched_out(struct task_struct *task,
				 struct task_struct *next)
P
Peter Zijlstra 已提交
1494 1495 1496 1497 1498 1499 1500
{
	int ctxn;

	for_each_task_context_nr(ctxn)
		perf_event_context_sched_out(task, ctxn, next);
}

1501 1502
static void task_ctx_sched_out(struct perf_event_context *ctx,
			       enum event_type_t event_type)
1503
{
P
Peter Zijlstra 已提交
1504
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1505

1506 1507
	if (!cpuctx->task_ctx)
		return;
1508 1509 1510 1511

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1512
	ctx_sched_out(ctx, cpuctx, event_type);
1513 1514 1515
	cpuctx->task_ctx = NULL;
}

1516 1517 1518 1519 1520 1521 1522
/*
 * Called with IRQs disabled
 */
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type)
{
	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1523 1524
}

1525
static void
1526
ctx_pinned_sched_in(struct perf_event_context *ctx,
1527
		    struct perf_cpu_context *cpuctx)
T
Thomas Gleixner 已提交
1528
{
1529
	struct perf_event *event;
T
Thomas Gleixner 已提交
1530

1531 1532
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF)
1533
			continue;
1534
		if (!event_filter_match(event))
1535 1536
			continue;

1537
		if (group_can_go_on(event, cpuctx, 1))
1538
			group_sched_in(event, cpuctx, ctx);
1539 1540 1541 1542 1543

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1544 1545 1546
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1547
		}
1548
	}
1549 1550 1551 1552
}

static void
ctx_flexible_sched_in(struct perf_event_context *ctx,
1553
		      struct perf_cpu_context *cpuctx)
1554 1555 1556
{
	struct perf_event *event;
	int can_add_hw = 1;
1557

1558 1559 1560
	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		/* Ignore events in OFF or ERROR state */
		if (event->state <= PERF_EVENT_STATE_OFF)
1561
			continue;
1562 1563
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1564
		 * of events:
1565
		 */
1566
		if (!event_filter_match(event))
T
Thomas Gleixner 已提交
1567 1568
			continue;

P
Peter Zijlstra 已提交
1569
		if (group_can_go_on(event, cpuctx, can_add_hw)) {
1570
			if (group_sched_in(event, cpuctx, ctx))
1571
				can_add_hw = 0;
P
Peter Zijlstra 已提交
1572
		}
T
Thomas Gleixner 已提交
1573
	}
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
}

static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
	     enum event_type_t event_type)
{
	raw_spin_lock(&ctx->lock);
	ctx->is_active = 1;
	if (likely(!ctx->nr_events))
		goto out;

	ctx->timestamp = perf_clock();

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
	if (event_type & EVENT_PINNED)
1593
		ctx_pinned_sched_in(ctx, cpuctx);
1594 1595 1596

	/* Then walk through the lower prio flexible groups */
	if (event_type & EVENT_FLEXIBLE)
1597
		ctx_flexible_sched_in(ctx, cpuctx);
1598

P
Peter Zijlstra 已提交
1599
out:
1600
	raw_spin_unlock(&ctx->lock);
1601 1602
}

1603 1604 1605 1606 1607 1608 1609 1610
static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
			     enum event_type_t event_type)
{
	struct perf_event_context *ctx = &cpuctx->ctx;

	ctx_sched_in(ctx, cpuctx, event_type);
}

P
Peter Zijlstra 已提交
1611
static void task_ctx_sched_in(struct perf_event_context *ctx,
1612 1613
			      enum event_type_t event_type)
{
P
Peter Zijlstra 已提交
1614
	struct perf_cpu_context *cpuctx;
1615

1616
	cpuctx = __get_cpu_context(ctx);
1617 1618
	if (cpuctx->task_ctx == ctx)
		return;
P
Peter Zijlstra 已提交
1619

1620 1621 1622
	ctx_sched_in(ctx, cpuctx, event_type);
	cpuctx->task_ctx = ctx;
}
T
Thomas Gleixner 已提交
1623

1624
static void perf_event_context_sched_in(struct perf_event_context *ctx)
1625
{
P
Peter Zijlstra 已提交
1626
	struct perf_cpu_context *cpuctx;
1627

P
Peter Zijlstra 已提交
1628
	cpuctx = __get_cpu_context(ctx);
1629 1630 1631
	if (cpuctx->task_ctx == ctx)
		return;

P
Peter Zijlstra 已提交
1632
	perf_pmu_disable(ctx->pmu);
1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644
	/*
	 * We want to keep the following priority order:
	 * cpu pinned (that don't need to move), task pinned,
	 * cpu flexible, task flexible.
	 */
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);

	ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
	ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);

	cpuctx->task_ctx = ctx;
1645

1646 1647 1648 1649
	/*
	 * Since these rotations are per-cpu, we need to ensure the
	 * cpu-context we got scheduled on is actually rotating.
	 */
P
Peter Zijlstra 已提交
1650
	perf_pmu_rotate_start(ctx->pmu);
P
Peter Zijlstra 已提交
1651
	perf_pmu_enable(ctx->pmu);
1652 1653
}

P
Peter Zijlstra 已提交
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664
/*
 * Called from scheduler to add the events of the current task
 * with interrupts disabled.
 *
 * We restore the event value and then enable it.
 *
 * This does not protect us against NMI, but enable()
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
 */
1665
void __perf_event_task_sched_in(struct task_struct *task)
P
Peter Zijlstra 已提交
1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
{
	struct perf_event_context *ctx;
	int ctxn;

	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (likely(!ctx))
			continue;

		perf_event_context_sched_in(ctx);
	}
1677 1678
}

1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
{
	u64 frequency = event->attr.sample_freq;
	u64 sec = NSEC_PER_SEC;
	u64 divisor, dividend;

	int count_fls, nsec_fls, frequency_fls, sec_fls;

	count_fls = fls64(count);
	nsec_fls = fls64(nsec);
	frequency_fls = fls64(frequency);
	sec_fls = 30;

	/*
	 * We got @count in @nsec, with a target of sample_freq HZ
	 * the target period becomes:
	 *
	 *             @count * 10^9
	 * period = -------------------
	 *          @nsec * sample_freq
	 *
	 */

	/*
	 * Reduce accuracy by one bit such that @a and @b converge
	 * to a similar magnitude.
	 */
1706
#define REDUCE_FLS(a, b)		\
1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
do {					\
	if (a##_fls > b##_fls) {	\
		a >>= 1;		\
		a##_fls--;		\
	} else {			\
		b >>= 1;		\
		b##_fls--;		\
	}				\
} while (0)

	/*
	 * Reduce accuracy until either term fits in a u64, then proceed with
	 * the other, so that finally we can do a u64/u64 division.
	 */
	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
		REDUCE_FLS(nsec, frequency);
		REDUCE_FLS(sec, count);
	}

	if (count_fls + sec_fls > 64) {
		divisor = nsec * frequency;

		while (count_fls + sec_fls > 64) {
			REDUCE_FLS(count, sec);
			divisor >>= 1;
		}

		dividend = count * sec;
	} else {
		dividend = count * sec;

		while (nsec_fls + frequency_fls > 64) {
			REDUCE_FLS(nsec, frequency);
			dividend >>= 1;
		}

		divisor = nsec * frequency;
	}

1746 1747 1748
	if (!divisor)
		return dividend;

1749 1750 1751 1752
	return div64_u64(dividend, divisor);
}

static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1753
{
1754
	struct hw_perf_event *hwc = &event->hw;
1755
	s64 period, sample_period;
1756 1757
	s64 delta;

1758
	period = perf_calculate_period(event, nsec, count);
1759 1760 1761 1762 1763 1764 1765 1766 1767 1768

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
1769

1770
	if (local64_read(&hwc->period_left) > 8*sample_period) {
P
Peter Zijlstra 已提交
1771
		event->pmu->stop(event, PERF_EF_UPDATE);
1772
		local64_set(&hwc->period_left, 0);
P
Peter Zijlstra 已提交
1773
		event->pmu->start(event, PERF_EF_RELOAD);
1774
	}
1775 1776
}

1777
static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
1778
{
1779 1780
	struct perf_event *event;
	struct hw_perf_event *hwc;
1781 1782
	u64 interrupts, now;
	s64 delta;
1783

1784
	raw_spin_lock(&ctx->lock);
1785
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1786
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1787 1788
			continue;

1789
		if (!event_filter_match(event))
1790 1791
			continue;

1792
		hwc = &event->hw;
1793 1794 1795

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1796

1797
		/*
1798
		 * unthrottle events on the tick
1799
		 */
1800
		if (interrupts == MAX_INTERRUPTS) {
1801
			perf_log_throttle(event, 1);
P
Peter Zijlstra 已提交
1802
			event->pmu->start(event, 0);
1803 1804
		}

1805
		if (!event->attr.freq || !event->attr.sample_freq)
1806 1807
			continue;

1808
		event->pmu->read(event);
1809
		now = local64_read(&event->count);
1810 1811
		delta = now - hwc->freq_count_stamp;
		hwc->freq_count_stamp = now;
1812

1813
		if (delta > 0)
1814
			perf_adjust_period(event, period, delta);
1815
	}
1816
	raw_spin_unlock(&ctx->lock);
1817 1818
}

1819
/*
1820
 * Round-robin a context's events:
1821
 */
1822
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1823
{
1824
	raw_spin_lock(&ctx->lock);
1825

1826 1827 1828 1829 1830 1831
	/*
	 * Rotate the first entry last of non-pinned groups. Rotation might be
	 * disabled by the inheritance code.
	 */
	if (!ctx->rotate_disable)
		list_rotate_left(&ctx->flexible_groups);
1832

1833
	raw_spin_unlock(&ctx->lock);
1834 1835
}

1836
/*
1837 1838 1839
 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
 * because they're strictly cpu affine and rotate_start is called with IRQs
 * disabled, while rotate_context is called from IRQ context.
1840
 */
1841
static void perf_rotate_context(struct perf_cpu_context *cpuctx)
1842
{
1843
	u64 interval = (u64)cpuctx->jiffies_interval * TICK_NSEC;
P
Peter Zijlstra 已提交
1844
	struct perf_event_context *ctx = NULL;
1845
	int rotate = 0, remove = 1;
1846

1847
	if (cpuctx->ctx.nr_events) {
1848
		remove = 0;
1849 1850 1851
		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
			rotate = 1;
	}
1852

P
Peter Zijlstra 已提交
1853
	ctx = cpuctx->task_ctx;
1854
	if (ctx && ctx->nr_events) {
1855
		remove = 0;
1856 1857 1858
		if (ctx->nr_events != ctx->nr_active)
			rotate = 1;
	}
1859

P
Peter Zijlstra 已提交
1860
	perf_pmu_disable(cpuctx->ctx.pmu);
1861
	perf_ctx_adjust_freq(&cpuctx->ctx, interval);
1862
	if (ctx)
1863
		perf_ctx_adjust_freq(ctx, interval);
1864

1865
	if (!rotate)
1866
		goto done;
1867

1868
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1869
	if (ctx)
1870
		task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
T
Thomas Gleixner 已提交
1871

1872
	rotate_ctx(&cpuctx->ctx);
1873 1874
	if (ctx)
		rotate_ctx(ctx);
1875

1876
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1877
	if (ctx)
P
Peter Zijlstra 已提交
1878
		task_ctx_sched_in(ctx, EVENT_FLEXIBLE);
1879 1880

done:
1881 1882 1883
	if (remove)
		list_del_init(&cpuctx->rotation_list);

P
Peter Zijlstra 已提交
1884
	perf_pmu_enable(cpuctx->ctx.pmu);
1885 1886 1887 1888 1889 1890
}

void perf_event_task_tick(void)
{
	struct list_head *head = &__get_cpu_var(rotation_list);
	struct perf_cpu_context *cpuctx, *tmp;
1891

1892 1893 1894 1895 1896 1897 1898
	WARN_ON(!irqs_disabled());

	list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
		if (cpuctx->jiffies_interval == 1 ||
				!(jiffies % cpuctx->jiffies_interval))
			perf_rotate_context(cpuctx);
	}
T
Thomas Gleixner 已提交
1899 1900
}

1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915
static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

	__perf_event_mark_enabled(event, ctx);

	return 1;
}

1916
/*
1917
 * Enable all of a task's events that have been marked enable-on-exec.
1918 1919
 * This expects task == current.
 */
P
Peter Zijlstra 已提交
1920
static void perf_event_enable_on_exec(struct perf_event_context *ctx)
1921
{
1922
	struct perf_event *event;
1923 1924
	unsigned long flags;
	int enabled = 0;
1925
	int ret;
1926 1927

	local_irq_save(flags);
1928
	if (!ctx || !ctx->nr_events)
1929 1930
		goto out;

P
Peter Zijlstra 已提交
1931
	task_ctx_sched_out(ctx, EVENT_ALL);
1932

1933
	raw_spin_lock(&ctx->lock);
1934

1935 1936 1937 1938 1939 1940 1941 1942 1943 1944
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
	}

	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
1945 1946 1947
	}

	/*
1948
	 * Unclone this context if we enabled any event.
1949
	 */
1950 1951
	if (enabled)
		unclone_ctx(ctx);
1952

1953
	raw_spin_unlock(&ctx->lock);
1954

P
Peter Zijlstra 已提交
1955
	perf_event_context_sched_in(ctx);
P
Peter Zijlstra 已提交
1956
out:
1957 1958 1959
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1960
/*
1961
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1962
 */
1963
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1964
{
1965 1966
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
P
Peter Zijlstra 已提交
1967
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
I
Ingo Molnar 已提交
1968

1969 1970 1971 1972
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1973 1974
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1975 1976 1977 1978
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

1979
	raw_spin_lock(&ctx->lock);
1980 1981
	if (ctx->is_active)
		update_context_time(ctx);
1982
	update_event_times(event);
1983 1984
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		event->pmu->read(event);
1985
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
1986 1987
}

P
Peter Zijlstra 已提交
1988 1989
static inline u64 perf_event_count(struct perf_event *event)
{
1990
	return local64_read(&event->count) + atomic64_read(&event->child_count);
P
Peter Zijlstra 已提交
1991 1992
}

1993
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1994 1995
{
	/*
1996 1997
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1998
	 */
1999 2000 2001 2002
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
2003 2004 2005
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

2006
		raw_spin_lock_irqsave(&ctx->lock, flags);
2007 2008 2009 2010 2011 2012 2013
		/*
		 * may read while context is not active
		 * (e.g., thread is blocked), in that case
		 * we cannot update context time
		 */
		if (ctx->is_active)
			update_context_time(ctx);
2014
		update_event_times(event);
2015
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
2016 2017
	}

P
Peter Zijlstra 已提交
2018
	return perf_event_count(event);
T
Thomas Gleixner 已提交
2019 2020
}

2021
/*
2022
 * Callchain support
2023
 */
2024 2025 2026 2027 2028 2029

struct callchain_cpus_entries {
	struct rcu_head			rcu_head;
	struct perf_callchain_entry	*cpu_entries[0];
};

2030
static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
2031 2032 2033 2034 2035 2036 2037
static atomic_t nr_callchain_events;
static DEFINE_MUTEX(callchain_mutex);
struct callchain_cpus_entries *callchain_cpus_entries;


__weak void perf_callchain_kernel(struct perf_callchain_entry *entry,
				  struct pt_regs *regs)
2038 2039 2040
{
}

2041 2042
__weak void perf_callchain_user(struct perf_callchain_entry *entry,
				struct pt_regs *regs)
T
Thomas Gleixner 已提交
2043
{
2044
}
T
Thomas Gleixner 已提交
2045

2046 2047 2048 2049
static void release_callchain_buffers_rcu(struct rcu_head *head)
{
	struct callchain_cpus_entries *entries;
	int cpu;
T
Thomas Gleixner 已提交
2050

2051
	entries = container_of(head, struct callchain_cpus_entries, rcu_head);
T
Thomas Gleixner 已提交
2052

2053 2054
	for_each_possible_cpu(cpu)
		kfree(entries->cpu_entries[cpu]);
T
Thomas Gleixner 已提交
2055

2056 2057
	kfree(entries);
}
T
Thomas Gleixner 已提交
2058

2059 2060 2061
static void release_callchain_buffers(void)
{
	struct callchain_cpus_entries *entries;
T
Thomas Gleixner 已提交
2062

2063 2064 2065 2066
	entries = callchain_cpus_entries;
	rcu_assign_pointer(callchain_cpus_entries, NULL);
	call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
}
T
Thomas Gleixner 已提交
2067

2068 2069 2070 2071 2072
static int alloc_callchain_buffers(void)
{
	int cpu;
	int size;
	struct callchain_cpus_entries *entries;
T
Thomas Gleixner 已提交
2073

2074
	/*
2075 2076 2077
	 * We can't use the percpu allocation API for data that can be
	 * accessed from NMI. Use a temporary manual per cpu allocation
	 * until that gets sorted out.
2078
	 */
2079
	size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
2080

2081 2082 2083
	entries = kzalloc(size, GFP_KERNEL);
	if (!entries)
		return -ENOMEM;
2084

2085
	size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS;
T
Thomas Gleixner 已提交
2086

2087 2088 2089 2090 2091
	for_each_possible_cpu(cpu) {
		entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
							 cpu_to_node(cpu));
		if (!entries->cpu_entries[cpu])
			goto fail;
2092 2093
	}

2094
	rcu_assign_pointer(callchain_cpus_entries, entries);
T
Thomas Gleixner 已提交
2095

2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229
	return 0;

fail:
	for_each_possible_cpu(cpu)
		kfree(entries->cpu_entries[cpu]);
	kfree(entries);

	return -ENOMEM;
}

static int get_callchain_buffers(void)
{
	int err = 0;
	int count;

	mutex_lock(&callchain_mutex);

	count = atomic_inc_return(&nr_callchain_events);
	if (WARN_ON_ONCE(count < 1)) {
		err = -EINVAL;
		goto exit;
	}

	if (count > 1) {
		/* If the allocation failed, give up */
		if (!callchain_cpus_entries)
			err = -ENOMEM;
		goto exit;
	}

	err = alloc_callchain_buffers();
	if (err)
		release_callchain_buffers();
exit:
	mutex_unlock(&callchain_mutex);

	return err;
}

static void put_callchain_buffers(void)
{
	if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
		release_callchain_buffers();
		mutex_unlock(&callchain_mutex);
	}
}

static int get_recursion_context(int *recursion)
{
	int rctx;

	if (in_nmi())
		rctx = 3;
	else if (in_irq())
		rctx = 2;
	else if (in_softirq())
		rctx = 1;
	else
		rctx = 0;

	if (recursion[rctx])
		return -1;

	recursion[rctx]++;
	barrier();

	return rctx;
}

static inline void put_recursion_context(int *recursion, int rctx)
{
	barrier();
	recursion[rctx]--;
}

static struct perf_callchain_entry *get_callchain_entry(int *rctx)
{
	int cpu;
	struct callchain_cpus_entries *entries;

	*rctx = get_recursion_context(__get_cpu_var(callchain_recursion));
	if (*rctx == -1)
		return NULL;

	entries = rcu_dereference(callchain_cpus_entries);
	if (!entries)
		return NULL;

	cpu = smp_processor_id();

	return &entries->cpu_entries[cpu][*rctx];
}

static void
put_callchain_entry(int rctx)
{
	put_recursion_context(__get_cpu_var(callchain_recursion), rctx);
}

static struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
{
	int rctx;
	struct perf_callchain_entry *entry;


	entry = get_callchain_entry(&rctx);
	if (rctx == -1)
		return NULL;

	if (!entry)
		goto exit_put;

	entry->nr = 0;

	if (!user_mode(regs)) {
		perf_callchain_store(entry, PERF_CONTEXT_KERNEL);
		perf_callchain_kernel(entry, regs);
		if (current->mm)
			regs = task_pt_regs(current);
		else
			regs = NULL;
	}

	if (regs) {
		perf_callchain_store(entry, PERF_CONTEXT_USER);
		perf_callchain_user(entry, regs);
	}

exit_put:
	put_callchain_entry(rctx);

	return entry;
}

2230
/*
2231
 * Initialize the perf_event context in a task_struct:
2232
 */
2233
static void __perf_event_init_context(struct perf_event_context *ctx)
2234
{
2235
	raw_spin_lock_init(&ctx->lock);
2236
	mutex_init(&ctx->mutex);
2237 2238
	INIT_LIST_HEAD(&ctx->pinned_groups);
	INIT_LIST_HEAD(&ctx->flexible_groups);
2239 2240
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255
}

static struct perf_event_context *
alloc_perf_context(struct pmu *pmu, struct task_struct *task)
{
	struct perf_event_context *ctx;

	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
	if (!ctx)
		return NULL;

	__perf_event_init_context(ctx);
	if (task) {
		ctx->task = task;
		get_task_struct(task);
T
Thomas Gleixner 已提交
2256
	}
2257 2258 2259
	ctx->pmu = pmu;

	return ctx;
2260 2261
}

2262 2263 2264 2265 2266
static struct task_struct *
find_lively_task_by_vpid(pid_t vpid)
{
	struct task_struct *task;
	int err;
T
Thomas Gleixner 已提交
2267 2268

	rcu_read_lock();
2269
	if (!vpid)
T
Thomas Gleixner 已提交
2270 2271
		task = current;
	else
2272
		task = find_task_by_vpid(vpid);
T
Thomas Gleixner 已提交
2273 2274 2275 2276 2277 2278 2279 2280
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

	/* Reuse ptrace permission checks for now. */
2281 2282 2283 2284
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

2285 2286 2287 2288 2289 2290 2291
	return task;
errout:
	put_task_struct(task);
	return ERR_PTR(err);

}

2292 2293 2294
/*
 * Returns a matching context with refcount and pincount.
 */
P
Peter Zijlstra 已提交
2295
static struct perf_event_context *
M
Matt Helsley 已提交
2296
find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
T
Thomas Gleixner 已提交
2297
{
2298
	struct perf_event_context *ctx;
2299
	struct perf_cpu_context *cpuctx;
2300
	unsigned long flags;
P
Peter Zijlstra 已提交
2301
	int ctxn, err;
T
Thomas Gleixner 已提交
2302

2303
	if (!task) {
2304
		/* Must be root to operate on a CPU event: */
2305
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
2306 2307 2308
			return ERR_PTR(-EACCES);

		/*
2309
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
2310 2311 2312
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
2313
		if (!cpu_online(cpu))
T
Thomas Gleixner 已提交
2314 2315
			return ERR_PTR(-ENODEV);

P
Peter Zijlstra 已提交
2316
		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
T
Thomas Gleixner 已提交
2317
		ctx = &cpuctx->ctx;
2318
		get_ctx(ctx);
2319
		++ctx->pin_count;
T
Thomas Gleixner 已提交
2320 2321 2322 2323

		return ctx;
	}

P
Peter Zijlstra 已提交
2324 2325 2326 2327 2328
	err = -EINVAL;
	ctxn = pmu->task_ctx_nr;
	if (ctxn < 0)
		goto errout;

P
Peter Zijlstra 已提交
2329
retry:
P
Peter Zijlstra 已提交
2330
	ctx = perf_lock_task_context(task, ctxn, &flags);
2331
	if (ctx) {
2332
		unclone_ctx(ctx);
2333
		++ctx->pin_count;
2334
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
2335 2336
	}

2337
	if (!ctx) {
2338
		ctx = alloc_perf_context(pmu, task);
2339 2340 2341
		err = -ENOMEM;
		if (!ctx)
			goto errout;
2342

2343
		get_ctx(ctx);
2344

2345 2346 2347 2348 2349 2350 2351 2352 2353 2354
		err = 0;
		mutex_lock(&task->perf_event_mutex);
		/*
		 * If it has already passed perf_event_exit_task().
		 * we must see PF_EXITING, it takes this mutex too.
		 */
		if (task->flags & PF_EXITING)
			err = -ESRCH;
		else if (task->perf_event_ctxp[ctxn])
			err = -EAGAIN;
2355 2356
		else {
			++ctx->pin_count;
2357
			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2358
		}
2359 2360 2361
		mutex_unlock(&task->perf_event_mutex);

		if (unlikely(err)) {
2362
			put_task_struct(task);
2363
			kfree(ctx);
2364 2365 2366 2367

			if (err == -EAGAIN)
				goto retry;
			goto errout;
2368 2369 2370
		}
	}

T
Thomas Gleixner 已提交
2371
	return ctx;
2372

P
Peter Zijlstra 已提交
2373
errout:
2374
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
2375 2376
}

L
Li Zefan 已提交
2377 2378
static void perf_event_free_filter(struct perf_event *event);

2379
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
2380
{
2381
	struct perf_event *event;
P
Peter Zijlstra 已提交
2382

2383 2384 2385
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
2386
	perf_event_free_filter(event);
2387
	kfree(event);
P
Peter Zijlstra 已提交
2388 2389
}

2390
static void perf_buffer_put(struct perf_buffer *buffer);
2391

2392
static void free_event(struct perf_event *event)
2393
{
2394
	irq_work_sync(&event->pending);
2395

2396
	if (!event->parent) {
2397 2398
		if (event->attach_state & PERF_ATTACH_TASK)
			jump_label_dec(&perf_task_events);
2399
		if (event->attr.mmap || event->attr.mmap_data)
2400 2401 2402 2403 2404
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
2405 2406
		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
			put_callchain_buffers();
2407
	}
2408

2409 2410 2411
	if (event->buffer) {
		perf_buffer_put(event->buffer);
		event->buffer = NULL;
2412 2413
	}

2414 2415
	if (event->destroy)
		event->destroy(event);
2416

P
Peter Zijlstra 已提交
2417 2418 2419
	if (event->ctx)
		put_ctx(event->ctx);

2420
	call_rcu(&event->rcu_head, free_event_rcu);
2421 2422
}

2423
int perf_event_release_kernel(struct perf_event *event)
T
Thomas Gleixner 已提交
2424
{
2425
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
2426

2427 2428 2429 2430 2431 2432
	/*
	 * Remove from the PMU, can't get re-enabled since we got
	 * here because the last ref went.
	 */
	perf_event_disable(event);

2433
	WARN_ON_ONCE(ctx->parent_ctx);
2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446
	/*
	 * There are two ways this annotation is useful:
	 *
	 *  1) there is a lock recursion from perf_event_exit_task
	 *     see the comment there.
	 *
	 *  2) there is a lock-inversion with mmap_sem through
	 *     perf_event_read_group(), which takes faults while
	 *     holding ctx->mutex, however this is called after
	 *     the last filedesc died, so there is no possibility
	 *     to trigger the AB-BA case.
	 */
	mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2447
	raw_spin_lock_irq(&ctx->lock);
2448
	perf_group_detach(event);
2449 2450
	list_del_event(event, ctx);
	raw_spin_unlock_irq(&ctx->lock);
2451
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
2452

2453
	free_event(event);
T
Thomas Gleixner 已提交
2454 2455 2456

	return 0;
}
2457
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
T
Thomas Gleixner 已提交
2458

2459 2460 2461 2462
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
2463
{
2464
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
2465
	struct task_struct *owner;
2466

2467
	file->private_data = NULL;
2468

P
Peter Zijlstra 已提交
2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501
	rcu_read_lock();
	owner = ACCESS_ONCE(event->owner);
	/*
	 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
	 * !owner it means the list deletion is complete and we can indeed
	 * free this event, otherwise we need to serialize on
	 * owner->perf_event_mutex.
	 */
	smp_read_barrier_depends();
	if (owner) {
		/*
		 * Since delayed_put_task_struct() also drops the last
		 * task reference we can safely take a new reference
		 * while holding the rcu_read_lock().
		 */
		get_task_struct(owner);
	}
	rcu_read_unlock();

	if (owner) {
		mutex_lock(&owner->perf_event_mutex);
		/*
		 * We have to re-check the event->owner field, if it is cleared
		 * we raced with perf_event_exit_task(), acquiring the mutex
		 * ensured they're done, and we can proceed with freeing the
		 * event.
		 */
		if (event->owner)
			list_del_init(&event->owner_entry);
		mutex_unlock(&owner->perf_event_mutex);
		put_task_struct(owner);
	}

2502
	return perf_event_release_kernel(event);
2503 2504
}

2505
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2506
{
2507
	struct perf_event *child;
2508 2509
	u64 total = 0;

2510 2511 2512
	*enabled = 0;
	*running = 0;

2513
	mutex_lock(&event->child_mutex);
2514
	total += perf_event_read(event);
2515 2516 2517 2518 2519 2520
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
2521
		total += perf_event_read(child);
2522 2523 2524
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
2525
	mutex_unlock(&event->child_mutex);
2526 2527 2528

	return total;
}
2529
EXPORT_SYMBOL_GPL(perf_event_read_value);
2530

2531
static int perf_event_read_group(struct perf_event *event,
2532 2533
				   u64 read_format, char __user *buf)
{
2534
	struct perf_event *leader = event->group_leader, *sub;
2535 2536
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
2537
	u64 values[5];
2538
	u64 count, enabled, running;
2539

2540
	mutex_lock(&ctx->mutex);
2541
	count = perf_event_read_value(leader, &enabled, &running);
2542 2543

	values[n++] = 1 + leader->nr_siblings;
2544 2545 2546 2547
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
2548 2549 2550
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
2551 2552 2553 2554

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
2555
		goto unlock;
2556

2557
	ret = size;
2558

2559
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2560
		n = 0;
2561

2562
		values[n++] = perf_event_read_value(sub, &enabled, &running);
2563 2564 2565 2566 2567
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

2568
		if (copy_to_user(buf + ret, values, size)) {
2569 2570 2571
			ret = -EFAULT;
			goto unlock;
		}
2572 2573

		ret += size;
2574
	}
2575 2576
unlock:
	mutex_unlock(&ctx->mutex);
2577

2578
	return ret;
2579 2580
}

2581
static int perf_event_read_one(struct perf_event *event,
2582 2583
				 u64 read_format, char __user *buf)
{
2584
	u64 enabled, running;
2585 2586 2587
	u64 values[4];
	int n = 0;

2588 2589 2590 2591 2592
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
2593
	if (read_format & PERF_FORMAT_ID)
2594
		values[n++] = primary_event_id(event);
2595 2596 2597 2598 2599 2600 2601

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
2602
/*
2603
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
2604 2605
 */
static ssize_t
2606
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
2607
{
2608
	u64 read_format = event->attr.read_format;
2609
	int ret;
T
Thomas Gleixner 已提交
2610

2611
	/*
2612
	 * Return end-of-file for a read on a event that is in
2613 2614 2615
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
2616
	if (event->state == PERF_EVENT_STATE_ERROR)
2617 2618
		return 0;

2619
	if (count < event->read_size)
2620 2621
		return -ENOSPC;

2622
	WARN_ON_ONCE(event->ctx->parent_ctx);
2623
	if (read_format & PERF_FORMAT_GROUP)
2624
		ret = perf_event_read_group(event, read_format, buf);
2625
	else
2626
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
2627

2628
	return ret;
T
Thomas Gleixner 已提交
2629 2630 2631 2632 2633
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
2634
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
2635

2636
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
2637 2638 2639 2640
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
2641
	struct perf_event *event = file->private_data;
2642
	struct perf_buffer *buffer;
2643
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
2644 2645

	rcu_read_lock();
2646 2647 2648
	buffer = rcu_dereference(event->buffer);
	if (buffer)
		events = atomic_xchg(&buffer->poll, 0);
P
Peter Zijlstra 已提交
2649
	rcu_read_unlock();
T
Thomas Gleixner 已提交
2650

2651
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
2652 2653 2654 2655

	return events;
}

2656
static void perf_event_reset(struct perf_event *event)
2657
{
2658
	(void)perf_event_read(event);
2659
	local64_set(&event->count, 0);
2660
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
2661 2662
}

2663
/*
2664 2665 2666 2667
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
2668
 */
2669 2670
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2671
{
2672
	struct perf_event *child;
P
Peter Zijlstra 已提交
2673

2674 2675 2676 2677
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
2678
		func(child);
2679
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
2680 2681
}

2682 2683
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2684
{
2685 2686
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
2687

2688 2689
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
2690
	event = event->group_leader;
2691

2692 2693 2694 2695
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
2696
	mutex_unlock(&ctx->mutex);
2697 2698
}

2699
static int perf_event_period(struct perf_event *event, u64 __user *arg)
2700
{
2701
	struct perf_event_context *ctx = event->ctx;
2702 2703 2704
	int ret = 0;
	u64 value;

2705
	if (!is_sampling_event(event))
2706 2707
		return -EINVAL;

2708
	if (copy_from_user(&value, arg, sizeof(value)))
2709 2710 2711 2712 2713
		return -EFAULT;

	if (!value)
		return -EINVAL;

2714
	raw_spin_lock_irq(&ctx->lock);
2715 2716
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
2717 2718 2719 2720
			ret = -EINVAL;
			goto unlock;
		}

2721
		event->attr.sample_freq = value;
2722
	} else {
2723 2724
		event->attr.sample_period = value;
		event->hw.sample_period = value;
2725 2726
	}
unlock:
2727
	raw_spin_unlock_irq(&ctx->lock);
2728 2729 2730 2731

	return ret;
}

2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752
static const struct file_operations perf_fops;

static struct perf_event *perf_fget_light(int fd, int *fput_needed)
{
	struct file *file;

	file = fget_light(fd, fput_needed);
	if (!file)
		return ERR_PTR(-EBADF);

	if (file->f_op != &perf_fops) {
		fput_light(file, *fput_needed);
		*fput_needed = 0;
		return ERR_PTR(-EBADF);
	}

	return file->private_data;
}

static int perf_event_set_output(struct perf_event *event,
				 struct perf_event *output_event);
L
Li Zefan 已提交
2753
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2754

2755 2756
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
2757 2758
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2759
	u32 flags = arg;
2760 2761

	switch (cmd) {
2762 2763
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2764
		break;
2765 2766
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2767
		break;
2768 2769
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2770
		break;
P
Peter Zijlstra 已提交
2771

2772 2773
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2774

2775 2776
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2777

2778
	case PERF_EVENT_IOC_SET_OUTPUT:
2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795
	{
		struct perf_event *output_event = NULL;
		int fput_needed = 0;
		int ret;

		if (arg != -1) {
			output_event = perf_fget_light(arg, &fput_needed);
			if (IS_ERR(output_event))
				return PTR_ERR(output_event);
		}

		ret = perf_event_set_output(event, output_event);
		if (output_event)
			fput_light(output_event->filp, fput_needed);

		return ret;
	}
2796

L
Li Zefan 已提交
2797 2798 2799
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2800
	default:
P
Peter Zijlstra 已提交
2801
		return -ENOTTY;
2802
	}
P
Peter Zijlstra 已提交
2803 2804

	if (flags & PERF_IOC_FLAG_GROUP)
2805
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2806
	else
2807
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2808 2809

	return 0;
2810 2811
}

2812
int perf_event_task_enable(void)
2813
{
2814
	struct perf_event *event;
2815

2816 2817 2818 2819
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2820 2821 2822 2823

	return 0;
}

2824
int perf_event_task_disable(void)
2825
{
2826
	struct perf_event *event;
2827

2828 2829 2830 2831
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2832 2833 2834 2835

	return 0;
}

2836 2837
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2838 2839
#endif

2840
static int perf_event_index(struct perf_event *event)
2841
{
P
Peter Zijlstra 已提交
2842 2843 2844
	if (event->hw.state & PERF_HES_STOPPED)
		return 0;

2845
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2846 2847
		return 0;

2848
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2849 2850
}

2851 2852 2853 2854 2855
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2856
void perf_event_update_userpage(struct perf_event *event)
2857
{
2858
	struct perf_event_mmap_page *userpg;
2859
	struct perf_buffer *buffer;
2860 2861

	rcu_read_lock();
2862 2863
	buffer = rcu_dereference(event->buffer);
	if (!buffer)
2864 2865
		goto unlock;

2866
	userpg = buffer->user_page;
2867

2868 2869 2870 2871 2872
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2873
	++userpg->lock;
2874
	barrier();
2875
	userpg->index = perf_event_index(event);
P
Peter Zijlstra 已提交
2876
	userpg->offset = perf_event_count(event);
2877
	if (event->state == PERF_EVENT_STATE_ACTIVE)
2878
		userpg->offset -= local64_read(&event->hw.prev_count);
2879

2880 2881
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2882

2883 2884
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2885

2886
	barrier();
2887
	++userpg->lock;
2888
	preempt_enable();
2889
unlock:
2890
	rcu_read_unlock();
2891 2892
}

2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
static unsigned long perf_data_size(struct perf_buffer *buffer);

static void
perf_buffer_init(struct perf_buffer *buffer, long watermark, int flags)
{
	long max_size = perf_data_size(buffer);

	if (watermark)
		buffer->watermark = min(max_size, watermark);

	if (!buffer->watermark)
		buffer->watermark = max_size / 2;

	if (flags & PERF_BUFFER_WRITABLE)
		buffer->writable = 1;

	atomic_set(&buffer->refcount, 1);
}

2912
#ifndef CONFIG_PERF_USE_VMALLOC
2913

2914 2915 2916
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2917

2918
static struct page *
2919
perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
2920
{
2921
	if (pgoff > buffer->nr_pages)
2922
		return NULL;
2923

2924
	if (pgoff == 0)
2925
		return virt_to_page(buffer->user_page);
2926

2927
	return virt_to_page(buffer->data_pages[pgoff - 1]);
2928 2929
}

2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942
static void *perf_mmap_alloc_page(int cpu)
{
	struct page *page;
	int node;

	node = (cpu == -1) ? cpu : cpu_to_node(cpu);
	page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
	if (!page)
		return NULL;

	return page_address(page);
}

2943
static struct perf_buffer *
2944
perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
2945
{
2946
	struct perf_buffer *buffer;
2947 2948 2949
	unsigned long size;
	int i;

2950
	size = sizeof(struct perf_buffer);
2951 2952
	size += nr_pages * sizeof(void *);

2953 2954
	buffer = kzalloc(size, GFP_KERNEL);
	if (!buffer)
2955 2956
		goto fail;

2957
	buffer->user_page = perf_mmap_alloc_page(cpu);
2958
	if (!buffer->user_page)
2959 2960 2961
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
2962
		buffer->data_pages[i] = perf_mmap_alloc_page(cpu);
2963
		if (!buffer->data_pages[i])
2964 2965 2966
			goto fail_data_pages;
	}

2967
	buffer->nr_pages = nr_pages;
2968

2969 2970
	perf_buffer_init(buffer, watermark, flags);

2971
	return buffer;
2972 2973 2974

fail_data_pages:
	for (i--; i >= 0; i--)
2975
		free_page((unsigned long)buffer->data_pages[i]);
2976

2977
	free_page((unsigned long)buffer->user_page);
2978 2979

fail_user_page:
2980
	kfree(buffer);
2981 2982

fail:
2983
	return NULL;
2984 2985
}

2986 2987
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2988
	struct page *page = virt_to_page((void *)addr);
2989 2990 2991 2992 2993

	page->mapping = NULL;
	__free_page(page);
}

2994
static void perf_buffer_free(struct perf_buffer *buffer)
2995 2996 2997
{
	int i;

2998 2999 3000 3001
	perf_mmap_free_page((unsigned long)buffer->user_page);
	for (i = 0; i < buffer->nr_pages; i++)
		perf_mmap_free_page((unsigned long)buffer->data_pages[i]);
	kfree(buffer);
3002 3003
}

3004
static inline int page_order(struct perf_buffer *buffer)
3005 3006 3007 3008
{
	return 0;
}

3009 3010 3011 3012 3013 3014 3015 3016
#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

3017
static inline int page_order(struct perf_buffer *buffer)
3018
{
3019
	return buffer->page_order;
3020 3021
}

3022
static struct page *
3023
perf_mmap_to_page(struct perf_buffer *buffer, unsigned long pgoff)
3024
{
3025
	if (pgoff > (1UL << page_order(buffer)))
3026 3027
		return NULL;

3028
	return vmalloc_to_page((void *)buffer->user_page + pgoff * PAGE_SIZE);
3029 3030 3031 3032 3033 3034 3035 3036 3037
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

3038
static void perf_buffer_free_work(struct work_struct *work)
3039
{
3040
	struct perf_buffer *buffer;
3041 3042 3043
	void *base;
	int i, nr;

3044 3045
	buffer = container_of(work, struct perf_buffer, work);
	nr = 1 << page_order(buffer);
3046

3047
	base = buffer->user_page;
3048 3049 3050 3051
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
3052
	kfree(buffer);
3053 3054
}

3055
static void perf_buffer_free(struct perf_buffer *buffer)
3056
{
3057
	schedule_work(&buffer->work);
3058 3059
}

3060
static struct perf_buffer *
3061
perf_buffer_alloc(int nr_pages, long watermark, int cpu, int flags)
3062
{
3063
	struct perf_buffer *buffer;
3064 3065 3066
	unsigned long size;
	void *all_buf;

3067
	size = sizeof(struct perf_buffer);
3068 3069
	size += sizeof(void *);

3070 3071
	buffer = kzalloc(size, GFP_KERNEL);
	if (!buffer)
3072 3073
		goto fail;

3074
	INIT_WORK(&buffer->work, perf_buffer_free_work);
3075 3076 3077 3078 3079

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

3080 3081 3082 3083
	buffer->user_page = all_buf;
	buffer->data_pages[0] = all_buf + PAGE_SIZE;
	buffer->page_order = ilog2(nr_pages);
	buffer->nr_pages = 1;
3084

3085 3086
	perf_buffer_init(buffer, watermark, flags);

3087
	return buffer;
3088 3089

fail_all_buf:
3090
	kfree(buffer);
3091 3092 3093 3094 3095 3096 3097

fail:
	return NULL;
}

#endif

3098
static unsigned long perf_data_size(struct perf_buffer *buffer)
3099
{
3100
	return buffer->nr_pages << (PAGE_SHIFT + page_order(buffer));
3101 3102
}

3103 3104 3105
static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
3106
	struct perf_buffer *buffer;
3107 3108 3109 3110 3111 3112 3113 3114 3115
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
3116 3117
	buffer = rcu_dereference(event->buffer);
	if (!buffer)
3118 3119 3120 3121 3122
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

3123
	vmf->page = perf_mmap_to_page(buffer, vmf->pgoff);
3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

3138
static void perf_buffer_free_rcu(struct rcu_head *rcu_head)
3139
{
3140
	struct perf_buffer *buffer;
3141

3142 3143
	buffer = container_of(rcu_head, struct perf_buffer, rcu_head);
	perf_buffer_free(buffer);
3144 3145
}

3146
static struct perf_buffer *perf_buffer_get(struct perf_event *event)
3147
{
3148
	struct perf_buffer *buffer;
3149

3150
	rcu_read_lock();
3151 3152 3153 3154
	buffer = rcu_dereference(event->buffer);
	if (buffer) {
		if (!atomic_inc_not_zero(&buffer->refcount))
			buffer = NULL;
3155 3156 3157
	}
	rcu_read_unlock();

3158
	return buffer;
3159 3160
}

3161
static void perf_buffer_put(struct perf_buffer *buffer)
3162
{
3163
	if (!atomic_dec_and_test(&buffer->refcount))
3164
		return;
3165

3166
	call_rcu(&buffer->rcu_head, perf_buffer_free_rcu);
3167 3168 3169 3170
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
3171
	struct perf_event *event = vma->vm_file->private_data;
3172

3173
	atomic_inc(&event->mmap_count);
3174 3175 3176 3177
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
3178
	struct perf_event *event = vma->vm_file->private_data;
3179

3180
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3181
		unsigned long size = perf_data_size(event->buffer);
3182
		struct user_struct *user = event->mmap_user;
3183
		struct perf_buffer *buffer = event->buffer;
3184

3185
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3186
		vma->vm_mm->locked_vm -= event->mmap_locked;
3187
		rcu_assign_pointer(event->buffer, NULL);
3188
		mutex_unlock(&event->mmap_mutex);
3189

3190
		perf_buffer_put(buffer);
3191
		free_uid(user);
3192
	}
3193 3194
}

3195
static const struct vm_operations_struct perf_mmap_vmops = {
3196 3197 3198 3199
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
3200 3201 3202 3203
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
3204
	struct perf_event *event = file->private_data;
3205
	unsigned long user_locked, user_lock_limit;
3206
	struct user_struct *user = current_user();
3207
	unsigned long locked, lock_limit;
3208
	struct perf_buffer *buffer;
3209 3210
	unsigned long vma_size;
	unsigned long nr_pages;
3211
	long user_extra, extra;
3212
	int ret = 0, flags = 0;
3213

3214 3215 3216 3217 3218 3219 3220 3221
	/*
	 * Don't allow mmap() of inherited per-task counters. This would
	 * create a performance issue due to all children writing to the
	 * same buffer.
	 */
	if (event->cpu == -1 && event->attr.inherit)
		return -EINVAL;

3222
	if (!(vma->vm_flags & VM_SHARED))
3223
		return -EINVAL;
3224 3225 3226 3227

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

3228
	/*
3229
	 * If we have buffer pages ensure they're a power-of-two number, so we
3230 3231 3232
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
3233 3234
		return -EINVAL;

3235
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
3236 3237
		return -EINVAL;

3238 3239
	if (vma->vm_pgoff != 0)
		return -EINVAL;
3240

3241 3242
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
3243 3244 3245
	if (event->buffer) {
		if (event->buffer->nr_pages == nr_pages)
			atomic_inc(&event->buffer->refcount);
3246
		else
3247 3248 3249 3250
			ret = -EINVAL;
		goto unlock;
	}

3251
	user_extra = nr_pages + 1;
3252
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
3253 3254 3255 3256 3257 3258

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

3259
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3260

3261 3262 3263
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
3264

3265
	lock_limit = rlimit(RLIMIT_MEMLOCK);
3266
	lock_limit >>= PAGE_SHIFT;
3267
	locked = vma->vm_mm->locked_vm + extra;
3268

3269 3270
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
3271 3272 3273
		ret = -EPERM;
		goto unlock;
	}
3274

3275
	WARN_ON(event->buffer);
3276

3277 3278 3279 3280 3281
	if (vma->vm_flags & VM_WRITE)
		flags |= PERF_BUFFER_WRITABLE;

	buffer = perf_buffer_alloc(nr_pages, event->attr.wakeup_watermark,
				   event->cpu, flags);
3282
	if (!buffer) {
3283
		ret = -ENOMEM;
3284
		goto unlock;
3285
	}
3286
	rcu_assign_pointer(event->buffer, buffer);
3287

3288 3289 3290 3291 3292
	atomic_long_add(user_extra, &user->locked_vm);
	event->mmap_locked = extra;
	event->mmap_user = get_current_user();
	vma->vm_mm->locked_vm += event->mmap_locked;

3293
unlock:
3294 3295
	if (!ret)
		atomic_inc(&event->mmap_count);
3296
	mutex_unlock(&event->mmap_mutex);
3297 3298 3299

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
3300 3301

	return ret;
3302 3303
}

P
Peter Zijlstra 已提交
3304 3305 3306
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
3307
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
3308 3309 3310
	int retval;

	mutex_lock(&inode->i_mutex);
3311
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
3312 3313 3314 3315 3316 3317 3318 3319
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
3320
static const struct file_operations perf_fops = {
3321
	.llseek			= no_llseek,
T
Thomas Gleixner 已提交
3322 3323 3324
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
3325 3326
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
3327
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
3328
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
3329 3330
};

3331
/*
3332
 * Perf event wakeup
3333 3334 3335 3336 3337
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

3338
void perf_event_wakeup(struct perf_event *event)
3339
{
3340
	wake_up_all(&event->waitq);
3341

3342 3343 3344
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
3345
	}
3346 3347
}

3348
static void perf_pending_event(struct irq_work *entry)
3349
{
3350 3351
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
3352

3353 3354 3355
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
3356 3357
	}

3358 3359 3360
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
3361 3362 3363
	}
}

3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384
/*
 * We assume there is only KVM supporting the callbacks.
 * Later on, we might change it to a list if there is
 * another virtualization implementation supporting the callbacks.
 */
struct perf_guest_info_callbacks *perf_guest_cbs;

int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = cbs;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);

int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = NULL;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);

3385 3386 3387
/*
 * Output
 */
3388
static bool perf_output_space(struct perf_buffer *buffer, unsigned long tail,
3389
			      unsigned long offset, unsigned long head)
3390 3391 3392
{
	unsigned long mask;

3393
	if (!buffer->writable)
3394 3395
		return true;

3396
	mask = perf_data_size(buffer) - 1;
3397 3398 3399 3400 3401 3402 3403 3404 3405 3406

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

3407
static void perf_output_wakeup(struct perf_output_handle *handle)
3408
{
3409
	atomic_set(&handle->buffer->poll, POLL_IN);
3410

3411
	if (handle->nmi) {
3412
		handle->event->pending_wakeup = 1;
3413
		irq_work_queue(&handle->event->pending);
3414
	} else
3415
		perf_event_wakeup(handle->event);
3416 3417
}

3418
/*
3419
 * We need to ensure a later event_id doesn't publish a head when a former
3420
 * event isn't done writing. However since we need to deal with NMIs we
3421 3422 3423
 * cannot fully serialize things.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
3424
 * event completes.
3425
 */
3426
static void perf_output_get_handle(struct perf_output_handle *handle)
3427
{
3428
	struct perf_buffer *buffer = handle->buffer;
3429

3430
	preempt_disable();
3431 3432
	local_inc(&buffer->nest);
	handle->wakeup = local_read(&buffer->wakeup);
3433 3434
}

3435
static void perf_output_put_handle(struct perf_output_handle *handle)
3436
{
3437
	struct perf_buffer *buffer = handle->buffer;
3438
	unsigned long head;
3439 3440

again:
3441
	head = local_read(&buffer->head);
3442 3443

	/*
3444
	 * IRQ/NMI can happen here, which means we can miss a head update.
3445 3446
	 */

3447
	if (!local_dec_and_test(&buffer->nest))
3448
		goto out;
3449 3450

	/*
3451
	 * Publish the known good head. Rely on the full barrier implied
3452
	 * by atomic_dec_and_test() order the buffer->head read and this
3453
	 * write.
3454
	 */
3455
	buffer->user_page->data_head = head;
3456

3457 3458
	/*
	 * Now check if we missed an update, rely on the (compiler)
3459
	 * barrier in atomic_dec_and_test() to re-read buffer->head.
3460
	 */
3461 3462
	if (unlikely(head != local_read(&buffer->head))) {
		local_inc(&buffer->nest);
3463 3464 3465
		goto again;
	}

3466
	if (handle->wakeup != local_read(&buffer->wakeup))
3467
		perf_output_wakeup(handle);
3468

P
Peter Zijlstra 已提交
3469
out:
3470
	preempt_enable();
3471 3472
}

3473
__always_inline void perf_output_copy(struct perf_output_handle *handle,
3474
		      const void *buf, unsigned int len)
3475
{
3476
	do {
3477
		unsigned long size = min_t(unsigned long, handle->size, len);
3478 3479 3480 3481 3482

		memcpy(handle->addr, buf, size);

		len -= size;
		handle->addr += size;
3483
		buf += size;
3484 3485
		handle->size -= size;
		if (!handle->size) {
3486
			struct perf_buffer *buffer = handle->buffer;
3487

3488
			handle->page++;
3489 3490 3491
			handle->page &= buffer->nr_pages - 1;
			handle->addr = buffer->data_pages[handle->page];
			handle->size = PAGE_SIZE << page_order(buffer);
3492 3493
		}
	} while (len);
3494 3495
}

3496 3497 3498
static void __perf_event_header__init_id(struct perf_event_header *header,
					 struct perf_sample_data *data,
					 struct perf_event *event)
3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525
{
	u64 sample_type = event->attr.sample_type;

	data->type = sample_type;
	header->size += event->id_header_size;

	if (sample_type & PERF_SAMPLE_TID) {
		/* namespace issues */
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
	}

	if (sample_type & PERF_SAMPLE_TIME)
		data->time = perf_clock();

	if (sample_type & PERF_SAMPLE_ID)
		data->id = primary_event_id(event);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		data->stream_id = event->id;

	if (sample_type & PERF_SAMPLE_CPU) {
		data->cpu_entry.cpu	 = raw_smp_processor_id();
		data->cpu_entry.reserved = 0;
	}
}

3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562
static void perf_event_header__init_id(struct perf_event_header *header,
				       struct perf_sample_data *data,
				       struct perf_event *event)
{
	if (event->attr.sample_id_all)
		__perf_event_header__init_id(header, data, event);
}

static void __perf_event__output_id_sample(struct perf_output_handle *handle,
					   struct perf_sample_data *data)
{
	u64 sample_type = data->type;

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);
}

static void perf_event__output_id_sample(struct perf_event *event,
					 struct perf_output_handle *handle,
					 struct perf_sample_data *sample)
{
	if (event->attr.sample_id_all)
		__perf_event__output_id_sample(handle, sample);
}

3563
int perf_output_begin(struct perf_output_handle *handle,
3564
		      struct perf_event *event, unsigned int size,
3565
		      int nmi, int sample)
3566
{
3567
	struct perf_buffer *buffer;
3568
	unsigned long tail, offset, head;
3569
	int have_lost;
3570
	struct perf_sample_data sample_data;
3571 3572 3573 3574 3575
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
3576

3577
	rcu_read_lock();
3578
	/*
3579
	 * For inherited events we send all the output towards the parent.
3580
	 */
3581 3582
	if (event->parent)
		event = event->parent;
3583

3584 3585
	buffer = rcu_dereference(event->buffer);
	if (!buffer)
3586 3587
		goto out;

3588
	handle->buffer	= buffer;
3589
	handle->event	= event;
3590 3591
	handle->nmi	= nmi;
	handle->sample	= sample;
3592

3593
	if (!buffer->nr_pages)
3594
		goto out;
3595

3596
	have_lost = local_read(&buffer->lost);
3597 3598 3599 3600 3601 3602
	if (have_lost) {
		lost_event.header.size = sizeof(lost_event);
		perf_event_header__init_id(&lost_event.header, &sample_data,
					   event);
		size += lost_event.header.size;
	}
3603

3604
	perf_output_get_handle(handle);
3605

3606
	do {
3607 3608 3609 3610 3611
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
3612
		tail = ACCESS_ONCE(buffer->user_page->data_tail);
3613
		smp_rmb();
3614
		offset = head = local_read(&buffer->head);
P
Peter Zijlstra 已提交
3615
		head += size;
3616
		if (unlikely(!perf_output_space(buffer, tail, offset, head)))
3617
			goto fail;
3618
	} while (local_cmpxchg(&buffer->head, offset, head) != offset);
3619

3620 3621
	if (head - local_read(&buffer->wakeup) > buffer->watermark)
		local_add(buffer->watermark, &buffer->wakeup);
3622

3623 3624 3625 3626
	handle->page = offset >> (PAGE_SHIFT + page_order(buffer));
	handle->page &= buffer->nr_pages - 1;
	handle->size = offset & ((PAGE_SIZE << page_order(buffer)) - 1);
	handle->addr = buffer->data_pages[handle->page];
3627
	handle->addr += handle->size;
3628
	handle->size = (PAGE_SIZE << page_order(buffer)) - handle->size;
3629

3630
	if (have_lost) {
3631
		lost_event.header.type = PERF_RECORD_LOST;
3632
		lost_event.header.misc = 0;
3633
		lost_event.id          = event->id;
3634
		lost_event.lost        = local_xchg(&buffer->lost, 0);
3635 3636

		perf_output_put(handle, lost_event);
3637
		perf_event__output_id_sample(event, handle, &sample_data);
3638 3639
	}

3640
	return 0;
3641

3642
fail:
3643
	local_inc(&buffer->lost);
3644
	perf_output_put_handle(handle);
3645 3646
out:
	rcu_read_unlock();
3647

3648 3649
	return -ENOSPC;
}
3650

3651
void perf_output_end(struct perf_output_handle *handle)
3652
{
3653
	struct perf_event *event = handle->event;
3654
	struct perf_buffer *buffer = handle->buffer;
3655

3656
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
3657

3658
	if (handle->sample && wakeup_events) {
3659
		int events = local_inc_return(&buffer->events);
P
Peter Zijlstra 已提交
3660
		if (events >= wakeup_events) {
3661 3662
			local_sub(wakeup_events, &buffer->events);
			local_inc(&buffer->wakeup);
P
Peter Zijlstra 已提交
3663
		}
3664 3665
	}

3666
	perf_output_put_handle(handle);
3667
	rcu_read_unlock();
3668 3669
}

3670
static void perf_output_read_one(struct perf_output_handle *handle,
3671 3672
				 struct perf_event *event,
				 u64 enabled, u64 running)
3673
{
3674
	u64 read_format = event->attr.read_format;
3675 3676 3677
	u64 values[4];
	int n = 0;

P
Peter Zijlstra 已提交
3678
	values[n++] = perf_event_count(event);
3679
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3680
		values[n++] = enabled +
3681
			atomic64_read(&event->child_total_time_enabled);
3682 3683
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3684
		values[n++] = running +
3685
			atomic64_read(&event->child_total_time_running);
3686 3687
	}
	if (read_format & PERF_FORMAT_ID)
3688
		values[n++] = primary_event_id(event);
3689 3690 3691 3692 3693

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
3694
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3695 3696
 */
static void perf_output_read_group(struct perf_output_handle *handle,
3697 3698
			    struct perf_event *event,
			    u64 enabled, u64 running)
3699
{
3700 3701
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
3702 3703 3704 3705 3706 3707
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3708
		values[n++] = enabled;
3709 3710

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3711
		values[n++] = running;
3712

3713
	if (leader != event)
3714 3715
		leader->pmu->read(leader);

P
Peter Zijlstra 已提交
3716
	values[n++] = perf_event_count(leader);
3717
	if (read_format & PERF_FORMAT_ID)
3718
		values[n++] = primary_event_id(leader);
3719 3720 3721

	perf_output_copy(handle, values, n * sizeof(u64));

3722
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3723 3724
		n = 0;

3725
		if (sub != event)
3726 3727
			sub->pmu->read(sub);

P
Peter Zijlstra 已提交
3728
		values[n++] = perf_event_count(sub);
3729
		if (read_format & PERF_FORMAT_ID)
3730
			values[n++] = primary_event_id(sub);
3731 3732 3733 3734 3735

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

3736 3737 3738
#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
				 PERF_FORMAT_TOTAL_TIME_RUNNING)

3739
static void perf_output_read(struct perf_output_handle *handle,
3740
			     struct perf_event *event)
3741
{
3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760
	u64 enabled = 0, running = 0, now, ctx_time;
	u64 read_format = event->attr.read_format;

	/*
	 * compute total_time_enabled, total_time_running
	 * based on snapshot values taken when the event
	 * was last scheduled in.
	 *
	 * we cannot simply called update_context_time()
	 * because of locking issue as we are called in
	 * NMI context
	 */
	if (read_format & PERF_FORMAT_TOTAL_TIMES) {
		now = perf_clock();
		ctx_time = event->shadow_ctx_time + now;
		enabled = ctx_time - event->tstamp_enabled;
		running = ctx_time - event->tstamp_running;
	}

3761
	if (event->attr.read_format & PERF_FORMAT_GROUP)
3762
		perf_output_read_group(handle, event, enabled, running);
3763
	else
3764
		perf_output_read_one(handle, event, enabled, running);
3765 3766
}

3767 3768 3769
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3770
			struct perf_event *event)
3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3801
		perf_output_read(handle, event);
3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3839
			 struct perf_event *event,
3840
			 struct pt_regs *regs)
3841
{
3842
	u64 sample_type = event->attr.sample_type;
3843

3844
	header->type = PERF_RECORD_SAMPLE;
3845
	header->size = sizeof(*header) + event->header_size;
3846 3847 3848

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3849

3850
	__perf_event_header__init_id(header, data, event);
3851

3852
	if (sample_type & PERF_SAMPLE_IP)
3853 3854
		data->ip = perf_instruction_pointer(regs);

3855
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3856
		int size = 1;
3857

3858 3859 3860 3861 3862 3863
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3864 3865
	}

3866
	if (sample_type & PERF_SAMPLE_RAW) {
3867 3868 3869 3870 3871 3872 3873 3874
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3875
		header->size += size;
3876
	}
3877
}
3878

3879
static void perf_event_output(struct perf_event *event, int nmi,
3880 3881 3882 3883 3884
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3885

3886 3887 3888
	/* protect the callchain buffers */
	rcu_read_lock();

3889
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3890

3891
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3892
		goto exit;
3893

3894
	perf_output_sample(&handle, &header, data, event);
3895

3896
	perf_output_end(&handle);
3897 3898 3899

exit:
	rcu_read_unlock();
3900 3901
}

3902
/*
3903
 * read event_id
3904 3905 3906 3907 3908 3909 3910 3911 3912 3913
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3914
perf_event_read_event(struct perf_event *event,
3915 3916 3917
			struct task_struct *task)
{
	struct perf_output_handle handle;
3918
	struct perf_sample_data sample;
3919
	struct perf_read_event read_event = {
3920
		.header = {
3921
			.type = PERF_RECORD_READ,
3922
			.misc = 0,
3923
			.size = sizeof(read_event) + event->read_size,
3924
		},
3925 3926
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3927
	};
3928
	int ret;
3929

3930
	perf_event_header__init_id(&read_event.header, &sample, event);
3931
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3932 3933 3934
	if (ret)
		return;

3935
	perf_output_put(&handle, read_event);
3936
	perf_output_read(&handle, event);
3937
	perf_event__output_id_sample(event, &handle, &sample);
3938

3939 3940 3941
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3942
/*
P
Peter Zijlstra 已提交
3943 3944
 * task tracking -- fork/exit
 *
3945
 * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
P
Peter Zijlstra 已提交
3946 3947
 */

P
Peter Zijlstra 已提交
3948
struct perf_task_event {
3949
	struct task_struct		*task;
3950
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3951 3952 3953 3954 3955 3956

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3957 3958
		u32				tid;
		u32				ptid;
3959
		u64				time;
3960
	} event_id;
P
Peter Zijlstra 已提交
3961 3962
};

3963
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3964
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3965 3966
{
	struct perf_output_handle handle;
3967
	struct perf_sample_data	sample;
P
Peter Zijlstra 已提交
3968
	struct task_struct *task = task_event->task;
3969
	int ret, size = task_event->event_id.header.size;
3970

3971
	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
P
Peter Zijlstra 已提交
3972

3973 3974
	ret = perf_output_begin(&handle, event,
				task_event->event_id.header.size, 0, 0);
3975
	if (ret)
3976
		goto out;
P
Peter Zijlstra 已提交
3977

3978 3979
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3980

3981 3982
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3983

3984
	perf_output_put(&handle, task_event->event_id);
3985

3986 3987
	perf_event__output_id_sample(event, &handle, &sample);

P
Peter Zijlstra 已提交
3988
	perf_output_end(&handle);
3989 3990
out:
	task_event->event_id.header.size = size;
P
Peter Zijlstra 已提交
3991 3992
}

3993
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3994
{
P
Peter Zijlstra 已提交
3995
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3996 3997
		return 0;

3998
	if (!event_filter_match(event))
3999 4000
		return 0;

4001 4002
	if (event->attr.comm || event->attr.mmap ||
	    event->attr.mmap_data || event->attr.task)
P
Peter Zijlstra 已提交
4003 4004 4005 4006 4007
		return 1;

	return 0;
}

4008
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
4009
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
4010
{
4011
	struct perf_event *event;
P
Peter Zijlstra 已提交
4012

4013 4014 4015
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
4016 4017 4018
	}
}

4019
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
4020 4021
{
	struct perf_cpu_context *cpuctx;
P
Peter Zijlstra 已提交
4022
	struct perf_event_context *ctx;
P
Peter Zijlstra 已提交
4023
	struct pmu *pmu;
P
Peter Zijlstra 已提交
4024
	int ctxn;
P
Peter Zijlstra 已提交
4025

4026
	rcu_read_lock();
P
Peter Zijlstra 已提交
4027
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4028
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4029 4030
		if (cpuctx->active_pmu != pmu)
			goto next;
P
Peter Zijlstra 已提交
4031
		perf_event_task_ctx(&cpuctx->ctx, task_event);
P
Peter Zijlstra 已提交
4032 4033 4034 4035 4036

		ctx = task_event->task_ctx;
		if (!ctx) {
			ctxn = pmu->task_ctx_nr;
			if (ctxn < 0)
4037
				goto next;
P
Peter Zijlstra 已提交
4038 4039 4040 4041
			ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		}
		if (ctx)
			perf_event_task_ctx(ctx, task_event);
4042 4043
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
4044
	}
P
Peter Zijlstra 已提交
4045 4046 4047
	rcu_read_unlock();
}

4048 4049
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
4050
			      int new)
P
Peter Zijlstra 已提交
4051
{
P
Peter Zijlstra 已提交
4052
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
4053

4054 4055 4056
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
4057 4058
		return;

P
Peter Zijlstra 已提交
4059
	task_event = (struct perf_task_event){
4060 4061
		.task	  = task,
		.task_ctx = task_ctx,
4062
		.event_id    = {
P
Peter Zijlstra 已提交
4063
			.header = {
4064
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4065
				.misc = 0,
4066
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
4067
			},
4068 4069
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
4070 4071
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
4072
			.time = perf_clock(),
P
Peter Zijlstra 已提交
4073 4074 4075
		},
	};

4076
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
4077 4078
}

4079
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
4080
{
4081
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
4082 4083
}

4084 4085 4086 4087 4088
/*
 * comm tracking
 */

struct perf_comm_event {
4089 4090
	struct task_struct	*task;
	char			*comm;
4091 4092 4093 4094 4095 4096 4097
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
4098
	} event_id;
4099 4100
};

4101
static void perf_event_comm_output(struct perf_event *event,
4102 4103 4104
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
4105
	struct perf_sample_data sample;
4106
	int size = comm_event->event_id.header.size;
4107 4108 4109 4110 4111
	int ret;

	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
	ret = perf_output_begin(&handle, event,
				comm_event->event_id.header.size, 0, 0);
4112 4113

	if (ret)
4114
		goto out;
4115

4116 4117
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4118

4119
	perf_output_put(&handle, comm_event->event_id);
4120 4121
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
4122 4123 4124

	perf_event__output_id_sample(event, &handle, &sample);

4125
	perf_output_end(&handle);
4126 4127
out:
	comm_event->event_id.header.size = size;
4128 4129
}

4130
static int perf_event_comm_match(struct perf_event *event)
4131
{
P
Peter Zijlstra 已提交
4132
	if (event->state < PERF_EVENT_STATE_INACTIVE)
4133 4134
		return 0;

4135
	if (!event_filter_match(event))
4136 4137
		return 0;

4138
	if (event->attr.comm)
4139 4140 4141 4142 4143
		return 1;

	return 0;
}

4144
static void perf_event_comm_ctx(struct perf_event_context *ctx,
4145 4146
				  struct perf_comm_event *comm_event)
{
4147
	struct perf_event *event;
4148

4149 4150 4151
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
4152 4153 4154
	}
}

4155
static void perf_event_comm_event(struct perf_comm_event *comm_event)
4156 4157
{
	struct perf_cpu_context *cpuctx;
4158
	struct perf_event_context *ctx;
4159
	char comm[TASK_COMM_LEN];
4160
	unsigned int size;
P
Peter Zijlstra 已提交
4161
	struct pmu *pmu;
P
Peter Zijlstra 已提交
4162
	int ctxn;
4163

4164
	memset(comm, 0, sizeof(comm));
4165
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
4166
	size = ALIGN(strlen(comm)+1, sizeof(u64));
4167 4168 4169 4170

	comm_event->comm = comm;
	comm_event->comm_size = size;

4171
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4172
	rcu_read_lock();
P
Peter Zijlstra 已提交
4173
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4174
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4175 4176
		if (cpuctx->active_pmu != pmu)
			goto next;
P
Peter Zijlstra 已提交
4177
		perf_event_comm_ctx(&cpuctx->ctx, comm_event);
P
Peter Zijlstra 已提交
4178 4179 4180

		ctxn = pmu->task_ctx_nr;
		if (ctxn < 0)
4181
			goto next;
P
Peter Zijlstra 已提交
4182 4183 4184 4185

		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		if (ctx)
			perf_event_comm_ctx(ctx, comm_event);
4186 4187
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
4188
	}
4189
	rcu_read_unlock();
4190 4191
}

4192
void perf_event_comm(struct task_struct *task)
4193
{
4194
	struct perf_comm_event comm_event;
P
Peter Zijlstra 已提交
4195 4196
	struct perf_event_context *ctx;
	int ctxn;
4197

P
Peter Zijlstra 已提交
4198 4199 4200 4201
	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (!ctx)
			continue;
4202

P
Peter Zijlstra 已提交
4203 4204
		perf_event_enable_on_exec(ctx);
	}
4205

4206
	if (!atomic_read(&nr_comm_events))
4207
		return;
4208

4209
	comm_event = (struct perf_comm_event){
4210
		.task	= task,
4211 4212
		/* .comm      */
		/* .comm_size */
4213
		.event_id  = {
4214
			.header = {
4215
				.type = PERF_RECORD_COMM,
4216 4217 4218 4219 4220
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
4221 4222 4223
		},
	};

4224
	perf_event_comm_event(&comm_event);
4225 4226
}

4227 4228 4229 4230 4231
/*
 * mmap tracking
 */

struct perf_mmap_event {
4232 4233 4234 4235
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
4236 4237 4238 4239 4240 4241 4242 4243 4244

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
4245
	} event_id;
4246 4247
};

4248
static void perf_event_mmap_output(struct perf_event *event,
4249 4250 4251
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
4252
	struct perf_sample_data sample;
4253
	int size = mmap_event->event_id.header.size;
4254
	int ret;
4255

4256 4257 4258
	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
	ret = perf_output_begin(&handle, event,
				mmap_event->event_id.header.size, 0, 0);
4259
	if (ret)
4260
		goto out;
4261

4262 4263
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
4264

4265
	perf_output_put(&handle, mmap_event->event_id);
4266 4267
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
4268 4269 4270

	perf_event__output_id_sample(event, &handle, &sample);

4271
	perf_output_end(&handle);
4272 4273
out:
	mmap_event->event_id.header.size = size;
4274 4275
}

4276
static int perf_event_mmap_match(struct perf_event *event,
4277 4278
				   struct perf_mmap_event *mmap_event,
				   int executable)
4279
{
P
Peter Zijlstra 已提交
4280
	if (event->state < PERF_EVENT_STATE_INACTIVE)
4281 4282
		return 0;

4283
	if (!event_filter_match(event))
4284 4285
		return 0;

4286 4287
	if ((!executable && event->attr.mmap_data) ||
	    (executable && event->attr.mmap))
4288 4289 4290 4291 4292
		return 1;

	return 0;
}

4293
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4294 4295
				  struct perf_mmap_event *mmap_event,
				  int executable)
4296
{
4297
	struct perf_event *event;
4298

4299
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4300
		if (perf_event_mmap_match(event, mmap_event, executable))
4301
			perf_event_mmap_output(event, mmap_event);
4302 4303 4304
	}
}

4305
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4306 4307
{
	struct perf_cpu_context *cpuctx;
4308
	struct perf_event_context *ctx;
4309 4310
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
4311 4312 4313
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
4314
	const char *name;
P
Peter Zijlstra 已提交
4315
	struct pmu *pmu;
P
Peter Zijlstra 已提交
4316
	int ctxn;
4317

4318 4319
	memset(tmp, 0, sizeof(tmp));

4320
	if (file) {
4321 4322 4323 4324 4325 4326
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4327 4328 4329 4330
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
4331
		name = d_path(&file->f_path, buf, PATH_MAX);
4332 4333 4334 4335 4336
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
4337 4338 4339
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
4340
			goto got_name;
4341
		}
4342 4343 4344 4345

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
4346 4347 4348 4349 4350 4351 4352 4353
		} else if (vma->vm_start <= vma->vm_mm->start_brk &&
				vma->vm_end >= vma->vm_mm->brk) {
			name = strncpy(tmp, "[heap]", sizeof(tmp));
			goto got_name;
		} else if (vma->vm_start <= vma->vm_mm->start_stack &&
				vma->vm_end >= vma->vm_mm->start_stack) {
			name = strncpy(tmp, "[stack]", sizeof(tmp));
			goto got_name;
4354 4355
		}

4356 4357 4358 4359 4360
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
4361
	size = ALIGN(strlen(name)+1, sizeof(u64));
4362 4363 4364 4365

	mmap_event->file_name = name;
	mmap_event->file_size = size;

4366
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4367

4368
	rcu_read_lock();
P
Peter Zijlstra 已提交
4369
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4370
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4371 4372
		if (cpuctx->active_pmu != pmu)
			goto next;
P
Peter Zijlstra 已提交
4373 4374
		perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
					vma->vm_flags & VM_EXEC);
P
Peter Zijlstra 已提交
4375 4376 4377

		ctxn = pmu->task_ctx_nr;
		if (ctxn < 0)
4378
			goto next;
P
Peter Zijlstra 已提交
4379 4380 4381 4382 4383 4384

		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		if (ctx) {
			perf_event_mmap_ctx(ctx, mmap_event,
					vma->vm_flags & VM_EXEC);
		}
4385 4386
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
4387
	}
4388 4389
	rcu_read_unlock();

4390 4391 4392
	kfree(buf);
}

4393
void perf_event_mmap(struct vm_area_struct *vma)
4394
{
4395 4396
	struct perf_mmap_event mmap_event;

4397
	if (!atomic_read(&nr_mmap_events))
4398 4399 4400
		return;

	mmap_event = (struct perf_mmap_event){
4401
		.vma	= vma,
4402 4403
		/* .file_name */
		/* .file_size */
4404
		.event_id  = {
4405
			.header = {
4406
				.type = PERF_RECORD_MMAP,
4407
				.misc = PERF_RECORD_MISC_USER,
4408 4409 4410 4411
				/* .size */
			},
			/* .pid */
			/* .tid */
4412 4413
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
4414
			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4415 4416 4417
		},
	};

4418
	perf_event_mmap_event(&mmap_event);
4419 4420
}

4421 4422 4423 4424
/*
 * IRQ throttle logging
 */

4425
static void perf_log_throttle(struct perf_event *event, int enable)
4426 4427
{
	struct perf_output_handle handle;
4428
	struct perf_sample_data sample;
4429 4430 4431 4432 4433
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
4434
		u64				id;
4435
		u64				stream_id;
4436 4437
	} throttle_event = {
		.header = {
4438
			.type = PERF_RECORD_THROTTLE,
4439 4440 4441
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
4442
		.time		= perf_clock(),
4443 4444
		.id		= primary_event_id(event),
		.stream_id	= event->id,
4445 4446
	};

4447
	if (enable)
4448
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4449

4450 4451 4452 4453
	perf_event_header__init_id(&throttle_event.header, &sample, event);

	ret = perf_output_begin(&handle, event,
				throttle_event.header.size, 1, 0);
4454 4455 4456 4457
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
4458
	perf_event__output_id_sample(event, &handle, &sample);
4459 4460 4461
	perf_output_end(&handle);
}

4462
/*
4463
 * Generic event overflow handling, sampling.
4464 4465
 */

4466
static int __perf_event_overflow(struct perf_event *event, int nmi,
4467 4468
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
4469
{
4470 4471
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
4472 4473
	int ret = 0;

4474 4475 4476 4477 4478 4479 4480
	/*
	 * Non-sampling counters might still use the PMI to fold short
	 * hardware counters, ignore those.
	 */
	if (unlikely(!is_sampling_event(event)))
		return 0;

4481
	if (!throttle) {
4482
		hwc->interrupts++;
4483
	} else {
4484 4485
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
4486
			if (HZ * hwc->interrupts >
4487
					(u64)sysctl_perf_event_sample_rate) {
4488
				hwc->interrupts = MAX_INTERRUPTS;
4489
				perf_log_throttle(event, 0);
4490 4491 4492 4493
				ret = 1;
			}
		} else {
			/*
4494
			 * Keep re-disabling events even though on the previous
4495
			 * pass we disabled it - just in case we raced with a
4496
			 * sched-in and the event got enabled again:
4497
			 */
4498 4499 4500
			ret = 1;
		}
	}
4501

4502
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
4503
		u64 now = perf_clock();
4504
		s64 delta = now - hwc->freq_time_stamp;
4505

4506
		hwc->freq_time_stamp = now;
4507

4508 4509
		if (delta > 0 && delta < 2*TICK_NSEC)
			perf_adjust_period(event, delta, hwc->last_period);
4510 4511
	}

4512 4513
	/*
	 * XXX event_limit might not quite work as expected on inherited
4514
	 * events
4515 4516
	 */

4517 4518
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
4519
		ret = 1;
4520
		event->pending_kill = POLL_HUP;
4521
		if (nmi) {
4522
			event->pending_disable = 1;
4523
			irq_work_queue(&event->pending);
4524
		} else
4525
			perf_event_disable(event);
4526 4527
	}

4528 4529 4530 4531 4532
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

4533
	return ret;
4534 4535
}

4536
int perf_event_overflow(struct perf_event *event, int nmi,
4537 4538
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
4539
{
4540
	return __perf_event_overflow(event, nmi, 1, data, regs);
4541 4542
}

4543
/*
4544
 * Generic software event infrastructure
4545 4546
 */

4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557
struct swevent_htable {
	struct swevent_hlist		*swevent_hlist;
	struct mutex			hlist_mutex;
	int				hlist_refcount;

	/* Recursion avoidance in each contexts */
	int				recursion[PERF_NR_CONTEXTS];
};

static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);

4558
/*
4559 4560
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
4561 4562 4563 4564
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

4565
static u64 perf_swevent_set_period(struct perf_event *event)
4566
{
4567
	struct hw_perf_event *hwc = &event->hw;
4568 4569 4570 4571 4572
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
4573 4574

again:
4575
	old = val = local64_read(&hwc->period_left);
4576 4577
	if (val < 0)
		return 0;
4578

4579 4580 4581
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
4582
	if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4583
		goto again;
4584

4585
	return nr;
4586 4587
}

4588
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4589 4590
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
4591
{
4592
	struct hw_perf_event *hwc = &event->hw;
4593
	int throttle = 0;
4594

4595
	data->period = event->hw.last_period;
4596 4597
	if (!overflow)
		overflow = perf_swevent_set_period(event);
4598

4599 4600
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
4601

4602
	for (; overflow; overflow--) {
4603
		if (__perf_event_overflow(event, nmi, throttle,
4604
					    data, regs)) {
4605 4606 4607 4608 4609 4610
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
4611
		throttle = 1;
4612
	}
4613 4614
}

P
Peter Zijlstra 已提交
4615
static void perf_swevent_event(struct perf_event *event, u64 nr,
4616 4617
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
4618
{
4619
	struct hw_perf_event *hwc = &event->hw;
4620

4621
	local64_add(nr, &event->count);
4622

4623 4624 4625
	if (!regs)
		return;

4626
	if (!is_sampling_event(event))
4627
		return;
4628

4629 4630 4631
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

4632
	if (local64_add_negative(nr, &hwc->period_left))
4633
		return;
4634

4635
	perf_swevent_overflow(event, 0, nmi, data, regs);
4636 4637
}

4638 4639 4640
static int perf_exclude_event(struct perf_event *event,
			      struct pt_regs *regs)
{
P
Peter Zijlstra 已提交
4641 4642 4643
	if (event->hw.state & PERF_HES_STOPPED)
		return 0;

4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654
	if (regs) {
		if (event->attr.exclude_user && user_mode(regs))
			return 1;

		if (event->attr.exclude_kernel && !user_mode(regs))
			return 1;
	}

	return 0;
}

4655
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
4656
				enum perf_type_id type,
L
Li Zefan 已提交
4657 4658 4659
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
4660
{
4661
	if (event->attr.type != type)
4662
		return 0;
4663

4664
	if (event->attr.config != event_id)
4665 4666
		return 0;

4667 4668
	if (perf_exclude_event(event, regs))
		return 0;
4669 4670 4671 4672

	return 1;
}

4673 4674 4675 4676 4677 4678 4679
static inline u64 swevent_hash(u64 type, u32 event_id)
{
	u64 val = event_id | (type << 32);

	return hash_64(val, SWEVENT_HLIST_BITS);
}

4680 4681
static inline struct hlist_head *
__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4682
{
4683 4684 4685 4686
	u64 hash = swevent_hash(type, event_id);

	return &hlist->heads[hash];
}
4687

4688 4689
/* For the read side: events when they trigger */
static inline struct hlist_head *
4690
find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4691 4692
{
	struct swevent_hlist *hlist;
4693

4694
	hlist = rcu_dereference(swhash->swevent_hlist);
4695 4696 4697
	if (!hlist)
		return NULL;

4698 4699 4700 4701 4702
	return __find_swevent_head(hlist, type, event_id);
}

/* For the event head insertion and removal in the hlist */
static inline struct hlist_head *
4703
find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4704 4705 4706 4707 4708 4709 4710 4711 4712 4713
{
	struct swevent_hlist *hlist;
	u32 event_id = event->attr.config;
	u64 type = event->attr.type;

	/*
	 * Event scheduling is always serialized against hlist allocation
	 * and release. Which makes the protected version suitable here.
	 * The context lock guarantees that.
	 */
4714
	hlist = rcu_dereference_protected(swhash->swevent_hlist,
4715 4716 4717 4718 4719
					  lockdep_is_held(&event->ctx->lock));
	if (!hlist)
		return NULL;

	return __find_swevent_head(hlist, type, event_id);
4720 4721 4722 4723 4724 4725
}

static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
				    u64 nr, int nmi,
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
4726
{
4727
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4728
	struct perf_event *event;
4729 4730
	struct hlist_node *node;
	struct hlist_head *head;
4731

4732
	rcu_read_lock();
4733
	head = find_swevent_head_rcu(swhash, type, event_id);
4734 4735 4736 4737
	if (!head)
		goto end;

	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
L
Li Zefan 已提交
4738
		if (perf_swevent_match(event, type, event_id, data, regs))
P
Peter Zijlstra 已提交
4739
			perf_swevent_event(event, nr, nmi, data, regs);
4740
	}
4741 4742
end:
	rcu_read_unlock();
4743 4744
}

4745
int perf_swevent_get_recursion_context(void)
P
Peter Zijlstra 已提交
4746
{
4747
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
P
Peter Zijlstra 已提交
4748

4749
	return get_recursion_context(swhash->recursion);
P
Peter Zijlstra 已提交
4750
}
I
Ingo Molnar 已提交
4751
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
4752

4753
inline void perf_swevent_put_recursion_context(int rctx)
4754
{
4755
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4756

4757
	put_recursion_context(swhash->recursion, rctx);
4758
}
4759

4760
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4761
			    struct pt_regs *regs, u64 addr)
4762
{
4763
	struct perf_sample_data data;
4764 4765
	int rctx;

4766
	preempt_disable_notrace();
4767 4768 4769
	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return;
4770

4771
	perf_sample_data_init(&data, addr);
4772

4773
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4774 4775

	perf_swevent_put_recursion_context(rctx);
4776
	preempt_enable_notrace();
4777 4778
}

4779
static void perf_swevent_read(struct perf_event *event)
4780 4781 4782
{
}

P
Peter Zijlstra 已提交
4783
static int perf_swevent_add(struct perf_event *event, int flags)
4784
{
4785
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4786
	struct hw_perf_event *hwc = &event->hw;
4787 4788
	struct hlist_head *head;

4789
	if (is_sampling_event(event)) {
4790
		hwc->last_period = hwc->sample_period;
4791
		perf_swevent_set_period(event);
4792
	}
4793

P
Peter Zijlstra 已提交
4794 4795
	hwc->state = !(flags & PERF_EF_START);

4796
	head = find_swevent_head(swhash, event);
4797 4798 4799 4800 4801
	if (WARN_ON_ONCE(!head))
		return -EINVAL;

	hlist_add_head_rcu(&event->hlist_entry, head);

4802 4803 4804
	return 0;
}

P
Peter Zijlstra 已提交
4805
static void perf_swevent_del(struct perf_event *event, int flags)
4806
{
4807
	hlist_del_rcu(&event->hlist_entry);
4808 4809
}

P
Peter Zijlstra 已提交
4810
static void perf_swevent_start(struct perf_event *event, int flags)
4811
{
P
Peter Zijlstra 已提交
4812
	event->hw.state = 0;
4813
}
I
Ingo Molnar 已提交
4814

P
Peter Zijlstra 已提交
4815
static void perf_swevent_stop(struct perf_event *event, int flags)
4816
{
P
Peter Zijlstra 已提交
4817
	event->hw.state = PERF_HES_STOPPED;
4818 4819
}

4820 4821
/* Deref the hlist from the update side */
static inline struct swevent_hlist *
4822
swevent_hlist_deref(struct swevent_htable *swhash)
4823
{
4824 4825
	return rcu_dereference_protected(swhash->swevent_hlist,
					 lockdep_is_held(&swhash->hlist_mutex));
4826 4827
}

4828 4829 4830 4831 4832 4833 4834 4835
static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
{
	struct swevent_hlist *hlist;

	hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
	kfree(hlist);
}

4836
static void swevent_hlist_release(struct swevent_htable *swhash)
4837
{
4838
	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
4839

4840
	if (!hlist)
4841 4842
		return;

4843
	rcu_assign_pointer(swhash->swevent_hlist, NULL);
4844 4845 4846 4847 4848
	call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
}

static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
{
4849
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4850

4851
	mutex_lock(&swhash->hlist_mutex);
4852

4853 4854
	if (!--swhash->hlist_refcount)
		swevent_hlist_release(swhash);
4855

4856
	mutex_unlock(&swhash->hlist_mutex);
4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873
}

static void swevent_hlist_put(struct perf_event *event)
{
	int cpu;

	if (event->cpu != -1) {
		swevent_hlist_put_cpu(event, event->cpu);
		return;
	}

	for_each_possible_cpu(cpu)
		swevent_hlist_put_cpu(event, cpu);
}

static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
{
4874
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4875 4876
	int err = 0;

4877
	mutex_lock(&swhash->hlist_mutex);
4878

4879
	if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
4880 4881 4882 4883 4884 4885 4886
		struct swevent_hlist *hlist;

		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
		if (!hlist) {
			err = -ENOMEM;
			goto exit;
		}
4887
		rcu_assign_pointer(swhash->swevent_hlist, hlist);
4888
	}
4889
	swhash->hlist_refcount++;
P
Peter Zijlstra 已提交
4890
exit:
4891
	mutex_unlock(&swhash->hlist_mutex);
4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914

	return err;
}

static int swevent_hlist_get(struct perf_event *event)
{
	int err;
	int cpu, failed_cpu;

	if (event->cpu != -1)
		return swevent_hlist_get_cpu(event, event->cpu);

	get_online_cpus();
	for_each_possible_cpu(cpu) {
		err = swevent_hlist_get_cpu(event, cpu);
		if (err) {
			failed_cpu = cpu;
			goto fail;
		}
	}
	put_online_cpus();

	return 0;
P
Peter Zijlstra 已提交
4915
fail:
4916 4917 4918 4919 4920 4921 4922 4923 4924 4925
	for_each_possible_cpu(cpu) {
		if (cpu == failed_cpu)
			break;
		swevent_hlist_put_cpu(event, cpu);
	}

	put_online_cpus();
	return err;
}

4926
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4927

4928 4929 4930
static void sw_perf_event_destroy(struct perf_event *event)
{
	u64 event_id = event->attr.config;
4931

4932 4933
	WARN_ON(event->parent);

P
Peter Zijlstra 已提交
4934
	jump_label_dec(&perf_swevent_enabled[event_id]);
4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953
	swevent_hlist_put(event);
}

static int perf_swevent_init(struct perf_event *event)
{
	int event_id = event->attr.config;

	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	switch (event_id) {
	case PERF_COUNT_SW_CPU_CLOCK:
	case PERF_COUNT_SW_TASK_CLOCK:
		return -ENOENT;

	default:
		break;
	}

4954
	if (event_id >= PERF_COUNT_SW_MAX)
4955 4956 4957 4958 4959 4960 4961 4962 4963
		return -ENOENT;

	if (!event->parent) {
		int err;

		err = swevent_hlist_get(event);
		if (err)
			return err;

P
Peter Zijlstra 已提交
4964
		jump_label_inc(&perf_swevent_enabled[event_id]);
4965 4966 4967 4968 4969 4970 4971
		event->destroy = sw_perf_event_destroy;
	}

	return 0;
}

static struct pmu perf_swevent = {
4972
	.task_ctx_nr	= perf_sw_context,
4973

4974
	.event_init	= perf_swevent_init,
P
Peter Zijlstra 已提交
4975 4976 4977 4978
	.add		= perf_swevent_add,
	.del		= perf_swevent_del,
	.start		= perf_swevent_start,
	.stop		= perf_swevent_stop,
4979 4980 4981
	.read		= perf_swevent_read,
};

4982 4983
#ifdef CONFIG_EVENT_TRACING

4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997
static int perf_tp_filter_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
4998 4999 5000 5001
	/*
	 * All tracepoints are from kernel-space.
	 */
	if (event->attr.exclude_kernel)
5002 5003 5004 5005 5006 5007 5008 5009 5010
		return 0;

	if (!perf_tp_filter_match(event, data))
		return 0;

	return 1;
}

void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5011
		   struct pt_regs *regs, struct hlist_head *head, int rctx)
5012 5013
{
	struct perf_sample_data data;
5014 5015 5016
	struct perf_event *event;
	struct hlist_node *node;

5017 5018 5019 5020 5021 5022 5023 5024
	struct perf_raw_record raw = {
		.size = entry_size,
		.data = record,
	};

	perf_sample_data_init(&data, addr);
	data.raw = &raw;

5025 5026
	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
		if (perf_tp_event_match(event, &data, regs))
P
Peter Zijlstra 已提交
5027
			perf_swevent_event(event, count, 1, &data, regs);
5028
	}
5029 5030

	perf_swevent_put_recursion_context(rctx);
5031 5032 5033
}
EXPORT_SYMBOL_GPL(perf_tp_event);

5034
static void tp_perf_event_destroy(struct perf_event *event)
5035
{
5036
	perf_trace_destroy(event);
5037 5038
}

5039
static int perf_tp_event_init(struct perf_event *event)
5040
{
5041 5042
	int err;

5043 5044 5045
	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -ENOENT;

5046 5047
	err = perf_trace_init(event);
	if (err)
5048
		return err;
5049

5050
	event->destroy = tp_perf_event_destroy;
5051

5052 5053 5054 5055
	return 0;
}

static struct pmu perf_tracepoint = {
5056 5057
	.task_ctx_nr	= perf_sw_context,

5058
	.event_init	= perf_tp_event_init,
P
Peter Zijlstra 已提交
5059 5060 5061 5062
	.add		= perf_trace_add,
	.del		= perf_trace_del,
	.start		= perf_swevent_start,
	.stop		= perf_swevent_stop,
5063 5064 5065 5066 5067
	.read		= perf_swevent_read,
};

static inline void perf_tp_register(void)
{
P
Peter Zijlstra 已提交
5068
	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5069
}
L
Li Zefan 已提交
5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

5094
#else
L
Li Zefan 已提交
5095

5096
static inline void perf_tp_register(void)
5097 5098
{
}
L
Li Zefan 已提交
5099 5100 5101 5102 5103 5104 5105 5106 5107 5108

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

5109
#endif /* CONFIG_EVENT_TRACING */
5110

5111
#ifdef CONFIG_HAVE_HW_BREAKPOINT
5112
void perf_bp_event(struct perf_event *bp, void *data)
5113
{
5114 5115 5116
	struct perf_sample_data sample;
	struct pt_regs *regs = data;

5117
	perf_sample_data_init(&sample, bp->attr.bp_addr);
5118

P
Peter Zijlstra 已提交
5119 5120
	if (!bp->hw.state && !perf_exclude_event(bp, regs))
		perf_swevent_event(bp, 1, 1, &sample, regs);
5121 5122 5123
}
#endif

5124 5125 5126
/*
 * hrtimer based swevent callback
 */
5127

5128
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5129
{
5130 5131 5132 5133 5134
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
	struct pt_regs *regs;
	struct perf_event *event;
	u64 period;
5135

5136 5137
	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
	event->pmu->read(event);
5138

5139 5140 5141 5142 5143 5144 5145 5146 5147
	perf_sample_data_init(&data, 0);
	data.period = event->hw.last_period;
	regs = get_irq_regs();

	if (regs && !perf_exclude_event(event, regs)) {
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
	}
5148

5149 5150
	period = max_t(u64, 10000, event->hw.sample_period);
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5151

5152
	return ret;
5153 5154
}

5155
static void perf_swevent_start_hrtimer(struct perf_event *event)
5156
{
5157
	struct hw_perf_event *hwc = &event->hw;
5158 5159 5160 5161
	s64 period;

	if (!is_sampling_event(event))
		return;
5162

5163 5164
	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
5165

5166 5167 5168 5169
	period = local64_read(&hwc->period_left);
	if (period) {
		if (period < 0)
			period = 10000;
P
Peter Zijlstra 已提交
5170

5171 5172 5173 5174 5175
		local64_set(&hwc->period_left, 0);
	} else {
		period = max_t(u64, 10000, hwc->sample_period);
	}
	__hrtimer_start_range_ns(&hwc->hrtimer,
5176
				ns_to_ktime(period), 0,
5177
				HRTIMER_MODE_REL_PINNED, 0);
5178
}
5179 5180

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5181
{
5182 5183
	struct hw_perf_event *hwc = &event->hw;

5184
	if (is_sampling_event(event)) {
5185
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
P
Peter Zijlstra 已提交
5186
		local64_set(&hwc->period_left, ktime_to_ns(remaining));
5187 5188 5189

		hrtimer_cancel(&hwc->hrtimer);
	}
5190 5191
}

5192 5193 5194 5195 5196
/*
 * Software event: cpu wall time clock
 */

static void cpu_clock_event_update(struct perf_event *event)
5197
{
5198 5199 5200
	s64 prev;
	u64 now;

P
Peter Zijlstra 已提交
5201
	now = local_clock();
5202 5203
	prev = local64_xchg(&event->hw.prev_count, now);
	local64_add(now - prev, &event->count);
5204 5205
}

P
Peter Zijlstra 已提交
5206
static void cpu_clock_event_start(struct perf_event *event, int flags)
5207
{
P
Peter Zijlstra 已提交
5208
	local64_set(&event->hw.prev_count, local_clock());
5209 5210 5211
	perf_swevent_start_hrtimer(event);
}

P
Peter Zijlstra 已提交
5212
static void cpu_clock_event_stop(struct perf_event *event, int flags)
5213
{
5214 5215 5216
	perf_swevent_cancel_hrtimer(event);
	cpu_clock_event_update(event);
}
5217

P
Peter Zijlstra 已提交
5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230
static int cpu_clock_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		cpu_clock_event_start(event, flags);

	return 0;
}

static void cpu_clock_event_del(struct perf_event *event, int flags)
{
	cpu_clock_event_stop(event, flags);
}

5231 5232 5233 5234
static void cpu_clock_event_read(struct perf_event *event)
{
	cpu_clock_event_update(event);
}
5235

5236 5237 5238 5239 5240 5241 5242 5243 5244
static int cpu_clock_event_init(struct perf_event *event)
{
	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
		return -ENOENT;

	return 0;
5245 5246
}

5247
static struct pmu perf_cpu_clock = {
5248 5249
	.task_ctx_nr	= perf_sw_context,

5250
	.event_init	= cpu_clock_event_init,
P
Peter Zijlstra 已提交
5251 5252 5253 5254
	.add		= cpu_clock_event_add,
	.del		= cpu_clock_event_del,
	.start		= cpu_clock_event_start,
	.stop		= cpu_clock_event_stop,
5255 5256 5257 5258 5259 5260 5261 5262
	.read		= cpu_clock_event_read,
};

/*
 * Software event: task time clock
 */

static void task_clock_event_update(struct perf_event *event, u64 now)
5263
{
5264 5265
	u64 prev;
	s64 delta;
5266

5267 5268 5269 5270
	prev = local64_xchg(&event->hw.prev_count, now);
	delta = now - prev;
	local64_add(delta, &event->count);
}
5271

P
Peter Zijlstra 已提交
5272
static void task_clock_event_start(struct perf_event *event, int flags)
5273
{
P
Peter Zijlstra 已提交
5274
	local64_set(&event->hw.prev_count, event->ctx->time);
5275 5276 5277
	perf_swevent_start_hrtimer(event);
}

P
Peter Zijlstra 已提交
5278
static void task_clock_event_stop(struct perf_event *event, int flags)
5279 5280 5281
{
	perf_swevent_cancel_hrtimer(event);
	task_clock_event_update(event, event->ctx->time);
P
Peter Zijlstra 已提交
5282 5283 5284 5285 5286 5287
}

static int task_clock_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		task_clock_event_start(event, flags);
5288

P
Peter Zijlstra 已提交
5289 5290 5291 5292 5293 5294
	return 0;
}

static void task_clock_event_del(struct perf_event *event, int flags)
{
	task_clock_event_stop(event, PERF_EF_UPDATE);
5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313
}

static void task_clock_event_read(struct perf_event *event)
{
	u64 time;

	if (!in_nmi()) {
		update_context_time(event->ctx);
		time = event->ctx->time;
	} else {
		u64 now = perf_clock();
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
	}

	task_clock_event_update(event, time);
}

static int task_clock_event_init(struct perf_event *event)
L
Li Zefan 已提交
5314
{
5315 5316 5317 5318 5319 5320 5321
	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
		return -ENOENT;

	return 0;
L
Li Zefan 已提交
5322 5323
}

5324
static struct pmu perf_task_clock = {
5325 5326
	.task_ctx_nr	= perf_sw_context,

5327
	.event_init	= task_clock_event_init,
P
Peter Zijlstra 已提交
5328 5329 5330 5331
	.add		= task_clock_event_add,
	.del		= task_clock_event_del,
	.start		= task_clock_event_start,
	.stop		= task_clock_event_stop,
5332 5333
	.read		= task_clock_event_read,
};
L
Li Zefan 已提交
5334

P
Peter Zijlstra 已提交
5335
static void perf_pmu_nop_void(struct pmu *pmu)
5336 5337
{
}
L
Li Zefan 已提交
5338

P
Peter Zijlstra 已提交
5339
static int perf_pmu_nop_int(struct pmu *pmu)
L
Li Zefan 已提交
5340
{
P
Peter Zijlstra 已提交
5341
	return 0;
L
Li Zefan 已提交
5342 5343
}

P
Peter Zijlstra 已提交
5344
static void perf_pmu_start_txn(struct pmu *pmu)
L
Li Zefan 已提交
5345
{
P
Peter Zijlstra 已提交
5346
	perf_pmu_disable(pmu);
L
Li Zefan 已提交
5347 5348
}

P
Peter Zijlstra 已提交
5349 5350 5351 5352 5353
static int perf_pmu_commit_txn(struct pmu *pmu)
{
	perf_pmu_enable(pmu);
	return 0;
}
5354

P
Peter Zijlstra 已提交
5355
static void perf_pmu_cancel_txn(struct pmu *pmu)
5356
{
P
Peter Zijlstra 已提交
5357
	perf_pmu_enable(pmu);
5358 5359
}

P
Peter Zijlstra 已提交
5360 5361 5362 5363 5364
/*
 * Ensures all contexts with the same task_ctx_nr have the same
 * pmu_cpu_context too.
 */
static void *find_pmu_context(int ctxn)
5365
{
P
Peter Zijlstra 已提交
5366
	struct pmu *pmu;
5367

P
Peter Zijlstra 已提交
5368 5369
	if (ctxn < 0)
		return NULL;
5370

P
Peter Zijlstra 已提交
5371 5372 5373 5374
	list_for_each_entry(pmu, &pmus, entry) {
		if (pmu->task_ctx_nr == ctxn)
			return pmu->pmu_cpu_context;
	}
5375

P
Peter Zijlstra 已提交
5376
	return NULL;
5377 5378
}

5379
static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5380
{
5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395
	int cpu;

	for_each_possible_cpu(cpu) {
		struct perf_cpu_context *cpuctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		if (cpuctx->active_pmu == old_pmu)
			cpuctx->active_pmu = pmu;
	}
}

static void free_pmu_context(struct pmu *pmu)
{
	struct pmu *i;
5396

P
Peter Zijlstra 已提交
5397
	mutex_lock(&pmus_lock);
5398
	/*
P
Peter Zijlstra 已提交
5399
	 * Like a real lame refcount.
5400
	 */
5401 5402 5403
	list_for_each_entry(i, &pmus, entry) {
		if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
			update_pmu_context(i, pmu);
P
Peter Zijlstra 已提交
5404
			goto out;
5405
		}
P
Peter Zijlstra 已提交
5406
	}
5407

5408
	free_percpu(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
5409 5410
out:
	mutex_unlock(&pmus_lock);
5411
}
P
Peter Zijlstra 已提交
5412
static struct idr pmu_idr;
5413

P
Peter Zijlstra 已提交
5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465
static ssize_t
type_show(struct device *dev, struct device_attribute *attr, char *page)
{
	struct pmu *pmu = dev_get_drvdata(dev);

	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
}

static struct device_attribute pmu_dev_attrs[] = {
       __ATTR_RO(type),
       __ATTR_NULL,
};

static int pmu_bus_running;
static struct bus_type pmu_bus = {
	.name		= "event_source",
	.dev_attrs	= pmu_dev_attrs,
};

static void pmu_dev_release(struct device *dev)
{
	kfree(dev);
}

static int pmu_dev_alloc(struct pmu *pmu)
{
	int ret = -ENOMEM;

	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
	if (!pmu->dev)
		goto out;

	device_initialize(pmu->dev);
	ret = dev_set_name(pmu->dev, "%s", pmu->name);
	if (ret)
		goto free_dev;

	dev_set_drvdata(pmu->dev, pmu);
	pmu->dev->bus = &pmu_bus;
	pmu->dev->release = pmu_dev_release;
	ret = device_add(pmu->dev);
	if (ret)
		goto free_dev;

out:
	return ret;

free_dev:
	put_device(pmu->dev);
	goto out;
}

5466 5467
static struct lock_class_key cpuctx_mutex;

P
Peter Zijlstra 已提交
5468
int perf_pmu_register(struct pmu *pmu, char *name, int type)
5469
{
P
Peter Zijlstra 已提交
5470
	int cpu, ret;
5471

5472
	mutex_lock(&pmus_lock);
P
Peter Zijlstra 已提交
5473 5474 5475 5476
	ret = -ENOMEM;
	pmu->pmu_disable_count = alloc_percpu(int);
	if (!pmu->pmu_disable_count)
		goto unlock;
5477

P
Peter Zijlstra 已提交
5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495
	pmu->type = -1;
	if (!name)
		goto skip_type;
	pmu->name = name;

	if (type < 0) {
		int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
		if (!err)
			goto free_pdc;

		err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
		if (err) {
			ret = err;
			goto free_pdc;
		}
	}
	pmu->type = type;

P
Peter Zijlstra 已提交
5496 5497 5498 5499 5500 5501
	if (pmu_bus_running) {
		ret = pmu_dev_alloc(pmu);
		if (ret)
			goto free_idr;
	}

P
Peter Zijlstra 已提交
5502
skip_type:
P
Peter Zijlstra 已提交
5503 5504 5505
	pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
	if (pmu->pmu_cpu_context)
		goto got_cpu_context;
5506

P
Peter Zijlstra 已提交
5507 5508
	pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
	if (!pmu->pmu_cpu_context)
P
Peter Zijlstra 已提交
5509
		goto free_dev;
5510

P
Peter Zijlstra 已提交
5511 5512 5513 5514
	for_each_possible_cpu(cpu) {
		struct perf_cpu_context *cpuctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5515
		__perf_event_init_context(&cpuctx->ctx);
5516
		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5517
		cpuctx->ctx.type = cpu_context;
P
Peter Zijlstra 已提交
5518
		cpuctx->ctx.pmu = pmu;
5519 5520
		cpuctx->jiffies_interval = 1;
		INIT_LIST_HEAD(&cpuctx->rotation_list);
5521
		cpuctx->active_pmu = pmu;
P
Peter Zijlstra 已提交
5522
	}
5523

P
Peter Zijlstra 已提交
5524
got_cpu_context:
P
Peter Zijlstra 已提交
5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538
	if (!pmu->start_txn) {
		if (pmu->pmu_enable) {
			/*
			 * If we have pmu_enable/pmu_disable calls, install
			 * transaction stubs that use that to try and batch
			 * hardware accesses.
			 */
			pmu->start_txn  = perf_pmu_start_txn;
			pmu->commit_txn = perf_pmu_commit_txn;
			pmu->cancel_txn = perf_pmu_cancel_txn;
		} else {
			pmu->start_txn  = perf_pmu_nop_void;
			pmu->commit_txn = perf_pmu_nop_int;
			pmu->cancel_txn = perf_pmu_nop_void;
5539
		}
5540
	}
5541

P
Peter Zijlstra 已提交
5542 5543 5544 5545 5546
	if (!pmu->pmu_enable) {
		pmu->pmu_enable  = perf_pmu_nop_void;
		pmu->pmu_disable = perf_pmu_nop_void;
	}

5547
	list_add_rcu(&pmu->entry, &pmus);
P
Peter Zijlstra 已提交
5548 5549
	ret = 0;
unlock:
5550 5551
	mutex_unlock(&pmus_lock);

P
Peter Zijlstra 已提交
5552
	return ret;
P
Peter Zijlstra 已提交
5553

P
Peter Zijlstra 已提交
5554 5555 5556 5557
free_dev:
	device_del(pmu->dev);
	put_device(pmu->dev);

P
Peter Zijlstra 已提交
5558 5559 5560 5561
free_idr:
	if (pmu->type >= PERF_TYPE_MAX)
		idr_remove(&pmu_idr, pmu->type);

P
Peter Zijlstra 已提交
5562 5563 5564
free_pdc:
	free_percpu(pmu->pmu_disable_count);
	goto unlock;
5565 5566
}

5567
void perf_pmu_unregister(struct pmu *pmu)
5568
{
5569 5570 5571
	mutex_lock(&pmus_lock);
	list_del_rcu(&pmu->entry);
	mutex_unlock(&pmus_lock);
5572

5573
	/*
P
Peter Zijlstra 已提交
5574 5575
	 * We dereference the pmu list under both SRCU and regular RCU, so
	 * synchronize against both of those.
5576
	 */
5577
	synchronize_srcu(&pmus_srcu);
P
Peter Zijlstra 已提交
5578
	synchronize_rcu();
5579

P
Peter Zijlstra 已提交
5580
	free_percpu(pmu->pmu_disable_count);
P
Peter Zijlstra 已提交
5581 5582
	if (pmu->type >= PERF_TYPE_MAX)
		idr_remove(&pmu_idr, pmu->type);
P
Peter Zijlstra 已提交
5583 5584
	device_del(pmu->dev);
	put_device(pmu->dev);
5585
	free_pmu_context(pmu);
5586
}
5587

5588 5589 5590 5591 5592 5593
struct pmu *perf_init_event(struct perf_event *event)
{
	struct pmu *pmu = NULL;
	int idx;

	idx = srcu_read_lock(&pmus_srcu);
P
Peter Zijlstra 已提交
5594 5595 5596 5597 5598 5599 5600

	rcu_read_lock();
	pmu = idr_find(&pmu_idr, event->attr.type);
	rcu_read_unlock();
	if (pmu)
		goto unlock;

5601 5602 5603
	list_for_each_entry_rcu(pmu, &pmus, entry) {
		int ret = pmu->event_init(event);
		if (!ret)
P
Peter Zijlstra 已提交
5604
			goto unlock;
5605

5606 5607
		if (ret != -ENOENT) {
			pmu = ERR_PTR(ret);
P
Peter Zijlstra 已提交
5608
			goto unlock;
5609
		}
5610
	}
P
Peter Zijlstra 已提交
5611 5612
	pmu = ERR_PTR(-ENOENT);
unlock:
5613
	srcu_read_unlock(&pmus_srcu, idx);
5614

5615
	return pmu;
5616 5617
}

T
Thomas Gleixner 已提交
5618
/*
5619
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
5620
 */
5621
static struct perf_event *
5622
perf_event_alloc(struct perf_event_attr *attr, int cpu,
5623 5624 5625 5626
		 struct task_struct *task,
		 struct perf_event *group_leader,
		 struct perf_event *parent_event,
		 perf_overflow_handler_t overflow_handler)
T
Thomas Gleixner 已提交
5627
{
P
Peter Zijlstra 已提交
5628
	struct pmu *pmu;
5629 5630
	struct perf_event *event;
	struct hw_perf_event *hwc;
5631
	long err;
T
Thomas Gleixner 已提交
5632

5633 5634 5635 5636 5637
	if ((unsigned)cpu >= nr_cpu_ids) {
		if (!task || cpu != -1)
			return ERR_PTR(-EINVAL);
	}

5638
	event = kzalloc(sizeof(*event), GFP_KERNEL);
5639
	if (!event)
5640
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
5641

5642
	/*
5643
	 * Single events are their own group leaders, with an
5644 5645 5646
	 * empty sibling list:
	 */
	if (!group_leader)
5647
		group_leader = event;
5648

5649 5650
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
5651

5652 5653 5654 5655
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
5656
	init_irq_work(&event->pending, perf_pending_event);
T
Thomas Gleixner 已提交
5657

5658
	mutex_init(&event->mmap_mutex);
5659

5660 5661 5662 5663 5664
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->oncpu		= -1;
5665

5666
	event->parent		= parent_event;
5667

5668 5669
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
5670

5671
	event->state		= PERF_EVENT_STATE_INACTIVE;
5672

5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683
	if (task) {
		event->attach_state = PERF_ATTACH_TASK;
#ifdef CONFIG_HAVE_HW_BREAKPOINT
		/*
		 * hw_breakpoint is a bit difficult here..
		 */
		if (attr->type == PERF_TYPE_BREAKPOINT)
			event->hw.bp_target = task;
#endif
	}

5684 5685
	if (!overflow_handler && parent_event)
		overflow_handler = parent_event->overflow_handler;
5686

5687
	event->overflow_handler	= overflow_handler;
5688

5689
	if (attr->disabled)
5690
		event->state = PERF_EVENT_STATE_OFF;
5691

5692
	pmu = NULL;
5693

5694
	hwc = &event->hw;
5695
	hwc->sample_period = attr->sample_period;
5696
	if (attr->freq && attr->sample_freq)
5697
		hwc->sample_period = 1;
5698
	hwc->last_period = hwc->sample_period;
5699

5700
	local64_set(&hwc->period_left, hwc->sample_period);
5701

5702
	/*
5703
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
5704
	 */
5705
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5706 5707
		goto done;

5708
	pmu = perf_init_event(event);
5709

5710 5711
done:
	err = 0;
5712
	if (!pmu)
5713
		err = -EINVAL;
5714 5715
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
5716

5717
	if (err) {
5718 5719 5720
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
5721
		return ERR_PTR(err);
I
Ingo Molnar 已提交
5722
	}
5723

5724
	event->pmu = pmu;
T
Thomas Gleixner 已提交
5725

5726
	if (!event->parent) {
5727 5728
		if (event->attach_state & PERF_ATTACH_TASK)
			jump_label_inc(&perf_task_events);
5729
		if (event->attr.mmap || event->attr.mmap_data)
5730 5731 5732 5733 5734
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
5735 5736 5737 5738 5739 5740 5741
		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
			err = get_callchain_buffers();
			if (err) {
				free_event(event);
				return ERR_PTR(err);
			}
		}
5742
	}
5743

5744
	return event;
T
Thomas Gleixner 已提交
5745 5746
}

5747 5748
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
5749 5750
{
	u32 size;
5751
	int ret;
5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
5776 5777 5778
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
5779 5780
	 */
	if (size > sizeof(*attr)) {
5781 5782 5783
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
5784

5785 5786
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
5787

5788
		for (; addr < end; addr++) {
5789 5790 5791 5792 5793 5794
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
5795
		size = sizeof(*attr);
5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

5809
	if (attr->__reserved_1)
5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

5827 5828
static int
perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
5829
{
5830
	struct perf_buffer *buffer = NULL, *old_buffer = NULL;
5831 5832
	int ret = -EINVAL;

5833
	if (!output_event)
5834 5835
		goto set;

5836 5837
	/* don't allow circular references */
	if (event == output_event)
5838 5839
		goto out;

5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851
	/*
	 * Don't allow cross-cpu buffers
	 */
	if (output_event->cpu != event->cpu)
		goto out;

	/*
	 * If its not a per-cpu buffer, it must be the same task.
	 */
	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
		goto out;

5852
set:
5853
	mutex_lock(&event->mmap_mutex);
5854 5855 5856
	/* Can't redirect output if we've got an active mmap() */
	if (atomic_read(&event->mmap_count))
		goto unlock;
5857

5858 5859
	if (output_event) {
		/* get the buffer we want to redirect to */
5860 5861
		buffer = perf_buffer_get(output_event);
		if (!buffer)
5862
			goto unlock;
5863 5864
	}

5865 5866
	old_buffer = event->buffer;
	rcu_assign_pointer(event->buffer, buffer);
5867
	ret = 0;
5868 5869 5870
unlock:
	mutex_unlock(&event->mmap_mutex);

5871 5872
	if (old_buffer)
		perf_buffer_put(old_buffer);
5873 5874 5875 5876
out:
	return ret;
}

T
Thomas Gleixner 已提交
5877
/**
5878
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
5879
 *
5880
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
5881
 * @pid:		target pid
I
Ingo Molnar 已提交
5882
 * @cpu:		target cpu
5883
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
5884
 */
5885 5886
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
5887
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
5888
{
5889 5890
	struct perf_event *group_leader = NULL, *output_event = NULL;
	struct perf_event *event, *sibling;
5891 5892 5893
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
5894
	struct file *group_file = NULL;
M
Matt Helsley 已提交
5895
	struct task_struct *task = NULL;
5896
	struct pmu *pmu;
5897
	int event_fd;
5898
	int move_group = 0;
5899
	int fput_needed = 0;
5900
	int err;
T
Thomas Gleixner 已提交
5901

5902
	/* for future expandability... */
5903
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
5904 5905
		return -EINVAL;

5906 5907 5908
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
5909

5910 5911 5912 5913 5914
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

5915
	if (attr.freq) {
5916
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
5917 5918 5919
			return -EINVAL;
	}

5920 5921 5922 5923
	event_fd = get_unused_fd_flags(O_RDWR);
	if (event_fd < 0)
		return event_fd;

5924 5925 5926 5927
	if (group_fd != -1) {
		group_leader = perf_fget_light(group_fd, &fput_needed);
		if (IS_ERR(group_leader)) {
			err = PTR_ERR(group_leader);
5928
			goto err_fd;
5929 5930 5931 5932 5933 5934 5935 5936
		}
		group_file = group_leader->filp;
		if (flags & PERF_FLAG_FD_OUTPUT)
			output_event = group_leader;
		if (flags & PERF_FLAG_FD_NO_GROUP)
			group_leader = NULL;
	}

5937 5938 5939 5940 5941 5942 5943 5944
	if (pid != -1) {
		task = find_lively_task_by_vpid(pid);
		if (IS_ERR(task)) {
			err = PTR_ERR(task);
			goto err_group_fd;
		}
	}

5945
	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
5946 5947
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
5948
		goto err_task;
5949 5950
	}

5951 5952 5953 5954 5955
	/*
	 * Special case software events and allow them to be part of
	 * any hardware group.
	 */
	pmu = event->pmu;
5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978

	if (group_leader &&
	    (is_software_event(event) != is_software_event(group_leader))) {
		if (is_software_event(event)) {
			/*
			 * If event and group_leader are not both a software
			 * event, and event is, then group leader is not.
			 *
			 * Allow the addition of software events to !software
			 * groups, this is safe because software events never
			 * fail to schedule.
			 */
			pmu = group_leader->pmu;
		} else if (is_software_event(group_leader) &&
			   (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
			/*
			 * In case the group is a pure software group, and we
			 * try to add a hardware event, move the whole group to
			 * the hardware context.
			 */
			move_group = 1;
		}
	}
5979 5980 5981 5982

	/*
	 * Get the target context (task or percpu):
	 */
M
Matt Helsley 已提交
5983
	ctx = find_get_context(pmu, task, cpu);
5984 5985
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
5986
		goto err_alloc;
5987 5988
	}

I
Ingo Molnar 已提交
5989
	/*
5990
	 * Look up the group leader (we will attach this event to it):
5991
	 */
5992
	if (group_leader) {
5993
		err = -EINVAL;
5994 5995

		/*
I
Ingo Molnar 已提交
5996 5997 5998 5999
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
6000
			goto err_context;
I
Ingo Molnar 已提交
6001 6002 6003
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
6004
		 */
6005 6006 6007 6008 6009 6010 6011 6012
		if (move_group) {
			if (group_leader->ctx->type != ctx->type)
				goto err_context;
		} else {
			if (group_leader->ctx != ctx)
				goto err_context;
		}

6013 6014 6015
		/*
		 * Only a group leader can be exclusive or pinned
		 */
6016
		if (attr.exclusive || attr.pinned)
6017
			goto err_context;
6018 6019 6020 6021 6022
	}

	if (output_event) {
		err = perf_event_set_output(event, output_event);
		if (err)
6023
			goto err_context;
6024
	}
T
Thomas Gleixner 已提交
6025

6026 6027 6028
	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
	if (IS_ERR(event_file)) {
		err = PTR_ERR(event_file);
6029
		goto err_context;
6030
	}
6031

6032 6033 6034 6035
	if (move_group) {
		struct perf_event_context *gctx = group_leader->ctx;

		mutex_lock(&gctx->mutex);
6036
		perf_remove_from_context(group_leader);
6037 6038
		list_for_each_entry(sibling, &group_leader->sibling_list,
				    group_entry) {
6039
			perf_remove_from_context(sibling);
6040 6041 6042 6043
			put_ctx(gctx);
		}
		mutex_unlock(&gctx->mutex);
		put_ctx(gctx);
6044
	}
6045

6046
	event->filp = event_file;
6047
	WARN_ON_ONCE(ctx->parent_ctx);
6048
	mutex_lock(&ctx->mutex);
6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059

	if (move_group) {
		perf_install_in_context(ctx, group_leader, cpu);
		get_ctx(ctx);
		list_for_each_entry(sibling, &group_leader->sibling_list,
				    group_entry) {
			perf_install_in_context(ctx, sibling, cpu);
			get_ctx(ctx);
		}
	}

6060
	perf_install_in_context(ctx, event, cpu);
6061
	++ctx->generation;
6062
	perf_unpin_context(ctx);
6063
	mutex_unlock(&ctx->mutex);
6064

6065
	event->owner = current;
P
Peter Zijlstra 已提交
6066

6067 6068 6069
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
6070

6071 6072 6073 6074
	/*
	 * Precalculate sample_data sizes
	 */
	perf_event__header_size(event);
6075
	perf_event__id_header_size(event);
6076

6077 6078 6079 6080 6081 6082
	/*
	 * Drop the reference on the group_event after placing the
	 * new event on the sibling_list. This ensures destruction
	 * of the group leader will find the pointer to itself in
	 * perf_group_detach().
	 */
6083 6084 6085
	fput_light(group_file, fput_needed);
	fd_install(event_fd, event_file);
	return event_fd;
T
Thomas Gleixner 已提交
6086

6087
err_context:
6088
	perf_unpin_context(ctx);
6089
	put_ctx(ctx);
6090
err_alloc:
6091
	free_event(event);
P
Peter Zijlstra 已提交
6092 6093 6094
err_task:
	if (task)
		put_task_struct(task);
6095
err_group_fd:
6096
	fput_light(group_file, fput_needed);
6097 6098
err_fd:
	put_unused_fd(event_fd);
6099
	return err;
T
Thomas Gleixner 已提交
6100 6101
}

6102 6103 6104 6105 6106
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
M
Matt Helsley 已提交
6107
 * @task: task to profile (NULL for percpu)
6108 6109 6110
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
M
Matt Helsley 已提交
6111
				 struct task_struct *task,
6112
				 perf_overflow_handler_t overflow_handler)
6113 6114
{
	struct perf_event_context *ctx;
6115
	struct perf_event *event;
6116
	int err;
6117

6118 6119 6120
	/*
	 * Get the target context (task or percpu):
	 */
6121

6122
	event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
6123 6124 6125 6126
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
		goto err;
	}
6127

M
Matt Helsley 已提交
6128
	ctx = find_get_context(event->pmu, task, cpu);
6129 6130
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
6131
		goto err_free;
6132
	}
6133 6134 6135 6136 6137 6138

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
6139
	perf_unpin_context(ctx);
6140 6141 6142 6143
	mutex_unlock(&ctx->mutex);

	return event;

6144 6145 6146
err_free:
	free_event(event);
err:
6147
	return ERR_PTR(err);
6148
}
6149
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6150

6151
static void sync_child_event(struct perf_event *child_event,
6152
			       struct task_struct *child)
6153
{
6154
	struct perf_event *parent_event = child_event->parent;
6155
	u64 child_val;
6156

6157 6158
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
6159

P
Peter Zijlstra 已提交
6160
	child_val = perf_event_count(child_event);
6161 6162 6163 6164

	/*
	 * Add back the child's count to the parent's count:
	 */
6165
	atomic64_add(child_val, &parent_event->child_count);
6166 6167 6168 6169
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
6170 6171

	/*
6172
	 * Remove this event from the parent's list
6173
	 */
6174 6175 6176 6177
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
6178 6179

	/*
6180
	 * Release the parent event, if this was the last
6181 6182
	 * reference to it.
	 */
6183
	fput(parent_event->filp);
6184 6185
}

6186
static void
6187 6188
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
6189
			 struct task_struct *child)
6190
{
6191
	struct perf_event *parent_event;
6192

6193
	perf_remove_from_context(child_event);
6194

6195
	parent_event = child_event->parent;
6196
	/*
6197
	 * It can happen that parent exits first, and has events
6198
	 * that are still around due to the child reference. These
6199
	 * events need to be zapped - but otherwise linger.
6200
	 */
6201 6202 6203
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
6204
	}
6205 6206
}

P
Peter Zijlstra 已提交
6207
static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6208
{
6209 6210
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
6211
	unsigned long flags;
6212

P
Peter Zijlstra 已提交
6213
	if (likely(!child->perf_event_ctxp[ctxn])) {
6214
		perf_event_task(child, NULL, 0);
6215
		return;
P
Peter Zijlstra 已提交
6216
	}
6217

6218
	local_irq_save(flags);
6219 6220 6221 6222 6223 6224
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
6225
	child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6226
	task_ctx_sched_out(child_ctx, EVENT_ALL);
6227 6228 6229

	/*
	 * Take the context lock here so that if find_get_context is
6230
	 * reading child->perf_event_ctxp, we wait until it has
6231 6232
	 * incremented the context's refcount before we do put_ctx below.
	 */
6233
	raw_spin_lock(&child_ctx->lock);
P
Peter Zijlstra 已提交
6234
	child->perf_event_ctxp[ctxn] = NULL;
6235 6236 6237
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
6238
	 * the events from it.
6239 6240
	 */
	unclone_ctx(child_ctx);
6241
	update_context_time(child_ctx);
6242
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
6243 6244

	/*
6245 6246 6247
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
6248
	 */
6249
	perf_event_task(child, child_ctx, 0);
6250

6251 6252 6253
	/*
	 * We can recurse on the same lock type through:
	 *
6254 6255 6256
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
6257 6258 6259 6260 6261
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
6262
	mutex_lock(&child_ctx->mutex);
6263

6264
again:
6265 6266 6267 6268 6269
	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
				 group_entry)
		__perf_event_exit_task(child_event, child_ctx, child);

	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6270
				 group_entry)
6271
		__perf_event_exit_task(child_event, child_ctx, child);
6272 6273

	/*
6274
	 * If the last event was a group event, it will have appended all
6275 6276 6277
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
6278 6279
	if (!list_empty(&child_ctx->pinned_groups) ||
	    !list_empty(&child_ctx->flexible_groups))
6280
		goto again;
6281 6282 6283 6284

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
6285 6286
}

P
Peter Zijlstra 已提交
6287 6288 6289 6290 6291
/*
 * When a child task exits, feed back event values to parent events.
 */
void perf_event_exit_task(struct task_struct *child)
{
P
Peter Zijlstra 已提交
6292
	struct perf_event *event, *tmp;
P
Peter Zijlstra 已提交
6293 6294
	int ctxn;

P
Peter Zijlstra 已提交
6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309
	mutex_lock(&child->perf_event_mutex);
	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
				 owner_entry) {
		list_del_init(&event->owner_entry);

		/*
		 * Ensure the list deletion is visible before we clear
		 * the owner, closes a race against perf_release() where
		 * we need to serialize on the owner->perf_event_mutex.
		 */
		smp_wmb();
		event->owner = NULL;
	}
	mutex_unlock(&child->perf_event_mutex);

P
Peter Zijlstra 已提交
6310 6311 6312 6313
	for_each_task_context_nr(ctxn)
		perf_event_exit_task_context(child, ctxn);
}

6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327
static void perf_free_event(struct perf_event *event,
			    struct perf_event_context *ctx)
{
	struct perf_event *parent = event->parent;

	if (WARN_ON_ONCE(!parent))
		return;

	mutex_lock(&parent->child_mutex);
	list_del_init(&event->child_list);
	mutex_unlock(&parent->child_mutex);

	fput(parent->filp);

6328
	perf_group_detach(event);
6329 6330 6331 6332
	list_del_event(event, ctx);
	free_event(event);
}

6333 6334
/*
 * free an unexposed, unused context as created by inheritance by
P
Peter Zijlstra 已提交
6335
 * perf_event_init_task below, used by fork() in case of fail.
6336
 */
6337
void perf_event_free_task(struct task_struct *task)
6338
{
P
Peter Zijlstra 已提交
6339
	struct perf_event_context *ctx;
6340
	struct perf_event *event, *tmp;
P
Peter Zijlstra 已提交
6341
	int ctxn;
6342

P
Peter Zijlstra 已提交
6343 6344 6345 6346
	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (!ctx)
			continue;
6347

P
Peter Zijlstra 已提交
6348
		mutex_lock(&ctx->mutex);
6349
again:
P
Peter Zijlstra 已提交
6350 6351 6352
		list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
				group_entry)
			perf_free_event(event, ctx);
6353

P
Peter Zijlstra 已提交
6354 6355 6356
		list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
				group_entry)
			perf_free_event(event, ctx);
6357

P
Peter Zijlstra 已提交
6358 6359 6360
		if (!list_empty(&ctx->pinned_groups) ||
				!list_empty(&ctx->flexible_groups))
			goto again;
6361

P
Peter Zijlstra 已提交
6362
		mutex_unlock(&ctx->mutex);
6363

P
Peter Zijlstra 已提交
6364 6365
		put_ctx(ctx);
	}
6366 6367
}

6368 6369 6370 6371 6372 6373 6374 6375
void perf_event_delayed_put(struct task_struct *task)
{
	int ctxn;

	for_each_task_context_nr(ctxn)
		WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
}

P
Peter Zijlstra 已提交
6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387
/*
 * inherit a event from parent task to child task:
 */
static struct perf_event *
inherit_event(struct perf_event *parent_event,
	      struct task_struct *parent,
	      struct perf_event_context *parent_ctx,
	      struct task_struct *child,
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
{
	struct perf_event *child_event;
6388
	unsigned long flags;
P
Peter Zijlstra 已提交
6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400

	/*
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
	if (parent_event->parent)
		parent_event = parent_event->parent;

	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu,
6401
					   child,
P
Peter Zijlstra 已提交
6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430
					   group_leader, parent_event,
					   NULL);
	if (IS_ERR(child_event))
		return child_event;
	get_ctx(child_ctx);

	/*
	 * Make the child state follow the state of the parent event,
	 * not its attr.disabled bit.  We hold the parent's mutex,
	 * so we won't race with perf_event_{en, dis}able_family.
	 */
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
	else
		child_event->state = PERF_EVENT_STATE_OFF;

	if (parent_event->attr.freq) {
		u64 sample_period = parent_event->hw.sample_period;
		struct hw_perf_event *hwc = &child_event->hw;

		hwc->sample_period = sample_period;
		hwc->last_period   = sample_period;

		local64_set(&hwc->period_left, sample_period);
	}

	child_event->ctx = child_ctx;
	child_event->overflow_handler = parent_event->overflow_handler;

6431 6432 6433 6434
	/*
	 * Precalculate sample_data sizes
	 */
	perf_event__header_size(child_event);
6435
	perf_event__id_header_size(child_event);
6436

P
Peter Zijlstra 已提交
6437 6438 6439
	/*
	 * Link it up in the child's context:
	 */
6440
	raw_spin_lock_irqsave(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
6441
	add_event_to_ctx(child_event, child_ctx);
6442
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483

	/*
	 * Get a reference to the parent filp - we will fput it
	 * when the child event exits. This is safe to do because
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
	atomic_long_inc(&parent_event->filp->f_count);

	/*
	 * Link this into the parent event's child list
	 */
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);

	return child_event;
}

static int inherit_group(struct perf_event *parent_event,
	      struct task_struct *parent,
	      struct perf_event_context *parent_ctx,
	      struct task_struct *child,
	      struct perf_event_context *child_ctx)
{
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;

	leader = inherit_event(parent_event, parent, parent_ctx,
				 child, NULL, child_ctx);
	if (IS_ERR(leader))
		return PTR_ERR(leader);
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
	}
	return 0;
6484 6485 6486 6487 6488
}

static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
		   struct perf_event_context *parent_ctx,
P
Peter Zijlstra 已提交
6489
		   struct task_struct *child, int ctxn,
6490 6491 6492
		   int *inherited_all)
{
	int ret;
P
Peter Zijlstra 已提交
6493
	struct perf_event_context *child_ctx;
6494 6495 6496 6497

	if (!event->attr.inherit) {
		*inherited_all = 0;
		return 0;
6498 6499
	}

6500
	child_ctx = child->perf_event_ctxp[ctxn];
6501 6502 6503 6504 6505 6506 6507
	if (!child_ctx) {
		/*
		 * This is executed from the parent task context, so
		 * inherit events that have been marked for cloning.
		 * First allocate and initialize a context for the
		 * child.
		 */
6508

6509
		child_ctx = alloc_perf_context(event->pmu, child);
6510 6511
		if (!child_ctx)
			return -ENOMEM;
6512

P
Peter Zijlstra 已提交
6513
		child->perf_event_ctxp[ctxn] = child_ctx;
6514 6515 6516 6517 6518 6519 6520 6521 6522
	}

	ret = inherit_group(event, parent, parent_ctx,
			    child, child_ctx);

	if (ret)
		*inherited_all = 0;

	return ret;
6523 6524
}

6525
/*
6526
 * Initialize the perf_event context in task_struct
6527
 */
P
Peter Zijlstra 已提交
6528
int perf_event_init_context(struct task_struct *child, int ctxn)
6529
{
6530
	struct perf_event_context *child_ctx, *parent_ctx;
6531 6532
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
6533
	struct task_struct *parent = current;
6534
	int inherited_all = 1;
6535
	unsigned long flags;
6536
	int ret = 0;
6537

P
Peter Zijlstra 已提交
6538
	if (likely(!parent->perf_event_ctxp[ctxn]))
6539 6540
		return 0;

6541
	/*
6542 6543
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
6544
	 */
P
Peter Zijlstra 已提交
6545
	parent_ctx = perf_pin_task_context(parent, ctxn);
6546

6547 6548 6549 6550 6551 6552 6553
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

6554 6555 6556 6557
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
6558
	mutex_lock(&parent_ctx->mutex);
6559 6560 6561 6562 6563

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
6564
	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
P
Peter Zijlstra 已提交
6565 6566
		ret = inherit_task_group(event, parent, parent_ctx,
					 child, ctxn, &inherited_all);
6567 6568 6569
		if (ret)
			break;
	}
6570

6571 6572 6573 6574 6575 6576 6577 6578 6579
	/*
	 * We can't hold ctx->lock when iterating the ->flexible_group list due
	 * to allocations, but we need to prevent rotation because
	 * rotate_ctx() will change the list from interrupt context.
	 */
	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
	parent_ctx->rotate_disable = 1;
	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);

6580
	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
P
Peter Zijlstra 已提交
6581 6582
		ret = inherit_task_group(event, parent, parent_ctx,
					 child, ctxn, &inherited_all);
6583
		if (ret)
6584
			break;
6585 6586
	}

6587 6588 6589
	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
	parent_ctx->rotate_disable = 0;

P
Peter Zijlstra 已提交
6590
	child_ctx = child->perf_event_ctxp[ctxn];
6591

6592
	if (child_ctx && inherited_all) {
6593 6594 6595
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
P
Peter Zijlstra 已提交
6596 6597 6598
		 *
		 * Note that if the parent is a clone, the holding of
		 * parent_ctx->lock avoids it from being uncloned.
6599
		 */
P
Peter Zijlstra 已提交
6600
		cloned_ctx = parent_ctx->parent_ctx;
6601 6602
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
6603
			child_ctx->parent_gen = parent_ctx->parent_gen;
6604 6605 6606 6607 6608
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
6609 6610
	}

P
Peter Zijlstra 已提交
6611
	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6612
	mutex_unlock(&parent_ctx->mutex);
6613

6614
	perf_unpin_context(parent_ctx);
6615
	put_ctx(parent_ctx);
6616

6617
	return ret;
6618 6619
}

P
Peter Zijlstra 已提交
6620 6621 6622 6623 6624 6625 6626
/*
 * Initialize the perf_event context in task_struct
 */
int perf_event_init_task(struct task_struct *child)
{
	int ctxn, ret;

6627 6628 6629 6630
	memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);

P
Peter Zijlstra 已提交
6631 6632 6633 6634 6635 6636 6637 6638 6639
	for_each_task_context_nr(ctxn) {
		ret = perf_event_init_context(child, ctxn);
		if (ret)
			return ret;
	}

	return 0;
}

6640 6641
static void __init perf_event_init_all_cpus(void)
{
6642
	struct swevent_htable *swhash;
6643 6644 6645
	int cpu;

	for_each_possible_cpu(cpu) {
6646 6647
		swhash = &per_cpu(swevent_htable, cpu);
		mutex_init(&swhash->hlist_mutex);
6648
		INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
6649 6650 6651
	}
}

6652
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
6653
{
P
Peter Zijlstra 已提交
6654
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
T
Thomas Gleixner 已提交
6655

6656 6657
	mutex_lock(&swhash->hlist_mutex);
	if (swhash->hlist_refcount > 0) {
6658 6659
		struct swevent_hlist *hlist;

6660 6661 6662
		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
		WARN_ON(!hlist);
		rcu_assign_pointer(swhash->swevent_hlist, hlist);
6663
	}
6664
	mutex_unlock(&swhash->hlist_mutex);
T
Thomas Gleixner 已提交
6665 6666
}

P
Peter Zijlstra 已提交
6667
#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
6668
static void perf_pmu_rotate_stop(struct pmu *pmu)
T
Thomas Gleixner 已提交
6669
{
6670 6671 6672 6673 6674 6675 6676
	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

	WARN_ON(!irqs_disabled());

	list_del_init(&cpuctx->rotation_list);
}

P
Peter Zijlstra 已提交
6677
static void __perf_event_exit_context(void *__info)
T
Thomas Gleixner 已提交
6678
{
P
Peter Zijlstra 已提交
6679
	struct perf_event_context *ctx = __info;
6680
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
6681

P
Peter Zijlstra 已提交
6682
	perf_pmu_rotate_stop(ctx->pmu);
6683

6684
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
6685
		__perf_remove_from_context(event);
6686
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
6687
		__perf_remove_from_context(event);
T
Thomas Gleixner 已提交
6688
}
P
Peter Zijlstra 已提交
6689 6690 6691 6692 6693 6694 6695 6696 6697

static void perf_event_exit_cpu_context(int cpu)
{
	struct perf_event_context *ctx;
	struct pmu *pmu;
	int idx;

	idx = srcu_read_lock(&pmus_srcu);
	list_for_each_entry_rcu(pmu, &pmus, entry) {
6698
		ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
P
Peter Zijlstra 已提交
6699 6700 6701 6702 6703 6704 6705 6706

		mutex_lock(&ctx->mutex);
		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
		mutex_unlock(&ctx->mutex);
	}
	srcu_read_unlock(&pmus_srcu, idx);
}

6707
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
6708
{
6709
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6710

6711 6712 6713
	mutex_lock(&swhash->hlist_mutex);
	swevent_hlist_release(swhash);
	mutex_unlock(&swhash->hlist_mutex);
6714

P
Peter Zijlstra 已提交
6715
	perf_event_exit_cpu_context(cpu);
T
Thomas Gleixner 已提交
6716 6717
}
#else
6718
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
6719 6720
#endif

P
Peter Zijlstra 已提交
6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740
static int
perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
{
	int cpu;

	for_each_online_cpu(cpu)
		perf_event_exit_cpu(cpu);

	return NOTIFY_OK;
}

/*
 * Run the perf reboot notifier at the very last possible moment so that
 * the generic watchdog code runs as long as possible.
 */
static struct notifier_block perf_reboot_notifier = {
	.notifier_call = perf_reboot,
	.priority = INT_MIN,
};

T
Thomas Gleixner 已提交
6741 6742 6743 6744 6745
static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

P
Peter Zijlstra 已提交
6746
	switch (action & ~CPU_TASKS_FROZEN) {
T
Thomas Gleixner 已提交
6747 6748

	case CPU_UP_PREPARE:
P
Peter Zijlstra 已提交
6749
	case CPU_DOWN_FAILED:
6750
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
6751 6752
		break;

P
Peter Zijlstra 已提交
6753
	case CPU_UP_CANCELED:
T
Thomas Gleixner 已提交
6754
	case CPU_DOWN_PREPARE:
6755
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
6756 6757 6758 6759 6760 6761 6762 6763 6764
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

6765
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
6766
{
6767 6768
	int ret;

P
Peter Zijlstra 已提交
6769 6770
	idr_init(&pmu_idr);

6771
	perf_event_init_all_cpus();
6772
	init_srcu_struct(&pmus_srcu);
P
Peter Zijlstra 已提交
6773 6774 6775
	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
	perf_pmu_register(&perf_cpu_clock, NULL, -1);
	perf_pmu_register(&perf_task_clock, NULL, -1);
6776 6777
	perf_tp_register();
	perf_cpu_notifier(perf_cpu_notify);
P
Peter Zijlstra 已提交
6778
	register_reboot_notifier(&perf_reboot_notifier);
6779 6780 6781

	ret = init_hw_breakpoint();
	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
T
Thomas Gleixner 已提交
6782
}
P
Peter Zijlstra 已提交
6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810

static int __init perf_event_sysfs_init(void)
{
	struct pmu *pmu;
	int ret;

	mutex_lock(&pmus_lock);

	ret = bus_register(&pmu_bus);
	if (ret)
		goto unlock;

	list_for_each_entry(pmu, &pmus, entry) {
		if (!pmu->name || pmu->type < 0)
			continue;

		ret = pmu_dev_alloc(pmu);
		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
	}
	pmu_bus_running = 1;
	ret = 0;

unlock:
	mutex_unlock(&pmus_lock);

	return ret;
}
device_initcall(perf_event_sysfs_init);