perf_event.c 126.1 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
16
#include <linux/file.h>
T
Thomas Gleixner 已提交
17 18
#include <linux/poll.h>
#include <linux/sysfs.h>
19
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
20
#include <linux/percpu.h>
21
#include <linux/ptrace.h>
22
#include <linux/vmstat.h>
23
#include <linux/vmalloc.h>
24 25
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
26 27 28
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
29
#include <linux/kernel_stat.h>
30
#include <linux/perf_event.h>
L
Li Zefan 已提交
31
#include <linux/ftrace_event.h>
32
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
33

34 35
#include <asm/irq_regs.h>

T
Thomas Gleixner 已提交
36
/*
37
 * Each CPU has a list of per CPU events:
T
Thomas Gleixner 已提交
38
 */
39
static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
T
Thomas Gleixner 已提交
40

41
int perf_max_events __read_mostly = 1;
T
Thomas Gleixner 已提交
42 43 44
static int perf_reserved_percpu __read_mostly;
static int perf_overcommit __read_mostly = 1;

45 46 47 48
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
49

50
/*
51
 * perf event paranoia level:
52 53
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
54
 *   1 - disallow cpu events for unpriv
55
 *   2 - disallow kernel profiling for unpriv
56
 */
57
int sysctl_perf_event_paranoid __read_mostly = 1;
58

59 60
static inline bool perf_paranoid_tracepoint_raw(void)
{
61
	return sysctl_perf_event_paranoid > -1;
62 63
}

64 65
static inline bool perf_paranoid_cpu(void)
{
66
	return sysctl_perf_event_paranoid > 0;
67 68 69 70
}

static inline bool perf_paranoid_kernel(void)
{
71
	return sysctl_perf_event_paranoid > 1;
72 73
}

74
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
75 76

/*
77
 * max perf event sample rate
78
 */
79
int sysctl_perf_event_sample_rate __read_mostly = 100000;
80

81
static atomic64_t perf_event_id;
82

T
Thomas Gleixner 已提交
83
/*
84
 * Lock for (sysadmin-configurable) event reservations:
T
Thomas Gleixner 已提交
85
 */
86
static DEFINE_SPINLOCK(perf_resource_lock);
T
Thomas Gleixner 已提交
87 88 89 90

/*
 * Architecture provided APIs - weak aliases:
 */
91
extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
T
Thomas Gleixner 已提交
92
{
93
	return NULL;
T
Thomas Gleixner 已提交
94 95
}

96 97 98
void __weak hw_perf_disable(void)		{ barrier(); }
void __weak hw_perf_enable(void)		{ barrier(); }

99 100
void __weak hw_perf_event_setup(int cpu)	{ barrier(); }
void __weak hw_perf_event_setup_online(int cpu)	{ barrier(); }
101
void __weak hw_perf_event_setup_offline(int cpu)	{ barrier(); }
102 103

int __weak
104
hw_perf_group_sched_in(struct perf_event *group_leader,
105
	       struct perf_cpu_context *cpuctx,
106
	       struct perf_event_context *ctx)
107 108 109
{
	return 0;
}
T
Thomas Gleixner 已提交
110

111
void __weak perf_event_print_debug(void)	{ }
112

113
static DEFINE_PER_CPU(int, perf_disable_count);
114 115 116

void __perf_disable(void)
{
117
	__get_cpu_var(perf_disable_count)++;
118 119 120 121
}

bool __perf_enable(void)
{
122
	return !--__get_cpu_var(perf_disable_count);
123 124 125 126 127 128 129 130 131 132 133 134 135 136
}

void perf_disable(void)
{
	__perf_disable();
	hw_perf_disable();
}

void perf_enable(void)
{
	if (__perf_enable())
		hw_perf_enable();
}

137
static void get_ctx(struct perf_event_context *ctx)
138
{
139
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
140 141
}

142 143
static void free_ctx(struct rcu_head *head)
{
144
	struct perf_event_context *ctx;
145

146
	ctx = container_of(head, struct perf_event_context, rcu_head);
147 148 149
	kfree(ctx);
}

150
static void put_ctx(struct perf_event_context *ctx)
151
{
152 153 154
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
155 156 157
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
158
	}
159 160
}

161
static void unclone_ctx(struct perf_event_context *ctx)
162 163 164 165 166 167 168
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

169
/*
170
 * If we inherit events we want to return the parent event id
171 172
 * to userspace.
 */
173
static u64 primary_event_id(struct perf_event *event)
174
{
175
	u64 id = event->id;
176

177 178
	if (event->parent)
		id = event->parent->id;
179 180 181 182

	return id;
}

183
/*
184
 * Get the perf_event_context for a task and lock it.
185 186 187
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
188
static struct perf_event_context *
189
perf_lock_task_context(struct task_struct *task, unsigned long *flags)
190
{
191
	struct perf_event_context *ctx;
192 193 194

	rcu_read_lock();
 retry:
195
	ctx = rcu_dereference(task->perf_event_ctxp);
196 197 198 199
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
200
		 * perf_event_task_sched_out, though the
201 202 203 204 205 206
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
207
		raw_spin_lock_irqsave(&ctx->lock, *flags);
208
		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
209
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
210 211
			goto retry;
		}
212 213

		if (!atomic_inc_not_zero(&ctx->refcount)) {
214
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
215 216
			ctx = NULL;
		}
217 218 219 220 221 222 223 224 225 226
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
227
static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
228
{
229
	struct perf_event_context *ctx;
230 231 232 233 234
	unsigned long flags;

	ctx = perf_lock_task_context(task, &flags);
	if (ctx) {
		++ctx->pin_count;
235
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
236 237 238 239
	}
	return ctx;
}

240
static void perf_unpin_context(struct perf_event_context *ctx)
241 242 243
{
	unsigned long flags;

244
	raw_spin_lock_irqsave(&ctx->lock, flags);
245
	--ctx->pin_count;
246
	raw_spin_unlock_irqrestore(&ctx->lock, flags);
247 248 249
	put_ctx(ctx);
}

250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
static inline u64 perf_clock(void)
{
	return cpu_clock(smp_processor_id());
}

/*
 * Update the record of the current time in a context.
 */
static void update_context_time(struct perf_event_context *ctx)
{
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
}

/*
 * Update the total_time_enabled and total_time_running fields for a event.
 */
static void update_event_times(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	u64 run_end;

	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
		return;

278 279 280 281 282 283
	if (ctx->is_active)
		run_end = ctx->time;
	else
		run_end = event->tstamp_stopped;

	event->total_time_enabled = run_end - event->tstamp_enabled;
284 285 286 287 288 289 290 291 292

	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
	else
		run_end = ctx->time;

	event->total_time_running = run_end - event->tstamp_running;
}

293 294 295 296 297 298 299 300 301
static struct list_head *
ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
{
	if (event->attr.pinned)
		return &ctx->pinned_groups;
	else
		return &ctx->flexible_groups;
}

302
/*
303
 * Add a event from the lists for its context.
304 305
 * Must be called with ctx->mutex and ctx->lock held.
 */
306
static void
307
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
308
{
309
	struct perf_event *group_leader = event->group_leader;
310 311

	/*
312 313
	 * Depending on whether it is a standalone or sibling event,
	 * add it straight to the context's event list, or to the group
314 315
	 * leader's sibling list:
	 */
316 317 318
	if (group_leader == event) {
		struct list_head *list;

319 320 321
		if (is_software_event(event))
			event->group_flags |= PERF_GROUP_SOFTWARE;

322 323 324
		list = ctx_group_list(event, ctx);
		list_add_tail(&event->group_entry, list);
	} else {
325 326 327 328
		if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
		    !is_software_event(event))
			group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;

329
		list_add_tail(&event->group_entry, &group_leader->sibling_list);
P
Peter Zijlstra 已提交
330 331
		group_leader->nr_siblings++;
	}
P
Peter Zijlstra 已提交
332

333 334 335
	list_add_rcu(&event->event_entry, &ctx->event_list);
	ctx->nr_events++;
	if (event->attr.inherit_stat)
336
		ctx->nr_stat++;
337 338
}

339
/*
340
 * Remove a event from the lists for its context.
341
 * Must be called with ctx->mutex and ctx->lock held.
342
 */
343
static void
344
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
345
{
346
	struct perf_event *sibling, *tmp;
347

348
	if (list_empty(&event->group_entry))
349
		return;
350 351
	ctx->nr_events--;
	if (event->attr.inherit_stat)
352
		ctx->nr_stat--;
353

354 355
	list_del_init(&event->group_entry);
	list_del_rcu(&event->event_entry);
356

357 358
	if (event->group_leader != event)
		event->group_leader->nr_siblings--;
P
Peter Zijlstra 已提交
359

360
	update_event_times(event);
361 362 363 364 365 366 367 368 369 370

	/*
	 * If event was in error state, then keep it
	 * that way, otherwise bogus counts will be
	 * returned on read(). The only way to get out
	 * of error state is by explicit re-enabling
	 * of the event
	 */
	if (event->state > PERF_EVENT_STATE_OFF)
		event->state = PERF_EVENT_STATE_OFF;
371

372
	/*
373 374
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
375 376
	 * to the context list directly:
	 */
377
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
378
		struct list_head *list;
379

380 381
		list = ctx_group_list(event, ctx);
		list_move_tail(&sibling->group_entry, list);
382
		sibling->group_leader = sibling;
383 384 385

		/* Inherit group flags from the previous leader */
		sibling->group_flags = event->group_flags;
386 387 388
	}
}

389
static void
390
event_sched_out(struct perf_event *event,
391
		  struct perf_cpu_context *cpuctx,
392
		  struct perf_event_context *ctx)
393
{
394
	if (event->state != PERF_EVENT_STATE_ACTIVE)
395 396
		return;

397 398 399 400
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
401
	}
402 403 404
	event->tstamp_stopped = ctx->time;
	event->pmu->disable(event);
	event->oncpu = -1;
405

406
	if (!is_software_event(event))
407 408
		cpuctx->active_oncpu--;
	ctx->nr_active--;
409
	if (event->attr.exclusive || !cpuctx->active_oncpu)
410 411 412
		cpuctx->exclusive = 0;
}

413
static void
414
group_sched_out(struct perf_event *group_event,
415
		struct perf_cpu_context *cpuctx,
416
		struct perf_event_context *ctx)
417
{
418
	struct perf_event *event;
419

420
	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
421 422
		return;

423
	event_sched_out(group_event, cpuctx, ctx);
424 425 426 427

	/*
	 * Schedule out siblings (if any):
	 */
428 429
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
430

431
	if (group_event->attr.exclusive)
432 433 434
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
435
/*
436
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
437
 *
438
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
439 440
 * remove it from the context list.
 */
441
static void __perf_event_remove_from_context(void *info)
T
Thomas Gleixner 已提交
442 443
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
444 445
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
446 447 448 449 450 451

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
	 */
452
	if (ctx->task && cpuctx->task_ctx != ctx)
T
Thomas Gleixner 已提交
453 454
		return;

455
	raw_spin_lock(&ctx->lock);
456 457
	/*
	 * Protect the list operation against NMI by disabling the
458
	 * events on a global level.
459 460
	 */
	perf_disable();
T
Thomas Gleixner 已提交
461

462
	event_sched_out(event, cpuctx, ctx);
463

464
	list_del_event(event, ctx);
T
Thomas Gleixner 已提交
465 466 467

	if (!ctx->task) {
		/*
468
		 * Allow more per task events with respect to the
T
Thomas Gleixner 已提交
469 470 471
		 * reservation:
		 */
		cpuctx->max_pertask =
472 473
			min(perf_max_events - ctx->nr_events,
			    perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
474 475
	}

476
	perf_enable();
477
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
478 479 480 481
}


/*
482
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
483
 *
484
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
485
 *
486
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
487
 * call when the task is on a CPU.
488
 *
489 490
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
491 492
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
493
 * When called from perf_event_exit_task, it's OK because the
494
 * context has been detached from its task.
T
Thomas Gleixner 已提交
495
 */
496
static void perf_event_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
497
{
498
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
499 500 501 502
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
503
		 * Per cpu events are removed via an smp call and
504
		 * the removal is always successful.
T
Thomas Gleixner 已提交
505
		 */
506 507 508
		smp_call_function_single(event->cpu,
					 __perf_event_remove_from_context,
					 event, 1);
T
Thomas Gleixner 已提交
509 510 511 512
		return;
	}

retry:
513 514
	task_oncpu_function_call(task, __perf_event_remove_from_context,
				 event);
T
Thomas Gleixner 已提交
515

516
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
517 518 519
	/*
	 * If the context is active we need to retry the smp call.
	 */
520
	if (ctx->nr_active && !list_empty(&event->group_entry)) {
521
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
522 523 524 525 526
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
527
	 * can remove the event safely, if the call above did not
T
Thomas Gleixner 已提交
528 529
	 * succeed.
	 */
P
Peter Zijlstra 已提交
530
	if (!list_empty(&event->group_entry))
531
		list_del_event(event, ctx);
532
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
533 534
}

535
/*
536
 * Update total_time_enabled and total_time_running for all events in a group.
537
 */
538
static void update_group_times(struct perf_event *leader)
539
{
540
	struct perf_event *event;
541

542 543 544
	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
545 546
}

547
/*
548
 * Cross CPU call to disable a performance event
549
 */
550
static void __perf_event_disable(void *info)
551
{
552
	struct perf_event *event = info;
553
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
554
	struct perf_event_context *ctx = event->ctx;
555 556

	/*
557 558
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
559
	 */
560
	if (ctx->task && cpuctx->task_ctx != ctx)
561 562
		return;

563
	raw_spin_lock(&ctx->lock);
564 565

	/*
566
	 * If the event is on, turn it off.
567 568
	 * If it is in error state, leave it in error state.
	 */
569
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
570
		update_context_time(ctx);
571 572 573
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
574
		else
575 576
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
577 578
	}

579
	raw_spin_unlock(&ctx->lock);
580 581 582
}

/*
583
 * Disable a event.
584
 *
585 586
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
587
 * remains valid.  This condition is satisifed when called through
588 589 590 591
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
592
 * is the current context on this CPU and preemption is disabled,
593
 * hence we can't get into perf_event_task_sched_out for this context.
594
 */
595
void perf_event_disable(struct perf_event *event)
596
{
597
	struct perf_event_context *ctx = event->ctx;
598 599 600 601
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
602
		 * Disable the event on the cpu that it's on
603
		 */
604 605
		smp_call_function_single(event->cpu, __perf_event_disable,
					 event, 1);
606 607 608 609
		return;
	}

 retry:
610
	task_oncpu_function_call(task, __perf_event_disable, event);
611

612
	raw_spin_lock_irq(&ctx->lock);
613
	/*
614
	 * If the event is still active, we need to retry the cross-call.
615
	 */
616
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
617
		raw_spin_unlock_irq(&ctx->lock);
618 619 620 621 622 623 624
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
625 626 627
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
628
	}
629

630
	raw_spin_unlock_irq(&ctx->lock);
631 632
}

633
static int
634
event_sched_in(struct perf_event *event,
635
		 struct perf_cpu_context *cpuctx,
636
		 struct perf_event_context *ctx)
637
{
638
	if (event->state <= PERF_EVENT_STATE_OFF)
639 640
		return 0;

641
	event->state = PERF_EVENT_STATE_ACTIVE;
642
	event->oncpu = smp_processor_id();
643 644 645 646 647
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

648 649 650
	if (event->pmu->enable(event)) {
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
651 652 653
		return -EAGAIN;
	}

654
	event->tstamp_running += ctx->time - event->tstamp_stopped;
655

656
	if (!is_software_event(event))
657
		cpuctx->active_oncpu++;
658 659
	ctx->nr_active++;

660
	if (event->attr.exclusive)
661 662
		cpuctx->exclusive = 1;

663 664 665
	return 0;
}

666
static int
667
group_sched_in(struct perf_event *group_event,
668
	       struct perf_cpu_context *cpuctx,
669
	       struct perf_event_context *ctx)
670
{
671
	struct perf_event *event, *partial_group;
672 673
	int ret;

674
	if (group_event->state == PERF_EVENT_STATE_OFF)
675 676
		return 0;

677
	ret = hw_perf_group_sched_in(group_event, cpuctx, ctx);
678 679 680
	if (ret)
		return ret < 0 ? ret : 0;

681
	if (event_sched_in(group_event, cpuctx, ctx))
682 683 684 685 686
		return -EAGAIN;

	/*
	 * Schedule in siblings as one group (if any):
	 */
687
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
688
		if (event_sched_in(event, cpuctx, ctx)) {
689
			partial_group = event;
690 691 692 693 694 695 696 697 698 699 700
			goto group_error;
		}
	}

	return 0;

group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
	 */
701 702
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
703
			break;
704
		event_sched_out(event, cpuctx, ctx);
705
	}
706
	event_sched_out(group_event, cpuctx, ctx);
707 708 709 710

	return -EAGAIN;
}

711
/*
712
 * Work out whether we can put this event group on the CPU now.
713
 */
714
static int group_can_go_on(struct perf_event *event,
715 716 717 718
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
719
	 * Groups consisting entirely of software events can always go on.
720
	 */
721
	if (event->group_flags & PERF_GROUP_SOFTWARE)
722 723 724
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
725
	 * events can go on.
726 727 728 729 730
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
731
	 * events on the CPU, it can't go on.
732
	 */
733
	if (event->attr.exclusive && cpuctx->active_oncpu)
734 735 736 737 738 739 740 741
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

742 743
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
744
{
745 746 747 748
	list_add_event(event, ctx);
	event->tstamp_enabled = ctx->time;
	event->tstamp_running = ctx->time;
	event->tstamp_stopped = ctx->time;
749 750
}

T
Thomas Gleixner 已提交
751
/*
752
 * Cross CPU call to install and enable a performance event
753 754
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
755 756 757 758
 */
static void __perf_install_in_context(void *info)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
759 760 761
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
762
	int err;
T
Thomas Gleixner 已提交
763 764 765 766 767

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
768
	 * Or possibly this is the right context but it isn't
769
	 * on this cpu because it had no events.
T
Thomas Gleixner 已提交
770
	 */
771
	if (ctx->task && cpuctx->task_ctx != ctx) {
772
		if (cpuctx->task_ctx || ctx->task != current)
773 774 775
			return;
		cpuctx->task_ctx = ctx;
	}
T
Thomas Gleixner 已提交
776

777
	raw_spin_lock(&ctx->lock);
778
	ctx->is_active = 1;
779
	update_context_time(ctx);
T
Thomas Gleixner 已提交
780 781 782

	/*
	 * Protect the list operation against NMI by disabling the
783
	 * events on a global level. NOP for non NMI based events.
T
Thomas Gleixner 已提交
784
	 */
785
	perf_disable();
T
Thomas Gleixner 已提交
786

787
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
788

789 790 791
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

792
	/*
793
	 * Don't put the event on if it is disabled or if
794 795
	 * it is in a group and the group isn't on.
	 */
796 797
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
798 799
		goto unlock;

800
	/*
801 802 803
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
804
	 */
805
	if (!group_can_go_on(event, cpuctx, 1))
806 807
		err = -EEXIST;
	else
808
		err = event_sched_in(event, cpuctx, ctx);
809

810 811
	if (err) {
		/*
812
		 * This event couldn't go on.  If it is in a group
813
		 * then we have to pull the whole group off.
814
		 * If the event group is pinned then put it in error state.
815
		 */
816
		if (leader != event)
817
			group_sched_out(leader, cpuctx, ctx);
818
		if (leader->attr.pinned) {
819
			update_group_times(leader);
820
			leader->state = PERF_EVENT_STATE_ERROR;
821
		}
822
	}
T
Thomas Gleixner 已提交
823

824
	if (!err && !ctx->task && cpuctx->max_pertask)
T
Thomas Gleixner 已提交
825 826
		cpuctx->max_pertask--;

827
 unlock:
828
	perf_enable();
829

830
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
831 832 833
}

/*
834
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
835
 *
836 837
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
838
 *
839
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
840 841
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
842 843
 *
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
844 845
 */
static void
846 847
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
848 849 850 851 852 853
			int cpu)
{
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
854
		 * Per cpu events are installed via an smp call and
855
		 * the install is always successful.
T
Thomas Gleixner 已提交
856 857
		 */
		smp_call_function_single(cpu, __perf_install_in_context,
858
					 event, 1);
T
Thomas Gleixner 已提交
859 860 861 862 863
		return;
	}

retry:
	task_oncpu_function_call(task, __perf_install_in_context,
864
				 event);
T
Thomas Gleixner 已提交
865

866
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
867 868 869
	/*
	 * we need to retry the smp call.
	 */
870
	if (ctx->is_active && list_empty(&event->group_entry)) {
871
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
872 873 874 875 876
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
877
	 * can add the event safely, if it the call above did not
T
Thomas Gleixner 已提交
878 879
	 * succeed.
	 */
880 881
	if (list_empty(&event->group_entry))
		add_event_to_ctx(event, ctx);
882
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
883 884
}

885
/*
886
 * Put a event into inactive state and update time fields.
887 888 889 890 891 892
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
893 894
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
895
{
896
	struct perf_event *sub;
897

898 899 900 901
	event->state = PERF_EVENT_STATE_INACTIVE;
	event->tstamp_enabled = ctx->time - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry)
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
902 903 904 905
			sub->tstamp_enabled =
				ctx->time - sub->total_time_enabled;
}

906
/*
907
 * Cross CPU call to enable a performance event
908
 */
909
static void __perf_event_enable(void *info)
910
{
911
	struct perf_event *event = info;
912
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
913 914
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
915
	int err;
916

917
	/*
918 919
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
920
	 */
921
	if (ctx->task && cpuctx->task_ctx != ctx) {
922
		if (cpuctx->task_ctx || ctx->task != current)
923 924 925
			return;
		cpuctx->task_ctx = ctx;
	}
926

927
	raw_spin_lock(&ctx->lock);
928
	ctx->is_active = 1;
929
	update_context_time(ctx);
930

931
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
932
		goto unlock;
933
	__perf_event_mark_enabled(event, ctx);
934

935 936 937
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

938
	/*
939
	 * If the event is in a group and isn't the group leader,
940
	 * then don't put it on unless the group is on.
941
	 */
942
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
943
		goto unlock;
944

945
	if (!group_can_go_on(event, cpuctx, 1)) {
946
		err = -EEXIST;
947
	} else {
948
		perf_disable();
949
		if (event == leader)
950
			err = group_sched_in(event, cpuctx, ctx);
951
		else
952
			err = event_sched_in(event, cpuctx, ctx);
953
		perf_enable();
954
	}
955 956 957

	if (err) {
		/*
958
		 * If this event can't go on and it's part of a
959 960
		 * group, then the whole group has to come off.
		 */
961
		if (leader != event)
962
			group_sched_out(leader, cpuctx, ctx);
963
		if (leader->attr.pinned) {
964
			update_group_times(leader);
965
			leader->state = PERF_EVENT_STATE_ERROR;
966
		}
967 968 969
	}

 unlock:
970
	raw_spin_unlock(&ctx->lock);
971 972 973
}

/*
974
 * Enable a event.
975
 *
976 977
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
978
 * remains valid.  This condition is satisfied when called through
979 980
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
981
 */
982
void perf_event_enable(struct perf_event *event)
983
{
984
	struct perf_event_context *ctx = event->ctx;
985 986 987 988
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
989
		 * Enable the event on the cpu that it's on
990
		 */
991 992
		smp_call_function_single(event->cpu, __perf_event_enable,
					 event, 1);
993 994 995
		return;
	}

996
	raw_spin_lock_irq(&ctx->lock);
997
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
998 999 1000
		goto out;

	/*
1001 1002
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
1003 1004 1005 1006
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
1007 1008
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
1009 1010

 retry:
1011
	raw_spin_unlock_irq(&ctx->lock);
1012
	task_oncpu_function_call(task, __perf_event_enable, event);
1013

1014
	raw_spin_lock_irq(&ctx->lock);
1015 1016

	/*
1017
	 * If the context is active and the event is still off,
1018 1019
	 * we need to retry the cross-call.
	 */
1020
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
1021 1022 1023 1024 1025 1026
		goto retry;

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1027 1028
	if (event->state == PERF_EVENT_STATE_OFF)
		__perf_event_mark_enabled(event, ctx);
1029

1030
 out:
1031
	raw_spin_unlock_irq(&ctx->lock);
1032 1033
}

1034
static int perf_event_refresh(struct perf_event *event, int refresh)
1035
{
1036
	/*
1037
	 * not supported on inherited events
1038
	 */
1039
	if (event->attr.inherit)
1040 1041
		return -EINVAL;

1042 1043
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1044 1045

	return 0;
1046 1047
}

1048 1049 1050 1051 1052 1053 1054 1055 1056
enum event_type_t {
	EVENT_FLEXIBLE = 0x1,
	EVENT_PINNED = 0x2,
	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
};

static void ctx_sched_out(struct perf_event_context *ctx,
			  struct perf_cpu_context *cpuctx,
			  enum event_type_t event_type)
1057
{
1058
	struct perf_event *event;
1059

1060
	raw_spin_lock(&ctx->lock);
1061
	ctx->is_active = 0;
1062
	if (likely(!ctx->nr_events))
1063
		goto out;
1064
	update_context_time(ctx);
1065

1066
	perf_disable();
1067 1068 1069 1070
	if (!ctx->nr_active)
		goto out_enable;

	if (event_type & EVENT_PINNED)
1071 1072 1073
		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
			group_sched_out(event, cpuctx, ctx);

1074
	if (event_type & EVENT_FLEXIBLE)
1075
		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1076
			group_sched_out(event, cpuctx, ctx);
1077 1078

 out_enable:
1079
	perf_enable();
1080
 out:
1081
	raw_spin_unlock(&ctx->lock);
1082 1083
}

1084 1085 1086
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1087 1088 1089 1090
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1091
 * in them directly with an fd; we can only enable/disable all
1092
 * events via prctl, or enable/disable all events in a family
1093 1094
 * via ioctl, which will have the same effect on both contexts.
 */
1095 1096
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1097 1098
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1099
		&& ctx1->parent_gen == ctx2->parent_gen
1100
		&& !ctx1->pin_count && !ctx2->pin_count;
1101 1102
}

1103 1104
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1105 1106 1107
{
	u64 value;

1108
	if (!event->attr.inherit_stat)
1109 1110 1111
		return;

	/*
1112
	 * Update the event value, we cannot use perf_event_read()
1113 1114
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1115
	 * we know the event must be on the current CPU, therefore we
1116 1117
	 * don't need to use it.
	 */
1118 1119
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1120 1121
		event->pmu->read(event);
		/* fall-through */
1122

1123 1124
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1125 1126 1127 1128 1129 1130 1131
		break;

	default:
		break;
	}

	/*
1132
	 * In order to keep per-task stats reliable we need to flip the event
1133 1134
	 * values when we flip the contexts.
	 */
1135 1136 1137
	value = atomic64_read(&next_event->count);
	value = atomic64_xchg(&event->count, value);
	atomic64_set(&next_event->count, value);
1138

1139 1140
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1141

1142
	/*
1143
	 * Since we swizzled the values, update the user visible data too.
1144
	 */
1145 1146
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1147 1148 1149 1150 1151
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1152 1153
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1154
{
1155
	struct perf_event *event, *next_event;
1156 1157 1158 1159

	if (!ctx->nr_stat)
		return;

1160 1161
	update_context_time(ctx);

1162 1163
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1164

1165 1166
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1167

1168 1169
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1170

1171
		__perf_event_sync_stat(event, next_event);
1172

1173 1174
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1175 1176 1177
	}
}

T
Thomas Gleixner 已提交
1178
/*
1179
 * Called from scheduler to remove the events of the current task,
T
Thomas Gleixner 已提交
1180 1181
 * with interrupts disabled.
 *
1182
 * We stop each event and update the event value in event->count.
T
Thomas Gleixner 已提交
1183
 *
I
Ingo Molnar 已提交
1184
 * This does not protect us against NMI, but disable()
1185 1186 1187
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
T
Thomas Gleixner 已提交
1188
 */
1189
void perf_event_task_sched_out(struct task_struct *task,
1190
				 struct task_struct *next)
T
Thomas Gleixner 已提交
1191
{
1192
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1193 1194 1195
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1196
	struct pt_regs *regs;
1197
	int do_switch = 1;
T
Thomas Gleixner 已提交
1198

1199
	regs = task_pt_regs(task);
1200
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1201

1202
	if (likely(!ctx || !cpuctx->task_ctx))
T
Thomas Gleixner 已提交
1203 1204
		return;

1205 1206
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1207
	next_ctx = next->perf_event_ctxp;
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
1219 1220
		raw_spin_lock(&ctx->lock);
		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1221
		if (context_equiv(ctx, next_ctx)) {
1222 1223
			/*
			 * XXX do we need a memory barrier of sorts
1224
			 * wrt to rcu_dereference() of perf_event_ctxp
1225
			 */
1226 1227
			task->perf_event_ctxp = next_ctx;
			next->perf_event_ctxp = ctx;
1228 1229 1230
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1231

1232
			perf_event_sync_stat(ctx, next_ctx);
1233
		}
1234 1235
		raw_spin_unlock(&next_ctx->lock);
		raw_spin_unlock(&ctx->lock);
1236
	}
1237
	rcu_read_unlock();
1238

1239
	if (do_switch) {
1240
		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1241 1242
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1243 1244
}

1245 1246
static void task_ctx_sched_out(struct perf_event_context *ctx,
			       enum event_type_t event_type)
1247 1248 1249
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

1250 1251
	if (!cpuctx->task_ctx)
		return;
1252 1253 1254 1255

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1256
	ctx_sched_out(ctx, cpuctx, event_type);
1257 1258 1259
	cpuctx->task_ctx = NULL;
}

1260 1261 1262
/*
 * Called with IRQs disabled
 */
1263
static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1264
{
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
	task_ctx_sched_out(ctx, EVENT_ALL);
}

/*
 * Called with IRQs disabled
 */
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type)
{
	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1275 1276
}

1277
static void
1278
ctx_pinned_sched_in(struct perf_event_context *ctx,
1279
		    struct perf_cpu_context *cpuctx)
T
Thomas Gleixner 已提交
1280
{
1281
	struct perf_event *event;
T
Thomas Gleixner 已提交
1282

1283 1284
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF)
1285
			continue;
1286
		if (event->cpu != -1 && event->cpu != smp_processor_id())
1287 1288
			continue;

1289
		if (group_can_go_on(event, cpuctx, 1))
1290
			group_sched_in(event, cpuctx, ctx);
1291 1292 1293 1294 1295

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1296 1297 1298
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1299
		}
1300
	}
1301 1302 1303 1304
}

static void
ctx_flexible_sched_in(struct perf_event_context *ctx,
1305
		      struct perf_cpu_context *cpuctx)
1306 1307 1308
{
	struct perf_event *event;
	int can_add_hw = 1;
1309

1310 1311 1312
	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		/* Ignore events in OFF or ERROR state */
		if (event->state <= PERF_EVENT_STATE_OFF)
1313
			continue;
1314 1315
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1316
		 * of events:
1317
		 */
1318
		if (event->cpu != -1 && event->cpu != smp_processor_id())
T
Thomas Gleixner 已提交
1319 1320
			continue;

1321
		if (group_can_go_on(event, cpuctx, can_add_hw))
1322
			if (group_sched_in(event, cpuctx, ctx))
1323
				can_add_hw = 0;
T
Thomas Gleixner 已提交
1324
	}
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345
}

static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
	     enum event_type_t event_type)
{
	raw_spin_lock(&ctx->lock);
	ctx->is_active = 1;
	if (likely(!ctx->nr_events))
		goto out;

	ctx->timestamp = perf_clock();

	perf_disable();

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
	if (event_type & EVENT_PINNED)
1346
		ctx_pinned_sched_in(ctx, cpuctx);
1347 1348 1349

	/* Then walk through the lower prio flexible groups */
	if (event_type & EVENT_FLEXIBLE)
1350
		ctx_flexible_sched_in(ctx, cpuctx);
1351

1352
	perf_enable();
1353
 out:
1354
	raw_spin_unlock(&ctx->lock);
1355 1356
}

1357 1358 1359 1360 1361 1362 1363 1364
static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
			     enum event_type_t event_type)
{
	struct perf_event_context *ctx = &cpuctx->ctx;

	ctx_sched_in(ctx, cpuctx, event_type);
}

1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
static void task_ctx_sched_in(struct task_struct *task,
			      enum event_type_t event_type)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_event_context *ctx = task->perf_event_ctxp;

	if (likely(!ctx))
		return;
	if (cpuctx->task_ctx == ctx)
		return;
	ctx_sched_in(ctx, cpuctx, event_type);
	cpuctx->task_ctx = ctx;
}
1378
/*
1379
 * Called from scheduler to add the events of the current task
1380 1381
 * with interrupts disabled.
 *
1382
 * We restore the event value and then enable it.
1383 1384
 *
 * This does not protect us against NMI, but enable()
1385 1386 1387
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
1388
 */
1389
void perf_event_task_sched_in(struct task_struct *task)
1390
{
1391 1392
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_event_context *ctx = task->perf_event_ctxp;
T
Thomas Gleixner 已提交
1393

1394 1395
	if (likely(!ctx))
		return;
1396

1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
	if (cpuctx->task_ctx == ctx)
		return;

	/*
	 * We want to keep the following priority order:
	 * cpu pinned (that don't need to move), task pinned,
	 * cpu flexible, task flexible.
	 */
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);

	ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
	ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);

	cpuctx->task_ctx = ctx;
1412 1413
}

1414 1415
#define MAX_INTERRUPTS (~0ULL)

1416
static void perf_log_throttle(struct perf_event *event, int enable);
1417

1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
{
	u64 frequency = event->attr.sample_freq;
	u64 sec = NSEC_PER_SEC;
	u64 divisor, dividend;

	int count_fls, nsec_fls, frequency_fls, sec_fls;

	count_fls = fls64(count);
	nsec_fls = fls64(nsec);
	frequency_fls = fls64(frequency);
	sec_fls = 30;

	/*
	 * We got @count in @nsec, with a target of sample_freq HZ
	 * the target period becomes:
	 *
	 *             @count * 10^9
	 * period = -------------------
	 *          @nsec * sample_freq
	 *
	 */

	/*
	 * Reduce accuracy by one bit such that @a and @b converge
	 * to a similar magnitude.
	 */
#define REDUCE_FLS(a, b) 		\
do {					\
	if (a##_fls > b##_fls) {	\
		a >>= 1;		\
		a##_fls--;		\
	} else {			\
		b >>= 1;		\
		b##_fls--;		\
	}				\
} while (0)

	/*
	 * Reduce accuracy until either term fits in a u64, then proceed with
	 * the other, so that finally we can do a u64/u64 division.
	 */
	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
		REDUCE_FLS(nsec, frequency);
		REDUCE_FLS(sec, count);
	}

	if (count_fls + sec_fls > 64) {
		divisor = nsec * frequency;

		while (count_fls + sec_fls > 64) {
			REDUCE_FLS(count, sec);
			divisor >>= 1;
		}

		dividend = count * sec;
	} else {
		dividend = count * sec;

		while (nsec_fls + frequency_fls > 64) {
			REDUCE_FLS(nsec, frequency);
			dividend >>= 1;
		}

		divisor = nsec * frequency;
	}

	return div64_u64(dividend, divisor);
}

1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
static void perf_event_stop(struct perf_event *event)
{
	if (!event->pmu->stop)
		return event->pmu->disable(event);

	return event->pmu->stop(event);
}

static int perf_event_start(struct perf_event *event)
{
	if (!event->pmu->start)
		return event->pmu->enable(event);

	return event->pmu->start(event);
}

1504
static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1505
{
1506
	struct hw_perf_event *hwc = &event->hw;
1507 1508 1509
	u64 period, sample_period;
	s64 delta;

1510
	period = perf_calculate_period(event, nsec, count);
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
1521 1522 1523

	if (atomic64_read(&hwc->period_left) > 8*sample_period) {
		perf_disable();
1524
		perf_event_stop(event);
1525
		atomic64_set(&hwc->period_left, 0);
1526
		perf_event_start(event);
1527 1528
		perf_enable();
	}
1529 1530
}

1531
static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1532
{
1533 1534
	struct perf_event *event;
	struct hw_perf_event *hwc;
1535 1536
	u64 interrupts, now;
	s64 delta;
1537

1538
	raw_spin_lock(&ctx->lock);
1539
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1540
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1541 1542
			continue;

1543 1544 1545
		if (event->cpu != -1 && event->cpu != smp_processor_id())
			continue;

1546
		hwc = &event->hw;
1547 1548 1549

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1550

1551
		/*
1552
		 * unthrottle events on the tick
1553
		 */
1554
		if (interrupts == MAX_INTERRUPTS) {
1555 1556
			perf_log_throttle(event, 1);
			event->pmu->unthrottle(event);
1557 1558
		}

1559
		if (!event->attr.freq || !event->attr.sample_freq)
1560 1561
			continue;

1562 1563 1564 1565
		event->pmu->read(event);
		now = atomic64_read(&event->count);
		delta = now - hwc->freq_count_stamp;
		hwc->freq_count_stamp = now;
1566

1567 1568
		if (delta > 0)
			perf_adjust_period(event, TICK_NSEC, delta);
1569
	}
1570
	raw_spin_unlock(&ctx->lock);
1571 1572
}

1573
/*
1574
 * Round-robin a context's events:
1575
 */
1576
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1577
{
1578
	if (!ctx->nr_events)
T
Thomas Gleixner 已提交
1579 1580
		return;

1581
	raw_spin_lock(&ctx->lock);
1582 1583 1584 1585

	/* Rotate the first entry last of non-pinned groups */
	list_rotate_left(&ctx->flexible_groups);

1586
	raw_spin_unlock(&ctx->lock);
1587 1588
}

1589
void perf_event_task_tick(struct task_struct *curr)
1590
{
1591
	struct perf_cpu_context *cpuctx;
1592
	struct perf_event_context *ctx;
1593

1594
	if (!atomic_read(&nr_events))
1595 1596
		return;

1597
	cpuctx = &__get_cpu_var(perf_cpu_context);
1598
	ctx = curr->perf_event_ctxp;
1599

1600 1601
	perf_disable();

1602
	perf_ctx_adjust_freq(&cpuctx->ctx);
1603
	if (ctx)
1604
		perf_ctx_adjust_freq(ctx);
1605

1606
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1607
	if (ctx)
1608
		task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
T
Thomas Gleixner 已提交
1609

1610
	rotate_ctx(&cpuctx->ctx);
1611 1612
	if (ctx)
		rotate_ctx(ctx);
1613

1614
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1615
	if (ctx)
1616
		task_ctx_sched_in(curr, EVENT_FLEXIBLE);
1617 1618

	perf_enable();
T
Thomas Gleixner 已提交
1619 1620
}

1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

	__perf_event_mark_enabled(event, ctx);

	return 1;
}

1636
/*
1637
 * Enable all of a task's events that have been marked enable-on-exec.
1638 1639
 * This expects task == current.
 */
1640
static void perf_event_enable_on_exec(struct task_struct *task)
1641
{
1642 1643
	struct perf_event_context *ctx;
	struct perf_event *event;
1644 1645
	unsigned long flags;
	int enabled = 0;
1646
	int ret;
1647 1648

	local_irq_save(flags);
1649 1650
	ctx = task->perf_event_ctxp;
	if (!ctx || !ctx->nr_events)
1651 1652
		goto out;

1653
	__perf_event_task_sched_out(ctx);
1654

1655
	raw_spin_lock(&ctx->lock);
1656

1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
	}

	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
1667 1668 1669
	}

	/*
1670
	 * Unclone this context if we enabled any event.
1671
	 */
1672 1673
	if (enabled)
		unclone_ctx(ctx);
1674

1675
	raw_spin_unlock(&ctx->lock);
1676

1677
	perf_event_task_sched_in(task);
1678 1679 1680 1681
 out:
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1682
/*
1683
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1684
 */
1685
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1686
{
1687
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1688 1689
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
I
Ingo Molnar 已提交
1690

1691 1692 1693 1694
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1695 1696
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1697 1698 1699 1700
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

1701
	raw_spin_lock(&ctx->lock);
P
Peter Zijlstra 已提交
1702
	update_context_time(ctx);
1703
	update_event_times(event);
1704
	raw_spin_unlock(&ctx->lock);
P
Peter Zijlstra 已提交
1705

P
Peter Zijlstra 已提交
1706
	event->pmu->read(event);
T
Thomas Gleixner 已提交
1707 1708
}

1709
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1710 1711
{
	/*
1712 1713
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1714
	 */
1715 1716 1717 1718
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
1719 1720 1721
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

1722
		raw_spin_lock_irqsave(&ctx->lock, flags);
P
Peter Zijlstra 已提交
1723
		update_context_time(ctx);
1724
		update_event_times(event);
1725
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1726 1727
	}

1728
	return atomic64_read(&event->count);
T
Thomas Gleixner 已提交
1729 1730
}

1731
/*
1732
 * Initialize the perf_event context in a task_struct:
1733 1734
 */
static void
1735
__perf_event_init_context(struct perf_event_context *ctx,
1736 1737
			    struct task_struct *task)
{
1738
	raw_spin_lock_init(&ctx->lock);
1739
	mutex_init(&ctx->mutex);
1740 1741
	INIT_LIST_HEAD(&ctx->pinned_groups);
	INIT_LIST_HEAD(&ctx->flexible_groups);
1742 1743 1744 1745 1746
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
	ctx->task = task;
}

1747
static struct perf_event_context *find_get_context(pid_t pid, int cpu)
T
Thomas Gleixner 已提交
1748
{
1749
	struct perf_event_context *ctx;
1750
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
1751
	struct task_struct *task;
1752
	unsigned long flags;
1753
	int err;
T
Thomas Gleixner 已提交
1754

1755
	if (pid == -1 && cpu != -1) {
1756
		/* Must be root to operate on a CPU event: */
1757
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
1758 1759
			return ERR_PTR(-EACCES);

1760
		if (cpu < 0 || cpu >= nr_cpumask_bits)
T
Thomas Gleixner 已提交
1761 1762 1763
			return ERR_PTR(-EINVAL);

		/*
1764
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
1765 1766 1767
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
1768
		if (!cpu_online(cpu))
T
Thomas Gleixner 已提交
1769 1770 1771 1772
			return ERR_PTR(-ENODEV);

		cpuctx = &per_cpu(perf_cpu_context, cpu);
		ctx = &cpuctx->ctx;
1773
		get_ctx(ctx);
T
Thomas Gleixner 已提交
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789

		return ctx;
	}

	rcu_read_lock();
	if (!pid)
		task = current;
	else
		task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

1790
	/*
1791
	 * Can't attach events to a dying task.
1792 1793 1794 1795 1796
	 */
	err = -ESRCH;
	if (task->flags & PF_EXITING)
		goto errout;

T
Thomas Gleixner 已提交
1797
	/* Reuse ptrace permission checks for now. */
1798 1799 1800 1801 1802
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

 retry:
1803
	ctx = perf_lock_task_context(task, &flags);
1804
	if (ctx) {
1805
		unclone_ctx(ctx);
1806
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1807 1808
	}

1809
	if (!ctx) {
1810
		ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1811 1812 1813
		err = -ENOMEM;
		if (!ctx)
			goto errout;
1814
		__perf_event_init_context(ctx, task);
1815
		get_ctx(ctx);
1816
		if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1817 1818 1819 1820 1821
			/*
			 * We raced with some other task; use
			 * the context they set.
			 */
			kfree(ctx);
1822
			goto retry;
1823
		}
1824
		get_task_struct(task);
1825 1826
	}

1827
	put_task_struct(task);
T
Thomas Gleixner 已提交
1828
	return ctx;
1829 1830 1831 1832

 errout:
	put_task_struct(task);
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
1833 1834
}

L
Li Zefan 已提交
1835 1836
static void perf_event_free_filter(struct perf_event *event);

1837
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
1838
{
1839
	struct perf_event *event;
P
Peter Zijlstra 已提交
1840

1841 1842 1843
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
1844
	perf_event_free_filter(event);
1845
	kfree(event);
P
Peter Zijlstra 已提交
1846 1847
}

1848
static void perf_pending_sync(struct perf_event *event);
1849

1850
static void free_event(struct perf_event *event)
1851
{
1852
	perf_pending_sync(event);
1853

1854 1855 1856 1857 1858 1859 1860 1861
	if (!event->parent) {
		atomic_dec(&nr_events);
		if (event->attr.mmap)
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
1862
	}
1863

1864 1865 1866
	if (event->output) {
		fput(event->output->filp);
		event->output = NULL;
1867 1868
	}

1869 1870
	if (event->destroy)
		event->destroy(event);
1871

1872 1873
	put_ctx(event->ctx);
	call_rcu(&event->rcu_head, free_event_rcu);
1874 1875
}

1876
int perf_event_release_kernel(struct perf_event *event)
T
Thomas Gleixner 已提交
1877
{
1878
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1879

1880
	WARN_ON_ONCE(ctx->parent_ctx);
1881
	mutex_lock(&ctx->mutex);
1882
	perf_event_remove_from_context(event);
1883
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
1884

1885 1886 1887 1888
	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);
1889

1890
	free_event(event);
T
Thomas Gleixner 已提交
1891 1892 1893

	return 0;
}
1894
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
T
Thomas Gleixner 已提交
1895

1896 1897 1898 1899
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
1900
{
1901
	struct perf_event *event = file->private_data;
1902

1903
	file->private_data = NULL;
1904

1905
	return perf_event_release_kernel(event);
1906 1907
}

1908
static int perf_event_read_size(struct perf_event *event)
1909 1910 1911 1912 1913
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

1914
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1915 1916
		size += sizeof(u64);

1917
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1918 1919
		size += sizeof(u64);

1920
	if (event->attr.read_format & PERF_FORMAT_ID)
1921 1922
		entry += sizeof(u64);

1923 1924
	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
1925 1926 1927 1928 1929 1930 1931 1932
		size += sizeof(u64);
	}

	size += entry * nr;

	return size;
}

1933
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
1934
{
1935
	struct perf_event *child;
1936 1937
	u64 total = 0;

1938 1939 1940
	*enabled = 0;
	*running = 0;

1941
	mutex_lock(&event->child_mutex);
1942
	total += perf_event_read(event);
1943 1944 1945 1946 1947 1948
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
1949
		total += perf_event_read(child);
1950 1951 1952
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
1953
	mutex_unlock(&event->child_mutex);
1954 1955 1956

	return total;
}
1957
EXPORT_SYMBOL_GPL(perf_event_read_value);
1958

1959
static int perf_event_read_group(struct perf_event *event,
1960 1961
				   u64 read_format, char __user *buf)
{
1962
	struct perf_event *leader = event->group_leader, *sub;
1963 1964
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
1965
	u64 values[5];
1966
	u64 count, enabled, running;
1967

1968
	mutex_lock(&ctx->mutex);
1969
	count = perf_event_read_value(leader, &enabled, &running);
1970 1971

	values[n++] = 1 + leader->nr_siblings;
1972 1973 1974 1975
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1976 1977 1978
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
1979 1980 1981 1982

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
1983
		goto unlock;
1984

1985
	ret = size;
1986

1987
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
1988
		n = 0;
1989

1990
		values[n++] = perf_event_read_value(sub, &enabled, &running);
1991 1992 1993 1994 1995
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

1996
		if (copy_to_user(buf + ret, values, size)) {
1997 1998 1999
			ret = -EFAULT;
			goto unlock;
		}
2000 2001

		ret += size;
2002
	}
2003 2004
unlock:
	mutex_unlock(&ctx->mutex);
2005

2006
	return ret;
2007 2008
}

2009
static int perf_event_read_one(struct perf_event *event,
2010 2011
				 u64 read_format, char __user *buf)
{
2012
	u64 enabled, running;
2013 2014 2015
	u64 values[4];
	int n = 0;

2016 2017 2018 2019 2020
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
2021
	if (read_format & PERF_FORMAT_ID)
2022
		values[n++] = primary_event_id(event);
2023 2024 2025 2026 2027 2028 2029

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
2030
/*
2031
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
2032 2033
 */
static ssize_t
2034
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
2035
{
2036
	u64 read_format = event->attr.read_format;
2037
	int ret;
T
Thomas Gleixner 已提交
2038

2039
	/*
2040
	 * Return end-of-file for a read on a event that is in
2041 2042 2043
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
2044
	if (event->state == PERF_EVENT_STATE_ERROR)
2045 2046
		return 0;

2047
	if (count < perf_event_read_size(event))
2048 2049
		return -ENOSPC;

2050
	WARN_ON_ONCE(event->ctx->parent_ctx);
2051
	if (read_format & PERF_FORMAT_GROUP)
2052
		ret = perf_event_read_group(event, read_format, buf);
2053
	else
2054
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
2055

2056
	return ret;
T
Thomas Gleixner 已提交
2057 2058 2059 2060 2061
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
2062
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
2063

2064
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
2065 2066 2067 2068
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
2069
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
2070
	struct perf_mmap_data *data;
2071
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
2072 2073

	rcu_read_lock();
2074
	data = rcu_dereference(event->data);
P
Peter Zijlstra 已提交
2075
	if (data)
2076
		events = atomic_xchg(&data->poll, 0);
P
Peter Zijlstra 已提交
2077
	rcu_read_unlock();
T
Thomas Gleixner 已提交
2078

2079
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
2080 2081 2082 2083

	return events;
}

2084
static void perf_event_reset(struct perf_event *event)
2085
{
2086 2087 2088
	(void)perf_event_read(event);
	atomic64_set(&event->count, 0);
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
2089 2090
}

2091
/*
2092 2093 2094 2095
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
2096
 */
2097 2098
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2099
{
2100
	struct perf_event *child;
P
Peter Zijlstra 已提交
2101

2102 2103 2104 2105
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
2106
		func(child);
2107
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
2108 2109
}

2110 2111
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2112
{
2113 2114
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
2115

2116 2117
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
2118
	event = event->group_leader;
2119

2120 2121 2122 2123
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
2124
	mutex_unlock(&ctx->mutex);
2125 2126
}

2127
static int perf_event_period(struct perf_event *event, u64 __user *arg)
2128
{
2129
	struct perf_event_context *ctx = event->ctx;
2130 2131 2132 2133
	unsigned long size;
	int ret = 0;
	u64 value;

2134
	if (!event->attr.sample_period)
2135 2136 2137 2138 2139 2140 2141 2142 2143
		return -EINVAL;

	size = copy_from_user(&value, arg, sizeof(value));
	if (size != sizeof(value))
		return -EFAULT;

	if (!value)
		return -EINVAL;

2144
	raw_spin_lock_irq(&ctx->lock);
2145 2146
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
2147 2148 2149 2150
			ret = -EINVAL;
			goto unlock;
		}

2151
		event->attr.sample_freq = value;
2152
	} else {
2153 2154
		event->attr.sample_period = value;
		event->hw.sample_period = value;
2155 2156
	}
unlock:
2157
	raw_spin_unlock_irq(&ctx->lock);
2158 2159 2160 2161

	return ret;
}

L
Li Zefan 已提交
2162 2163
static int perf_event_set_output(struct perf_event *event, int output_fd);
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2164

2165 2166
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
2167 2168
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2169
	u32 flags = arg;
2170 2171

	switch (cmd) {
2172 2173
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2174
		break;
2175 2176
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2177
		break;
2178 2179
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2180
		break;
P
Peter Zijlstra 已提交
2181

2182 2183
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2184

2185 2186
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2187

2188 2189
	case PERF_EVENT_IOC_SET_OUTPUT:
		return perf_event_set_output(event, arg);
2190

L
Li Zefan 已提交
2191 2192 2193
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2194
	default:
P
Peter Zijlstra 已提交
2195
		return -ENOTTY;
2196
	}
P
Peter Zijlstra 已提交
2197 2198

	if (flags & PERF_IOC_FLAG_GROUP)
2199
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2200
	else
2201
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2202 2203

	return 0;
2204 2205
}

2206
int perf_event_task_enable(void)
2207
{
2208
	struct perf_event *event;
2209

2210 2211 2212 2213
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2214 2215 2216 2217

	return 0;
}

2218
int perf_event_task_disable(void)
2219
{
2220
	struct perf_event *event;
2221

2222 2223 2224 2225
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2226 2227 2228 2229

	return 0;
}

2230 2231
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2232 2233
#endif

2234
static int perf_event_index(struct perf_event *event)
2235
{
2236
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2237 2238
		return 0;

2239
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2240 2241
}

2242 2243 2244 2245 2246
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2247
void perf_event_update_userpage(struct perf_event *event)
2248
{
2249
	struct perf_event_mmap_page *userpg;
2250
	struct perf_mmap_data *data;
2251 2252

	rcu_read_lock();
2253
	data = rcu_dereference(event->data);
2254 2255 2256 2257
	if (!data)
		goto unlock;

	userpg = data->user_page;
2258

2259 2260 2261 2262 2263
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2264
	++userpg->lock;
2265
	barrier();
2266 2267 2268 2269
	userpg->index = perf_event_index(event);
	userpg->offset = atomic64_read(&event->count);
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		userpg->offset -= atomic64_read(&event->hw.prev_count);
2270

2271 2272
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2273

2274 2275
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2276

2277
	barrier();
2278
	++userpg->lock;
2279
	preempt_enable();
2280
unlock:
2281
	rcu_read_unlock();
2282 2283
}

2284
static unsigned long perf_data_size(struct perf_mmap_data *data)
2285
{
2286 2287
	return data->nr_pages << (PAGE_SHIFT + data->data_order);
}
2288

2289
#ifndef CONFIG_PERF_USE_VMALLOC
2290

2291 2292 2293
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2294

2295 2296 2297 2298 2299
static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > data->nr_pages)
		return NULL;
2300

2301 2302
	if (pgoff == 0)
		return virt_to_page(data->user_page);
2303

2304
	return virt_to_page(data->data_pages[pgoff - 1]);
2305 2306
}

2307 2308
static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2309 2310 2311 2312 2313
{
	struct perf_mmap_data *data;
	unsigned long size;
	int i;

2314
	WARN_ON(atomic_read(&event->mmap_count));
2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332

	size = sizeof(struct perf_mmap_data);
	size += nr_pages * sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
	if (!data->user_page)
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
		data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
		if (!data->data_pages[i])
			goto fail_data_pages;
	}

2333
	data->data_order = 0;
2334 2335
	data->nr_pages = nr_pages;

2336
	return data;
2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347

fail_data_pages:
	for (i--; i >= 0; i--)
		free_page((unsigned long)data->data_pages[i]);

	free_page((unsigned long)data->user_page);

fail_user_page:
	kfree(data);

fail:
2348
	return NULL;
2349 2350
}

2351 2352
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2353
	struct page *page = virt_to_page((void *)addr);
2354 2355 2356 2357 2358

	page->mapping = NULL;
	__free_page(page);
}

2359
static void perf_mmap_data_free(struct perf_mmap_data *data)
2360 2361 2362
{
	int i;

2363
	perf_mmap_free_page((unsigned long)data->user_page);
2364
	for (i = 0; i < data->nr_pages; i++)
2365
		perf_mmap_free_page((unsigned long)data->data_pages[i]);
2366
	kfree(data);
2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406
}

#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > (1UL << data->data_order))
		return NULL;

	return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

static void perf_mmap_data_free_work(struct work_struct *work)
{
	struct perf_mmap_data *data;
	void *base;
	int i, nr;

	data = container_of(work, struct perf_mmap_data, work);
	nr = 1 << data->data_order;

	base = data->user_page;
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
2407
	kfree(data);
2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422
}

static void perf_mmap_data_free(struct perf_mmap_data *data)
{
	schedule_work(&data->work);
}

static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
{
	struct perf_mmap_data *data;
	unsigned long size;
	void *all_buf;

	WARN_ON(atomic_read(&event->mmap_count));
2423

2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500
	size = sizeof(struct perf_mmap_data);
	size += sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	INIT_WORK(&data->work, perf_mmap_data_free_work);

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

	data->user_page = all_buf;
	data->data_pages[0] = all_buf + PAGE_SIZE;
	data->data_order = ilog2(nr_pages);
	data->nr_pages = 1;

	return data;

fail_all_buf:
	kfree(data);

fail:
	return NULL;
}

#endif

static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
	struct perf_mmap_data *data;
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	data = rcu_dereference(event->data);
	if (!data)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(data, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

static void
perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
{
	long max_size = perf_data_size(data);

	atomic_set(&data->lock, -1);

	if (event->attr.watermark) {
		data->watermark = min_t(long, max_size,
					event->attr.wakeup_watermark);
	}

	if (!data->watermark)
2501
		data->watermark = max_size / 2;
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512


	rcu_assign_pointer(event->data, data);
}

static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
{
	struct perf_mmap_data *data;

	data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
	perf_mmap_data_free(data);
2513 2514
}

2515
static void perf_mmap_data_release(struct perf_event *event)
2516
{
2517
	struct perf_mmap_data *data = event->data;
2518

2519
	WARN_ON(atomic_read(&event->mmap_count));
2520

2521
	rcu_assign_pointer(event->data, NULL);
2522
	call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2523 2524 2525 2526
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
2527
	struct perf_event *event = vma->vm_file->private_data;
2528

2529
	atomic_inc(&event->mmap_count);
2530 2531 2532 2533
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
2534
	struct perf_event *event = vma->vm_file->private_data;
2535

2536 2537
	WARN_ON_ONCE(event->ctx->parent_ctx);
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2538
		unsigned long size = perf_data_size(event->data);
2539 2540
		struct user_struct *user = current_user();

2541
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2542
		vma->vm_mm->locked_vm -= event->data->nr_locked;
2543
		perf_mmap_data_release(event);
2544
		mutex_unlock(&event->mmap_mutex);
2545
	}
2546 2547
}

2548
static const struct vm_operations_struct perf_mmap_vmops = {
2549 2550 2551 2552
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
2553 2554 2555 2556
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
2557
	struct perf_event *event = file->private_data;
2558
	unsigned long user_locked, user_lock_limit;
2559
	struct user_struct *user = current_user();
2560
	unsigned long locked, lock_limit;
2561
	struct perf_mmap_data *data;
2562 2563
	unsigned long vma_size;
	unsigned long nr_pages;
2564
	long user_extra, extra;
2565
	int ret = 0;
2566

2567
	if (!(vma->vm_flags & VM_SHARED))
2568
		return -EINVAL;
2569 2570 2571 2572

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

2573 2574 2575 2576 2577
	/*
	 * If we have data pages ensure they're a power-of-two number, so we
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
2578 2579
		return -EINVAL;

2580
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
2581 2582
		return -EINVAL;

2583 2584
	if (vma->vm_pgoff != 0)
		return -EINVAL;
2585

2586 2587 2588
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
	if (event->output) {
2589 2590 2591 2592
		ret = -EINVAL;
		goto unlock;
	}

2593 2594
	if (atomic_inc_not_zero(&event->mmap_count)) {
		if (nr_pages != event->data->nr_pages)
2595 2596 2597 2598
			ret = -EINVAL;
		goto unlock;
	}

2599
	user_extra = nr_pages + 1;
2600
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
2601 2602 2603 2604 2605 2606

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

2607
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2608

2609 2610 2611
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
2612 2613 2614

	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
	lock_limit >>= PAGE_SHIFT;
2615
	locked = vma->vm_mm->locked_vm + extra;
2616

2617 2618
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
2619 2620 2621
		ret = -EPERM;
		goto unlock;
	}
2622

2623
	WARN_ON(event->data);
2624 2625 2626 2627

	data = perf_mmap_data_alloc(event, nr_pages);
	ret = -ENOMEM;
	if (!data)
2628 2629
		goto unlock;

2630 2631 2632
	ret = 0;
	perf_mmap_data_init(event, data);

2633
	atomic_set(&event->mmap_count, 1);
2634
	atomic_long_add(user_extra, &user->locked_vm);
2635
	vma->vm_mm->locked_vm += extra;
2636
	event->data->nr_locked = extra;
2637
	if (vma->vm_flags & VM_WRITE)
2638
		event->data->writable = 1;
2639

2640
unlock:
2641
	mutex_unlock(&event->mmap_mutex);
2642 2643 2644

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
2645 2646

	return ret;
2647 2648
}

P
Peter Zijlstra 已提交
2649 2650 2651
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
2652
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
2653 2654 2655
	int retval;

	mutex_lock(&inode->i_mutex);
2656
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
2657 2658 2659 2660 2661 2662 2663 2664
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
2665 2666 2667 2668
static const struct file_operations perf_fops = {
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
2669 2670
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
2671
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
2672
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
2673 2674
};

2675
/*
2676
 * Perf event wakeup
2677 2678 2679 2680 2681
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

2682
void perf_event_wakeup(struct perf_event *event)
2683
{
2684
	wake_up_all(&event->waitq);
2685

2686 2687 2688
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
2689
	}
2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700
}

/*
 * Pending wakeups
 *
 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
 *
 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
 * single linked list and use cmpxchg() to add entries lockless.
 */

2701
static void perf_pending_event(struct perf_pending_entry *entry)
2702
{
2703 2704
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
2705

2706 2707 2708
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
2709 2710
	}

2711 2712 2713
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
2714 2715 2716
	}
}

2717
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2718

2719
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2720 2721 2722
	PENDING_TAIL,
};

2723 2724
static void perf_pending_queue(struct perf_pending_entry *entry,
			       void (*func)(struct perf_pending_entry *))
2725
{
2726
	struct perf_pending_entry **head;
2727

2728
	if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2729 2730
		return;

2731 2732 2733
	entry->func = func;

	head = &get_cpu_var(perf_pending_head);
2734 2735

	do {
2736 2737
		entry->next = *head;
	} while (cmpxchg(head, entry->next, entry) != entry->next);
2738

2739
	set_perf_event_pending();
2740

2741
	put_cpu_var(perf_pending_head);
2742 2743 2744 2745
}

static int __perf_pending_run(void)
{
2746
	struct perf_pending_entry *list;
2747 2748
	int nr = 0;

2749
	list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2750
	while (list != PENDING_TAIL) {
2751 2752
		void (*func)(struct perf_pending_entry *);
		struct perf_pending_entry *entry = list;
2753 2754 2755

		list = list->next;

2756 2757
		func = entry->func;
		entry->next = NULL;
2758 2759 2760 2761 2762 2763 2764
		/*
		 * Ensure we observe the unqueue before we issue the wakeup,
		 * so that we won't be waiting forever.
		 * -- see perf_not_pending().
		 */
		smp_wmb();

2765
		func(entry);
2766 2767 2768 2769 2770 2771
		nr++;
	}

	return nr;
}

2772
static inline int perf_not_pending(struct perf_event *event)
2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
{
	/*
	 * If we flush on whatever cpu we run, there is a chance we don't
	 * need to wait.
	 */
	get_cpu();
	__perf_pending_run();
	put_cpu();

	/*
	 * Ensure we see the proper queue state before going to sleep
	 * so that we do not miss the wakeup. -- see perf_pending_handle()
	 */
	smp_rmb();
2787
	return event->pending.next == NULL;
2788 2789
}

2790
static void perf_pending_sync(struct perf_event *event)
2791
{
2792
	wait_event(event->waitq, perf_not_pending(event));
2793 2794
}

2795
void perf_event_do_pending(void)
2796 2797 2798 2799
{
	__perf_pending_run();
}

2800 2801 2802 2803
/*
 * Callchain support -- arch specific
 */

2804
__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2805 2806 2807 2808
{
	return NULL;
}

2809 2810 2811
/*
 * Output
 */
2812 2813
static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
			      unsigned long offset, unsigned long head)
2814 2815 2816 2817 2818 2819
{
	unsigned long mask;

	if (!data->writable)
		return true;

2820
	mask = perf_data_size(data) - 1;
2821 2822 2823 2824 2825 2826 2827 2828 2829 2830

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

2831
static void perf_output_wakeup(struct perf_output_handle *handle)
2832
{
2833 2834
	atomic_set(&handle->data->poll, POLL_IN);

2835
	if (handle->nmi) {
2836 2837 2838
		handle->event->pending_wakeup = 1;
		perf_pending_queue(&handle->event->pending,
				   perf_pending_event);
2839
	} else
2840
		perf_event_wakeup(handle->event);
2841 2842
}

2843 2844 2845
/*
 * Curious locking construct.
 *
2846 2847
 * We need to ensure a later event_id doesn't publish a head when a former
 * event_id isn't done writing. However since we need to deal with NMIs we
2848 2849 2850 2851 2852 2853
 * cannot fully serialize things.
 *
 * What we do is serialize between CPUs so we only have to deal with NMI
 * nesting on a single CPU.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
2854
 * event_id completes.
2855 2856 2857 2858
 */
static void perf_output_lock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2859
	int cur, cpu = get_cpu();
2860 2861 2862

	handle->locked = 0;

2863 2864 2865 2866 2867 2868 2869 2870
	for (;;) {
		cur = atomic_cmpxchg(&data->lock, -1, cpu);
		if (cur == -1) {
			handle->locked = 1;
			break;
		}
		if (cur == cpu)
			break;
2871 2872

		cpu_relax();
2873
	}
2874 2875 2876 2877 2878
}

static void perf_output_unlock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2879 2880
	unsigned long head;
	int cpu;
2881

2882
	data->done_head = data->head;
2883 2884 2885 2886 2887 2888 2889 2890 2891 2892

	if (!handle->locked)
		goto out;

again:
	/*
	 * The xchg implies a full barrier that ensures all writes are done
	 * before we publish the new head, matched by a rmb() in userspace when
	 * reading this position.
	 */
2893
	while ((head = atomic_long_xchg(&data->done_head, 0)))
2894 2895 2896
		data->user_page->data_head = head;

	/*
2897
	 * NMI can happen here, which means we can miss a done_head update.
2898 2899
	 */

2900
	cpu = atomic_xchg(&data->lock, -1);
2901 2902 2903 2904 2905
	WARN_ON_ONCE(cpu != smp_processor_id());

	/*
	 * Therefore we have to validate we did not indeed do so.
	 */
2906
	if (unlikely(atomic_long_read(&data->done_head))) {
2907 2908 2909
		/*
		 * Since we had it locked, we can lock it again.
		 */
2910
		while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2911 2912 2913 2914 2915
			cpu_relax();

		goto again;
	}

2916
	if (atomic_xchg(&data->wakeup, 0))
2917 2918
		perf_output_wakeup(handle);
out:
2919
	put_cpu();
2920 2921
}

2922 2923
void perf_output_copy(struct perf_output_handle *handle,
		      const void *buf, unsigned int len)
2924 2925
{
	unsigned int pages_mask;
2926
	unsigned long offset;
2927 2928 2929 2930 2931 2932 2933 2934
	unsigned int size;
	void **pages;

	offset		= handle->offset;
	pages_mask	= handle->data->nr_pages - 1;
	pages		= handle->data->data_pages;

	do {
2935 2936
		unsigned long page_offset;
		unsigned long page_size;
2937 2938 2939
		int nr;

		nr	    = (offset >> PAGE_SHIFT) & pages_mask;
2940 2941 2942
		page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
		page_offset = offset & (page_size - 1);
		size	    = min_t(unsigned int, page_size - page_offset, len);
2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959

		memcpy(pages[nr] + page_offset, buf, size);

		len	    -= size;
		buf	    += size;
		offset	    += size;
	} while (len);

	handle->offset = offset;

	/*
	 * Check we didn't copy past our reservation window, taking the
	 * possible unsigned int wrap into account.
	 */
	WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
}

2960
int perf_output_begin(struct perf_output_handle *handle,
2961
		      struct perf_event *event, unsigned int size,
2962
		      int nmi, int sample)
2963
{
2964
	struct perf_event *output_event;
2965
	struct perf_mmap_data *data;
2966
	unsigned long tail, offset, head;
2967 2968 2969 2970 2971 2972
	int have_lost;
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
2973

2974
	rcu_read_lock();
2975
	/*
2976
	 * For inherited events we send all the output towards the parent.
2977
	 */
2978 2979
	if (event->parent)
		event = event->parent;
2980

2981 2982 2983
	output_event = rcu_dereference(event->output);
	if (output_event)
		event = output_event;
2984

2985
	data = rcu_dereference(event->data);
2986 2987 2988
	if (!data)
		goto out;

2989
	handle->data	= data;
2990
	handle->event	= event;
2991 2992
	handle->nmi	= nmi;
	handle->sample	= sample;
2993

2994
	if (!data->nr_pages)
2995
		goto fail;
2996

2997 2998 2999 3000
	have_lost = atomic_read(&data->lost);
	if (have_lost)
		size += sizeof(lost_event);

3001 3002
	perf_output_lock(handle);

3003
	do {
3004 3005 3006 3007 3008 3009 3010
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
		tail = ACCESS_ONCE(data->user_page->data_tail);
		smp_rmb();
3011
		offset = head = atomic_long_read(&data->head);
P
Peter Zijlstra 已提交
3012
		head += size;
3013
		if (unlikely(!perf_output_space(data, tail, offset, head)))
3014
			goto fail;
3015
	} while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
3016

3017
	handle->offset	= offset;
3018
	handle->head	= head;
3019

3020
	if (head - tail > data->watermark)
3021
		atomic_set(&data->wakeup, 1);
3022

3023
	if (have_lost) {
3024
		lost_event.header.type = PERF_RECORD_LOST;
3025 3026
		lost_event.header.misc = 0;
		lost_event.header.size = sizeof(lost_event);
3027
		lost_event.id          = event->id;
3028 3029 3030 3031 3032
		lost_event.lost        = atomic_xchg(&data->lost, 0);

		perf_output_put(handle, lost_event);
	}

3033
	return 0;
3034

3035
fail:
3036 3037
	atomic_inc(&data->lost);
	perf_output_unlock(handle);
3038 3039
out:
	rcu_read_unlock();
3040

3041 3042
	return -ENOSPC;
}
3043

3044
void perf_output_end(struct perf_output_handle *handle)
3045
{
3046
	struct perf_event *event = handle->event;
3047 3048
	struct perf_mmap_data *data = handle->data;

3049
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
3050

3051
	if (handle->sample && wakeup_events) {
3052
		int events = atomic_inc_return(&data->events);
P
Peter Zijlstra 已提交
3053
		if (events >= wakeup_events) {
3054
			atomic_sub(wakeup_events, &data->events);
3055
			atomic_set(&data->wakeup, 1);
P
Peter Zijlstra 已提交
3056
		}
3057 3058 3059
	}

	perf_output_unlock(handle);
3060
	rcu_read_unlock();
3061 3062
}

3063
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
3064 3065
{
	/*
3066
	 * only top level events have the pid namespace they were created in
3067
	 */
3068 3069
	if (event->parent)
		event = event->parent;
3070

3071
	return task_tgid_nr_ns(p, event->ns);
3072 3073
}

3074
static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
3075 3076
{
	/*
3077
	 * only top level events have the pid namespace they were created in
3078
	 */
3079 3080
	if (event->parent)
		event = event->parent;
3081

3082
	return task_pid_nr_ns(p, event->ns);
3083 3084
}

3085
static void perf_output_read_one(struct perf_output_handle *handle,
3086
				 struct perf_event *event)
3087
{
3088
	u64 read_format = event->attr.read_format;
3089 3090 3091
	u64 values[4];
	int n = 0;

3092
	values[n++] = atomic64_read(&event->count);
3093
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3094 3095
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
3096 3097
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3098 3099
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
3100 3101
	}
	if (read_format & PERF_FORMAT_ID)
3102
		values[n++] = primary_event_id(event);
3103 3104 3105 3106 3107

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
3108
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3109 3110
 */
static void perf_output_read_group(struct perf_output_handle *handle,
3111
			    struct perf_event *event)
3112
{
3113 3114
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = leader->total_time_enabled;

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = leader->total_time_running;

3126
	if (leader != event)
3127 3128 3129 3130
		leader->pmu->read(leader);

	values[n++] = atomic64_read(&leader->count);
	if (read_format & PERF_FORMAT_ID)
3131
		values[n++] = primary_event_id(leader);
3132 3133 3134

	perf_output_copy(handle, values, n * sizeof(u64));

3135
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3136 3137
		n = 0;

3138
		if (sub != event)
3139 3140 3141 3142
			sub->pmu->read(sub);

		values[n++] = atomic64_read(&sub->count);
		if (read_format & PERF_FORMAT_ID)
3143
			values[n++] = primary_event_id(sub);
3144 3145 3146 3147 3148 3149

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

static void perf_output_read(struct perf_output_handle *handle,
3150
			     struct perf_event *event)
3151
{
3152 3153
	if (event->attr.read_format & PERF_FORMAT_GROUP)
		perf_output_read_group(handle, event);
3154
	else
3155
		perf_output_read_one(handle, event);
3156 3157
}

3158 3159 3160
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3161
			struct perf_event *event)
3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3192
		perf_output_read(handle, event);
3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3230
			 struct perf_event *event,
3231
			 struct pt_regs *regs)
3232
{
3233
	u64 sample_type = event->attr.sample_type;
3234

3235
	data->type = sample_type;
3236

3237
	header->type = PERF_RECORD_SAMPLE;
3238 3239 3240 3241
	header->size = sizeof(*header);

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3242

3243
	if (sample_type & PERF_SAMPLE_IP) {
3244 3245 3246
		data->ip = perf_instruction_pointer(regs);

		header->size += sizeof(data->ip);
3247
	}
3248

3249
	if (sample_type & PERF_SAMPLE_TID) {
3250
		/* namespace issues */
3251 3252
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
3253

3254
		header->size += sizeof(data->tid_entry);
3255 3256
	}

3257
	if (sample_type & PERF_SAMPLE_TIME) {
P
Peter Zijlstra 已提交
3258
		data->time = perf_clock();
3259

3260
		header->size += sizeof(data->time);
3261 3262
	}

3263
	if (sample_type & PERF_SAMPLE_ADDR)
3264
		header->size += sizeof(data->addr);
3265

3266
	if (sample_type & PERF_SAMPLE_ID) {
3267
		data->id = primary_event_id(event);
3268

3269 3270 3271 3272
		header->size += sizeof(data->id);
	}

	if (sample_type & PERF_SAMPLE_STREAM_ID) {
3273
		data->stream_id = event->id;
3274 3275 3276

		header->size += sizeof(data->stream_id);
	}
3277

3278
	if (sample_type & PERF_SAMPLE_CPU) {
3279 3280
		data->cpu_entry.cpu		= raw_smp_processor_id();
		data->cpu_entry.reserved	= 0;
3281

3282
		header->size += sizeof(data->cpu_entry);
3283 3284
	}

3285
	if (sample_type & PERF_SAMPLE_PERIOD)
3286
		header->size += sizeof(data->period);
3287

3288
	if (sample_type & PERF_SAMPLE_READ)
3289
		header->size += perf_event_read_size(event);
3290

3291
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3292
		int size = 1;
3293

3294 3295 3296 3297 3298 3299
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3300 3301
	}

3302
	if (sample_type & PERF_SAMPLE_RAW) {
3303 3304 3305 3306 3307 3308 3309 3310
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3311
		header->size += size;
3312
	}
3313
}
3314

3315
static void perf_event_output(struct perf_event *event, int nmi,
3316 3317 3318 3319 3320
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3321

3322
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3323

3324
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3325
		return;
3326

3327
	perf_output_sample(&handle, &header, data, event);
3328

3329
	perf_output_end(&handle);
3330 3331
}

3332
/*
3333
 * read event_id
3334 3335 3336 3337 3338 3339 3340 3341 3342 3343
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3344
perf_event_read_event(struct perf_event *event,
3345 3346 3347
			struct task_struct *task)
{
	struct perf_output_handle handle;
3348
	struct perf_read_event read_event = {
3349
		.header = {
3350
			.type = PERF_RECORD_READ,
3351
			.misc = 0,
3352
			.size = sizeof(read_event) + perf_event_read_size(event),
3353
		},
3354 3355
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3356
	};
3357
	int ret;
3358

3359
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3360 3361 3362
	if (ret)
		return;

3363
	perf_output_put(&handle, read_event);
3364
	perf_output_read(&handle, event);
3365

3366 3367 3368
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3369
/*
P
Peter Zijlstra 已提交
3370 3371 3372
 * task tracking -- fork/exit
 *
 * enabled by: attr.comm | attr.mmap | attr.task
P
Peter Zijlstra 已提交
3373 3374
 */

P
Peter Zijlstra 已提交
3375
struct perf_task_event {
3376
	struct task_struct		*task;
3377
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3378 3379 3380 3381 3382 3383

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3384 3385
		u32				tid;
		u32				ptid;
3386
		u64				time;
3387
	} event_id;
P
Peter Zijlstra 已提交
3388 3389
};

3390
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3391
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3392 3393
{
	struct perf_output_handle handle;
3394
	int size;
P
Peter Zijlstra 已提交
3395
	struct task_struct *task = task_event->task;
3396 3397
	int ret;

3398 3399
	size  = task_event->event_id.header.size;
	ret = perf_output_begin(&handle, event, size, 0, 0);
P
Peter Zijlstra 已提交
3400 3401 3402 3403

	if (ret)
		return;

3404 3405
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3406

3407 3408
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3409

3410
	task_event->event_id.time = perf_clock();
3411

3412
	perf_output_put(&handle, task_event->event_id);
3413

P
Peter Zijlstra 已提交
3414 3415 3416
	perf_output_end(&handle);
}

3417
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3418
{
3419 3420 3421
	if (event->state != PERF_EVENT_STATE_ACTIVE)
		return 0;

3422 3423 3424
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3425
	if (event->attr.comm || event->attr.mmap || event->attr.task)
P
Peter Zijlstra 已提交
3426 3427 3428 3429 3430
		return 1;

	return 0;
}

3431
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
3432
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3433
{
3434
	struct perf_event *event;
P
Peter Zijlstra 已提交
3435

3436 3437 3438
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
3439 3440 3441
	}
}

3442
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3443 3444
{
	struct perf_cpu_context *cpuctx;
3445
	struct perf_event_context *ctx = task_event->task_ctx;
P
Peter Zijlstra 已提交
3446

3447
	rcu_read_lock();
P
Peter Zijlstra 已提交
3448
	cpuctx = &get_cpu_var(perf_cpu_context);
3449
	perf_event_task_ctx(&cpuctx->ctx, task_event);
3450
	if (!ctx)
3451
		ctx = rcu_dereference(task_event->task->perf_event_ctxp);
P
Peter Zijlstra 已提交
3452
	if (ctx)
3453
		perf_event_task_ctx(ctx, task_event);
3454
	put_cpu_var(perf_cpu_context);
P
Peter Zijlstra 已提交
3455 3456 3457
	rcu_read_unlock();
}

3458 3459
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3460
			      int new)
P
Peter Zijlstra 已提交
3461
{
P
Peter Zijlstra 已提交
3462
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
3463

3464 3465 3466
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
3467 3468
		return;

P
Peter Zijlstra 已提交
3469
	task_event = (struct perf_task_event){
3470 3471
		.task	  = task,
		.task_ctx = task_ctx,
3472
		.event_id    = {
P
Peter Zijlstra 已提交
3473
			.header = {
3474
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3475
				.misc = 0,
3476
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
3477
			},
3478 3479
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
3480 3481
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
3482 3483 3484
		},
	};

3485
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
3486 3487
}

3488
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
3489
{
3490
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
3491 3492
}

3493 3494 3495 3496 3497
/*
 * comm tracking
 */

struct perf_comm_event {
3498 3499
	struct task_struct	*task;
	char			*comm;
3500 3501 3502 3503 3504 3505 3506
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
3507
	} event_id;
3508 3509
};

3510
static void perf_event_comm_output(struct perf_event *event,
3511 3512 3513
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
3514 3515
	int size = comm_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3516 3517 3518 3519

	if (ret)
		return;

3520 3521
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3522

3523
	perf_output_put(&handle, comm_event->event_id);
3524 3525 3526 3527 3528
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
	perf_output_end(&handle);
}

3529
static int perf_event_comm_match(struct perf_event *event)
3530
{
3531 3532 3533
	if (event->state != PERF_EVENT_STATE_ACTIVE)
		return 0;

3534 3535 3536
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3537
	if (event->attr.comm)
3538 3539 3540 3541 3542
		return 1;

	return 0;
}

3543
static void perf_event_comm_ctx(struct perf_event_context *ctx,
3544 3545
				  struct perf_comm_event *comm_event)
{
3546
	struct perf_event *event;
3547

3548 3549 3550
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
3551 3552 3553
	}
}

3554
static void perf_event_comm_event(struct perf_comm_event *comm_event)
3555 3556
{
	struct perf_cpu_context *cpuctx;
3557
	struct perf_event_context *ctx;
3558
	unsigned int size;
3559
	char comm[TASK_COMM_LEN];
3560

3561
	memset(comm, 0, sizeof(comm));
3562
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
3563
	size = ALIGN(strlen(comm)+1, sizeof(u64));
3564 3565 3566 3567

	comm_event->comm = comm;
	comm_event->comm_size = size;

3568
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3569

3570
	rcu_read_lock();
3571
	cpuctx = &get_cpu_var(perf_cpu_context);
3572 3573
	perf_event_comm_ctx(&cpuctx->ctx, comm_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3574
	if (ctx)
3575
		perf_event_comm_ctx(ctx, comm_event);
3576
	put_cpu_var(perf_cpu_context);
3577
	rcu_read_unlock();
3578 3579
}

3580
void perf_event_comm(struct task_struct *task)
3581
{
3582 3583
	struct perf_comm_event comm_event;

3584 3585
	if (task->perf_event_ctxp)
		perf_event_enable_on_exec(task);
3586

3587
	if (!atomic_read(&nr_comm_events))
3588
		return;
3589

3590
	comm_event = (struct perf_comm_event){
3591
		.task	= task,
3592 3593
		/* .comm      */
		/* .comm_size */
3594
		.event_id  = {
3595
			.header = {
3596
				.type = PERF_RECORD_COMM,
3597 3598 3599 3600 3601
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3602 3603 3604
		},
	};

3605
	perf_event_comm_event(&comm_event);
3606 3607
}

3608 3609 3610 3611 3612
/*
 * mmap tracking
 */

struct perf_mmap_event {
3613 3614 3615 3616
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
3617 3618 3619 3620 3621 3622 3623 3624 3625

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
3626
	} event_id;
3627 3628
};

3629
static void perf_event_mmap_output(struct perf_event *event,
3630 3631 3632
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
3633 3634
	int size = mmap_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3635 3636 3637 3638

	if (ret)
		return;

3639 3640
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
3641

3642
	perf_output_put(&handle, mmap_event->event_id);
3643 3644
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
3645
	perf_output_end(&handle);
3646 3647
}

3648
static int perf_event_mmap_match(struct perf_event *event,
3649 3650
				   struct perf_mmap_event *mmap_event)
{
3651 3652 3653
	if (event->state != PERF_EVENT_STATE_ACTIVE)
		return 0;

3654 3655 3656
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3657
	if (event->attr.mmap)
3658 3659 3660 3661 3662
		return 1;

	return 0;
}

3663
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3664 3665
				  struct perf_mmap_event *mmap_event)
{
3666
	struct perf_event *event;
3667

3668 3669 3670
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_mmap_match(event, mmap_event))
			perf_event_mmap_output(event, mmap_event);
3671 3672 3673
	}
}

3674
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3675 3676
{
	struct perf_cpu_context *cpuctx;
3677
	struct perf_event_context *ctx;
3678 3679
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
3680 3681 3682
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
3683
	const char *name;
3684

3685 3686
	memset(tmp, 0, sizeof(tmp));

3687
	if (file) {
3688 3689 3690 3691 3692 3693
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3694 3695 3696 3697
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
3698
		name = d_path(&file->f_path, buf, PATH_MAX);
3699 3700 3701 3702 3703
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
3704 3705 3706
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
3707
			goto got_name;
3708
		}
3709 3710 3711 3712 3713 3714

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
		}

3715 3716 3717 3718 3719
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
3720
	size = ALIGN(strlen(name)+1, sizeof(u64));
3721 3722 3723 3724

	mmap_event->file_name = name;
	mmap_event->file_size = size;

3725
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3726

3727
	rcu_read_lock();
3728
	cpuctx = &get_cpu_var(perf_cpu_context);
3729 3730
	perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3731
	if (ctx)
3732
		perf_event_mmap_ctx(ctx, mmap_event);
3733
	put_cpu_var(perf_cpu_context);
3734 3735
	rcu_read_unlock();

3736 3737 3738
	kfree(buf);
}

3739
void __perf_event_mmap(struct vm_area_struct *vma)
3740
{
3741 3742
	struct perf_mmap_event mmap_event;

3743
	if (!atomic_read(&nr_mmap_events))
3744 3745 3746
		return;

	mmap_event = (struct perf_mmap_event){
3747
		.vma	= vma,
3748 3749
		/* .file_name */
		/* .file_size */
3750
		.event_id  = {
3751
			.header = {
3752
				.type = PERF_RECORD_MMAP,
3753 3754 3755 3756 3757
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3758 3759
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
3760
			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
3761 3762 3763
		},
	};

3764
	perf_event_mmap_event(&mmap_event);
3765 3766
}

3767 3768 3769 3770
/*
 * IRQ throttle logging
 */

3771
static void perf_log_throttle(struct perf_event *event, int enable)
3772 3773 3774 3775 3776 3777 3778
{
	struct perf_output_handle handle;
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
3779
		u64				id;
3780
		u64				stream_id;
3781 3782
	} throttle_event = {
		.header = {
3783
			.type = PERF_RECORD_THROTTLE,
3784 3785 3786
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
3787
		.time		= perf_clock(),
3788 3789
		.id		= primary_event_id(event),
		.stream_id	= event->id,
3790 3791
	};

3792
	if (enable)
3793
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3794

3795
	ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3796 3797 3798 3799 3800 3801 3802
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
	perf_output_end(&handle);
}

3803
/*
3804
 * Generic event overflow handling, sampling.
3805 3806
 */

3807
static int __perf_event_overflow(struct perf_event *event, int nmi,
3808 3809
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
3810
{
3811 3812
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
3813 3814
	int ret = 0;

3815
	throttle = (throttle && event->pmu->unthrottle != NULL);
3816

3817
	if (!throttle) {
3818
		hwc->interrupts++;
3819
	} else {
3820 3821
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
3822
			if (HZ * hwc->interrupts >
3823
					(u64)sysctl_perf_event_sample_rate) {
3824
				hwc->interrupts = MAX_INTERRUPTS;
3825
				perf_log_throttle(event, 0);
3826 3827 3828 3829
				ret = 1;
			}
		} else {
			/*
3830
			 * Keep re-disabling events even though on the previous
3831
			 * pass we disabled it - just in case we raced with a
3832
			 * sched-in and the event got enabled again:
3833
			 */
3834 3835 3836
			ret = 1;
		}
	}
3837

3838
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
3839
		u64 now = perf_clock();
3840
		s64 delta = now - hwc->freq_time_stamp;
3841

3842
		hwc->freq_time_stamp = now;
3843

3844 3845
		if (delta > 0 && delta < 2*TICK_NSEC)
			perf_adjust_period(event, delta, hwc->last_period);
3846 3847
	}

3848 3849
	/*
	 * XXX event_limit might not quite work as expected on inherited
3850
	 * events
3851 3852
	 */

3853 3854
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
3855
		ret = 1;
3856
		event->pending_kill = POLL_HUP;
3857
		if (nmi) {
3858 3859 3860
			event->pending_disable = 1;
			perf_pending_queue(&event->pending,
					   perf_pending_event);
3861
		} else
3862
			perf_event_disable(event);
3863 3864
	}

3865 3866 3867 3868 3869
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

3870
	return ret;
3871 3872
}

3873
int perf_event_overflow(struct perf_event *event, int nmi,
3874 3875
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
3876
{
3877
	return __perf_event_overflow(event, nmi, 1, data, regs);
3878 3879
}

3880
/*
3881
 * Generic software event infrastructure
3882 3883
 */

3884
/*
3885 3886
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
3887 3888 3889 3890
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

3891
static u64 perf_swevent_set_period(struct perf_event *event)
3892
{
3893
	struct hw_perf_event *hwc = &event->hw;
3894 3895 3896 3897 3898
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
3899 3900

again:
3901 3902 3903
	old = val = atomic64_read(&hwc->period_left);
	if (val < 0)
		return 0;
3904

3905 3906 3907 3908 3909
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
	if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
		goto again;
3910

3911
	return nr;
3912 3913
}

3914
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3915 3916
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
3917
{
3918
	struct hw_perf_event *hwc = &event->hw;
3919
	int throttle = 0;
3920

3921
	data->period = event->hw.last_period;
3922 3923
	if (!overflow)
		overflow = perf_swevent_set_period(event);
3924

3925 3926
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
3927

3928
	for (; overflow; overflow--) {
3929
		if (__perf_event_overflow(event, nmi, throttle,
3930
					    data, regs)) {
3931 3932 3933 3934 3935 3936
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
3937
		throttle = 1;
3938
	}
3939 3940
}

3941
static void perf_swevent_unthrottle(struct perf_event *event)
3942 3943
{
	/*
3944
	 * Nothing to do, we already reset hwc->interrupts.
3945
	 */
3946
}
3947

3948
static void perf_swevent_add(struct perf_event *event, u64 nr,
3949 3950
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
3951
{
3952
	struct hw_perf_event *hwc = &event->hw;
3953

3954
	atomic64_add(nr, &event->count);
3955

3956 3957 3958
	if (!regs)
		return;

3959 3960
	if (!hwc->sample_period)
		return;
3961

3962 3963 3964 3965
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

	if (atomic64_add_negative(nr, &hwc->period_left))
3966
		return;
3967

3968
	perf_swevent_overflow(event, 0, nmi, data, regs);
3969 3970
}

3971
static int perf_swevent_is_counting(struct perf_event *event)
3972
{
3973
	/*
3974
	 * The event is active, we're good!
3975
	 */
3976
	if (event->state == PERF_EVENT_STATE_ACTIVE)
3977 3978
		return 1;

3979
	/*
3980
	 * The event is off/error, not counting.
3981
	 */
3982
	if (event->state != PERF_EVENT_STATE_INACTIVE)
3983 3984 3985
		return 0;

	/*
3986
	 * The event is inactive, if the context is active
3987 3988
	 * we're part of a group that didn't make it on the 'pmu',
	 * not counting.
3989
	 */
3990
	if (event->ctx->is_active)
3991 3992 3993 3994 3995 3996 3997 3998
		return 0;

	/*
	 * We're inactive and the context is too, this means the
	 * task is scheduled out, we're counting events that happen
	 * to us, like migration events.
	 */
	return 1;
3999 4000
}

L
Li Zefan 已提交
4001 4002 4003
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data);

4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017
static int perf_exclude_event(struct perf_event *event,
			      struct pt_regs *regs)
{
	if (regs) {
		if (event->attr.exclude_user && user_mode(regs))
			return 1;

		if (event->attr.exclude_kernel && !user_mode(regs))
			return 1;
	}

	return 0;
}

4018
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
4019
				enum perf_type_id type,
L
Li Zefan 已提交
4020 4021 4022
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
4023
{
4024 4025 4026
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

4027
	if (!perf_swevent_is_counting(event))
4028 4029
		return 0;

4030
	if (event->attr.type != type)
4031
		return 0;
4032

4033
	if (event->attr.config != event_id)
4034 4035
		return 0;

4036 4037
	if (perf_exclude_event(event, regs))
		return 0;
4038

L
Li Zefan 已提交
4039 4040 4041 4042
	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
	    !perf_tp_event_match(event, data))
		return 0;

4043 4044 4045
	return 1;
}

4046
static void perf_swevent_ctx_event(struct perf_event_context *ctx,
4047
				     enum perf_type_id type,
4048
				     u32 event_id, u64 nr, int nmi,
4049 4050
				     struct perf_sample_data *data,
				     struct pt_regs *regs)
4051
{
4052
	struct perf_event *event;
4053

4054
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
L
Li Zefan 已提交
4055
		if (perf_swevent_match(event, type, event_id, data, regs))
4056
			perf_swevent_add(event, nr, nmi, data, regs);
4057 4058 4059
	}
}

4060
int perf_swevent_get_recursion_context(void)
P
Peter Zijlstra 已提交
4061
{
4062 4063
	struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
	int rctx;
4064

P
Peter Zijlstra 已提交
4065
	if (in_nmi())
4066
		rctx = 3;
4067
	else if (in_irq())
4068
		rctx = 2;
4069
	else if (in_softirq())
4070
		rctx = 1;
4071
	else
4072
		rctx = 0;
P
Peter Zijlstra 已提交
4073

4074 4075
	if (cpuctx->recursion[rctx]) {
		put_cpu_var(perf_cpu_context);
4076
		return -1;
4077
	}
P
Peter Zijlstra 已提交
4078

4079 4080
	cpuctx->recursion[rctx]++;
	barrier();
P
Peter Zijlstra 已提交
4081

4082
	return rctx;
P
Peter Zijlstra 已提交
4083
}
I
Ingo Molnar 已提交
4084
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
4085

4086
void perf_swevent_put_recursion_context(int rctx)
4087
{
4088 4089
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	barrier();
4090
	cpuctx->recursion[rctx]--;
4091
	put_cpu_var(perf_cpu_context);
4092
}
I
Ingo Molnar 已提交
4093
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
P
Peter Zijlstra 已提交
4094

4095 4096 4097 4098
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
				    u64 nr, int nmi,
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
4099
{
4100
	struct perf_cpu_context *cpuctx;
4101
	struct perf_event_context *ctx;
4102

4103
	cpuctx = &__get_cpu_var(perf_cpu_context);
4104
	rcu_read_lock();
4105
	perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
4106
				 nr, nmi, data, regs);
4107 4108 4109 4110
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
4111
	ctx = rcu_dereference(current->perf_event_ctxp);
4112
	if (ctx)
4113
		perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs);
4114
	rcu_read_unlock();
4115
}
4116

4117
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4118
			    struct pt_regs *regs, u64 addr)
4119
{
4120
	struct perf_sample_data data;
4121 4122 4123 4124 4125
	int rctx;

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return;
4126 4127 4128

	data.addr = addr;
	data.raw  = NULL;
4129

4130
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4131 4132

	perf_swevent_put_recursion_context(rctx);
4133 4134
}

4135
static void perf_swevent_read(struct perf_event *event)
4136 4137 4138
{
}

4139
static int perf_swevent_enable(struct perf_event *event)
4140
{
4141
	struct hw_perf_event *hwc = &event->hw;
4142 4143 4144

	if (hwc->sample_period) {
		hwc->last_period = hwc->sample_period;
4145
		perf_swevent_set_period(event);
4146
	}
4147 4148 4149
	return 0;
}

4150
static void perf_swevent_disable(struct perf_event *event)
4151 4152 4153
{
}

4154
static const struct pmu perf_ops_generic = {
4155 4156 4157 4158
	.enable		= perf_swevent_enable,
	.disable	= perf_swevent_disable,
	.read		= perf_swevent_read,
	.unthrottle	= perf_swevent_unthrottle,
4159 4160
};

4161
/*
4162
 * hrtimer based swevent callback
4163 4164
 */

4165
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4166 4167 4168
{
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
4169
	struct pt_regs *regs;
4170
	struct perf_event *event;
4171 4172
	u64 period;

4173 4174
	event	= container_of(hrtimer, struct perf_event, hw.hrtimer);
	event->pmu->read(event);
4175 4176

	data.addr = 0;
4177
	data.raw = NULL;
4178
	data.period = event->hw.last_period;
4179
	regs = get_irq_regs();
4180 4181 4182 4183
	/*
	 * In case we exclude kernel IPs or are somehow not in interrupt
	 * context, provide the next best thing, the user IP.
	 */
4184 4185
	if ((event->attr.exclude_kernel || !regs) &&
			!event->attr.exclude_user)
4186
		regs = task_pt_regs(current);
4187

4188
	if (regs) {
4189 4190 4191
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
4192 4193
	}

4194
	period = max_t(u64, 10000, event->hw.sample_period);
4195 4196 4197 4198 4199
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));

	return ret;
}

4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235
static void perf_swevent_start_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
	if (hwc->sample_period) {
		u64 period;

		if (hwc->remaining) {
			if (hwc->remaining < 0)
				period = 10000;
			else
				period = hwc->remaining;
			hwc->remaining = 0;
		} else {
			period = max_t(u64, 10000, hwc->sample_period);
		}
		__hrtimer_start_range_ns(&hwc->hrtimer,
				ns_to_ktime(period), 0,
				HRTIMER_MODE_REL, 0);
	}
}

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (hwc->sample_period) {
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
		hwc->remaining = ktime_to_ns(remaining);

		hrtimer_cancel(&hwc->hrtimer);
	}
}

4236
/*
4237
 * Software event: cpu wall time clock
4238 4239
 */

4240
static void cpu_clock_perf_event_update(struct perf_event *event)
4241 4242 4243 4244 4245 4246
{
	int cpu = raw_smp_processor_id();
	s64 prev;
	u64 now;

	now = cpu_clock(cpu);
4247
	prev = atomic64_xchg(&event->hw.prev_count, now);
4248
	atomic64_add(now - prev, &event->count);
4249 4250
}

4251
static int cpu_clock_perf_event_enable(struct perf_event *event)
4252
{
4253
	struct hw_perf_event *hwc = &event->hw;
4254 4255 4256
	int cpu = raw_smp_processor_id();

	atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4257
	perf_swevent_start_hrtimer(event);
4258 4259 4260 4261

	return 0;
}

4262
static void cpu_clock_perf_event_disable(struct perf_event *event)
4263
{
4264
	perf_swevent_cancel_hrtimer(event);
4265
	cpu_clock_perf_event_update(event);
4266 4267
}

4268
static void cpu_clock_perf_event_read(struct perf_event *event)
4269
{
4270
	cpu_clock_perf_event_update(event);
4271 4272
}

4273
static const struct pmu perf_ops_cpu_clock = {
4274 4275 4276
	.enable		= cpu_clock_perf_event_enable,
	.disable	= cpu_clock_perf_event_disable,
	.read		= cpu_clock_perf_event_read,
4277 4278
};

4279
/*
4280
 * Software event: task time clock
4281 4282
 */

4283
static void task_clock_perf_event_update(struct perf_event *event, u64 now)
I
Ingo Molnar 已提交
4284
{
4285
	u64 prev;
I
Ingo Molnar 已提交
4286 4287
	s64 delta;

4288
	prev = atomic64_xchg(&event->hw.prev_count, now);
I
Ingo Molnar 已提交
4289
	delta = now - prev;
4290
	atomic64_add(delta, &event->count);
4291 4292
}

4293
static int task_clock_perf_event_enable(struct perf_event *event)
I
Ingo Molnar 已提交
4294
{
4295
	struct hw_perf_event *hwc = &event->hw;
4296 4297
	u64 now;

4298
	now = event->ctx->time;
4299

4300
	atomic64_set(&hwc->prev_count, now);
4301 4302

	perf_swevent_start_hrtimer(event);
4303 4304

	return 0;
I
Ingo Molnar 已提交
4305 4306
}

4307
static void task_clock_perf_event_disable(struct perf_event *event)
4308
{
4309
	perf_swevent_cancel_hrtimer(event);
4310
	task_clock_perf_event_update(event, event->ctx->time);
4311

4312
}
I
Ingo Molnar 已提交
4313

4314
static void task_clock_perf_event_read(struct perf_event *event)
4315
{
4316 4317 4318
	u64 time;

	if (!in_nmi()) {
4319 4320
		update_context_time(event->ctx);
		time = event->ctx->time;
4321 4322
	} else {
		u64 now = perf_clock();
4323 4324
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
4325 4326
	}

4327
	task_clock_perf_event_update(event, time);
4328 4329
}

4330
static const struct pmu perf_ops_task_clock = {
4331 4332 4333
	.enable		= task_clock_perf_event_enable,
	.disable	= task_clock_perf_event_disable,
	.read		= task_clock_perf_event_read,
4334 4335
};

4336
#ifdef CONFIG_EVENT_TRACING
L
Li Zefan 已提交
4337

4338
void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4339
			  int entry_size)
4340
{
4341
	struct perf_raw_record raw = {
4342
		.size = entry_size,
4343
		.data = record,
4344 4345
	};

4346
	struct perf_sample_data data = {
4347
		.addr = addr,
4348
		.raw = &raw,
4349
	};
4350

4351 4352 4353 4354
	struct pt_regs *regs = get_irq_regs();

	if (!regs)
		regs = task_pt_regs(current);
4355

4356
	/* Trace events already protected against recursion */
4357
	do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4358
				&data, regs);
4359
}
4360
EXPORT_SYMBOL_GPL(perf_tp_event);
4361

L
Li Zefan 已提交
4362 4363 4364 4365 4366 4367 4368 4369 4370
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}
4371

4372
static void tp_perf_event_destroy(struct perf_event *event)
4373
{
4374
	ftrace_profile_disable(event->attr.config);
4375 4376
}

4377
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4378
{
4379 4380 4381 4382
	/*
	 * Raw tracepoint data is a severe data leak, only allow root to
	 * have these.
	 */
4383
	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4384
			perf_paranoid_tracepoint_raw() &&
4385 4386 4387
			!capable(CAP_SYS_ADMIN))
		return ERR_PTR(-EPERM);

4388
	if (ftrace_profile_enable(event->attr.config))
4389 4390
		return NULL;

4391
	event->destroy = tp_perf_event_destroy;
4392 4393 4394

	return &perf_ops_generic;
}
L
Li Zefan 已提交
4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

4419
#else
L
Li Zefan 已提交
4420 4421 4422 4423 4424 4425 4426

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	return 1;
}

4427
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4428 4429 4430
{
	return NULL;
}
L
Li Zefan 已提交
4431 4432 4433 4434 4435 4436 4437 4438 4439 4440

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

4441
#endif /* CONFIG_EVENT_TRACING */
4442

4443 4444 4445 4446 4447 4448 4449 4450 4451
#ifdef CONFIG_HAVE_HW_BREAKPOINT
static void bp_perf_event_destroy(struct perf_event *event)
{
	release_bp_slot(event);
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	int err;
4452 4453

	err = register_perf_hw_breakpoint(bp);
4454 4455 4456 4457 4458 4459 4460 4461
	if (err)
		return ERR_PTR(err);

	bp->destroy = bp_perf_event_destroy;

	return &perf_ops_bp;
}

4462
void perf_bp_event(struct perf_event *bp, void *data)
4463
{
4464 4465 4466
	struct perf_sample_data sample;
	struct pt_regs *regs = data;

4467
	sample.raw = NULL;
4468 4469 4470 4471
	sample.addr = bp->attr.bp_addr;

	if (!perf_exclude_event(bp, regs))
		perf_swevent_add(bp, 1, 1, &sample, regs);
4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483
}
#else
static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	return NULL;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
}
#endif

4484
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4485

4486
static void sw_perf_event_destroy(struct perf_event *event)
4487
{
4488
	u64 event_id = event->attr.config;
4489

4490
	WARN_ON(event->parent);
4491

4492
	atomic_dec(&perf_swevent_enabled[event_id]);
4493 4494
}

4495
static const struct pmu *sw_perf_event_init(struct perf_event *event)
4496
{
4497
	const struct pmu *pmu = NULL;
4498
	u64 event_id = event->attr.config;
4499

4500
	/*
4501
	 * Software events (currently) can't in general distinguish
4502 4503 4504 4505 4506
	 * between user, kernel and hypervisor events.
	 * However, context switches and cpu migrations are considered
	 * to be kernel events, and page faults are never hypervisor
	 * events.
	 */
4507
	switch (event_id) {
4508
	case PERF_COUNT_SW_CPU_CLOCK:
4509
		pmu = &perf_ops_cpu_clock;
4510

4511
		break;
4512
	case PERF_COUNT_SW_TASK_CLOCK:
4513
		/*
4514 4515
		 * If the user instantiates this as a per-cpu event,
		 * use the cpu_clock event instead.
4516
		 */
4517
		if (event->ctx->task)
4518
			pmu = &perf_ops_task_clock;
4519
		else
4520
			pmu = &perf_ops_cpu_clock;
4521

4522
		break;
4523 4524 4525 4526 4527
	case PERF_COUNT_SW_PAGE_FAULTS:
	case PERF_COUNT_SW_PAGE_FAULTS_MIN:
	case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
	case PERF_COUNT_SW_CONTEXT_SWITCHES:
	case PERF_COUNT_SW_CPU_MIGRATIONS:
4528 4529
	case PERF_COUNT_SW_ALIGNMENT_FAULTS:
	case PERF_COUNT_SW_EMULATION_FAULTS:
4530 4531 4532
		if (!event->parent) {
			atomic_inc(&perf_swevent_enabled[event_id]);
			event->destroy = sw_perf_event_destroy;
4533
		}
4534
		pmu = &perf_ops_generic;
4535
		break;
4536
	}
4537

4538
	return pmu;
4539 4540
}

T
Thomas Gleixner 已提交
4541
/*
4542
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
4543
 */
4544 4545
static struct perf_event *
perf_event_alloc(struct perf_event_attr *attr,
4546
		   int cpu,
4547 4548 4549
		   struct perf_event_context *ctx,
		   struct perf_event *group_leader,
		   struct perf_event *parent_event,
4550
		   perf_overflow_handler_t overflow_handler,
4551
		   gfp_t gfpflags)
T
Thomas Gleixner 已提交
4552
{
4553
	const struct pmu *pmu;
4554 4555
	struct perf_event *event;
	struct hw_perf_event *hwc;
4556
	long err;
T
Thomas Gleixner 已提交
4557

4558 4559
	event = kzalloc(sizeof(*event), gfpflags);
	if (!event)
4560
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
4561

4562
	/*
4563
	 * Single events are their own group leaders, with an
4564 4565 4566
	 * empty sibling list:
	 */
	if (!group_leader)
4567
		group_leader = event;
4568

4569 4570
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
4571

4572 4573 4574 4575
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
T
Thomas Gleixner 已提交
4576

4577
	mutex_init(&event->mmap_mutex);
4578

4579 4580 4581 4582 4583 4584
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->ctx		= ctx;
	event->oncpu		= -1;
4585

4586
	event->parent		= parent_event;
4587

4588 4589
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
4590

4591
	event->state		= PERF_EVENT_STATE_INACTIVE;
4592

4593 4594
	if (!overflow_handler && parent_event)
		overflow_handler = parent_event->overflow_handler;
4595
	
4596
	event->overflow_handler	= overflow_handler;
4597

4598
	if (attr->disabled)
4599
		event->state = PERF_EVENT_STATE_OFF;
4600

4601
	pmu = NULL;
4602

4603
	hwc = &event->hw;
4604
	hwc->sample_period = attr->sample_period;
4605
	if (attr->freq && attr->sample_freq)
4606
		hwc->sample_period = 1;
4607
	hwc->last_period = hwc->sample_period;
4608 4609

	atomic64_set(&hwc->period_left, hwc->sample_period);
4610

4611
	/*
4612
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
4613
	 */
4614
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4615 4616
		goto done;

4617
	switch (attr->type) {
4618
	case PERF_TYPE_RAW:
4619
	case PERF_TYPE_HARDWARE:
4620
	case PERF_TYPE_HW_CACHE:
4621
		pmu = hw_perf_event_init(event);
4622 4623 4624
		break;

	case PERF_TYPE_SOFTWARE:
4625
		pmu = sw_perf_event_init(event);
4626 4627 4628
		break;

	case PERF_TYPE_TRACEPOINT:
4629
		pmu = tp_perf_event_init(event);
4630
		break;
4631

4632 4633 4634 4635 4636
	case PERF_TYPE_BREAKPOINT:
		pmu = bp_perf_event_init(event);
		break;


4637 4638
	default:
		break;
4639
	}
4640 4641
done:
	err = 0;
4642
	if (!pmu)
4643
		err = -EINVAL;
4644 4645
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
4646

4647
	if (err) {
4648 4649 4650
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
4651
		return ERR_PTR(err);
I
Ingo Molnar 已提交
4652
	}
4653

4654
	event->pmu = pmu;
T
Thomas Gleixner 已提交
4655

4656 4657 4658 4659 4660 4661 4662 4663
	if (!event->parent) {
		atomic_inc(&nr_events);
		if (event->attr.mmap)
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
4664
	}
4665

4666
	return event;
T
Thomas Gleixner 已提交
4667 4668
}

4669 4670
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
4671 4672
{
	u32 size;
4673
	int ret;
4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
4698 4699 4700
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
4701 4702
	 */
	if (size > sizeof(*attr)) {
4703 4704 4705
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
4706

4707 4708
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
4709

4710
		for (; addr < end; addr++) {
4711 4712 4713 4714 4715 4716
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
4717
		size = sizeof(*attr);
4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

4731
	if (attr->__reserved_1 || attr->__reserved_2)
4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Li Zefan 已提交
4749
static int perf_event_set_output(struct perf_event *event, int output_fd)
4750
{
4751
	struct perf_event *output_event = NULL;
4752
	struct file *output_file = NULL;
4753
	struct perf_event *old_output;
4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766
	int fput_needed = 0;
	int ret = -EINVAL;

	if (!output_fd)
		goto set;

	output_file = fget_light(output_fd, &fput_needed);
	if (!output_file)
		return -EBADF;

	if (output_file->f_op != &perf_fops)
		goto out;

4767
	output_event = output_file->private_data;
4768 4769

	/* Don't chain output fds */
4770
	if (output_event->output)
4771 4772 4773
		goto out;

	/* Don't set an output fd when we already have an output channel */
4774
	if (event->data)
4775 4776 4777 4778 4779
		goto out;

	atomic_long_inc(&output_file->f_count);

set:
4780 4781 4782 4783
	mutex_lock(&event->mmap_mutex);
	old_output = event->output;
	rcu_assign_pointer(event->output, output_event);
	mutex_unlock(&event->mmap_mutex);
4784 4785 4786 4787

	if (old_output) {
		/*
		 * we need to make sure no existing perf_output_*()
4788
		 * is still referencing this event.
4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799
		 */
		synchronize_rcu();
		fput(old_output->filp);
	}

	ret = 0;
out:
	fput_light(output_file, fput_needed);
	return ret;
}

T
Thomas Gleixner 已提交
4800
/**
4801
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
4802
 *
4803
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
4804
 * @pid:		target pid
I
Ingo Molnar 已提交
4805
 * @cpu:		target cpu
4806
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
4807
 */
4808 4809
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
4810
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
4811
{
4812 4813 4814 4815
	struct perf_event *event, *group_leader;
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
4816 4817
	struct file *group_file = NULL;
	int fput_needed = 0;
4818
	int fput_needed2 = 0;
4819
	int err;
T
Thomas Gleixner 已提交
4820

4821
	/* for future expandability... */
4822
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4823 4824
		return -EINVAL;

4825 4826 4827
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
4828

4829 4830 4831 4832 4833
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

4834
	if (attr.freq) {
4835
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
4836 4837 4838
			return -EINVAL;
	}

4839
	/*
I
Ingo Molnar 已提交
4840 4841 4842 4843 4844 4845 4846
	 * Get the target context (task or percpu):
	 */
	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

	/*
4847
	 * Look up the group leader (we will attach this event to it):
4848 4849
	 */
	group_leader = NULL;
4850
	if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4851
		err = -EINVAL;
4852 4853
		group_file = fget_light(group_fd, &fput_needed);
		if (!group_file)
I
Ingo Molnar 已提交
4854
			goto err_put_context;
4855
		if (group_file->f_op != &perf_fops)
I
Ingo Molnar 已提交
4856
			goto err_put_context;
4857 4858 4859

		group_leader = group_file->private_data;
		/*
I
Ingo Molnar 已提交
4860 4861 4862 4863 4864 4865 4866 4867
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
			goto err_put_context;
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
4868
		 */
I
Ingo Molnar 已提交
4869 4870
		if (group_leader->ctx != ctx)
			goto err_put_context;
4871 4872 4873
		/*
		 * Only a group leader can be exclusive or pinned
		 */
4874
		if (attr.exclusive || attr.pinned)
4875
			goto err_put_context;
4876 4877
	}

4878
	event = perf_event_alloc(&attr, cpu, ctx, group_leader,
4879
				     NULL, NULL, GFP_KERNEL);
4880 4881
	err = PTR_ERR(event);
	if (IS_ERR(event))
T
Thomas Gleixner 已提交
4882 4883
		goto err_put_context;

4884
	err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
4885
	if (err < 0)
4886 4887
		goto err_free_put_context;

4888 4889
	event_file = fget_light(err, &fput_needed2);
	if (!event_file)
4890 4891
		goto err_free_put_context;

4892
	if (flags & PERF_FLAG_FD_OUTPUT) {
4893
		err = perf_event_set_output(event, group_fd);
4894 4895
		if (err)
			goto err_fput_free_put_context;
4896 4897
	}

4898
	event->filp = event_file;
4899
	WARN_ON_ONCE(ctx->parent_ctx);
4900
	mutex_lock(&ctx->mutex);
4901
	perf_install_in_context(ctx, event, cpu);
4902
	++ctx->generation;
4903
	mutex_unlock(&ctx->mutex);
4904

4905
	event->owner = current;
4906
	get_task_struct(current);
4907 4908 4909
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
4910

4911
err_fput_free_put_context:
4912
	fput_light(event_file, fput_needed2);
T
Thomas Gleixner 已提交
4913

4914
err_free_put_context:
4915
	if (err < 0)
4916
		kfree(event);
T
Thomas Gleixner 已提交
4917 4918

err_put_context:
4919 4920 4921 4922
	if (err < 0)
		put_ctx(ctx);

	fput_light(group_file, fput_needed);
T
Thomas Gleixner 已提交
4923

4924
	return err;
T
Thomas Gleixner 已提交
4925 4926
}

4927 4928 4929 4930 4931 4932 4933 4934 4935
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
 * @pid: task to profile
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4936 4937
				 pid_t pid,
				 perf_overflow_handler_t overflow_handler)
4938 4939 4940 4941 4942 4943 4944 4945 4946 4947
{
	struct perf_event *event;
	struct perf_event_context *ctx;
	int err;

	/*
	 * Get the target context (task or percpu):
	 */

	ctx = find_get_context(pid, cpu);
4948 4949 4950 4951
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
		goto err_exit;
	}
4952 4953

	event = perf_event_alloc(attr, cpu, ctx, NULL,
4954
				 NULL, overflow_handler, GFP_KERNEL);
4955 4956
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
4957
		goto err_put_context;
4958
	}
4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
	mutex_unlock(&ctx->mutex);

	event->owner = current;
	get_task_struct(current);
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);

	return event;

4975 4976 4977 4978
 err_put_context:
	put_ctx(ctx);
 err_exit:
	return ERR_PTR(err);
4979 4980 4981
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);

4982
/*
4983
 * inherit a event from parent task to child task:
4984
 */
4985 4986
static struct perf_event *
inherit_event(struct perf_event *parent_event,
4987
	      struct task_struct *parent,
4988
	      struct perf_event_context *parent_ctx,
4989
	      struct task_struct *child,
4990 4991
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
4992
{
4993
	struct perf_event *child_event;
4994

4995
	/*
4996 4997
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
4998 4999 5000
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
5001 5002
	if (parent_event->parent)
		parent_event = parent_event->parent;
5003

5004 5005 5006
	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu, child_ctx,
					   group_leader, parent_event,
5007
					   NULL, GFP_KERNEL);
5008 5009
	if (IS_ERR(child_event))
		return child_event;
5010
	get_ctx(child_ctx);
5011

5012
	/*
5013
	 * Make the child state follow the state of the parent event,
5014
	 * not its attr.disabled bit.  We hold the parent's mutex,
5015
	 * so we won't race with perf_event_{en, dis}able_family.
5016
	 */
5017 5018
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
5019
	else
5020
		child_event->state = PERF_EVENT_STATE_OFF;
5021

5022 5023 5024 5025 5026 5027 5028 5029 5030
	if (parent_event->attr.freq) {
		u64 sample_period = parent_event->hw.sample_period;
		struct hw_perf_event *hwc = &child_event->hw;

		hwc->sample_period = sample_period;
		hwc->last_period   = sample_period;

		atomic64_set(&hwc->period_left, sample_period);
	}
5031

5032 5033
	child_event->overflow_handler = parent_event->overflow_handler;

5034 5035 5036
	/*
	 * Link it up in the child's context:
	 */
5037
	add_event_to_ctx(child_event, child_ctx);
5038 5039 5040

	/*
	 * Get a reference to the parent filp - we will fput it
5041
	 * when the child event exits. This is safe to do because
5042 5043 5044
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
5045
	atomic_long_inc(&parent_event->filp->f_count);
5046

5047
	/*
5048
	 * Link this into the parent event's child list
5049
	 */
5050 5051 5052 5053
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
5054

5055
	return child_event;
5056 5057
}

5058
static int inherit_group(struct perf_event *parent_event,
5059
	      struct task_struct *parent,
5060
	      struct perf_event_context *parent_ctx,
5061
	      struct task_struct *child,
5062
	      struct perf_event_context *child_ctx)
5063
{
5064 5065 5066
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;
5067

5068
	leader = inherit_event(parent_event, parent, parent_ctx,
5069
				 child, NULL, child_ctx);
5070 5071
	if (IS_ERR(leader))
		return PTR_ERR(leader);
5072 5073
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
5074 5075 5076
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
5077
	}
5078 5079 5080
	return 0;
}

5081
static void sync_child_event(struct perf_event *child_event,
5082
			       struct task_struct *child)
5083
{
5084
	struct perf_event *parent_event = child_event->parent;
5085
	u64 child_val;
5086

5087 5088
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
5089

5090
	child_val = atomic64_read(&child_event->count);
5091 5092 5093 5094

	/*
	 * Add back the child's count to the parent's count:
	 */
5095 5096 5097 5098 5099
	atomic64_add(child_val, &parent_event->count);
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
5100 5101

	/*
5102
	 * Remove this event from the parent's list
5103
	 */
5104 5105 5106 5107
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
5108 5109

	/*
5110
	 * Release the parent event, if this was the last
5111 5112
	 * reference to it.
	 */
5113
	fput(parent_event->filp);
5114 5115
}

5116
static void
5117 5118
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
5119
			 struct task_struct *child)
5120
{
5121
	struct perf_event *parent_event;
5122

5123
	perf_event_remove_from_context(child_event);
5124

5125
	parent_event = child_event->parent;
5126
	/*
5127
	 * It can happen that parent exits first, and has events
5128
	 * that are still around due to the child reference. These
5129
	 * events need to be zapped - but otherwise linger.
5130
	 */
5131 5132 5133
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
5134
	}
5135 5136 5137
}

/*
5138
 * When a child task exits, feed back event values to parent events.
5139
 */
5140
void perf_event_exit_task(struct task_struct *child)
5141
{
5142 5143
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
5144
	unsigned long flags;
5145

5146 5147
	if (likely(!child->perf_event_ctxp)) {
		perf_event_task(child, NULL, 0);
5148
		return;
P
Peter Zijlstra 已提交
5149
	}
5150

5151
	local_irq_save(flags);
5152 5153 5154 5155 5156 5157
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
5158 5159
	child_ctx = child->perf_event_ctxp;
	__perf_event_task_sched_out(child_ctx);
5160 5161 5162

	/*
	 * Take the context lock here so that if find_get_context is
5163
	 * reading child->perf_event_ctxp, we wait until it has
5164 5165
	 * incremented the context's refcount before we do put_ctx below.
	 */
5166
	raw_spin_lock(&child_ctx->lock);
5167
	child->perf_event_ctxp = NULL;
5168 5169 5170
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
5171
	 * the events from it.
5172 5173
	 */
	unclone_ctx(child_ctx);
5174
	update_context_time(child_ctx);
5175
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
5176 5177

	/*
5178 5179 5180
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
5181
	 */
5182
	perf_event_task(child, child_ctx, 0);
5183

5184 5185 5186
	/*
	 * We can recurse on the same lock type through:
	 *
5187 5188 5189
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
5190 5191 5192 5193 5194 5195
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
	mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
5196

5197
again:
5198 5199 5200 5201 5202
	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
				 group_entry)
		__perf_event_exit_task(child_event, child_ctx, child);

	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
5203
				 group_entry)
5204
		__perf_event_exit_task(child_event, child_ctx, child);
5205 5206

	/*
5207
	 * If the last event was a group event, it will have appended all
5208 5209 5210
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
5211 5212
	if (!list_empty(&child_ctx->pinned_groups) ||
	    !list_empty(&child_ctx->flexible_groups))
5213
		goto again;
5214 5215 5216 5217

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
5218 5219
}

5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237
static void perf_free_event(struct perf_event *event,
			    struct perf_event_context *ctx)
{
	struct perf_event *parent = event->parent;

	if (WARN_ON_ONCE(!parent))
		return;

	mutex_lock(&parent->child_mutex);
	list_del_init(&event->child_list);
	mutex_unlock(&parent->child_mutex);

	fput(parent->filp);

	list_del_event(event, ctx);
	free_event(event);
}

5238 5239 5240 5241
/*
 * free an unexposed, unused context as created by inheritance by
 * init_task below, used by fork() in case of fail.
 */
5242
void perf_event_free_task(struct task_struct *task)
5243
{
5244 5245
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event *event, *tmp;
5246 5247 5248 5249 5250 5251

	if (!ctx)
		return;

	mutex_lock(&ctx->mutex);
again:
5252 5253
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		perf_free_event(event, ctx);
5254

5255 5256 5257
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
				 group_entry)
		perf_free_event(event, ctx);
5258

5259 5260 5261
	if (!list_empty(&ctx->pinned_groups) ||
	    !list_empty(&ctx->flexible_groups))
		goto again;
5262

5263
	mutex_unlock(&ctx->mutex);
5264

5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279
	put_ctx(ctx);
}

static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
		   struct perf_event_context *parent_ctx,
		   struct task_struct *child,
		   int *inherited_all)
{
	int ret;
	struct perf_event_context *child_ctx = child->perf_event_ctxp;

	if (!event->attr.inherit) {
		*inherited_all = 0;
		return 0;
5280 5281
	}

5282 5283 5284 5285 5286 5287 5288
	if (!child_ctx) {
		/*
		 * This is executed from the parent task context, so
		 * inherit events that have been marked for cloning.
		 * First allocate and initialize a context for the
		 * child.
		 */
5289

5290 5291 5292 5293
		child_ctx = kzalloc(sizeof(struct perf_event_context),
				    GFP_KERNEL);
		if (!child_ctx)
			return -ENOMEM;
5294

5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306
		__perf_event_init_context(child_ctx, child);
		child->perf_event_ctxp = child_ctx;
		get_task_struct(child);
	}

	ret = inherit_group(event, parent, parent_ctx,
			    child, child_ctx);

	if (ret)
		*inherited_all = 0;

	return ret;
5307 5308
}

5309

5310
/*
5311
 * Initialize the perf_event context in task_struct
5312
 */
5313
int perf_event_init_task(struct task_struct *child)
5314
{
5315
	struct perf_event_context *child_ctx, *parent_ctx;
5316 5317
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
5318
	struct task_struct *parent = current;
5319
	int inherited_all = 1;
5320
	int ret = 0;
5321

5322
	child->perf_event_ctxp = NULL;
5323

5324 5325
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);
5326

5327
	if (likely(!parent->perf_event_ctxp))
5328 5329
		return 0;

5330
	/*
5331 5332
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
5333
	 */
5334 5335
	parent_ctx = perf_pin_task_context(parent);

5336 5337 5338 5339 5340 5341 5342
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

5343 5344 5345 5346
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
5347
	mutex_lock(&parent_ctx->mutex);
5348 5349 5350 5351 5352

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
5353 5354 5355 5356 5357 5358
	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
			break;
	}
5359

5360 5361 5362 5363
	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
5364
			break;
5365 5366
	}

5367 5368
	child_ctx = child->perf_event_ctxp;

5369
	if (child_ctx && inherited_all) {
5370 5371 5372
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
5373 5374
		 * Note that if the parent is a clone, it could get
		 * uncloned at any point, but that doesn't matter
5375
		 * because the list of events and the generation
5376
		 * count can't have changed since we took the mutex.
5377
		 */
5378 5379 5380
		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
5381
			child_ctx->parent_gen = parent_ctx->parent_gen;
5382 5383 5384 5385 5386
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
5387 5388
	}

5389
	mutex_unlock(&parent_ctx->mutex);
5390

5391
	perf_unpin_context(parent_ctx);
5392

5393
	return ret;
5394 5395
}

5396
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
5397
{
5398
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
5399

5400
	cpuctx = &per_cpu(perf_cpu_context, cpu);
5401
	__perf_event_init_context(&cpuctx->ctx, NULL);
T
Thomas Gleixner 已提交
5402

5403
	spin_lock(&perf_resource_lock);
5404
	cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5405
	spin_unlock(&perf_resource_lock);
5406

5407
	hw_perf_event_setup(cpu);
T
Thomas Gleixner 已提交
5408 5409 5410
}

#ifdef CONFIG_HOTPLUG_CPU
5411
static void __perf_event_exit_cpu(void *info)
T
Thomas Gleixner 已提交
5412 5413
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5414 5415
	struct perf_event_context *ctx = &cpuctx->ctx;
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
5416

5417 5418 5419
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		__perf_event_remove_from_context(event);
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
5420
		__perf_event_remove_from_context(event);
T
Thomas Gleixner 已提交
5421
}
5422
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
5423
{
5424
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5425
	struct perf_event_context *ctx = &cpuctx->ctx;
5426 5427

	mutex_lock(&ctx->mutex);
5428
	smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5429
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
5430 5431
}
#else
5432
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443
#endif

static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action) {

	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5444
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
5445 5446
		break;

5447 5448
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
5449
		hw_perf_event_setup_online(cpu);
5450 5451
		break;

T
Thomas Gleixner 已提交
5452 5453
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
5454
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
5455 5456
		break;

5457 5458 5459 5460
	case CPU_DEAD:
		hw_perf_event_setup_offline(cpu);
		break;

T
Thomas Gleixner 已提交
5461 5462 5463 5464 5465 5466 5467
	default:
		break;
	}

	return NOTIFY_OK;
}

5468 5469 5470
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
T
Thomas Gleixner 已提交
5471 5472
static struct notifier_block __cpuinitdata perf_cpu_nb = {
	.notifier_call		= perf_cpu_notify,
5473
	.priority		= 20,
T
Thomas Gleixner 已提交
5474 5475
};

5476
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
5477 5478 5479
{
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
			(void *)(long)smp_processor_id());
5480 5481
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
			(void *)(long)smp_processor_id());
T
Thomas Gleixner 已提交
5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501
	register_cpu_notifier(&perf_cpu_nb);
}

static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_reserved_percpu);
}

static ssize_t
perf_set_reserve_percpu(struct sysdev_class *class,
			const char *buf,
			size_t count)
{
	struct perf_cpu_context *cpuctx;
	unsigned long val;
	int err, cpu, mpt;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
5502
	if (val > perf_max_events)
T
Thomas Gleixner 已提交
5503 5504
		return -EINVAL;

5505
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5506 5507 5508
	perf_reserved_percpu = val;
	for_each_online_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
5509
		raw_spin_lock_irq(&cpuctx->ctx.lock);
5510 5511
		mpt = min(perf_max_events - cpuctx->ctx.nr_events,
			  perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
5512
		cpuctx->max_pertask = mpt;
5513
		raw_spin_unlock_irq(&cpuctx->ctx.lock);
T
Thomas Gleixner 已提交
5514
	}
5515
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536

	return count;
}

static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_overcommit);
}

static ssize_t
perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
{
	unsigned long val;
	int err;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
	if (val > 1)
		return -EINVAL;

5537
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5538
	perf_overcommit = val;
5539
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565

	return count;
}

static SYSDEV_CLASS_ATTR(
				reserve_percpu,
				0644,
				perf_show_reserve_percpu,
				perf_set_reserve_percpu
			);

static SYSDEV_CLASS_ATTR(
				overcommit,
				0644,
				perf_show_overcommit,
				perf_set_overcommit
			);

static struct attribute *perfclass_attrs[] = {
	&attr_reserve_percpu.attr,
	&attr_overcommit.attr,
	NULL
};

static struct attribute_group perfclass_attr_group = {
	.attrs			= perfclass_attrs,
5566
	.name			= "perf_events",
T
Thomas Gleixner 已提交
5567 5568
};

5569
static int __init perf_event_sysfs_init(void)
T
Thomas Gleixner 已提交
5570 5571 5572 5573
{
	return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
				  &perfclass_attr_group);
}
5574
device_initcall(perf_event_sysfs_init);