perf_event.c 125.5 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
16
#include <linux/file.h>
T
Thomas Gleixner 已提交
17 18
#include <linux/poll.h>
#include <linux/sysfs.h>
19
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
20
#include <linux/percpu.h>
21
#include <linux/ptrace.h>
22
#include <linux/vmstat.h>
23
#include <linux/vmalloc.h>
24 25
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
26 27 28
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
29
#include <linux/kernel_stat.h>
30
#include <linux/perf_event.h>
L
Li Zefan 已提交
31
#include <linux/ftrace_event.h>
32
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
33

34 35
#include <asm/irq_regs.h>

T
Thomas Gleixner 已提交
36
/*
37
 * Each CPU has a list of per CPU events:
T
Thomas Gleixner 已提交
38
 */
39
static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
T
Thomas Gleixner 已提交
40

41
int perf_max_events __read_mostly = 1;
T
Thomas Gleixner 已提交
42 43 44
static int perf_reserved_percpu __read_mostly;
static int perf_overcommit __read_mostly = 1;

45 46 47 48
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
49

50
/*
51
 * perf event paranoia level:
52 53
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
54
 *   1 - disallow cpu events for unpriv
55
 *   2 - disallow kernel profiling for unpriv
56
 */
57
int sysctl_perf_event_paranoid __read_mostly = 1;
58

59
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
60 61

/*
62
 * max perf event sample rate
63
 */
64
int sysctl_perf_event_sample_rate __read_mostly = 100000;
65

66
static atomic64_t perf_event_id;
67

T
Thomas Gleixner 已提交
68
/*
69
 * Lock for (sysadmin-configurable) event reservations:
T
Thomas Gleixner 已提交
70
 */
71
static DEFINE_SPINLOCK(perf_resource_lock);
T
Thomas Gleixner 已提交
72 73 74 75

/*
 * Architecture provided APIs - weak aliases:
 */
76
extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
T
Thomas Gleixner 已提交
77
{
78
	return NULL;
T
Thomas Gleixner 已提交
79 80
}

81 82 83
void __weak hw_perf_disable(void)		{ barrier(); }
void __weak hw_perf_enable(void)		{ barrier(); }

84
int __weak
85
hw_perf_group_sched_in(struct perf_event *group_leader,
86
	       struct perf_cpu_context *cpuctx,
87
	       struct perf_event_context *ctx)
88 89 90
{
	return 0;
}
T
Thomas Gleixner 已提交
91

92
void __weak perf_event_print_debug(void)	{ }
93

94
static DEFINE_PER_CPU(int, perf_disable_count);
95 96 97

void perf_disable(void)
{
P
Peter Zijlstra 已提交
98 99
	if (!__get_cpu_var(perf_disable_count)++)
		hw_perf_disable();
100 101 102 103
}

void perf_enable(void)
{
P
Peter Zijlstra 已提交
104
	if (!--__get_cpu_var(perf_disable_count))
105 106 107
		hw_perf_enable();
}

108
static void get_ctx(struct perf_event_context *ctx)
109
{
110
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
111 112
}

113 114
static void free_ctx(struct rcu_head *head)
{
115
	struct perf_event_context *ctx;
116

117
	ctx = container_of(head, struct perf_event_context, rcu_head);
118 119 120
	kfree(ctx);
}

121
static void put_ctx(struct perf_event_context *ctx)
122
{
123 124 125
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
126 127 128
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
129
	}
130 131
}

132
static void unclone_ctx(struct perf_event_context *ctx)
133 134 135 136 137 138 139
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

140
/*
141
 * If we inherit events we want to return the parent event id
142 143
 * to userspace.
 */
144
static u64 primary_event_id(struct perf_event *event)
145
{
146
	u64 id = event->id;
147

148 149
	if (event->parent)
		id = event->parent->id;
150 151 152 153

	return id;
}

154
/*
155
 * Get the perf_event_context for a task and lock it.
156 157 158
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
159
static struct perf_event_context *
160
perf_lock_task_context(struct task_struct *task, unsigned long *flags)
161
{
162
	struct perf_event_context *ctx;
163 164 165

	rcu_read_lock();
 retry:
166
	ctx = rcu_dereference(task->perf_event_ctxp);
167 168 169 170
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
171
		 * perf_event_task_sched_out, though the
172 173 174 175 176 177
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
178
		raw_spin_lock_irqsave(&ctx->lock, *flags);
179
		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
180
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
181 182
			goto retry;
		}
183 184

		if (!atomic_inc_not_zero(&ctx->refcount)) {
185
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
186 187
			ctx = NULL;
		}
188 189 190 191 192 193 194 195 196 197
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
198
static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
199
{
200
	struct perf_event_context *ctx;
201 202 203 204 205
	unsigned long flags;

	ctx = perf_lock_task_context(task, &flags);
	if (ctx) {
		++ctx->pin_count;
206
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
207 208 209 210
	}
	return ctx;
}

211
static void perf_unpin_context(struct perf_event_context *ctx)
212 213 214
{
	unsigned long flags;

215
	raw_spin_lock_irqsave(&ctx->lock, flags);
216
	--ctx->pin_count;
217
	raw_spin_unlock_irqrestore(&ctx->lock, flags);
218 219 220
	put_ctx(ctx);
}

221 222
static inline u64 perf_clock(void)
{
223
	return cpu_clock(raw_smp_processor_id());
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
}

/*
 * Update the record of the current time in a context.
 */
static void update_context_time(struct perf_event_context *ctx)
{
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
}

/*
 * Update the total_time_enabled and total_time_running fields for a event.
 */
static void update_event_times(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	u64 run_end;

	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
		return;

249 250 251 252 253 254
	if (ctx->is_active)
		run_end = ctx->time;
	else
		run_end = event->tstamp_stopped;

	event->total_time_enabled = run_end - event->tstamp_enabled;
255 256 257 258 259 260 261 262 263

	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
	else
		run_end = ctx->time;

	event->total_time_running = run_end - event->tstamp_running;
}

264 265 266 267 268 269 270 271 272
static struct list_head *
ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
{
	if (event->attr.pinned)
		return &ctx->pinned_groups;
	else
		return &ctx->flexible_groups;
}

273
/*
274
 * Add a event from the lists for its context.
275 276
 * Must be called with ctx->mutex and ctx->lock held.
 */
277
static void
278
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
279
{
280
	struct perf_event *group_leader = event->group_leader;
281 282

	/*
283 284
	 * Depending on whether it is a standalone or sibling event,
	 * add it straight to the context's event list, or to the group
285 286
	 * leader's sibling list:
	 */
287 288 289
	if (group_leader == event) {
		struct list_head *list;

290 291 292
		if (is_software_event(event))
			event->group_flags |= PERF_GROUP_SOFTWARE;

293 294 295
		list = ctx_group_list(event, ctx);
		list_add_tail(&event->group_entry, list);
	} else {
296 297 298 299
		if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
		    !is_software_event(event))
			group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;

300
		list_add_tail(&event->group_entry, &group_leader->sibling_list);
P
Peter Zijlstra 已提交
301 302
		group_leader->nr_siblings++;
	}
P
Peter Zijlstra 已提交
303

304 305 306
	list_add_rcu(&event->event_entry, &ctx->event_list);
	ctx->nr_events++;
	if (event->attr.inherit_stat)
307
		ctx->nr_stat++;
308 309
}

310
/*
311
 * Remove a event from the lists for its context.
312
 * Must be called with ctx->mutex and ctx->lock held.
313
 */
314
static void
315
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
316
{
317
	struct perf_event *sibling, *tmp;
318

319
	if (list_empty(&event->group_entry))
320
		return;
321 322
	ctx->nr_events--;
	if (event->attr.inherit_stat)
323
		ctx->nr_stat--;
324

325 326
	list_del_init(&event->group_entry);
	list_del_rcu(&event->event_entry);
327

328 329
	if (event->group_leader != event)
		event->group_leader->nr_siblings--;
P
Peter Zijlstra 已提交
330

331
	update_event_times(event);
332 333 334 335 336 337 338 339 340 341

	/*
	 * If event was in error state, then keep it
	 * that way, otherwise bogus counts will be
	 * returned on read(). The only way to get out
	 * of error state is by explicit re-enabling
	 * of the event
	 */
	if (event->state > PERF_EVENT_STATE_OFF)
		event->state = PERF_EVENT_STATE_OFF;
342

343
	/*
344 345
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
346 347
	 * to the context list directly:
	 */
348
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
349
		struct list_head *list;
350

351 352
		list = ctx_group_list(event, ctx);
		list_move_tail(&sibling->group_entry, list);
353
		sibling->group_leader = sibling;
354 355 356

		/* Inherit group flags from the previous leader */
		sibling->group_flags = event->group_flags;
357 358 359
	}
}

360
static void
361
event_sched_out(struct perf_event *event,
362
		  struct perf_cpu_context *cpuctx,
363
		  struct perf_event_context *ctx)
364
{
365
	if (event->state != PERF_EVENT_STATE_ACTIVE)
366 367
		return;

368 369 370 371
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
372
	}
373 374 375
	event->tstamp_stopped = ctx->time;
	event->pmu->disable(event);
	event->oncpu = -1;
376

377
	if (!is_software_event(event))
378 379
		cpuctx->active_oncpu--;
	ctx->nr_active--;
380
	if (event->attr.exclusive || !cpuctx->active_oncpu)
381 382 383
		cpuctx->exclusive = 0;
}

384
static void
385
group_sched_out(struct perf_event *group_event,
386
		struct perf_cpu_context *cpuctx,
387
		struct perf_event_context *ctx)
388
{
389
	struct perf_event *event;
390

391
	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
392 393
		return;

394
	event_sched_out(group_event, cpuctx, ctx);
395 396 397 398

	/*
	 * Schedule out siblings (if any):
	 */
399 400
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
401

402
	if (group_event->attr.exclusive)
403 404 405
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
406
/*
407
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
408
 *
409
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
410 411
 * remove it from the context list.
 */
412
static void __perf_event_remove_from_context(void *info)
T
Thomas Gleixner 已提交
413 414
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
415 416
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
417 418 419 420 421 422

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
	 */
423
	if (ctx->task && cpuctx->task_ctx != ctx)
T
Thomas Gleixner 已提交
424 425
		return;

426
	raw_spin_lock(&ctx->lock);
427 428
	/*
	 * Protect the list operation against NMI by disabling the
429
	 * events on a global level.
430 431
	 */
	perf_disable();
T
Thomas Gleixner 已提交
432

433
	event_sched_out(event, cpuctx, ctx);
434

435
	list_del_event(event, ctx);
T
Thomas Gleixner 已提交
436 437 438

	if (!ctx->task) {
		/*
439
		 * Allow more per task events with respect to the
T
Thomas Gleixner 已提交
440 441 442
		 * reservation:
		 */
		cpuctx->max_pertask =
443 444
			min(perf_max_events - ctx->nr_events,
			    perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
445 446
	}

447
	perf_enable();
448
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
449 450 451 452
}


/*
453
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
454
 *
455
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
456
 *
457
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
458
 * call when the task is on a CPU.
459
 *
460 461
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
462 463
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
464
 * When called from perf_event_exit_task, it's OK because the
465
 * context has been detached from its task.
T
Thomas Gleixner 已提交
466
 */
467
static void perf_event_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
468
{
469
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
470 471 472 473
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
474
		 * Per cpu events are removed via an smp call and
475
		 * the removal is always successful.
T
Thomas Gleixner 已提交
476
		 */
477 478 479
		smp_call_function_single(event->cpu,
					 __perf_event_remove_from_context,
					 event, 1);
T
Thomas Gleixner 已提交
480 481 482 483
		return;
	}

retry:
484 485
	task_oncpu_function_call(task, __perf_event_remove_from_context,
				 event);
T
Thomas Gleixner 已提交
486

487
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
488 489 490
	/*
	 * If the context is active we need to retry the smp call.
	 */
491
	if (ctx->nr_active && !list_empty(&event->group_entry)) {
492
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
493 494 495 496 497
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
498
	 * can remove the event safely, if the call above did not
T
Thomas Gleixner 已提交
499 500
	 * succeed.
	 */
P
Peter Zijlstra 已提交
501
	if (!list_empty(&event->group_entry))
502
		list_del_event(event, ctx);
503
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
504 505
}

506
/*
507
 * Update total_time_enabled and total_time_running for all events in a group.
508
 */
509
static void update_group_times(struct perf_event *leader)
510
{
511
	struct perf_event *event;
512

513 514 515
	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
516 517
}

518
/*
519
 * Cross CPU call to disable a performance event
520
 */
521
static void __perf_event_disable(void *info)
522
{
523
	struct perf_event *event = info;
524
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
525
	struct perf_event_context *ctx = event->ctx;
526 527

	/*
528 529
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
530
	 */
531
	if (ctx->task && cpuctx->task_ctx != ctx)
532 533
		return;

534
	raw_spin_lock(&ctx->lock);
535 536

	/*
537
	 * If the event is on, turn it off.
538 539
	 * If it is in error state, leave it in error state.
	 */
540
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
541
		update_context_time(ctx);
542 543 544
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
545
		else
546 547
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
548 549
	}

550
	raw_spin_unlock(&ctx->lock);
551 552 553
}

/*
554
 * Disable a event.
555
 *
556 557
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
558
 * remains valid.  This condition is satisifed when called through
559 560 561 562
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
563
 * is the current context on this CPU and preemption is disabled,
564
 * hence we can't get into perf_event_task_sched_out for this context.
565
 */
566
void perf_event_disable(struct perf_event *event)
567
{
568
	struct perf_event_context *ctx = event->ctx;
569 570 571 572
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
573
		 * Disable the event on the cpu that it's on
574
		 */
575 576
		smp_call_function_single(event->cpu, __perf_event_disable,
					 event, 1);
577 578 579 580
		return;
	}

 retry:
581
	task_oncpu_function_call(task, __perf_event_disable, event);
582

583
	raw_spin_lock_irq(&ctx->lock);
584
	/*
585
	 * If the event is still active, we need to retry the cross-call.
586
	 */
587
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
588
		raw_spin_unlock_irq(&ctx->lock);
589 590 591 592 593 594 595
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
596 597 598
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
599
	}
600

601
	raw_spin_unlock_irq(&ctx->lock);
602 603
}

604
static int
605
event_sched_in(struct perf_event *event,
606
		 struct perf_cpu_context *cpuctx,
607
		 struct perf_event_context *ctx)
608
{
609
	if (event->state <= PERF_EVENT_STATE_OFF)
610 611
		return 0;

612
	event->state = PERF_EVENT_STATE_ACTIVE;
613
	event->oncpu = smp_processor_id();
614 615 616 617 618
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

619 620 621
	if (event->pmu->enable(event)) {
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
622 623 624
		return -EAGAIN;
	}

625
	event->tstamp_running += ctx->time - event->tstamp_stopped;
626

627
	if (!is_software_event(event))
628
		cpuctx->active_oncpu++;
629 630
	ctx->nr_active++;

631
	if (event->attr.exclusive)
632 633
		cpuctx->exclusive = 1;

634 635 636
	return 0;
}

637
static int
638
group_sched_in(struct perf_event *group_event,
639
	       struct perf_cpu_context *cpuctx,
640
	       struct perf_event_context *ctx)
641
{
642
	struct perf_event *event, *partial_group;
643 644
	int ret;

645
	if (group_event->state == PERF_EVENT_STATE_OFF)
646 647
		return 0;

648
	ret = hw_perf_group_sched_in(group_event, cpuctx, ctx);
649 650 651
	if (ret)
		return ret < 0 ? ret : 0;

652
	if (event_sched_in(group_event, cpuctx, ctx))
653 654 655 656 657
		return -EAGAIN;

	/*
	 * Schedule in siblings as one group (if any):
	 */
658
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
659
		if (event_sched_in(event, cpuctx, ctx)) {
660
			partial_group = event;
661 662 663 664 665 666 667 668 669 670 671
			goto group_error;
		}
	}

	return 0;

group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
	 */
672 673
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
674
			break;
675
		event_sched_out(event, cpuctx, ctx);
676
	}
677
	event_sched_out(group_event, cpuctx, ctx);
678 679 680 681

	return -EAGAIN;
}

682
/*
683
 * Work out whether we can put this event group on the CPU now.
684
 */
685
static int group_can_go_on(struct perf_event *event,
686 687 688 689
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
690
	 * Groups consisting entirely of software events can always go on.
691
	 */
692
	if (event->group_flags & PERF_GROUP_SOFTWARE)
693 694 695
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
696
	 * events can go on.
697 698 699 700 701
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
702
	 * events on the CPU, it can't go on.
703
	 */
704
	if (event->attr.exclusive && cpuctx->active_oncpu)
705 706 707 708 709 710 711 712
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

713 714
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
715
{
716 717 718 719
	list_add_event(event, ctx);
	event->tstamp_enabled = ctx->time;
	event->tstamp_running = ctx->time;
	event->tstamp_stopped = ctx->time;
720 721
}

T
Thomas Gleixner 已提交
722
/*
723
 * Cross CPU call to install and enable a performance event
724 725
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
726 727 728 729
 */
static void __perf_install_in_context(void *info)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
730 731 732
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
733
	int err;
T
Thomas Gleixner 已提交
734 735 736 737 738

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
739
	 * Or possibly this is the right context but it isn't
740
	 * on this cpu because it had no events.
T
Thomas Gleixner 已提交
741
	 */
742
	if (ctx->task && cpuctx->task_ctx != ctx) {
743
		if (cpuctx->task_ctx || ctx->task != current)
744 745 746
			return;
		cpuctx->task_ctx = ctx;
	}
T
Thomas Gleixner 已提交
747

748
	raw_spin_lock(&ctx->lock);
749
	ctx->is_active = 1;
750
	update_context_time(ctx);
T
Thomas Gleixner 已提交
751 752 753

	/*
	 * Protect the list operation against NMI by disabling the
754
	 * events on a global level. NOP for non NMI based events.
T
Thomas Gleixner 已提交
755
	 */
756
	perf_disable();
T
Thomas Gleixner 已提交
757

758
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
759

760 761 762
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

763
	/*
764
	 * Don't put the event on if it is disabled or if
765 766
	 * it is in a group and the group isn't on.
	 */
767 768
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
769 770
		goto unlock;

771
	/*
772 773 774
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
775
	 */
776
	if (!group_can_go_on(event, cpuctx, 1))
777 778
		err = -EEXIST;
	else
779
		err = event_sched_in(event, cpuctx, ctx);
780

781 782
	if (err) {
		/*
783
		 * This event couldn't go on.  If it is in a group
784
		 * then we have to pull the whole group off.
785
		 * If the event group is pinned then put it in error state.
786
		 */
787
		if (leader != event)
788
			group_sched_out(leader, cpuctx, ctx);
789
		if (leader->attr.pinned) {
790
			update_group_times(leader);
791
			leader->state = PERF_EVENT_STATE_ERROR;
792
		}
793
	}
T
Thomas Gleixner 已提交
794

795
	if (!err && !ctx->task && cpuctx->max_pertask)
T
Thomas Gleixner 已提交
796 797
		cpuctx->max_pertask--;

798
 unlock:
799
	perf_enable();
800

801
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
802 803 804
}

/*
805
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
806
 *
807 808
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
809
 *
810
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
811 812
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
813 814
 *
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
815 816
 */
static void
817 818
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
819 820 821 822 823 824
			int cpu)
{
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
825
		 * Per cpu events are installed via an smp call and
826
		 * the install is always successful.
T
Thomas Gleixner 已提交
827 828
		 */
		smp_call_function_single(cpu, __perf_install_in_context,
829
					 event, 1);
T
Thomas Gleixner 已提交
830 831 832 833 834
		return;
	}

retry:
	task_oncpu_function_call(task, __perf_install_in_context,
835
				 event);
T
Thomas Gleixner 已提交
836

837
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
838 839 840
	/*
	 * we need to retry the smp call.
	 */
841
	if (ctx->is_active && list_empty(&event->group_entry)) {
842
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
843 844 845 846 847
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
848
	 * can add the event safely, if it the call above did not
T
Thomas Gleixner 已提交
849 850
	 * succeed.
	 */
851 852
	if (list_empty(&event->group_entry))
		add_event_to_ctx(event, ctx);
853
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
854 855
}

856
/*
857
 * Put a event into inactive state and update time fields.
858 859 860 861 862 863
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
864 865
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
866
{
867
	struct perf_event *sub;
868

869 870 871 872
	event->state = PERF_EVENT_STATE_INACTIVE;
	event->tstamp_enabled = ctx->time - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry)
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
873 874 875 876
			sub->tstamp_enabled =
				ctx->time - sub->total_time_enabled;
}

877
/*
878
 * Cross CPU call to enable a performance event
879
 */
880
static void __perf_event_enable(void *info)
881
{
882
	struct perf_event *event = info;
883
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
884 885
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
886
	int err;
887

888
	/*
889 890
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
891
	 */
892
	if (ctx->task && cpuctx->task_ctx != ctx) {
893
		if (cpuctx->task_ctx || ctx->task != current)
894 895 896
			return;
		cpuctx->task_ctx = ctx;
	}
897

898
	raw_spin_lock(&ctx->lock);
899
	ctx->is_active = 1;
900
	update_context_time(ctx);
901

902
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
903
		goto unlock;
904
	__perf_event_mark_enabled(event, ctx);
905

906 907 908
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

909
	/*
910
	 * If the event is in a group and isn't the group leader,
911
	 * then don't put it on unless the group is on.
912
	 */
913
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
914
		goto unlock;
915

916
	if (!group_can_go_on(event, cpuctx, 1)) {
917
		err = -EEXIST;
918
	} else {
919
		perf_disable();
920
		if (event == leader)
921
			err = group_sched_in(event, cpuctx, ctx);
922
		else
923
			err = event_sched_in(event, cpuctx, ctx);
924
		perf_enable();
925
	}
926 927 928

	if (err) {
		/*
929
		 * If this event can't go on and it's part of a
930 931
		 * group, then the whole group has to come off.
		 */
932
		if (leader != event)
933
			group_sched_out(leader, cpuctx, ctx);
934
		if (leader->attr.pinned) {
935
			update_group_times(leader);
936
			leader->state = PERF_EVENT_STATE_ERROR;
937
		}
938 939 940
	}

 unlock:
941
	raw_spin_unlock(&ctx->lock);
942 943 944
}

/*
945
 * Enable a event.
946
 *
947 948
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
949
 * remains valid.  This condition is satisfied when called through
950 951
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
952
 */
953
void perf_event_enable(struct perf_event *event)
954
{
955
	struct perf_event_context *ctx = event->ctx;
956 957 958 959
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
960
		 * Enable the event on the cpu that it's on
961
		 */
962 963
		smp_call_function_single(event->cpu, __perf_event_enable,
					 event, 1);
964 965 966
		return;
	}

967
	raw_spin_lock_irq(&ctx->lock);
968
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
969 970 971
		goto out;

	/*
972 973
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
974 975 976 977
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
978 979
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
980 981

 retry:
982
	raw_spin_unlock_irq(&ctx->lock);
983
	task_oncpu_function_call(task, __perf_event_enable, event);
984

985
	raw_spin_lock_irq(&ctx->lock);
986 987

	/*
988
	 * If the context is active and the event is still off,
989 990
	 * we need to retry the cross-call.
	 */
991
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
992 993 994 995 996 997
		goto retry;

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
998 999
	if (event->state == PERF_EVENT_STATE_OFF)
		__perf_event_mark_enabled(event, ctx);
1000

1001
 out:
1002
	raw_spin_unlock_irq(&ctx->lock);
1003 1004
}

1005
static int perf_event_refresh(struct perf_event *event, int refresh)
1006
{
1007
	/*
1008
	 * not supported on inherited events
1009
	 */
1010
	if (event->attr.inherit)
1011 1012
		return -EINVAL;

1013 1014
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1015 1016

	return 0;
1017 1018
}

1019 1020 1021 1022 1023 1024 1025 1026 1027
enum event_type_t {
	EVENT_FLEXIBLE = 0x1,
	EVENT_PINNED = 0x2,
	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
};

static void ctx_sched_out(struct perf_event_context *ctx,
			  struct perf_cpu_context *cpuctx,
			  enum event_type_t event_type)
1028
{
1029
	struct perf_event *event;
1030

1031
	raw_spin_lock(&ctx->lock);
1032
	ctx->is_active = 0;
1033
	if (likely(!ctx->nr_events))
1034
		goto out;
1035
	update_context_time(ctx);
1036

1037
	perf_disable();
1038 1039 1040 1041
	if (!ctx->nr_active)
		goto out_enable;

	if (event_type & EVENT_PINNED)
1042 1043 1044
		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
			group_sched_out(event, cpuctx, ctx);

1045
	if (event_type & EVENT_FLEXIBLE)
1046
		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1047
			group_sched_out(event, cpuctx, ctx);
1048 1049

 out_enable:
1050
	perf_enable();
1051
 out:
1052
	raw_spin_unlock(&ctx->lock);
1053 1054
}

1055 1056 1057
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1058 1059 1060 1061
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1062
 * in them directly with an fd; we can only enable/disable all
1063
 * events via prctl, or enable/disable all events in a family
1064 1065
 * via ioctl, which will have the same effect on both contexts.
 */
1066 1067
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1068 1069
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1070
		&& ctx1->parent_gen == ctx2->parent_gen
1071
		&& !ctx1->pin_count && !ctx2->pin_count;
1072 1073
}

1074 1075
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1076 1077 1078
{
	u64 value;

1079
	if (!event->attr.inherit_stat)
1080 1081 1082
		return;

	/*
1083
	 * Update the event value, we cannot use perf_event_read()
1084 1085
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1086
	 * we know the event must be on the current CPU, therefore we
1087 1088
	 * don't need to use it.
	 */
1089 1090
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1091 1092
		event->pmu->read(event);
		/* fall-through */
1093

1094 1095
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1096 1097 1098 1099 1100 1101 1102
		break;

	default:
		break;
	}

	/*
1103
	 * In order to keep per-task stats reliable we need to flip the event
1104 1105
	 * values when we flip the contexts.
	 */
1106 1107 1108
	value = atomic64_read(&next_event->count);
	value = atomic64_xchg(&event->count, value);
	atomic64_set(&next_event->count, value);
1109

1110 1111
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1112

1113
	/*
1114
	 * Since we swizzled the values, update the user visible data too.
1115
	 */
1116 1117
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1118 1119 1120 1121 1122
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1123 1124
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1125
{
1126
	struct perf_event *event, *next_event;
1127 1128 1129 1130

	if (!ctx->nr_stat)
		return;

1131 1132
	update_context_time(ctx);

1133 1134
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1135

1136 1137
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1138

1139 1140
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1141

1142
		__perf_event_sync_stat(event, next_event);
1143

1144 1145
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1146 1147 1148
	}
}

T
Thomas Gleixner 已提交
1149
/*
1150
 * Called from scheduler to remove the events of the current task,
T
Thomas Gleixner 已提交
1151 1152
 * with interrupts disabled.
 *
1153
 * We stop each event and update the event value in event->count.
T
Thomas Gleixner 已提交
1154
 *
I
Ingo Molnar 已提交
1155
 * This does not protect us against NMI, but disable()
1156 1157 1158
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
T
Thomas Gleixner 已提交
1159
 */
1160
void perf_event_task_sched_out(struct task_struct *task,
1161
				 struct task_struct *next)
T
Thomas Gleixner 已提交
1162
{
1163
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1164 1165 1166
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1167
	struct pt_regs *regs;
1168
	int do_switch = 1;
T
Thomas Gleixner 已提交
1169

1170
	regs = task_pt_regs(task);
1171
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1172

1173
	if (likely(!ctx || !cpuctx->task_ctx))
T
Thomas Gleixner 已提交
1174 1175
		return;

1176 1177
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1178
	next_ctx = next->perf_event_ctxp;
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
1190 1191
		raw_spin_lock(&ctx->lock);
		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1192
		if (context_equiv(ctx, next_ctx)) {
1193 1194
			/*
			 * XXX do we need a memory barrier of sorts
1195
			 * wrt to rcu_dereference() of perf_event_ctxp
1196
			 */
1197 1198
			task->perf_event_ctxp = next_ctx;
			next->perf_event_ctxp = ctx;
1199 1200 1201
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1202

1203
			perf_event_sync_stat(ctx, next_ctx);
1204
		}
1205 1206
		raw_spin_unlock(&next_ctx->lock);
		raw_spin_unlock(&ctx->lock);
1207
	}
1208
	rcu_read_unlock();
1209

1210
	if (do_switch) {
1211
		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1212 1213
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1214 1215
}

1216 1217
static void task_ctx_sched_out(struct perf_event_context *ctx,
			       enum event_type_t event_type)
1218 1219 1220
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

1221 1222
	if (!cpuctx->task_ctx)
		return;
1223 1224 1225 1226

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1227
	ctx_sched_out(ctx, cpuctx, event_type);
1228 1229 1230
	cpuctx->task_ctx = NULL;
}

1231 1232 1233
/*
 * Called with IRQs disabled
 */
1234
static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1235
{
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
	task_ctx_sched_out(ctx, EVENT_ALL);
}

/*
 * Called with IRQs disabled
 */
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type)
{
	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1246 1247
}

1248
static void
1249
ctx_pinned_sched_in(struct perf_event_context *ctx,
1250
		    struct perf_cpu_context *cpuctx)
T
Thomas Gleixner 已提交
1251
{
1252
	struct perf_event *event;
T
Thomas Gleixner 已提交
1253

1254 1255
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF)
1256
			continue;
1257
		if (event->cpu != -1 && event->cpu != smp_processor_id())
1258 1259
			continue;

1260
		if (group_can_go_on(event, cpuctx, 1))
1261
			group_sched_in(event, cpuctx, ctx);
1262 1263 1264 1265 1266

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1267 1268 1269
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1270
		}
1271
	}
1272 1273 1274 1275
}

static void
ctx_flexible_sched_in(struct perf_event_context *ctx,
1276
		      struct perf_cpu_context *cpuctx)
1277 1278 1279
{
	struct perf_event *event;
	int can_add_hw = 1;
1280

1281 1282 1283
	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		/* Ignore events in OFF or ERROR state */
		if (event->state <= PERF_EVENT_STATE_OFF)
1284
			continue;
1285 1286
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1287
		 * of events:
1288
		 */
1289
		if (event->cpu != -1 && event->cpu != smp_processor_id())
T
Thomas Gleixner 已提交
1290 1291
			continue;

1292
		if (group_can_go_on(event, cpuctx, can_add_hw))
1293
			if (group_sched_in(event, cpuctx, ctx))
1294
				can_add_hw = 0;
T
Thomas Gleixner 已提交
1295
	}
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
}

static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
	     enum event_type_t event_type)
{
	raw_spin_lock(&ctx->lock);
	ctx->is_active = 1;
	if (likely(!ctx->nr_events))
		goto out;

	ctx->timestamp = perf_clock();

	perf_disable();

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
	if (event_type & EVENT_PINNED)
1317
		ctx_pinned_sched_in(ctx, cpuctx);
1318 1319 1320

	/* Then walk through the lower prio flexible groups */
	if (event_type & EVENT_FLEXIBLE)
1321
		ctx_flexible_sched_in(ctx, cpuctx);
1322

1323
	perf_enable();
1324
 out:
1325
	raw_spin_unlock(&ctx->lock);
1326 1327
}

1328 1329 1330 1331 1332 1333 1334 1335
static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
			     enum event_type_t event_type)
{
	struct perf_event_context *ctx = &cpuctx->ctx;

	ctx_sched_in(ctx, cpuctx, event_type);
}

1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
static void task_ctx_sched_in(struct task_struct *task,
			      enum event_type_t event_type)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_event_context *ctx = task->perf_event_ctxp;

	if (likely(!ctx))
		return;
	if (cpuctx->task_ctx == ctx)
		return;
	ctx_sched_in(ctx, cpuctx, event_type);
	cpuctx->task_ctx = ctx;
}
1349
/*
1350
 * Called from scheduler to add the events of the current task
1351 1352
 * with interrupts disabled.
 *
1353
 * We restore the event value and then enable it.
1354 1355
 *
 * This does not protect us against NMI, but enable()
1356 1357 1358
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
1359
 */
1360
void perf_event_task_sched_in(struct task_struct *task)
1361
{
1362 1363
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_event_context *ctx = task->perf_event_ctxp;
T
Thomas Gleixner 已提交
1364

1365 1366
	if (likely(!ctx))
		return;
1367

1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
	if (cpuctx->task_ctx == ctx)
		return;

	/*
	 * We want to keep the following priority order:
	 * cpu pinned (that don't need to move), task pinned,
	 * cpu flexible, task flexible.
	 */
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);

	ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
	ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);

	cpuctx->task_ctx = ctx;
1383 1384
}

1385 1386
#define MAX_INTERRUPTS (~0ULL)

1387
static void perf_log_throttle(struct perf_event *event, int enable);
1388

1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
{
	u64 frequency = event->attr.sample_freq;
	u64 sec = NSEC_PER_SEC;
	u64 divisor, dividend;

	int count_fls, nsec_fls, frequency_fls, sec_fls;

	count_fls = fls64(count);
	nsec_fls = fls64(nsec);
	frequency_fls = fls64(frequency);
	sec_fls = 30;

	/*
	 * We got @count in @nsec, with a target of sample_freq HZ
	 * the target period becomes:
	 *
	 *             @count * 10^9
	 * period = -------------------
	 *          @nsec * sample_freq
	 *
	 */

	/*
	 * Reduce accuracy by one bit such that @a and @b converge
	 * to a similar magnitude.
	 */
#define REDUCE_FLS(a, b) 		\
do {					\
	if (a##_fls > b##_fls) {	\
		a >>= 1;		\
		a##_fls--;		\
	} else {			\
		b >>= 1;		\
		b##_fls--;		\
	}				\
} while (0)

	/*
	 * Reduce accuracy until either term fits in a u64, then proceed with
	 * the other, so that finally we can do a u64/u64 division.
	 */
	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
		REDUCE_FLS(nsec, frequency);
		REDUCE_FLS(sec, count);
	}

	if (count_fls + sec_fls > 64) {
		divisor = nsec * frequency;

		while (count_fls + sec_fls > 64) {
			REDUCE_FLS(count, sec);
			divisor >>= 1;
		}

		dividend = count * sec;
	} else {
		dividend = count * sec;

		while (nsec_fls + frequency_fls > 64) {
			REDUCE_FLS(nsec, frequency);
			dividend >>= 1;
		}

		divisor = nsec * frequency;
	}

	return div64_u64(dividend, divisor);
}

1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
static void perf_event_stop(struct perf_event *event)
{
	if (!event->pmu->stop)
		return event->pmu->disable(event);

	return event->pmu->stop(event);
}

static int perf_event_start(struct perf_event *event)
{
	if (!event->pmu->start)
		return event->pmu->enable(event);

	return event->pmu->start(event);
}

1475
static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1476
{
1477
	struct hw_perf_event *hwc = &event->hw;
1478 1479 1480
	u64 period, sample_period;
	s64 delta;

1481
	period = perf_calculate_period(event, nsec, count);
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
1492 1493 1494

	if (atomic64_read(&hwc->period_left) > 8*sample_period) {
		perf_disable();
1495
		perf_event_stop(event);
1496
		atomic64_set(&hwc->period_left, 0);
1497
		perf_event_start(event);
1498 1499
		perf_enable();
	}
1500 1501
}

1502
static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1503
{
1504 1505
	struct perf_event *event;
	struct hw_perf_event *hwc;
1506 1507
	u64 interrupts, now;
	s64 delta;
1508

1509
	raw_spin_lock(&ctx->lock);
1510
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1511
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1512 1513
			continue;

1514 1515 1516
		if (event->cpu != -1 && event->cpu != smp_processor_id())
			continue;

1517
		hwc = &event->hw;
1518 1519 1520

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1521

1522
		/*
1523
		 * unthrottle events on the tick
1524
		 */
1525
		if (interrupts == MAX_INTERRUPTS) {
1526 1527
			perf_log_throttle(event, 1);
			event->pmu->unthrottle(event);
1528 1529
		}

1530
		if (!event->attr.freq || !event->attr.sample_freq)
1531 1532
			continue;

1533 1534 1535 1536
		event->pmu->read(event);
		now = atomic64_read(&event->count);
		delta = now - hwc->freq_count_stamp;
		hwc->freq_count_stamp = now;
1537

1538 1539
		if (delta > 0)
			perf_adjust_period(event, TICK_NSEC, delta);
1540
	}
1541
	raw_spin_unlock(&ctx->lock);
1542 1543
}

1544
/*
1545
 * Round-robin a context's events:
1546
 */
1547
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1548
{
1549
	if (!ctx->nr_events)
T
Thomas Gleixner 已提交
1550 1551
		return;

1552
	raw_spin_lock(&ctx->lock);
1553 1554 1555 1556

	/* Rotate the first entry last of non-pinned groups */
	list_rotate_left(&ctx->flexible_groups);

1557
	raw_spin_unlock(&ctx->lock);
1558 1559
}

1560
void perf_event_task_tick(struct task_struct *curr)
1561
{
1562
	struct perf_cpu_context *cpuctx;
1563
	struct perf_event_context *ctx;
1564

1565
	if (!atomic_read(&nr_events))
1566 1567
		return;

1568
	cpuctx = &__get_cpu_var(perf_cpu_context);
1569
	ctx = curr->perf_event_ctxp;
1570

1571 1572
	perf_disable();

1573
	perf_ctx_adjust_freq(&cpuctx->ctx);
1574
	if (ctx)
1575
		perf_ctx_adjust_freq(ctx);
1576

1577
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1578
	if (ctx)
1579
		task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
T
Thomas Gleixner 已提交
1580

1581
	rotate_ctx(&cpuctx->ctx);
1582 1583
	if (ctx)
		rotate_ctx(ctx);
1584

1585
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1586
	if (ctx)
1587
		task_ctx_sched_in(curr, EVENT_FLEXIBLE);
1588 1589

	perf_enable();
T
Thomas Gleixner 已提交
1590 1591
}

1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606
static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

	__perf_event_mark_enabled(event, ctx);

	return 1;
}

1607
/*
1608
 * Enable all of a task's events that have been marked enable-on-exec.
1609 1610
 * This expects task == current.
 */
1611
static void perf_event_enable_on_exec(struct task_struct *task)
1612
{
1613 1614
	struct perf_event_context *ctx;
	struct perf_event *event;
1615 1616
	unsigned long flags;
	int enabled = 0;
1617
	int ret;
1618 1619

	local_irq_save(flags);
1620 1621
	ctx = task->perf_event_ctxp;
	if (!ctx || !ctx->nr_events)
1622 1623
		goto out;

1624
	__perf_event_task_sched_out(ctx);
1625

1626
	raw_spin_lock(&ctx->lock);
1627

1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
	}

	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
1638 1639 1640
	}

	/*
1641
	 * Unclone this context if we enabled any event.
1642
	 */
1643 1644
	if (enabled)
		unclone_ctx(ctx);
1645

1646
	raw_spin_unlock(&ctx->lock);
1647

1648
	perf_event_task_sched_in(task);
1649 1650 1651 1652
 out:
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1653
/*
1654
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1655
 */
1656
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1657
{
1658
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1659 1660
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
I
Ingo Molnar 已提交
1661

1662 1663 1664 1665
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1666 1667
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1668 1669 1670 1671
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

1672
	raw_spin_lock(&ctx->lock);
P
Peter Zijlstra 已提交
1673
	update_context_time(ctx);
1674
	update_event_times(event);
1675
	raw_spin_unlock(&ctx->lock);
P
Peter Zijlstra 已提交
1676

P
Peter Zijlstra 已提交
1677
	event->pmu->read(event);
T
Thomas Gleixner 已提交
1678 1679
}

1680
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1681 1682
{
	/*
1683 1684
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1685
	 */
1686 1687 1688 1689
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
1690 1691 1692
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

1693
		raw_spin_lock_irqsave(&ctx->lock, flags);
P
Peter Zijlstra 已提交
1694
		update_context_time(ctx);
1695
		update_event_times(event);
1696
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1697 1698
	}

1699
	return atomic64_read(&event->count);
T
Thomas Gleixner 已提交
1700 1701
}

1702
/*
1703
 * Initialize the perf_event context in a task_struct:
1704 1705
 */
static void
1706
__perf_event_init_context(struct perf_event_context *ctx,
1707 1708
			    struct task_struct *task)
{
1709
	raw_spin_lock_init(&ctx->lock);
1710
	mutex_init(&ctx->mutex);
1711 1712
	INIT_LIST_HEAD(&ctx->pinned_groups);
	INIT_LIST_HEAD(&ctx->flexible_groups);
1713 1714 1715 1716 1717
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
	ctx->task = task;
}

1718
static struct perf_event_context *find_get_context(pid_t pid, int cpu)
T
Thomas Gleixner 已提交
1719
{
1720
	struct perf_event_context *ctx;
1721
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
1722
	struct task_struct *task;
1723
	unsigned long flags;
1724
	int err;
T
Thomas Gleixner 已提交
1725

1726
	if (pid == -1 && cpu != -1) {
1727
		/* Must be root to operate on a CPU event: */
1728
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
1729 1730
			return ERR_PTR(-EACCES);

1731
		if (cpu < 0 || cpu >= nr_cpumask_bits)
T
Thomas Gleixner 已提交
1732 1733 1734
			return ERR_PTR(-EINVAL);

		/*
1735
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
1736 1737 1738
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
1739
		if (!cpu_online(cpu))
T
Thomas Gleixner 已提交
1740 1741 1742 1743
			return ERR_PTR(-ENODEV);

		cpuctx = &per_cpu(perf_cpu_context, cpu);
		ctx = &cpuctx->ctx;
1744
		get_ctx(ctx);
T
Thomas Gleixner 已提交
1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760

		return ctx;
	}

	rcu_read_lock();
	if (!pid)
		task = current;
	else
		task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

1761
	/*
1762
	 * Can't attach events to a dying task.
1763 1764 1765 1766 1767
	 */
	err = -ESRCH;
	if (task->flags & PF_EXITING)
		goto errout;

T
Thomas Gleixner 已提交
1768
	/* Reuse ptrace permission checks for now. */
1769 1770 1771 1772 1773
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

 retry:
1774
	ctx = perf_lock_task_context(task, &flags);
1775
	if (ctx) {
1776
		unclone_ctx(ctx);
1777
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1778 1779
	}

1780
	if (!ctx) {
1781
		ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1782 1783 1784
		err = -ENOMEM;
		if (!ctx)
			goto errout;
1785
		__perf_event_init_context(ctx, task);
1786
		get_ctx(ctx);
1787
		if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1788 1789 1790 1791 1792
			/*
			 * We raced with some other task; use
			 * the context they set.
			 */
			kfree(ctx);
1793
			goto retry;
1794
		}
1795
		get_task_struct(task);
1796 1797
	}

1798
	put_task_struct(task);
T
Thomas Gleixner 已提交
1799
	return ctx;
1800 1801 1802 1803

 errout:
	put_task_struct(task);
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
1804 1805
}

L
Li Zefan 已提交
1806 1807
static void perf_event_free_filter(struct perf_event *event);

1808
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
1809
{
1810
	struct perf_event *event;
P
Peter Zijlstra 已提交
1811

1812 1813 1814
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
1815
	perf_event_free_filter(event);
1816
	kfree(event);
P
Peter Zijlstra 已提交
1817 1818
}

1819
static void perf_pending_sync(struct perf_event *event);
1820

1821
static void free_event(struct perf_event *event)
1822
{
1823
	perf_pending_sync(event);
1824

1825 1826 1827 1828 1829 1830 1831 1832
	if (!event->parent) {
		atomic_dec(&nr_events);
		if (event->attr.mmap)
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
1833
	}
1834

1835 1836 1837
	if (event->output) {
		fput(event->output->filp);
		event->output = NULL;
1838 1839
	}

1840 1841
	if (event->destroy)
		event->destroy(event);
1842

1843 1844
	put_ctx(event->ctx);
	call_rcu(&event->rcu_head, free_event_rcu);
1845 1846
}

1847
int perf_event_release_kernel(struct perf_event *event)
T
Thomas Gleixner 已提交
1848
{
1849
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1850

1851
	WARN_ON_ONCE(ctx->parent_ctx);
1852
	mutex_lock(&ctx->mutex);
1853
	perf_event_remove_from_context(event);
1854
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
1855

1856 1857 1858 1859
	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);
1860

1861
	free_event(event);
T
Thomas Gleixner 已提交
1862 1863 1864

	return 0;
}
1865
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
T
Thomas Gleixner 已提交
1866

1867 1868 1869 1870
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
1871
{
1872
	struct perf_event *event = file->private_data;
1873

1874
	file->private_data = NULL;
1875

1876
	return perf_event_release_kernel(event);
1877 1878
}

1879
static int perf_event_read_size(struct perf_event *event)
1880 1881 1882 1883 1884
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

1885
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1886 1887
		size += sizeof(u64);

1888
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1889 1890
		size += sizeof(u64);

1891
	if (event->attr.read_format & PERF_FORMAT_ID)
1892 1893
		entry += sizeof(u64);

1894 1895
	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
1896 1897 1898 1899 1900 1901 1902 1903
		size += sizeof(u64);
	}

	size += entry * nr;

	return size;
}

1904
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
1905
{
1906
	struct perf_event *child;
1907 1908
	u64 total = 0;

1909 1910 1911
	*enabled = 0;
	*running = 0;

1912
	mutex_lock(&event->child_mutex);
1913
	total += perf_event_read(event);
1914 1915 1916 1917 1918 1919
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
1920
		total += perf_event_read(child);
1921 1922 1923
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
1924
	mutex_unlock(&event->child_mutex);
1925 1926 1927

	return total;
}
1928
EXPORT_SYMBOL_GPL(perf_event_read_value);
1929

1930
static int perf_event_read_group(struct perf_event *event,
1931 1932
				   u64 read_format, char __user *buf)
{
1933
	struct perf_event *leader = event->group_leader, *sub;
1934 1935
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
1936
	u64 values[5];
1937
	u64 count, enabled, running;
1938

1939
	mutex_lock(&ctx->mutex);
1940
	count = perf_event_read_value(leader, &enabled, &running);
1941 1942

	values[n++] = 1 + leader->nr_siblings;
1943 1944 1945 1946
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1947 1948 1949
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
1950 1951 1952 1953

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
1954
		goto unlock;
1955

1956
	ret = size;
1957

1958
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
1959
		n = 0;
1960

1961
		values[n++] = perf_event_read_value(sub, &enabled, &running);
1962 1963 1964 1965 1966
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

1967
		if (copy_to_user(buf + ret, values, size)) {
1968 1969 1970
			ret = -EFAULT;
			goto unlock;
		}
1971 1972

		ret += size;
1973
	}
1974 1975
unlock:
	mutex_unlock(&ctx->mutex);
1976

1977
	return ret;
1978 1979
}

1980
static int perf_event_read_one(struct perf_event *event,
1981 1982
				 u64 read_format, char __user *buf)
{
1983
	u64 enabled, running;
1984 1985 1986
	u64 values[4];
	int n = 0;

1987 1988 1989 1990 1991
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1992
	if (read_format & PERF_FORMAT_ID)
1993
		values[n++] = primary_event_id(event);
1994 1995 1996 1997 1998 1999 2000

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
2001
/*
2002
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
2003 2004
 */
static ssize_t
2005
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
2006
{
2007
	u64 read_format = event->attr.read_format;
2008
	int ret;
T
Thomas Gleixner 已提交
2009

2010
	/*
2011
	 * Return end-of-file for a read on a event that is in
2012 2013 2014
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
2015
	if (event->state == PERF_EVENT_STATE_ERROR)
2016 2017
		return 0;

2018
	if (count < perf_event_read_size(event))
2019 2020
		return -ENOSPC;

2021
	WARN_ON_ONCE(event->ctx->parent_ctx);
2022
	if (read_format & PERF_FORMAT_GROUP)
2023
		ret = perf_event_read_group(event, read_format, buf);
2024
	else
2025
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
2026

2027
	return ret;
T
Thomas Gleixner 已提交
2028 2029 2030 2031 2032
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
2033
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
2034

2035
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
2036 2037 2038 2039
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
2040
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
2041
	struct perf_mmap_data *data;
2042
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
2043 2044

	rcu_read_lock();
2045
	data = rcu_dereference(event->data);
P
Peter Zijlstra 已提交
2046
	if (data)
2047
		events = atomic_xchg(&data->poll, 0);
P
Peter Zijlstra 已提交
2048
	rcu_read_unlock();
T
Thomas Gleixner 已提交
2049

2050
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
2051 2052 2053 2054

	return events;
}

2055
static void perf_event_reset(struct perf_event *event)
2056
{
2057 2058 2059
	(void)perf_event_read(event);
	atomic64_set(&event->count, 0);
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
2060 2061
}

2062
/*
2063 2064 2065 2066
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
2067
 */
2068 2069
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2070
{
2071
	struct perf_event *child;
P
Peter Zijlstra 已提交
2072

2073 2074 2075 2076
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
2077
		func(child);
2078
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
2079 2080
}

2081 2082
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2083
{
2084 2085
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
2086

2087 2088
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
2089
	event = event->group_leader;
2090

2091 2092 2093 2094
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
2095
	mutex_unlock(&ctx->mutex);
2096 2097
}

2098
static int perf_event_period(struct perf_event *event, u64 __user *arg)
2099
{
2100
	struct perf_event_context *ctx = event->ctx;
2101 2102 2103 2104
	unsigned long size;
	int ret = 0;
	u64 value;

2105
	if (!event->attr.sample_period)
2106 2107 2108 2109 2110 2111 2112 2113 2114
		return -EINVAL;

	size = copy_from_user(&value, arg, sizeof(value));
	if (size != sizeof(value))
		return -EFAULT;

	if (!value)
		return -EINVAL;

2115
	raw_spin_lock_irq(&ctx->lock);
2116 2117
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
2118 2119 2120 2121
			ret = -EINVAL;
			goto unlock;
		}

2122
		event->attr.sample_freq = value;
2123
	} else {
2124 2125
		event->attr.sample_period = value;
		event->hw.sample_period = value;
2126 2127
	}
unlock:
2128
	raw_spin_unlock_irq(&ctx->lock);
2129 2130 2131 2132

	return ret;
}

L
Li Zefan 已提交
2133 2134
static int perf_event_set_output(struct perf_event *event, int output_fd);
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2135

2136 2137
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
2138 2139
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2140
	u32 flags = arg;
2141 2142

	switch (cmd) {
2143 2144
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2145
		break;
2146 2147
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2148
		break;
2149 2150
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2151
		break;
P
Peter Zijlstra 已提交
2152

2153 2154
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2155

2156 2157
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2158

2159 2160
	case PERF_EVENT_IOC_SET_OUTPUT:
		return perf_event_set_output(event, arg);
2161

L
Li Zefan 已提交
2162 2163 2164
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2165
	default:
P
Peter Zijlstra 已提交
2166
		return -ENOTTY;
2167
	}
P
Peter Zijlstra 已提交
2168 2169

	if (flags & PERF_IOC_FLAG_GROUP)
2170
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2171
	else
2172
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2173 2174

	return 0;
2175 2176
}

2177
int perf_event_task_enable(void)
2178
{
2179
	struct perf_event *event;
2180

2181 2182 2183 2184
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2185 2186 2187 2188

	return 0;
}

2189
int perf_event_task_disable(void)
2190
{
2191
	struct perf_event *event;
2192

2193 2194 2195 2196
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2197 2198 2199 2200

	return 0;
}

2201 2202
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2203 2204
#endif

2205
static int perf_event_index(struct perf_event *event)
2206
{
2207
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2208 2209
		return 0;

2210
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2211 2212
}

2213 2214 2215 2216 2217
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2218
void perf_event_update_userpage(struct perf_event *event)
2219
{
2220
	struct perf_event_mmap_page *userpg;
2221
	struct perf_mmap_data *data;
2222 2223

	rcu_read_lock();
2224
	data = rcu_dereference(event->data);
2225 2226 2227 2228
	if (!data)
		goto unlock;

	userpg = data->user_page;
2229

2230 2231 2232 2233 2234
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2235
	++userpg->lock;
2236
	barrier();
2237 2238 2239 2240
	userpg->index = perf_event_index(event);
	userpg->offset = atomic64_read(&event->count);
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		userpg->offset -= atomic64_read(&event->hw.prev_count);
2241

2242 2243
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2244

2245 2246
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2247

2248
	barrier();
2249
	++userpg->lock;
2250
	preempt_enable();
2251
unlock:
2252
	rcu_read_unlock();
2253 2254
}

2255
static unsigned long perf_data_size(struct perf_mmap_data *data)
2256
{
2257 2258
	return data->nr_pages << (PAGE_SHIFT + data->data_order);
}
2259

2260
#ifndef CONFIG_PERF_USE_VMALLOC
2261

2262 2263 2264
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2265

2266 2267 2268 2269 2270
static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > data->nr_pages)
		return NULL;
2271

2272 2273
	if (pgoff == 0)
		return virt_to_page(data->user_page);
2274

2275
	return virt_to_page(data->data_pages[pgoff - 1]);
2276 2277
}

2278 2279
static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2280 2281 2282 2283 2284
{
	struct perf_mmap_data *data;
	unsigned long size;
	int i;

2285
	WARN_ON(atomic_read(&event->mmap_count));
2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303

	size = sizeof(struct perf_mmap_data);
	size += nr_pages * sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
	if (!data->user_page)
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
		data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
		if (!data->data_pages[i])
			goto fail_data_pages;
	}

2304
	data->data_order = 0;
2305 2306
	data->nr_pages = nr_pages;

2307
	return data;
2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318

fail_data_pages:
	for (i--; i >= 0; i--)
		free_page((unsigned long)data->data_pages[i]);

	free_page((unsigned long)data->user_page);

fail_user_page:
	kfree(data);

fail:
2319
	return NULL;
2320 2321
}

2322 2323
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2324
	struct page *page = virt_to_page((void *)addr);
2325 2326 2327 2328 2329

	page->mapping = NULL;
	__free_page(page);
}

2330
static void perf_mmap_data_free(struct perf_mmap_data *data)
2331 2332 2333
{
	int i;

2334
	perf_mmap_free_page((unsigned long)data->user_page);
2335
	for (i = 0; i < data->nr_pages; i++)
2336
		perf_mmap_free_page((unsigned long)data->data_pages[i]);
2337
	kfree(data);
2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377
}

#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > (1UL << data->data_order))
		return NULL;

	return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

static void perf_mmap_data_free_work(struct work_struct *work)
{
	struct perf_mmap_data *data;
	void *base;
	int i, nr;

	data = container_of(work, struct perf_mmap_data, work);
	nr = 1 << data->data_order;

	base = data->user_page;
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
2378
	kfree(data);
2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
}

static void perf_mmap_data_free(struct perf_mmap_data *data)
{
	schedule_work(&data->work);
}

static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
{
	struct perf_mmap_data *data;
	unsigned long size;
	void *all_buf;

	WARN_ON(atomic_read(&event->mmap_count));
2394

2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471
	size = sizeof(struct perf_mmap_data);
	size += sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	INIT_WORK(&data->work, perf_mmap_data_free_work);

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

	data->user_page = all_buf;
	data->data_pages[0] = all_buf + PAGE_SIZE;
	data->data_order = ilog2(nr_pages);
	data->nr_pages = 1;

	return data;

fail_all_buf:
	kfree(data);

fail:
	return NULL;
}

#endif

static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
	struct perf_mmap_data *data;
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	data = rcu_dereference(event->data);
	if (!data)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(data, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

static void
perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
{
	long max_size = perf_data_size(data);

	atomic_set(&data->lock, -1);

	if (event->attr.watermark) {
		data->watermark = min_t(long, max_size,
					event->attr.wakeup_watermark);
	}

	if (!data->watermark)
2472
		data->watermark = max_size / 2;
2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483


	rcu_assign_pointer(event->data, data);
}

static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
{
	struct perf_mmap_data *data;

	data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
	perf_mmap_data_free(data);
2484 2485
}

2486
static void perf_mmap_data_release(struct perf_event *event)
2487
{
2488
	struct perf_mmap_data *data = event->data;
2489

2490
	WARN_ON(atomic_read(&event->mmap_count));
2491

2492
	rcu_assign_pointer(event->data, NULL);
2493
	call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2494 2495 2496 2497
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
2498
	struct perf_event *event = vma->vm_file->private_data;
2499

2500
	atomic_inc(&event->mmap_count);
2501 2502 2503 2504
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
2505
	struct perf_event *event = vma->vm_file->private_data;
2506

2507 2508
	WARN_ON_ONCE(event->ctx->parent_ctx);
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2509
		unsigned long size = perf_data_size(event->data);
2510 2511
		struct user_struct *user = current_user();

2512
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2513
		vma->vm_mm->locked_vm -= event->data->nr_locked;
2514
		perf_mmap_data_release(event);
2515
		mutex_unlock(&event->mmap_mutex);
2516
	}
2517 2518
}

2519
static const struct vm_operations_struct perf_mmap_vmops = {
2520 2521 2522 2523
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
2524 2525 2526 2527
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
2528
	struct perf_event *event = file->private_data;
2529
	unsigned long user_locked, user_lock_limit;
2530
	struct user_struct *user = current_user();
2531
	unsigned long locked, lock_limit;
2532
	struct perf_mmap_data *data;
2533 2534
	unsigned long vma_size;
	unsigned long nr_pages;
2535
	long user_extra, extra;
2536
	int ret = 0;
2537

2538
	if (!(vma->vm_flags & VM_SHARED))
2539
		return -EINVAL;
2540 2541 2542 2543

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

2544 2545 2546 2547 2548
	/*
	 * If we have data pages ensure they're a power-of-two number, so we
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
2549 2550
		return -EINVAL;

2551
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
2552 2553
		return -EINVAL;

2554 2555
	if (vma->vm_pgoff != 0)
		return -EINVAL;
2556

2557 2558 2559
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
	if (event->output) {
2560 2561 2562 2563
		ret = -EINVAL;
		goto unlock;
	}

2564 2565
	if (atomic_inc_not_zero(&event->mmap_count)) {
		if (nr_pages != event->data->nr_pages)
2566 2567 2568 2569
			ret = -EINVAL;
		goto unlock;
	}

2570
	user_extra = nr_pages + 1;
2571
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
2572 2573 2574 2575 2576 2577

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

2578
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2579

2580 2581 2582
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
2583

2584
	lock_limit = rlimit(RLIMIT_MEMLOCK);
2585
	lock_limit >>= PAGE_SHIFT;
2586
	locked = vma->vm_mm->locked_vm + extra;
2587

2588 2589
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
2590 2591 2592
		ret = -EPERM;
		goto unlock;
	}
2593

2594
	WARN_ON(event->data);
2595 2596 2597 2598

	data = perf_mmap_data_alloc(event, nr_pages);
	ret = -ENOMEM;
	if (!data)
2599 2600
		goto unlock;

2601 2602 2603
	ret = 0;
	perf_mmap_data_init(event, data);

2604
	atomic_set(&event->mmap_count, 1);
2605
	atomic_long_add(user_extra, &user->locked_vm);
2606
	vma->vm_mm->locked_vm += extra;
2607
	event->data->nr_locked = extra;
2608
	if (vma->vm_flags & VM_WRITE)
2609
		event->data->writable = 1;
2610

2611
unlock:
2612
	mutex_unlock(&event->mmap_mutex);
2613 2614 2615

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
2616 2617

	return ret;
2618 2619
}

P
Peter Zijlstra 已提交
2620 2621 2622
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
2623
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
2624 2625 2626
	int retval;

	mutex_lock(&inode->i_mutex);
2627
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
2628 2629 2630 2631 2632 2633 2634 2635
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
2636 2637 2638 2639
static const struct file_operations perf_fops = {
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
2640 2641
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
2642
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
2643
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
2644 2645
};

2646
/*
2647
 * Perf event wakeup
2648 2649 2650 2651 2652
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

2653
void perf_event_wakeup(struct perf_event *event)
2654
{
2655
	wake_up_all(&event->waitq);
2656

2657 2658 2659
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
2660
	}
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671
}

/*
 * Pending wakeups
 *
 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
 *
 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
 * single linked list and use cmpxchg() to add entries lockless.
 */

2672
static void perf_pending_event(struct perf_pending_entry *entry)
2673
{
2674 2675
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
2676

2677 2678 2679
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
2680 2681
	}

2682 2683 2684
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
2685 2686 2687
	}
}

2688
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2689

2690
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2691 2692 2693
	PENDING_TAIL,
};

2694 2695
static void perf_pending_queue(struct perf_pending_entry *entry,
			       void (*func)(struct perf_pending_entry *))
2696
{
2697
	struct perf_pending_entry **head;
2698

2699
	if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2700 2701
		return;

2702 2703 2704
	entry->func = func;

	head = &get_cpu_var(perf_pending_head);
2705 2706

	do {
2707 2708
		entry->next = *head;
	} while (cmpxchg(head, entry->next, entry) != entry->next);
2709

2710
	set_perf_event_pending();
2711

2712
	put_cpu_var(perf_pending_head);
2713 2714 2715 2716
}

static int __perf_pending_run(void)
{
2717
	struct perf_pending_entry *list;
2718 2719
	int nr = 0;

2720
	list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2721
	while (list != PENDING_TAIL) {
2722 2723
		void (*func)(struct perf_pending_entry *);
		struct perf_pending_entry *entry = list;
2724 2725 2726

		list = list->next;

2727 2728
		func = entry->func;
		entry->next = NULL;
2729 2730 2731 2732 2733 2734 2735
		/*
		 * Ensure we observe the unqueue before we issue the wakeup,
		 * so that we won't be waiting forever.
		 * -- see perf_not_pending().
		 */
		smp_wmb();

2736
		func(entry);
2737 2738 2739 2740 2741 2742
		nr++;
	}

	return nr;
}

2743
static inline int perf_not_pending(struct perf_event *event)
2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
{
	/*
	 * If we flush on whatever cpu we run, there is a chance we don't
	 * need to wait.
	 */
	get_cpu();
	__perf_pending_run();
	put_cpu();

	/*
	 * Ensure we see the proper queue state before going to sleep
	 * so that we do not miss the wakeup. -- see perf_pending_handle()
	 */
	smp_rmb();
2758
	return event->pending.next == NULL;
2759 2760
}

2761
static void perf_pending_sync(struct perf_event *event)
2762
{
2763
	wait_event(event->waitq, perf_not_pending(event));
2764 2765
}

2766
void perf_event_do_pending(void)
2767 2768 2769 2770
{
	__perf_pending_run();
}

2771 2772 2773 2774
/*
 * Callchain support -- arch specific
 */

2775
__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2776 2777 2778 2779
{
	return NULL;
}

2780 2781 2782
/*
 * Output
 */
2783 2784
static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
			      unsigned long offset, unsigned long head)
2785 2786 2787 2788 2789 2790
{
	unsigned long mask;

	if (!data->writable)
		return true;

2791
	mask = perf_data_size(data) - 1;
2792 2793 2794 2795 2796 2797 2798 2799 2800 2801

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

2802
static void perf_output_wakeup(struct perf_output_handle *handle)
2803
{
2804 2805
	atomic_set(&handle->data->poll, POLL_IN);

2806
	if (handle->nmi) {
2807 2808 2809
		handle->event->pending_wakeup = 1;
		perf_pending_queue(&handle->event->pending,
				   perf_pending_event);
2810
	} else
2811
		perf_event_wakeup(handle->event);
2812 2813
}

2814 2815 2816
/*
 * Curious locking construct.
 *
2817 2818
 * We need to ensure a later event_id doesn't publish a head when a former
 * event_id isn't done writing. However since we need to deal with NMIs we
2819 2820 2821 2822 2823 2824
 * cannot fully serialize things.
 *
 * What we do is serialize between CPUs so we only have to deal with NMI
 * nesting on a single CPU.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
2825
 * event_id completes.
2826 2827 2828 2829
 */
static void perf_output_lock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2830
	int cur, cpu = get_cpu();
2831 2832 2833

	handle->locked = 0;

2834 2835 2836 2837 2838 2839 2840 2841
	for (;;) {
		cur = atomic_cmpxchg(&data->lock, -1, cpu);
		if (cur == -1) {
			handle->locked = 1;
			break;
		}
		if (cur == cpu)
			break;
2842 2843

		cpu_relax();
2844
	}
2845 2846 2847 2848 2849
}

static void perf_output_unlock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2850 2851
	unsigned long head;
	int cpu;
2852

2853
	data->done_head = data->head;
2854 2855 2856 2857 2858 2859 2860 2861 2862 2863

	if (!handle->locked)
		goto out;

again:
	/*
	 * The xchg implies a full barrier that ensures all writes are done
	 * before we publish the new head, matched by a rmb() in userspace when
	 * reading this position.
	 */
2864
	while ((head = atomic_long_xchg(&data->done_head, 0)))
2865 2866 2867
		data->user_page->data_head = head;

	/*
2868
	 * NMI can happen here, which means we can miss a done_head update.
2869 2870
	 */

2871
	cpu = atomic_xchg(&data->lock, -1);
2872 2873 2874 2875 2876
	WARN_ON_ONCE(cpu != smp_processor_id());

	/*
	 * Therefore we have to validate we did not indeed do so.
	 */
2877
	if (unlikely(atomic_long_read(&data->done_head))) {
2878 2879 2880
		/*
		 * Since we had it locked, we can lock it again.
		 */
2881
		while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2882 2883 2884 2885 2886
			cpu_relax();

		goto again;
	}

2887
	if (atomic_xchg(&data->wakeup, 0))
2888 2889
		perf_output_wakeup(handle);
out:
2890
	put_cpu();
2891 2892
}

2893 2894
void perf_output_copy(struct perf_output_handle *handle,
		      const void *buf, unsigned int len)
2895 2896
{
	unsigned int pages_mask;
2897
	unsigned long offset;
2898 2899 2900 2901 2902 2903 2904 2905
	unsigned int size;
	void **pages;

	offset		= handle->offset;
	pages_mask	= handle->data->nr_pages - 1;
	pages		= handle->data->data_pages;

	do {
2906 2907
		unsigned long page_offset;
		unsigned long page_size;
2908 2909 2910
		int nr;

		nr	    = (offset >> PAGE_SHIFT) & pages_mask;
2911 2912 2913
		page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
		page_offset = offset & (page_size - 1);
		size	    = min_t(unsigned int, page_size - page_offset, len);
2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930

		memcpy(pages[nr] + page_offset, buf, size);

		len	    -= size;
		buf	    += size;
		offset	    += size;
	} while (len);

	handle->offset = offset;

	/*
	 * Check we didn't copy past our reservation window, taking the
	 * possible unsigned int wrap into account.
	 */
	WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
}

2931
int perf_output_begin(struct perf_output_handle *handle,
2932
		      struct perf_event *event, unsigned int size,
2933
		      int nmi, int sample)
2934
{
2935
	struct perf_event *output_event;
2936
	struct perf_mmap_data *data;
2937
	unsigned long tail, offset, head;
2938 2939 2940 2941 2942 2943
	int have_lost;
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
2944

2945
	rcu_read_lock();
2946
	/*
2947
	 * For inherited events we send all the output towards the parent.
2948
	 */
2949 2950
	if (event->parent)
		event = event->parent;
2951

2952 2953 2954
	output_event = rcu_dereference(event->output);
	if (output_event)
		event = output_event;
2955

2956
	data = rcu_dereference(event->data);
2957 2958 2959
	if (!data)
		goto out;

2960
	handle->data	= data;
2961
	handle->event	= event;
2962 2963
	handle->nmi	= nmi;
	handle->sample	= sample;
2964

2965
	if (!data->nr_pages)
2966
		goto fail;
2967

2968 2969 2970 2971
	have_lost = atomic_read(&data->lost);
	if (have_lost)
		size += sizeof(lost_event);

2972 2973
	perf_output_lock(handle);

2974
	do {
2975 2976 2977 2978 2979 2980 2981
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
		tail = ACCESS_ONCE(data->user_page->data_tail);
		smp_rmb();
2982
		offset = head = atomic_long_read(&data->head);
P
Peter Zijlstra 已提交
2983
		head += size;
2984
		if (unlikely(!perf_output_space(data, tail, offset, head)))
2985
			goto fail;
2986
	} while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2987

2988
	handle->offset	= offset;
2989
	handle->head	= head;
2990

2991
	if (head - tail > data->watermark)
2992
		atomic_set(&data->wakeup, 1);
2993

2994
	if (have_lost) {
2995
		lost_event.header.type = PERF_RECORD_LOST;
2996 2997
		lost_event.header.misc = 0;
		lost_event.header.size = sizeof(lost_event);
2998
		lost_event.id          = event->id;
2999 3000 3001 3002 3003
		lost_event.lost        = atomic_xchg(&data->lost, 0);

		perf_output_put(handle, lost_event);
	}

3004
	return 0;
3005

3006
fail:
3007 3008
	atomic_inc(&data->lost);
	perf_output_unlock(handle);
3009 3010
out:
	rcu_read_unlock();
3011

3012 3013
	return -ENOSPC;
}
3014

3015
void perf_output_end(struct perf_output_handle *handle)
3016
{
3017
	struct perf_event *event = handle->event;
3018 3019
	struct perf_mmap_data *data = handle->data;

3020
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
3021

3022
	if (handle->sample && wakeup_events) {
3023
		int events = atomic_inc_return(&data->events);
P
Peter Zijlstra 已提交
3024
		if (events >= wakeup_events) {
3025
			atomic_sub(wakeup_events, &data->events);
3026
			atomic_set(&data->wakeup, 1);
P
Peter Zijlstra 已提交
3027
		}
3028 3029 3030
	}

	perf_output_unlock(handle);
3031
	rcu_read_unlock();
3032 3033
}

3034
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
3035 3036
{
	/*
3037
	 * only top level events have the pid namespace they were created in
3038
	 */
3039 3040
	if (event->parent)
		event = event->parent;
3041

3042
	return task_tgid_nr_ns(p, event->ns);
3043 3044
}

3045
static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
3046 3047
{
	/*
3048
	 * only top level events have the pid namespace they were created in
3049
	 */
3050 3051
	if (event->parent)
		event = event->parent;
3052

3053
	return task_pid_nr_ns(p, event->ns);
3054 3055
}

3056
static void perf_output_read_one(struct perf_output_handle *handle,
3057
				 struct perf_event *event)
3058
{
3059
	u64 read_format = event->attr.read_format;
3060 3061 3062
	u64 values[4];
	int n = 0;

3063
	values[n++] = atomic64_read(&event->count);
3064
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3065 3066
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
3067 3068
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3069 3070
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
3071 3072
	}
	if (read_format & PERF_FORMAT_ID)
3073
		values[n++] = primary_event_id(event);
3074 3075 3076 3077 3078

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
3079
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3080 3081
 */
static void perf_output_read_group(struct perf_output_handle *handle,
3082
			    struct perf_event *event)
3083
{
3084 3085
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = leader->total_time_enabled;

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = leader->total_time_running;

3097
	if (leader != event)
3098 3099 3100 3101
		leader->pmu->read(leader);

	values[n++] = atomic64_read(&leader->count);
	if (read_format & PERF_FORMAT_ID)
3102
		values[n++] = primary_event_id(leader);
3103 3104 3105

	perf_output_copy(handle, values, n * sizeof(u64));

3106
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3107 3108
		n = 0;

3109
		if (sub != event)
3110 3111 3112 3113
			sub->pmu->read(sub);

		values[n++] = atomic64_read(&sub->count);
		if (read_format & PERF_FORMAT_ID)
3114
			values[n++] = primary_event_id(sub);
3115 3116 3117 3118 3119 3120

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

static void perf_output_read(struct perf_output_handle *handle,
3121
			     struct perf_event *event)
3122
{
3123 3124
	if (event->attr.read_format & PERF_FORMAT_GROUP)
		perf_output_read_group(handle, event);
3125
	else
3126
		perf_output_read_one(handle, event);
3127 3128
}

3129 3130 3131
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3132
			struct perf_event *event)
3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3163
		perf_output_read(handle, event);
3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3201
			 struct perf_event *event,
3202
			 struct pt_regs *regs)
3203
{
3204
	u64 sample_type = event->attr.sample_type;
3205

3206
	data->type = sample_type;
3207

3208
	header->type = PERF_RECORD_SAMPLE;
3209 3210 3211 3212
	header->size = sizeof(*header);

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3213

3214
	if (sample_type & PERF_SAMPLE_IP) {
3215 3216 3217
		data->ip = perf_instruction_pointer(regs);

		header->size += sizeof(data->ip);
3218
	}
3219

3220
	if (sample_type & PERF_SAMPLE_TID) {
3221
		/* namespace issues */
3222 3223
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
3224

3225
		header->size += sizeof(data->tid_entry);
3226 3227
	}

3228
	if (sample_type & PERF_SAMPLE_TIME) {
P
Peter Zijlstra 已提交
3229
		data->time = perf_clock();
3230

3231
		header->size += sizeof(data->time);
3232 3233
	}

3234
	if (sample_type & PERF_SAMPLE_ADDR)
3235
		header->size += sizeof(data->addr);
3236

3237
	if (sample_type & PERF_SAMPLE_ID) {
3238
		data->id = primary_event_id(event);
3239

3240 3241 3242 3243
		header->size += sizeof(data->id);
	}

	if (sample_type & PERF_SAMPLE_STREAM_ID) {
3244
		data->stream_id = event->id;
3245 3246 3247

		header->size += sizeof(data->stream_id);
	}
3248

3249
	if (sample_type & PERF_SAMPLE_CPU) {
3250 3251
		data->cpu_entry.cpu		= raw_smp_processor_id();
		data->cpu_entry.reserved	= 0;
3252

3253
		header->size += sizeof(data->cpu_entry);
3254 3255
	}

3256
	if (sample_type & PERF_SAMPLE_PERIOD)
3257
		header->size += sizeof(data->period);
3258

3259
	if (sample_type & PERF_SAMPLE_READ)
3260
		header->size += perf_event_read_size(event);
3261

3262
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3263
		int size = 1;
3264

3265 3266 3267 3268 3269 3270
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3271 3272
	}

3273
	if (sample_type & PERF_SAMPLE_RAW) {
3274 3275 3276 3277 3278 3279 3280 3281
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3282
		header->size += size;
3283
	}
3284
}
3285

3286
static void perf_event_output(struct perf_event *event, int nmi,
3287 3288 3289 3290 3291
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3292

3293
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3294

3295
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3296
		return;
3297

3298
	perf_output_sample(&handle, &header, data, event);
3299

3300
	perf_output_end(&handle);
3301 3302
}

3303
/*
3304
 * read event_id
3305 3306 3307 3308 3309 3310 3311 3312 3313 3314
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3315
perf_event_read_event(struct perf_event *event,
3316 3317 3318
			struct task_struct *task)
{
	struct perf_output_handle handle;
3319
	struct perf_read_event read_event = {
3320
		.header = {
3321
			.type = PERF_RECORD_READ,
3322
			.misc = 0,
3323
			.size = sizeof(read_event) + perf_event_read_size(event),
3324
		},
3325 3326
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3327
	};
3328
	int ret;
3329

3330
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3331 3332 3333
	if (ret)
		return;

3334
	perf_output_put(&handle, read_event);
3335
	perf_output_read(&handle, event);
3336

3337 3338 3339
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3340
/*
P
Peter Zijlstra 已提交
3341 3342 3343
 * task tracking -- fork/exit
 *
 * enabled by: attr.comm | attr.mmap | attr.task
P
Peter Zijlstra 已提交
3344 3345
 */

P
Peter Zijlstra 已提交
3346
struct perf_task_event {
3347
	struct task_struct		*task;
3348
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3349 3350 3351 3352 3353 3354

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3355 3356
		u32				tid;
		u32				ptid;
3357
		u64				time;
3358
	} event_id;
P
Peter Zijlstra 已提交
3359 3360
};

3361
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3362
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3363 3364
{
	struct perf_output_handle handle;
3365
	int size;
P
Peter Zijlstra 已提交
3366
	struct task_struct *task = task_event->task;
3367 3368
	int ret;

3369 3370
	size  = task_event->event_id.header.size;
	ret = perf_output_begin(&handle, event, size, 0, 0);
P
Peter Zijlstra 已提交
3371 3372 3373 3374

	if (ret)
		return;

3375 3376
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3377

3378 3379
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3380

3381
	perf_output_put(&handle, task_event->event_id);
3382

P
Peter Zijlstra 已提交
3383 3384 3385
	perf_output_end(&handle);
}

3386
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3387
{
P
Peter Zijlstra 已提交
3388
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3389 3390
		return 0;

3391 3392 3393
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3394
	if (event->attr.comm || event->attr.mmap || event->attr.task)
P
Peter Zijlstra 已提交
3395 3396 3397 3398 3399
		return 1;

	return 0;
}

3400
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
3401
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3402
{
3403
	struct perf_event *event;
P
Peter Zijlstra 已提交
3404

3405 3406 3407
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
3408 3409 3410
	}
}

3411
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3412 3413
{
	struct perf_cpu_context *cpuctx;
3414
	struct perf_event_context *ctx = task_event->task_ctx;
P
Peter Zijlstra 已提交
3415

3416
	rcu_read_lock();
P
Peter Zijlstra 已提交
3417
	cpuctx = &get_cpu_var(perf_cpu_context);
3418
	perf_event_task_ctx(&cpuctx->ctx, task_event);
3419
	if (!ctx)
P
Peter Zijlstra 已提交
3420
		ctx = rcu_dereference(current->perf_event_ctxp);
P
Peter Zijlstra 已提交
3421
	if (ctx)
3422
		perf_event_task_ctx(ctx, task_event);
3423
	put_cpu_var(perf_cpu_context);
P
Peter Zijlstra 已提交
3424 3425 3426
	rcu_read_unlock();
}

3427 3428
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3429
			      int new)
P
Peter Zijlstra 已提交
3430
{
P
Peter Zijlstra 已提交
3431
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
3432

3433 3434 3435
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
3436 3437
		return;

P
Peter Zijlstra 已提交
3438
	task_event = (struct perf_task_event){
3439 3440
		.task	  = task,
		.task_ctx = task_ctx,
3441
		.event_id    = {
P
Peter Zijlstra 已提交
3442
			.header = {
3443
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3444
				.misc = 0,
3445
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
3446
			},
3447 3448
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
3449 3450
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
3451
			.time = perf_clock(),
P
Peter Zijlstra 已提交
3452 3453 3454
		},
	};

3455
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
3456 3457
}

3458
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
3459
{
3460
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
3461 3462
}

3463 3464 3465 3466 3467
/*
 * comm tracking
 */

struct perf_comm_event {
3468 3469
	struct task_struct	*task;
	char			*comm;
3470 3471 3472 3473 3474 3475 3476
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
3477
	} event_id;
3478 3479
};

3480
static void perf_event_comm_output(struct perf_event *event,
3481 3482 3483
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
3484 3485
	int size = comm_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3486 3487 3488 3489

	if (ret)
		return;

3490 3491
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3492

3493
	perf_output_put(&handle, comm_event->event_id);
3494 3495 3496 3497 3498
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
	perf_output_end(&handle);
}

3499
static int perf_event_comm_match(struct perf_event *event)
3500
{
P
Peter Zijlstra 已提交
3501
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3502 3503
		return 0;

3504 3505 3506
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3507
	if (event->attr.comm)
3508 3509 3510 3511 3512
		return 1;

	return 0;
}

3513
static void perf_event_comm_ctx(struct perf_event_context *ctx,
3514 3515
				  struct perf_comm_event *comm_event)
{
3516
	struct perf_event *event;
3517

3518 3519 3520
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
3521 3522 3523
	}
}

3524
static void perf_event_comm_event(struct perf_comm_event *comm_event)
3525 3526
{
	struct perf_cpu_context *cpuctx;
3527
	struct perf_event_context *ctx;
3528
	unsigned int size;
3529
	char comm[TASK_COMM_LEN];
3530

3531
	memset(comm, 0, sizeof(comm));
3532
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
3533
	size = ALIGN(strlen(comm)+1, sizeof(u64));
3534 3535 3536 3537

	comm_event->comm = comm;
	comm_event->comm_size = size;

3538
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3539

3540
	rcu_read_lock();
3541
	cpuctx = &get_cpu_var(perf_cpu_context);
3542 3543
	perf_event_comm_ctx(&cpuctx->ctx, comm_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3544
	if (ctx)
3545
		perf_event_comm_ctx(ctx, comm_event);
3546
	put_cpu_var(perf_cpu_context);
3547
	rcu_read_unlock();
3548 3549
}

3550
void perf_event_comm(struct task_struct *task)
3551
{
3552 3553
	struct perf_comm_event comm_event;

3554 3555
	if (task->perf_event_ctxp)
		perf_event_enable_on_exec(task);
3556

3557
	if (!atomic_read(&nr_comm_events))
3558
		return;
3559

3560
	comm_event = (struct perf_comm_event){
3561
		.task	= task,
3562 3563
		/* .comm      */
		/* .comm_size */
3564
		.event_id  = {
3565
			.header = {
3566
				.type = PERF_RECORD_COMM,
3567 3568 3569 3570 3571
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3572 3573 3574
		},
	};

3575
	perf_event_comm_event(&comm_event);
3576 3577
}

3578 3579 3580 3581 3582
/*
 * mmap tracking
 */

struct perf_mmap_event {
3583 3584 3585 3586
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
3587 3588 3589 3590 3591 3592 3593 3594 3595

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
3596
	} event_id;
3597 3598
};

3599
static void perf_event_mmap_output(struct perf_event *event,
3600 3601 3602
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
3603 3604
	int size = mmap_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3605 3606 3607 3608

	if (ret)
		return;

3609 3610
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
3611

3612
	perf_output_put(&handle, mmap_event->event_id);
3613 3614
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
3615
	perf_output_end(&handle);
3616 3617
}

3618
static int perf_event_mmap_match(struct perf_event *event,
3619 3620
				   struct perf_mmap_event *mmap_event)
{
P
Peter Zijlstra 已提交
3621
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3622 3623
		return 0;

3624 3625 3626
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3627
	if (event->attr.mmap)
3628 3629 3630 3631 3632
		return 1;

	return 0;
}

3633
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3634 3635
				  struct perf_mmap_event *mmap_event)
{
3636
	struct perf_event *event;
3637

3638 3639 3640
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_mmap_match(event, mmap_event))
			perf_event_mmap_output(event, mmap_event);
3641 3642 3643
	}
}

3644
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3645 3646
{
	struct perf_cpu_context *cpuctx;
3647
	struct perf_event_context *ctx;
3648 3649
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
3650 3651 3652
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
3653
	const char *name;
3654

3655 3656
	memset(tmp, 0, sizeof(tmp));

3657
	if (file) {
3658 3659 3660 3661 3662 3663
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3664 3665 3666 3667
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
3668
		name = d_path(&file->f_path, buf, PATH_MAX);
3669 3670 3671 3672 3673
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
3674 3675 3676
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
3677
			goto got_name;
3678
		}
3679 3680 3681 3682 3683 3684

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
		}

3685 3686 3687 3688 3689
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
3690
	size = ALIGN(strlen(name)+1, sizeof(u64));
3691 3692 3693 3694

	mmap_event->file_name = name;
	mmap_event->file_size = size;

3695
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3696

3697
	rcu_read_lock();
3698
	cpuctx = &get_cpu_var(perf_cpu_context);
3699 3700
	perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3701
	if (ctx)
3702
		perf_event_mmap_ctx(ctx, mmap_event);
3703
	put_cpu_var(perf_cpu_context);
3704 3705
	rcu_read_unlock();

3706 3707 3708
	kfree(buf);
}

3709
void __perf_event_mmap(struct vm_area_struct *vma)
3710
{
3711 3712
	struct perf_mmap_event mmap_event;

3713
	if (!atomic_read(&nr_mmap_events))
3714 3715 3716
		return;

	mmap_event = (struct perf_mmap_event){
3717
		.vma	= vma,
3718 3719
		/* .file_name */
		/* .file_size */
3720
		.event_id  = {
3721
			.header = {
3722
				.type = PERF_RECORD_MMAP,
3723 3724 3725 3726 3727
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3728 3729
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
3730
			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
3731 3732 3733
		},
	};

3734
	perf_event_mmap_event(&mmap_event);
3735 3736
}

3737 3738 3739 3740
/*
 * IRQ throttle logging
 */

3741
static void perf_log_throttle(struct perf_event *event, int enable)
3742 3743 3744 3745 3746 3747 3748
{
	struct perf_output_handle handle;
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
3749
		u64				id;
3750
		u64				stream_id;
3751 3752
	} throttle_event = {
		.header = {
3753
			.type = PERF_RECORD_THROTTLE,
3754 3755 3756
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
3757
		.time		= perf_clock(),
3758 3759
		.id		= primary_event_id(event),
		.stream_id	= event->id,
3760 3761
	};

3762
	if (enable)
3763
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3764

3765
	ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3766 3767 3768 3769 3770 3771 3772
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
	perf_output_end(&handle);
}

3773
/*
3774
 * Generic event overflow handling, sampling.
3775 3776
 */

3777
static int __perf_event_overflow(struct perf_event *event, int nmi,
3778 3779
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
3780
{
3781 3782
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
3783 3784
	int ret = 0;

3785
	throttle = (throttle && event->pmu->unthrottle != NULL);
3786

3787
	if (!throttle) {
3788
		hwc->interrupts++;
3789
	} else {
3790 3791
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
3792
			if (HZ * hwc->interrupts >
3793
					(u64)sysctl_perf_event_sample_rate) {
3794
				hwc->interrupts = MAX_INTERRUPTS;
3795
				perf_log_throttle(event, 0);
3796 3797 3798 3799
				ret = 1;
			}
		} else {
			/*
3800
			 * Keep re-disabling events even though on the previous
3801
			 * pass we disabled it - just in case we raced with a
3802
			 * sched-in and the event got enabled again:
3803
			 */
3804 3805 3806
			ret = 1;
		}
	}
3807

3808
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
3809
		u64 now = perf_clock();
3810
		s64 delta = now - hwc->freq_time_stamp;
3811

3812
		hwc->freq_time_stamp = now;
3813

3814 3815
		if (delta > 0 && delta < 2*TICK_NSEC)
			perf_adjust_period(event, delta, hwc->last_period);
3816 3817
	}

3818 3819
	/*
	 * XXX event_limit might not quite work as expected on inherited
3820
	 * events
3821 3822
	 */

3823 3824
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
3825
		ret = 1;
3826
		event->pending_kill = POLL_HUP;
3827
		if (nmi) {
3828 3829 3830
			event->pending_disable = 1;
			perf_pending_queue(&event->pending,
					   perf_pending_event);
3831
		} else
3832
			perf_event_disable(event);
3833 3834
	}

3835 3836 3837 3838 3839
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

3840
	return ret;
3841 3842
}

3843
int perf_event_overflow(struct perf_event *event, int nmi,
3844 3845
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
3846
{
3847
	return __perf_event_overflow(event, nmi, 1, data, regs);
3848 3849
}

3850
/*
3851
 * Generic software event infrastructure
3852 3853
 */

3854
/*
3855 3856
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
3857 3858 3859 3860
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

3861
static u64 perf_swevent_set_period(struct perf_event *event)
3862
{
3863
	struct hw_perf_event *hwc = &event->hw;
3864 3865 3866 3867 3868
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
3869 3870

again:
3871 3872 3873
	old = val = atomic64_read(&hwc->period_left);
	if (val < 0)
		return 0;
3874

3875 3876 3877 3878 3879
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
	if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
		goto again;
3880

3881
	return nr;
3882 3883
}

3884
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3885 3886
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
3887
{
3888
	struct hw_perf_event *hwc = &event->hw;
3889
	int throttle = 0;
3890

3891
	data->period = event->hw.last_period;
3892 3893
	if (!overflow)
		overflow = perf_swevent_set_period(event);
3894

3895 3896
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
3897

3898
	for (; overflow; overflow--) {
3899
		if (__perf_event_overflow(event, nmi, throttle,
3900
					    data, regs)) {
3901 3902 3903 3904 3905 3906
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
3907
		throttle = 1;
3908
	}
3909 3910
}

3911
static void perf_swevent_unthrottle(struct perf_event *event)
3912 3913
{
	/*
3914
	 * Nothing to do, we already reset hwc->interrupts.
3915
	 */
3916
}
3917

3918
static void perf_swevent_add(struct perf_event *event, u64 nr,
3919 3920
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
3921
{
3922
	struct hw_perf_event *hwc = &event->hw;
3923

3924
	atomic64_add(nr, &event->count);
3925

3926 3927 3928
	if (!regs)
		return;

3929 3930
	if (!hwc->sample_period)
		return;
3931

3932 3933 3934 3935
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

	if (atomic64_add_negative(nr, &hwc->period_left))
3936
		return;
3937

3938
	perf_swevent_overflow(event, 0, nmi, data, regs);
3939 3940
}

3941
static int perf_swevent_is_counting(struct perf_event *event)
3942
{
3943
	/*
3944
	 * The event is active, we're good!
3945
	 */
3946
	if (event->state == PERF_EVENT_STATE_ACTIVE)
3947 3948
		return 1;

3949
	/*
3950
	 * The event is off/error, not counting.
3951
	 */
3952
	if (event->state != PERF_EVENT_STATE_INACTIVE)
3953 3954 3955
		return 0;

	/*
3956
	 * The event is inactive, if the context is active
3957 3958
	 * we're part of a group that didn't make it on the 'pmu',
	 * not counting.
3959
	 */
3960
	if (event->ctx->is_active)
3961 3962 3963 3964 3965 3966 3967 3968
		return 0;

	/*
	 * We're inactive and the context is too, this means the
	 * task is scheduled out, we're counting events that happen
	 * to us, like migration events.
	 */
	return 1;
3969 3970
}

L
Li Zefan 已提交
3971 3972 3973
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data);

3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987
static int perf_exclude_event(struct perf_event *event,
			      struct pt_regs *regs)
{
	if (regs) {
		if (event->attr.exclude_user && user_mode(regs))
			return 1;

		if (event->attr.exclude_kernel && !user_mode(regs))
			return 1;
	}

	return 0;
}

3988
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
3989
				enum perf_type_id type,
L
Li Zefan 已提交
3990 3991 3992
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
3993
{
3994 3995 3996
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3997
	if (!perf_swevent_is_counting(event))
3998 3999
		return 0;

4000
	if (event->attr.type != type)
4001
		return 0;
4002

4003
	if (event->attr.config != event_id)
4004 4005
		return 0;

4006 4007
	if (perf_exclude_event(event, regs))
		return 0;
4008

L
Li Zefan 已提交
4009 4010 4011 4012
	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
	    !perf_tp_event_match(event, data))
		return 0;

4013 4014 4015
	return 1;
}

4016
static void perf_swevent_ctx_event(struct perf_event_context *ctx,
4017
				     enum perf_type_id type,
4018
				     u32 event_id, u64 nr, int nmi,
4019 4020
				     struct perf_sample_data *data,
				     struct pt_regs *regs)
4021
{
4022
	struct perf_event *event;
4023

4024
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
L
Li Zefan 已提交
4025
		if (perf_swevent_match(event, type, event_id, data, regs))
4026
			perf_swevent_add(event, nr, nmi, data, regs);
4027 4028 4029
	}
}

4030
int perf_swevent_get_recursion_context(void)
P
Peter Zijlstra 已提交
4031
{
4032 4033
	struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
	int rctx;
4034

P
Peter Zijlstra 已提交
4035
	if (in_nmi())
4036
		rctx = 3;
4037
	else if (in_irq())
4038
		rctx = 2;
4039
	else if (in_softirq())
4040
		rctx = 1;
4041
	else
4042
		rctx = 0;
P
Peter Zijlstra 已提交
4043

4044 4045
	if (cpuctx->recursion[rctx]) {
		put_cpu_var(perf_cpu_context);
4046
		return -1;
4047
	}
P
Peter Zijlstra 已提交
4048

4049 4050
	cpuctx->recursion[rctx]++;
	barrier();
P
Peter Zijlstra 已提交
4051

4052
	return rctx;
P
Peter Zijlstra 已提交
4053
}
I
Ingo Molnar 已提交
4054
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
4055

4056
void perf_swevent_put_recursion_context(int rctx)
4057
{
4058 4059
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	barrier();
4060
	cpuctx->recursion[rctx]--;
4061
	put_cpu_var(perf_cpu_context);
4062
}
I
Ingo Molnar 已提交
4063
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
P
Peter Zijlstra 已提交
4064

4065 4066 4067 4068
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
				    u64 nr, int nmi,
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
4069
{
4070
	struct perf_cpu_context *cpuctx;
4071
	struct perf_event_context *ctx;
4072

4073
	cpuctx = &__get_cpu_var(perf_cpu_context);
4074
	rcu_read_lock();
4075
	perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
4076
				 nr, nmi, data, regs);
4077 4078 4079 4080
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
4081
	ctx = rcu_dereference(current->perf_event_ctxp);
4082
	if (ctx)
4083
		perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs);
4084
	rcu_read_unlock();
4085
}
4086

4087
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4088
			    struct pt_regs *regs, u64 addr)
4089
{
4090
	struct perf_sample_data data;
4091 4092 4093 4094 4095
	int rctx;

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return;
4096

4097
	perf_sample_data_init(&data, addr);
4098

4099
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4100 4101

	perf_swevent_put_recursion_context(rctx);
4102 4103
}

4104
static void perf_swevent_read(struct perf_event *event)
4105 4106 4107
{
}

4108
static int perf_swevent_enable(struct perf_event *event)
4109
{
4110
	struct hw_perf_event *hwc = &event->hw;
4111 4112 4113

	if (hwc->sample_period) {
		hwc->last_period = hwc->sample_period;
4114
		perf_swevent_set_period(event);
4115
	}
4116 4117 4118
	return 0;
}

4119
static void perf_swevent_disable(struct perf_event *event)
4120 4121 4122
{
}

4123
static const struct pmu perf_ops_generic = {
4124 4125 4126 4127
	.enable		= perf_swevent_enable,
	.disable	= perf_swevent_disable,
	.read		= perf_swevent_read,
	.unthrottle	= perf_swevent_unthrottle,
4128 4129
};

4130
/*
4131
 * hrtimer based swevent callback
4132 4133
 */

4134
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4135 4136 4137
{
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
4138
	struct pt_regs *regs;
4139
	struct perf_event *event;
4140 4141
	u64 period;

4142
	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
4143
	event->pmu->read(event);
4144

4145
	perf_sample_data_init(&data, 0);
4146
	data.period = event->hw.last_period;
4147
	regs = get_irq_regs();
4148 4149 4150 4151
	/*
	 * In case we exclude kernel IPs or are somehow not in interrupt
	 * context, provide the next best thing, the user IP.
	 */
4152 4153
	if ((event->attr.exclude_kernel || !regs) &&
			!event->attr.exclude_user)
4154
		regs = task_pt_regs(current);
4155

4156
	if (regs) {
4157 4158 4159
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
4160 4161
	}

4162
	period = max_t(u64, 10000, event->hw.sample_period);
4163 4164 4165 4166 4167
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));

	return ret;
}

4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203
static void perf_swevent_start_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
	if (hwc->sample_period) {
		u64 period;

		if (hwc->remaining) {
			if (hwc->remaining < 0)
				period = 10000;
			else
				period = hwc->remaining;
			hwc->remaining = 0;
		} else {
			period = max_t(u64, 10000, hwc->sample_period);
		}
		__hrtimer_start_range_ns(&hwc->hrtimer,
				ns_to_ktime(period), 0,
				HRTIMER_MODE_REL, 0);
	}
}

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (hwc->sample_period) {
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
		hwc->remaining = ktime_to_ns(remaining);

		hrtimer_cancel(&hwc->hrtimer);
	}
}

4204
/*
4205
 * Software event: cpu wall time clock
4206 4207
 */

4208
static void cpu_clock_perf_event_update(struct perf_event *event)
4209 4210 4211 4212 4213 4214
{
	int cpu = raw_smp_processor_id();
	s64 prev;
	u64 now;

	now = cpu_clock(cpu);
4215
	prev = atomic64_xchg(&event->hw.prev_count, now);
4216
	atomic64_add(now - prev, &event->count);
4217 4218
}

4219
static int cpu_clock_perf_event_enable(struct perf_event *event)
4220
{
4221
	struct hw_perf_event *hwc = &event->hw;
4222 4223 4224
	int cpu = raw_smp_processor_id();

	atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4225
	perf_swevent_start_hrtimer(event);
4226 4227 4228 4229

	return 0;
}

4230
static void cpu_clock_perf_event_disable(struct perf_event *event)
4231
{
4232
	perf_swevent_cancel_hrtimer(event);
4233
	cpu_clock_perf_event_update(event);
4234 4235
}

4236
static void cpu_clock_perf_event_read(struct perf_event *event)
4237
{
4238
	cpu_clock_perf_event_update(event);
4239 4240
}

4241
static const struct pmu perf_ops_cpu_clock = {
4242 4243 4244
	.enable		= cpu_clock_perf_event_enable,
	.disable	= cpu_clock_perf_event_disable,
	.read		= cpu_clock_perf_event_read,
4245 4246
};

4247
/*
4248
 * Software event: task time clock
4249 4250
 */

4251
static void task_clock_perf_event_update(struct perf_event *event, u64 now)
I
Ingo Molnar 已提交
4252
{
4253
	u64 prev;
I
Ingo Molnar 已提交
4254 4255
	s64 delta;

4256
	prev = atomic64_xchg(&event->hw.prev_count, now);
I
Ingo Molnar 已提交
4257
	delta = now - prev;
4258
	atomic64_add(delta, &event->count);
4259 4260
}

4261
static int task_clock_perf_event_enable(struct perf_event *event)
I
Ingo Molnar 已提交
4262
{
4263
	struct hw_perf_event *hwc = &event->hw;
4264 4265
	u64 now;

4266
	now = event->ctx->time;
4267

4268
	atomic64_set(&hwc->prev_count, now);
4269 4270

	perf_swevent_start_hrtimer(event);
4271 4272

	return 0;
I
Ingo Molnar 已提交
4273 4274
}

4275
static void task_clock_perf_event_disable(struct perf_event *event)
4276
{
4277
	perf_swevent_cancel_hrtimer(event);
4278
	task_clock_perf_event_update(event, event->ctx->time);
4279

4280
}
I
Ingo Molnar 已提交
4281

4282
static void task_clock_perf_event_read(struct perf_event *event)
4283
{
4284 4285 4286
	u64 time;

	if (!in_nmi()) {
4287 4288
		update_context_time(event->ctx);
		time = event->ctx->time;
4289 4290
	} else {
		u64 now = perf_clock();
4291 4292
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
4293 4294
	}

4295
	task_clock_perf_event_update(event, time);
4296 4297
}

4298
static const struct pmu perf_ops_task_clock = {
4299 4300 4301
	.enable		= task_clock_perf_event_enable,
	.disable	= task_clock_perf_event_disable,
	.read		= task_clock_perf_event_read,
4302 4303
};

4304
#ifdef CONFIG_EVENT_TRACING
L
Li Zefan 已提交
4305

4306
void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4307
			  int entry_size)
4308
{
4309 4310
	struct pt_regs *regs = get_irq_regs();
	struct perf_sample_data data;
4311
	struct perf_raw_record raw = {
4312
		.size = entry_size,
4313
		.data = record,
4314 4315
	};

4316 4317
	perf_sample_data_init(&data, addr);
	data.raw = &raw;
4318 4319 4320

	if (!regs)
		regs = task_pt_regs(current);
4321

4322
	/* Trace events already protected against recursion */
4323
	do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4324
				&data, regs);
4325
}
4326
EXPORT_SYMBOL_GPL(perf_tp_event);
4327

L
Li Zefan 已提交
4328 4329 4330 4331 4332 4333 4334 4335 4336
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}
4337

4338
static void tp_perf_event_destroy(struct perf_event *event)
4339
{
4340
	ftrace_profile_disable(event->attr.config);
4341 4342
}

4343
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4344
{
4345 4346 4347 4348
	/*
	 * Raw tracepoint data is a severe data leak, only allow root to
	 * have these.
	 */
4349
	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4350
			perf_paranoid_tracepoint_raw() &&
4351 4352 4353
			!capable(CAP_SYS_ADMIN))
		return ERR_PTR(-EPERM);

4354
	if (ftrace_profile_enable(event->attr.config))
4355 4356
		return NULL;

4357
	event->destroy = tp_perf_event_destroy;
4358 4359 4360

	return &perf_ops_generic;
}
L
Li Zefan 已提交
4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

4385
#else
L
Li Zefan 已提交
4386 4387 4388 4389 4390 4391 4392

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	return 1;
}

4393
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4394 4395 4396
{
	return NULL;
}
L
Li Zefan 已提交
4397 4398 4399 4400 4401 4402 4403 4404 4405 4406

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

4407
#endif /* CONFIG_EVENT_TRACING */
4408

4409 4410 4411 4412 4413 4414 4415 4416 4417
#ifdef CONFIG_HAVE_HW_BREAKPOINT
static void bp_perf_event_destroy(struct perf_event *event)
{
	release_bp_slot(event);
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	int err;
4418 4419

	err = register_perf_hw_breakpoint(bp);
4420 4421 4422 4423 4424 4425 4426 4427
	if (err)
		return ERR_PTR(err);

	bp->destroy = bp_perf_event_destroy;

	return &perf_ops_bp;
}

4428
void perf_bp_event(struct perf_event *bp, void *data)
4429
{
4430 4431 4432
	struct perf_sample_data sample;
	struct pt_regs *regs = data;

4433
	perf_sample_data_init(&sample, bp->attr.bp_addr);
4434 4435 4436

	if (!perf_exclude_event(bp, regs))
		perf_swevent_add(bp, 1, 1, &sample, regs);
4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448
}
#else
static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	return NULL;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
}
#endif

4449
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4450

4451
static void sw_perf_event_destroy(struct perf_event *event)
4452
{
4453
	u64 event_id = event->attr.config;
4454

4455
	WARN_ON(event->parent);
4456

4457
	atomic_dec(&perf_swevent_enabled[event_id]);
4458 4459
}

4460
static const struct pmu *sw_perf_event_init(struct perf_event *event)
4461
{
4462
	const struct pmu *pmu = NULL;
4463
	u64 event_id = event->attr.config;
4464

4465
	/*
4466
	 * Software events (currently) can't in general distinguish
4467 4468 4469 4470 4471
	 * between user, kernel and hypervisor events.
	 * However, context switches and cpu migrations are considered
	 * to be kernel events, and page faults are never hypervisor
	 * events.
	 */
4472
	switch (event_id) {
4473
	case PERF_COUNT_SW_CPU_CLOCK:
4474
		pmu = &perf_ops_cpu_clock;
4475

4476
		break;
4477
	case PERF_COUNT_SW_TASK_CLOCK:
4478
		/*
4479 4480
		 * If the user instantiates this as a per-cpu event,
		 * use the cpu_clock event instead.
4481
		 */
4482
		if (event->ctx->task)
4483
			pmu = &perf_ops_task_clock;
4484
		else
4485
			pmu = &perf_ops_cpu_clock;
4486

4487
		break;
4488 4489 4490 4491 4492
	case PERF_COUNT_SW_PAGE_FAULTS:
	case PERF_COUNT_SW_PAGE_FAULTS_MIN:
	case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
	case PERF_COUNT_SW_CONTEXT_SWITCHES:
	case PERF_COUNT_SW_CPU_MIGRATIONS:
4493 4494
	case PERF_COUNT_SW_ALIGNMENT_FAULTS:
	case PERF_COUNT_SW_EMULATION_FAULTS:
4495 4496 4497
		if (!event->parent) {
			atomic_inc(&perf_swevent_enabled[event_id]);
			event->destroy = sw_perf_event_destroy;
4498
		}
4499
		pmu = &perf_ops_generic;
4500
		break;
4501
	}
4502

4503
	return pmu;
4504 4505
}

T
Thomas Gleixner 已提交
4506
/*
4507
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
4508
 */
4509 4510
static struct perf_event *
perf_event_alloc(struct perf_event_attr *attr,
4511
		   int cpu,
4512 4513 4514
		   struct perf_event_context *ctx,
		   struct perf_event *group_leader,
		   struct perf_event *parent_event,
4515
		   perf_overflow_handler_t overflow_handler,
4516
		   gfp_t gfpflags)
T
Thomas Gleixner 已提交
4517
{
4518
	const struct pmu *pmu;
4519 4520
	struct perf_event *event;
	struct hw_perf_event *hwc;
4521
	long err;
T
Thomas Gleixner 已提交
4522

4523 4524
	event = kzalloc(sizeof(*event), gfpflags);
	if (!event)
4525
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
4526

4527
	/*
4528
	 * Single events are their own group leaders, with an
4529 4530 4531
	 * empty sibling list:
	 */
	if (!group_leader)
4532
		group_leader = event;
4533

4534 4535
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
4536

4537 4538 4539 4540
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
T
Thomas Gleixner 已提交
4541

4542
	mutex_init(&event->mmap_mutex);
4543

4544 4545 4546 4547 4548 4549
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->ctx		= ctx;
	event->oncpu		= -1;
4550

4551
	event->parent		= parent_event;
4552

4553 4554
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
4555

4556
	event->state		= PERF_EVENT_STATE_INACTIVE;
4557

4558 4559
	if (!overflow_handler && parent_event)
		overflow_handler = parent_event->overflow_handler;
4560
	
4561
	event->overflow_handler	= overflow_handler;
4562

4563
	if (attr->disabled)
4564
		event->state = PERF_EVENT_STATE_OFF;
4565

4566
	pmu = NULL;
4567

4568
	hwc = &event->hw;
4569
	hwc->sample_period = attr->sample_period;
4570
	if (attr->freq && attr->sample_freq)
4571
		hwc->sample_period = 1;
4572
	hwc->last_period = hwc->sample_period;
4573 4574

	atomic64_set(&hwc->period_left, hwc->sample_period);
4575

4576
	/*
4577
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
4578
	 */
4579
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4580 4581
		goto done;

4582
	switch (attr->type) {
4583
	case PERF_TYPE_RAW:
4584
	case PERF_TYPE_HARDWARE:
4585
	case PERF_TYPE_HW_CACHE:
4586
		pmu = hw_perf_event_init(event);
4587 4588 4589
		break;

	case PERF_TYPE_SOFTWARE:
4590
		pmu = sw_perf_event_init(event);
4591 4592 4593
		break;

	case PERF_TYPE_TRACEPOINT:
4594
		pmu = tp_perf_event_init(event);
4595
		break;
4596

4597 4598 4599 4600 4601
	case PERF_TYPE_BREAKPOINT:
		pmu = bp_perf_event_init(event);
		break;


4602 4603
	default:
		break;
4604
	}
4605 4606
done:
	err = 0;
4607
	if (!pmu)
4608
		err = -EINVAL;
4609 4610
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
4611

4612
	if (err) {
4613 4614 4615
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
4616
		return ERR_PTR(err);
I
Ingo Molnar 已提交
4617
	}
4618

4619
	event->pmu = pmu;
T
Thomas Gleixner 已提交
4620

4621 4622 4623 4624 4625 4626 4627 4628
	if (!event->parent) {
		atomic_inc(&nr_events);
		if (event->attr.mmap)
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
4629
	}
4630

4631
	return event;
T
Thomas Gleixner 已提交
4632 4633
}

4634 4635
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
4636 4637
{
	u32 size;
4638
	int ret;
4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
4663 4664 4665
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
4666 4667
	 */
	if (size > sizeof(*attr)) {
4668 4669 4670
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
4671

4672 4673
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
4674

4675
		for (; addr < end; addr++) {
4676 4677 4678 4679 4680 4681
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
4682
		size = sizeof(*attr);
4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

4696
	if (attr->__reserved_1)
4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Li Zefan 已提交
4714
static int perf_event_set_output(struct perf_event *event, int output_fd)
4715
{
4716
	struct perf_event *output_event = NULL;
4717
	struct file *output_file = NULL;
4718
	struct perf_event *old_output;
4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731
	int fput_needed = 0;
	int ret = -EINVAL;

	if (!output_fd)
		goto set;

	output_file = fget_light(output_fd, &fput_needed);
	if (!output_file)
		return -EBADF;

	if (output_file->f_op != &perf_fops)
		goto out;

4732
	output_event = output_file->private_data;
4733 4734

	/* Don't chain output fds */
4735
	if (output_event->output)
4736 4737 4738
		goto out;

	/* Don't set an output fd when we already have an output channel */
4739
	if (event->data)
4740 4741 4742 4743 4744
		goto out;

	atomic_long_inc(&output_file->f_count);

set:
4745 4746 4747 4748
	mutex_lock(&event->mmap_mutex);
	old_output = event->output;
	rcu_assign_pointer(event->output, output_event);
	mutex_unlock(&event->mmap_mutex);
4749 4750 4751 4752

	if (old_output) {
		/*
		 * we need to make sure no existing perf_output_*()
4753
		 * is still referencing this event.
4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764
		 */
		synchronize_rcu();
		fput(old_output->filp);
	}

	ret = 0;
out:
	fput_light(output_file, fput_needed);
	return ret;
}

T
Thomas Gleixner 已提交
4765
/**
4766
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
4767
 *
4768
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
4769
 * @pid:		target pid
I
Ingo Molnar 已提交
4770
 * @cpu:		target cpu
4771
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
4772
 */
4773 4774
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
4775
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
4776
{
4777 4778 4779 4780
	struct perf_event *event, *group_leader;
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
4781 4782
	struct file *group_file = NULL;
	int fput_needed = 0;
4783
	int fput_needed2 = 0;
4784
	int err;
T
Thomas Gleixner 已提交
4785

4786
	/* for future expandability... */
4787
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4788 4789
		return -EINVAL;

4790 4791 4792
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
4793

4794 4795 4796 4797 4798
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

4799
	if (attr.freq) {
4800
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
4801 4802 4803
			return -EINVAL;
	}

4804
	/*
I
Ingo Molnar 已提交
4805 4806 4807 4808 4809 4810 4811
	 * Get the target context (task or percpu):
	 */
	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

	/*
4812
	 * Look up the group leader (we will attach this event to it):
4813 4814
	 */
	group_leader = NULL;
4815
	if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4816
		err = -EINVAL;
4817 4818
		group_file = fget_light(group_fd, &fput_needed);
		if (!group_file)
I
Ingo Molnar 已提交
4819
			goto err_put_context;
4820
		if (group_file->f_op != &perf_fops)
I
Ingo Molnar 已提交
4821
			goto err_put_context;
4822 4823 4824

		group_leader = group_file->private_data;
		/*
I
Ingo Molnar 已提交
4825 4826 4827 4828 4829 4830 4831 4832
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
			goto err_put_context;
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
4833
		 */
I
Ingo Molnar 已提交
4834 4835
		if (group_leader->ctx != ctx)
			goto err_put_context;
4836 4837 4838
		/*
		 * Only a group leader can be exclusive or pinned
		 */
4839
		if (attr.exclusive || attr.pinned)
4840
			goto err_put_context;
4841 4842
	}

4843
	event = perf_event_alloc(&attr, cpu, ctx, group_leader,
4844
				     NULL, NULL, GFP_KERNEL);
4845 4846
	err = PTR_ERR(event);
	if (IS_ERR(event))
T
Thomas Gleixner 已提交
4847 4848
		goto err_put_context;

4849
	err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
4850
	if (err < 0)
4851 4852
		goto err_free_put_context;

4853 4854
	event_file = fget_light(err, &fput_needed2);
	if (!event_file)
4855 4856
		goto err_free_put_context;

4857
	if (flags & PERF_FLAG_FD_OUTPUT) {
4858
		err = perf_event_set_output(event, group_fd);
4859 4860
		if (err)
			goto err_fput_free_put_context;
4861 4862
	}

4863
	event->filp = event_file;
4864
	WARN_ON_ONCE(ctx->parent_ctx);
4865
	mutex_lock(&ctx->mutex);
4866
	perf_install_in_context(ctx, event, cpu);
4867
	++ctx->generation;
4868
	mutex_unlock(&ctx->mutex);
4869

4870
	event->owner = current;
4871
	get_task_struct(current);
4872 4873 4874
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
4875

4876
err_fput_free_put_context:
4877
	fput_light(event_file, fput_needed2);
T
Thomas Gleixner 已提交
4878

4879
err_free_put_context:
4880
	if (err < 0)
4881
		kfree(event);
T
Thomas Gleixner 已提交
4882 4883

err_put_context:
4884 4885 4886 4887
	if (err < 0)
		put_ctx(ctx);

	fput_light(group_file, fput_needed);
T
Thomas Gleixner 已提交
4888

4889
	return err;
T
Thomas Gleixner 已提交
4890 4891
}

4892 4893 4894 4895 4896 4897 4898 4899 4900
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
 * @pid: task to profile
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4901 4902
				 pid_t pid,
				 perf_overflow_handler_t overflow_handler)
4903 4904 4905 4906 4907 4908 4909 4910 4911 4912
{
	struct perf_event *event;
	struct perf_event_context *ctx;
	int err;

	/*
	 * Get the target context (task or percpu):
	 */

	ctx = find_get_context(pid, cpu);
4913 4914 4915 4916
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
		goto err_exit;
	}
4917 4918

	event = perf_event_alloc(attr, cpu, ctx, NULL,
4919
				 NULL, overflow_handler, GFP_KERNEL);
4920 4921
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
4922
		goto err_put_context;
4923
	}
4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
	mutex_unlock(&ctx->mutex);

	event->owner = current;
	get_task_struct(current);
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);

	return event;

4940 4941 4942 4943
 err_put_context:
	put_ctx(ctx);
 err_exit:
	return ERR_PTR(err);
4944 4945 4946
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);

4947
/*
4948
 * inherit a event from parent task to child task:
4949
 */
4950 4951
static struct perf_event *
inherit_event(struct perf_event *parent_event,
4952
	      struct task_struct *parent,
4953
	      struct perf_event_context *parent_ctx,
4954
	      struct task_struct *child,
4955 4956
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
4957
{
4958
	struct perf_event *child_event;
4959

4960
	/*
4961 4962
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
4963 4964 4965
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
4966 4967
	if (parent_event->parent)
		parent_event = parent_event->parent;
4968

4969 4970 4971
	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu, child_ctx,
					   group_leader, parent_event,
4972
					   NULL, GFP_KERNEL);
4973 4974
	if (IS_ERR(child_event))
		return child_event;
4975
	get_ctx(child_ctx);
4976

4977
	/*
4978
	 * Make the child state follow the state of the parent event,
4979
	 * not its attr.disabled bit.  We hold the parent's mutex,
4980
	 * so we won't race with perf_event_{en, dis}able_family.
4981
	 */
4982 4983
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
4984
	else
4985
		child_event->state = PERF_EVENT_STATE_OFF;
4986

4987 4988 4989 4990 4991 4992 4993 4994 4995
	if (parent_event->attr.freq) {
		u64 sample_period = parent_event->hw.sample_period;
		struct hw_perf_event *hwc = &child_event->hw;

		hwc->sample_period = sample_period;
		hwc->last_period   = sample_period;

		atomic64_set(&hwc->period_left, sample_period);
	}
4996

4997 4998
	child_event->overflow_handler = parent_event->overflow_handler;

4999 5000 5001
	/*
	 * Link it up in the child's context:
	 */
5002
	add_event_to_ctx(child_event, child_ctx);
5003 5004 5005

	/*
	 * Get a reference to the parent filp - we will fput it
5006
	 * when the child event exits. This is safe to do because
5007 5008 5009
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
5010
	atomic_long_inc(&parent_event->filp->f_count);
5011

5012
	/*
5013
	 * Link this into the parent event's child list
5014
	 */
5015 5016 5017 5018
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
5019

5020
	return child_event;
5021 5022
}

5023
static int inherit_group(struct perf_event *parent_event,
5024
	      struct task_struct *parent,
5025
	      struct perf_event_context *parent_ctx,
5026
	      struct task_struct *child,
5027
	      struct perf_event_context *child_ctx)
5028
{
5029 5030 5031
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;
5032

5033
	leader = inherit_event(parent_event, parent, parent_ctx,
5034
				 child, NULL, child_ctx);
5035 5036
	if (IS_ERR(leader))
		return PTR_ERR(leader);
5037 5038
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
5039 5040 5041
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
5042
	}
5043 5044 5045
	return 0;
}

5046
static void sync_child_event(struct perf_event *child_event,
5047
			       struct task_struct *child)
5048
{
5049
	struct perf_event *parent_event = child_event->parent;
5050
	u64 child_val;
5051

5052 5053
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
5054

5055
	child_val = atomic64_read(&child_event->count);
5056 5057 5058 5059

	/*
	 * Add back the child's count to the parent's count:
	 */
5060 5061 5062 5063 5064
	atomic64_add(child_val, &parent_event->count);
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
5065 5066

	/*
5067
	 * Remove this event from the parent's list
5068
	 */
5069 5070 5071 5072
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
5073 5074

	/*
5075
	 * Release the parent event, if this was the last
5076 5077
	 * reference to it.
	 */
5078
	fput(parent_event->filp);
5079 5080
}

5081
static void
5082 5083
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
5084
			 struct task_struct *child)
5085
{
5086
	struct perf_event *parent_event;
5087

5088
	perf_event_remove_from_context(child_event);
5089

5090
	parent_event = child_event->parent;
5091
	/*
5092
	 * It can happen that parent exits first, and has events
5093
	 * that are still around due to the child reference. These
5094
	 * events need to be zapped - but otherwise linger.
5095
	 */
5096 5097 5098
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
5099
	}
5100 5101 5102
}

/*
5103
 * When a child task exits, feed back event values to parent events.
5104
 */
5105
void perf_event_exit_task(struct task_struct *child)
5106
{
5107 5108
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
5109
	unsigned long flags;
5110

5111 5112
	if (likely(!child->perf_event_ctxp)) {
		perf_event_task(child, NULL, 0);
5113
		return;
P
Peter Zijlstra 已提交
5114
	}
5115

5116
	local_irq_save(flags);
5117 5118 5119 5120 5121 5122
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
5123 5124
	child_ctx = child->perf_event_ctxp;
	__perf_event_task_sched_out(child_ctx);
5125 5126 5127

	/*
	 * Take the context lock here so that if find_get_context is
5128
	 * reading child->perf_event_ctxp, we wait until it has
5129 5130
	 * incremented the context's refcount before we do put_ctx below.
	 */
5131
	raw_spin_lock(&child_ctx->lock);
5132
	child->perf_event_ctxp = NULL;
5133 5134 5135
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
5136
	 * the events from it.
5137 5138
	 */
	unclone_ctx(child_ctx);
5139
	update_context_time(child_ctx);
5140
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
5141 5142

	/*
5143 5144 5145
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
5146
	 */
5147
	perf_event_task(child, child_ctx, 0);
5148

5149 5150 5151
	/*
	 * We can recurse on the same lock type through:
	 *
5152 5153 5154
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
5155 5156 5157 5158 5159 5160
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
	mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
5161

5162
again:
5163 5164 5165 5166 5167
	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
				 group_entry)
		__perf_event_exit_task(child_event, child_ctx, child);

	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
5168
				 group_entry)
5169
		__perf_event_exit_task(child_event, child_ctx, child);
5170 5171

	/*
5172
	 * If the last event was a group event, it will have appended all
5173 5174 5175
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
5176 5177
	if (!list_empty(&child_ctx->pinned_groups) ||
	    !list_empty(&child_ctx->flexible_groups))
5178
		goto again;
5179 5180 5181 5182

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
5183 5184
}

5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202
static void perf_free_event(struct perf_event *event,
			    struct perf_event_context *ctx)
{
	struct perf_event *parent = event->parent;

	if (WARN_ON_ONCE(!parent))
		return;

	mutex_lock(&parent->child_mutex);
	list_del_init(&event->child_list);
	mutex_unlock(&parent->child_mutex);

	fput(parent->filp);

	list_del_event(event, ctx);
	free_event(event);
}

5203 5204 5205 5206
/*
 * free an unexposed, unused context as created by inheritance by
 * init_task below, used by fork() in case of fail.
 */
5207
void perf_event_free_task(struct task_struct *task)
5208
{
5209 5210
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event *event, *tmp;
5211 5212 5213 5214 5215 5216

	if (!ctx)
		return;

	mutex_lock(&ctx->mutex);
again:
5217 5218
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		perf_free_event(event, ctx);
5219

5220 5221 5222
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
				 group_entry)
		perf_free_event(event, ctx);
5223

5224 5225 5226
	if (!list_empty(&ctx->pinned_groups) ||
	    !list_empty(&ctx->flexible_groups))
		goto again;
5227

5228
	mutex_unlock(&ctx->mutex);
5229

5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244
	put_ctx(ctx);
}

static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
		   struct perf_event_context *parent_ctx,
		   struct task_struct *child,
		   int *inherited_all)
{
	int ret;
	struct perf_event_context *child_ctx = child->perf_event_ctxp;

	if (!event->attr.inherit) {
		*inherited_all = 0;
		return 0;
5245 5246
	}

5247 5248 5249 5250 5251 5252 5253
	if (!child_ctx) {
		/*
		 * This is executed from the parent task context, so
		 * inherit events that have been marked for cloning.
		 * First allocate and initialize a context for the
		 * child.
		 */
5254

5255 5256 5257 5258
		child_ctx = kzalloc(sizeof(struct perf_event_context),
				    GFP_KERNEL);
		if (!child_ctx)
			return -ENOMEM;
5259

5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271
		__perf_event_init_context(child_ctx, child);
		child->perf_event_ctxp = child_ctx;
		get_task_struct(child);
	}

	ret = inherit_group(event, parent, parent_ctx,
			    child, child_ctx);

	if (ret)
		*inherited_all = 0;

	return ret;
5272 5273
}

5274

5275
/*
5276
 * Initialize the perf_event context in task_struct
5277
 */
5278
int perf_event_init_task(struct task_struct *child)
5279
{
5280
	struct perf_event_context *child_ctx, *parent_ctx;
5281 5282
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
5283
	struct task_struct *parent = current;
5284
	int inherited_all = 1;
5285
	int ret = 0;
5286

5287
	child->perf_event_ctxp = NULL;
5288

5289 5290
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);
5291

5292
	if (likely(!parent->perf_event_ctxp))
5293 5294
		return 0;

5295
	/*
5296 5297
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
5298
	 */
5299 5300
	parent_ctx = perf_pin_task_context(parent);

5301 5302 5303 5304 5305 5306 5307
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

5308 5309 5310 5311
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
5312
	mutex_lock(&parent_ctx->mutex);
5313 5314 5315 5316 5317

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
5318 5319 5320 5321 5322 5323
	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
			break;
	}
5324

5325 5326 5327 5328
	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
5329
			break;
5330 5331
	}

5332 5333
	child_ctx = child->perf_event_ctxp;

5334
	if (child_ctx && inherited_all) {
5335 5336 5337
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
5338 5339
		 * Note that if the parent is a clone, it could get
		 * uncloned at any point, but that doesn't matter
5340
		 * because the list of events and the generation
5341
		 * count can't have changed since we took the mutex.
5342
		 */
5343 5344 5345
		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
5346
			child_ctx->parent_gen = parent_ctx->parent_gen;
5347 5348 5349 5350 5351
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
5352 5353
	}

5354
	mutex_unlock(&parent_ctx->mutex);
5355

5356
	perf_unpin_context(parent_ctx);
5357

5358
	return ret;
5359 5360
}

5361
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
5362
{
5363
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
5364

5365
	cpuctx = &per_cpu(perf_cpu_context, cpu);
5366
	__perf_event_init_context(&cpuctx->ctx, NULL);
T
Thomas Gleixner 已提交
5367

5368
	spin_lock(&perf_resource_lock);
5369
	cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5370
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5371 5372 5373
}

#ifdef CONFIG_HOTPLUG_CPU
5374
static void __perf_event_exit_cpu(void *info)
T
Thomas Gleixner 已提交
5375 5376
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5377 5378
	struct perf_event_context *ctx = &cpuctx->ctx;
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
5379

5380 5381 5382
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		__perf_event_remove_from_context(event);
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
5383
		__perf_event_remove_from_context(event);
T
Thomas Gleixner 已提交
5384
}
5385
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
5386
{
5387
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5388
	struct perf_event_context *ctx = &cpuctx->ctx;
5389 5390

	mutex_lock(&ctx->mutex);
5391
	smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5392
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
5393 5394
}
#else
5395
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406
#endif

static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action) {

	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5407
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
5408 5409 5410 5411
		break;

	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
5412
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
5413 5414 5415 5416 5417 5418 5419 5420 5421
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

5422 5423 5424
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
T
Thomas Gleixner 已提交
5425 5426
static struct notifier_block __cpuinitdata perf_cpu_nb = {
	.notifier_call		= perf_cpu_notify,
5427
	.priority		= 20,
T
Thomas Gleixner 已提交
5428 5429
};

5430
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
5431 5432 5433
{
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
			(void *)(long)smp_processor_id());
5434 5435
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
			(void *)(long)smp_processor_id());
T
Thomas Gleixner 已提交
5436 5437 5438
	register_cpu_notifier(&perf_cpu_nb);
}

5439 5440 5441
static ssize_t perf_show_reserve_percpu(struct sysdev_class *class,
					struct sysdev_class_attribute *attr,
					char *buf)
T
Thomas Gleixner 已提交
5442 5443 5444 5445 5446 5447
{
	return sprintf(buf, "%d\n", perf_reserved_percpu);
}

static ssize_t
perf_set_reserve_percpu(struct sysdev_class *class,
5448
			struct sysdev_class_attribute *attr,
T
Thomas Gleixner 已提交
5449 5450 5451 5452 5453 5454 5455 5456 5457 5458
			const char *buf,
			size_t count)
{
	struct perf_cpu_context *cpuctx;
	unsigned long val;
	int err, cpu, mpt;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
5459
	if (val > perf_max_events)
T
Thomas Gleixner 已提交
5460 5461
		return -EINVAL;

5462
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5463 5464 5465
	perf_reserved_percpu = val;
	for_each_online_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
5466
		raw_spin_lock_irq(&cpuctx->ctx.lock);
5467 5468
		mpt = min(perf_max_events - cpuctx->ctx.nr_events,
			  perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
5469
		cpuctx->max_pertask = mpt;
5470
		raw_spin_unlock_irq(&cpuctx->ctx.lock);
T
Thomas Gleixner 已提交
5471
	}
5472
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5473 5474 5475 5476

	return count;
}

5477 5478 5479
static ssize_t perf_show_overcommit(struct sysdev_class *class,
				    struct sysdev_class_attribute *attr,
				    char *buf)
T
Thomas Gleixner 已提交
5480 5481 5482 5483 5484
{
	return sprintf(buf, "%d\n", perf_overcommit);
}

static ssize_t
5485 5486 5487
perf_set_overcommit(struct sysdev_class *class,
		    struct sysdev_class_attribute *attr,
		    const char *buf, size_t count)
T
Thomas Gleixner 已提交
5488 5489 5490 5491 5492 5493 5494 5495 5496 5497
{
	unsigned long val;
	int err;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
	if (val > 1)
		return -EINVAL;

5498
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5499
	perf_overcommit = val;
5500
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526

	return count;
}

static SYSDEV_CLASS_ATTR(
				reserve_percpu,
				0644,
				perf_show_reserve_percpu,
				perf_set_reserve_percpu
			);

static SYSDEV_CLASS_ATTR(
				overcommit,
				0644,
				perf_show_overcommit,
				perf_set_overcommit
			);

static struct attribute *perfclass_attrs[] = {
	&attr_reserve_percpu.attr,
	&attr_overcommit.attr,
	NULL
};

static struct attribute_group perfclass_attr_group = {
	.attrs			= perfclass_attrs,
5527
	.name			= "perf_events",
T
Thomas Gleixner 已提交
5528 5529
};

5530
static int __init perf_event_sysfs_init(void)
T
Thomas Gleixner 已提交
5531 5532 5533 5534
{
	return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
				  &perfclass_attr_group);
}
5535
device_initcall(perf_event_sysfs_init);