perf_event.c 123.3 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
16
#include <linux/file.h>
T
Thomas Gleixner 已提交
17 18
#include <linux/poll.h>
#include <linux/sysfs.h>
19
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
20
#include <linux/percpu.h>
21
#include <linux/ptrace.h>
22
#include <linux/vmstat.h>
23
#include <linux/vmalloc.h>
24 25
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
26 27 28
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
29
#include <linux/kernel_stat.h>
30
#include <linux/perf_event.h>
L
Li Zefan 已提交
31
#include <linux/ftrace_event.h>
32
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
33

34 35
#include <asm/irq_regs.h>

T
Thomas Gleixner 已提交
36
/*
37
 * Each CPU has a list of per CPU events:
T
Thomas Gleixner 已提交
38
 */
39
static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
T
Thomas Gleixner 已提交
40

41
int perf_max_events __read_mostly = 1;
T
Thomas Gleixner 已提交
42 43 44
static int perf_reserved_percpu __read_mostly;
static int perf_overcommit __read_mostly = 1;

45 46 47 48
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
49

50
/*
51
 * perf event paranoia level:
52 53
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
54
 *   1 - disallow cpu events for unpriv
55
 *   2 - disallow kernel profiling for unpriv
56
 */
57
int sysctl_perf_event_paranoid __read_mostly = 1;
58

59 60
static inline bool perf_paranoid_tracepoint_raw(void)
{
61
	return sysctl_perf_event_paranoid > -1;
62 63
}

64 65
static inline bool perf_paranoid_cpu(void)
{
66
	return sysctl_perf_event_paranoid > 0;
67 68 69 70
}

static inline bool perf_paranoid_kernel(void)
{
71
	return sysctl_perf_event_paranoid > 1;
72 73
}

74
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
75 76

/*
77
 * max perf event sample rate
78
 */
79
int sysctl_perf_event_sample_rate __read_mostly = 100000;
80

81
static atomic64_t perf_event_id;
82

T
Thomas Gleixner 已提交
83
/*
84
 * Lock for (sysadmin-configurable) event reservations:
T
Thomas Gleixner 已提交
85
 */
86
static DEFINE_SPINLOCK(perf_resource_lock);
T
Thomas Gleixner 已提交
87 88 89 90

/*
 * Architecture provided APIs - weak aliases:
 */
91
extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
T
Thomas Gleixner 已提交
92
{
93
	return NULL;
T
Thomas Gleixner 已提交
94 95
}

96 97 98
void __weak hw_perf_disable(void)		{ barrier(); }
void __weak hw_perf_enable(void)		{ barrier(); }

99 100
void __weak hw_perf_event_setup(int cpu)	{ barrier(); }
void __weak hw_perf_event_setup_online(int cpu)	{ barrier(); }
101 102

int __weak
103
hw_perf_group_sched_in(struct perf_event *group_leader,
104
	       struct perf_cpu_context *cpuctx,
105
	       struct perf_event_context *ctx, int cpu)
106 107 108
{
	return 0;
}
T
Thomas Gleixner 已提交
109

110
void __weak perf_event_print_debug(void)	{ }
111

112
static DEFINE_PER_CPU(int, perf_disable_count);
113 114 115

void __perf_disable(void)
{
116
	__get_cpu_var(perf_disable_count)++;
117 118 119 120
}

bool __perf_enable(void)
{
121
	return !--__get_cpu_var(perf_disable_count);
122 123 124 125 126 127 128 129 130 131 132 133 134 135
}

void perf_disable(void)
{
	__perf_disable();
	hw_perf_disable();
}

void perf_enable(void)
{
	if (__perf_enable())
		hw_perf_enable();
}

136
static void get_ctx(struct perf_event_context *ctx)
137
{
138
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
139 140
}

141 142
static void free_ctx(struct rcu_head *head)
{
143
	struct perf_event_context *ctx;
144

145
	ctx = container_of(head, struct perf_event_context, rcu_head);
146 147 148
	kfree(ctx);
}

149
static void put_ctx(struct perf_event_context *ctx)
150
{
151 152 153
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
154 155 156
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
157
	}
158 159
}

160
static void unclone_ctx(struct perf_event_context *ctx)
161 162 163 164 165 166 167
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

168
/*
169
 * If we inherit events we want to return the parent event id
170 171
 * to userspace.
 */
172
static u64 primary_event_id(struct perf_event *event)
173
{
174
	u64 id = event->id;
175

176 177
	if (event->parent)
		id = event->parent->id;
178 179 180 181

	return id;
}

182
/*
183
 * Get the perf_event_context for a task and lock it.
184 185 186
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
187
static struct perf_event_context *
188
perf_lock_task_context(struct task_struct *task, unsigned long *flags)
189
{
190
	struct perf_event_context *ctx;
191 192 193

	rcu_read_lock();
 retry:
194
	ctx = rcu_dereference(task->perf_event_ctxp);
195 196 197 198
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
199
		 * perf_event_task_sched_out, though the
200 201 202 203 204 205
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
206
		raw_spin_lock_irqsave(&ctx->lock, *flags);
207
		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
208
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
209 210
			goto retry;
		}
211 212

		if (!atomic_inc_not_zero(&ctx->refcount)) {
213
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
214 215
			ctx = NULL;
		}
216 217 218 219 220 221 222 223 224 225
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
226
static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
227
{
228
	struct perf_event_context *ctx;
229 230 231 232 233
	unsigned long flags;

	ctx = perf_lock_task_context(task, &flags);
	if (ctx) {
		++ctx->pin_count;
234
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
235 236 237 238
	}
	return ctx;
}

239
static void perf_unpin_context(struct perf_event_context *ctx)
240 241 242
{
	unsigned long flags;

243
	raw_spin_lock_irqsave(&ctx->lock, flags);
244
	--ctx->pin_count;
245
	raw_spin_unlock_irqrestore(&ctx->lock, flags);
246 247 248
	put_ctx(ctx);
}

249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
static inline u64 perf_clock(void)
{
	return cpu_clock(smp_processor_id());
}

/*
 * Update the record of the current time in a context.
 */
static void update_context_time(struct perf_event_context *ctx)
{
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
}

/*
 * Update the total_time_enabled and total_time_running fields for a event.
 */
static void update_event_times(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	u64 run_end;

	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
		return;

277 278 279 280 281 282
	if (ctx->is_active)
		run_end = ctx->time;
	else
		run_end = event->tstamp_stopped;

	event->total_time_enabled = run_end - event->tstamp_enabled;
283 284 285 286 287 288 289 290 291

	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
	else
		run_end = ctx->time;

	event->total_time_running = run_end - event->tstamp_running;
}

292 293 294 295 296 297 298 299 300
static struct list_head *
ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
{
	if (event->attr.pinned)
		return &ctx->pinned_groups;
	else
		return &ctx->flexible_groups;
}

301
/*
302
 * Add a event from the lists for its context.
303 304
 * Must be called with ctx->mutex and ctx->lock held.
 */
305
static void
306
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
307
{
308
	struct perf_event *group_leader = event->group_leader;
309 310

	/*
311 312
	 * Depending on whether it is a standalone or sibling event,
	 * add it straight to the context's event list, or to the group
313 314
	 * leader's sibling list:
	 */
315 316 317 318 319 320
	if (group_leader == event) {
		struct list_head *list;

		list = ctx_group_list(event, ctx);
		list_add_tail(&event->group_entry, list);
	} else {
321
		list_add_tail(&event->group_entry, &group_leader->sibling_list);
P
Peter Zijlstra 已提交
322 323
		group_leader->nr_siblings++;
	}
P
Peter Zijlstra 已提交
324

325 326 327
	list_add_rcu(&event->event_entry, &ctx->event_list);
	ctx->nr_events++;
	if (event->attr.inherit_stat)
328
		ctx->nr_stat++;
329 330
}

331
/*
332
 * Remove a event from the lists for its context.
333
 * Must be called with ctx->mutex and ctx->lock held.
334
 */
335
static void
336
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
337
{
338
	struct perf_event *sibling, *tmp;
339

340
	if (list_empty(&event->group_entry))
341
		return;
342 343
	ctx->nr_events--;
	if (event->attr.inherit_stat)
344
		ctx->nr_stat--;
345

346 347
	list_del_init(&event->group_entry);
	list_del_rcu(&event->event_entry);
348

349 350
	if (event->group_leader != event)
		event->group_leader->nr_siblings--;
P
Peter Zijlstra 已提交
351

352
	update_event_times(event);
353 354 355 356 357 358 359 360 361 362

	/*
	 * If event was in error state, then keep it
	 * that way, otherwise bogus counts will be
	 * returned on read(). The only way to get out
	 * of error state is by explicit re-enabling
	 * of the event
	 */
	if (event->state > PERF_EVENT_STATE_OFF)
		event->state = PERF_EVENT_STATE_OFF;
363

364
	/*
365 366
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
367 368
	 * to the context list directly:
	 */
369
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
370
		struct list_head *list;
371

372 373
		list = ctx_group_list(event, ctx);
		list_move_tail(&sibling->group_entry, list);
374 375 376 377
		sibling->group_leader = sibling;
	}
}

378
static void
379
event_sched_out(struct perf_event *event,
380
		  struct perf_cpu_context *cpuctx,
381
		  struct perf_event_context *ctx)
382
{
383
	if (event->state != PERF_EVENT_STATE_ACTIVE)
384 385
		return;

386 387 388 389
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
390
	}
391 392 393
	event->tstamp_stopped = ctx->time;
	event->pmu->disable(event);
	event->oncpu = -1;
394

395
	if (!is_software_event(event))
396 397
		cpuctx->active_oncpu--;
	ctx->nr_active--;
398
	if (event->attr.exclusive || !cpuctx->active_oncpu)
399 400 401
		cpuctx->exclusive = 0;
}

402
static void
403
group_sched_out(struct perf_event *group_event,
404
		struct perf_cpu_context *cpuctx,
405
		struct perf_event_context *ctx)
406
{
407
	struct perf_event *event;
408

409
	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
410 411
		return;

412
	event_sched_out(group_event, cpuctx, ctx);
413 414 415 416

	/*
	 * Schedule out siblings (if any):
	 */
417 418
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
419

420
	if (group_event->attr.exclusive)
421 422 423
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
424
/*
425
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
426
 *
427
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
428 429
 * remove it from the context list.
 */
430
static void __perf_event_remove_from_context(void *info)
T
Thomas Gleixner 已提交
431 432
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
433 434
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
435 436 437 438 439 440

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
	 */
441
	if (ctx->task && cpuctx->task_ctx != ctx)
T
Thomas Gleixner 已提交
442 443
		return;

444
	raw_spin_lock(&ctx->lock);
445 446
	/*
	 * Protect the list operation against NMI by disabling the
447
	 * events on a global level.
448 449
	 */
	perf_disable();
T
Thomas Gleixner 已提交
450

451
	event_sched_out(event, cpuctx, ctx);
452

453
	list_del_event(event, ctx);
T
Thomas Gleixner 已提交
454 455 456

	if (!ctx->task) {
		/*
457
		 * Allow more per task events with respect to the
T
Thomas Gleixner 已提交
458 459 460
		 * reservation:
		 */
		cpuctx->max_pertask =
461 462
			min(perf_max_events - ctx->nr_events,
			    perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
463 464
	}

465
	perf_enable();
466
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
467 468 469 470
}


/*
471
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
472
 *
473
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
474
 *
475
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
476
 * call when the task is on a CPU.
477
 *
478 479
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
480 481
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
482
 * When called from perf_event_exit_task, it's OK because the
483
 * context has been detached from its task.
T
Thomas Gleixner 已提交
484
 */
485
static void perf_event_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
486
{
487
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
488 489 490 491
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
492
		 * Per cpu events are removed via an smp call and
493
		 * the removal is always successful.
T
Thomas Gleixner 已提交
494
		 */
495 496 497
		smp_call_function_single(event->cpu,
					 __perf_event_remove_from_context,
					 event, 1);
T
Thomas Gleixner 已提交
498 499 500 501
		return;
	}

retry:
502 503
	task_oncpu_function_call(task, __perf_event_remove_from_context,
				 event);
T
Thomas Gleixner 已提交
504

505
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
506 507 508
	/*
	 * If the context is active we need to retry the smp call.
	 */
509
	if (ctx->nr_active && !list_empty(&event->group_entry)) {
510
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
511 512 513 514 515
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
516
	 * can remove the event safely, if the call above did not
T
Thomas Gleixner 已提交
517 518
	 * succeed.
	 */
P
Peter Zijlstra 已提交
519
	if (!list_empty(&event->group_entry))
520
		list_del_event(event, ctx);
521
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
522 523
}

524
/*
525
 * Update total_time_enabled and total_time_running for all events in a group.
526
 */
527
static void update_group_times(struct perf_event *leader)
528
{
529
	struct perf_event *event;
530

531 532 533
	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
534 535
}

536
/*
537
 * Cross CPU call to disable a performance event
538
 */
539
static void __perf_event_disable(void *info)
540
{
541
	struct perf_event *event = info;
542
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
543
	struct perf_event_context *ctx = event->ctx;
544 545

	/*
546 547
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
548
	 */
549
	if (ctx->task && cpuctx->task_ctx != ctx)
550 551
		return;

552
	raw_spin_lock(&ctx->lock);
553 554

	/*
555
	 * If the event is on, turn it off.
556 557
	 * If it is in error state, leave it in error state.
	 */
558
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
559
		update_context_time(ctx);
560 561 562
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
563
		else
564 565
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
566 567
	}

568
	raw_spin_unlock(&ctx->lock);
569 570 571
}

/*
572
 * Disable a event.
573
 *
574 575
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
576
 * remains valid.  This condition is satisifed when called through
577 578 579 580
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
581
 * is the current context on this CPU and preemption is disabled,
582
 * hence we can't get into perf_event_task_sched_out for this context.
583
 */
584
void perf_event_disable(struct perf_event *event)
585
{
586
	struct perf_event_context *ctx = event->ctx;
587 588 589 590
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
591
		 * Disable the event on the cpu that it's on
592
		 */
593 594
		smp_call_function_single(event->cpu, __perf_event_disable,
					 event, 1);
595 596 597 598
		return;
	}

 retry:
599
	task_oncpu_function_call(task, __perf_event_disable, event);
600

601
	raw_spin_lock_irq(&ctx->lock);
602
	/*
603
	 * If the event is still active, we need to retry the cross-call.
604
	 */
605
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
606
		raw_spin_unlock_irq(&ctx->lock);
607 608 609 610 611 612 613
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
614 615 616
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
617
	}
618

619
	raw_spin_unlock_irq(&ctx->lock);
620 621
}

622
static int
623
event_sched_in(struct perf_event *event,
624
		 struct perf_cpu_context *cpuctx,
625
		 struct perf_event_context *ctx,
626 627
		 int cpu)
{
628
	if (event->state <= PERF_EVENT_STATE_OFF)
629 630
		return 0;

631 632
	event->state = PERF_EVENT_STATE_ACTIVE;
	event->oncpu = cpu;	/* TODO: put 'cpu' into cpuctx->cpu */
633 634 635 636 637
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

638 639 640
	if (event->pmu->enable(event)) {
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
641 642 643
		return -EAGAIN;
	}

644
	event->tstamp_running += ctx->time - event->tstamp_stopped;
645

646
	if (!is_software_event(event))
647
		cpuctx->active_oncpu++;
648 649
	ctx->nr_active++;

650
	if (event->attr.exclusive)
651 652
		cpuctx->exclusive = 1;

653 654 655
	return 0;
}

656
static int
657
group_sched_in(struct perf_event *group_event,
658
	       struct perf_cpu_context *cpuctx,
659
	       struct perf_event_context *ctx,
660 661
	       int cpu)
{
662
	struct perf_event *event, *partial_group;
663 664
	int ret;

665
	if (group_event->state == PERF_EVENT_STATE_OFF)
666 667
		return 0;

668
	ret = hw_perf_group_sched_in(group_event, cpuctx, ctx, cpu);
669 670 671
	if (ret)
		return ret < 0 ? ret : 0;

672
	if (event_sched_in(group_event, cpuctx, ctx, cpu))
673 674 675 676 677
		return -EAGAIN;

	/*
	 * Schedule in siblings as one group (if any):
	 */
678 679 680
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event_sched_in(event, cpuctx, ctx, cpu)) {
			partial_group = event;
681 682 683 684 685 686 687 688 689 690 691
			goto group_error;
		}
	}

	return 0;

group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
	 */
692 693
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
694
			break;
695
		event_sched_out(event, cpuctx, ctx);
696
	}
697
	event_sched_out(group_event, cpuctx, ctx);
698 699 700 701

	return -EAGAIN;
}

702
/*
703 704
 * Return 1 for a group consisting entirely of software events,
 * 0 if the group contains any hardware events.
705
 */
706
static int is_software_only_group(struct perf_event *leader)
707
{
708
	struct perf_event *event;
709

710
	if (!is_software_event(leader))
711
		return 0;
P
Peter Zijlstra 已提交
712

713 714
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		if (!is_software_event(event))
715
			return 0;
P
Peter Zijlstra 已提交
716

717 718 719 720
	return 1;
}

/*
721
 * Work out whether we can put this event group on the CPU now.
722
 */
723
static int group_can_go_on(struct perf_event *event,
724 725 726 727
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
728
	 * Groups consisting entirely of software events can always go on.
729
	 */
730
	if (is_software_only_group(event))
731 732 733
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
734
	 * events can go on.
735 736 737 738 739
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
740
	 * events on the CPU, it can't go on.
741
	 */
742
	if (event->attr.exclusive && cpuctx->active_oncpu)
743 744 745 746 747 748 749 750
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

751 752
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
753
{
754 755 756 757
	list_add_event(event, ctx);
	event->tstamp_enabled = ctx->time;
	event->tstamp_running = ctx->time;
	event->tstamp_stopped = ctx->time;
758 759
}

T
Thomas Gleixner 已提交
760
/*
761
 * Cross CPU call to install and enable a performance event
762 763
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
764 765 766 767
 */
static void __perf_install_in_context(void *info)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
768 769 770
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
T
Thomas Gleixner 已提交
771
	int cpu = smp_processor_id();
772
	int err;
T
Thomas Gleixner 已提交
773 774 775 776 777

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
778
	 * Or possibly this is the right context but it isn't
779
	 * on this cpu because it had no events.
T
Thomas Gleixner 已提交
780
	 */
781
	if (ctx->task && cpuctx->task_ctx != ctx) {
782
		if (cpuctx->task_ctx || ctx->task != current)
783 784 785
			return;
		cpuctx->task_ctx = ctx;
	}
T
Thomas Gleixner 已提交
786

787
	raw_spin_lock(&ctx->lock);
788
	ctx->is_active = 1;
789
	update_context_time(ctx);
T
Thomas Gleixner 已提交
790 791 792

	/*
	 * Protect the list operation against NMI by disabling the
793
	 * events on a global level. NOP for non NMI based events.
T
Thomas Gleixner 已提交
794
	 */
795
	perf_disable();
T
Thomas Gleixner 已提交
796

797
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
798

799 800 801
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

802
	/*
803
	 * Don't put the event on if it is disabled or if
804 805
	 * it is in a group and the group isn't on.
	 */
806 807
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
808 809
		goto unlock;

810
	/*
811 812 813
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
814
	 */
815
	if (!group_can_go_on(event, cpuctx, 1))
816 817
		err = -EEXIST;
	else
818
		err = event_sched_in(event, cpuctx, ctx, cpu);
819

820 821
	if (err) {
		/*
822
		 * This event couldn't go on.  If it is in a group
823
		 * then we have to pull the whole group off.
824
		 * If the event group is pinned then put it in error state.
825
		 */
826
		if (leader != event)
827
			group_sched_out(leader, cpuctx, ctx);
828
		if (leader->attr.pinned) {
829
			update_group_times(leader);
830
			leader->state = PERF_EVENT_STATE_ERROR;
831
		}
832
	}
T
Thomas Gleixner 已提交
833

834
	if (!err && !ctx->task && cpuctx->max_pertask)
T
Thomas Gleixner 已提交
835 836
		cpuctx->max_pertask--;

837
 unlock:
838
	perf_enable();
839

840
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
841 842 843
}

/*
844
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
845
 *
846 847
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
848
 *
849
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
850 851
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
852 853
 *
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
854 855
 */
static void
856 857
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
858 859 860 861 862 863
			int cpu)
{
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
864
		 * Per cpu events are installed via an smp call and
865
		 * the install is always successful.
T
Thomas Gleixner 已提交
866 867
		 */
		smp_call_function_single(cpu, __perf_install_in_context,
868
					 event, 1);
T
Thomas Gleixner 已提交
869 870 871 872 873
		return;
	}

retry:
	task_oncpu_function_call(task, __perf_install_in_context,
874
				 event);
T
Thomas Gleixner 已提交
875

876
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
877 878 879
	/*
	 * we need to retry the smp call.
	 */
880
	if (ctx->is_active && list_empty(&event->group_entry)) {
881
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
882 883 884 885 886
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
887
	 * can add the event safely, if it the call above did not
T
Thomas Gleixner 已提交
888 889
	 * succeed.
	 */
890 891
	if (list_empty(&event->group_entry))
		add_event_to_ctx(event, ctx);
892
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
893 894
}

895
/*
896
 * Put a event into inactive state and update time fields.
897 898 899 900 901 902
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
903 904
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
905
{
906
	struct perf_event *sub;
907

908 909 910 911
	event->state = PERF_EVENT_STATE_INACTIVE;
	event->tstamp_enabled = ctx->time - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry)
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
912 913 914 915
			sub->tstamp_enabled =
				ctx->time - sub->total_time_enabled;
}

916
/*
917
 * Cross CPU call to enable a performance event
918
 */
919
static void __perf_event_enable(void *info)
920
{
921
	struct perf_event *event = info;
922
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
923 924
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
925
	int err;
926

927
	/*
928 929
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
930
	 */
931
	if (ctx->task && cpuctx->task_ctx != ctx) {
932
		if (cpuctx->task_ctx || ctx->task != current)
933 934 935
			return;
		cpuctx->task_ctx = ctx;
	}
936

937
	raw_spin_lock(&ctx->lock);
938
	ctx->is_active = 1;
939
	update_context_time(ctx);
940

941
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
942
		goto unlock;
943
	__perf_event_mark_enabled(event, ctx);
944

945 946 947
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

948
	/*
949
	 * If the event is in a group and isn't the group leader,
950
	 * then don't put it on unless the group is on.
951
	 */
952
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
953
		goto unlock;
954

955
	if (!group_can_go_on(event, cpuctx, 1)) {
956
		err = -EEXIST;
957
	} else {
958
		perf_disable();
959 960
		if (event == leader)
			err = group_sched_in(event, cpuctx, ctx,
961 962
					     smp_processor_id());
		else
963
			err = event_sched_in(event, cpuctx, ctx,
964
					       smp_processor_id());
965
		perf_enable();
966
	}
967 968 969

	if (err) {
		/*
970
		 * If this event can't go on and it's part of a
971 972
		 * group, then the whole group has to come off.
		 */
973
		if (leader != event)
974
			group_sched_out(leader, cpuctx, ctx);
975
		if (leader->attr.pinned) {
976
			update_group_times(leader);
977
			leader->state = PERF_EVENT_STATE_ERROR;
978
		}
979 980 981
	}

 unlock:
982
	raw_spin_unlock(&ctx->lock);
983 984 985
}

/*
986
 * Enable a event.
987
 *
988 989
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
990
 * remains valid.  This condition is satisfied when called through
991 992
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
993
 */
994
void perf_event_enable(struct perf_event *event)
995
{
996
	struct perf_event_context *ctx = event->ctx;
997 998 999 1000
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
1001
		 * Enable the event on the cpu that it's on
1002
		 */
1003 1004
		smp_call_function_single(event->cpu, __perf_event_enable,
					 event, 1);
1005 1006 1007
		return;
	}

1008
	raw_spin_lock_irq(&ctx->lock);
1009
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1010 1011 1012
		goto out;

	/*
1013 1014
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
1015 1016 1017 1018
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
1019 1020
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
1021 1022

 retry:
1023
	raw_spin_unlock_irq(&ctx->lock);
1024
	task_oncpu_function_call(task, __perf_event_enable, event);
1025

1026
	raw_spin_lock_irq(&ctx->lock);
1027 1028

	/*
1029
	 * If the context is active and the event is still off,
1030 1031
	 * we need to retry the cross-call.
	 */
1032
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
1033 1034 1035 1036 1037 1038
		goto retry;

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1039 1040
	if (event->state == PERF_EVENT_STATE_OFF)
		__perf_event_mark_enabled(event, ctx);
1041

1042
 out:
1043
	raw_spin_unlock_irq(&ctx->lock);
1044 1045
}

1046
static int perf_event_refresh(struct perf_event *event, int refresh)
1047
{
1048
	/*
1049
	 * not supported on inherited events
1050
	 */
1051
	if (event->attr.inherit)
1052 1053
		return -EINVAL;

1054 1055
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1056 1057

	return 0;
1058 1059
}

1060
void __perf_event_sched_out(struct perf_event_context *ctx,
1061 1062
			      struct perf_cpu_context *cpuctx)
{
1063
	struct perf_event *event;
1064

1065
	raw_spin_lock(&ctx->lock);
1066
	ctx->is_active = 0;
1067
	if (likely(!ctx->nr_events))
1068
		goto out;
1069
	update_context_time(ctx);
1070

1071
	perf_disable();
P
Peter Zijlstra 已提交
1072
	if (ctx->nr_active) {
1073 1074 1075 1076
		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
			group_sched_out(event, cpuctx, ctx);

		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1077
			group_sched_out(event, cpuctx, ctx);
P
Peter Zijlstra 已提交
1078
	}
1079
	perf_enable();
1080
 out:
1081
	raw_spin_unlock(&ctx->lock);
1082 1083
}

1084 1085 1086
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1087 1088 1089 1090
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1091
 * in them directly with an fd; we can only enable/disable all
1092
 * events via prctl, or enable/disable all events in a family
1093 1094
 * via ioctl, which will have the same effect on both contexts.
 */
1095 1096
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1097 1098
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1099
		&& ctx1->parent_gen == ctx2->parent_gen
1100
		&& !ctx1->pin_count && !ctx2->pin_count;
1101 1102
}

1103 1104
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1105 1106 1107
{
	u64 value;

1108
	if (!event->attr.inherit_stat)
1109 1110 1111
		return;

	/*
1112
	 * Update the event value, we cannot use perf_event_read()
1113 1114
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1115
	 * we know the event must be on the current CPU, therefore we
1116 1117
	 * don't need to use it.
	 */
1118 1119
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1120 1121
		event->pmu->read(event);
		/* fall-through */
1122

1123 1124
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1125 1126 1127 1128 1129 1130 1131
		break;

	default:
		break;
	}

	/*
1132
	 * In order to keep per-task stats reliable we need to flip the event
1133 1134
	 * values when we flip the contexts.
	 */
1135 1136 1137
	value = atomic64_read(&next_event->count);
	value = atomic64_xchg(&event->count, value);
	atomic64_set(&next_event->count, value);
1138

1139 1140
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1141

1142
	/*
1143
	 * Since we swizzled the values, update the user visible data too.
1144
	 */
1145 1146
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1147 1148 1149 1150 1151
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1152 1153
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1154
{
1155
	struct perf_event *event, *next_event;
1156 1157 1158 1159

	if (!ctx->nr_stat)
		return;

1160 1161
	update_context_time(ctx);

1162 1163
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1164

1165 1166
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1167

1168 1169
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1170

1171
		__perf_event_sync_stat(event, next_event);
1172

1173 1174
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1175 1176 1177
	}
}

T
Thomas Gleixner 已提交
1178
/*
1179
 * Called from scheduler to remove the events of the current task,
T
Thomas Gleixner 已提交
1180 1181
 * with interrupts disabled.
 *
1182
 * We stop each event and update the event value in event->count.
T
Thomas Gleixner 已提交
1183
 *
I
Ingo Molnar 已提交
1184
 * This does not protect us against NMI, but disable()
1185 1186 1187
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
T
Thomas Gleixner 已提交
1188
 */
1189
void perf_event_task_sched_out(struct task_struct *task,
1190
				 struct task_struct *next)
T
Thomas Gleixner 已提交
1191
{
1192
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1193 1194 1195
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1196
	struct pt_regs *regs;
1197
	int do_switch = 1;
T
Thomas Gleixner 已提交
1198

1199
	regs = task_pt_regs(task);
1200
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1201

1202
	if (likely(!ctx || !cpuctx->task_ctx))
T
Thomas Gleixner 已提交
1203 1204
		return;

1205 1206
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1207
	next_ctx = next->perf_event_ctxp;
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
1219 1220
		raw_spin_lock(&ctx->lock);
		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1221
		if (context_equiv(ctx, next_ctx)) {
1222 1223
			/*
			 * XXX do we need a memory barrier of sorts
1224
			 * wrt to rcu_dereference() of perf_event_ctxp
1225
			 */
1226 1227
			task->perf_event_ctxp = next_ctx;
			next->perf_event_ctxp = ctx;
1228 1229 1230
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1231

1232
			perf_event_sync_stat(ctx, next_ctx);
1233
		}
1234 1235
		raw_spin_unlock(&next_ctx->lock);
		raw_spin_unlock(&ctx->lock);
1236
	}
1237
	rcu_read_unlock();
1238

1239
	if (do_switch) {
1240
		__perf_event_sched_out(ctx, cpuctx);
1241 1242
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1243 1244
}

1245 1246 1247
/*
 * Called with IRQs disabled
 */
1248
static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1249 1250 1251
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

1252 1253
	if (!cpuctx->task_ctx)
		return;
1254 1255 1256 1257

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1258
	__perf_event_sched_out(ctx, cpuctx);
1259 1260 1261
	cpuctx->task_ctx = NULL;
}

1262 1263 1264
/*
 * Called with IRQs disabled
 */
1265
static void perf_event_cpu_sched_out(struct perf_cpu_context *cpuctx)
1266
{
1267
	__perf_event_sched_out(&cpuctx->ctx, cpuctx);
1268 1269
}

1270
static void
1271
__perf_event_sched_in(struct perf_event_context *ctx,
1272
			struct perf_cpu_context *cpuctx)
T
Thomas Gleixner 已提交
1273
{
1274
	int cpu = smp_processor_id();
1275
	struct perf_event *event;
1276
	int can_add_hw = 1;
T
Thomas Gleixner 已提交
1277

1278
	raw_spin_lock(&ctx->lock);
1279
	ctx->is_active = 1;
1280
	if (likely(!ctx->nr_events))
1281
		goto out;
T
Thomas Gleixner 已提交
1282

1283
	ctx->timestamp = perf_clock();
1284

1285
	perf_disable();
1286 1287 1288 1289 1290

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
1291 1292
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF)
1293
			continue;
1294
		if (event->cpu != -1 && event->cpu != cpu)
1295 1296
			continue;

1297 1298
		if (group_can_go_on(event, cpuctx, 1))
			group_sched_in(event, cpuctx, ctx, cpu);
1299 1300 1301 1302 1303

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1304 1305 1306
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1307
		}
1308 1309
	}

1310 1311 1312
	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		/* Ignore events in OFF or ERROR state */
		if (event->state <= PERF_EVENT_STATE_OFF)
1313
			continue;
1314 1315
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1316
		 * of events:
1317
		 */
1318
		if (event->cpu != -1 && event->cpu != cpu)
T
Thomas Gleixner 已提交
1319 1320
			continue;

1321 1322
		if (group_can_go_on(event, cpuctx, can_add_hw))
			if (group_sched_in(event, cpuctx, ctx, cpu))
1323
				can_add_hw = 0;
T
Thomas Gleixner 已提交
1324
	}
1325
	perf_enable();
1326
 out:
1327
	raw_spin_unlock(&ctx->lock);
1328 1329 1330
}

/*
1331
 * Called from scheduler to add the events of the current task
1332 1333
 * with interrupts disabled.
 *
1334
 * We restore the event value and then enable it.
1335 1336
 *
 * This does not protect us against NMI, but enable()
1337 1338 1339
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
1340
 */
1341
void perf_event_task_sched_in(struct task_struct *task)
1342
{
1343
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1344
	struct perf_event_context *ctx = task->perf_event_ctxp;
1345

1346 1347
	if (likely(!ctx))
		return;
1348 1349
	if (cpuctx->task_ctx == ctx)
		return;
1350
	__perf_event_sched_in(ctx, cpuctx);
T
Thomas Gleixner 已提交
1351 1352 1353
	cpuctx->task_ctx = ctx;
}

1354
static void perf_event_cpu_sched_in(struct perf_cpu_context *cpuctx)
1355
{
1356
	struct perf_event_context *ctx = &cpuctx->ctx;
1357

1358
	__perf_event_sched_in(ctx, cpuctx);
1359 1360
}

1361 1362
#define MAX_INTERRUPTS (~0ULL)

1363
static void perf_log_throttle(struct perf_event *event, int enable);
1364

1365
static void perf_adjust_period(struct perf_event *event, u64 events)
1366
{
1367
	struct hw_perf_event *hwc = &event->hw;
1368 1369 1370 1371
	u64 period, sample_period;
	s64 delta;

	events *= hwc->sample_period;
1372
	period = div64_u64(events, event->attr.sample_freq);
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
}

1385
static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1386
{
1387 1388
	struct perf_event *event;
	struct hw_perf_event *hwc;
1389
	u64 interrupts, freq;
1390

1391
	raw_spin_lock(&ctx->lock);
1392
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1393
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1394 1395
			continue;

1396 1397 1398
		if (event->cpu != -1 && event->cpu != smp_processor_id())
			continue;

1399
		hwc = &event->hw;
1400 1401 1402

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1403

1404
		/*
1405
		 * unthrottle events on the tick
1406
		 */
1407
		if (interrupts == MAX_INTERRUPTS) {
1408 1409 1410
			perf_log_throttle(event, 1);
			event->pmu->unthrottle(event);
			interrupts = 2*sysctl_perf_event_sample_rate/HZ;
1411 1412
		}

1413
		if (!event->attr.freq || !event->attr.sample_freq)
1414 1415
			continue;

1416 1417 1418
		/*
		 * if the specified freq < HZ then we need to skip ticks
		 */
1419 1420
		if (event->attr.sample_freq < HZ) {
			freq = event->attr.sample_freq;
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433

			hwc->freq_count += freq;
			hwc->freq_interrupts += interrupts;

			if (hwc->freq_count < HZ)
				continue;

			interrupts = hwc->freq_interrupts;
			hwc->freq_interrupts = 0;
			hwc->freq_count -= HZ;
		} else
			freq = HZ;

1434
		perf_adjust_period(event, freq * interrupts);
1435

1436 1437 1438 1439 1440 1441 1442
		/*
		 * In order to avoid being stalled by an (accidental) huge
		 * sample period, force reset the sample period if we didn't
		 * get any events in this freq period.
		 */
		if (!interrupts) {
			perf_disable();
1443
			event->pmu->disable(event);
1444
			atomic64_set(&hwc->period_left, 0);
1445
			event->pmu->enable(event);
1446 1447
			perf_enable();
		}
1448
	}
1449
	raw_spin_unlock(&ctx->lock);
1450 1451
}

1452
/*
1453
 * Round-robin a context's events:
1454
 */
1455
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1456
{
1457
	struct perf_event *event;
T
Thomas Gleixner 已提交
1458

1459
	if (!ctx->nr_events)
T
Thomas Gleixner 已提交
1460 1461
		return;

1462
	raw_spin_lock(&ctx->lock);
T
Thomas Gleixner 已提交
1463
	/*
1464
	 * Rotate the first entry last (works just fine for group events too):
T
Thomas Gleixner 已提交
1465
	 */
1466
	perf_disable();
1467 1468 1469 1470 1471 1472 1473
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		list_move_tail(&event->group_entry, &ctx->pinned_groups);
		break;
	}

	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		list_move_tail(&event->group_entry, &ctx->flexible_groups);
T
Thomas Gleixner 已提交
1474 1475
		break;
	}
1476
	perf_enable();
T
Thomas Gleixner 已提交
1477

1478
	raw_spin_unlock(&ctx->lock);
1479 1480
}

1481
void perf_event_task_tick(struct task_struct *curr)
1482
{
1483
	struct perf_cpu_context *cpuctx;
1484
	struct perf_event_context *ctx;
1485

1486
	if (!atomic_read(&nr_events))
1487 1488
		return;

1489
	cpuctx = &__get_cpu_var(perf_cpu_context);
1490
	ctx = curr->perf_event_ctxp;
1491

1492
	perf_ctx_adjust_freq(&cpuctx->ctx);
1493
	if (ctx)
1494
		perf_ctx_adjust_freq(ctx);
1495

1496
	perf_event_cpu_sched_out(cpuctx);
1497
	if (ctx)
1498
		__perf_event_task_sched_out(ctx);
T
Thomas Gleixner 已提交
1499

1500
	rotate_ctx(&cpuctx->ctx);
1501 1502
	if (ctx)
		rotate_ctx(ctx);
1503

1504
	perf_event_cpu_sched_in(cpuctx);
1505
	if (ctx)
1506
		perf_event_task_sched_in(curr);
T
Thomas Gleixner 已提交
1507 1508
}

1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

	__perf_event_mark_enabled(event, ctx);

	return 1;
}

1524
/*
1525
 * Enable all of a task's events that have been marked enable-on-exec.
1526 1527
 * This expects task == current.
 */
1528
static void perf_event_enable_on_exec(struct task_struct *task)
1529
{
1530 1531
	struct perf_event_context *ctx;
	struct perf_event *event;
1532 1533
	unsigned long flags;
	int enabled = 0;
1534
	int ret;
1535 1536

	local_irq_save(flags);
1537 1538
	ctx = task->perf_event_ctxp;
	if (!ctx || !ctx->nr_events)
1539 1540
		goto out;

1541
	__perf_event_task_sched_out(ctx);
1542

1543
	raw_spin_lock(&ctx->lock);
1544

1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
	}

	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
1555 1556 1557
	}

	/*
1558
	 * Unclone this context if we enabled any event.
1559
	 */
1560 1561
	if (enabled)
		unclone_ctx(ctx);
1562

1563
	raw_spin_unlock(&ctx->lock);
1564

1565
	perf_event_task_sched_in(task);
1566 1567 1568 1569
 out:
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1570
/*
1571
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1572
 */
1573
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1574
{
1575
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1576 1577
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
I
Ingo Molnar 已提交
1578

1579 1580 1581 1582
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1583 1584
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1585 1586 1587 1588
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

1589
	raw_spin_lock(&ctx->lock);
P
Peter Zijlstra 已提交
1590
	update_context_time(ctx);
1591
	update_event_times(event);
1592
	raw_spin_unlock(&ctx->lock);
P
Peter Zijlstra 已提交
1593

P
Peter Zijlstra 已提交
1594
	event->pmu->read(event);
T
Thomas Gleixner 已提交
1595 1596
}

1597
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1598 1599
{
	/*
1600 1601
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1602
	 */
1603 1604 1605 1606
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
1607 1608 1609
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

1610
		raw_spin_lock_irqsave(&ctx->lock, flags);
P
Peter Zijlstra 已提交
1611
		update_context_time(ctx);
1612
		update_event_times(event);
1613
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1614 1615
	}

1616
	return atomic64_read(&event->count);
T
Thomas Gleixner 已提交
1617 1618
}

1619
/*
1620
 * Initialize the perf_event context in a task_struct:
1621 1622
 */
static void
1623
__perf_event_init_context(struct perf_event_context *ctx,
1624 1625
			    struct task_struct *task)
{
1626
	raw_spin_lock_init(&ctx->lock);
1627
	mutex_init(&ctx->mutex);
1628 1629
	INIT_LIST_HEAD(&ctx->pinned_groups);
	INIT_LIST_HEAD(&ctx->flexible_groups);
1630 1631 1632 1633 1634
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
	ctx->task = task;
}

1635
static struct perf_event_context *find_get_context(pid_t pid, int cpu)
T
Thomas Gleixner 已提交
1636
{
1637
	struct perf_event_context *ctx;
1638
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
1639
	struct task_struct *task;
1640
	unsigned long flags;
1641
	int err;
T
Thomas Gleixner 已提交
1642

1643
	if (pid == -1 && cpu != -1) {
1644
		/* Must be root to operate on a CPU event: */
1645
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
1646 1647
			return ERR_PTR(-EACCES);

1648
		if (cpu < 0 || cpu >= nr_cpumask_bits)
T
Thomas Gleixner 已提交
1649 1650 1651
			return ERR_PTR(-EINVAL);

		/*
1652
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
1653 1654 1655
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
1656
		if (!cpu_online(cpu))
T
Thomas Gleixner 已提交
1657 1658 1659 1660
			return ERR_PTR(-ENODEV);

		cpuctx = &per_cpu(perf_cpu_context, cpu);
		ctx = &cpuctx->ctx;
1661
		get_ctx(ctx);
T
Thomas Gleixner 已提交
1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677

		return ctx;
	}

	rcu_read_lock();
	if (!pid)
		task = current;
	else
		task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

1678
	/*
1679
	 * Can't attach events to a dying task.
1680 1681 1682 1683 1684
	 */
	err = -ESRCH;
	if (task->flags & PF_EXITING)
		goto errout;

T
Thomas Gleixner 已提交
1685
	/* Reuse ptrace permission checks for now. */
1686 1687 1688 1689 1690
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

 retry:
1691
	ctx = perf_lock_task_context(task, &flags);
1692
	if (ctx) {
1693
		unclone_ctx(ctx);
1694
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1695 1696
	}

1697
	if (!ctx) {
1698
		ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1699 1700 1701
		err = -ENOMEM;
		if (!ctx)
			goto errout;
1702
		__perf_event_init_context(ctx, task);
1703
		get_ctx(ctx);
1704
		if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1705 1706 1707 1708 1709
			/*
			 * We raced with some other task; use
			 * the context they set.
			 */
			kfree(ctx);
1710
			goto retry;
1711
		}
1712
		get_task_struct(task);
1713 1714
	}

1715
	put_task_struct(task);
T
Thomas Gleixner 已提交
1716
	return ctx;
1717 1718 1719 1720

 errout:
	put_task_struct(task);
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
1721 1722
}

L
Li Zefan 已提交
1723 1724
static void perf_event_free_filter(struct perf_event *event);

1725
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
1726
{
1727
	struct perf_event *event;
P
Peter Zijlstra 已提交
1728

1729 1730 1731
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
1732
	perf_event_free_filter(event);
1733
	kfree(event);
P
Peter Zijlstra 已提交
1734 1735
}

1736
static void perf_pending_sync(struct perf_event *event);
1737

1738
static void free_event(struct perf_event *event)
1739
{
1740
	perf_pending_sync(event);
1741

1742 1743 1744 1745 1746 1747 1748 1749
	if (!event->parent) {
		atomic_dec(&nr_events);
		if (event->attr.mmap)
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
1750
	}
1751

1752 1753 1754
	if (event->output) {
		fput(event->output->filp);
		event->output = NULL;
1755 1756
	}

1757 1758
	if (event->destroy)
		event->destroy(event);
1759

1760 1761
	put_ctx(event->ctx);
	call_rcu(&event->rcu_head, free_event_rcu);
1762 1763
}

1764
int perf_event_release_kernel(struct perf_event *event)
T
Thomas Gleixner 已提交
1765
{
1766
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1767

1768
	WARN_ON_ONCE(ctx->parent_ctx);
1769
	mutex_lock(&ctx->mutex);
1770
	perf_event_remove_from_context(event);
1771
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
1772

1773 1774 1775 1776
	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);
1777

1778
	free_event(event);
T
Thomas Gleixner 已提交
1779 1780 1781

	return 0;
}
1782
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
T
Thomas Gleixner 已提交
1783

1784 1785 1786 1787
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
1788
{
1789
	struct perf_event *event = file->private_data;
1790

1791
	file->private_data = NULL;
1792

1793
	return perf_event_release_kernel(event);
1794 1795
}

1796
static int perf_event_read_size(struct perf_event *event)
1797 1798 1799 1800 1801
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

1802
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1803 1804
		size += sizeof(u64);

1805
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1806 1807
		size += sizeof(u64);

1808
	if (event->attr.read_format & PERF_FORMAT_ID)
1809 1810
		entry += sizeof(u64);

1811 1812
	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
1813 1814 1815 1816 1817 1818 1819 1820
		size += sizeof(u64);
	}

	size += entry * nr;

	return size;
}

1821
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
1822
{
1823
	struct perf_event *child;
1824 1825
	u64 total = 0;

1826 1827 1828
	*enabled = 0;
	*running = 0;

1829
	mutex_lock(&event->child_mutex);
1830
	total += perf_event_read(event);
1831 1832 1833 1834 1835 1836
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
1837
		total += perf_event_read(child);
1838 1839 1840
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
1841
	mutex_unlock(&event->child_mutex);
1842 1843 1844

	return total;
}
1845
EXPORT_SYMBOL_GPL(perf_event_read_value);
1846

1847
static int perf_event_read_group(struct perf_event *event,
1848 1849
				   u64 read_format, char __user *buf)
{
1850
	struct perf_event *leader = event->group_leader, *sub;
1851 1852
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
1853
	u64 values[5];
1854
	u64 count, enabled, running;
1855

1856
	mutex_lock(&ctx->mutex);
1857
	count = perf_event_read_value(leader, &enabled, &running);
1858 1859

	values[n++] = 1 + leader->nr_siblings;
1860 1861 1862 1863
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1864 1865 1866
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
1867 1868 1869 1870

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
1871
		goto unlock;
1872

1873
	ret = size;
1874

1875
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
1876
		n = 0;
1877

1878
		values[n++] = perf_event_read_value(sub, &enabled, &running);
1879 1880 1881 1882 1883
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

1884
		if (copy_to_user(buf + ret, values, size)) {
1885 1886 1887
			ret = -EFAULT;
			goto unlock;
		}
1888 1889

		ret += size;
1890
	}
1891 1892
unlock:
	mutex_unlock(&ctx->mutex);
1893

1894
	return ret;
1895 1896
}

1897
static int perf_event_read_one(struct perf_event *event,
1898 1899
				 u64 read_format, char __user *buf)
{
1900
	u64 enabled, running;
1901 1902 1903
	u64 values[4];
	int n = 0;

1904 1905 1906 1907 1908
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1909
	if (read_format & PERF_FORMAT_ID)
1910
		values[n++] = primary_event_id(event);
1911 1912 1913 1914 1915 1916 1917

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
1918
/*
1919
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
1920 1921
 */
static ssize_t
1922
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
1923
{
1924
	u64 read_format = event->attr.read_format;
1925
	int ret;
T
Thomas Gleixner 已提交
1926

1927
	/*
1928
	 * Return end-of-file for a read on a event that is in
1929 1930 1931
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
1932
	if (event->state == PERF_EVENT_STATE_ERROR)
1933 1934
		return 0;

1935
	if (count < perf_event_read_size(event))
1936 1937
		return -ENOSPC;

1938
	WARN_ON_ONCE(event->ctx->parent_ctx);
1939
	if (read_format & PERF_FORMAT_GROUP)
1940
		ret = perf_event_read_group(event, read_format, buf);
1941
	else
1942
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
1943

1944
	return ret;
T
Thomas Gleixner 已提交
1945 1946 1947 1948 1949
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
1950
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
1951

1952
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
1953 1954 1955 1956
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
1957
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
1958
	struct perf_mmap_data *data;
1959
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
1960 1961

	rcu_read_lock();
1962
	data = rcu_dereference(event->data);
P
Peter Zijlstra 已提交
1963
	if (data)
1964
		events = atomic_xchg(&data->poll, 0);
P
Peter Zijlstra 已提交
1965
	rcu_read_unlock();
T
Thomas Gleixner 已提交
1966

1967
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
1968 1969 1970 1971

	return events;
}

1972
static void perf_event_reset(struct perf_event *event)
1973
{
1974 1975 1976
	(void)perf_event_read(event);
	atomic64_set(&event->count, 0);
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
1977 1978
}

1979
/*
1980 1981 1982 1983
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
1984
 */
1985 1986
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
1987
{
1988
	struct perf_event *child;
P
Peter Zijlstra 已提交
1989

1990 1991 1992 1993
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
1994
		func(child);
1995
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
1996 1997
}

1998 1999
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2000
{
2001 2002
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
2003

2004 2005
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
2006
	event = event->group_leader;
2007

2008 2009 2010 2011
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
2012
	mutex_unlock(&ctx->mutex);
2013 2014
}

2015
static int perf_event_period(struct perf_event *event, u64 __user *arg)
2016
{
2017
	struct perf_event_context *ctx = event->ctx;
2018 2019 2020 2021
	unsigned long size;
	int ret = 0;
	u64 value;

2022
	if (!event->attr.sample_period)
2023 2024 2025 2026 2027 2028 2029 2030 2031
		return -EINVAL;

	size = copy_from_user(&value, arg, sizeof(value));
	if (size != sizeof(value))
		return -EFAULT;

	if (!value)
		return -EINVAL;

2032
	raw_spin_lock_irq(&ctx->lock);
2033 2034
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
2035 2036 2037 2038
			ret = -EINVAL;
			goto unlock;
		}

2039
		event->attr.sample_freq = value;
2040
	} else {
2041 2042
		event->attr.sample_period = value;
		event->hw.sample_period = value;
2043 2044
	}
unlock:
2045
	raw_spin_unlock_irq(&ctx->lock);
2046 2047 2048 2049

	return ret;
}

L
Li Zefan 已提交
2050 2051
static int perf_event_set_output(struct perf_event *event, int output_fd);
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2052

2053 2054
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
2055 2056
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2057
	u32 flags = arg;
2058 2059

	switch (cmd) {
2060 2061
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2062
		break;
2063 2064
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2065
		break;
2066 2067
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2068
		break;
P
Peter Zijlstra 已提交
2069

2070 2071
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2072

2073 2074
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2075

2076 2077
	case PERF_EVENT_IOC_SET_OUTPUT:
		return perf_event_set_output(event, arg);
2078

L
Li Zefan 已提交
2079 2080 2081
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2082
	default:
P
Peter Zijlstra 已提交
2083
		return -ENOTTY;
2084
	}
P
Peter Zijlstra 已提交
2085 2086

	if (flags & PERF_IOC_FLAG_GROUP)
2087
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2088
	else
2089
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2090 2091

	return 0;
2092 2093
}

2094
int perf_event_task_enable(void)
2095
{
2096
	struct perf_event *event;
2097

2098 2099 2100 2101
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2102 2103 2104 2105

	return 0;
}

2106
int perf_event_task_disable(void)
2107
{
2108
	struct perf_event *event;
2109

2110 2111 2112 2113
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2114 2115 2116 2117

	return 0;
}

2118 2119
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2120 2121
#endif

2122
static int perf_event_index(struct perf_event *event)
2123
{
2124
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2125 2126
		return 0;

2127
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2128 2129
}

2130 2131 2132 2133 2134
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2135
void perf_event_update_userpage(struct perf_event *event)
2136
{
2137
	struct perf_event_mmap_page *userpg;
2138
	struct perf_mmap_data *data;
2139 2140

	rcu_read_lock();
2141
	data = rcu_dereference(event->data);
2142 2143 2144 2145
	if (!data)
		goto unlock;

	userpg = data->user_page;
2146

2147 2148 2149 2150 2151
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2152
	++userpg->lock;
2153
	barrier();
2154 2155 2156 2157
	userpg->index = perf_event_index(event);
	userpg->offset = atomic64_read(&event->count);
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		userpg->offset -= atomic64_read(&event->hw.prev_count);
2158

2159 2160
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2161

2162 2163
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2164

2165
	barrier();
2166
	++userpg->lock;
2167
	preempt_enable();
2168
unlock:
2169
	rcu_read_unlock();
2170 2171
}

2172
static unsigned long perf_data_size(struct perf_mmap_data *data)
2173
{
2174 2175
	return data->nr_pages << (PAGE_SHIFT + data->data_order);
}
2176

2177
#ifndef CONFIG_PERF_USE_VMALLOC
2178

2179 2180 2181
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2182

2183 2184 2185 2186 2187
static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > data->nr_pages)
		return NULL;
2188

2189 2190
	if (pgoff == 0)
		return virt_to_page(data->user_page);
2191

2192
	return virt_to_page(data->data_pages[pgoff - 1]);
2193 2194
}

2195 2196
static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2197 2198 2199 2200 2201
{
	struct perf_mmap_data *data;
	unsigned long size;
	int i;

2202
	WARN_ON(atomic_read(&event->mmap_count));
2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220

	size = sizeof(struct perf_mmap_data);
	size += nr_pages * sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
	if (!data->user_page)
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
		data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
		if (!data->data_pages[i])
			goto fail_data_pages;
	}

2221
	data->data_order = 0;
2222 2223
	data->nr_pages = nr_pages;

2224
	return data;
2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235

fail_data_pages:
	for (i--; i >= 0; i--)
		free_page((unsigned long)data->data_pages[i]);

	free_page((unsigned long)data->user_page);

fail_user_page:
	kfree(data);

fail:
2236
	return NULL;
2237 2238
}

2239 2240
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2241
	struct page *page = virt_to_page((void *)addr);
2242 2243 2244 2245 2246

	page->mapping = NULL;
	__free_page(page);
}

2247
static void perf_mmap_data_free(struct perf_mmap_data *data)
2248 2249 2250
{
	int i;

2251
	perf_mmap_free_page((unsigned long)data->user_page);
2252
	for (i = 0; i < data->nr_pages; i++)
2253
		perf_mmap_free_page((unsigned long)data->data_pages[i]);
2254
	kfree(data);
2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294
}

#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > (1UL << data->data_order))
		return NULL;

	return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

static void perf_mmap_data_free_work(struct work_struct *work)
{
	struct perf_mmap_data *data;
	void *base;
	int i, nr;

	data = container_of(work, struct perf_mmap_data, work);
	nr = 1 << data->data_order;

	base = data->user_page;
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
2295
	kfree(data);
2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
}

static void perf_mmap_data_free(struct perf_mmap_data *data)
{
	schedule_work(&data->work);
}

static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
{
	struct perf_mmap_data *data;
	unsigned long size;
	void *all_buf;

	WARN_ON(atomic_read(&event->mmap_count));
2311

2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388
	size = sizeof(struct perf_mmap_data);
	size += sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	INIT_WORK(&data->work, perf_mmap_data_free_work);

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

	data->user_page = all_buf;
	data->data_pages[0] = all_buf + PAGE_SIZE;
	data->data_order = ilog2(nr_pages);
	data->nr_pages = 1;

	return data;

fail_all_buf:
	kfree(data);

fail:
	return NULL;
}

#endif

static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
	struct perf_mmap_data *data;
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	data = rcu_dereference(event->data);
	if (!data)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(data, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

static void
perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
{
	long max_size = perf_data_size(data);

	atomic_set(&data->lock, -1);

	if (event->attr.watermark) {
		data->watermark = min_t(long, max_size,
					event->attr.wakeup_watermark);
	}

	if (!data->watermark)
2389
		data->watermark = max_size / 2;
2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400


	rcu_assign_pointer(event->data, data);
}

static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
{
	struct perf_mmap_data *data;

	data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
	perf_mmap_data_free(data);
2401 2402
}

2403
static void perf_mmap_data_release(struct perf_event *event)
2404
{
2405
	struct perf_mmap_data *data = event->data;
2406

2407
	WARN_ON(atomic_read(&event->mmap_count));
2408

2409
	rcu_assign_pointer(event->data, NULL);
2410
	call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2411 2412 2413 2414
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
2415
	struct perf_event *event = vma->vm_file->private_data;
2416

2417
	atomic_inc(&event->mmap_count);
2418 2419 2420 2421
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
2422
	struct perf_event *event = vma->vm_file->private_data;
2423

2424 2425
	WARN_ON_ONCE(event->ctx->parent_ctx);
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2426
		unsigned long size = perf_data_size(event->data);
2427 2428
		struct user_struct *user = current_user();

2429
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2430
		vma->vm_mm->locked_vm -= event->data->nr_locked;
2431
		perf_mmap_data_release(event);
2432
		mutex_unlock(&event->mmap_mutex);
2433
	}
2434 2435
}

2436
static const struct vm_operations_struct perf_mmap_vmops = {
2437 2438 2439 2440
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
2441 2442 2443 2444
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
2445
	struct perf_event *event = file->private_data;
2446
	unsigned long user_locked, user_lock_limit;
2447
	struct user_struct *user = current_user();
2448
	unsigned long locked, lock_limit;
2449
	struct perf_mmap_data *data;
2450 2451
	unsigned long vma_size;
	unsigned long nr_pages;
2452
	long user_extra, extra;
2453
	int ret = 0;
2454

2455
	if (!(vma->vm_flags & VM_SHARED))
2456
		return -EINVAL;
2457 2458 2459 2460

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

2461 2462 2463 2464 2465
	/*
	 * If we have data pages ensure they're a power-of-two number, so we
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
2466 2467
		return -EINVAL;

2468
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
2469 2470
		return -EINVAL;

2471 2472
	if (vma->vm_pgoff != 0)
		return -EINVAL;
2473

2474 2475 2476
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
	if (event->output) {
2477 2478 2479 2480
		ret = -EINVAL;
		goto unlock;
	}

2481 2482
	if (atomic_inc_not_zero(&event->mmap_count)) {
		if (nr_pages != event->data->nr_pages)
2483 2484 2485 2486
			ret = -EINVAL;
		goto unlock;
	}

2487
	user_extra = nr_pages + 1;
2488
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
2489 2490 2491 2492 2493 2494

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

2495
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2496

2497 2498 2499
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
2500 2501 2502

	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
	lock_limit >>= PAGE_SHIFT;
2503
	locked = vma->vm_mm->locked_vm + extra;
2504

2505 2506
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
2507 2508 2509
		ret = -EPERM;
		goto unlock;
	}
2510

2511
	WARN_ON(event->data);
2512 2513 2514 2515

	data = perf_mmap_data_alloc(event, nr_pages);
	ret = -ENOMEM;
	if (!data)
2516 2517
		goto unlock;

2518 2519 2520
	ret = 0;
	perf_mmap_data_init(event, data);

2521
	atomic_set(&event->mmap_count, 1);
2522
	atomic_long_add(user_extra, &user->locked_vm);
2523
	vma->vm_mm->locked_vm += extra;
2524
	event->data->nr_locked = extra;
2525
	if (vma->vm_flags & VM_WRITE)
2526
		event->data->writable = 1;
2527

2528
unlock:
2529
	mutex_unlock(&event->mmap_mutex);
2530 2531 2532

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
2533 2534

	return ret;
2535 2536
}

P
Peter Zijlstra 已提交
2537 2538 2539
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
2540
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
2541 2542 2543
	int retval;

	mutex_lock(&inode->i_mutex);
2544
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
2545 2546 2547 2548 2549 2550 2551 2552
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
2553 2554 2555 2556
static const struct file_operations perf_fops = {
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
2557 2558
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
2559
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
2560
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
2561 2562
};

2563
/*
2564
 * Perf event wakeup
2565 2566 2567 2568 2569
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

2570
void perf_event_wakeup(struct perf_event *event)
2571
{
2572
	wake_up_all(&event->waitq);
2573

2574 2575 2576
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
2577
	}
2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588
}

/*
 * Pending wakeups
 *
 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
 *
 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
 * single linked list and use cmpxchg() to add entries lockless.
 */

2589
static void perf_pending_event(struct perf_pending_entry *entry)
2590
{
2591 2592
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
2593

2594 2595 2596
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
2597 2598
	}

2599 2600 2601
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
2602 2603 2604
	}
}

2605
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2606

2607
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2608 2609 2610
	PENDING_TAIL,
};

2611 2612
static void perf_pending_queue(struct perf_pending_entry *entry,
			       void (*func)(struct perf_pending_entry *))
2613
{
2614
	struct perf_pending_entry **head;
2615

2616
	if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2617 2618
		return;

2619 2620 2621
	entry->func = func;

	head = &get_cpu_var(perf_pending_head);
2622 2623

	do {
2624 2625
		entry->next = *head;
	} while (cmpxchg(head, entry->next, entry) != entry->next);
2626

2627
	set_perf_event_pending();
2628

2629
	put_cpu_var(perf_pending_head);
2630 2631 2632 2633
}

static int __perf_pending_run(void)
{
2634
	struct perf_pending_entry *list;
2635 2636
	int nr = 0;

2637
	list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2638
	while (list != PENDING_TAIL) {
2639 2640
		void (*func)(struct perf_pending_entry *);
		struct perf_pending_entry *entry = list;
2641 2642 2643

		list = list->next;

2644 2645
		func = entry->func;
		entry->next = NULL;
2646 2647 2648 2649 2650 2651 2652
		/*
		 * Ensure we observe the unqueue before we issue the wakeup,
		 * so that we won't be waiting forever.
		 * -- see perf_not_pending().
		 */
		smp_wmb();

2653
		func(entry);
2654 2655 2656 2657 2658 2659
		nr++;
	}

	return nr;
}

2660
static inline int perf_not_pending(struct perf_event *event)
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674
{
	/*
	 * If we flush on whatever cpu we run, there is a chance we don't
	 * need to wait.
	 */
	get_cpu();
	__perf_pending_run();
	put_cpu();

	/*
	 * Ensure we see the proper queue state before going to sleep
	 * so that we do not miss the wakeup. -- see perf_pending_handle()
	 */
	smp_rmb();
2675
	return event->pending.next == NULL;
2676 2677
}

2678
static void perf_pending_sync(struct perf_event *event)
2679
{
2680
	wait_event(event->waitq, perf_not_pending(event));
2681 2682
}

2683
void perf_event_do_pending(void)
2684 2685 2686 2687
{
	__perf_pending_run();
}

2688 2689 2690 2691
/*
 * Callchain support -- arch specific
 */

2692
__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2693 2694 2695 2696
{
	return NULL;
}

2697 2698 2699
/*
 * Output
 */
2700 2701
static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
			      unsigned long offset, unsigned long head)
2702 2703 2704 2705 2706 2707
{
	unsigned long mask;

	if (!data->writable)
		return true;

2708
	mask = perf_data_size(data) - 1;
2709 2710 2711 2712 2713 2714 2715 2716 2717 2718

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

2719
static void perf_output_wakeup(struct perf_output_handle *handle)
2720
{
2721 2722
	atomic_set(&handle->data->poll, POLL_IN);

2723
	if (handle->nmi) {
2724 2725 2726
		handle->event->pending_wakeup = 1;
		perf_pending_queue(&handle->event->pending,
				   perf_pending_event);
2727
	} else
2728
		perf_event_wakeup(handle->event);
2729 2730
}

2731 2732 2733
/*
 * Curious locking construct.
 *
2734 2735
 * We need to ensure a later event_id doesn't publish a head when a former
 * event_id isn't done writing. However since we need to deal with NMIs we
2736 2737 2738 2739 2740 2741
 * cannot fully serialize things.
 *
 * What we do is serialize between CPUs so we only have to deal with NMI
 * nesting on a single CPU.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
2742
 * event_id completes.
2743 2744 2745 2746
 */
static void perf_output_lock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2747
	int cur, cpu = get_cpu();
2748 2749 2750

	handle->locked = 0;

2751 2752 2753 2754 2755 2756 2757 2758
	for (;;) {
		cur = atomic_cmpxchg(&data->lock, -1, cpu);
		if (cur == -1) {
			handle->locked = 1;
			break;
		}
		if (cur == cpu)
			break;
2759 2760

		cpu_relax();
2761
	}
2762 2763 2764 2765 2766
}

static void perf_output_unlock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2767 2768
	unsigned long head;
	int cpu;
2769

2770
	data->done_head = data->head;
2771 2772 2773 2774 2775 2776 2777 2778 2779 2780

	if (!handle->locked)
		goto out;

again:
	/*
	 * The xchg implies a full barrier that ensures all writes are done
	 * before we publish the new head, matched by a rmb() in userspace when
	 * reading this position.
	 */
2781
	while ((head = atomic_long_xchg(&data->done_head, 0)))
2782 2783 2784
		data->user_page->data_head = head;

	/*
2785
	 * NMI can happen here, which means we can miss a done_head update.
2786 2787
	 */

2788
	cpu = atomic_xchg(&data->lock, -1);
2789 2790 2791 2792 2793
	WARN_ON_ONCE(cpu != smp_processor_id());

	/*
	 * Therefore we have to validate we did not indeed do so.
	 */
2794
	if (unlikely(atomic_long_read(&data->done_head))) {
2795 2796 2797
		/*
		 * Since we had it locked, we can lock it again.
		 */
2798
		while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2799 2800 2801 2802 2803
			cpu_relax();

		goto again;
	}

2804
	if (atomic_xchg(&data->wakeup, 0))
2805 2806
		perf_output_wakeup(handle);
out:
2807
	put_cpu();
2808 2809
}

2810 2811
void perf_output_copy(struct perf_output_handle *handle,
		      const void *buf, unsigned int len)
2812 2813
{
	unsigned int pages_mask;
2814
	unsigned long offset;
2815 2816 2817 2818 2819 2820 2821 2822
	unsigned int size;
	void **pages;

	offset		= handle->offset;
	pages_mask	= handle->data->nr_pages - 1;
	pages		= handle->data->data_pages;

	do {
2823 2824
		unsigned long page_offset;
		unsigned long page_size;
2825 2826 2827
		int nr;

		nr	    = (offset >> PAGE_SHIFT) & pages_mask;
2828 2829 2830
		page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
		page_offset = offset & (page_size - 1);
		size	    = min_t(unsigned int, page_size - page_offset, len);
2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847

		memcpy(pages[nr] + page_offset, buf, size);

		len	    -= size;
		buf	    += size;
		offset	    += size;
	} while (len);

	handle->offset = offset;

	/*
	 * Check we didn't copy past our reservation window, taking the
	 * possible unsigned int wrap into account.
	 */
	WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
}

2848
int perf_output_begin(struct perf_output_handle *handle,
2849
		      struct perf_event *event, unsigned int size,
2850
		      int nmi, int sample)
2851
{
2852
	struct perf_event *output_event;
2853
	struct perf_mmap_data *data;
2854
	unsigned long tail, offset, head;
2855 2856 2857 2858 2859 2860
	int have_lost;
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
2861

2862
	rcu_read_lock();
2863
	/*
2864
	 * For inherited events we send all the output towards the parent.
2865
	 */
2866 2867
	if (event->parent)
		event = event->parent;
2868

2869 2870 2871
	output_event = rcu_dereference(event->output);
	if (output_event)
		event = output_event;
2872

2873
	data = rcu_dereference(event->data);
2874 2875 2876
	if (!data)
		goto out;

2877
	handle->data	= data;
2878
	handle->event	= event;
2879 2880
	handle->nmi	= nmi;
	handle->sample	= sample;
2881

2882
	if (!data->nr_pages)
2883
		goto fail;
2884

2885 2886 2887 2888
	have_lost = atomic_read(&data->lost);
	if (have_lost)
		size += sizeof(lost_event);

2889 2890
	perf_output_lock(handle);

2891
	do {
2892 2893 2894 2895 2896 2897 2898
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
		tail = ACCESS_ONCE(data->user_page->data_tail);
		smp_rmb();
2899
		offset = head = atomic_long_read(&data->head);
P
Peter Zijlstra 已提交
2900
		head += size;
2901
		if (unlikely(!perf_output_space(data, tail, offset, head)))
2902
			goto fail;
2903
	} while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2904

2905
	handle->offset	= offset;
2906
	handle->head	= head;
2907

2908
	if (head - tail > data->watermark)
2909
		atomic_set(&data->wakeup, 1);
2910

2911
	if (have_lost) {
2912
		lost_event.header.type = PERF_RECORD_LOST;
2913 2914
		lost_event.header.misc = 0;
		lost_event.header.size = sizeof(lost_event);
2915
		lost_event.id          = event->id;
2916 2917 2918 2919 2920
		lost_event.lost        = atomic_xchg(&data->lost, 0);

		perf_output_put(handle, lost_event);
	}

2921
	return 0;
2922

2923
fail:
2924 2925
	atomic_inc(&data->lost);
	perf_output_unlock(handle);
2926 2927
out:
	rcu_read_unlock();
2928

2929 2930
	return -ENOSPC;
}
2931

2932
void perf_output_end(struct perf_output_handle *handle)
2933
{
2934
	struct perf_event *event = handle->event;
2935 2936
	struct perf_mmap_data *data = handle->data;

2937
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
2938

2939
	if (handle->sample && wakeup_events) {
2940
		int events = atomic_inc_return(&data->events);
P
Peter Zijlstra 已提交
2941
		if (events >= wakeup_events) {
2942
			atomic_sub(wakeup_events, &data->events);
2943
			atomic_set(&data->wakeup, 1);
P
Peter Zijlstra 已提交
2944
		}
2945 2946 2947
	}

	perf_output_unlock(handle);
2948
	rcu_read_unlock();
2949 2950
}

2951
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
2952 2953
{
	/*
2954
	 * only top level events have the pid namespace they were created in
2955
	 */
2956 2957
	if (event->parent)
		event = event->parent;
2958

2959
	return task_tgid_nr_ns(p, event->ns);
2960 2961
}

2962
static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
2963 2964
{
	/*
2965
	 * only top level events have the pid namespace they were created in
2966
	 */
2967 2968
	if (event->parent)
		event = event->parent;
2969

2970
	return task_pid_nr_ns(p, event->ns);
2971 2972
}

2973
static void perf_output_read_one(struct perf_output_handle *handle,
2974
				 struct perf_event *event)
2975
{
2976
	u64 read_format = event->attr.read_format;
2977 2978 2979
	u64 values[4];
	int n = 0;

2980
	values[n++] = atomic64_read(&event->count);
2981
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2982 2983
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2984 2985
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2986 2987
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2988 2989
	}
	if (read_format & PERF_FORMAT_ID)
2990
		values[n++] = primary_event_id(event);
2991 2992 2993 2994 2995

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
2996
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
2997 2998
 */
static void perf_output_read_group(struct perf_output_handle *handle,
2999
			    struct perf_event *event)
3000
{
3001 3002
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = leader->total_time_enabled;

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = leader->total_time_running;

3014
	if (leader != event)
3015 3016 3017 3018
		leader->pmu->read(leader);

	values[n++] = atomic64_read(&leader->count);
	if (read_format & PERF_FORMAT_ID)
3019
		values[n++] = primary_event_id(leader);
3020 3021 3022

	perf_output_copy(handle, values, n * sizeof(u64));

3023
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3024 3025
		n = 0;

3026
		if (sub != event)
3027 3028 3029 3030
			sub->pmu->read(sub);

		values[n++] = atomic64_read(&sub->count);
		if (read_format & PERF_FORMAT_ID)
3031
			values[n++] = primary_event_id(sub);
3032 3033 3034 3035 3036 3037

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

static void perf_output_read(struct perf_output_handle *handle,
3038
			     struct perf_event *event)
3039
{
3040 3041
	if (event->attr.read_format & PERF_FORMAT_GROUP)
		perf_output_read_group(handle, event);
3042
	else
3043
		perf_output_read_one(handle, event);
3044 3045
}

3046 3047 3048
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3049
			struct perf_event *event)
3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3080
		perf_output_read(handle, event);
3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3118
			 struct perf_event *event,
3119
			 struct pt_regs *regs)
3120
{
3121
	u64 sample_type = event->attr.sample_type;
3122

3123
	data->type = sample_type;
3124

3125
	header->type = PERF_RECORD_SAMPLE;
3126 3127 3128 3129
	header->size = sizeof(*header);

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3130

3131
	if (sample_type & PERF_SAMPLE_IP) {
3132 3133 3134
		data->ip = perf_instruction_pointer(regs);

		header->size += sizeof(data->ip);
3135
	}
3136

3137
	if (sample_type & PERF_SAMPLE_TID) {
3138
		/* namespace issues */
3139 3140
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
3141

3142
		header->size += sizeof(data->tid_entry);
3143 3144
	}

3145
	if (sample_type & PERF_SAMPLE_TIME) {
P
Peter Zijlstra 已提交
3146
		data->time = perf_clock();
3147

3148
		header->size += sizeof(data->time);
3149 3150
	}

3151
	if (sample_type & PERF_SAMPLE_ADDR)
3152
		header->size += sizeof(data->addr);
3153

3154
	if (sample_type & PERF_SAMPLE_ID) {
3155
		data->id = primary_event_id(event);
3156

3157 3158 3159 3160
		header->size += sizeof(data->id);
	}

	if (sample_type & PERF_SAMPLE_STREAM_ID) {
3161
		data->stream_id = event->id;
3162 3163 3164

		header->size += sizeof(data->stream_id);
	}
3165

3166
	if (sample_type & PERF_SAMPLE_CPU) {
3167 3168
		data->cpu_entry.cpu		= raw_smp_processor_id();
		data->cpu_entry.reserved	= 0;
3169

3170
		header->size += sizeof(data->cpu_entry);
3171 3172
	}

3173
	if (sample_type & PERF_SAMPLE_PERIOD)
3174
		header->size += sizeof(data->period);
3175

3176
	if (sample_type & PERF_SAMPLE_READ)
3177
		header->size += perf_event_read_size(event);
3178

3179
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3180
		int size = 1;
3181

3182 3183 3184 3185 3186 3187
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3188 3189
	}

3190
	if (sample_type & PERF_SAMPLE_RAW) {
3191 3192 3193 3194 3195 3196 3197 3198
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3199
		header->size += size;
3200
	}
3201
}
3202

3203
static void perf_event_output(struct perf_event *event, int nmi,
3204 3205 3206 3207 3208
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3209

3210
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3211

3212
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3213
		return;
3214

3215
	perf_output_sample(&handle, &header, data, event);
3216

3217
	perf_output_end(&handle);
3218 3219
}

3220
/*
3221
 * read event_id
3222 3223 3224 3225 3226 3227 3228 3229 3230 3231
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3232
perf_event_read_event(struct perf_event *event,
3233 3234 3235
			struct task_struct *task)
{
	struct perf_output_handle handle;
3236
	struct perf_read_event read_event = {
3237
		.header = {
3238
			.type = PERF_RECORD_READ,
3239
			.misc = 0,
3240
			.size = sizeof(read_event) + perf_event_read_size(event),
3241
		},
3242 3243
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3244
	};
3245
	int ret;
3246

3247
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3248 3249 3250
	if (ret)
		return;

3251
	perf_output_put(&handle, read_event);
3252
	perf_output_read(&handle, event);
3253

3254 3255 3256
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3257
/*
P
Peter Zijlstra 已提交
3258 3259 3260
 * task tracking -- fork/exit
 *
 * enabled by: attr.comm | attr.mmap | attr.task
P
Peter Zijlstra 已提交
3261 3262
 */

P
Peter Zijlstra 已提交
3263
struct perf_task_event {
3264
	struct task_struct		*task;
3265
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3266 3267 3268 3269 3270 3271

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3272 3273
		u32				tid;
		u32				ptid;
3274
		u64				time;
3275
	} event_id;
P
Peter Zijlstra 已提交
3276 3277
};

3278
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3279
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3280 3281
{
	struct perf_output_handle handle;
3282
	int size;
P
Peter Zijlstra 已提交
3283
	struct task_struct *task = task_event->task;
3284 3285
	int ret;

3286 3287
	size  = task_event->event_id.header.size;
	ret = perf_output_begin(&handle, event, size, 0, 0);
P
Peter Zijlstra 已提交
3288 3289 3290 3291

	if (ret)
		return;

3292 3293
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3294

3295 3296
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3297

3298
	task_event->event_id.time = perf_clock();
3299

3300
	perf_output_put(&handle, task_event->event_id);
3301

P
Peter Zijlstra 已提交
3302 3303 3304
	perf_output_end(&handle);
}

3305
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3306
{
3307 3308 3309
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3310
	if (event->attr.comm || event->attr.mmap || event->attr.task)
P
Peter Zijlstra 已提交
3311 3312 3313 3314 3315
		return 1;

	return 0;
}

3316
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
3317
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3318
{
3319
	struct perf_event *event;
P
Peter Zijlstra 已提交
3320

3321 3322 3323
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
3324 3325 3326
	}
}

3327
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3328 3329
{
	struct perf_cpu_context *cpuctx;
3330
	struct perf_event_context *ctx = task_event->task_ctx;
P
Peter Zijlstra 已提交
3331

3332
	rcu_read_lock();
P
Peter Zijlstra 已提交
3333
	cpuctx = &get_cpu_var(perf_cpu_context);
3334
	perf_event_task_ctx(&cpuctx->ctx, task_event);
3335
	if (!ctx)
3336
		ctx = rcu_dereference(task_event->task->perf_event_ctxp);
P
Peter Zijlstra 已提交
3337
	if (ctx)
3338
		perf_event_task_ctx(ctx, task_event);
3339
	put_cpu_var(perf_cpu_context);
P
Peter Zijlstra 已提交
3340 3341 3342
	rcu_read_unlock();
}

3343 3344
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3345
			      int new)
P
Peter Zijlstra 已提交
3346
{
P
Peter Zijlstra 已提交
3347
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
3348

3349 3350 3351
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
3352 3353
		return;

P
Peter Zijlstra 已提交
3354
	task_event = (struct perf_task_event){
3355 3356
		.task	  = task,
		.task_ctx = task_ctx,
3357
		.event_id    = {
P
Peter Zijlstra 已提交
3358
			.header = {
3359
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3360
				.misc = 0,
3361
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
3362
			},
3363 3364
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
3365 3366
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
3367 3368 3369
		},
	};

3370
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
3371 3372
}

3373
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
3374
{
3375
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
3376 3377
}

3378 3379 3380 3381 3382
/*
 * comm tracking
 */

struct perf_comm_event {
3383 3384
	struct task_struct	*task;
	char			*comm;
3385 3386 3387 3388 3389 3390 3391
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
3392
	} event_id;
3393 3394
};

3395
static void perf_event_comm_output(struct perf_event *event,
3396 3397 3398
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
3399 3400
	int size = comm_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3401 3402 3403 3404

	if (ret)
		return;

3405 3406
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3407

3408
	perf_output_put(&handle, comm_event->event_id);
3409 3410 3411 3412 3413
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
	perf_output_end(&handle);
}

3414
static int perf_event_comm_match(struct perf_event *event)
3415
{
3416 3417 3418
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3419
	if (event->attr.comm)
3420 3421 3422 3423 3424
		return 1;

	return 0;
}

3425
static void perf_event_comm_ctx(struct perf_event_context *ctx,
3426 3427
				  struct perf_comm_event *comm_event)
{
3428
	struct perf_event *event;
3429

3430 3431 3432
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
3433 3434 3435
	}
}

3436
static void perf_event_comm_event(struct perf_comm_event *comm_event)
3437 3438
{
	struct perf_cpu_context *cpuctx;
3439
	struct perf_event_context *ctx;
3440
	unsigned int size;
3441
	char comm[TASK_COMM_LEN];
3442

3443
	memset(comm, 0, sizeof(comm));
3444
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
3445
	size = ALIGN(strlen(comm)+1, sizeof(u64));
3446 3447 3448 3449

	comm_event->comm = comm;
	comm_event->comm_size = size;

3450
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3451

3452
	rcu_read_lock();
3453
	cpuctx = &get_cpu_var(perf_cpu_context);
3454 3455
	perf_event_comm_ctx(&cpuctx->ctx, comm_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3456
	if (ctx)
3457
		perf_event_comm_ctx(ctx, comm_event);
3458
	put_cpu_var(perf_cpu_context);
3459
	rcu_read_unlock();
3460 3461
}

3462
void perf_event_comm(struct task_struct *task)
3463
{
3464 3465
	struct perf_comm_event comm_event;

3466 3467
	if (task->perf_event_ctxp)
		perf_event_enable_on_exec(task);
3468

3469
	if (!atomic_read(&nr_comm_events))
3470
		return;
3471

3472
	comm_event = (struct perf_comm_event){
3473
		.task	= task,
3474 3475
		/* .comm      */
		/* .comm_size */
3476
		.event_id  = {
3477
			.header = {
3478
				.type = PERF_RECORD_COMM,
3479 3480 3481 3482 3483
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3484 3485 3486
		},
	};

3487
	perf_event_comm_event(&comm_event);
3488 3489
}

3490 3491 3492 3493 3494
/*
 * mmap tracking
 */

struct perf_mmap_event {
3495 3496 3497 3498
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
3499 3500 3501 3502 3503 3504 3505 3506 3507

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
3508
	} event_id;
3509 3510
};

3511
static void perf_event_mmap_output(struct perf_event *event,
3512 3513 3514
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
3515 3516
	int size = mmap_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3517 3518 3519 3520

	if (ret)
		return;

3521 3522
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
3523

3524
	perf_output_put(&handle, mmap_event->event_id);
3525 3526
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
3527
	perf_output_end(&handle);
3528 3529
}

3530
static int perf_event_mmap_match(struct perf_event *event,
3531 3532
				   struct perf_mmap_event *mmap_event)
{
3533 3534 3535
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3536
	if (event->attr.mmap)
3537 3538 3539 3540 3541
		return 1;

	return 0;
}

3542
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3543 3544
				  struct perf_mmap_event *mmap_event)
{
3545
	struct perf_event *event;
3546

3547 3548 3549
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_mmap_match(event, mmap_event))
			perf_event_mmap_output(event, mmap_event);
3550 3551 3552
	}
}

3553
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3554 3555
{
	struct perf_cpu_context *cpuctx;
3556
	struct perf_event_context *ctx;
3557 3558
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
3559 3560 3561
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
3562
	const char *name;
3563

3564 3565
	memset(tmp, 0, sizeof(tmp));

3566
	if (file) {
3567 3568 3569 3570 3571 3572
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3573 3574 3575 3576
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
3577
		name = d_path(&file->f_path, buf, PATH_MAX);
3578 3579 3580 3581 3582
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
3583 3584 3585
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
3586
			goto got_name;
3587
		}
3588 3589 3590 3591 3592 3593

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
		}

3594 3595 3596 3597 3598
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
3599
	size = ALIGN(strlen(name)+1, sizeof(u64));
3600 3601 3602 3603

	mmap_event->file_name = name;
	mmap_event->file_size = size;

3604
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3605

3606
	rcu_read_lock();
3607
	cpuctx = &get_cpu_var(perf_cpu_context);
3608 3609
	perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3610
	if (ctx)
3611
		perf_event_mmap_ctx(ctx, mmap_event);
3612
	put_cpu_var(perf_cpu_context);
3613 3614
	rcu_read_unlock();

3615 3616 3617
	kfree(buf);
}

3618
void __perf_event_mmap(struct vm_area_struct *vma)
3619
{
3620 3621
	struct perf_mmap_event mmap_event;

3622
	if (!atomic_read(&nr_mmap_events))
3623 3624 3625
		return;

	mmap_event = (struct perf_mmap_event){
3626
		.vma	= vma,
3627 3628
		/* .file_name */
		/* .file_size */
3629
		.event_id  = {
3630
			.header = {
3631
				.type = PERF_RECORD_MMAP,
3632 3633 3634 3635 3636
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3637 3638 3639
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
			.pgoff  = vma->vm_pgoff,
3640 3641 3642
		},
	};

3643
	perf_event_mmap_event(&mmap_event);
3644 3645
}

3646 3647 3648 3649
/*
 * IRQ throttle logging
 */

3650
static void perf_log_throttle(struct perf_event *event, int enable)
3651 3652 3653 3654 3655 3656 3657
{
	struct perf_output_handle handle;
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
3658
		u64				id;
3659
		u64				stream_id;
3660 3661
	} throttle_event = {
		.header = {
3662
			.type = PERF_RECORD_THROTTLE,
3663 3664 3665
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
3666
		.time		= perf_clock(),
3667 3668
		.id		= primary_event_id(event),
		.stream_id	= event->id,
3669 3670
	};

3671
	if (enable)
3672
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3673

3674
	ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3675 3676 3677 3678 3679 3680 3681
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
	perf_output_end(&handle);
}

3682
/*
3683
 * Generic event overflow handling, sampling.
3684 3685
 */

3686
static int __perf_event_overflow(struct perf_event *event, int nmi,
3687 3688
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
3689
{
3690 3691
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
3692 3693
	int ret = 0;

3694
	throttle = (throttle && event->pmu->unthrottle != NULL);
3695

3696
	if (!throttle) {
3697
		hwc->interrupts++;
3698
	} else {
3699 3700
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
3701
			if (HZ * hwc->interrupts >
3702
					(u64)sysctl_perf_event_sample_rate) {
3703
				hwc->interrupts = MAX_INTERRUPTS;
3704
				perf_log_throttle(event, 0);
3705 3706 3707 3708
				ret = 1;
			}
		} else {
			/*
3709
			 * Keep re-disabling events even though on the previous
3710
			 * pass we disabled it - just in case we raced with a
3711
			 * sched-in and the event got enabled again:
3712
			 */
3713 3714 3715
			ret = 1;
		}
	}
3716

3717
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
3718
		u64 now = perf_clock();
3719 3720 3721 3722 3723
		s64 delta = now - hwc->freq_stamp;

		hwc->freq_stamp = now;

		if (delta > 0 && delta < TICK_NSEC)
3724
			perf_adjust_period(event, NSEC_PER_SEC / (int)delta);
3725 3726
	}

3727 3728
	/*
	 * XXX event_limit might not quite work as expected on inherited
3729
	 * events
3730 3731
	 */

3732 3733
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
3734
		ret = 1;
3735
		event->pending_kill = POLL_HUP;
3736
		if (nmi) {
3737 3738 3739
			event->pending_disable = 1;
			perf_pending_queue(&event->pending,
					   perf_pending_event);
3740
		} else
3741
			perf_event_disable(event);
3742 3743
	}

3744 3745 3746 3747 3748
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

3749
	return ret;
3750 3751
}

3752
int perf_event_overflow(struct perf_event *event, int nmi,
3753 3754
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
3755
{
3756
	return __perf_event_overflow(event, nmi, 1, data, regs);
3757 3758
}

3759
/*
3760
 * Generic software event infrastructure
3761 3762
 */

3763
/*
3764 3765
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
3766 3767 3768 3769
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

3770
static u64 perf_swevent_set_period(struct perf_event *event)
3771
{
3772
	struct hw_perf_event *hwc = &event->hw;
3773 3774 3775 3776 3777
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
3778 3779

again:
3780 3781 3782
	old = val = atomic64_read(&hwc->period_left);
	if (val < 0)
		return 0;
3783

3784 3785 3786 3787 3788
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
	if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
		goto again;
3789

3790
	return nr;
3791 3792
}

3793
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3794 3795
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
3796
{
3797
	struct hw_perf_event *hwc = &event->hw;
3798
	int throttle = 0;
3799

3800
	data->period = event->hw.last_period;
3801 3802
	if (!overflow)
		overflow = perf_swevent_set_period(event);
3803

3804 3805
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
3806

3807
	for (; overflow; overflow--) {
3808
		if (__perf_event_overflow(event, nmi, throttle,
3809
					    data, regs)) {
3810 3811 3812 3813 3814 3815
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
3816
		throttle = 1;
3817
	}
3818 3819
}

3820
static void perf_swevent_unthrottle(struct perf_event *event)
3821 3822
{
	/*
3823
	 * Nothing to do, we already reset hwc->interrupts.
3824
	 */
3825
}
3826

3827
static void perf_swevent_add(struct perf_event *event, u64 nr,
3828 3829
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
3830
{
3831
	struct hw_perf_event *hwc = &event->hw;
3832

3833
	atomic64_add(nr, &event->count);
3834

3835 3836 3837
	if (!regs)
		return;

3838 3839
	if (!hwc->sample_period)
		return;
3840

3841 3842 3843 3844
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

	if (atomic64_add_negative(nr, &hwc->period_left))
3845
		return;
3846

3847
	perf_swevent_overflow(event, 0, nmi, data, regs);
3848 3849
}

3850
static int perf_swevent_is_counting(struct perf_event *event)
3851
{
3852
	/*
3853
	 * The event is active, we're good!
3854
	 */
3855
	if (event->state == PERF_EVENT_STATE_ACTIVE)
3856 3857
		return 1;

3858
	/*
3859
	 * The event is off/error, not counting.
3860
	 */
3861
	if (event->state != PERF_EVENT_STATE_INACTIVE)
3862 3863 3864
		return 0;

	/*
3865
	 * The event is inactive, if the context is active
3866 3867
	 * we're part of a group that didn't make it on the 'pmu',
	 * not counting.
3868
	 */
3869
	if (event->ctx->is_active)
3870 3871 3872 3873 3874 3875 3876 3877
		return 0;

	/*
	 * We're inactive and the context is too, this means the
	 * task is scheduled out, we're counting events that happen
	 * to us, like migration events.
	 */
	return 1;
3878 3879
}

L
Li Zefan 已提交
3880 3881 3882
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data);

3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896
static int perf_exclude_event(struct perf_event *event,
			      struct pt_regs *regs)
{
	if (regs) {
		if (event->attr.exclude_user && user_mode(regs))
			return 1;

		if (event->attr.exclude_kernel && !user_mode(regs))
			return 1;
	}

	return 0;
}

3897
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
3898
				enum perf_type_id type,
L
Li Zefan 已提交
3899 3900 3901
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
3902
{
3903 3904 3905
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3906
	if (!perf_swevent_is_counting(event))
3907 3908
		return 0;

3909
	if (event->attr.type != type)
3910
		return 0;
3911

3912
	if (event->attr.config != event_id)
3913 3914
		return 0;

3915 3916
	if (perf_exclude_event(event, regs))
		return 0;
3917

L
Li Zefan 已提交
3918 3919 3920 3921
	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
	    !perf_tp_event_match(event, data))
		return 0;

3922 3923 3924
	return 1;
}

3925
static void perf_swevent_ctx_event(struct perf_event_context *ctx,
3926
				     enum perf_type_id type,
3927
				     u32 event_id, u64 nr, int nmi,
3928 3929
				     struct perf_sample_data *data,
				     struct pt_regs *regs)
3930
{
3931
	struct perf_event *event;
3932

3933
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
L
Li Zefan 已提交
3934
		if (perf_swevent_match(event, type, event_id, data, regs))
3935
			perf_swevent_add(event, nr, nmi, data, regs);
3936 3937 3938
	}
}

3939
int perf_swevent_get_recursion_context(void)
P
Peter Zijlstra 已提交
3940
{
3941 3942
	struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
	int rctx;
3943

P
Peter Zijlstra 已提交
3944
	if (in_nmi())
3945
		rctx = 3;
3946
	else if (in_irq())
3947
		rctx = 2;
3948
	else if (in_softirq())
3949
		rctx = 1;
3950
	else
3951
		rctx = 0;
P
Peter Zijlstra 已提交
3952

3953 3954
	if (cpuctx->recursion[rctx]) {
		put_cpu_var(perf_cpu_context);
3955
		return -1;
3956
	}
P
Peter Zijlstra 已提交
3957

3958 3959
	cpuctx->recursion[rctx]++;
	barrier();
P
Peter Zijlstra 已提交
3960

3961
	return rctx;
P
Peter Zijlstra 已提交
3962
}
I
Ingo Molnar 已提交
3963
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
3964

3965
void perf_swevent_put_recursion_context(int rctx)
3966
{
3967 3968
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	barrier();
3969
	cpuctx->recursion[rctx]--;
3970
	put_cpu_var(perf_cpu_context);
3971
}
I
Ingo Molnar 已提交
3972
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
P
Peter Zijlstra 已提交
3973

3974 3975 3976 3977
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
				    u64 nr, int nmi,
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
3978
{
3979
	struct perf_cpu_context *cpuctx;
3980
	struct perf_event_context *ctx;
3981

3982
	cpuctx = &__get_cpu_var(perf_cpu_context);
3983
	rcu_read_lock();
3984
	perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
3985
				 nr, nmi, data, regs);
3986 3987 3988 3989
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3990
	ctx = rcu_dereference(current->perf_event_ctxp);
3991
	if (ctx)
3992
		perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs);
3993
	rcu_read_unlock();
3994
}
3995

3996
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
3997
			    struct pt_regs *regs, u64 addr)
3998
{
3999
	struct perf_sample_data data;
4000 4001 4002 4003 4004
	int rctx;

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return;
4005 4006 4007

	data.addr = addr;
	data.raw  = NULL;
4008

4009
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4010 4011

	perf_swevent_put_recursion_context(rctx);
4012 4013
}

4014
static void perf_swevent_read(struct perf_event *event)
4015 4016 4017
{
}

4018
static int perf_swevent_enable(struct perf_event *event)
4019
{
4020
	struct hw_perf_event *hwc = &event->hw;
4021 4022 4023

	if (hwc->sample_period) {
		hwc->last_period = hwc->sample_period;
4024
		perf_swevent_set_period(event);
4025
	}
4026 4027 4028
	return 0;
}

4029
static void perf_swevent_disable(struct perf_event *event)
4030 4031 4032
{
}

4033
static const struct pmu perf_ops_generic = {
4034 4035 4036 4037
	.enable		= perf_swevent_enable,
	.disable	= perf_swevent_disable,
	.read		= perf_swevent_read,
	.unthrottle	= perf_swevent_unthrottle,
4038 4039
};

4040
/*
4041
 * hrtimer based swevent callback
4042 4043
 */

4044
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4045 4046 4047
{
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
4048
	struct pt_regs *regs;
4049
	struct perf_event *event;
4050 4051
	u64 period;

4052 4053
	event	= container_of(hrtimer, struct perf_event, hw.hrtimer);
	event->pmu->read(event);
4054 4055

	data.addr = 0;
4056
	data.raw = NULL;
4057
	data.period = event->hw.last_period;
4058
	regs = get_irq_regs();
4059 4060 4061 4062
	/*
	 * In case we exclude kernel IPs or are somehow not in interrupt
	 * context, provide the next best thing, the user IP.
	 */
4063 4064
	if ((event->attr.exclude_kernel || !regs) &&
			!event->attr.exclude_user)
4065
		regs = task_pt_regs(current);
4066

4067
	if (regs) {
4068 4069 4070
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
4071 4072
	}

4073
	period = max_t(u64, 10000, event->hw.sample_period);
4074 4075 4076 4077 4078
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));

	return ret;
}

4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114
static void perf_swevent_start_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
	if (hwc->sample_period) {
		u64 period;

		if (hwc->remaining) {
			if (hwc->remaining < 0)
				period = 10000;
			else
				period = hwc->remaining;
			hwc->remaining = 0;
		} else {
			period = max_t(u64, 10000, hwc->sample_period);
		}
		__hrtimer_start_range_ns(&hwc->hrtimer,
				ns_to_ktime(period), 0,
				HRTIMER_MODE_REL, 0);
	}
}

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (hwc->sample_period) {
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
		hwc->remaining = ktime_to_ns(remaining);

		hrtimer_cancel(&hwc->hrtimer);
	}
}

4115
/*
4116
 * Software event: cpu wall time clock
4117 4118
 */

4119
static void cpu_clock_perf_event_update(struct perf_event *event)
4120 4121 4122 4123 4124 4125
{
	int cpu = raw_smp_processor_id();
	s64 prev;
	u64 now;

	now = cpu_clock(cpu);
4126
	prev = atomic64_xchg(&event->hw.prev_count, now);
4127
	atomic64_add(now - prev, &event->count);
4128 4129
}

4130
static int cpu_clock_perf_event_enable(struct perf_event *event)
4131
{
4132
	struct hw_perf_event *hwc = &event->hw;
4133 4134 4135
	int cpu = raw_smp_processor_id();

	atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4136
	perf_swevent_start_hrtimer(event);
4137 4138 4139 4140

	return 0;
}

4141
static void cpu_clock_perf_event_disable(struct perf_event *event)
4142
{
4143
	perf_swevent_cancel_hrtimer(event);
4144
	cpu_clock_perf_event_update(event);
4145 4146
}

4147
static void cpu_clock_perf_event_read(struct perf_event *event)
4148
{
4149
	cpu_clock_perf_event_update(event);
4150 4151
}

4152
static const struct pmu perf_ops_cpu_clock = {
4153 4154 4155
	.enable		= cpu_clock_perf_event_enable,
	.disable	= cpu_clock_perf_event_disable,
	.read		= cpu_clock_perf_event_read,
4156 4157
};

4158
/*
4159
 * Software event: task time clock
4160 4161
 */

4162
static void task_clock_perf_event_update(struct perf_event *event, u64 now)
I
Ingo Molnar 已提交
4163
{
4164
	u64 prev;
I
Ingo Molnar 已提交
4165 4166
	s64 delta;

4167
	prev = atomic64_xchg(&event->hw.prev_count, now);
I
Ingo Molnar 已提交
4168
	delta = now - prev;
4169
	atomic64_add(delta, &event->count);
4170 4171
}

4172
static int task_clock_perf_event_enable(struct perf_event *event)
I
Ingo Molnar 已提交
4173
{
4174
	struct hw_perf_event *hwc = &event->hw;
4175 4176
	u64 now;

4177
	now = event->ctx->time;
4178

4179
	atomic64_set(&hwc->prev_count, now);
4180 4181

	perf_swevent_start_hrtimer(event);
4182 4183

	return 0;
I
Ingo Molnar 已提交
4184 4185
}

4186
static void task_clock_perf_event_disable(struct perf_event *event)
4187
{
4188
	perf_swevent_cancel_hrtimer(event);
4189
	task_clock_perf_event_update(event, event->ctx->time);
4190

4191
}
I
Ingo Molnar 已提交
4192

4193
static void task_clock_perf_event_read(struct perf_event *event)
4194
{
4195 4196 4197
	u64 time;

	if (!in_nmi()) {
4198 4199
		update_context_time(event->ctx);
		time = event->ctx->time;
4200 4201
	} else {
		u64 now = perf_clock();
4202 4203
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
4204 4205
	}

4206
	task_clock_perf_event_update(event, time);
4207 4208
}

4209
static const struct pmu perf_ops_task_clock = {
4210 4211 4212
	.enable		= task_clock_perf_event_enable,
	.disable	= task_clock_perf_event_disable,
	.read		= task_clock_perf_event_read,
4213 4214
};

4215
#ifdef CONFIG_EVENT_TRACING
L
Li Zefan 已提交
4216

4217
void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4218
			  int entry_size)
4219
{
4220
	struct perf_raw_record raw = {
4221
		.size = entry_size,
4222
		.data = record,
4223 4224
	};

4225
	struct perf_sample_data data = {
4226
		.addr = addr,
4227
		.raw = &raw,
4228
	};
4229

4230 4231 4232 4233
	struct pt_regs *regs = get_irq_regs();

	if (!regs)
		regs = task_pt_regs(current);
4234

4235
	/* Trace events already protected against recursion */
4236
	do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4237
				&data, regs);
4238
}
4239
EXPORT_SYMBOL_GPL(perf_tp_event);
4240

L
Li Zefan 已提交
4241 4242 4243 4244 4245 4246 4247 4248 4249
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}
4250

4251
static void tp_perf_event_destroy(struct perf_event *event)
4252
{
4253
	ftrace_profile_disable(event->attr.config);
4254 4255
}

4256
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4257
{
4258 4259 4260 4261
	/*
	 * Raw tracepoint data is a severe data leak, only allow root to
	 * have these.
	 */
4262
	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4263
			perf_paranoid_tracepoint_raw() &&
4264 4265 4266
			!capable(CAP_SYS_ADMIN))
		return ERR_PTR(-EPERM);

4267
	if (ftrace_profile_enable(event->attr.config))
4268 4269
		return NULL;

4270
	event->destroy = tp_perf_event_destroy;
4271 4272 4273

	return &perf_ops_generic;
}
L
Li Zefan 已提交
4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

4298
#else
L
Li Zefan 已提交
4299 4300 4301 4302 4303 4304 4305

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	return 1;
}

4306
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4307 4308 4309
{
	return NULL;
}
L
Li Zefan 已提交
4310 4311 4312 4313 4314 4315 4316 4317 4318 4319

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

4320
#endif /* CONFIG_EVENT_TRACING */
4321

4322 4323 4324 4325 4326 4327 4328 4329 4330
#ifdef CONFIG_HAVE_HW_BREAKPOINT
static void bp_perf_event_destroy(struct perf_event *event)
{
	release_bp_slot(event);
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	int err;
4331 4332

	err = register_perf_hw_breakpoint(bp);
4333 4334 4335 4336 4337 4338 4339 4340
	if (err)
		return ERR_PTR(err);

	bp->destroy = bp_perf_event_destroy;

	return &perf_ops_bp;
}

4341
void perf_bp_event(struct perf_event *bp, void *data)
4342
{
4343 4344 4345
	struct perf_sample_data sample;
	struct pt_regs *regs = data;

4346
	sample.raw = NULL;
4347 4348 4349 4350
	sample.addr = bp->attr.bp_addr;

	if (!perf_exclude_event(bp, regs))
		perf_swevent_add(bp, 1, 1, &sample, regs);
4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362
}
#else
static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	return NULL;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
}
#endif

4363
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4364

4365
static void sw_perf_event_destroy(struct perf_event *event)
4366
{
4367
	u64 event_id = event->attr.config;
4368

4369
	WARN_ON(event->parent);
4370

4371
	atomic_dec(&perf_swevent_enabled[event_id]);
4372 4373
}

4374
static const struct pmu *sw_perf_event_init(struct perf_event *event)
4375
{
4376
	const struct pmu *pmu = NULL;
4377
	u64 event_id = event->attr.config;
4378

4379
	/*
4380
	 * Software events (currently) can't in general distinguish
4381 4382 4383 4384 4385
	 * between user, kernel and hypervisor events.
	 * However, context switches and cpu migrations are considered
	 * to be kernel events, and page faults are never hypervisor
	 * events.
	 */
4386
	switch (event_id) {
4387
	case PERF_COUNT_SW_CPU_CLOCK:
4388
		pmu = &perf_ops_cpu_clock;
4389

4390
		break;
4391
	case PERF_COUNT_SW_TASK_CLOCK:
4392
		/*
4393 4394
		 * If the user instantiates this as a per-cpu event,
		 * use the cpu_clock event instead.
4395
		 */
4396
		if (event->ctx->task)
4397
			pmu = &perf_ops_task_clock;
4398
		else
4399
			pmu = &perf_ops_cpu_clock;
4400

4401
		break;
4402 4403 4404 4405 4406
	case PERF_COUNT_SW_PAGE_FAULTS:
	case PERF_COUNT_SW_PAGE_FAULTS_MIN:
	case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
	case PERF_COUNT_SW_CONTEXT_SWITCHES:
	case PERF_COUNT_SW_CPU_MIGRATIONS:
4407 4408
	case PERF_COUNT_SW_ALIGNMENT_FAULTS:
	case PERF_COUNT_SW_EMULATION_FAULTS:
4409 4410 4411
		if (!event->parent) {
			atomic_inc(&perf_swevent_enabled[event_id]);
			event->destroy = sw_perf_event_destroy;
4412
		}
4413
		pmu = &perf_ops_generic;
4414
		break;
4415
	}
4416

4417
	return pmu;
4418 4419
}

T
Thomas Gleixner 已提交
4420
/*
4421
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
4422
 */
4423 4424
static struct perf_event *
perf_event_alloc(struct perf_event_attr *attr,
4425
		   int cpu,
4426 4427 4428
		   struct perf_event_context *ctx,
		   struct perf_event *group_leader,
		   struct perf_event *parent_event,
4429
		   perf_overflow_handler_t overflow_handler,
4430
		   gfp_t gfpflags)
T
Thomas Gleixner 已提交
4431
{
4432
	const struct pmu *pmu;
4433 4434
	struct perf_event *event;
	struct hw_perf_event *hwc;
4435
	long err;
T
Thomas Gleixner 已提交
4436

4437 4438
	event = kzalloc(sizeof(*event), gfpflags);
	if (!event)
4439
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
4440

4441
	/*
4442
	 * Single events are their own group leaders, with an
4443 4444 4445
	 * empty sibling list:
	 */
	if (!group_leader)
4446
		group_leader = event;
4447

4448 4449
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
4450

4451 4452 4453 4454
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
T
Thomas Gleixner 已提交
4455

4456
	mutex_init(&event->mmap_mutex);
4457

4458 4459 4460 4461 4462 4463
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->ctx		= ctx;
	event->oncpu		= -1;
4464

4465
	event->parent		= parent_event;
4466

4467 4468
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
4469

4470
	event->state		= PERF_EVENT_STATE_INACTIVE;
4471

4472 4473
	if (!overflow_handler && parent_event)
		overflow_handler = parent_event->overflow_handler;
4474
	
4475
	event->overflow_handler	= overflow_handler;
4476

4477
	if (attr->disabled)
4478
		event->state = PERF_EVENT_STATE_OFF;
4479

4480
	pmu = NULL;
4481

4482
	hwc = &event->hw;
4483
	hwc->sample_period = attr->sample_period;
4484
	if (attr->freq && attr->sample_freq)
4485
		hwc->sample_period = 1;
4486
	hwc->last_period = hwc->sample_period;
4487 4488

	atomic64_set(&hwc->period_left, hwc->sample_period);
4489

4490
	/*
4491
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
4492
	 */
4493
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4494 4495
		goto done;

4496
	switch (attr->type) {
4497
	case PERF_TYPE_RAW:
4498
	case PERF_TYPE_HARDWARE:
4499
	case PERF_TYPE_HW_CACHE:
4500
		pmu = hw_perf_event_init(event);
4501 4502 4503
		break;

	case PERF_TYPE_SOFTWARE:
4504
		pmu = sw_perf_event_init(event);
4505 4506 4507
		break;

	case PERF_TYPE_TRACEPOINT:
4508
		pmu = tp_perf_event_init(event);
4509
		break;
4510

4511 4512 4513 4514 4515
	case PERF_TYPE_BREAKPOINT:
		pmu = bp_perf_event_init(event);
		break;


4516 4517
	default:
		break;
4518
	}
4519 4520
done:
	err = 0;
4521
	if (!pmu)
4522
		err = -EINVAL;
4523 4524
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
4525

4526
	if (err) {
4527 4528 4529
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
4530
		return ERR_PTR(err);
I
Ingo Molnar 已提交
4531
	}
4532

4533
	event->pmu = pmu;
T
Thomas Gleixner 已提交
4534

4535 4536 4537 4538 4539 4540 4541 4542
	if (!event->parent) {
		atomic_inc(&nr_events);
		if (event->attr.mmap)
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
4543
	}
4544

4545
	return event;
T
Thomas Gleixner 已提交
4546 4547
}

4548 4549
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
4550 4551
{
	u32 size;
4552
	int ret;
4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
4577 4578 4579
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
4580 4581
	 */
	if (size > sizeof(*attr)) {
4582 4583 4584
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
4585

4586 4587
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
4588

4589
		for (; addr < end; addr++) {
4590 4591 4592 4593 4594 4595
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
4596
		size = sizeof(*attr);
4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

4610
	if (attr->__reserved_1 || attr->__reserved_2)
4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Li Zefan 已提交
4628
static int perf_event_set_output(struct perf_event *event, int output_fd)
4629
{
4630
	struct perf_event *output_event = NULL;
4631
	struct file *output_file = NULL;
4632
	struct perf_event *old_output;
4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645
	int fput_needed = 0;
	int ret = -EINVAL;

	if (!output_fd)
		goto set;

	output_file = fget_light(output_fd, &fput_needed);
	if (!output_file)
		return -EBADF;

	if (output_file->f_op != &perf_fops)
		goto out;

4646
	output_event = output_file->private_data;
4647 4648

	/* Don't chain output fds */
4649
	if (output_event->output)
4650 4651 4652
		goto out;

	/* Don't set an output fd when we already have an output channel */
4653
	if (event->data)
4654 4655 4656 4657 4658
		goto out;

	atomic_long_inc(&output_file->f_count);

set:
4659 4660 4661 4662
	mutex_lock(&event->mmap_mutex);
	old_output = event->output;
	rcu_assign_pointer(event->output, output_event);
	mutex_unlock(&event->mmap_mutex);
4663 4664 4665 4666

	if (old_output) {
		/*
		 * we need to make sure no existing perf_output_*()
4667
		 * is still referencing this event.
4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678
		 */
		synchronize_rcu();
		fput(old_output->filp);
	}

	ret = 0;
out:
	fput_light(output_file, fput_needed);
	return ret;
}

T
Thomas Gleixner 已提交
4679
/**
4680
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
4681
 *
4682
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
4683
 * @pid:		target pid
I
Ingo Molnar 已提交
4684
 * @cpu:		target cpu
4685
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
4686
 */
4687 4688
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
4689
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
4690
{
4691 4692 4693 4694
	struct perf_event *event, *group_leader;
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
4695 4696
	struct file *group_file = NULL;
	int fput_needed = 0;
4697
	int fput_needed2 = 0;
4698
	int err;
T
Thomas Gleixner 已提交
4699

4700
	/* for future expandability... */
4701
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4702 4703
		return -EINVAL;

4704 4705 4706
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
4707

4708 4709 4710 4711 4712
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

4713
	if (attr.freq) {
4714
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
4715 4716 4717
			return -EINVAL;
	}

4718
	/*
I
Ingo Molnar 已提交
4719 4720 4721 4722 4723 4724 4725
	 * Get the target context (task or percpu):
	 */
	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

	/*
4726
	 * Look up the group leader (we will attach this event to it):
4727 4728
	 */
	group_leader = NULL;
4729
	if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4730
		err = -EINVAL;
4731 4732
		group_file = fget_light(group_fd, &fput_needed);
		if (!group_file)
I
Ingo Molnar 已提交
4733
			goto err_put_context;
4734
		if (group_file->f_op != &perf_fops)
I
Ingo Molnar 已提交
4735
			goto err_put_context;
4736 4737 4738

		group_leader = group_file->private_data;
		/*
I
Ingo Molnar 已提交
4739 4740 4741 4742 4743 4744 4745 4746
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
			goto err_put_context;
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
4747
		 */
I
Ingo Molnar 已提交
4748 4749
		if (group_leader->ctx != ctx)
			goto err_put_context;
4750 4751 4752
		/*
		 * Only a group leader can be exclusive or pinned
		 */
4753
		if (attr.exclusive || attr.pinned)
4754
			goto err_put_context;
4755 4756
	}

4757
	event = perf_event_alloc(&attr, cpu, ctx, group_leader,
4758
				     NULL, NULL, GFP_KERNEL);
4759 4760
	err = PTR_ERR(event);
	if (IS_ERR(event))
T
Thomas Gleixner 已提交
4761 4762
		goto err_put_context;

4763
	err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
4764
	if (err < 0)
4765 4766
		goto err_free_put_context;

4767 4768
	event_file = fget_light(err, &fput_needed2);
	if (!event_file)
4769 4770
		goto err_free_put_context;

4771
	if (flags & PERF_FLAG_FD_OUTPUT) {
4772
		err = perf_event_set_output(event, group_fd);
4773 4774
		if (err)
			goto err_fput_free_put_context;
4775 4776
	}

4777
	event->filp = event_file;
4778
	WARN_ON_ONCE(ctx->parent_ctx);
4779
	mutex_lock(&ctx->mutex);
4780
	perf_install_in_context(ctx, event, cpu);
4781
	++ctx->generation;
4782
	mutex_unlock(&ctx->mutex);
4783

4784
	event->owner = current;
4785
	get_task_struct(current);
4786 4787 4788
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
4789

4790
err_fput_free_put_context:
4791
	fput_light(event_file, fput_needed2);
T
Thomas Gleixner 已提交
4792

4793
err_free_put_context:
4794
	if (err < 0)
4795
		kfree(event);
T
Thomas Gleixner 已提交
4796 4797

err_put_context:
4798 4799 4800 4801
	if (err < 0)
		put_ctx(ctx);

	fput_light(group_file, fput_needed);
T
Thomas Gleixner 已提交
4802

4803
	return err;
T
Thomas Gleixner 已提交
4804 4805
}

4806 4807 4808 4809 4810 4811 4812 4813 4814
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
 * @pid: task to profile
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4815 4816
				 pid_t pid,
				 perf_overflow_handler_t overflow_handler)
4817 4818 4819 4820 4821 4822 4823 4824 4825 4826
{
	struct perf_event *event;
	struct perf_event_context *ctx;
	int err;

	/*
	 * Get the target context (task or percpu):
	 */

	ctx = find_get_context(pid, cpu);
4827 4828 4829 4830
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
		goto err_exit;
	}
4831 4832

	event = perf_event_alloc(attr, cpu, ctx, NULL,
4833
				 NULL, overflow_handler, GFP_KERNEL);
4834 4835
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
4836
		goto err_put_context;
4837
	}
4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
	mutex_unlock(&ctx->mutex);

	event->owner = current;
	get_task_struct(current);
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);

	return event;

4854 4855 4856 4857
 err_put_context:
	put_ctx(ctx);
 err_exit:
	return ERR_PTR(err);
4858 4859 4860
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);

4861
/*
4862
 * inherit a event from parent task to child task:
4863
 */
4864 4865
static struct perf_event *
inherit_event(struct perf_event *parent_event,
4866
	      struct task_struct *parent,
4867
	      struct perf_event_context *parent_ctx,
4868
	      struct task_struct *child,
4869 4870
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
4871
{
4872
	struct perf_event *child_event;
4873

4874
	/*
4875 4876
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
4877 4878 4879
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
4880 4881
	if (parent_event->parent)
		parent_event = parent_event->parent;
4882

4883 4884 4885
	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu, child_ctx,
					   group_leader, parent_event,
4886
					   NULL, GFP_KERNEL);
4887 4888
	if (IS_ERR(child_event))
		return child_event;
4889
	get_ctx(child_ctx);
4890

4891
	/*
4892
	 * Make the child state follow the state of the parent event,
4893
	 * not its attr.disabled bit.  We hold the parent's mutex,
4894
	 * so we won't race with perf_event_{en, dis}able_family.
4895
	 */
4896 4897
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
4898
	else
4899
		child_event->state = PERF_EVENT_STATE_OFF;
4900

4901 4902
	if (parent_event->attr.freq)
		child_event->hw.sample_period = parent_event->hw.sample_period;
4903

4904 4905
	child_event->overflow_handler = parent_event->overflow_handler;

4906 4907 4908
	/*
	 * Link it up in the child's context:
	 */
4909
	add_event_to_ctx(child_event, child_ctx);
4910 4911 4912

	/*
	 * Get a reference to the parent filp - we will fput it
4913
	 * when the child event exits. This is safe to do because
4914 4915 4916
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
4917
	atomic_long_inc(&parent_event->filp->f_count);
4918

4919
	/*
4920
	 * Link this into the parent event's child list
4921
	 */
4922 4923 4924 4925
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4926

4927
	return child_event;
4928 4929
}

4930
static int inherit_group(struct perf_event *parent_event,
4931
	      struct task_struct *parent,
4932
	      struct perf_event_context *parent_ctx,
4933
	      struct task_struct *child,
4934
	      struct perf_event_context *child_ctx)
4935
{
4936 4937 4938
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;
4939

4940
	leader = inherit_event(parent_event, parent, parent_ctx,
4941
				 child, NULL, child_ctx);
4942 4943
	if (IS_ERR(leader))
		return PTR_ERR(leader);
4944 4945
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
4946 4947 4948
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
4949
	}
4950 4951 4952
	return 0;
}

4953
static void sync_child_event(struct perf_event *child_event,
4954
			       struct task_struct *child)
4955
{
4956
	struct perf_event *parent_event = child_event->parent;
4957
	u64 child_val;
4958

4959 4960
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
4961

4962
	child_val = atomic64_read(&child_event->count);
4963 4964 4965 4966

	/*
	 * Add back the child's count to the parent's count:
	 */
4967 4968 4969 4970 4971
	atomic64_add(child_val, &parent_event->count);
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
4972 4973

	/*
4974
	 * Remove this event from the parent's list
4975
	 */
4976 4977 4978 4979
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4980 4981

	/*
4982
	 * Release the parent event, if this was the last
4983 4984
	 * reference to it.
	 */
4985
	fput(parent_event->filp);
4986 4987
}

4988
static void
4989 4990
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
4991
			 struct task_struct *child)
4992
{
4993
	struct perf_event *parent_event;
4994

4995
	perf_event_remove_from_context(child_event);
4996

4997
	parent_event = child_event->parent;
4998
	/*
4999
	 * It can happen that parent exits first, and has events
5000
	 * that are still around due to the child reference. These
5001
	 * events need to be zapped - but otherwise linger.
5002
	 */
5003 5004 5005
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
5006
	}
5007 5008 5009
}

/*
5010
 * When a child task exits, feed back event values to parent events.
5011
 */
5012
void perf_event_exit_task(struct task_struct *child)
5013
{
5014 5015
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
5016
	unsigned long flags;
5017

5018 5019
	if (likely(!child->perf_event_ctxp)) {
		perf_event_task(child, NULL, 0);
5020
		return;
P
Peter Zijlstra 已提交
5021
	}
5022

5023
	local_irq_save(flags);
5024 5025 5026 5027 5028 5029
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
5030 5031
	child_ctx = child->perf_event_ctxp;
	__perf_event_task_sched_out(child_ctx);
5032 5033 5034

	/*
	 * Take the context lock here so that if find_get_context is
5035
	 * reading child->perf_event_ctxp, we wait until it has
5036 5037
	 * incremented the context's refcount before we do put_ctx below.
	 */
5038
	raw_spin_lock(&child_ctx->lock);
5039
	child->perf_event_ctxp = NULL;
5040 5041 5042
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
5043
	 * the events from it.
5044 5045
	 */
	unclone_ctx(child_ctx);
5046
	update_context_time(child_ctx);
5047
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
5048 5049

	/*
5050 5051 5052
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
5053
	 */
5054
	perf_event_task(child, child_ctx, 0);
5055

5056 5057 5058
	/*
	 * We can recurse on the same lock type through:
	 *
5059 5060 5061
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
5062 5063 5064 5065 5066 5067
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
	mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
5068

5069
again:
5070 5071 5072 5073 5074
	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
				 group_entry)
		__perf_event_exit_task(child_event, child_ctx, child);

	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
5075
				 group_entry)
5076
		__perf_event_exit_task(child_event, child_ctx, child);
5077 5078

	/*
5079
	 * If the last event was a group event, it will have appended all
5080 5081 5082
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
5083 5084
	if (!list_empty(&child_ctx->pinned_groups) ||
	    !list_empty(&child_ctx->flexible_groups))
5085
		goto again;
5086 5087 5088 5089

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
5090 5091
}

5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109
static void perf_free_event(struct perf_event *event,
			    struct perf_event_context *ctx)
{
	struct perf_event *parent = event->parent;

	if (WARN_ON_ONCE(!parent))
		return;

	mutex_lock(&parent->child_mutex);
	list_del_init(&event->child_list);
	mutex_unlock(&parent->child_mutex);

	fput(parent->filp);

	list_del_event(event, ctx);
	free_event(event);
}

5110 5111 5112 5113
/*
 * free an unexposed, unused context as created by inheritance by
 * init_task below, used by fork() in case of fail.
 */
5114
void perf_event_free_task(struct task_struct *task)
5115
{
5116 5117
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event *event, *tmp;
5118 5119 5120 5121 5122 5123

	if (!ctx)
		return;

	mutex_lock(&ctx->mutex);
again:
5124 5125
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		perf_free_event(event, ctx);
5126

5127 5128 5129
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
				 group_entry)
		perf_free_event(event, ctx);
5130

5131 5132 5133
	if (!list_empty(&ctx->pinned_groups) ||
	    !list_empty(&ctx->flexible_groups))
		goto again;
5134

5135
	mutex_unlock(&ctx->mutex);
5136

5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151
	put_ctx(ctx);
}

static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
		   struct perf_event_context *parent_ctx,
		   struct task_struct *child,
		   int *inherited_all)
{
	int ret;
	struct perf_event_context *child_ctx = child->perf_event_ctxp;

	if (!event->attr.inherit) {
		*inherited_all = 0;
		return 0;
5152 5153
	}

5154 5155 5156 5157 5158 5159 5160
	if (!child_ctx) {
		/*
		 * This is executed from the parent task context, so
		 * inherit events that have been marked for cloning.
		 * First allocate and initialize a context for the
		 * child.
		 */
5161

5162 5163 5164 5165
		child_ctx = kzalloc(sizeof(struct perf_event_context),
				    GFP_KERNEL);
		if (!child_ctx)
			return -ENOMEM;
5166

5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178
		__perf_event_init_context(child_ctx, child);
		child->perf_event_ctxp = child_ctx;
		get_task_struct(child);
	}

	ret = inherit_group(event, parent, parent_ctx,
			    child, child_ctx);

	if (ret)
		*inherited_all = 0;

	return ret;
5179 5180
}

5181

5182
/*
5183
 * Initialize the perf_event context in task_struct
5184
 */
5185
int perf_event_init_task(struct task_struct *child)
5186
{
5187
	struct perf_event_context *child_ctx, *parent_ctx;
5188 5189
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
5190
	struct task_struct *parent = current;
5191
	int inherited_all = 1;
5192
	int ret = 0;
5193

5194
	child->perf_event_ctxp = NULL;
5195

5196 5197
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);
5198

5199
	if (likely(!parent->perf_event_ctxp))
5200 5201
		return 0;

5202
	/*
5203 5204
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
5205
	 */
5206 5207
	parent_ctx = perf_pin_task_context(parent);

5208 5209 5210 5211 5212 5213 5214
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

5215 5216 5217 5218
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
5219
	mutex_lock(&parent_ctx->mutex);
5220 5221 5222 5223 5224

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
5225 5226 5227 5228 5229 5230
	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
			break;
	}
5231

5232 5233 5234 5235
	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
5236
			break;
5237 5238
	}

5239 5240
	child_ctx = child->perf_event_ctxp;

5241
	if (child_ctx && inherited_all) {
5242 5243 5244
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
5245 5246
		 * Note that if the parent is a clone, it could get
		 * uncloned at any point, but that doesn't matter
5247
		 * because the list of events and the generation
5248
		 * count can't have changed since we took the mutex.
5249
		 */
5250 5251 5252
		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
5253
			child_ctx->parent_gen = parent_ctx->parent_gen;
5254 5255 5256 5257 5258
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
5259 5260
	}

5261
	mutex_unlock(&parent_ctx->mutex);
5262

5263
	perf_unpin_context(parent_ctx);
5264

5265
	return ret;
5266 5267
}

5268
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
5269
{
5270
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
5271

5272
	cpuctx = &per_cpu(perf_cpu_context, cpu);
5273
	__perf_event_init_context(&cpuctx->ctx, NULL);
T
Thomas Gleixner 已提交
5274

5275
	spin_lock(&perf_resource_lock);
5276
	cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5277
	spin_unlock(&perf_resource_lock);
5278

5279
	hw_perf_event_setup(cpu);
T
Thomas Gleixner 已提交
5280 5281 5282
}

#ifdef CONFIG_HOTPLUG_CPU
5283
static void __perf_event_exit_cpu(void *info)
T
Thomas Gleixner 已提交
5284 5285
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5286 5287
	struct perf_event_context *ctx = &cpuctx->ctx;
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
5288

5289 5290 5291
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		__perf_event_remove_from_context(event);
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
5292
		__perf_event_remove_from_context(event);
T
Thomas Gleixner 已提交
5293
}
5294
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
5295
{
5296
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5297
	struct perf_event_context *ctx = &cpuctx->ctx;
5298 5299

	mutex_lock(&ctx->mutex);
5300
	smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5301
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
5302 5303
}
#else
5304
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315
#endif

static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action) {

	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5316
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
5317 5318
		break;

5319 5320
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
5321
		hw_perf_event_setup_online(cpu);
5322 5323
		break;

T
Thomas Gleixner 已提交
5324 5325
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
5326
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
5327 5328 5329 5330 5331 5332 5333 5334 5335
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

5336 5337 5338
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
T
Thomas Gleixner 已提交
5339 5340
static struct notifier_block __cpuinitdata perf_cpu_nb = {
	.notifier_call		= perf_cpu_notify,
5341
	.priority		= 20,
T
Thomas Gleixner 已提交
5342 5343
};

5344
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
5345 5346 5347
{
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
			(void *)(long)smp_processor_id());
5348 5349
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
			(void *)(long)smp_processor_id());
T
Thomas Gleixner 已提交
5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369
	register_cpu_notifier(&perf_cpu_nb);
}

static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_reserved_percpu);
}

static ssize_t
perf_set_reserve_percpu(struct sysdev_class *class,
			const char *buf,
			size_t count)
{
	struct perf_cpu_context *cpuctx;
	unsigned long val;
	int err, cpu, mpt;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
5370
	if (val > perf_max_events)
T
Thomas Gleixner 已提交
5371 5372
		return -EINVAL;

5373
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5374 5375 5376
	perf_reserved_percpu = val;
	for_each_online_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
5377
		raw_spin_lock_irq(&cpuctx->ctx.lock);
5378 5379
		mpt = min(perf_max_events - cpuctx->ctx.nr_events,
			  perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
5380
		cpuctx->max_pertask = mpt;
5381
		raw_spin_unlock_irq(&cpuctx->ctx.lock);
T
Thomas Gleixner 已提交
5382
	}
5383
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404

	return count;
}

static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_overcommit);
}

static ssize_t
perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
{
	unsigned long val;
	int err;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
	if (val > 1)
		return -EINVAL;

5405
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5406
	perf_overcommit = val;
5407
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433

	return count;
}

static SYSDEV_CLASS_ATTR(
				reserve_percpu,
				0644,
				perf_show_reserve_percpu,
				perf_set_reserve_percpu
			);

static SYSDEV_CLASS_ATTR(
				overcommit,
				0644,
				perf_show_overcommit,
				perf_set_overcommit
			);

static struct attribute *perfclass_attrs[] = {
	&attr_reserve_percpu.attr,
	&attr_overcommit.attr,
	NULL
};

static struct attribute_group perfclass_attr_group = {
	.attrs			= perfclass_attrs,
5434
	.name			= "perf_events",
T
Thomas Gleixner 已提交
5435 5436
};

5437
static int __init perf_event_sysfs_init(void)
T
Thomas Gleixner 已提交
5438 5439 5440 5441
{
	return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
				  &perfclass_attr_group);
}
5442
device_initcall(perf_event_sysfs_init);