perf_event.c 129.6 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
16
#include <linux/file.h>
T
Thomas Gleixner 已提交
17
#include <linux/poll.h>
18
#include <linux/slab.h>
19
#include <linux/hash.h>
T
Thomas Gleixner 已提交
20
#include <linux/sysfs.h>
21
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
22
#include <linux/percpu.h>
23
#include <linux/ptrace.h>
24
#include <linux/vmstat.h>
25
#include <linux/vmalloc.h>
26 27
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
28 29 30
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
31
#include <linux/kernel_stat.h>
32
#include <linux/perf_event.h>
L
Li Zefan 已提交
33
#include <linux/ftrace_event.h>
34
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
35

36 37
#include <asm/irq_regs.h>

T
Thomas Gleixner 已提交
38
/*
39
 * Each CPU has a list of per CPU events:
T
Thomas Gleixner 已提交
40
 */
41
static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
T
Thomas Gleixner 已提交
42

43
int perf_max_events __read_mostly = 1;
T
Thomas Gleixner 已提交
44 45 46
static int perf_reserved_percpu __read_mostly;
static int perf_overcommit __read_mostly = 1;

47 48 49 50
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
51

52
/*
53
 * perf event paranoia level:
54 55
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
56
 *   1 - disallow cpu events for unpriv
57
 *   2 - disallow kernel profiling for unpriv
58
 */
59
int sysctl_perf_event_paranoid __read_mostly = 1;
60

61
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
62 63

/*
64
 * max perf event sample rate
65
 */
66
int sysctl_perf_event_sample_rate __read_mostly = 100000;
67

68
static atomic64_t perf_event_id;
69

T
Thomas Gleixner 已提交
70
/*
71
 * Lock for (sysadmin-configurable) event reservations:
T
Thomas Gleixner 已提交
72
 */
73
static DEFINE_SPINLOCK(perf_resource_lock);
T
Thomas Gleixner 已提交
74 75 76 77

/*
 * Architecture provided APIs - weak aliases:
 */
78
extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
T
Thomas Gleixner 已提交
79
{
80
	return NULL;
T
Thomas Gleixner 已提交
81 82
}

83 84 85
void __weak hw_perf_disable(void)		{ barrier(); }
void __weak hw_perf_enable(void)		{ barrier(); }

86
void __weak perf_event_print_debug(void)	{ }
87

88
static DEFINE_PER_CPU(int, perf_disable_count);
89 90 91

void perf_disable(void)
{
P
Peter Zijlstra 已提交
92 93
	if (!__get_cpu_var(perf_disable_count)++)
		hw_perf_disable();
94 95 96 97
}

void perf_enable(void)
{
P
Peter Zijlstra 已提交
98
	if (!--__get_cpu_var(perf_disable_count))
99 100 101
		hw_perf_enable();
}

102
static void get_ctx(struct perf_event_context *ctx)
103
{
104
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
105 106
}

107 108
static void free_ctx(struct rcu_head *head)
{
109
	struct perf_event_context *ctx;
110

111
	ctx = container_of(head, struct perf_event_context, rcu_head);
112 113 114
	kfree(ctx);
}

115
static void put_ctx(struct perf_event_context *ctx)
116
{
117 118 119
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
120 121 122
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
123
	}
124 125
}

126
static void unclone_ctx(struct perf_event_context *ctx)
127 128 129 130 131 132 133
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

134
/*
135
 * If we inherit events we want to return the parent event id
136 137
 * to userspace.
 */
138
static u64 primary_event_id(struct perf_event *event)
139
{
140
	u64 id = event->id;
141

142 143
	if (event->parent)
		id = event->parent->id;
144 145 146 147

	return id;
}

148
/*
149
 * Get the perf_event_context for a task and lock it.
150 151 152
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
153
static struct perf_event_context *
154
perf_lock_task_context(struct task_struct *task, unsigned long *flags)
155
{
156
	struct perf_event_context *ctx;
157 158 159

	rcu_read_lock();
 retry:
160
	ctx = rcu_dereference(task->perf_event_ctxp);
161 162 163 164
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
165
		 * perf_event_task_sched_out, though the
166 167 168 169 170 171
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
172
		raw_spin_lock_irqsave(&ctx->lock, *flags);
173
		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
174
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
175 176
			goto retry;
		}
177 178

		if (!atomic_inc_not_zero(&ctx->refcount)) {
179
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
180 181
			ctx = NULL;
		}
182 183 184 185 186 187 188 189 190 191
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
192
static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
193
{
194
	struct perf_event_context *ctx;
195 196 197 198 199
	unsigned long flags;

	ctx = perf_lock_task_context(task, &flags);
	if (ctx) {
		++ctx->pin_count;
200
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
201 202 203 204
	}
	return ctx;
}

205
static void perf_unpin_context(struct perf_event_context *ctx)
206 207 208
{
	unsigned long flags;

209
	raw_spin_lock_irqsave(&ctx->lock, flags);
210
	--ctx->pin_count;
211
	raw_spin_unlock_irqrestore(&ctx->lock, flags);
212 213 214
	put_ctx(ctx);
}

215 216
static inline u64 perf_clock(void)
{
217
	return cpu_clock(raw_smp_processor_id());
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
}

/*
 * Update the record of the current time in a context.
 */
static void update_context_time(struct perf_event_context *ctx)
{
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
}

/*
 * Update the total_time_enabled and total_time_running fields for a event.
 */
static void update_event_times(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	u64 run_end;

	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
		return;

243 244 245 246 247 248
	if (ctx->is_active)
		run_end = ctx->time;
	else
		run_end = event->tstamp_stopped;

	event->total_time_enabled = run_end - event->tstamp_enabled;
249 250 251 252 253 254 255 256 257

	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
	else
		run_end = ctx->time;

	event->total_time_running = run_end - event->tstamp_running;
}

258 259 260 261 262 263 264 265 266 267 268 269
/*
 * Update total_time_enabled and total_time_running for all events in a group.
 */
static void update_group_times(struct perf_event *leader)
{
	struct perf_event *event;

	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
}

270 271 272 273 274 275 276 277 278
static struct list_head *
ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
{
	if (event->attr.pinned)
		return &ctx->pinned_groups;
	else
		return &ctx->flexible_groups;
}

279
/*
280
 * Add a event from the lists for its context.
281 282
 * Must be called with ctx->mutex and ctx->lock held.
 */
283
static void
284
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
285
{
286
	struct perf_event *group_leader = event->group_leader;
287 288

	/*
289 290
	 * Depending on whether it is a standalone or sibling event,
	 * add it straight to the context's event list, or to the group
291 292
	 * leader's sibling list:
	 */
293 294 295
	if (group_leader == event) {
		struct list_head *list;

296 297 298
		if (is_software_event(event))
			event->group_flags |= PERF_GROUP_SOFTWARE;

299 300 301
		list = ctx_group_list(event, ctx);
		list_add_tail(&event->group_entry, list);
	} else {
302 303 304 305
		if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
		    !is_software_event(event))
			group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;

306
		list_add_tail(&event->group_entry, &group_leader->sibling_list);
P
Peter Zijlstra 已提交
307 308
		group_leader->nr_siblings++;
	}
P
Peter Zijlstra 已提交
309

310 311 312
	list_add_rcu(&event->event_entry, &ctx->event_list);
	ctx->nr_events++;
	if (event->attr.inherit_stat)
313
		ctx->nr_stat++;
314 315
}

316
/*
317
 * Remove a event from the lists for its context.
318
 * Must be called with ctx->mutex and ctx->lock held.
319
 */
320
static void
321
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
322
{
323
	if (list_empty(&event->group_entry))
324
		return;
325 326
	ctx->nr_events--;
	if (event->attr.inherit_stat)
327
		ctx->nr_stat--;
328

329 330
	list_del_init(&event->group_entry);
	list_del_rcu(&event->event_entry);
331

332 333
	if (event->group_leader != event)
		event->group_leader->nr_siblings--;
P
Peter Zijlstra 已提交
334

335
	update_group_times(event);
336 337 338 339 340 341 342 343 344 345

	/*
	 * If event was in error state, then keep it
	 * that way, otherwise bogus counts will be
	 * returned on read(). The only way to get out
	 * of error state is by explicit re-enabling
	 * of the event
	 */
	if (event->state > PERF_EVENT_STATE_OFF)
		event->state = PERF_EVENT_STATE_OFF;
346 347 348 349 350 351
}

static void
perf_destroy_group(struct perf_event *event, struct perf_event_context *ctx)
{
	struct perf_event *sibling, *tmp;
352

353
	/*
354 355
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
356 357
	 * to the context list directly:
	 */
358
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
359
		struct list_head *list;
360

361 362
		list = ctx_group_list(event, ctx);
		list_move_tail(&sibling->group_entry, list);
363
		sibling->group_leader = sibling;
364 365 366

		/* Inherit group flags from the previous leader */
		sibling->group_flags = event->group_flags;
367 368 369
	}
}

370
static void
371
event_sched_out(struct perf_event *event,
372
		  struct perf_cpu_context *cpuctx,
373
		  struct perf_event_context *ctx)
374
{
375
	if (event->state != PERF_EVENT_STATE_ACTIVE)
376 377
		return;

378 379 380 381
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
382
	}
383 384 385
	event->tstamp_stopped = ctx->time;
	event->pmu->disable(event);
	event->oncpu = -1;
386

387
	if (!is_software_event(event))
388 389
		cpuctx->active_oncpu--;
	ctx->nr_active--;
390
	if (event->attr.exclusive || !cpuctx->active_oncpu)
391 392 393
		cpuctx->exclusive = 0;
}

394
static void
395
group_sched_out(struct perf_event *group_event,
396
		struct perf_cpu_context *cpuctx,
397
		struct perf_event_context *ctx)
398
{
399
	struct perf_event *event;
400

401
	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
402 403
		return;

404
	event_sched_out(group_event, cpuctx, ctx);
405 406 407 408

	/*
	 * Schedule out siblings (if any):
	 */
409 410
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
411

412
	if (group_event->attr.exclusive)
413 414 415
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
416
/*
417
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
418
 *
419
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
420 421
 * remove it from the context list.
 */
422
static void __perf_event_remove_from_context(void *info)
T
Thomas Gleixner 已提交
423 424
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
425 426
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
427 428 429 430 431 432

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
	 */
433
	if (ctx->task && cpuctx->task_ctx != ctx)
T
Thomas Gleixner 已提交
434 435
		return;

436
	raw_spin_lock(&ctx->lock);
437 438
	/*
	 * Protect the list operation against NMI by disabling the
439
	 * events on a global level.
440 441
	 */
	perf_disable();
T
Thomas Gleixner 已提交
442

443
	event_sched_out(event, cpuctx, ctx);
444

445
	list_del_event(event, ctx);
T
Thomas Gleixner 已提交
446 447 448

	if (!ctx->task) {
		/*
449
		 * Allow more per task events with respect to the
T
Thomas Gleixner 已提交
450 451 452
		 * reservation:
		 */
		cpuctx->max_pertask =
453 454
			min(perf_max_events - ctx->nr_events,
			    perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
455 456
	}

457
	perf_enable();
458
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
459 460 461 462
}


/*
463
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
464
 *
465
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
466
 *
467
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
468
 * call when the task is on a CPU.
469
 *
470 471
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
472 473
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
474
 * When called from perf_event_exit_task, it's OK because the
475
 * context has been detached from its task.
T
Thomas Gleixner 已提交
476
 */
477
static void perf_event_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
478
{
479
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
480 481 482 483
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
484
		 * Per cpu events are removed via an smp call and
485
		 * the removal is always successful.
T
Thomas Gleixner 已提交
486
		 */
487 488 489
		smp_call_function_single(event->cpu,
					 __perf_event_remove_from_context,
					 event, 1);
T
Thomas Gleixner 已提交
490 491 492 493
		return;
	}

retry:
494 495
	task_oncpu_function_call(task, __perf_event_remove_from_context,
				 event);
T
Thomas Gleixner 已提交
496

497
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
498 499 500
	/*
	 * If the context is active we need to retry the smp call.
	 */
501
	if (ctx->nr_active && !list_empty(&event->group_entry)) {
502
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
503 504 505 506 507
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
508
	 * can remove the event safely, if the call above did not
T
Thomas Gleixner 已提交
509 510
	 * succeed.
	 */
P
Peter Zijlstra 已提交
511
	if (!list_empty(&event->group_entry))
512
		list_del_event(event, ctx);
513
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
514 515
}

516
/*
517
 * Cross CPU call to disable a performance event
518
 */
519
static void __perf_event_disable(void *info)
520
{
521
	struct perf_event *event = info;
522
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
523
	struct perf_event_context *ctx = event->ctx;
524 525

	/*
526 527
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
528
	 */
529
	if (ctx->task && cpuctx->task_ctx != ctx)
530 531
		return;

532
	raw_spin_lock(&ctx->lock);
533 534

	/*
535
	 * If the event is on, turn it off.
536 537
	 * If it is in error state, leave it in error state.
	 */
538
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
539
		update_context_time(ctx);
540 541 542
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
543
		else
544 545
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
546 547
	}

548
	raw_spin_unlock(&ctx->lock);
549 550 551
}

/*
552
 * Disable a event.
553
 *
554 555
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
556
 * remains valid.  This condition is satisifed when called through
557 558 559 560
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
561
 * is the current context on this CPU and preemption is disabled,
562
 * hence we can't get into perf_event_task_sched_out for this context.
563
 */
564
void perf_event_disable(struct perf_event *event)
565
{
566
	struct perf_event_context *ctx = event->ctx;
567 568 569 570
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
571
		 * Disable the event on the cpu that it's on
572
		 */
573 574
		smp_call_function_single(event->cpu, __perf_event_disable,
					 event, 1);
575 576 577 578
		return;
	}

 retry:
579
	task_oncpu_function_call(task, __perf_event_disable, event);
580

581
	raw_spin_lock_irq(&ctx->lock);
582
	/*
583
	 * If the event is still active, we need to retry the cross-call.
584
	 */
585
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
586
		raw_spin_unlock_irq(&ctx->lock);
587 588 589 590 591 592 593
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
594 595 596
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
597
	}
598

599
	raw_spin_unlock_irq(&ctx->lock);
600 601
}

602
static int
603
event_sched_in(struct perf_event *event,
604
		 struct perf_cpu_context *cpuctx,
605
		 struct perf_event_context *ctx)
606
{
607
	if (event->state <= PERF_EVENT_STATE_OFF)
608 609
		return 0;

610
	event->state = PERF_EVENT_STATE_ACTIVE;
611
	event->oncpu = smp_processor_id();
612 613 614 615 616
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

617 618 619
	if (event->pmu->enable(event)) {
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
620 621 622
		return -EAGAIN;
	}

623
	event->tstamp_running += ctx->time - event->tstamp_stopped;
624

625
	if (!is_software_event(event))
626
		cpuctx->active_oncpu++;
627 628
	ctx->nr_active++;

629
	if (event->attr.exclusive)
630 631
		cpuctx->exclusive = 1;

632 633 634
	return 0;
}

635
static int
636
group_sched_in(struct perf_event *group_event,
637
	       struct perf_cpu_context *cpuctx,
638
	       struct perf_event_context *ctx)
639
{
640 641 642
	struct perf_event *event, *partial_group = NULL;
	const struct pmu *pmu = group_event->pmu;
	bool txn = false;
643 644
	int ret;

645
	if (group_event->state == PERF_EVENT_STATE_OFF)
646 647
		return 0;

648 649 650 651 652 653
	/* Check if group transaction availabe */
	if (pmu->start_txn)
		txn = true;

	if (txn)
		pmu->start_txn(pmu);
654

655
	if (event_sched_in(group_event, cpuctx, ctx))
656 657 658 659 660
		return -EAGAIN;

	/*
	 * Schedule in siblings as one group (if any):
	 */
661
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
662
		if (event_sched_in(event, cpuctx, ctx)) {
663
			partial_group = event;
664 665 666 667
			goto group_error;
		}
	}

668 669
	if (!txn)
		return 0;
670

671 672 673 674
	ret = pmu->commit_txn(pmu);
	if (!ret) {
		pmu->cancel_txn(pmu);
		return 0;
675
	}
676 677

group_error:
678 679 680
	if (txn)
		pmu->cancel_txn(pmu);

681 682 683 684
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
	 */
685 686
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
687
			break;
688
		event_sched_out(event, cpuctx, ctx);
689
	}
690
	event_sched_out(group_event, cpuctx, ctx);
691 692 693 694

	return -EAGAIN;
}

695
/*
696
 * Work out whether we can put this event group on the CPU now.
697
 */
698
static int group_can_go_on(struct perf_event *event,
699 700 701 702
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
703
	 * Groups consisting entirely of software events can always go on.
704
	 */
705
	if (event->group_flags & PERF_GROUP_SOFTWARE)
706 707 708
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
709
	 * events can go on.
710 711 712 713 714
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
715
	 * events on the CPU, it can't go on.
716
	 */
717
	if (event->attr.exclusive && cpuctx->active_oncpu)
718 719 720 721 722 723 724 725
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

726 727
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
728
{
729 730 731 732
	list_add_event(event, ctx);
	event->tstamp_enabled = ctx->time;
	event->tstamp_running = ctx->time;
	event->tstamp_stopped = ctx->time;
733 734
}

T
Thomas Gleixner 已提交
735
/*
736
 * Cross CPU call to install and enable a performance event
737 738
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
739 740 741 742
 */
static void __perf_install_in_context(void *info)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
743 744 745
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
746
	int err;
T
Thomas Gleixner 已提交
747 748 749 750 751

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
752
	 * Or possibly this is the right context but it isn't
753
	 * on this cpu because it had no events.
T
Thomas Gleixner 已提交
754
	 */
755
	if (ctx->task && cpuctx->task_ctx != ctx) {
756
		if (cpuctx->task_ctx || ctx->task != current)
757 758 759
			return;
		cpuctx->task_ctx = ctx;
	}
T
Thomas Gleixner 已提交
760

761
	raw_spin_lock(&ctx->lock);
762
	ctx->is_active = 1;
763
	update_context_time(ctx);
T
Thomas Gleixner 已提交
764 765 766

	/*
	 * Protect the list operation against NMI by disabling the
767
	 * events on a global level. NOP for non NMI based events.
T
Thomas Gleixner 已提交
768
	 */
769
	perf_disable();
T
Thomas Gleixner 已提交
770

771
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
772

773 774 775
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

776
	/*
777
	 * Don't put the event on if it is disabled or if
778 779
	 * it is in a group and the group isn't on.
	 */
780 781
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
782 783
		goto unlock;

784
	/*
785 786 787
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
788
	 */
789
	if (!group_can_go_on(event, cpuctx, 1))
790 791
		err = -EEXIST;
	else
792
		err = event_sched_in(event, cpuctx, ctx);
793

794 795
	if (err) {
		/*
796
		 * This event couldn't go on.  If it is in a group
797
		 * then we have to pull the whole group off.
798
		 * If the event group is pinned then put it in error state.
799
		 */
800
		if (leader != event)
801
			group_sched_out(leader, cpuctx, ctx);
802
		if (leader->attr.pinned) {
803
			update_group_times(leader);
804
			leader->state = PERF_EVENT_STATE_ERROR;
805
		}
806
	}
T
Thomas Gleixner 已提交
807

808
	if (!err && !ctx->task && cpuctx->max_pertask)
T
Thomas Gleixner 已提交
809 810
		cpuctx->max_pertask--;

811
 unlock:
812
	perf_enable();
813

814
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
815 816 817
}

/*
818
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
819
 *
820 821
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
822
 *
823
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
824 825
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
826 827
 *
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
828 829
 */
static void
830 831
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
832 833 834 835 836 837
			int cpu)
{
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
838
		 * Per cpu events are installed via an smp call and
839
		 * the install is always successful.
T
Thomas Gleixner 已提交
840 841
		 */
		smp_call_function_single(cpu, __perf_install_in_context,
842
					 event, 1);
T
Thomas Gleixner 已提交
843 844 845 846 847
		return;
	}

retry:
	task_oncpu_function_call(task, __perf_install_in_context,
848
				 event);
T
Thomas Gleixner 已提交
849

850
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
851 852 853
	/*
	 * we need to retry the smp call.
	 */
854
	if (ctx->is_active && list_empty(&event->group_entry)) {
855
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
856 857 858 859 860
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
861
	 * can add the event safely, if it the call above did not
T
Thomas Gleixner 已提交
862 863
	 * succeed.
	 */
864 865
	if (list_empty(&event->group_entry))
		add_event_to_ctx(event, ctx);
866
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
867 868
}

869
/*
870
 * Put a event into inactive state and update time fields.
871 872 873 874 875 876
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
877 878
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
879
{
880
	struct perf_event *sub;
881

882 883 884 885
	event->state = PERF_EVENT_STATE_INACTIVE;
	event->tstamp_enabled = ctx->time - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry)
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
886 887 888 889
			sub->tstamp_enabled =
				ctx->time - sub->total_time_enabled;
}

890
/*
891
 * Cross CPU call to enable a performance event
892
 */
893
static void __perf_event_enable(void *info)
894
{
895
	struct perf_event *event = info;
896
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
897 898
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
899
	int err;
900

901
	/*
902 903
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
904
	 */
905
	if (ctx->task && cpuctx->task_ctx != ctx) {
906
		if (cpuctx->task_ctx || ctx->task != current)
907 908 909
			return;
		cpuctx->task_ctx = ctx;
	}
910

911
	raw_spin_lock(&ctx->lock);
912
	ctx->is_active = 1;
913
	update_context_time(ctx);
914

915
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
916
		goto unlock;
917
	__perf_event_mark_enabled(event, ctx);
918

919 920 921
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		goto unlock;

922
	/*
923
	 * If the event is in a group and isn't the group leader,
924
	 * then don't put it on unless the group is on.
925
	 */
926
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
927
		goto unlock;
928

929
	if (!group_can_go_on(event, cpuctx, 1)) {
930
		err = -EEXIST;
931
	} else {
932
		perf_disable();
933
		if (event == leader)
934
			err = group_sched_in(event, cpuctx, ctx);
935
		else
936
			err = event_sched_in(event, cpuctx, ctx);
937
		perf_enable();
938
	}
939 940 941

	if (err) {
		/*
942
		 * If this event can't go on and it's part of a
943 944
		 * group, then the whole group has to come off.
		 */
945
		if (leader != event)
946
			group_sched_out(leader, cpuctx, ctx);
947
		if (leader->attr.pinned) {
948
			update_group_times(leader);
949
			leader->state = PERF_EVENT_STATE_ERROR;
950
		}
951 952 953
	}

 unlock:
954
	raw_spin_unlock(&ctx->lock);
955 956 957
}

/*
958
 * Enable a event.
959
 *
960 961
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
962
 * remains valid.  This condition is satisfied when called through
963 964
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
965
 */
966
void perf_event_enable(struct perf_event *event)
967
{
968
	struct perf_event_context *ctx = event->ctx;
969 970 971 972
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
973
		 * Enable the event on the cpu that it's on
974
		 */
975 976
		smp_call_function_single(event->cpu, __perf_event_enable,
					 event, 1);
977 978 979
		return;
	}

980
	raw_spin_lock_irq(&ctx->lock);
981
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
982 983 984
		goto out;

	/*
985 986
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
987 988 989 990
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
991 992
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
993 994

 retry:
995
	raw_spin_unlock_irq(&ctx->lock);
996
	task_oncpu_function_call(task, __perf_event_enable, event);
997

998
	raw_spin_lock_irq(&ctx->lock);
999 1000

	/*
1001
	 * If the context is active and the event is still off,
1002 1003
	 * we need to retry the cross-call.
	 */
1004
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
1005 1006 1007 1008 1009 1010
		goto retry;

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1011 1012
	if (event->state == PERF_EVENT_STATE_OFF)
		__perf_event_mark_enabled(event, ctx);
1013

1014
 out:
1015
	raw_spin_unlock_irq(&ctx->lock);
1016 1017
}

1018
static int perf_event_refresh(struct perf_event *event, int refresh)
1019
{
1020
	/*
1021
	 * not supported on inherited events
1022
	 */
1023
	if (event->attr.inherit)
1024 1025
		return -EINVAL;

1026 1027
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1028 1029

	return 0;
1030 1031
}

1032 1033 1034 1035 1036 1037 1038 1039 1040
enum event_type_t {
	EVENT_FLEXIBLE = 0x1,
	EVENT_PINNED = 0x2,
	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
};

static void ctx_sched_out(struct perf_event_context *ctx,
			  struct perf_cpu_context *cpuctx,
			  enum event_type_t event_type)
1041
{
1042
	struct perf_event *event;
1043

1044
	raw_spin_lock(&ctx->lock);
1045
	ctx->is_active = 0;
1046
	if (likely(!ctx->nr_events))
1047
		goto out;
1048
	update_context_time(ctx);
1049

1050
	perf_disable();
1051 1052 1053 1054
	if (!ctx->nr_active)
		goto out_enable;

	if (event_type & EVENT_PINNED)
1055 1056 1057
		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
			group_sched_out(event, cpuctx, ctx);

1058
	if (event_type & EVENT_FLEXIBLE)
1059
		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1060
			group_sched_out(event, cpuctx, ctx);
1061 1062

 out_enable:
1063
	perf_enable();
1064
 out:
1065
	raw_spin_unlock(&ctx->lock);
1066 1067
}

1068 1069 1070
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1071 1072 1073 1074
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1075
 * in them directly with an fd; we can only enable/disable all
1076
 * events via prctl, or enable/disable all events in a family
1077 1078
 * via ioctl, which will have the same effect on both contexts.
 */
1079 1080
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1081 1082
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1083
		&& ctx1->parent_gen == ctx2->parent_gen
1084
		&& !ctx1->pin_count && !ctx2->pin_count;
1085 1086
}

1087 1088
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1089 1090 1091
{
	u64 value;

1092
	if (!event->attr.inherit_stat)
1093 1094 1095
		return;

	/*
1096
	 * Update the event value, we cannot use perf_event_read()
1097 1098
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1099
	 * we know the event must be on the current CPU, therefore we
1100 1101
	 * don't need to use it.
	 */
1102 1103
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1104 1105
		event->pmu->read(event);
		/* fall-through */
1106

1107 1108
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1109 1110 1111 1112 1113 1114 1115
		break;

	default:
		break;
	}

	/*
1116
	 * In order to keep per-task stats reliable we need to flip the event
1117 1118
	 * values when we flip the contexts.
	 */
1119 1120 1121
	value = atomic64_read(&next_event->count);
	value = atomic64_xchg(&event->count, value);
	atomic64_set(&next_event->count, value);
1122

1123 1124
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1125

1126
	/*
1127
	 * Since we swizzled the values, update the user visible data too.
1128
	 */
1129 1130
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1131 1132 1133 1134 1135
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1136 1137
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1138
{
1139
	struct perf_event *event, *next_event;
1140 1141 1142 1143

	if (!ctx->nr_stat)
		return;

1144 1145
	update_context_time(ctx);

1146 1147
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1148

1149 1150
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1151

1152 1153
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1154

1155
		__perf_event_sync_stat(event, next_event);
1156

1157 1158
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1159 1160 1161
	}
}

T
Thomas Gleixner 已提交
1162
/*
1163
 * Called from scheduler to remove the events of the current task,
T
Thomas Gleixner 已提交
1164 1165
 * with interrupts disabled.
 *
1166
 * We stop each event and update the event value in event->count.
T
Thomas Gleixner 已提交
1167
 *
I
Ingo Molnar 已提交
1168
 * This does not protect us against NMI, but disable()
1169 1170 1171
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
T
Thomas Gleixner 已提交
1172
 */
1173
void perf_event_task_sched_out(struct task_struct *task,
1174
				 struct task_struct *next)
T
Thomas Gleixner 已提交
1175
{
1176
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1177 1178 1179
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1180
	int do_switch = 1;
T
Thomas Gleixner 已提交
1181

1182
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
1183

1184
	if (likely(!ctx || !cpuctx->task_ctx))
T
Thomas Gleixner 已提交
1185 1186
		return;

1187 1188
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1189
	next_ctx = next->perf_event_ctxp;
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
1201 1202
		raw_spin_lock(&ctx->lock);
		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1203
		if (context_equiv(ctx, next_ctx)) {
1204 1205
			/*
			 * XXX do we need a memory barrier of sorts
1206
			 * wrt to rcu_dereference() of perf_event_ctxp
1207
			 */
1208 1209
			task->perf_event_ctxp = next_ctx;
			next->perf_event_ctxp = ctx;
1210 1211 1212
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1213

1214
			perf_event_sync_stat(ctx, next_ctx);
1215
		}
1216 1217
		raw_spin_unlock(&next_ctx->lock);
		raw_spin_unlock(&ctx->lock);
1218
	}
1219
	rcu_read_unlock();
1220

1221
	if (do_switch) {
1222
		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
1223 1224
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1225 1226
}

1227 1228
static void task_ctx_sched_out(struct perf_event_context *ctx,
			       enum event_type_t event_type)
1229 1230 1231
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

1232 1233
	if (!cpuctx->task_ctx)
		return;
1234 1235 1236 1237

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1238
	ctx_sched_out(ctx, cpuctx, event_type);
1239 1240 1241
	cpuctx->task_ctx = NULL;
}

1242 1243 1244
/*
 * Called with IRQs disabled
 */
1245
static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1246
{
1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
	task_ctx_sched_out(ctx, EVENT_ALL);
}

/*
 * Called with IRQs disabled
 */
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type)
{
	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
1257 1258
}

1259
static void
1260
ctx_pinned_sched_in(struct perf_event_context *ctx,
1261
		    struct perf_cpu_context *cpuctx)
T
Thomas Gleixner 已提交
1262
{
1263
	struct perf_event *event;
T
Thomas Gleixner 已提交
1264

1265 1266
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF)
1267
			continue;
1268
		if (event->cpu != -1 && event->cpu != smp_processor_id())
1269 1270
			continue;

1271
		if (group_can_go_on(event, cpuctx, 1))
1272
			group_sched_in(event, cpuctx, ctx);
1273 1274 1275 1276 1277

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1278 1279 1280
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1281
		}
1282
	}
1283 1284 1285 1286
}

static void
ctx_flexible_sched_in(struct perf_event_context *ctx,
1287
		      struct perf_cpu_context *cpuctx)
1288 1289 1290
{
	struct perf_event *event;
	int can_add_hw = 1;
1291

1292 1293 1294
	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		/* Ignore events in OFF or ERROR state */
		if (event->state <= PERF_EVENT_STATE_OFF)
1295
			continue;
1296 1297
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1298
		 * of events:
1299
		 */
1300
		if (event->cpu != -1 && event->cpu != smp_processor_id())
T
Thomas Gleixner 已提交
1301 1302
			continue;

1303
		if (group_can_go_on(event, cpuctx, can_add_hw))
1304
			if (group_sched_in(event, cpuctx, ctx))
1305
				can_add_hw = 0;
T
Thomas Gleixner 已提交
1306
	}
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
}

static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
	     enum event_type_t event_type)
{
	raw_spin_lock(&ctx->lock);
	ctx->is_active = 1;
	if (likely(!ctx->nr_events))
		goto out;

	ctx->timestamp = perf_clock();

	perf_disable();

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
	if (event_type & EVENT_PINNED)
1328
		ctx_pinned_sched_in(ctx, cpuctx);
1329 1330 1331

	/* Then walk through the lower prio flexible groups */
	if (event_type & EVENT_FLEXIBLE)
1332
		ctx_flexible_sched_in(ctx, cpuctx);
1333

1334
	perf_enable();
1335
 out:
1336
	raw_spin_unlock(&ctx->lock);
1337 1338
}

1339 1340 1341 1342 1343 1344 1345 1346
static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
			     enum event_type_t event_type)
{
	struct perf_event_context *ctx = &cpuctx->ctx;

	ctx_sched_in(ctx, cpuctx, event_type);
}

1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
static void task_ctx_sched_in(struct task_struct *task,
			      enum event_type_t event_type)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_event_context *ctx = task->perf_event_ctxp;

	if (likely(!ctx))
		return;
	if (cpuctx->task_ctx == ctx)
		return;
	ctx_sched_in(ctx, cpuctx, event_type);
	cpuctx->task_ctx = ctx;
}
1360
/*
1361
 * Called from scheduler to add the events of the current task
1362 1363
 * with interrupts disabled.
 *
1364
 * We restore the event value and then enable it.
1365 1366
 *
 * This does not protect us against NMI, but enable()
1367 1368 1369
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
1370
 */
1371
void perf_event_task_sched_in(struct task_struct *task)
1372
{
1373 1374
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	struct perf_event_context *ctx = task->perf_event_ctxp;
T
Thomas Gleixner 已提交
1375

1376 1377
	if (likely(!ctx))
		return;
1378

1379 1380 1381
	if (cpuctx->task_ctx == ctx)
		return;

1382 1383
	perf_disable();

1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	/*
	 * We want to keep the following priority order:
	 * cpu pinned (that don't need to move), task pinned,
	 * cpu flexible, task flexible.
	 */
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);

	ctx_sched_in(ctx, cpuctx, EVENT_PINNED);
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
	ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);

	cpuctx->task_ctx = ctx;
1396 1397

	perf_enable();
1398 1399
}

1400 1401
#define MAX_INTERRUPTS (~0ULL)

1402
static void perf_log_throttle(struct perf_event *event, int enable);
1403

1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
{
	u64 frequency = event->attr.sample_freq;
	u64 sec = NSEC_PER_SEC;
	u64 divisor, dividend;

	int count_fls, nsec_fls, frequency_fls, sec_fls;

	count_fls = fls64(count);
	nsec_fls = fls64(nsec);
	frequency_fls = fls64(frequency);
	sec_fls = 30;

	/*
	 * We got @count in @nsec, with a target of sample_freq HZ
	 * the target period becomes:
	 *
	 *             @count * 10^9
	 * period = -------------------
	 *          @nsec * sample_freq
	 *
	 */

	/*
	 * Reduce accuracy by one bit such that @a and @b converge
	 * to a similar magnitude.
	 */
#define REDUCE_FLS(a, b) 		\
do {					\
	if (a##_fls > b##_fls) {	\
		a >>= 1;		\
		a##_fls--;		\
	} else {			\
		b >>= 1;		\
		b##_fls--;		\
	}				\
} while (0)

	/*
	 * Reduce accuracy until either term fits in a u64, then proceed with
	 * the other, so that finally we can do a u64/u64 division.
	 */
	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
		REDUCE_FLS(nsec, frequency);
		REDUCE_FLS(sec, count);
	}

	if (count_fls + sec_fls > 64) {
		divisor = nsec * frequency;

		while (count_fls + sec_fls > 64) {
			REDUCE_FLS(count, sec);
			divisor >>= 1;
		}

		dividend = count * sec;
	} else {
		dividend = count * sec;

		while (nsec_fls + frequency_fls > 64) {
			REDUCE_FLS(nsec, frequency);
			dividend >>= 1;
		}

		divisor = nsec * frequency;
	}

	return div64_u64(dividend, divisor);
}

1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
static void perf_event_stop(struct perf_event *event)
{
	if (!event->pmu->stop)
		return event->pmu->disable(event);

	return event->pmu->stop(event);
}

static int perf_event_start(struct perf_event *event)
{
	if (!event->pmu->start)
		return event->pmu->enable(event);

	return event->pmu->start(event);
}

1490
static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1491
{
1492
	struct hw_perf_event *hwc = &event->hw;
1493 1494 1495
	u64 period, sample_period;
	s64 delta;

1496
	period = perf_calculate_period(event, nsec, count);
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
1507 1508 1509

	if (atomic64_read(&hwc->period_left) > 8*sample_period) {
		perf_disable();
1510
		perf_event_stop(event);
1511
		atomic64_set(&hwc->period_left, 0);
1512
		perf_event_start(event);
1513 1514
		perf_enable();
	}
1515 1516
}

1517
static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1518
{
1519 1520
	struct perf_event *event;
	struct hw_perf_event *hwc;
1521 1522
	u64 interrupts, now;
	s64 delta;
1523

1524
	raw_spin_lock(&ctx->lock);
1525
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1526
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1527 1528
			continue;

1529 1530 1531
		if (event->cpu != -1 && event->cpu != smp_processor_id())
			continue;

1532
		hwc = &event->hw;
1533 1534 1535

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1536

1537
		/*
1538
		 * unthrottle events on the tick
1539
		 */
1540
		if (interrupts == MAX_INTERRUPTS) {
1541
			perf_log_throttle(event, 1);
1542
			perf_disable();
1543
			event->pmu->unthrottle(event);
1544
			perf_enable();
1545 1546
		}

1547
		if (!event->attr.freq || !event->attr.sample_freq)
1548 1549
			continue;

1550
		perf_disable();
1551 1552 1553 1554
		event->pmu->read(event);
		now = atomic64_read(&event->count);
		delta = now - hwc->freq_count_stamp;
		hwc->freq_count_stamp = now;
1555

1556 1557
		if (delta > 0)
			perf_adjust_period(event, TICK_NSEC, delta);
1558
		perf_enable();
1559
	}
1560
	raw_spin_unlock(&ctx->lock);
1561 1562
}

1563
/*
1564
 * Round-robin a context's events:
1565
 */
1566
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1567
{
1568
	raw_spin_lock(&ctx->lock);
1569 1570 1571 1572

	/* Rotate the first entry last of non-pinned groups */
	list_rotate_left(&ctx->flexible_groups);

1573
	raw_spin_unlock(&ctx->lock);
1574 1575
}

1576
void perf_event_task_tick(struct task_struct *curr)
1577
{
1578
	struct perf_cpu_context *cpuctx;
1579
	struct perf_event_context *ctx;
1580
	int rotate = 0;
1581

1582
	if (!atomic_read(&nr_events))
1583 1584
		return;

1585
	cpuctx = &__get_cpu_var(perf_cpu_context);
1586 1587 1588
	if (cpuctx->ctx.nr_events &&
	    cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
		rotate = 1;
1589

1590 1591 1592
	ctx = curr->perf_event_ctxp;
	if (ctx && ctx->nr_events && ctx->nr_events != ctx->nr_active)
		rotate = 1;
1593

1594
	perf_ctx_adjust_freq(&cpuctx->ctx);
1595
	if (ctx)
1596
		perf_ctx_adjust_freq(ctx);
1597

1598 1599 1600 1601
	if (!rotate)
		return;

	perf_disable();
1602
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
1603
	if (ctx)
1604
		task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
T
Thomas Gleixner 已提交
1605

1606
	rotate_ctx(&cpuctx->ctx);
1607 1608
	if (ctx)
		rotate_ctx(ctx);
1609

1610
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
1611
	if (ctx)
1612
		task_ctx_sched_in(curr, EVENT_FLEXIBLE);
1613
	perf_enable();
T
Thomas Gleixner 已提交
1614 1615
}

1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

	__perf_event_mark_enabled(event, ctx);

	return 1;
}

1631
/*
1632
 * Enable all of a task's events that have been marked enable-on-exec.
1633 1634
 * This expects task == current.
 */
1635
static void perf_event_enable_on_exec(struct task_struct *task)
1636
{
1637 1638
	struct perf_event_context *ctx;
	struct perf_event *event;
1639 1640
	unsigned long flags;
	int enabled = 0;
1641
	int ret;
1642 1643

	local_irq_save(flags);
1644 1645
	ctx = task->perf_event_ctxp;
	if (!ctx || !ctx->nr_events)
1646 1647
		goto out;

1648
	__perf_event_task_sched_out(ctx);
1649

1650
	raw_spin_lock(&ctx->lock);
1651

1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
	}

	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
1662 1663 1664
	}

	/*
1665
	 * Unclone this context if we enabled any event.
1666
	 */
1667 1668
	if (enabled)
		unclone_ctx(ctx);
1669

1670
	raw_spin_unlock(&ctx->lock);
1671

1672
	perf_event_task_sched_in(task);
1673 1674 1675 1676
 out:
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1677
/*
1678
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1679
 */
1680
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1681
{
1682
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1683 1684
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
I
Ingo Molnar 已提交
1685

1686 1687 1688 1689
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1690 1691
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1692 1693 1694 1695
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

1696
	raw_spin_lock(&ctx->lock);
P
Peter Zijlstra 已提交
1697
	update_context_time(ctx);
1698
	update_event_times(event);
1699
	raw_spin_unlock(&ctx->lock);
P
Peter Zijlstra 已提交
1700

P
Peter Zijlstra 已提交
1701
	event->pmu->read(event);
T
Thomas Gleixner 已提交
1702 1703
}

1704
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1705 1706
{
	/*
1707 1708
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1709
	 */
1710 1711 1712 1713
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
1714 1715 1716
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

1717
		raw_spin_lock_irqsave(&ctx->lock, flags);
P
Peter Zijlstra 已提交
1718
		update_context_time(ctx);
1719
		update_event_times(event);
1720
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1721 1722
	}

1723
	return atomic64_read(&event->count);
T
Thomas Gleixner 已提交
1724 1725
}

1726
/*
1727
 * Initialize the perf_event context in a task_struct:
1728 1729
 */
static void
1730
__perf_event_init_context(struct perf_event_context *ctx,
1731 1732
			    struct task_struct *task)
{
1733
	raw_spin_lock_init(&ctx->lock);
1734
	mutex_init(&ctx->mutex);
1735 1736
	INIT_LIST_HEAD(&ctx->pinned_groups);
	INIT_LIST_HEAD(&ctx->flexible_groups);
1737 1738 1739 1740 1741
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
	ctx->task = task;
}

1742
static struct perf_event_context *find_get_context(pid_t pid, int cpu)
T
Thomas Gleixner 已提交
1743
{
1744
	struct perf_event_context *ctx;
1745
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
1746
	struct task_struct *task;
1747
	unsigned long flags;
1748
	int err;
T
Thomas Gleixner 已提交
1749

1750
	if (pid == -1 && cpu != -1) {
1751
		/* Must be root to operate on a CPU event: */
1752
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
1753 1754
			return ERR_PTR(-EACCES);

1755
		if (cpu < 0 || cpu >= nr_cpumask_bits)
T
Thomas Gleixner 已提交
1756 1757 1758
			return ERR_PTR(-EINVAL);

		/*
1759
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
1760 1761 1762
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
1763
		if (!cpu_online(cpu))
T
Thomas Gleixner 已提交
1764 1765 1766 1767
			return ERR_PTR(-ENODEV);

		cpuctx = &per_cpu(perf_cpu_context, cpu);
		ctx = &cpuctx->ctx;
1768
		get_ctx(ctx);
T
Thomas Gleixner 已提交
1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784

		return ctx;
	}

	rcu_read_lock();
	if (!pid)
		task = current;
	else
		task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

1785
	/*
1786
	 * Can't attach events to a dying task.
1787 1788 1789 1790 1791
	 */
	err = -ESRCH;
	if (task->flags & PF_EXITING)
		goto errout;

T
Thomas Gleixner 已提交
1792
	/* Reuse ptrace permission checks for now. */
1793 1794 1795 1796 1797
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

 retry:
1798
	ctx = perf_lock_task_context(task, &flags);
1799
	if (ctx) {
1800
		unclone_ctx(ctx);
1801
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1802 1803
	}

1804
	if (!ctx) {
1805
		ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1806 1807 1808
		err = -ENOMEM;
		if (!ctx)
			goto errout;
1809
		__perf_event_init_context(ctx, task);
1810
		get_ctx(ctx);
1811
		if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1812 1813 1814 1815 1816
			/*
			 * We raced with some other task; use
			 * the context they set.
			 */
			kfree(ctx);
1817
			goto retry;
1818
		}
1819
		get_task_struct(task);
1820 1821
	}

1822
	put_task_struct(task);
T
Thomas Gleixner 已提交
1823
	return ctx;
1824 1825 1826 1827

 errout:
	put_task_struct(task);
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
1828 1829
}

L
Li Zefan 已提交
1830 1831
static void perf_event_free_filter(struct perf_event *event);

1832
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
1833
{
1834
	struct perf_event *event;
P
Peter Zijlstra 已提交
1835

1836 1837 1838
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
1839
	perf_event_free_filter(event);
1840
	kfree(event);
P
Peter Zijlstra 已提交
1841 1842
}

1843
static void perf_pending_sync(struct perf_event *event);
1844

1845
static void free_event(struct perf_event *event)
1846
{
1847
	perf_pending_sync(event);
1848

1849 1850 1851 1852 1853 1854 1855 1856
	if (!event->parent) {
		atomic_dec(&nr_events);
		if (event->attr.mmap)
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
1857
	}
1858

1859 1860 1861
	if (event->output) {
		fput(event->output->filp);
		event->output = NULL;
1862 1863
	}

1864 1865
	if (event->destroy)
		event->destroy(event);
1866

1867 1868
	put_ctx(event->ctx);
	call_rcu(&event->rcu_head, free_event_rcu);
1869 1870
}

1871
int perf_event_release_kernel(struct perf_event *event)
T
Thomas Gleixner 已提交
1872
{
1873
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1874

1875 1876 1877 1878 1879 1880
	/*
	 * Remove from the PMU, can't get re-enabled since we got
	 * here because the last ref went.
	 */
	perf_event_disable(event);

1881
	WARN_ON_ONCE(ctx->parent_ctx);
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894
	/*
	 * There are two ways this annotation is useful:
	 *
	 *  1) there is a lock recursion from perf_event_exit_task
	 *     see the comment there.
	 *
	 *  2) there is a lock-inversion with mmap_sem through
	 *     perf_event_read_group(), which takes faults while
	 *     holding ctx->mutex, however this is called after
	 *     the last filedesc died, so there is no possibility
	 *     to trigger the AB-BA case.
	 */
	mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
1895 1896 1897 1898
	raw_spin_lock_irq(&ctx->lock);
	list_del_event(event, ctx);
	perf_destroy_group(event, ctx);
	raw_spin_unlock_irq(&ctx->lock);
1899
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
1900

1901 1902 1903 1904
	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);
1905

1906
	free_event(event);
T
Thomas Gleixner 已提交
1907 1908 1909

	return 0;
}
1910
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
T
Thomas Gleixner 已提交
1911

1912 1913 1914 1915
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
1916
{
1917
	struct perf_event *event = file->private_data;
1918

1919
	file->private_data = NULL;
1920

1921
	return perf_event_release_kernel(event);
1922 1923
}

1924
static int perf_event_read_size(struct perf_event *event)
1925 1926 1927 1928 1929
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

1930
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1931 1932
		size += sizeof(u64);

1933
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1934 1935
		size += sizeof(u64);

1936
	if (event->attr.read_format & PERF_FORMAT_ID)
1937 1938
		entry += sizeof(u64);

1939 1940
	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
1941 1942 1943 1944 1945 1946 1947 1948
		size += sizeof(u64);
	}

	size += entry * nr;

	return size;
}

1949
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
1950
{
1951
	struct perf_event *child;
1952 1953
	u64 total = 0;

1954 1955 1956
	*enabled = 0;
	*running = 0;

1957
	mutex_lock(&event->child_mutex);
1958
	total += perf_event_read(event);
1959 1960 1961 1962 1963 1964
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
1965
		total += perf_event_read(child);
1966 1967 1968
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
1969
	mutex_unlock(&event->child_mutex);
1970 1971 1972

	return total;
}
1973
EXPORT_SYMBOL_GPL(perf_event_read_value);
1974

1975
static int perf_event_read_group(struct perf_event *event,
1976 1977
				   u64 read_format, char __user *buf)
{
1978
	struct perf_event *leader = event->group_leader, *sub;
1979 1980
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
1981
	u64 values[5];
1982
	u64 count, enabled, running;
1983

1984
	mutex_lock(&ctx->mutex);
1985
	count = perf_event_read_value(leader, &enabled, &running);
1986 1987

	values[n++] = 1 + leader->nr_siblings;
1988 1989 1990 1991
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1992 1993 1994
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
1995 1996 1997 1998

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
1999
		goto unlock;
2000

2001
	ret = size;
2002

2003
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2004
		n = 0;
2005

2006
		values[n++] = perf_event_read_value(sub, &enabled, &running);
2007 2008 2009 2010 2011
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

2012
		if (copy_to_user(buf + ret, values, size)) {
2013 2014 2015
			ret = -EFAULT;
			goto unlock;
		}
2016 2017

		ret += size;
2018
	}
2019 2020
unlock:
	mutex_unlock(&ctx->mutex);
2021

2022
	return ret;
2023 2024
}

2025
static int perf_event_read_one(struct perf_event *event,
2026 2027
				 u64 read_format, char __user *buf)
{
2028
	u64 enabled, running;
2029 2030 2031
	u64 values[4];
	int n = 0;

2032 2033 2034 2035 2036
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
2037
	if (read_format & PERF_FORMAT_ID)
2038
		values[n++] = primary_event_id(event);
2039 2040 2041 2042 2043 2044 2045

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
2046
/*
2047
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
2048 2049
 */
static ssize_t
2050
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
2051
{
2052
	u64 read_format = event->attr.read_format;
2053
	int ret;
T
Thomas Gleixner 已提交
2054

2055
	/*
2056
	 * Return end-of-file for a read on a event that is in
2057 2058 2059
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
2060
	if (event->state == PERF_EVENT_STATE_ERROR)
2061 2062
		return 0;

2063
	if (count < perf_event_read_size(event))
2064 2065
		return -ENOSPC;

2066
	WARN_ON_ONCE(event->ctx->parent_ctx);
2067
	if (read_format & PERF_FORMAT_GROUP)
2068
		ret = perf_event_read_group(event, read_format, buf);
2069
	else
2070
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
2071

2072
	return ret;
T
Thomas Gleixner 已提交
2073 2074 2075 2076 2077
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
2078
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
2079

2080
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
2081 2082 2083 2084
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
2085
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
2086
	struct perf_mmap_data *data;
2087
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
2088 2089

	rcu_read_lock();
2090
	data = rcu_dereference(event->data);
P
Peter Zijlstra 已提交
2091
	if (data)
2092
		events = atomic_xchg(&data->poll, 0);
P
Peter Zijlstra 已提交
2093
	rcu_read_unlock();
T
Thomas Gleixner 已提交
2094

2095
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
2096 2097 2098 2099

	return events;
}

2100
static void perf_event_reset(struct perf_event *event)
2101
{
2102 2103 2104
	(void)perf_event_read(event);
	atomic64_set(&event->count, 0);
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
2105 2106
}

2107
/*
2108 2109 2110 2111
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
2112
 */
2113 2114
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2115
{
2116
	struct perf_event *child;
P
Peter Zijlstra 已提交
2117

2118 2119 2120 2121
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
2122
		func(child);
2123
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
2124 2125
}

2126 2127
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
2128
{
2129 2130
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
2131

2132 2133
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
2134
	event = event->group_leader;
2135

2136 2137 2138 2139
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
2140
	mutex_unlock(&ctx->mutex);
2141 2142
}

2143
static int perf_event_period(struct perf_event *event, u64 __user *arg)
2144
{
2145
	struct perf_event_context *ctx = event->ctx;
2146 2147 2148 2149
	unsigned long size;
	int ret = 0;
	u64 value;

2150
	if (!event->attr.sample_period)
2151 2152 2153 2154 2155 2156 2157 2158 2159
		return -EINVAL;

	size = copy_from_user(&value, arg, sizeof(value));
	if (size != sizeof(value))
		return -EFAULT;

	if (!value)
		return -EINVAL;

2160
	raw_spin_lock_irq(&ctx->lock);
2161 2162
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
2163 2164 2165 2166
			ret = -EINVAL;
			goto unlock;
		}

2167
		event->attr.sample_freq = value;
2168
	} else {
2169 2170
		event->attr.sample_period = value;
		event->hw.sample_period = value;
2171 2172
	}
unlock:
2173
	raw_spin_unlock_irq(&ctx->lock);
2174 2175 2176 2177

	return ret;
}

L
Li Zefan 已提交
2178 2179
static int perf_event_set_output(struct perf_event *event, int output_fd);
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2180

2181 2182
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
2183 2184
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2185
	u32 flags = arg;
2186 2187

	switch (cmd) {
2188 2189
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2190
		break;
2191 2192
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2193
		break;
2194 2195
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2196
		break;
P
Peter Zijlstra 已提交
2197

2198 2199
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2200

2201 2202
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2203

2204 2205
	case PERF_EVENT_IOC_SET_OUTPUT:
		return perf_event_set_output(event, arg);
2206

L
Li Zefan 已提交
2207 2208 2209
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2210
	default:
P
Peter Zijlstra 已提交
2211
		return -ENOTTY;
2212
	}
P
Peter Zijlstra 已提交
2213 2214

	if (flags & PERF_IOC_FLAG_GROUP)
2215
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2216
	else
2217
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2218 2219

	return 0;
2220 2221
}

2222
int perf_event_task_enable(void)
2223
{
2224
	struct perf_event *event;
2225

2226 2227 2228 2229
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2230 2231 2232 2233

	return 0;
}

2234
int perf_event_task_disable(void)
2235
{
2236
	struct perf_event *event;
2237

2238 2239 2240 2241
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2242 2243 2244 2245

	return 0;
}

2246 2247
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2248 2249
#endif

2250
static int perf_event_index(struct perf_event *event)
2251
{
2252
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2253 2254
		return 0;

2255
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2256 2257
}

2258 2259 2260 2261 2262
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2263
void perf_event_update_userpage(struct perf_event *event)
2264
{
2265
	struct perf_event_mmap_page *userpg;
2266
	struct perf_mmap_data *data;
2267 2268

	rcu_read_lock();
2269
	data = rcu_dereference(event->data);
2270 2271 2272 2273
	if (!data)
		goto unlock;

	userpg = data->user_page;
2274

2275 2276 2277 2278 2279
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2280
	++userpg->lock;
2281
	barrier();
2282 2283 2284 2285
	userpg->index = perf_event_index(event);
	userpg->offset = atomic64_read(&event->count);
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		userpg->offset -= atomic64_read(&event->hw.prev_count);
2286

2287 2288
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2289

2290 2291
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2292

2293
	barrier();
2294
	++userpg->lock;
2295
	preempt_enable();
2296
unlock:
2297
	rcu_read_unlock();
2298 2299
}

2300
static unsigned long perf_data_size(struct perf_mmap_data *data)
2301
{
2302 2303
	return data->nr_pages << (PAGE_SHIFT + data->data_order);
}
2304

2305
#ifndef CONFIG_PERF_USE_VMALLOC
2306

2307 2308 2309
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2310

2311 2312 2313 2314 2315
static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > data->nr_pages)
		return NULL;
2316

2317 2318
	if (pgoff == 0)
		return virt_to_page(data->user_page);
2319

2320
	return virt_to_page(data->data_pages[pgoff - 1]);
2321 2322
}

2323 2324
static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2325 2326 2327 2328 2329
{
	struct perf_mmap_data *data;
	unsigned long size;
	int i;

2330
	WARN_ON(atomic_read(&event->mmap_count));
2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348

	size = sizeof(struct perf_mmap_data);
	size += nr_pages * sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
	if (!data->user_page)
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
		data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
		if (!data->data_pages[i])
			goto fail_data_pages;
	}

2349
	data->data_order = 0;
2350 2351
	data->nr_pages = nr_pages;

2352
	return data;
2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363

fail_data_pages:
	for (i--; i >= 0; i--)
		free_page((unsigned long)data->data_pages[i]);

	free_page((unsigned long)data->user_page);

fail_user_page:
	kfree(data);

fail:
2364
	return NULL;
2365 2366
}

2367 2368
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2369
	struct page *page = virt_to_page((void *)addr);
2370 2371 2372 2373 2374

	page->mapping = NULL;
	__free_page(page);
}

2375
static void perf_mmap_data_free(struct perf_mmap_data *data)
2376 2377 2378
{
	int i;

2379
	perf_mmap_free_page((unsigned long)data->user_page);
2380
	for (i = 0; i < data->nr_pages; i++)
2381
		perf_mmap_free_page((unsigned long)data->data_pages[i]);
2382
	kfree(data);
2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422
}

#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > (1UL << data->data_order))
		return NULL;

	return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

static void perf_mmap_data_free_work(struct work_struct *work)
{
	struct perf_mmap_data *data;
	void *base;
	int i, nr;

	data = container_of(work, struct perf_mmap_data, work);
	nr = 1 << data->data_order;

	base = data->user_page;
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
2423
	kfree(data);
2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438
}

static void perf_mmap_data_free(struct perf_mmap_data *data)
{
	schedule_work(&data->work);
}

static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
{
	struct perf_mmap_data *data;
	unsigned long size;
	void *all_buf;

	WARN_ON(atomic_read(&event->mmap_count));
2439

2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516
	size = sizeof(struct perf_mmap_data);
	size += sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	INIT_WORK(&data->work, perf_mmap_data_free_work);

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

	data->user_page = all_buf;
	data->data_pages[0] = all_buf + PAGE_SIZE;
	data->data_order = ilog2(nr_pages);
	data->nr_pages = 1;

	return data;

fail_all_buf:
	kfree(data);

fail:
	return NULL;
}

#endif

static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
	struct perf_mmap_data *data;
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	data = rcu_dereference(event->data);
	if (!data)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(data, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

static void
perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
{
	long max_size = perf_data_size(data);

	atomic_set(&data->lock, -1);

	if (event->attr.watermark) {
		data->watermark = min_t(long, max_size,
					event->attr.wakeup_watermark);
	}

	if (!data->watermark)
2517
		data->watermark = max_size / 2;
2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528


	rcu_assign_pointer(event->data, data);
}

static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
{
	struct perf_mmap_data *data;

	data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
	perf_mmap_data_free(data);
2529 2530
}

2531
static void perf_mmap_data_release(struct perf_event *event)
2532
{
2533
	struct perf_mmap_data *data = event->data;
2534

2535
	WARN_ON(atomic_read(&event->mmap_count));
2536

2537
	rcu_assign_pointer(event->data, NULL);
2538
	call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2539 2540 2541 2542
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
2543
	struct perf_event *event = vma->vm_file->private_data;
2544

2545
	atomic_inc(&event->mmap_count);
2546 2547 2548 2549
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
2550
	struct perf_event *event = vma->vm_file->private_data;
2551

2552 2553
	WARN_ON_ONCE(event->ctx->parent_ctx);
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2554
		unsigned long size = perf_data_size(event->data);
2555 2556
		struct user_struct *user = current_user();

2557
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2558
		vma->vm_mm->locked_vm -= event->data->nr_locked;
2559
		perf_mmap_data_release(event);
2560
		mutex_unlock(&event->mmap_mutex);
2561
	}
2562 2563
}

2564
static const struct vm_operations_struct perf_mmap_vmops = {
2565 2566 2567 2568
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
2569 2570 2571 2572
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
2573
	struct perf_event *event = file->private_data;
2574
	unsigned long user_locked, user_lock_limit;
2575
	struct user_struct *user = current_user();
2576
	unsigned long locked, lock_limit;
2577
	struct perf_mmap_data *data;
2578 2579
	unsigned long vma_size;
	unsigned long nr_pages;
2580
	long user_extra, extra;
2581
	int ret = 0;
2582

2583
	if (!(vma->vm_flags & VM_SHARED))
2584
		return -EINVAL;
2585 2586 2587 2588

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

2589 2590 2591 2592 2593
	/*
	 * If we have data pages ensure they're a power-of-two number, so we
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
2594 2595
		return -EINVAL;

2596
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
2597 2598
		return -EINVAL;

2599 2600
	if (vma->vm_pgoff != 0)
		return -EINVAL;
2601

2602 2603 2604
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
	if (event->output) {
2605 2606 2607 2608
		ret = -EINVAL;
		goto unlock;
	}

2609 2610
	if (atomic_inc_not_zero(&event->mmap_count)) {
		if (nr_pages != event->data->nr_pages)
2611 2612 2613 2614
			ret = -EINVAL;
		goto unlock;
	}

2615
	user_extra = nr_pages + 1;
2616
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
2617 2618 2619 2620 2621 2622

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

2623
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2624

2625 2626 2627
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
2628

2629
	lock_limit = rlimit(RLIMIT_MEMLOCK);
2630
	lock_limit >>= PAGE_SHIFT;
2631
	locked = vma->vm_mm->locked_vm + extra;
2632

2633 2634
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
2635 2636 2637
		ret = -EPERM;
		goto unlock;
	}
2638

2639
	WARN_ON(event->data);
2640 2641 2642 2643

	data = perf_mmap_data_alloc(event, nr_pages);
	ret = -ENOMEM;
	if (!data)
2644 2645
		goto unlock;

2646 2647 2648
	ret = 0;
	perf_mmap_data_init(event, data);

2649
	atomic_set(&event->mmap_count, 1);
2650
	atomic_long_add(user_extra, &user->locked_vm);
2651
	vma->vm_mm->locked_vm += extra;
2652
	event->data->nr_locked = extra;
2653
	if (vma->vm_flags & VM_WRITE)
2654
		event->data->writable = 1;
2655

2656
unlock:
2657
	mutex_unlock(&event->mmap_mutex);
2658 2659 2660

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
2661 2662

	return ret;
2663 2664
}

P
Peter Zijlstra 已提交
2665 2666 2667
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
2668
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
2669 2670 2671
	int retval;

	mutex_lock(&inode->i_mutex);
2672
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
2673 2674 2675 2676 2677 2678 2679 2680
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
2681
static const struct file_operations perf_fops = {
2682
	.llseek			= no_llseek,
T
Thomas Gleixner 已提交
2683 2684 2685
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
2686 2687
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
2688
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
2689
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
2690 2691
};

2692
/*
2693
 * Perf event wakeup
2694 2695 2696 2697 2698
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

2699
void perf_event_wakeup(struct perf_event *event)
2700
{
2701
	wake_up_all(&event->waitq);
2702

2703 2704 2705
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
2706
	}
2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717
}

/*
 * Pending wakeups
 *
 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
 *
 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
 * single linked list and use cmpxchg() to add entries lockless.
 */

2718
static void perf_pending_event(struct perf_pending_entry *entry)
2719
{
2720 2721
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
2722

2723 2724 2725
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
2726 2727
	}

2728 2729 2730
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
2731 2732 2733
	}
}

2734
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2735

2736
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2737 2738 2739
	PENDING_TAIL,
};

2740 2741
static void perf_pending_queue(struct perf_pending_entry *entry,
			       void (*func)(struct perf_pending_entry *))
2742
{
2743
	struct perf_pending_entry **head;
2744

2745
	if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2746 2747
		return;

2748 2749 2750
	entry->func = func;

	head = &get_cpu_var(perf_pending_head);
2751 2752

	do {
2753 2754
		entry->next = *head;
	} while (cmpxchg(head, entry->next, entry) != entry->next);
2755

2756
	set_perf_event_pending();
2757

2758
	put_cpu_var(perf_pending_head);
2759 2760 2761 2762
}

static int __perf_pending_run(void)
{
2763
	struct perf_pending_entry *list;
2764 2765
	int nr = 0;

2766
	list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2767
	while (list != PENDING_TAIL) {
2768 2769
		void (*func)(struct perf_pending_entry *);
		struct perf_pending_entry *entry = list;
2770 2771 2772

		list = list->next;

2773 2774
		func = entry->func;
		entry->next = NULL;
2775 2776 2777 2778 2779 2780 2781
		/*
		 * Ensure we observe the unqueue before we issue the wakeup,
		 * so that we won't be waiting forever.
		 * -- see perf_not_pending().
		 */
		smp_wmb();

2782
		func(entry);
2783 2784 2785 2786 2787 2788
		nr++;
	}

	return nr;
}

2789
static inline int perf_not_pending(struct perf_event *event)
2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803
{
	/*
	 * If we flush on whatever cpu we run, there is a chance we don't
	 * need to wait.
	 */
	get_cpu();
	__perf_pending_run();
	put_cpu();

	/*
	 * Ensure we see the proper queue state before going to sleep
	 * so that we do not miss the wakeup. -- see perf_pending_handle()
	 */
	smp_rmb();
2804
	return event->pending.next == NULL;
2805 2806
}

2807
static void perf_pending_sync(struct perf_event *event)
2808
{
2809
	wait_event(event->waitq, perf_not_pending(event));
2810 2811
}

2812
void perf_event_do_pending(void)
2813 2814 2815 2816
{
	__perf_pending_run();
}

2817 2818 2819 2820
/*
 * Callchain support -- arch specific
 */

2821
__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2822 2823 2824 2825
{
	return NULL;
}

2826 2827 2828 2829
__weak
void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip)
{
}
2830

2831

2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852
/*
 * We assume there is only KVM supporting the callbacks.
 * Later on, we might change it to a list if there is
 * another virtualization implementation supporting the callbacks.
 */
struct perf_guest_info_callbacks *perf_guest_cbs;

int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = cbs;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);

int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = NULL;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);

2853 2854 2855
/*
 * Output
 */
2856 2857
static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
			      unsigned long offset, unsigned long head)
2858 2859 2860 2861 2862 2863
{
	unsigned long mask;

	if (!data->writable)
		return true;

2864
	mask = perf_data_size(data) - 1;
2865 2866 2867 2868 2869 2870 2871 2872 2873 2874

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

2875
static void perf_output_wakeup(struct perf_output_handle *handle)
2876
{
2877 2878
	atomic_set(&handle->data->poll, POLL_IN);

2879
	if (handle->nmi) {
2880 2881 2882
		handle->event->pending_wakeup = 1;
		perf_pending_queue(&handle->event->pending,
				   perf_pending_event);
2883
	} else
2884
		perf_event_wakeup(handle->event);
2885 2886
}

2887 2888 2889
/*
 * Curious locking construct.
 *
2890 2891
 * We need to ensure a later event_id doesn't publish a head when a former
 * event_id isn't done writing. However since we need to deal with NMIs we
2892 2893 2894 2895 2896 2897
 * cannot fully serialize things.
 *
 * What we do is serialize between CPUs so we only have to deal with NMI
 * nesting on a single CPU.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
2898
 * event_id completes.
2899 2900 2901 2902
 */
static void perf_output_lock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2903
	int cur, cpu = get_cpu();
2904 2905 2906

	handle->locked = 0;

2907 2908 2909 2910 2911 2912 2913 2914
	for (;;) {
		cur = atomic_cmpxchg(&data->lock, -1, cpu);
		if (cur == -1) {
			handle->locked = 1;
			break;
		}
		if (cur == cpu)
			break;
2915 2916

		cpu_relax();
2917
	}
2918 2919 2920 2921 2922
}

static void perf_output_unlock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2923 2924
	unsigned long head;
	int cpu;
2925

2926
	data->done_head = data->head;
2927 2928 2929 2930 2931 2932 2933 2934 2935 2936

	if (!handle->locked)
		goto out;

again:
	/*
	 * The xchg implies a full barrier that ensures all writes are done
	 * before we publish the new head, matched by a rmb() in userspace when
	 * reading this position.
	 */
2937
	while ((head = atomic_long_xchg(&data->done_head, 0)))
2938 2939 2940
		data->user_page->data_head = head;

	/*
2941
	 * NMI can happen here, which means we can miss a done_head update.
2942 2943
	 */

2944
	cpu = atomic_xchg(&data->lock, -1);
2945 2946 2947 2948 2949
	WARN_ON_ONCE(cpu != smp_processor_id());

	/*
	 * Therefore we have to validate we did not indeed do so.
	 */
2950
	if (unlikely(atomic_long_read(&data->done_head))) {
2951 2952 2953
		/*
		 * Since we had it locked, we can lock it again.
		 */
2954
		while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2955 2956 2957 2958 2959
			cpu_relax();

		goto again;
	}

2960
	if (atomic_xchg(&data->wakeup, 0))
2961 2962
		perf_output_wakeup(handle);
out:
2963
	put_cpu();
2964 2965
}

2966 2967
void perf_output_copy(struct perf_output_handle *handle,
		      const void *buf, unsigned int len)
2968 2969
{
	unsigned int pages_mask;
2970
	unsigned long offset;
2971 2972 2973 2974 2975 2976 2977 2978
	unsigned int size;
	void **pages;

	offset		= handle->offset;
	pages_mask	= handle->data->nr_pages - 1;
	pages		= handle->data->data_pages;

	do {
2979 2980
		unsigned long page_offset;
		unsigned long page_size;
2981 2982 2983
		int nr;

		nr	    = (offset >> PAGE_SHIFT) & pages_mask;
2984 2985 2986
		page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
		page_offset = offset & (page_size - 1);
		size	    = min_t(unsigned int, page_size - page_offset, len);
2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003

		memcpy(pages[nr] + page_offset, buf, size);

		len	    -= size;
		buf	    += size;
		offset	    += size;
	} while (len);

	handle->offset = offset;

	/*
	 * Check we didn't copy past our reservation window, taking the
	 * possible unsigned int wrap into account.
	 */
	WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
}

3004
int perf_output_begin(struct perf_output_handle *handle,
3005
		      struct perf_event *event, unsigned int size,
3006
		      int nmi, int sample)
3007
{
3008
	struct perf_event *output_event;
3009
	struct perf_mmap_data *data;
3010
	unsigned long tail, offset, head;
3011 3012 3013 3014 3015 3016
	int have_lost;
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
3017

3018
	rcu_read_lock();
3019
	/*
3020
	 * For inherited events we send all the output towards the parent.
3021
	 */
3022 3023
	if (event->parent)
		event = event->parent;
3024

3025 3026 3027
	output_event = rcu_dereference(event->output);
	if (output_event)
		event = output_event;
3028

3029
	data = rcu_dereference(event->data);
3030 3031 3032
	if (!data)
		goto out;

3033
	handle->data	= data;
3034
	handle->event	= event;
3035 3036
	handle->nmi	= nmi;
	handle->sample	= sample;
3037

3038
	if (!data->nr_pages)
3039
		goto fail;
3040

3041 3042 3043 3044
	have_lost = atomic_read(&data->lost);
	if (have_lost)
		size += sizeof(lost_event);

3045 3046
	perf_output_lock(handle);

3047
	do {
3048 3049 3050 3051 3052 3053 3054
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
		tail = ACCESS_ONCE(data->user_page->data_tail);
		smp_rmb();
3055
		offset = head = atomic_long_read(&data->head);
P
Peter Zijlstra 已提交
3056
		head += size;
3057
		if (unlikely(!perf_output_space(data, tail, offset, head)))
3058
			goto fail;
3059
	} while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
3060

3061
	handle->offset	= offset;
3062
	handle->head	= head;
3063

3064
	if (head - tail > data->watermark)
3065
		atomic_set(&data->wakeup, 1);
3066

3067
	if (have_lost) {
3068
		lost_event.header.type = PERF_RECORD_LOST;
3069 3070
		lost_event.header.misc = 0;
		lost_event.header.size = sizeof(lost_event);
3071
		lost_event.id          = event->id;
3072 3073 3074 3075 3076
		lost_event.lost        = atomic_xchg(&data->lost, 0);

		perf_output_put(handle, lost_event);
	}

3077
	return 0;
3078

3079
fail:
3080 3081
	atomic_inc(&data->lost);
	perf_output_unlock(handle);
3082 3083
out:
	rcu_read_unlock();
3084

3085 3086
	return -ENOSPC;
}
3087

3088
void perf_output_end(struct perf_output_handle *handle)
3089
{
3090
	struct perf_event *event = handle->event;
3091 3092
	struct perf_mmap_data *data = handle->data;

3093
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
3094

3095
	if (handle->sample && wakeup_events) {
3096
		int events = atomic_inc_return(&data->events);
P
Peter Zijlstra 已提交
3097
		if (events >= wakeup_events) {
3098
			atomic_sub(wakeup_events, &data->events);
3099
			atomic_set(&data->wakeup, 1);
P
Peter Zijlstra 已提交
3100
		}
3101 3102 3103
	}

	perf_output_unlock(handle);
3104
	rcu_read_unlock();
3105 3106
}

3107
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
3108 3109
{
	/*
3110
	 * only top level events have the pid namespace they were created in
3111
	 */
3112 3113
	if (event->parent)
		event = event->parent;
3114

3115
	return task_tgid_nr_ns(p, event->ns);
3116 3117
}

3118
static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
3119 3120
{
	/*
3121
	 * only top level events have the pid namespace they were created in
3122
	 */
3123 3124
	if (event->parent)
		event = event->parent;
3125

3126
	return task_pid_nr_ns(p, event->ns);
3127 3128
}

3129
static void perf_output_read_one(struct perf_output_handle *handle,
3130
				 struct perf_event *event)
3131
{
3132
	u64 read_format = event->attr.read_format;
3133 3134 3135
	u64 values[4];
	int n = 0;

3136
	values[n++] = atomic64_read(&event->count);
3137
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3138 3139
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
3140 3141
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3142 3143
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
3144 3145
	}
	if (read_format & PERF_FORMAT_ID)
3146
		values[n++] = primary_event_id(event);
3147 3148 3149 3150 3151

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
3152
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3153 3154
 */
static void perf_output_read_group(struct perf_output_handle *handle,
3155
			    struct perf_event *event)
3156
{
3157 3158
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = leader->total_time_enabled;

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = leader->total_time_running;

3170
	if (leader != event)
3171 3172 3173 3174
		leader->pmu->read(leader);

	values[n++] = atomic64_read(&leader->count);
	if (read_format & PERF_FORMAT_ID)
3175
		values[n++] = primary_event_id(leader);
3176 3177 3178

	perf_output_copy(handle, values, n * sizeof(u64));

3179
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3180 3181
		n = 0;

3182
		if (sub != event)
3183 3184 3185 3186
			sub->pmu->read(sub);

		values[n++] = atomic64_read(&sub->count);
		if (read_format & PERF_FORMAT_ID)
3187
			values[n++] = primary_event_id(sub);
3188 3189 3190 3191 3192 3193

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

static void perf_output_read(struct perf_output_handle *handle,
3194
			     struct perf_event *event)
3195
{
3196 3197
	if (event->attr.read_format & PERF_FORMAT_GROUP)
		perf_output_read_group(handle, event);
3198
	else
3199
		perf_output_read_one(handle, event);
3200 3201
}

3202 3203 3204
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3205
			struct perf_event *event)
3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3236
		perf_output_read(handle, event);
3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3274
			 struct perf_event *event,
3275
			 struct pt_regs *regs)
3276
{
3277
	u64 sample_type = event->attr.sample_type;
3278

3279
	data->type = sample_type;
3280

3281
	header->type = PERF_RECORD_SAMPLE;
3282 3283 3284 3285
	header->size = sizeof(*header);

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3286

3287
	if (sample_type & PERF_SAMPLE_IP) {
3288 3289 3290
		data->ip = perf_instruction_pointer(regs);

		header->size += sizeof(data->ip);
3291
	}
3292

3293
	if (sample_type & PERF_SAMPLE_TID) {
3294
		/* namespace issues */
3295 3296
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
3297

3298
		header->size += sizeof(data->tid_entry);
3299 3300
	}

3301
	if (sample_type & PERF_SAMPLE_TIME) {
P
Peter Zijlstra 已提交
3302
		data->time = perf_clock();
3303

3304
		header->size += sizeof(data->time);
3305 3306
	}

3307
	if (sample_type & PERF_SAMPLE_ADDR)
3308
		header->size += sizeof(data->addr);
3309

3310
	if (sample_type & PERF_SAMPLE_ID) {
3311
		data->id = primary_event_id(event);
3312

3313 3314 3315 3316
		header->size += sizeof(data->id);
	}

	if (sample_type & PERF_SAMPLE_STREAM_ID) {
3317
		data->stream_id = event->id;
3318 3319 3320

		header->size += sizeof(data->stream_id);
	}
3321

3322
	if (sample_type & PERF_SAMPLE_CPU) {
3323 3324
		data->cpu_entry.cpu		= raw_smp_processor_id();
		data->cpu_entry.reserved	= 0;
3325

3326
		header->size += sizeof(data->cpu_entry);
3327 3328
	}

3329
	if (sample_type & PERF_SAMPLE_PERIOD)
3330
		header->size += sizeof(data->period);
3331

3332
	if (sample_type & PERF_SAMPLE_READ)
3333
		header->size += perf_event_read_size(event);
3334

3335
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3336
		int size = 1;
3337

3338 3339 3340 3341 3342 3343
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3344 3345
	}

3346
	if (sample_type & PERF_SAMPLE_RAW) {
3347 3348 3349 3350 3351 3352 3353 3354
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3355
		header->size += size;
3356
	}
3357
}
3358

3359
static void perf_event_output(struct perf_event *event, int nmi,
3360 3361 3362 3363 3364
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3365

3366
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3367

3368
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3369
		return;
3370

3371
	perf_output_sample(&handle, &header, data, event);
3372

3373
	perf_output_end(&handle);
3374 3375
}

3376
/*
3377
 * read event_id
3378 3379 3380 3381 3382 3383 3384 3385 3386 3387
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3388
perf_event_read_event(struct perf_event *event,
3389 3390 3391
			struct task_struct *task)
{
	struct perf_output_handle handle;
3392
	struct perf_read_event read_event = {
3393
		.header = {
3394
			.type = PERF_RECORD_READ,
3395
			.misc = 0,
3396
			.size = sizeof(read_event) + perf_event_read_size(event),
3397
		},
3398 3399
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3400
	};
3401
	int ret;
3402

3403
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3404 3405 3406
	if (ret)
		return;

3407
	perf_output_put(&handle, read_event);
3408
	perf_output_read(&handle, event);
3409

3410 3411 3412
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3413
/*
P
Peter Zijlstra 已提交
3414 3415 3416
 * task tracking -- fork/exit
 *
 * enabled by: attr.comm | attr.mmap | attr.task
P
Peter Zijlstra 已提交
3417 3418
 */

P
Peter Zijlstra 已提交
3419
struct perf_task_event {
3420
	struct task_struct		*task;
3421
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3422 3423 3424 3425 3426 3427

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3428 3429
		u32				tid;
		u32				ptid;
3430
		u64				time;
3431
	} event_id;
P
Peter Zijlstra 已提交
3432 3433
};

3434
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3435
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3436 3437
{
	struct perf_output_handle handle;
P
Peter Zijlstra 已提交
3438
	struct task_struct *task = task_event->task;
3439 3440 3441 3442 3443 3444 3445 3446
	unsigned long flags;
	int size, ret;

	/*
	 * If this CPU attempts to acquire an rq lock held by a CPU spinning
	 * in perf_output_lock() from interrupt context, it's game over.
	 */
	local_irq_save(flags);
3447

3448 3449
	size  = task_event->event_id.header.size;
	ret = perf_output_begin(&handle, event, size, 0, 0);
P
Peter Zijlstra 已提交
3450

3451 3452
	if (ret) {
		local_irq_restore(flags);
P
Peter Zijlstra 已提交
3453
		return;
3454
	}
P
Peter Zijlstra 已提交
3455

3456 3457
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3458

3459 3460
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3461

3462
	perf_output_put(&handle, task_event->event_id);
3463

P
Peter Zijlstra 已提交
3464
	perf_output_end(&handle);
3465
	local_irq_restore(flags);
P
Peter Zijlstra 已提交
3466 3467
}

3468
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3469
{
P
Peter Zijlstra 已提交
3470
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3471 3472
		return 0;

3473 3474 3475
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3476
	if (event->attr.comm || event->attr.mmap || event->attr.task)
P
Peter Zijlstra 已提交
3477 3478 3479 3480 3481
		return 1;

	return 0;
}

3482
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
3483
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3484
{
3485
	struct perf_event *event;
P
Peter Zijlstra 已提交
3486

3487 3488 3489
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
3490 3491 3492
	}
}

3493
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3494 3495
{
	struct perf_cpu_context *cpuctx;
3496
	struct perf_event_context *ctx = task_event->task_ctx;
P
Peter Zijlstra 已提交
3497

3498
	rcu_read_lock();
P
Peter Zijlstra 已提交
3499
	cpuctx = &get_cpu_var(perf_cpu_context);
3500
	perf_event_task_ctx(&cpuctx->ctx, task_event);
3501
	if (!ctx)
P
Peter Zijlstra 已提交
3502
		ctx = rcu_dereference(current->perf_event_ctxp);
P
Peter Zijlstra 已提交
3503
	if (ctx)
3504
		perf_event_task_ctx(ctx, task_event);
3505
	put_cpu_var(perf_cpu_context);
P
Peter Zijlstra 已提交
3506 3507 3508
	rcu_read_unlock();
}

3509 3510
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3511
			      int new)
P
Peter Zijlstra 已提交
3512
{
P
Peter Zijlstra 已提交
3513
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
3514

3515 3516 3517
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
3518 3519
		return;

P
Peter Zijlstra 已提交
3520
	task_event = (struct perf_task_event){
3521 3522
		.task	  = task,
		.task_ctx = task_ctx,
3523
		.event_id    = {
P
Peter Zijlstra 已提交
3524
			.header = {
3525
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3526
				.misc = 0,
3527
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
3528
			},
3529 3530
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
3531 3532
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
3533
			.time = perf_clock(),
P
Peter Zijlstra 已提交
3534 3535 3536
		},
	};

3537
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
3538 3539
}

3540
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
3541
{
3542
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
3543 3544
}

3545 3546 3547 3548 3549
/*
 * comm tracking
 */

struct perf_comm_event {
3550 3551
	struct task_struct	*task;
	char			*comm;
3552 3553 3554 3555 3556 3557 3558
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
3559
	} event_id;
3560 3561
};

3562
static void perf_event_comm_output(struct perf_event *event,
3563 3564 3565
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
3566 3567
	int size = comm_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3568 3569 3570 3571

	if (ret)
		return;

3572 3573
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3574

3575
	perf_output_put(&handle, comm_event->event_id);
3576 3577 3578 3579 3580
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
	perf_output_end(&handle);
}

3581
static int perf_event_comm_match(struct perf_event *event)
3582
{
P
Peter Zijlstra 已提交
3583
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3584 3585
		return 0;

3586 3587 3588
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3589
	if (event->attr.comm)
3590 3591 3592 3593 3594
		return 1;

	return 0;
}

3595
static void perf_event_comm_ctx(struct perf_event_context *ctx,
3596 3597
				  struct perf_comm_event *comm_event)
{
3598
	struct perf_event *event;
3599

3600 3601 3602
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
3603 3604 3605
	}
}

3606
static void perf_event_comm_event(struct perf_comm_event *comm_event)
3607 3608
{
	struct perf_cpu_context *cpuctx;
3609
	struct perf_event_context *ctx;
3610
	unsigned int size;
3611
	char comm[TASK_COMM_LEN];
3612

3613
	memset(comm, 0, sizeof(comm));
3614
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
3615
	size = ALIGN(strlen(comm)+1, sizeof(u64));
3616 3617 3618 3619

	comm_event->comm = comm;
	comm_event->comm_size = size;

3620
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3621

3622
	rcu_read_lock();
3623
	cpuctx = &get_cpu_var(perf_cpu_context);
3624 3625
	perf_event_comm_ctx(&cpuctx->ctx, comm_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3626
	if (ctx)
3627
		perf_event_comm_ctx(ctx, comm_event);
3628
	put_cpu_var(perf_cpu_context);
3629
	rcu_read_unlock();
3630 3631
}

3632
void perf_event_comm(struct task_struct *task)
3633
{
3634 3635
	struct perf_comm_event comm_event;

3636 3637
	if (task->perf_event_ctxp)
		perf_event_enable_on_exec(task);
3638

3639
	if (!atomic_read(&nr_comm_events))
3640
		return;
3641

3642
	comm_event = (struct perf_comm_event){
3643
		.task	= task,
3644 3645
		/* .comm      */
		/* .comm_size */
3646
		.event_id  = {
3647
			.header = {
3648
				.type = PERF_RECORD_COMM,
3649 3650 3651 3652 3653
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3654 3655 3656
		},
	};

3657
	perf_event_comm_event(&comm_event);
3658 3659
}

3660 3661 3662 3663 3664
/*
 * mmap tracking
 */

struct perf_mmap_event {
3665 3666 3667 3668
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
3669 3670 3671 3672 3673 3674 3675 3676 3677

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
3678
	} event_id;
3679 3680
};

3681
static void perf_event_mmap_output(struct perf_event *event,
3682 3683 3684
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
3685 3686
	int size = mmap_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3687 3688 3689 3690

	if (ret)
		return;

3691 3692
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
3693

3694
	perf_output_put(&handle, mmap_event->event_id);
3695 3696
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
3697
	perf_output_end(&handle);
3698 3699
}

3700
static int perf_event_mmap_match(struct perf_event *event,
3701 3702
				   struct perf_mmap_event *mmap_event)
{
P
Peter Zijlstra 已提交
3703
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3704 3705
		return 0;

3706 3707 3708
	if (event->cpu != -1 && event->cpu != smp_processor_id())
		return 0;

3709
	if (event->attr.mmap)
3710 3711 3712 3713 3714
		return 1;

	return 0;
}

3715
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3716 3717
				  struct perf_mmap_event *mmap_event)
{
3718
	struct perf_event *event;
3719

3720 3721 3722
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_mmap_match(event, mmap_event))
			perf_event_mmap_output(event, mmap_event);
3723 3724 3725
	}
}

3726
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3727 3728
{
	struct perf_cpu_context *cpuctx;
3729
	struct perf_event_context *ctx;
3730 3731
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
3732 3733 3734
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
3735
	const char *name;
3736

3737 3738
	memset(tmp, 0, sizeof(tmp));

3739
	if (file) {
3740 3741 3742 3743 3744 3745
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3746 3747 3748 3749
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
3750
		name = d_path(&file->f_path, buf, PATH_MAX);
3751 3752 3753 3754 3755
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
3756 3757 3758
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
3759
			goto got_name;
3760
		}
3761 3762 3763 3764 3765 3766

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
		}

3767 3768 3769 3770 3771
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
3772
	size = ALIGN(strlen(name)+1, sizeof(u64));
3773 3774 3775 3776

	mmap_event->file_name = name;
	mmap_event->file_size = size;

3777
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3778

3779
	rcu_read_lock();
3780
	cpuctx = &get_cpu_var(perf_cpu_context);
3781 3782
	perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
	ctx = rcu_dereference(current->perf_event_ctxp);
3783
	if (ctx)
3784
		perf_event_mmap_ctx(ctx, mmap_event);
3785
	put_cpu_var(perf_cpu_context);
3786 3787
	rcu_read_unlock();

3788 3789 3790
	kfree(buf);
}

3791
void __perf_event_mmap(struct vm_area_struct *vma)
3792
{
3793 3794
	struct perf_mmap_event mmap_event;

3795
	if (!atomic_read(&nr_mmap_events))
3796 3797 3798
		return;

	mmap_event = (struct perf_mmap_event){
3799
		.vma	= vma,
3800 3801
		/* .file_name */
		/* .file_size */
3802
		.event_id  = {
3803
			.header = {
3804
				.type = PERF_RECORD_MMAP,
3805
				.misc = PERF_RECORD_MISC_USER,
3806 3807 3808 3809
				/* .size */
			},
			/* .pid */
			/* .tid */
3810 3811
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
3812
			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
3813 3814 3815
		},
	};

3816
	perf_event_mmap_event(&mmap_event);
3817 3818
}

3819 3820 3821 3822
/*
 * IRQ throttle logging
 */

3823
static void perf_log_throttle(struct perf_event *event, int enable)
3824 3825 3826 3827 3828 3829 3830
{
	struct perf_output_handle handle;
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
3831
		u64				id;
3832
		u64				stream_id;
3833 3834
	} throttle_event = {
		.header = {
3835
			.type = PERF_RECORD_THROTTLE,
3836 3837 3838
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
3839
		.time		= perf_clock(),
3840 3841
		.id		= primary_event_id(event),
		.stream_id	= event->id,
3842 3843
	};

3844
	if (enable)
3845
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3846

3847
	ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3848 3849 3850 3851 3852 3853 3854
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
	perf_output_end(&handle);
}

3855
/*
3856
 * Generic event overflow handling, sampling.
3857 3858
 */

3859
static int __perf_event_overflow(struct perf_event *event, int nmi,
3860 3861
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
3862
{
3863 3864
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
3865 3866
	int ret = 0;

3867
	throttle = (throttle && event->pmu->unthrottle != NULL);
3868

3869
	if (!throttle) {
3870
		hwc->interrupts++;
3871
	} else {
3872 3873
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
3874
			if (HZ * hwc->interrupts >
3875
					(u64)sysctl_perf_event_sample_rate) {
3876
				hwc->interrupts = MAX_INTERRUPTS;
3877
				perf_log_throttle(event, 0);
3878 3879 3880 3881
				ret = 1;
			}
		} else {
			/*
3882
			 * Keep re-disabling events even though on the previous
3883
			 * pass we disabled it - just in case we raced with a
3884
			 * sched-in and the event got enabled again:
3885
			 */
3886 3887 3888
			ret = 1;
		}
	}
3889

3890
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
3891
		u64 now = perf_clock();
3892
		s64 delta = now - hwc->freq_time_stamp;
3893

3894
		hwc->freq_time_stamp = now;
3895

3896 3897
		if (delta > 0 && delta < 2*TICK_NSEC)
			perf_adjust_period(event, delta, hwc->last_period);
3898 3899
	}

3900 3901
	/*
	 * XXX event_limit might not quite work as expected on inherited
3902
	 * events
3903 3904
	 */

3905 3906
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
3907
		ret = 1;
3908
		event->pending_kill = POLL_HUP;
3909
		if (nmi) {
3910 3911 3912
			event->pending_disable = 1;
			perf_pending_queue(&event->pending,
					   perf_pending_event);
3913
		} else
3914
			perf_event_disable(event);
3915 3916
	}

3917 3918 3919 3920 3921
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

3922
	return ret;
3923 3924
}

3925
int perf_event_overflow(struct perf_event *event, int nmi,
3926 3927
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
3928
{
3929
	return __perf_event_overflow(event, nmi, 1, data, regs);
3930 3931
}

3932
/*
3933
 * Generic software event infrastructure
3934 3935
 */

3936
/*
3937 3938
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
3939 3940 3941 3942
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

3943
static u64 perf_swevent_set_period(struct perf_event *event)
3944
{
3945
	struct hw_perf_event *hwc = &event->hw;
3946 3947 3948 3949 3950
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
3951 3952

again:
3953 3954 3955
	old = val = atomic64_read(&hwc->period_left);
	if (val < 0)
		return 0;
3956

3957 3958 3959 3960 3961
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
	if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
		goto again;
3962

3963
	return nr;
3964 3965
}

3966
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3967 3968
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
3969
{
3970
	struct hw_perf_event *hwc = &event->hw;
3971
	int throttle = 0;
3972

3973
	data->period = event->hw.last_period;
3974 3975
	if (!overflow)
		overflow = perf_swevent_set_period(event);
3976

3977 3978
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
3979

3980
	for (; overflow; overflow--) {
3981
		if (__perf_event_overflow(event, nmi, throttle,
3982
					    data, regs)) {
3983 3984 3985 3986 3987 3988
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
3989
		throttle = 1;
3990
	}
3991 3992
}

3993
static void perf_swevent_unthrottle(struct perf_event *event)
3994 3995
{
	/*
3996
	 * Nothing to do, we already reset hwc->interrupts.
3997
	 */
3998
}
3999

4000
static void perf_swevent_add(struct perf_event *event, u64 nr,
4001 4002
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
4003
{
4004
	struct hw_perf_event *hwc = &event->hw;
4005

4006
	atomic64_add(nr, &event->count);
4007

4008 4009 4010
	if (!regs)
		return;

4011 4012
	if (!hwc->sample_period)
		return;
4013

4014 4015 4016 4017
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

	if (atomic64_add_negative(nr, &hwc->period_left))
4018
		return;
4019

4020
	perf_swevent_overflow(event, 0, nmi, data, regs);
4021 4022
}

L
Li Zefan 已提交
4023 4024 4025
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data);

4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039
static int perf_exclude_event(struct perf_event *event,
			      struct pt_regs *regs)
{
	if (regs) {
		if (event->attr.exclude_user && user_mode(regs))
			return 1;

		if (event->attr.exclude_kernel && !user_mode(regs))
			return 1;
	}

	return 0;
}

4040
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
4041
				enum perf_type_id type,
L
Li Zefan 已提交
4042 4043 4044
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
4045
{
4046
	if (event->attr.type != type)
4047
		return 0;
4048

4049
	if (event->attr.config != event_id)
4050 4051
		return 0;

4052 4053
	if (perf_exclude_event(event, regs))
		return 0;
4054

L
Li Zefan 已提交
4055 4056 4057 4058
	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
	    !perf_tp_event_match(event, data))
		return 0;

4059 4060 4061
	return 1;
}

4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087
static inline u64 swevent_hash(u64 type, u32 event_id)
{
	u64 val = event_id | (type << 32);

	return hash_64(val, SWEVENT_HLIST_BITS);
}

static struct hlist_head *
find_swevent_head(struct perf_cpu_context *ctx, u64 type, u32 event_id)
{
	u64 hash;
	struct swevent_hlist *hlist;

	hash = swevent_hash(type, event_id);

	hlist = rcu_dereference(ctx->swevent_hlist);
	if (!hlist)
		return NULL;

	return &hlist->heads[hash];
}

static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
				    u64 nr, int nmi,
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
4088
{
4089
	struct perf_cpu_context *cpuctx;
4090
	struct perf_event *event;
4091 4092
	struct hlist_node *node;
	struct hlist_head *head;
4093

4094 4095 4096 4097 4098 4099 4100 4101 4102 4103
	cpuctx = &__get_cpu_var(perf_cpu_context);

	rcu_read_lock();

	head = find_swevent_head(cpuctx, type, event_id);

	if (!head)
		goto end;

	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
L
Li Zefan 已提交
4104
		if (perf_swevent_match(event, type, event_id, data, regs))
4105
			perf_swevent_add(event, nr, nmi, data, regs);
4106
	}
4107 4108
end:
	rcu_read_unlock();
4109 4110
}

4111
int perf_swevent_get_recursion_context(void)
P
Peter Zijlstra 已提交
4112
{
4113 4114
	struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
	int rctx;
4115

P
Peter Zijlstra 已提交
4116
	if (in_nmi())
4117
		rctx = 3;
4118
	else if (in_irq())
4119
		rctx = 2;
4120
	else if (in_softirq())
4121
		rctx = 1;
4122
	else
4123
		rctx = 0;
P
Peter Zijlstra 已提交
4124

4125 4126
	if (cpuctx->recursion[rctx]) {
		put_cpu_var(perf_cpu_context);
4127
		return -1;
4128
	}
P
Peter Zijlstra 已提交
4129

4130 4131
	cpuctx->recursion[rctx]++;
	barrier();
P
Peter Zijlstra 已提交
4132

4133
	return rctx;
P
Peter Zijlstra 已提交
4134
}
I
Ingo Molnar 已提交
4135
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
4136

4137
void perf_swevent_put_recursion_context(int rctx)
4138
{
4139 4140
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
	barrier();
4141
	cpuctx->recursion[rctx]--;
4142
	put_cpu_var(perf_cpu_context);
4143
}
I
Ingo Molnar 已提交
4144
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
P
Peter Zijlstra 已提交
4145

4146

4147
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
4148
			    struct pt_regs *regs, u64 addr)
4149
{
4150
	struct perf_sample_data data;
4151 4152 4153 4154 4155
	int rctx;

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return;
4156

4157
	perf_sample_data_init(&data, addr);
4158

4159
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
4160 4161

	perf_swevent_put_recursion_context(rctx);
4162 4163
}

4164
static void perf_swevent_read(struct perf_event *event)
4165 4166 4167
{
}

4168
static int perf_swevent_enable(struct perf_event *event)
4169
{
4170
	struct hw_perf_event *hwc = &event->hw;
4171 4172 4173 4174
	struct perf_cpu_context *cpuctx;
	struct hlist_head *head;

	cpuctx = &__get_cpu_var(perf_cpu_context);
4175 4176 4177

	if (hwc->sample_period) {
		hwc->last_period = hwc->sample_period;
4178
		perf_swevent_set_period(event);
4179
	}
4180 4181 4182 4183 4184 4185 4186

	head = find_swevent_head(cpuctx, event->attr.type, event->attr.config);
	if (WARN_ON_ONCE(!head))
		return -EINVAL;

	hlist_add_head_rcu(&event->hlist_entry, head);

4187 4188 4189
	return 0;
}

4190
static void perf_swevent_disable(struct perf_event *event)
4191
{
4192
	hlist_del_rcu(&event->hlist_entry);
4193 4194
}

4195
static const struct pmu perf_ops_generic = {
4196 4197 4198 4199
	.enable		= perf_swevent_enable,
	.disable	= perf_swevent_disable,
	.read		= perf_swevent_read,
	.unthrottle	= perf_swevent_unthrottle,
4200 4201
};

4202
/*
4203
 * hrtimer based swevent callback
4204 4205
 */

4206
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
4207 4208 4209
{
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
4210
	struct pt_regs *regs;
4211
	struct perf_event *event;
4212 4213
	u64 period;

4214
	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
4215
	event->pmu->read(event);
4216

4217
	perf_sample_data_init(&data, 0);
4218
	data.period = event->hw.last_period;
4219
	regs = get_irq_regs();
4220

4221
	if (regs && !perf_exclude_event(event, regs)) {
4222 4223 4224
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
4225 4226
	}

4227
	period = max_t(u64, 10000, event->hw.sample_period);
4228 4229 4230 4231 4232
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));

	return ret;
}

4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268
static void perf_swevent_start_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
	if (hwc->sample_period) {
		u64 period;

		if (hwc->remaining) {
			if (hwc->remaining < 0)
				period = 10000;
			else
				period = hwc->remaining;
			hwc->remaining = 0;
		} else {
			period = max_t(u64, 10000, hwc->sample_period);
		}
		__hrtimer_start_range_ns(&hwc->hrtimer,
				ns_to_ktime(period), 0,
				HRTIMER_MODE_REL, 0);
	}
}

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (hwc->sample_period) {
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
		hwc->remaining = ktime_to_ns(remaining);

		hrtimer_cancel(&hwc->hrtimer);
	}
}

4269
/*
4270
 * Software event: cpu wall time clock
4271 4272
 */

4273
static void cpu_clock_perf_event_update(struct perf_event *event)
4274 4275 4276 4277 4278 4279
{
	int cpu = raw_smp_processor_id();
	s64 prev;
	u64 now;

	now = cpu_clock(cpu);
4280
	prev = atomic64_xchg(&event->hw.prev_count, now);
4281
	atomic64_add(now - prev, &event->count);
4282 4283
}

4284
static int cpu_clock_perf_event_enable(struct perf_event *event)
4285
{
4286
	struct hw_perf_event *hwc = &event->hw;
4287 4288 4289
	int cpu = raw_smp_processor_id();

	atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4290
	perf_swevent_start_hrtimer(event);
4291 4292 4293 4294

	return 0;
}

4295
static void cpu_clock_perf_event_disable(struct perf_event *event)
4296
{
4297
	perf_swevent_cancel_hrtimer(event);
4298
	cpu_clock_perf_event_update(event);
4299 4300
}

4301
static void cpu_clock_perf_event_read(struct perf_event *event)
4302
{
4303
	cpu_clock_perf_event_update(event);
4304 4305
}

4306
static const struct pmu perf_ops_cpu_clock = {
4307 4308 4309
	.enable		= cpu_clock_perf_event_enable,
	.disable	= cpu_clock_perf_event_disable,
	.read		= cpu_clock_perf_event_read,
4310 4311
};

4312
/*
4313
 * Software event: task time clock
4314 4315
 */

4316
static void task_clock_perf_event_update(struct perf_event *event, u64 now)
I
Ingo Molnar 已提交
4317
{
4318
	u64 prev;
I
Ingo Molnar 已提交
4319 4320
	s64 delta;

4321
	prev = atomic64_xchg(&event->hw.prev_count, now);
I
Ingo Molnar 已提交
4322
	delta = now - prev;
4323
	atomic64_add(delta, &event->count);
4324 4325
}

4326
static int task_clock_perf_event_enable(struct perf_event *event)
I
Ingo Molnar 已提交
4327
{
4328
	struct hw_perf_event *hwc = &event->hw;
4329 4330
	u64 now;

4331
	now = event->ctx->time;
4332

4333
	atomic64_set(&hwc->prev_count, now);
4334 4335

	perf_swevent_start_hrtimer(event);
4336 4337

	return 0;
I
Ingo Molnar 已提交
4338 4339
}

4340
static void task_clock_perf_event_disable(struct perf_event *event)
4341
{
4342
	perf_swevent_cancel_hrtimer(event);
4343
	task_clock_perf_event_update(event, event->ctx->time);
4344

4345
}
I
Ingo Molnar 已提交
4346

4347
static void task_clock_perf_event_read(struct perf_event *event)
4348
{
4349 4350 4351
	u64 time;

	if (!in_nmi()) {
4352 4353
		update_context_time(event->ctx);
		time = event->ctx->time;
4354 4355
	} else {
		u64 now = perf_clock();
4356 4357
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
4358 4359
	}

4360
	task_clock_perf_event_update(event, time);
4361 4362
}

4363
static const struct pmu perf_ops_task_clock = {
4364 4365 4366
	.enable		= task_clock_perf_event_enable,
	.disable	= task_clock_perf_event_disable,
	.read		= task_clock_perf_event_read,
4367 4368
};

4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467
static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
{
	struct swevent_hlist *hlist;

	hlist = container_of(rcu_head, struct swevent_hlist, rcu_head);
	kfree(hlist);
}

static void swevent_hlist_release(struct perf_cpu_context *cpuctx)
{
	struct swevent_hlist *hlist;

	if (!cpuctx->swevent_hlist)
		return;

	hlist = cpuctx->swevent_hlist;
	rcu_assign_pointer(cpuctx->swevent_hlist, NULL);
	call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
}

static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);

	mutex_lock(&cpuctx->hlist_mutex);

	if (!--cpuctx->hlist_refcount)
		swevent_hlist_release(cpuctx);

	mutex_unlock(&cpuctx->hlist_mutex);
}

static void swevent_hlist_put(struct perf_event *event)
{
	int cpu;

	if (event->cpu != -1) {
		swevent_hlist_put_cpu(event, event->cpu);
		return;
	}

	for_each_possible_cpu(cpu)
		swevent_hlist_put_cpu(event, cpu);
}

static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
	int err = 0;

	mutex_lock(&cpuctx->hlist_mutex);

	if (!cpuctx->swevent_hlist && cpu_online(cpu)) {
		struct swevent_hlist *hlist;

		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
		if (!hlist) {
			err = -ENOMEM;
			goto exit;
		}
		rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
	}
	cpuctx->hlist_refcount++;
 exit:
	mutex_unlock(&cpuctx->hlist_mutex);

	return err;
}

static int swevent_hlist_get(struct perf_event *event)
{
	int err;
	int cpu, failed_cpu;

	if (event->cpu != -1)
		return swevent_hlist_get_cpu(event, event->cpu);

	get_online_cpus();
	for_each_possible_cpu(cpu) {
		err = swevent_hlist_get_cpu(event, cpu);
		if (err) {
			failed_cpu = cpu;
			goto fail;
		}
	}
	put_online_cpus();

	return 0;
 fail:
	for_each_possible_cpu(cpu) {
		if (cpu == failed_cpu)
			break;
		swevent_hlist_put_cpu(event, cpu);
	}

	put_online_cpus();
	return err;
}

4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497
#ifdef CONFIG_EVENT_TRACING

void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
		   int entry_size, struct pt_regs *regs)
{
	struct perf_sample_data data;
	struct perf_raw_record raw = {
		.size = entry_size,
		.data = record,
	};

	perf_sample_data_init(&data, addr);
	data.raw = &raw;

	/* Trace events already protected against recursion */
	do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
			 &data, regs);
}
EXPORT_SYMBOL_GPL(perf_tp_event);

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}

4498
static void tp_perf_event_destroy(struct perf_event *event)
4499
{
4500
	perf_trace_disable(event->attr.config);
4501
	swevent_hlist_put(event);
4502 4503
}

4504
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4505
{
4506 4507
	int err;

4508 4509 4510 4511
	/*
	 * Raw tracepoint data is a severe data leak, only allow root to
	 * have these.
	 */
4512
	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4513
			perf_paranoid_tracepoint_raw() &&
4514 4515 4516
			!capable(CAP_SYS_ADMIN))
		return ERR_PTR(-EPERM);

4517
	if (perf_trace_enable(event->attr.config))
4518 4519
		return NULL;

4520
	event->destroy = tp_perf_event_destroy;
4521 4522 4523 4524 4525
	err = swevent_hlist_get(event);
	if (err) {
		perf_trace_disable(event->attr.config);
		return ERR_PTR(err);
	}
4526 4527 4528

	return &perf_ops_generic;
}
L
Li Zefan 已提交
4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

4553
#else
L
Li Zefan 已提交
4554 4555 4556 4557 4558 4559 4560

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	return 1;
}

4561
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4562 4563 4564
{
	return NULL;
}
L
Li Zefan 已提交
4565 4566 4567 4568 4569 4570 4571 4572 4573 4574

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

4575
#endif /* CONFIG_EVENT_TRACING */
4576

4577 4578 4579 4580 4581 4582 4583 4584 4585
#ifdef CONFIG_HAVE_HW_BREAKPOINT
static void bp_perf_event_destroy(struct perf_event *event)
{
	release_bp_slot(event);
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	int err;
4586 4587

	err = register_perf_hw_breakpoint(bp);
4588 4589 4590 4591 4592 4593 4594 4595
	if (err)
		return ERR_PTR(err);

	bp->destroy = bp_perf_event_destroy;

	return &perf_ops_bp;
}

4596
void perf_bp_event(struct perf_event *bp, void *data)
4597
{
4598 4599 4600
	struct perf_sample_data sample;
	struct pt_regs *regs = data;

4601
	perf_sample_data_init(&sample, bp->attr.bp_addr);
4602 4603 4604

	if (!perf_exclude_event(bp, regs))
		perf_swevent_add(bp, 1, 1, &sample, regs);
4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616
}
#else
static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	return NULL;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
}
#endif

4617
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4618

4619
static void sw_perf_event_destroy(struct perf_event *event)
4620
{
4621
	u64 event_id = event->attr.config;
4622

4623
	WARN_ON(event->parent);
4624

4625
	atomic_dec(&perf_swevent_enabled[event_id]);
4626
	swevent_hlist_put(event);
4627 4628
}

4629
static const struct pmu *sw_perf_event_init(struct perf_event *event)
4630
{
4631
	const struct pmu *pmu = NULL;
4632
	u64 event_id = event->attr.config;
4633

4634
	/*
4635
	 * Software events (currently) can't in general distinguish
4636 4637 4638 4639 4640
	 * between user, kernel and hypervisor events.
	 * However, context switches and cpu migrations are considered
	 * to be kernel events, and page faults are never hypervisor
	 * events.
	 */
4641
	switch (event_id) {
4642
	case PERF_COUNT_SW_CPU_CLOCK:
4643
		pmu = &perf_ops_cpu_clock;
4644

4645
		break;
4646
	case PERF_COUNT_SW_TASK_CLOCK:
4647
		/*
4648 4649
		 * If the user instantiates this as a per-cpu event,
		 * use the cpu_clock event instead.
4650
		 */
4651
		if (event->ctx->task)
4652
			pmu = &perf_ops_task_clock;
4653
		else
4654
			pmu = &perf_ops_cpu_clock;
4655

4656
		break;
4657 4658 4659 4660 4661
	case PERF_COUNT_SW_PAGE_FAULTS:
	case PERF_COUNT_SW_PAGE_FAULTS_MIN:
	case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
	case PERF_COUNT_SW_CONTEXT_SWITCHES:
	case PERF_COUNT_SW_CPU_MIGRATIONS:
4662 4663
	case PERF_COUNT_SW_ALIGNMENT_FAULTS:
	case PERF_COUNT_SW_EMULATION_FAULTS:
4664
		if (!event->parent) {
4665 4666 4667 4668 4669 4670
			int err;

			err = swevent_hlist_get(event);
			if (err)
				return ERR_PTR(err);

4671 4672
			atomic_inc(&perf_swevent_enabled[event_id]);
			event->destroy = sw_perf_event_destroy;
4673
		}
4674
		pmu = &perf_ops_generic;
4675
		break;
4676
	}
4677

4678
	return pmu;
4679 4680
}

T
Thomas Gleixner 已提交
4681
/*
4682
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
4683
 */
4684 4685
static struct perf_event *
perf_event_alloc(struct perf_event_attr *attr,
4686
		   int cpu,
4687 4688 4689
		   struct perf_event_context *ctx,
		   struct perf_event *group_leader,
		   struct perf_event *parent_event,
4690
		   perf_overflow_handler_t overflow_handler,
4691
		   gfp_t gfpflags)
T
Thomas Gleixner 已提交
4692
{
4693
	const struct pmu *pmu;
4694 4695
	struct perf_event *event;
	struct hw_perf_event *hwc;
4696
	long err;
T
Thomas Gleixner 已提交
4697

4698 4699
	event = kzalloc(sizeof(*event), gfpflags);
	if (!event)
4700
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
4701

4702
	/*
4703
	 * Single events are their own group leaders, with an
4704 4705 4706
	 * empty sibling list:
	 */
	if (!group_leader)
4707
		group_leader = event;
4708

4709 4710
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
4711

4712 4713 4714 4715
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
T
Thomas Gleixner 已提交
4716

4717
	mutex_init(&event->mmap_mutex);
4718

4719 4720 4721 4722 4723 4724
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->ctx		= ctx;
	event->oncpu		= -1;
4725

4726
	event->parent		= parent_event;
4727

4728 4729
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
4730

4731
	event->state		= PERF_EVENT_STATE_INACTIVE;
4732

4733 4734
	if (!overflow_handler && parent_event)
		overflow_handler = parent_event->overflow_handler;
4735
	
4736
	event->overflow_handler	= overflow_handler;
4737

4738
	if (attr->disabled)
4739
		event->state = PERF_EVENT_STATE_OFF;
4740

4741
	pmu = NULL;
4742

4743
	hwc = &event->hw;
4744
	hwc->sample_period = attr->sample_period;
4745
	if (attr->freq && attr->sample_freq)
4746
		hwc->sample_period = 1;
4747
	hwc->last_period = hwc->sample_period;
4748 4749

	atomic64_set(&hwc->period_left, hwc->sample_period);
4750

4751
	/*
4752
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
4753
	 */
4754
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4755 4756
		goto done;

4757
	switch (attr->type) {
4758
	case PERF_TYPE_RAW:
4759
	case PERF_TYPE_HARDWARE:
4760
	case PERF_TYPE_HW_CACHE:
4761
		pmu = hw_perf_event_init(event);
4762 4763 4764
		break;

	case PERF_TYPE_SOFTWARE:
4765
		pmu = sw_perf_event_init(event);
4766 4767 4768
		break;

	case PERF_TYPE_TRACEPOINT:
4769
		pmu = tp_perf_event_init(event);
4770
		break;
4771

4772 4773 4774 4775 4776
	case PERF_TYPE_BREAKPOINT:
		pmu = bp_perf_event_init(event);
		break;


4777 4778
	default:
		break;
4779
	}
4780 4781
done:
	err = 0;
4782
	if (!pmu)
4783
		err = -EINVAL;
4784 4785
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
4786

4787
	if (err) {
4788 4789 4790
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
4791
		return ERR_PTR(err);
I
Ingo Molnar 已提交
4792
	}
4793

4794
	event->pmu = pmu;
T
Thomas Gleixner 已提交
4795

4796 4797 4798 4799 4800 4801 4802 4803
	if (!event->parent) {
		atomic_inc(&nr_events);
		if (event->attr.mmap)
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
4804
	}
4805

4806
	return event;
T
Thomas Gleixner 已提交
4807 4808
}

4809 4810
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
4811 4812
{
	u32 size;
4813
	int ret;
4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
4838 4839 4840
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
4841 4842
	 */
	if (size > sizeof(*attr)) {
4843 4844 4845
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
4846

4847 4848
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
4849

4850
		for (; addr < end; addr++) {
4851 4852 4853 4854 4855 4856
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
4857
		size = sizeof(*attr);
4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

4871
	if (attr->__reserved_1)
4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Li Zefan 已提交
4889
static int perf_event_set_output(struct perf_event *event, int output_fd)
4890
{
4891
	struct perf_event *output_event = NULL;
4892
	struct file *output_file = NULL;
4893
	struct perf_event *old_output;
4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906
	int fput_needed = 0;
	int ret = -EINVAL;

	if (!output_fd)
		goto set;

	output_file = fget_light(output_fd, &fput_needed);
	if (!output_file)
		return -EBADF;

	if (output_file->f_op != &perf_fops)
		goto out;

4907
	output_event = output_file->private_data;
4908 4909

	/* Don't chain output fds */
4910
	if (output_event->output)
4911 4912 4913
		goto out;

	/* Don't set an output fd when we already have an output channel */
4914
	if (event->data)
4915 4916 4917 4918 4919
		goto out;

	atomic_long_inc(&output_file->f_count);

set:
4920 4921 4922 4923
	mutex_lock(&event->mmap_mutex);
	old_output = event->output;
	rcu_assign_pointer(event->output, output_event);
	mutex_unlock(&event->mmap_mutex);
4924 4925 4926 4927

	if (old_output) {
		/*
		 * we need to make sure no existing perf_output_*()
4928
		 * is still referencing this event.
4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939
		 */
		synchronize_rcu();
		fput(old_output->filp);
	}

	ret = 0;
out:
	fput_light(output_file, fput_needed);
	return ret;
}

T
Thomas Gleixner 已提交
4940
/**
4941
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
4942
 *
4943
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
4944
 * @pid:		target pid
I
Ingo Molnar 已提交
4945
 * @cpu:		target cpu
4946
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
4947
 */
4948 4949
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
4950
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
4951
{
4952 4953 4954 4955
	struct perf_event *event, *group_leader;
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
4956 4957
	struct file *group_file = NULL;
	int fput_needed = 0;
4958
	int fput_needed2 = 0;
4959
	int err;
T
Thomas Gleixner 已提交
4960

4961
	/* for future expandability... */
4962
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4963 4964
		return -EINVAL;

4965 4966 4967
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
4968

4969 4970 4971 4972 4973
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

4974
	if (attr.freq) {
4975
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
4976 4977 4978
			return -EINVAL;
	}

4979
	/*
I
Ingo Molnar 已提交
4980 4981 4982 4983 4984 4985 4986
	 * Get the target context (task or percpu):
	 */
	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

	/*
4987
	 * Look up the group leader (we will attach this event to it):
4988 4989
	 */
	group_leader = NULL;
4990
	if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4991
		err = -EINVAL;
4992 4993
		group_file = fget_light(group_fd, &fput_needed);
		if (!group_file)
I
Ingo Molnar 已提交
4994
			goto err_put_context;
4995
		if (group_file->f_op != &perf_fops)
I
Ingo Molnar 已提交
4996
			goto err_put_context;
4997 4998 4999

		group_leader = group_file->private_data;
		/*
I
Ingo Molnar 已提交
5000 5001 5002 5003 5004 5005 5006 5007
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
			goto err_put_context;
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
5008
		 */
I
Ingo Molnar 已提交
5009 5010
		if (group_leader->ctx != ctx)
			goto err_put_context;
5011 5012 5013
		/*
		 * Only a group leader can be exclusive or pinned
		 */
5014
		if (attr.exclusive || attr.pinned)
5015
			goto err_put_context;
5016 5017
	}

5018
	event = perf_event_alloc(&attr, cpu, ctx, group_leader,
5019
				     NULL, NULL, GFP_KERNEL);
5020 5021
	err = PTR_ERR(event);
	if (IS_ERR(event))
T
Thomas Gleixner 已提交
5022 5023
		goto err_put_context;

5024
	err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
5025
	if (err < 0)
5026 5027
		goto err_free_put_context;

5028 5029
	event_file = fget_light(err, &fput_needed2);
	if (!event_file)
5030 5031
		goto err_free_put_context;

5032
	if (flags & PERF_FLAG_FD_OUTPUT) {
5033
		err = perf_event_set_output(event, group_fd);
5034 5035
		if (err)
			goto err_fput_free_put_context;
5036 5037
	}

5038
	event->filp = event_file;
5039
	WARN_ON_ONCE(ctx->parent_ctx);
5040
	mutex_lock(&ctx->mutex);
5041
	perf_install_in_context(ctx, event, cpu);
5042
	++ctx->generation;
5043
	mutex_unlock(&ctx->mutex);
5044

5045
	event->owner = current;
5046
	get_task_struct(current);
5047 5048 5049
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
5050

5051
err_fput_free_put_context:
5052
	fput_light(event_file, fput_needed2);
T
Thomas Gleixner 已提交
5053

5054
err_free_put_context:
5055
	if (err < 0)
5056
		free_event(event);
T
Thomas Gleixner 已提交
5057 5058

err_put_context:
5059 5060 5061 5062
	if (err < 0)
		put_ctx(ctx);

	fput_light(group_file, fput_needed);
T
Thomas Gleixner 已提交
5063

5064
	return err;
T
Thomas Gleixner 已提交
5065 5066
}

5067 5068 5069 5070 5071 5072 5073 5074 5075
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
 * @pid: task to profile
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
5076 5077
				 pid_t pid,
				 perf_overflow_handler_t overflow_handler)
5078 5079 5080 5081 5082 5083 5084 5085 5086 5087
{
	struct perf_event *event;
	struct perf_event_context *ctx;
	int err;

	/*
	 * Get the target context (task or percpu):
	 */

	ctx = find_get_context(pid, cpu);
5088 5089 5090 5091
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
		goto err_exit;
	}
5092 5093

	event = perf_event_alloc(attr, cpu, ctx, NULL,
5094
				 NULL, overflow_handler, GFP_KERNEL);
5095 5096
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
5097
		goto err_put_context;
5098
	}
5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
	mutex_unlock(&ctx->mutex);

	event->owner = current;
	get_task_struct(current);
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);

	return event;

5115 5116 5117 5118
 err_put_context:
	put_ctx(ctx);
 err_exit:
	return ERR_PTR(err);
5119 5120 5121
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);

5122
/*
5123
 * inherit a event from parent task to child task:
5124
 */
5125 5126
static struct perf_event *
inherit_event(struct perf_event *parent_event,
5127
	      struct task_struct *parent,
5128
	      struct perf_event_context *parent_ctx,
5129
	      struct task_struct *child,
5130 5131
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
5132
{
5133
	struct perf_event *child_event;
5134

5135
	/*
5136 5137
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
5138 5139 5140
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
5141 5142
	if (parent_event->parent)
		parent_event = parent_event->parent;
5143

5144 5145 5146
	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu, child_ctx,
					   group_leader, parent_event,
5147
					   NULL, GFP_KERNEL);
5148 5149
	if (IS_ERR(child_event))
		return child_event;
5150
	get_ctx(child_ctx);
5151

5152
	/*
5153
	 * Make the child state follow the state of the parent event,
5154
	 * not its attr.disabled bit.  We hold the parent's mutex,
5155
	 * so we won't race with perf_event_{en, dis}able_family.
5156
	 */
5157 5158
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
5159
	else
5160
		child_event->state = PERF_EVENT_STATE_OFF;
5161

5162 5163 5164 5165 5166 5167 5168 5169 5170
	if (parent_event->attr.freq) {
		u64 sample_period = parent_event->hw.sample_period;
		struct hw_perf_event *hwc = &child_event->hw;

		hwc->sample_period = sample_period;
		hwc->last_period   = sample_period;

		atomic64_set(&hwc->period_left, sample_period);
	}
5171

5172 5173
	child_event->overflow_handler = parent_event->overflow_handler;

5174 5175 5176
	/*
	 * Link it up in the child's context:
	 */
5177
	add_event_to_ctx(child_event, child_ctx);
5178 5179 5180

	/*
	 * Get a reference to the parent filp - we will fput it
5181
	 * when the child event exits. This is safe to do because
5182 5183 5184
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
5185
	atomic_long_inc(&parent_event->filp->f_count);
5186

5187
	/*
5188
	 * Link this into the parent event's child list
5189
	 */
5190 5191 5192 5193
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
5194

5195
	return child_event;
5196 5197
}

5198
static int inherit_group(struct perf_event *parent_event,
5199
	      struct task_struct *parent,
5200
	      struct perf_event_context *parent_ctx,
5201
	      struct task_struct *child,
5202
	      struct perf_event_context *child_ctx)
5203
{
5204 5205 5206
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;
5207

5208
	leader = inherit_event(parent_event, parent, parent_ctx,
5209
				 child, NULL, child_ctx);
5210 5211
	if (IS_ERR(leader))
		return PTR_ERR(leader);
5212 5213
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
5214 5215 5216
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
5217
	}
5218 5219 5220
	return 0;
}

5221
static void sync_child_event(struct perf_event *child_event,
5222
			       struct task_struct *child)
5223
{
5224
	struct perf_event *parent_event = child_event->parent;
5225
	u64 child_val;
5226

5227 5228
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
5229

5230
	child_val = atomic64_read(&child_event->count);
5231 5232 5233 5234

	/*
	 * Add back the child's count to the parent's count:
	 */
5235 5236 5237 5238 5239
	atomic64_add(child_val, &parent_event->count);
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
5240 5241

	/*
5242
	 * Remove this event from the parent's list
5243
	 */
5244 5245 5246 5247
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
5248 5249

	/*
5250
	 * Release the parent event, if this was the last
5251 5252
	 * reference to it.
	 */
5253
	fput(parent_event->filp);
5254 5255
}

5256
static void
5257 5258
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
5259
			 struct task_struct *child)
5260
{
5261
	struct perf_event *parent_event;
5262

5263
	perf_event_remove_from_context(child_event);
5264

5265
	parent_event = child_event->parent;
5266
	/*
5267
	 * It can happen that parent exits first, and has events
5268
	 * that are still around due to the child reference. These
5269
	 * events need to be zapped - but otherwise linger.
5270
	 */
5271 5272 5273
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
5274
	}
5275 5276 5277
}

/*
5278
 * When a child task exits, feed back event values to parent events.
5279
 */
5280
void perf_event_exit_task(struct task_struct *child)
5281
{
5282 5283
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
5284
	unsigned long flags;
5285

5286 5287
	if (likely(!child->perf_event_ctxp)) {
		perf_event_task(child, NULL, 0);
5288
		return;
P
Peter Zijlstra 已提交
5289
	}
5290

5291
	local_irq_save(flags);
5292 5293 5294 5295 5296 5297
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
5298 5299
	child_ctx = child->perf_event_ctxp;
	__perf_event_task_sched_out(child_ctx);
5300 5301 5302

	/*
	 * Take the context lock here so that if find_get_context is
5303
	 * reading child->perf_event_ctxp, we wait until it has
5304 5305
	 * incremented the context's refcount before we do put_ctx below.
	 */
5306
	raw_spin_lock(&child_ctx->lock);
5307
	child->perf_event_ctxp = NULL;
5308 5309 5310
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
5311
	 * the events from it.
5312 5313
	 */
	unclone_ctx(child_ctx);
5314
	update_context_time(child_ctx);
5315
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
5316 5317

	/*
5318 5319 5320
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
5321
	 */
5322
	perf_event_task(child, child_ctx, 0);
5323

5324 5325 5326
	/*
	 * We can recurse on the same lock type through:
	 *
5327 5328 5329
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
5330 5331 5332 5333 5334
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
5335
	mutex_lock(&child_ctx->mutex);
5336

5337
again:
5338 5339 5340 5341 5342
	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
				 group_entry)
		__perf_event_exit_task(child_event, child_ctx, child);

	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
5343
				 group_entry)
5344
		__perf_event_exit_task(child_event, child_ctx, child);
5345 5346

	/*
5347
	 * If the last event was a group event, it will have appended all
5348 5349 5350
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
5351 5352
	if (!list_empty(&child_ctx->pinned_groups) ||
	    !list_empty(&child_ctx->flexible_groups))
5353
		goto again;
5354 5355 5356 5357

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
5358 5359
}

5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377
static void perf_free_event(struct perf_event *event,
			    struct perf_event_context *ctx)
{
	struct perf_event *parent = event->parent;

	if (WARN_ON_ONCE(!parent))
		return;

	mutex_lock(&parent->child_mutex);
	list_del_init(&event->child_list);
	mutex_unlock(&parent->child_mutex);

	fput(parent->filp);

	list_del_event(event, ctx);
	free_event(event);
}

5378 5379 5380 5381
/*
 * free an unexposed, unused context as created by inheritance by
 * init_task below, used by fork() in case of fail.
 */
5382
void perf_event_free_task(struct task_struct *task)
5383
{
5384 5385
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event *event, *tmp;
5386 5387 5388 5389 5390 5391

	if (!ctx)
		return;

	mutex_lock(&ctx->mutex);
again:
5392 5393
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		perf_free_event(event, ctx);
5394

5395 5396 5397
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
				 group_entry)
		perf_free_event(event, ctx);
5398

5399 5400 5401
	if (!list_empty(&ctx->pinned_groups) ||
	    !list_empty(&ctx->flexible_groups))
		goto again;
5402

5403
	mutex_unlock(&ctx->mutex);
5404

5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419
	put_ctx(ctx);
}

static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
		   struct perf_event_context *parent_ctx,
		   struct task_struct *child,
		   int *inherited_all)
{
	int ret;
	struct perf_event_context *child_ctx = child->perf_event_ctxp;

	if (!event->attr.inherit) {
		*inherited_all = 0;
		return 0;
5420 5421
	}

5422 5423 5424 5425 5426 5427 5428
	if (!child_ctx) {
		/*
		 * This is executed from the parent task context, so
		 * inherit events that have been marked for cloning.
		 * First allocate and initialize a context for the
		 * child.
		 */
5429

5430 5431 5432 5433
		child_ctx = kzalloc(sizeof(struct perf_event_context),
				    GFP_KERNEL);
		if (!child_ctx)
			return -ENOMEM;
5434

5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446
		__perf_event_init_context(child_ctx, child);
		child->perf_event_ctxp = child_ctx;
		get_task_struct(child);
	}

	ret = inherit_group(event, parent, parent_ctx,
			    child, child_ctx);

	if (ret)
		*inherited_all = 0;

	return ret;
5447 5448
}

5449

5450
/*
5451
 * Initialize the perf_event context in task_struct
5452
 */
5453
int perf_event_init_task(struct task_struct *child)
5454
{
5455
	struct perf_event_context *child_ctx, *parent_ctx;
5456 5457
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
5458
	struct task_struct *parent = current;
5459
	int inherited_all = 1;
5460
	int ret = 0;
5461

5462
	child->perf_event_ctxp = NULL;
5463

5464 5465
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);
5466

5467
	if (likely(!parent->perf_event_ctxp))
5468 5469
		return 0;

5470
	/*
5471 5472
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
5473
	 */
5474 5475
	parent_ctx = perf_pin_task_context(parent);

5476 5477 5478 5479 5480 5481 5482
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

5483 5484 5485 5486
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
5487
	mutex_lock(&parent_ctx->mutex);
5488 5489 5490 5491 5492

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
5493 5494 5495 5496 5497 5498
	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
			break;
	}
5499

5500 5501 5502 5503
	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
		ret = inherit_task_group(event, parent, parent_ctx, child,
					 &inherited_all);
		if (ret)
5504
			break;
5505 5506
	}

5507 5508
	child_ctx = child->perf_event_ctxp;

5509
	if (child_ctx && inherited_all) {
5510 5511 5512
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
5513 5514
		 * Note that if the parent is a clone, it could get
		 * uncloned at any point, but that doesn't matter
5515
		 * because the list of events and the generation
5516
		 * count can't have changed since we took the mutex.
5517
		 */
5518 5519 5520
		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
5521
			child_ctx->parent_gen = parent_ctx->parent_gen;
5522 5523 5524 5525 5526
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
5527 5528
	}

5529
	mutex_unlock(&parent_ctx->mutex);
5530

5531
	perf_unpin_context(parent_ctx);
5532

5533
	return ret;
5534 5535
}

5536 5537 5538 5539 5540 5541 5542
static void __init perf_event_init_all_cpus(void)
{
	int cpu;
	struct perf_cpu_context *cpuctx;

	for_each_possible_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
5543
		mutex_init(&cpuctx->hlist_mutex);
5544 5545 5546 5547
		__perf_event_init_context(&cpuctx->ctx, NULL);
	}
}

5548
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
5549
{
5550
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
5551

5552
	cpuctx = &per_cpu(perf_cpu_context, cpu);
T
Thomas Gleixner 已提交
5553

5554
	spin_lock(&perf_resource_lock);
5555
	cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5556
	spin_unlock(&perf_resource_lock);
5557 5558 5559 5560 5561 5562 5563 5564 5565 5566

	mutex_lock(&cpuctx->hlist_mutex);
	if (cpuctx->hlist_refcount > 0) {
		struct swevent_hlist *hlist;

		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
		WARN_ON_ONCE(!hlist);
		rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
	}
	mutex_unlock(&cpuctx->hlist_mutex);
T
Thomas Gleixner 已提交
5567 5568 5569
}

#ifdef CONFIG_HOTPLUG_CPU
5570
static void __perf_event_exit_cpu(void *info)
T
Thomas Gleixner 已提交
5571 5572
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5573 5574
	struct perf_event_context *ctx = &cpuctx->ctx;
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
5575

5576 5577 5578
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
		__perf_event_remove_from_context(event);
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
5579
		__perf_event_remove_from_context(event);
T
Thomas Gleixner 已提交
5580
}
5581
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
5582
{
5583
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5584
	struct perf_event_context *ctx = &cpuctx->ctx;
5585

5586 5587 5588 5589
	mutex_lock(&cpuctx->hlist_mutex);
	swevent_hlist_release(cpuctx);
	mutex_unlock(&cpuctx->hlist_mutex);

5590
	mutex_lock(&ctx->mutex);
5591
	smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5592
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
5593 5594
}
#else
5595
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606
#endif

static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action) {

	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5607
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
5608 5609 5610 5611
		break;

	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
5612
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
5613 5614 5615 5616 5617 5618 5619 5620 5621
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

5622 5623 5624
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
T
Thomas Gleixner 已提交
5625 5626
static struct notifier_block __cpuinitdata perf_cpu_nb = {
	.notifier_call		= perf_cpu_notify,
5627
	.priority		= 20,
T
Thomas Gleixner 已提交
5628 5629
};

5630
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
5631
{
5632
	perf_event_init_all_cpus();
T
Thomas Gleixner 已提交
5633 5634
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
			(void *)(long)smp_processor_id());
5635 5636
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
			(void *)(long)smp_processor_id());
T
Thomas Gleixner 已提交
5637 5638 5639
	register_cpu_notifier(&perf_cpu_nb);
}

5640 5641 5642
static ssize_t perf_show_reserve_percpu(struct sysdev_class *class,
					struct sysdev_class_attribute *attr,
					char *buf)
T
Thomas Gleixner 已提交
5643 5644 5645 5646 5647 5648
{
	return sprintf(buf, "%d\n", perf_reserved_percpu);
}

static ssize_t
perf_set_reserve_percpu(struct sysdev_class *class,
5649
			struct sysdev_class_attribute *attr,
T
Thomas Gleixner 已提交
5650 5651 5652 5653 5654 5655 5656 5657 5658 5659
			const char *buf,
			size_t count)
{
	struct perf_cpu_context *cpuctx;
	unsigned long val;
	int err, cpu, mpt;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
5660
	if (val > perf_max_events)
T
Thomas Gleixner 已提交
5661 5662
		return -EINVAL;

5663
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5664 5665 5666
	perf_reserved_percpu = val;
	for_each_online_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
5667
		raw_spin_lock_irq(&cpuctx->ctx.lock);
5668 5669
		mpt = min(perf_max_events - cpuctx->ctx.nr_events,
			  perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
5670
		cpuctx->max_pertask = mpt;
5671
		raw_spin_unlock_irq(&cpuctx->ctx.lock);
T
Thomas Gleixner 已提交
5672
	}
5673
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5674 5675 5676 5677

	return count;
}

5678 5679 5680
static ssize_t perf_show_overcommit(struct sysdev_class *class,
				    struct sysdev_class_attribute *attr,
				    char *buf)
T
Thomas Gleixner 已提交
5681 5682 5683 5684 5685
{
	return sprintf(buf, "%d\n", perf_overcommit);
}

static ssize_t
5686 5687 5688
perf_set_overcommit(struct sysdev_class *class,
		    struct sysdev_class_attribute *attr,
		    const char *buf, size_t count)
T
Thomas Gleixner 已提交
5689 5690 5691 5692 5693 5694 5695 5696 5697 5698
{
	unsigned long val;
	int err;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
	if (val > 1)
		return -EINVAL;

5699
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5700
	perf_overcommit = val;
5701
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727

	return count;
}

static SYSDEV_CLASS_ATTR(
				reserve_percpu,
				0644,
				perf_show_reserve_percpu,
				perf_set_reserve_percpu
			);

static SYSDEV_CLASS_ATTR(
				overcommit,
				0644,
				perf_show_overcommit,
				perf_set_overcommit
			);

static struct attribute *perfclass_attrs[] = {
	&attr_reserve_percpu.attr,
	&attr_overcommit.attr,
	NULL
};

static struct attribute_group perfclass_attr_group = {
	.attrs			= perfclass_attrs,
5728
	.name			= "perf_events",
T
Thomas Gleixner 已提交
5729 5730
};

5731
static int __init perf_event_sysfs_init(void)
T
Thomas Gleixner 已提交
5732 5733 5734 5735
{
	return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
				  &perfclass_attr_group);
}
5736
device_initcall(perf_event_sysfs_init);