perf_event.c 120.7 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
16
#include <linux/file.h>
T
Thomas Gleixner 已提交
17 18
#include <linux/poll.h>
#include <linux/sysfs.h>
19
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
20
#include <linux/percpu.h>
21
#include <linux/ptrace.h>
22
#include <linux/vmstat.h>
23
#include <linux/vmalloc.h>
24 25
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
26 27 28
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
29
#include <linux/kernel_stat.h>
30
#include <linux/perf_event.h>
L
Li Zefan 已提交
31
#include <linux/ftrace_event.h>
32
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
33

34 35
#include <asm/irq_regs.h>

T
Thomas Gleixner 已提交
36
/*
37
 * Each CPU has a list of per CPU events:
T
Thomas Gleixner 已提交
38 39 40
 */
DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);

41
int perf_max_events __read_mostly = 1;
T
Thomas Gleixner 已提交
42 43 44
static int perf_reserved_percpu __read_mostly;
static int perf_overcommit __read_mostly = 1;

45 46 47 48
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
49

50
/*
51
 * perf event paranoia level:
52 53
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
54
 *   1 - disallow cpu events for unpriv
55
 *   2 - disallow kernel profiling for unpriv
56
 */
57
int sysctl_perf_event_paranoid __read_mostly = 1;
58

59 60
static inline bool perf_paranoid_tracepoint_raw(void)
{
61
	return sysctl_perf_event_paranoid > -1;
62 63
}

64 65
static inline bool perf_paranoid_cpu(void)
{
66
	return sysctl_perf_event_paranoid > 0;
67 68 69 70
}

static inline bool perf_paranoid_kernel(void)
{
71
	return sysctl_perf_event_paranoid > 1;
72 73
}

74
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
75 76

/*
77
 * max perf event sample rate
78
 */
79
int sysctl_perf_event_sample_rate __read_mostly = 100000;
80

81
static atomic64_t perf_event_id;
82

T
Thomas Gleixner 已提交
83
/*
84
 * Lock for (sysadmin-configurable) event reservations:
T
Thomas Gleixner 已提交
85
 */
86
static DEFINE_SPINLOCK(perf_resource_lock);
T
Thomas Gleixner 已提交
87 88 89 90

/*
 * Architecture provided APIs - weak aliases:
 */
91
extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
T
Thomas Gleixner 已提交
92
{
93
	return NULL;
T
Thomas Gleixner 已提交
94 95
}

96 97 98
void __weak hw_perf_disable(void)		{ barrier(); }
void __weak hw_perf_enable(void)		{ barrier(); }

99 100
void __weak hw_perf_event_setup(int cpu)	{ barrier(); }
void __weak hw_perf_event_setup_online(int cpu)	{ barrier(); }
101 102

int __weak
103
hw_perf_group_sched_in(struct perf_event *group_leader,
104
	       struct perf_cpu_context *cpuctx,
105
	       struct perf_event_context *ctx, int cpu)
106 107 108
{
	return 0;
}
T
Thomas Gleixner 已提交
109

110
void __weak perf_event_print_debug(void)	{ }
111

112
static DEFINE_PER_CPU(int, perf_disable_count);
113 114 115

void __perf_disable(void)
{
116
	__get_cpu_var(perf_disable_count)++;
117 118 119 120
}

bool __perf_enable(void)
{
121
	return !--__get_cpu_var(perf_disable_count);
122 123 124 125 126 127 128 129 130 131 132 133 134 135
}

void perf_disable(void)
{
	__perf_disable();
	hw_perf_disable();
}

void perf_enable(void)
{
	if (__perf_enable())
		hw_perf_enable();
}

136
static void get_ctx(struct perf_event_context *ctx)
137
{
138
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
139 140
}

141 142
static void free_ctx(struct rcu_head *head)
{
143
	struct perf_event_context *ctx;
144

145
	ctx = container_of(head, struct perf_event_context, rcu_head);
146 147 148
	kfree(ctx);
}

149
static void put_ctx(struct perf_event_context *ctx)
150
{
151 152 153
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
154 155 156
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
157
	}
158 159
}

160
static void unclone_ctx(struct perf_event_context *ctx)
161 162 163 164 165 166 167
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

168
/*
169
 * If we inherit events we want to return the parent event id
170 171
 * to userspace.
 */
172
static u64 primary_event_id(struct perf_event *event)
173
{
174
	u64 id = event->id;
175

176 177
	if (event->parent)
		id = event->parent->id;
178 179 180 181

	return id;
}

182
/*
183
 * Get the perf_event_context for a task and lock it.
184 185 186
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
187
static struct perf_event_context *
188
perf_lock_task_context(struct task_struct *task, unsigned long *flags)
189
{
190
	struct perf_event_context *ctx;
191 192 193

	rcu_read_lock();
 retry:
194
	ctx = rcu_dereference(task->perf_event_ctxp);
195 196 197 198
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
199
		 * perf_event_task_sched_out, though the
200 201 202 203 204 205 206
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
		spin_lock_irqsave(&ctx->lock, *flags);
207
		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
208 209 210
			spin_unlock_irqrestore(&ctx->lock, *flags);
			goto retry;
		}
211 212 213 214 215

		if (!atomic_inc_not_zero(&ctx->refcount)) {
			spin_unlock_irqrestore(&ctx->lock, *flags);
			ctx = NULL;
		}
216 217 218 219 220 221 222 223 224 225
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
226
static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
227
{
228
	struct perf_event_context *ctx;
229 230 231 232 233 234 235 236 237 238
	unsigned long flags;

	ctx = perf_lock_task_context(task, &flags);
	if (ctx) {
		++ctx->pin_count;
		spin_unlock_irqrestore(&ctx->lock, flags);
	}
	return ctx;
}

239
static void perf_unpin_context(struct perf_event_context *ctx)
240 241 242 243 244 245 246 247 248
{
	unsigned long flags;

	spin_lock_irqsave(&ctx->lock, flags);
	--ctx->pin_count;
	spin_unlock_irqrestore(&ctx->lock, flags);
	put_ctx(ctx);
}

249
/*
250
 * Add a event from the lists for its context.
251 252
 * Must be called with ctx->mutex and ctx->lock held.
 */
253
static void
254
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
255
{
256
	struct perf_event *group_leader = event->group_leader;
257 258

	/*
259 260
	 * Depending on whether it is a standalone or sibling event,
	 * add it straight to the context's event list, or to the group
261 262
	 * leader's sibling list:
	 */
263 264
	if (group_leader == event)
		list_add_tail(&event->group_entry, &ctx->group_list);
P
Peter Zijlstra 已提交
265
	else {
266
		list_add_tail(&event->group_entry, &group_leader->sibling_list);
P
Peter Zijlstra 已提交
267 268
		group_leader->nr_siblings++;
	}
P
Peter Zijlstra 已提交
269

270 271 272
	list_add_rcu(&event->event_entry, &ctx->event_list);
	ctx->nr_events++;
	if (event->attr.inherit_stat)
273
		ctx->nr_stat++;
274 275
}

276
/*
277
 * Remove a event from the lists for its context.
278
 * Must be called with ctx->mutex and ctx->lock held.
279
 */
280
static void
281
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
282
{
283
	struct perf_event *sibling, *tmp;
284

285
	if (list_empty(&event->group_entry))
286
		return;
287 288
	ctx->nr_events--;
	if (event->attr.inherit_stat)
289
		ctx->nr_stat--;
290

291 292
	list_del_init(&event->group_entry);
	list_del_rcu(&event->event_entry);
293

294 295
	if (event->group_leader != event)
		event->group_leader->nr_siblings--;
P
Peter Zijlstra 已提交
296

297 298
	event->state = PERF_EVENT_STATE_OFF;

299
	/*
300 301
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
302 303
	 * to the context list directly:
	 */
304
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
305

306
		list_move_tail(&sibling->group_entry, &ctx->group_list);
307 308 309 310
		sibling->group_leader = sibling;
	}
}

311
static void
312
event_sched_out(struct perf_event *event,
313
		  struct perf_cpu_context *cpuctx,
314
		  struct perf_event_context *ctx)
315
{
316
	if (event->state != PERF_EVENT_STATE_ACTIVE)
317 318
		return;

319 320 321 322
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
323
	}
324 325 326
	event->tstamp_stopped = ctx->time;
	event->pmu->disable(event);
	event->oncpu = -1;
327

328
	if (!is_software_event(event))
329 330
		cpuctx->active_oncpu--;
	ctx->nr_active--;
331
	if (event->attr.exclusive || !cpuctx->active_oncpu)
332 333 334
		cpuctx->exclusive = 0;
}

335
static void
336
group_sched_out(struct perf_event *group_event,
337
		struct perf_cpu_context *cpuctx,
338
		struct perf_event_context *ctx)
339
{
340
	struct perf_event *event;
341

342
	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
343 344
		return;

345
	event_sched_out(group_event, cpuctx, ctx);
346 347 348 349

	/*
	 * Schedule out siblings (if any):
	 */
350 351
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
352

353
	if (group_event->attr.exclusive)
354 355 356
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
357
/*
358
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
359
 *
360
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
361 362
 * remove it from the context list.
 */
363
static void __perf_event_remove_from_context(void *info)
T
Thomas Gleixner 已提交
364 365
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
366 367
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
368 369 370 371 372 373

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
	 */
374
	if (ctx->task && cpuctx->task_ctx != ctx)
T
Thomas Gleixner 已提交
375 376
		return;

377
	spin_lock(&ctx->lock);
378 379
	/*
	 * Protect the list operation against NMI by disabling the
380
	 * events on a global level.
381 382
	 */
	perf_disable();
T
Thomas Gleixner 已提交
383

384
	event_sched_out(event, cpuctx, ctx);
385

386
	list_del_event(event, ctx);
T
Thomas Gleixner 已提交
387 388 389

	if (!ctx->task) {
		/*
390
		 * Allow more per task events with respect to the
T
Thomas Gleixner 已提交
391 392 393
		 * reservation:
		 */
		cpuctx->max_pertask =
394 395
			min(perf_max_events - ctx->nr_events,
			    perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
396 397
	}

398
	perf_enable();
399
	spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
400 401 402 403
}


/*
404
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
405
 *
406
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
407
 *
408
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
409
 * call when the task is on a CPU.
410
 *
411 412
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
413 414
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
415
 * When called from perf_event_exit_task, it's OK because the
416
 * context has been detached from its task.
T
Thomas Gleixner 已提交
417
 */
418
static void perf_event_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
419
{
420
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
421 422 423 424
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
425
		 * Per cpu events are removed via an smp call and
T
Thomas Gleixner 已提交
426 427
		 * the removal is always sucessful.
		 */
428 429 430
		smp_call_function_single(event->cpu,
					 __perf_event_remove_from_context,
					 event, 1);
T
Thomas Gleixner 已提交
431 432 433 434
		return;
	}

retry:
435 436
	task_oncpu_function_call(task, __perf_event_remove_from_context,
				 event);
T
Thomas Gleixner 已提交
437 438 439 440 441

	spin_lock_irq(&ctx->lock);
	/*
	 * If the context is active we need to retry the smp call.
	 */
442
	if (ctx->nr_active && !list_empty(&event->group_entry)) {
T
Thomas Gleixner 已提交
443 444 445 446 447 448
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
449
	 * can remove the event safely, if the call above did not
T
Thomas Gleixner 已提交
450 451
	 * succeed.
	 */
P
Peter Zijlstra 已提交
452
	if (!list_empty(&event->group_entry))
453
		list_del_event(event, ctx);
T
Thomas Gleixner 已提交
454 455 456
	spin_unlock_irq(&ctx->lock);
}

457
static inline u64 perf_clock(void)
458
{
459
	return cpu_clock(smp_processor_id());
460 461 462 463 464
}

/*
 * Update the record of the current time in a context.
 */
465
static void update_context_time(struct perf_event_context *ctx)
466
{
467 468 469 470
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
471 472 473
}

/*
474
 * Update the total_time_enabled and total_time_running fields for a event.
475
 */
476
static void update_event_times(struct perf_event *event)
477
{
478
	struct perf_event_context *ctx = event->ctx;
479 480
	u64 run_end;

481 482
	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
483 484
		return;

485
	event->total_time_enabled = ctx->time - event->tstamp_enabled;
486

487 488
	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
489 490 491
	else
		run_end = ctx->time;

492
	event->total_time_running = run_end - event->tstamp_running;
493 494 495
}

/*
496
 * Update total_time_enabled and total_time_running for all events in a group.
497
 */
498
static void update_group_times(struct perf_event *leader)
499
{
500
	struct perf_event *event;
501

502 503 504
	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
505 506
}

507
/*
508
 * Cross CPU call to disable a performance event
509
 */
510
static void __perf_event_disable(void *info)
511
{
512
	struct perf_event *event = info;
513
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
514
	struct perf_event_context *ctx = event->ctx;
515 516

	/*
517 518
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
519
	 */
520
	if (ctx->task && cpuctx->task_ctx != ctx)
521 522
		return;

523
	spin_lock(&ctx->lock);
524 525

	/*
526
	 * If the event is on, turn it off.
527 528
	 * If it is in error state, leave it in error state.
	 */
529
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
530
		update_context_time(ctx);
531 532 533
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
534
		else
535 536
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
537 538
	}

539
	spin_unlock(&ctx->lock);
540 541 542
}

/*
543
 * Disable a event.
544
 *
545 546
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
547
 * remains valid.  This condition is satisifed when called through
548 549 550 551
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
552
 * is the current context on this CPU and preemption is disabled,
553
 * hence we can't get into perf_event_task_sched_out for this context.
554
 */
555
static void perf_event_disable(struct perf_event *event)
556
{
557
	struct perf_event_context *ctx = event->ctx;
558 559 560 561
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
562
		 * Disable the event on the cpu that it's on
563
		 */
564 565
		smp_call_function_single(event->cpu, __perf_event_disable,
					 event, 1);
566 567 568 569
		return;
	}

 retry:
570
	task_oncpu_function_call(task, __perf_event_disable, event);
571 572 573

	spin_lock_irq(&ctx->lock);
	/*
574
	 * If the event is still active, we need to retry the cross-call.
575
	 */
576
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
577 578 579 580 581 582 583 584
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
585 586 587
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
588
	}
589 590 591 592

	spin_unlock_irq(&ctx->lock);
}

593
static int
594
event_sched_in(struct perf_event *event,
595
		 struct perf_cpu_context *cpuctx,
596
		 struct perf_event_context *ctx,
597 598
		 int cpu)
{
599
	if (event->state <= PERF_EVENT_STATE_OFF)
600 601
		return 0;

602 603
	event->state = PERF_EVENT_STATE_ACTIVE;
	event->oncpu = cpu;	/* TODO: put 'cpu' into cpuctx->cpu */
604 605 606 607 608
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

609 610 611
	if (event->pmu->enable(event)) {
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
612 613 614
		return -EAGAIN;
	}

615
	event->tstamp_running += ctx->time - event->tstamp_stopped;
616

617
	if (!is_software_event(event))
618
		cpuctx->active_oncpu++;
619 620
	ctx->nr_active++;

621
	if (event->attr.exclusive)
622 623
		cpuctx->exclusive = 1;

624 625 626
	return 0;
}

627
static int
628
group_sched_in(struct perf_event *group_event,
629
	       struct perf_cpu_context *cpuctx,
630
	       struct perf_event_context *ctx,
631 632
	       int cpu)
{
633
	struct perf_event *event, *partial_group;
634 635
	int ret;

636
	if (group_event->state == PERF_EVENT_STATE_OFF)
637 638
		return 0;

639
	ret = hw_perf_group_sched_in(group_event, cpuctx, ctx, cpu);
640 641 642
	if (ret)
		return ret < 0 ? ret : 0;

643
	if (event_sched_in(group_event, cpuctx, ctx, cpu))
644 645 646 647 648
		return -EAGAIN;

	/*
	 * Schedule in siblings as one group (if any):
	 */
649 650 651
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event_sched_in(event, cpuctx, ctx, cpu)) {
			partial_group = event;
652 653 654 655 656 657 658 659 660 661 662
			goto group_error;
		}
	}

	return 0;

group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
	 */
663 664
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
665
			break;
666
		event_sched_out(event, cpuctx, ctx);
667
	}
668
	event_sched_out(group_event, cpuctx, ctx);
669 670 671 672

	return -EAGAIN;
}

673
/*
674 675
 * Return 1 for a group consisting entirely of software events,
 * 0 if the group contains any hardware events.
676
 */
677
static int is_software_only_group(struct perf_event *leader)
678
{
679
	struct perf_event *event;
680

681
	if (!is_software_event(leader))
682
		return 0;
P
Peter Zijlstra 已提交
683

684 685
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		if (!is_software_event(event))
686
			return 0;
P
Peter Zijlstra 已提交
687

688 689 690 691
	return 1;
}

/*
692
 * Work out whether we can put this event group on the CPU now.
693
 */
694
static int group_can_go_on(struct perf_event *event,
695 696 697 698
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
699
	 * Groups consisting entirely of software events can always go on.
700
	 */
701
	if (is_software_only_group(event))
702 703 704
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
705
	 * events can go on.
706 707 708 709 710
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
711
	 * events on the CPU, it can't go on.
712
	 */
713
	if (event->attr.exclusive && cpuctx->active_oncpu)
714 715 716 717 718 719 720 721
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

722 723
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
724
{
725 726 727 728
	list_add_event(event, ctx);
	event->tstamp_enabled = ctx->time;
	event->tstamp_running = ctx->time;
	event->tstamp_stopped = ctx->time;
729 730
}

T
Thomas Gleixner 已提交
731
/*
732
 * Cross CPU call to install and enable a performance event
733 734
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
735 736 737 738
 */
static void __perf_install_in_context(void *info)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
739 740 741
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
T
Thomas Gleixner 已提交
742
	int cpu = smp_processor_id();
743
	int err;
T
Thomas Gleixner 已提交
744 745 746 747 748

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
749
	 * Or possibly this is the right context but it isn't
750
	 * on this cpu because it had no events.
T
Thomas Gleixner 已提交
751
	 */
752
	if (ctx->task && cpuctx->task_ctx != ctx) {
753
		if (cpuctx->task_ctx || ctx->task != current)
754 755 756
			return;
		cpuctx->task_ctx = ctx;
	}
T
Thomas Gleixner 已提交
757

758
	spin_lock(&ctx->lock);
759
	ctx->is_active = 1;
760
	update_context_time(ctx);
T
Thomas Gleixner 已提交
761 762 763

	/*
	 * Protect the list operation against NMI by disabling the
764
	 * events on a global level. NOP for non NMI based events.
T
Thomas Gleixner 已提交
765
	 */
766
	perf_disable();
T
Thomas Gleixner 已提交
767

768
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
769

770
	/*
771
	 * Don't put the event on if it is disabled or if
772 773
	 * it is in a group and the group isn't on.
	 */
774 775
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
776 777
		goto unlock;

778
	/*
779 780 781
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
782
	 */
783
	if (!group_can_go_on(event, cpuctx, 1))
784 785
		err = -EEXIST;
	else
786
		err = event_sched_in(event, cpuctx, ctx, cpu);
787

788 789
	if (err) {
		/*
790
		 * This event couldn't go on.  If it is in a group
791
		 * then we have to pull the whole group off.
792
		 * If the event group is pinned then put it in error state.
793
		 */
794
		if (leader != event)
795
			group_sched_out(leader, cpuctx, ctx);
796
		if (leader->attr.pinned) {
797
			update_group_times(leader);
798
			leader->state = PERF_EVENT_STATE_ERROR;
799
		}
800
	}
T
Thomas Gleixner 已提交
801

802
	if (!err && !ctx->task && cpuctx->max_pertask)
T
Thomas Gleixner 已提交
803 804
		cpuctx->max_pertask--;

805
 unlock:
806
	perf_enable();
807

808
	spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
809 810 811
}

/*
812
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
813
 *
814 815
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
816
 *
817
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
818 819
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
820 821
 *
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
822 823
 */
static void
824 825
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
826 827 828 829 830 831
			int cpu)
{
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
832
		 * Per cpu events are installed via an smp call and
T
Thomas Gleixner 已提交
833 834 835
		 * the install is always sucessful.
		 */
		smp_call_function_single(cpu, __perf_install_in_context,
836
					 event, 1);
T
Thomas Gleixner 已提交
837 838 839 840 841
		return;
	}

retry:
	task_oncpu_function_call(task, __perf_install_in_context,
842
				 event);
T
Thomas Gleixner 已提交
843 844 845 846 847

	spin_lock_irq(&ctx->lock);
	/*
	 * we need to retry the smp call.
	 */
848
	if (ctx->is_active && list_empty(&event->group_entry)) {
T
Thomas Gleixner 已提交
849 850 851 852 853 854
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
855
	 * can add the event safely, if it the call above did not
T
Thomas Gleixner 已提交
856 857
	 * succeed.
	 */
858 859
	if (list_empty(&event->group_entry))
		add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
860 861 862
	spin_unlock_irq(&ctx->lock);
}

863
/*
864
 * Put a event into inactive state and update time fields.
865 866 867 868 869 870
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
871 872
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
873
{
874
	struct perf_event *sub;
875

876 877 878 879
	event->state = PERF_EVENT_STATE_INACTIVE;
	event->tstamp_enabled = ctx->time - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry)
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
880 881 882 883
			sub->tstamp_enabled =
				ctx->time - sub->total_time_enabled;
}

884
/*
885
 * Cross CPU call to enable a performance event
886
 */
887
static void __perf_event_enable(void *info)
888
{
889
	struct perf_event *event = info;
890
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
891 892
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
893
	int err;
894

895
	/*
896 897
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
898
	 */
899
	if (ctx->task && cpuctx->task_ctx != ctx) {
900
		if (cpuctx->task_ctx || ctx->task != current)
901 902 903
			return;
		cpuctx->task_ctx = ctx;
	}
904

905
	spin_lock(&ctx->lock);
906
	ctx->is_active = 1;
907
	update_context_time(ctx);
908

909
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
910
		goto unlock;
911
	__perf_event_mark_enabled(event, ctx);
912 913

	/*
914
	 * If the event is in a group and isn't the group leader,
915
	 * then don't put it on unless the group is on.
916
	 */
917
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
918
		goto unlock;
919

920
	if (!group_can_go_on(event, cpuctx, 1)) {
921
		err = -EEXIST;
922
	} else {
923
		perf_disable();
924 925
		if (event == leader)
			err = group_sched_in(event, cpuctx, ctx,
926 927
					     smp_processor_id());
		else
928
			err = event_sched_in(event, cpuctx, ctx,
929
					       smp_processor_id());
930
		perf_enable();
931
	}
932 933 934

	if (err) {
		/*
935
		 * If this event can't go on and it's part of a
936 937
		 * group, then the whole group has to come off.
		 */
938
		if (leader != event)
939
			group_sched_out(leader, cpuctx, ctx);
940
		if (leader->attr.pinned) {
941
			update_group_times(leader);
942
			leader->state = PERF_EVENT_STATE_ERROR;
943
		}
944 945 946
	}

 unlock:
947
	spin_unlock(&ctx->lock);
948 949 950
}

/*
951
 * Enable a event.
952
 *
953 954
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
955
 * remains valid.  This condition is satisfied when called through
956 957
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
958
 */
959
static void perf_event_enable(struct perf_event *event)
960
{
961
	struct perf_event_context *ctx = event->ctx;
962 963 964 965
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
966
		 * Enable the event on the cpu that it's on
967
		 */
968 969
		smp_call_function_single(event->cpu, __perf_event_enable,
					 event, 1);
970 971 972 973
		return;
	}

	spin_lock_irq(&ctx->lock);
974
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
975 976 977
		goto out;

	/*
978 979
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
980 981 982 983
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
984 985
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
986 987 988

 retry:
	spin_unlock_irq(&ctx->lock);
989
	task_oncpu_function_call(task, __perf_event_enable, event);
990 991 992 993

	spin_lock_irq(&ctx->lock);

	/*
994
	 * If the context is active and the event is still off,
995 996
	 * we need to retry the cross-call.
	 */
997
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
998 999 1000 1001 1002 1003
		goto retry;

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1004 1005
	if (event->state == PERF_EVENT_STATE_OFF)
		__perf_event_mark_enabled(event, ctx);
1006

1007 1008 1009 1010
 out:
	spin_unlock_irq(&ctx->lock);
}

1011
static int perf_event_refresh(struct perf_event *event, int refresh)
1012
{
1013
	/*
1014
	 * not supported on inherited events
1015
	 */
1016
	if (event->attr.inherit)
1017 1018
		return -EINVAL;

1019 1020
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1021 1022

	return 0;
1023 1024
}

1025
void __perf_event_sched_out(struct perf_event_context *ctx,
1026 1027
			      struct perf_cpu_context *cpuctx)
{
1028
	struct perf_event *event;
1029

1030 1031
	spin_lock(&ctx->lock);
	ctx->is_active = 0;
1032
	if (likely(!ctx->nr_events))
1033
		goto out;
1034
	update_context_time(ctx);
1035

1036
	perf_disable();
P
Peter Zijlstra 已提交
1037
	if (ctx->nr_active) {
1038 1039
		list_for_each_entry(event, &ctx->group_list, group_entry)
			group_sched_out(event, cpuctx, ctx);
P
Peter Zijlstra 已提交
1040
	}
1041
	perf_enable();
1042
 out:
1043 1044 1045
	spin_unlock(&ctx->lock);
}

1046 1047 1048
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1049 1050 1051 1052
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1053
 * in them directly with an fd; we can only enable/disable all
1054
 * events via prctl, or enable/disable all events in a family
1055 1056
 * via ioctl, which will have the same effect on both contexts.
 */
1057 1058
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1059 1060
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1061
		&& ctx1->parent_gen == ctx2->parent_gen
1062
		&& !ctx1->pin_count && !ctx2->pin_count;
1063 1064
}

1065 1066
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1067 1068 1069
{
	u64 value;

1070
	if (!event->attr.inherit_stat)
1071 1072 1073
		return;

	/*
1074
	 * Update the event value, we cannot use perf_event_read()
1075 1076
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1077
	 * we know the event must be on the current CPU, therefore we
1078 1079
	 * don't need to use it.
	 */
1080 1081
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1082 1083
		event->pmu->read(event);
		/* fall-through */
1084

1085 1086
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1087 1088 1089 1090 1091 1092 1093
		break;

	default:
		break;
	}

	/*
1094
	 * In order to keep per-task stats reliable we need to flip the event
1095 1096
	 * values when we flip the contexts.
	 */
1097 1098 1099
	value = atomic64_read(&next_event->count);
	value = atomic64_xchg(&event->count, value);
	atomic64_set(&next_event->count, value);
1100

1101 1102
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1103

1104
	/*
1105
	 * Since we swizzled the values, update the user visible data too.
1106
	 */
1107 1108
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1109 1110 1111 1112 1113
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1114 1115
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1116
{
1117
	struct perf_event *event, *next_event;
1118 1119 1120 1121

	if (!ctx->nr_stat)
		return;

1122 1123
	update_context_time(ctx);

1124 1125
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1126

1127 1128
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1129

1130 1131
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1132

1133
		__perf_event_sync_stat(event, next_event);
1134

1135 1136
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1137 1138 1139
	}
}

T
Thomas Gleixner 已提交
1140
/*
1141
 * Called from scheduler to remove the events of the current task,
T
Thomas Gleixner 已提交
1142 1143
 * with interrupts disabled.
 *
1144
 * We stop each event and update the event value in event->count.
T
Thomas Gleixner 已提交
1145
 *
I
Ingo Molnar 已提交
1146
 * This does not protect us against NMI, but disable()
1147 1148 1149
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
T
Thomas Gleixner 已提交
1150
 */
1151
void perf_event_task_sched_out(struct task_struct *task,
1152
				 struct task_struct *next, int cpu)
T
Thomas Gleixner 已提交
1153 1154
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1155 1156 1157
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1158
	struct pt_regs *regs;
1159
	int do_switch = 1;
T
Thomas Gleixner 已提交
1160

1161
	regs = task_pt_regs(task);
1162
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1163

1164
	if (likely(!ctx || !cpuctx->task_ctx))
T
Thomas Gleixner 已提交
1165 1166
		return;

1167 1168
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1169
	next_ctx = next->perf_event_ctxp;
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
		spin_lock(&ctx->lock);
		spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
		if (context_equiv(ctx, next_ctx)) {
1184 1185
			/*
			 * XXX do we need a memory barrier of sorts
1186
			 * wrt to rcu_dereference() of perf_event_ctxp
1187
			 */
1188 1189
			task->perf_event_ctxp = next_ctx;
			next->perf_event_ctxp = ctx;
1190 1191 1192
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1193

1194
			perf_event_sync_stat(ctx, next_ctx);
1195 1196 1197
		}
		spin_unlock(&next_ctx->lock);
		spin_unlock(&ctx->lock);
1198
	}
1199
	rcu_read_unlock();
1200

1201
	if (do_switch) {
1202
		__perf_event_sched_out(ctx, cpuctx);
1203 1204
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1205 1206
}

1207 1208 1209
/*
 * Called with IRQs disabled
 */
1210
static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1211 1212 1213
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

1214 1215
	if (!cpuctx->task_ctx)
		return;
1216 1217 1218 1219

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1220
	__perf_event_sched_out(ctx, cpuctx);
1221 1222 1223
	cpuctx->task_ctx = NULL;
}

1224 1225 1226
/*
 * Called with IRQs disabled
 */
1227
static void perf_event_cpu_sched_out(struct perf_cpu_context *cpuctx)
1228
{
1229
	__perf_event_sched_out(&cpuctx->ctx, cpuctx);
1230 1231
}

1232
static void
1233
__perf_event_sched_in(struct perf_event_context *ctx,
1234
			struct perf_cpu_context *cpuctx, int cpu)
T
Thomas Gleixner 已提交
1235
{
1236
	struct perf_event *event;
1237
	int can_add_hw = 1;
T
Thomas Gleixner 已提交
1238

1239 1240
	spin_lock(&ctx->lock);
	ctx->is_active = 1;
1241
	if (likely(!ctx->nr_events))
1242
		goto out;
T
Thomas Gleixner 已提交
1243

1244
	ctx->timestamp = perf_clock();
1245

1246
	perf_disable();
1247 1248 1249 1250 1251

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
1252 1253 1254
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF ||
		    !event->attr.pinned)
1255
			continue;
1256
		if (event->cpu != -1 && event->cpu != cpu)
1257 1258
			continue;

1259 1260
		if (group_can_go_on(event, cpuctx, 1))
			group_sched_in(event, cpuctx, ctx, cpu);
1261 1262 1263 1264 1265

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1266 1267 1268
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1269
		}
1270 1271
	}

1272
	list_for_each_entry(event, &ctx->group_list, group_entry) {
1273
		/*
1274 1275
		 * Ignore events in OFF or ERROR state, and
		 * ignore pinned events since we did them already.
1276
		 */
1277 1278
		if (event->state <= PERF_EVENT_STATE_OFF ||
		    event->attr.pinned)
1279 1280
			continue;

1281 1282
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1283
		 * of events:
1284
		 */
1285
		if (event->cpu != -1 && event->cpu != cpu)
T
Thomas Gleixner 已提交
1286 1287
			continue;

1288 1289
		if (group_can_go_on(event, cpuctx, can_add_hw))
			if (group_sched_in(event, cpuctx, ctx, cpu))
1290
				can_add_hw = 0;
T
Thomas Gleixner 已提交
1291
	}
1292
	perf_enable();
1293
 out:
T
Thomas Gleixner 已提交
1294
	spin_unlock(&ctx->lock);
1295 1296 1297
}

/*
1298
 * Called from scheduler to add the events of the current task
1299 1300
 * with interrupts disabled.
 *
1301
 * We restore the event value and then enable it.
1302 1303
 *
 * This does not protect us against NMI, but enable()
1304 1305 1306
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
1307
 */
1308
void perf_event_task_sched_in(struct task_struct *task, int cpu)
1309 1310
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1311
	struct perf_event_context *ctx = task->perf_event_ctxp;
1312

1313 1314
	if (likely(!ctx))
		return;
1315 1316
	if (cpuctx->task_ctx == ctx)
		return;
1317
	__perf_event_sched_in(ctx, cpuctx, cpu);
T
Thomas Gleixner 已提交
1318 1319 1320
	cpuctx->task_ctx = ctx;
}

1321
static void perf_event_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1322
{
1323
	struct perf_event_context *ctx = &cpuctx->ctx;
1324

1325
	__perf_event_sched_in(ctx, cpuctx, cpu);
1326 1327
}

1328 1329
#define MAX_INTERRUPTS (~0ULL)

1330
static void perf_log_throttle(struct perf_event *event, int enable);
1331

1332
static void perf_adjust_period(struct perf_event *event, u64 events)
1333
{
1334
	struct hw_perf_event *hwc = &event->hw;
1335 1336 1337 1338
	u64 period, sample_period;
	s64 delta;

	events *= hwc->sample_period;
1339
	period = div64_u64(events, event->attr.sample_freq);
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
}

1352
static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1353
{
1354 1355
	struct perf_event *event;
	struct hw_perf_event *hwc;
1356
	u64 interrupts, freq;
1357 1358

	spin_lock(&ctx->lock);
1359
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1360
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1361 1362
			continue;

1363
		hwc = &event->hw;
1364 1365 1366

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1367

1368
		/*
1369
		 * unthrottle events on the tick
1370
		 */
1371
		if (interrupts == MAX_INTERRUPTS) {
1372 1373 1374
			perf_log_throttle(event, 1);
			event->pmu->unthrottle(event);
			interrupts = 2*sysctl_perf_event_sample_rate/HZ;
1375 1376
		}

1377
		if (!event->attr.freq || !event->attr.sample_freq)
1378 1379
			continue;

1380 1381 1382
		/*
		 * if the specified freq < HZ then we need to skip ticks
		 */
1383 1384
		if (event->attr.sample_freq < HZ) {
			freq = event->attr.sample_freq;
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397

			hwc->freq_count += freq;
			hwc->freq_interrupts += interrupts;

			if (hwc->freq_count < HZ)
				continue;

			interrupts = hwc->freq_interrupts;
			hwc->freq_interrupts = 0;
			hwc->freq_count -= HZ;
		} else
			freq = HZ;

1398
		perf_adjust_period(event, freq * interrupts);
1399

1400 1401 1402 1403 1404 1405 1406
		/*
		 * In order to avoid being stalled by an (accidental) huge
		 * sample period, force reset the sample period if we didn't
		 * get any events in this freq period.
		 */
		if (!interrupts) {
			perf_disable();
1407
			event->pmu->disable(event);
1408
			atomic64_set(&hwc->period_left, 0);
1409
			event->pmu->enable(event);
1410 1411
			perf_enable();
		}
1412 1413 1414 1415
	}
	spin_unlock(&ctx->lock);
}

1416
/*
1417
 * Round-robin a context's events:
1418
 */
1419
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1420
{
1421
	struct perf_event *event;
T
Thomas Gleixner 已提交
1422

1423
	if (!ctx->nr_events)
T
Thomas Gleixner 已提交
1424 1425 1426 1427
		return;

	spin_lock(&ctx->lock);
	/*
1428
	 * Rotate the first entry last (works just fine for group events too):
T
Thomas Gleixner 已提交
1429
	 */
1430
	perf_disable();
1431 1432
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		list_move_tail(&event->group_entry, &ctx->group_list);
T
Thomas Gleixner 已提交
1433 1434
		break;
	}
1435
	perf_enable();
T
Thomas Gleixner 已提交
1436 1437

	spin_unlock(&ctx->lock);
1438 1439
}

1440
void perf_event_task_tick(struct task_struct *curr, int cpu)
1441
{
1442
	struct perf_cpu_context *cpuctx;
1443
	struct perf_event_context *ctx;
1444

1445
	if (!atomic_read(&nr_events))
1446 1447 1448
		return;

	cpuctx = &per_cpu(perf_cpu_context, cpu);
1449
	ctx = curr->perf_event_ctxp;
1450

1451
	perf_ctx_adjust_freq(&cpuctx->ctx);
1452
	if (ctx)
1453
		perf_ctx_adjust_freq(ctx);
1454

1455
	perf_event_cpu_sched_out(cpuctx);
1456
	if (ctx)
1457
		__perf_event_task_sched_out(ctx);
T
Thomas Gleixner 已提交
1458

1459
	rotate_ctx(&cpuctx->ctx);
1460 1461
	if (ctx)
		rotate_ctx(ctx);
1462

1463
	perf_event_cpu_sched_in(cpuctx, cpu);
1464
	if (ctx)
1465
		perf_event_task_sched_in(curr, cpu);
T
Thomas Gleixner 已提交
1466 1467
}

1468
/*
1469
 * Enable all of a task's events that have been marked enable-on-exec.
1470 1471
 * This expects task == current.
 */
1472
static void perf_event_enable_on_exec(struct task_struct *task)
1473
{
1474 1475
	struct perf_event_context *ctx;
	struct perf_event *event;
1476 1477 1478 1479
	unsigned long flags;
	int enabled = 0;

	local_irq_save(flags);
1480 1481
	ctx = task->perf_event_ctxp;
	if (!ctx || !ctx->nr_events)
1482 1483
		goto out;

1484
	__perf_event_task_sched_out(ctx);
1485 1486 1487

	spin_lock(&ctx->lock);

1488 1489
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		if (!event->attr.enable_on_exec)
1490
			continue;
1491 1492
		event->attr.enable_on_exec = 0;
		if (event->state >= PERF_EVENT_STATE_INACTIVE)
1493
			continue;
1494
		__perf_event_mark_enabled(event, ctx);
1495 1496 1497 1498
		enabled = 1;
	}

	/*
1499
	 * Unclone this context if we enabled any event.
1500
	 */
1501 1502
	if (enabled)
		unclone_ctx(ctx);
1503 1504 1505

	spin_unlock(&ctx->lock);

1506
	perf_event_task_sched_in(task, smp_processor_id());
1507 1508 1509 1510
 out:
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1511
/*
1512
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1513
 */
1514
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1515
{
1516
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1517 1518
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
I
Ingo Molnar 已提交
1519

1520 1521 1522 1523
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1524 1525
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1526 1527 1528 1529
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

P
Peter Zijlstra 已提交
1530
	spin_lock(&ctx->lock);
P
Peter Zijlstra 已提交
1531
	update_context_time(ctx);
1532
	update_event_times(event);
P
Peter Zijlstra 已提交
1533 1534
	spin_unlock(&ctx->lock);

P
Peter Zijlstra 已提交
1535
	event->pmu->read(event);
T
Thomas Gleixner 已提交
1536 1537
}

1538
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1539 1540
{
	/*
1541 1542
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1543
	 */
1544 1545 1546 1547
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
1548 1549 1550 1551 1552
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

		spin_lock_irqsave(&ctx->lock, flags);
		update_context_time(ctx);
1553
		update_event_times(event);
P
Peter Zijlstra 已提交
1554
		spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1555 1556
	}

1557
	return atomic64_read(&event->count);
T
Thomas Gleixner 已提交
1558 1559
}

1560
/*
1561
 * Initialize the perf_event context in a task_struct:
1562 1563
 */
static void
1564
__perf_event_init_context(struct perf_event_context *ctx,
1565 1566 1567 1568 1569
			    struct task_struct *task)
{
	memset(ctx, 0, sizeof(*ctx));
	spin_lock_init(&ctx->lock);
	mutex_init(&ctx->mutex);
1570
	INIT_LIST_HEAD(&ctx->group_list);
1571 1572 1573 1574 1575
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
	ctx->task = task;
}

1576
static struct perf_event_context *find_get_context(pid_t pid, int cpu)
T
Thomas Gleixner 已提交
1577
{
1578
	struct perf_event_context *ctx;
1579
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
1580
	struct task_struct *task;
1581
	unsigned long flags;
1582
	int err;
T
Thomas Gleixner 已提交
1583 1584

	/*
1585
	 * If cpu is not a wildcard then this is a percpu event:
T
Thomas Gleixner 已提交
1586 1587
	 */
	if (cpu != -1) {
1588
		/* Must be root to operate on a CPU event: */
1589
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
1590 1591 1592 1593 1594 1595
			return ERR_PTR(-EACCES);

		if (cpu < 0 || cpu > num_possible_cpus())
			return ERR_PTR(-EINVAL);

		/*
1596
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
1597 1598 1599 1600 1601 1602 1603 1604
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
		if (!cpu_isset(cpu, cpu_online_map))
			return ERR_PTR(-ENODEV);

		cpuctx = &per_cpu(perf_cpu_context, cpu);
		ctx = &cpuctx->ctx;
1605
		get_ctx(ctx);
T
Thomas Gleixner 已提交
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621

		return ctx;
	}

	rcu_read_lock();
	if (!pid)
		task = current;
	else
		task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

1622
	/*
1623
	 * Can't attach events to a dying task.
1624 1625 1626 1627 1628
	 */
	err = -ESRCH;
	if (task->flags & PF_EXITING)
		goto errout;

T
Thomas Gleixner 已提交
1629
	/* Reuse ptrace permission checks for now. */
1630 1631 1632 1633 1634
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

 retry:
1635
	ctx = perf_lock_task_context(task, &flags);
1636
	if (ctx) {
1637
		unclone_ctx(ctx);
1638
		spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1639 1640
	}

1641
	if (!ctx) {
1642
		ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1643 1644 1645
		err = -ENOMEM;
		if (!ctx)
			goto errout;
1646
		__perf_event_init_context(ctx, task);
1647
		get_ctx(ctx);
1648
		if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1649 1650 1651 1652 1653
			/*
			 * We raced with some other task; use
			 * the context they set.
			 */
			kfree(ctx);
1654
			goto retry;
1655
		}
1656
		get_task_struct(task);
1657 1658
	}

1659
	put_task_struct(task);
T
Thomas Gleixner 已提交
1660
	return ctx;
1661 1662 1663 1664

 errout:
	put_task_struct(task);
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
1665 1666
}

L
Li Zefan 已提交
1667 1668
static void perf_event_free_filter(struct perf_event *event);

1669
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
1670
{
1671
	struct perf_event *event;
P
Peter Zijlstra 已提交
1672

1673 1674 1675
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
1676
	perf_event_free_filter(event);
1677
	kfree(event);
P
Peter Zijlstra 已提交
1678 1679
}

1680
static void perf_pending_sync(struct perf_event *event);
1681

1682
static void free_event(struct perf_event *event)
1683
{
1684
	perf_pending_sync(event);
1685

1686 1687 1688 1689 1690 1691 1692 1693
	if (!event->parent) {
		atomic_dec(&nr_events);
		if (event->attr.mmap)
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
1694
	}
1695

1696 1697 1698
	if (event->output) {
		fput(event->output->filp);
		event->output = NULL;
1699 1700
	}

1701 1702
	if (event->destroy)
		event->destroy(event);
1703

1704 1705
	put_ctx(event->ctx);
	call_rcu(&event->rcu_head, free_event_rcu);
1706 1707
}

1708
int perf_event_release_kernel(struct perf_event *event)
T
Thomas Gleixner 已提交
1709
{
1710
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1711

1712
	WARN_ON_ONCE(ctx->parent_ctx);
1713
	mutex_lock(&ctx->mutex);
1714
	perf_event_remove_from_context(event);
1715
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
1716

1717 1718 1719 1720
	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);
1721

1722
	free_event(event);
T
Thomas Gleixner 已提交
1723 1724 1725

	return 0;
}
1726
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
T
Thomas Gleixner 已提交
1727

1728 1729 1730 1731
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
1732
{
1733
	struct perf_event *event = file->private_data;
1734

1735
	file->private_data = NULL;
1736

1737
	return perf_event_release_kernel(event);
1738 1739
}

1740
static int perf_event_read_size(struct perf_event *event)
1741 1742 1743 1744 1745
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

1746
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1747 1748
		size += sizeof(u64);

1749
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1750 1751
		size += sizeof(u64);

1752
	if (event->attr.read_format & PERF_FORMAT_ID)
1753 1754
		entry += sizeof(u64);

1755 1756
	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
1757 1758 1759 1760 1761 1762 1763 1764
		size += sizeof(u64);
	}

	size += entry * nr;

	return size;
}

1765
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
1766
{
1767
	struct perf_event *child;
1768 1769
	u64 total = 0;

1770 1771 1772
	*enabled = 0;
	*running = 0;

1773
	mutex_lock(&event->child_mutex);
1774
	total += perf_event_read(event);
1775 1776 1777 1778 1779 1780
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
1781
		total += perf_event_read(child);
1782 1783 1784
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
1785
	mutex_unlock(&event->child_mutex);
1786 1787 1788

	return total;
}
1789
EXPORT_SYMBOL_GPL(perf_event_read_value);
1790

1791
static int perf_event_read_group(struct perf_event *event,
1792 1793
				   u64 read_format, char __user *buf)
{
1794
	struct perf_event *leader = event->group_leader, *sub;
1795 1796
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
1797
	u64 values[5];
1798
	u64 count, enabled, running;
1799

1800
	mutex_lock(&ctx->mutex);
1801
	count = perf_event_read_value(leader, &enabled, &running);
1802 1803

	values[n++] = 1 + leader->nr_siblings;
1804 1805 1806 1807
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1808 1809 1810
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
1811 1812 1813 1814

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
1815
		goto unlock;
1816

1817
	ret = size;
1818

1819
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
1820
		n = 0;
1821

1822
		values[n++] = perf_event_read_value(sub, &enabled, &running);
1823 1824 1825 1826 1827
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

1828 1829 1830 1831
		if (copy_to_user(buf + size, values, size)) {
			ret = -EFAULT;
			goto unlock;
		}
1832 1833

		ret += size;
1834
	}
1835 1836
unlock:
	mutex_unlock(&ctx->mutex);
1837

1838
	return ret;
1839 1840
}

1841
static int perf_event_read_one(struct perf_event *event,
1842 1843
				 u64 read_format, char __user *buf)
{
1844
	u64 enabled, running;
1845 1846 1847
	u64 values[4];
	int n = 0;

1848 1849 1850 1851 1852
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1853
	if (read_format & PERF_FORMAT_ID)
1854
		values[n++] = primary_event_id(event);
1855 1856 1857 1858 1859 1860 1861

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
1862
/*
1863
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
1864 1865
 */
static ssize_t
1866
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
1867
{
1868
	u64 read_format = event->attr.read_format;
1869
	int ret;
T
Thomas Gleixner 已提交
1870

1871
	/*
1872
	 * Return end-of-file for a read on a event that is in
1873 1874 1875
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
1876
	if (event->state == PERF_EVENT_STATE_ERROR)
1877 1878
		return 0;

1879
	if (count < perf_event_read_size(event))
1880 1881
		return -ENOSPC;

1882
	WARN_ON_ONCE(event->ctx->parent_ctx);
1883
	if (read_format & PERF_FORMAT_GROUP)
1884
		ret = perf_event_read_group(event, read_format, buf);
1885
	else
1886
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
1887

1888
	return ret;
T
Thomas Gleixner 已提交
1889 1890 1891 1892 1893
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
1894
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
1895

1896
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
1897 1898 1899 1900
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
1901
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
1902
	struct perf_mmap_data *data;
1903
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
1904 1905

	rcu_read_lock();
1906
	data = rcu_dereference(event->data);
P
Peter Zijlstra 已提交
1907
	if (data)
1908
		events = atomic_xchg(&data->poll, 0);
P
Peter Zijlstra 已提交
1909
	rcu_read_unlock();
T
Thomas Gleixner 已提交
1910

1911
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
1912 1913 1914 1915

	return events;
}

1916
static void perf_event_reset(struct perf_event *event)
1917
{
1918 1919 1920
	(void)perf_event_read(event);
	atomic64_set(&event->count, 0);
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
1921 1922
}

1923
/*
1924 1925 1926 1927
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
1928
 */
1929 1930
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
1931
{
1932
	struct perf_event *child;
P
Peter Zijlstra 已提交
1933

1934 1935 1936 1937
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
1938
		func(child);
1939
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
1940 1941
}

1942 1943
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
1944
{
1945 1946
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
1947

1948 1949
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
1950
	event = event->group_leader;
1951

1952 1953 1954 1955
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
1956
	mutex_unlock(&ctx->mutex);
1957 1958
}

1959
static int perf_event_period(struct perf_event *event, u64 __user *arg)
1960
{
1961
	struct perf_event_context *ctx = event->ctx;
1962 1963 1964 1965
	unsigned long size;
	int ret = 0;
	u64 value;

1966
	if (!event->attr.sample_period)
1967 1968 1969 1970 1971 1972 1973 1974 1975 1976
		return -EINVAL;

	size = copy_from_user(&value, arg, sizeof(value));
	if (size != sizeof(value))
		return -EFAULT;

	if (!value)
		return -EINVAL;

	spin_lock_irq(&ctx->lock);
1977 1978
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
1979 1980 1981 1982
			ret = -EINVAL;
			goto unlock;
		}

1983
		event->attr.sample_freq = value;
1984
	} else {
1985 1986
		event->attr.sample_period = value;
		event->hw.sample_period = value;
1987 1988 1989 1990 1991 1992 1993
	}
unlock:
	spin_unlock_irq(&ctx->lock);

	return ret;
}

L
Li Zefan 已提交
1994 1995
static int perf_event_set_output(struct perf_event *event, int output_fd);
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
1996

1997 1998
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
1999 2000
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2001
	u32 flags = arg;
2002 2003

	switch (cmd) {
2004 2005
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2006
		break;
2007 2008
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2009
		break;
2010 2011
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2012
		break;
P
Peter Zijlstra 已提交
2013

2014 2015
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2016

2017 2018
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2019

2020 2021
	case PERF_EVENT_IOC_SET_OUTPUT:
		return perf_event_set_output(event, arg);
2022

L
Li Zefan 已提交
2023 2024 2025
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2026
	default:
P
Peter Zijlstra 已提交
2027
		return -ENOTTY;
2028
	}
P
Peter Zijlstra 已提交
2029 2030

	if (flags & PERF_IOC_FLAG_GROUP)
2031
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2032
	else
2033
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2034 2035

	return 0;
2036 2037
}

2038
int perf_event_task_enable(void)
2039
{
2040
	struct perf_event *event;
2041

2042 2043 2044 2045
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2046 2047 2048 2049

	return 0;
}

2050
int perf_event_task_disable(void)
2051
{
2052
	struct perf_event *event;
2053

2054 2055 2056 2057
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2058 2059 2060 2061

	return 0;
}

2062 2063
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2064 2065
#endif

2066
static int perf_event_index(struct perf_event *event)
2067
{
2068
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2069 2070
		return 0;

2071
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2072 2073
}

2074 2075 2076 2077 2078
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2079
void perf_event_update_userpage(struct perf_event *event)
2080
{
2081
	struct perf_event_mmap_page *userpg;
2082
	struct perf_mmap_data *data;
2083 2084

	rcu_read_lock();
2085
	data = rcu_dereference(event->data);
2086 2087 2088 2089
	if (!data)
		goto unlock;

	userpg = data->user_page;
2090

2091 2092 2093 2094 2095
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2096
	++userpg->lock;
2097
	barrier();
2098 2099 2100 2101
	userpg->index = perf_event_index(event);
	userpg->offset = atomic64_read(&event->count);
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		userpg->offset -= atomic64_read(&event->hw.prev_count);
2102

2103 2104
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2105

2106 2107
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2108

2109
	barrier();
2110
	++userpg->lock;
2111
	preempt_enable();
2112
unlock:
2113
	rcu_read_unlock();
2114 2115
}

2116
static unsigned long perf_data_size(struct perf_mmap_data *data)
2117
{
2118 2119
	return data->nr_pages << (PAGE_SHIFT + data->data_order);
}
2120

2121
#ifndef CONFIG_PERF_USE_VMALLOC
2122

2123 2124 2125
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2126

2127 2128 2129 2130 2131
static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > data->nr_pages)
		return NULL;
2132

2133 2134
	if (pgoff == 0)
		return virt_to_page(data->user_page);
2135

2136
	return virt_to_page(data->data_pages[pgoff - 1]);
2137 2138
}

2139 2140
static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2141 2142 2143 2144 2145
{
	struct perf_mmap_data *data;
	unsigned long size;
	int i;

2146
	WARN_ON(atomic_read(&event->mmap_count));
2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164

	size = sizeof(struct perf_mmap_data);
	size += nr_pages * sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
	if (!data->user_page)
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
		data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
		if (!data->data_pages[i])
			goto fail_data_pages;
	}

2165
	data->data_order = 0;
2166 2167
	data->nr_pages = nr_pages;

2168
	return data;
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179

fail_data_pages:
	for (i--; i >= 0; i--)
		free_page((unsigned long)data->data_pages[i]);

	free_page((unsigned long)data->user_page);

fail_user_page:
	kfree(data);

fail:
2180
	return NULL;
2181 2182
}

2183 2184
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2185
	struct page *page = virt_to_page((void *)addr);
2186 2187 2188 2189 2190

	page->mapping = NULL;
	__free_page(page);
}

2191
static void perf_mmap_data_free(struct perf_mmap_data *data)
2192 2193 2194
{
	int i;

2195
	perf_mmap_free_page((unsigned long)data->user_page);
2196
	for (i = 0; i < data->nr_pages; i++)
2197
		perf_mmap_free_page((unsigned long)data->data_pages[i]);
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252
}

#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > (1UL << data->data_order))
		return NULL;

	return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

static void perf_mmap_data_free_work(struct work_struct *work)
{
	struct perf_mmap_data *data;
	void *base;
	int i, nr;

	data = container_of(work, struct perf_mmap_data, work);
	nr = 1 << data->data_order;

	base = data->user_page;
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
}

static void perf_mmap_data_free(struct perf_mmap_data *data)
{
	schedule_work(&data->work);
}

static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
{
	struct perf_mmap_data *data;
	unsigned long size;
	void *all_buf;

	WARN_ON(atomic_read(&event->mmap_count));
2253

2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330
	size = sizeof(struct perf_mmap_data);
	size += sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	INIT_WORK(&data->work, perf_mmap_data_free_work);

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

	data->user_page = all_buf;
	data->data_pages[0] = all_buf + PAGE_SIZE;
	data->data_order = ilog2(nr_pages);
	data->nr_pages = 1;

	return data;

fail_all_buf:
	kfree(data);

fail:
	return NULL;
}

#endif

static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
	struct perf_mmap_data *data;
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	data = rcu_dereference(event->data);
	if (!data)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(data, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

static void
perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
{
	long max_size = perf_data_size(data);

	atomic_set(&data->lock, -1);

	if (event->attr.watermark) {
		data->watermark = min_t(long, max_size,
					event->attr.wakeup_watermark);
	}

	if (!data->watermark)
2331
		data->watermark = max_size / 2;
2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342


	rcu_assign_pointer(event->data, data);
}

static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
{
	struct perf_mmap_data *data;

	data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
	perf_mmap_data_free(data);
2343 2344 2345
	kfree(data);
}

2346
static void perf_mmap_data_release(struct perf_event *event)
2347
{
2348
	struct perf_mmap_data *data = event->data;
2349

2350
	WARN_ON(atomic_read(&event->mmap_count));
2351

2352
	rcu_assign_pointer(event->data, NULL);
2353
	call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2354 2355 2356 2357
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
2358
	struct perf_event *event = vma->vm_file->private_data;
2359

2360
	atomic_inc(&event->mmap_count);
2361 2362 2363 2364
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
2365
	struct perf_event *event = vma->vm_file->private_data;
2366

2367 2368
	WARN_ON_ONCE(event->ctx->parent_ctx);
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2369
		unsigned long size = perf_data_size(event->data);
2370 2371
		struct user_struct *user = current_user();

2372
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2373
		vma->vm_mm->locked_vm -= event->data->nr_locked;
2374
		perf_mmap_data_release(event);
2375
		mutex_unlock(&event->mmap_mutex);
2376
	}
2377 2378
}

2379
static const struct vm_operations_struct perf_mmap_vmops = {
2380 2381 2382 2383
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
2384 2385 2386 2387
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
2388
	struct perf_event *event = file->private_data;
2389
	unsigned long user_locked, user_lock_limit;
2390
	struct user_struct *user = current_user();
2391
	unsigned long locked, lock_limit;
2392
	struct perf_mmap_data *data;
2393 2394
	unsigned long vma_size;
	unsigned long nr_pages;
2395
	long user_extra, extra;
2396
	int ret = 0;
2397

2398
	if (!(vma->vm_flags & VM_SHARED))
2399
		return -EINVAL;
2400 2401 2402 2403

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

2404 2405 2406 2407 2408
	/*
	 * If we have data pages ensure they're a power-of-two number, so we
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
2409 2410
		return -EINVAL;

2411
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
2412 2413
		return -EINVAL;

2414 2415
	if (vma->vm_pgoff != 0)
		return -EINVAL;
2416

2417 2418 2419
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
	if (event->output) {
2420 2421 2422 2423
		ret = -EINVAL;
		goto unlock;
	}

2424 2425
	if (atomic_inc_not_zero(&event->mmap_count)) {
		if (nr_pages != event->data->nr_pages)
2426 2427 2428 2429
			ret = -EINVAL;
		goto unlock;
	}

2430
	user_extra = nr_pages + 1;
2431
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
2432 2433 2434 2435 2436 2437

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

2438
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2439

2440 2441 2442
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
2443 2444 2445

	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
	lock_limit >>= PAGE_SHIFT;
2446
	locked = vma->vm_mm->locked_vm + extra;
2447

2448 2449
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
2450 2451 2452
		ret = -EPERM;
		goto unlock;
	}
2453

2454
	WARN_ON(event->data);
2455 2456 2457 2458

	data = perf_mmap_data_alloc(event, nr_pages);
	ret = -ENOMEM;
	if (!data)
2459 2460
		goto unlock;

2461 2462 2463
	ret = 0;
	perf_mmap_data_init(event, data);

2464
	atomic_set(&event->mmap_count, 1);
2465
	atomic_long_add(user_extra, &user->locked_vm);
2466
	vma->vm_mm->locked_vm += extra;
2467
	event->data->nr_locked = extra;
2468
	if (vma->vm_flags & VM_WRITE)
2469
		event->data->writable = 1;
2470

2471
unlock:
2472
	mutex_unlock(&event->mmap_mutex);
2473 2474 2475

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
2476 2477

	return ret;
2478 2479
}

P
Peter Zijlstra 已提交
2480 2481 2482
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
2483
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
2484 2485 2486
	int retval;

	mutex_lock(&inode->i_mutex);
2487
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
2488 2489 2490 2491 2492 2493 2494 2495
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
2496 2497 2498 2499
static const struct file_operations perf_fops = {
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
2500 2501
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
2502
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
2503
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
2504 2505
};

2506
/*
2507
 * Perf event wakeup
2508 2509 2510 2511 2512
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

2513
void perf_event_wakeup(struct perf_event *event)
2514
{
2515
	wake_up_all(&event->waitq);
2516

2517 2518 2519
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
2520
	}
2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531
}

/*
 * Pending wakeups
 *
 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
 *
 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
 * single linked list and use cmpxchg() to add entries lockless.
 */

2532
static void perf_pending_event(struct perf_pending_entry *entry)
2533
{
2534 2535
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
2536

2537 2538 2539
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
2540 2541
	}

2542 2543 2544
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
2545 2546 2547
	}
}

2548
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2549

2550
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2551 2552 2553
	PENDING_TAIL,
};

2554 2555
static void perf_pending_queue(struct perf_pending_entry *entry,
			       void (*func)(struct perf_pending_entry *))
2556
{
2557
	struct perf_pending_entry **head;
2558

2559
	if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2560 2561
		return;

2562 2563 2564
	entry->func = func;

	head = &get_cpu_var(perf_pending_head);
2565 2566

	do {
2567 2568
		entry->next = *head;
	} while (cmpxchg(head, entry->next, entry) != entry->next);
2569

2570
	set_perf_event_pending();
2571

2572
	put_cpu_var(perf_pending_head);
2573 2574 2575 2576
}

static int __perf_pending_run(void)
{
2577
	struct perf_pending_entry *list;
2578 2579
	int nr = 0;

2580
	list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2581
	while (list != PENDING_TAIL) {
2582 2583
		void (*func)(struct perf_pending_entry *);
		struct perf_pending_entry *entry = list;
2584 2585 2586

		list = list->next;

2587 2588
		func = entry->func;
		entry->next = NULL;
2589 2590 2591 2592 2593 2594 2595
		/*
		 * Ensure we observe the unqueue before we issue the wakeup,
		 * so that we won't be waiting forever.
		 * -- see perf_not_pending().
		 */
		smp_wmb();

2596
		func(entry);
2597 2598 2599 2600 2601 2602
		nr++;
	}

	return nr;
}

2603
static inline int perf_not_pending(struct perf_event *event)
2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
{
	/*
	 * If we flush on whatever cpu we run, there is a chance we don't
	 * need to wait.
	 */
	get_cpu();
	__perf_pending_run();
	put_cpu();

	/*
	 * Ensure we see the proper queue state before going to sleep
	 * so that we do not miss the wakeup. -- see perf_pending_handle()
	 */
	smp_rmb();
2618
	return event->pending.next == NULL;
2619 2620
}

2621
static void perf_pending_sync(struct perf_event *event)
2622
{
2623
	wait_event(event->waitq, perf_not_pending(event));
2624 2625
}

2626
void perf_event_do_pending(void)
2627 2628 2629 2630
{
	__perf_pending_run();
}

2631 2632 2633 2634
/*
 * Callchain support -- arch specific
 */

2635
__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2636 2637 2638 2639
{
	return NULL;
}

2640 2641 2642
/*
 * Output
 */
2643 2644
static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
			      unsigned long offset, unsigned long head)
2645 2646 2647 2648 2649 2650
{
	unsigned long mask;

	if (!data->writable)
		return true;

2651
	mask = perf_data_size(data) - 1;
2652 2653 2654 2655 2656 2657 2658 2659 2660 2661

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

2662
static void perf_output_wakeup(struct perf_output_handle *handle)
2663
{
2664 2665
	atomic_set(&handle->data->poll, POLL_IN);

2666
	if (handle->nmi) {
2667 2668 2669
		handle->event->pending_wakeup = 1;
		perf_pending_queue(&handle->event->pending,
				   perf_pending_event);
2670
	} else
2671
		perf_event_wakeup(handle->event);
2672 2673
}

2674 2675 2676
/*
 * Curious locking construct.
 *
2677 2678
 * We need to ensure a later event_id doesn't publish a head when a former
 * event_id isn't done writing. However since we need to deal with NMIs we
2679 2680 2681 2682 2683 2684
 * cannot fully serialize things.
 *
 * What we do is serialize between CPUs so we only have to deal with NMI
 * nesting on a single CPU.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
2685
 * event_id completes.
2686 2687 2688 2689
 */
static void perf_output_lock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2690
	int cur, cpu = get_cpu();
2691 2692 2693

	handle->locked = 0;

2694 2695 2696 2697 2698 2699 2700 2701
	for (;;) {
		cur = atomic_cmpxchg(&data->lock, -1, cpu);
		if (cur == -1) {
			handle->locked = 1;
			break;
		}
		if (cur == cpu)
			break;
2702 2703

		cpu_relax();
2704
	}
2705 2706 2707 2708 2709
}

static void perf_output_unlock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2710 2711
	unsigned long head;
	int cpu;
2712

2713
	data->done_head = data->head;
2714 2715 2716 2717 2718 2719 2720 2721 2722 2723

	if (!handle->locked)
		goto out;

again:
	/*
	 * The xchg implies a full barrier that ensures all writes are done
	 * before we publish the new head, matched by a rmb() in userspace when
	 * reading this position.
	 */
2724
	while ((head = atomic_long_xchg(&data->done_head, 0)))
2725 2726 2727
		data->user_page->data_head = head;

	/*
2728
	 * NMI can happen here, which means we can miss a done_head update.
2729 2730
	 */

2731
	cpu = atomic_xchg(&data->lock, -1);
2732 2733 2734 2735 2736
	WARN_ON_ONCE(cpu != smp_processor_id());

	/*
	 * Therefore we have to validate we did not indeed do so.
	 */
2737
	if (unlikely(atomic_long_read(&data->done_head))) {
2738 2739 2740
		/*
		 * Since we had it locked, we can lock it again.
		 */
2741
		while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2742 2743 2744 2745 2746
			cpu_relax();

		goto again;
	}

2747
	if (atomic_xchg(&data->wakeup, 0))
2748 2749
		perf_output_wakeup(handle);
out:
2750
	put_cpu();
2751 2752
}

2753 2754
void perf_output_copy(struct perf_output_handle *handle,
		      const void *buf, unsigned int len)
2755 2756
{
	unsigned int pages_mask;
2757
	unsigned long offset;
2758 2759 2760 2761 2762 2763 2764 2765
	unsigned int size;
	void **pages;

	offset		= handle->offset;
	pages_mask	= handle->data->nr_pages - 1;
	pages		= handle->data->data_pages;

	do {
2766 2767
		unsigned long page_offset;
		unsigned long page_size;
2768 2769 2770
		int nr;

		nr	    = (offset >> PAGE_SHIFT) & pages_mask;
2771 2772 2773
		page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
		page_offset = offset & (page_size - 1);
		size	    = min_t(unsigned int, page_size - page_offset, len);
2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790

		memcpy(pages[nr] + page_offset, buf, size);

		len	    -= size;
		buf	    += size;
		offset	    += size;
	} while (len);

	handle->offset = offset;

	/*
	 * Check we didn't copy past our reservation window, taking the
	 * possible unsigned int wrap into account.
	 */
	WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
}

2791
int perf_output_begin(struct perf_output_handle *handle,
2792
		      struct perf_event *event, unsigned int size,
2793
		      int nmi, int sample)
2794
{
2795
	struct perf_event *output_event;
2796
	struct perf_mmap_data *data;
2797
	unsigned long tail, offset, head;
2798 2799 2800 2801 2802 2803
	int have_lost;
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
2804

2805
	rcu_read_lock();
2806
	/*
2807
	 * For inherited events we send all the output towards the parent.
2808
	 */
2809 2810
	if (event->parent)
		event = event->parent;
2811

2812 2813 2814
	output_event = rcu_dereference(event->output);
	if (output_event)
		event = output_event;
2815

2816
	data = rcu_dereference(event->data);
2817 2818 2819
	if (!data)
		goto out;

2820
	handle->data	= data;
2821
	handle->event	= event;
2822 2823
	handle->nmi	= nmi;
	handle->sample	= sample;
2824

2825
	if (!data->nr_pages)
2826
		goto fail;
2827

2828 2829 2830 2831
	have_lost = atomic_read(&data->lost);
	if (have_lost)
		size += sizeof(lost_event);

2832 2833
	perf_output_lock(handle);

2834
	do {
2835 2836 2837 2838 2839 2840 2841
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
		tail = ACCESS_ONCE(data->user_page->data_tail);
		smp_rmb();
2842
		offset = head = atomic_long_read(&data->head);
P
Peter Zijlstra 已提交
2843
		head += size;
2844
		if (unlikely(!perf_output_space(data, tail, offset, head)))
2845
			goto fail;
2846
	} while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2847

2848
	handle->offset	= offset;
2849
	handle->head	= head;
2850

2851
	if (head - tail > data->watermark)
2852
		atomic_set(&data->wakeup, 1);
2853

2854
	if (have_lost) {
2855
		lost_event.header.type = PERF_RECORD_LOST;
2856 2857
		lost_event.header.misc = 0;
		lost_event.header.size = sizeof(lost_event);
2858
		lost_event.id          = event->id;
2859 2860 2861 2862 2863
		lost_event.lost        = atomic_xchg(&data->lost, 0);

		perf_output_put(handle, lost_event);
	}

2864
	return 0;
2865

2866
fail:
2867 2868
	atomic_inc(&data->lost);
	perf_output_unlock(handle);
2869 2870
out:
	rcu_read_unlock();
2871

2872 2873
	return -ENOSPC;
}
2874

2875
void perf_output_end(struct perf_output_handle *handle)
2876
{
2877
	struct perf_event *event = handle->event;
2878 2879
	struct perf_mmap_data *data = handle->data;

2880
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
2881

2882
	if (handle->sample && wakeup_events) {
2883
		int events = atomic_inc_return(&data->events);
P
Peter Zijlstra 已提交
2884
		if (events >= wakeup_events) {
2885
			atomic_sub(wakeup_events, &data->events);
2886
			atomic_set(&data->wakeup, 1);
P
Peter Zijlstra 已提交
2887
		}
2888 2889 2890
	}

	perf_output_unlock(handle);
2891
	rcu_read_unlock();
2892 2893
}

2894
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
2895 2896
{
	/*
2897
	 * only top level events have the pid namespace they were created in
2898
	 */
2899 2900
	if (event->parent)
		event = event->parent;
2901

2902
	return task_tgid_nr_ns(p, event->ns);
2903 2904
}

2905
static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
2906 2907
{
	/*
2908
	 * only top level events have the pid namespace they were created in
2909
	 */
2910 2911
	if (event->parent)
		event = event->parent;
2912

2913
	return task_pid_nr_ns(p, event->ns);
2914 2915
}

2916
static void perf_output_read_one(struct perf_output_handle *handle,
2917
				 struct perf_event *event)
2918
{
2919
	u64 read_format = event->attr.read_format;
2920 2921 2922
	u64 values[4];
	int n = 0;

2923
	values[n++] = atomic64_read(&event->count);
2924
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2925 2926
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2927 2928
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2929 2930
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2931 2932
	}
	if (read_format & PERF_FORMAT_ID)
2933
		values[n++] = primary_event_id(event);
2934 2935 2936 2937 2938

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
2939
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
2940 2941
 */
static void perf_output_read_group(struct perf_output_handle *handle,
2942
			    struct perf_event *event)
2943
{
2944 2945
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = leader->total_time_enabled;

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = leader->total_time_running;

2957
	if (leader != event)
2958 2959 2960 2961
		leader->pmu->read(leader);

	values[n++] = atomic64_read(&leader->count);
	if (read_format & PERF_FORMAT_ID)
2962
		values[n++] = primary_event_id(leader);
2963 2964 2965

	perf_output_copy(handle, values, n * sizeof(u64));

2966
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2967 2968
		n = 0;

2969
		if (sub != event)
2970 2971 2972 2973
			sub->pmu->read(sub);

		values[n++] = atomic64_read(&sub->count);
		if (read_format & PERF_FORMAT_ID)
2974
			values[n++] = primary_event_id(sub);
2975 2976 2977 2978 2979 2980

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

static void perf_output_read(struct perf_output_handle *handle,
2981
			     struct perf_event *event)
2982
{
2983 2984
	if (event->attr.read_format & PERF_FORMAT_GROUP)
		perf_output_read_group(handle, event);
2985
	else
2986
		perf_output_read_one(handle, event);
2987 2988
}

2989 2990 2991
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
2992
			struct perf_event *event)
2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3023
		perf_output_read(handle, event);
3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3061
			 struct perf_event *event,
3062
			 struct pt_regs *regs)
3063
{
3064
	u64 sample_type = event->attr.sample_type;
3065

3066
	data->type = sample_type;
3067

3068
	header->type = PERF_RECORD_SAMPLE;
3069 3070 3071 3072
	header->size = sizeof(*header);

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3073

3074
	if (sample_type & PERF_SAMPLE_IP) {
3075 3076 3077
		data->ip = perf_instruction_pointer(regs);

		header->size += sizeof(data->ip);
3078
	}
3079

3080
	if (sample_type & PERF_SAMPLE_TID) {
3081
		/* namespace issues */
3082 3083
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
3084

3085
		header->size += sizeof(data->tid_entry);
3086 3087
	}

3088
	if (sample_type & PERF_SAMPLE_TIME) {
P
Peter Zijlstra 已提交
3089
		data->time = perf_clock();
3090

3091
		header->size += sizeof(data->time);
3092 3093
	}

3094
	if (sample_type & PERF_SAMPLE_ADDR)
3095
		header->size += sizeof(data->addr);
3096

3097
	if (sample_type & PERF_SAMPLE_ID) {
3098
		data->id = primary_event_id(event);
3099

3100 3101 3102 3103
		header->size += sizeof(data->id);
	}

	if (sample_type & PERF_SAMPLE_STREAM_ID) {
3104
		data->stream_id = event->id;
3105 3106 3107

		header->size += sizeof(data->stream_id);
	}
3108

3109
	if (sample_type & PERF_SAMPLE_CPU) {
3110 3111
		data->cpu_entry.cpu		= raw_smp_processor_id();
		data->cpu_entry.reserved	= 0;
3112

3113
		header->size += sizeof(data->cpu_entry);
3114 3115
	}

3116
	if (sample_type & PERF_SAMPLE_PERIOD)
3117
		header->size += sizeof(data->period);
3118

3119
	if (sample_type & PERF_SAMPLE_READ)
3120
		header->size += perf_event_read_size(event);
3121

3122
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3123
		int size = 1;
3124

3125 3126 3127 3128 3129 3130
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3131 3132
	}

3133
	if (sample_type & PERF_SAMPLE_RAW) {
3134 3135 3136 3137 3138 3139 3140 3141
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3142
		header->size += size;
3143
	}
3144
}
3145

3146
static void perf_event_output(struct perf_event *event, int nmi,
3147 3148 3149 3150 3151
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3152

3153
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3154

3155
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3156
		return;
3157

3158
	perf_output_sample(&handle, &header, data, event);
3159

3160
	perf_output_end(&handle);
3161 3162
}

3163
/*
3164
 * read event_id
3165 3166 3167 3168 3169 3170 3171 3172 3173 3174
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3175
perf_event_read_event(struct perf_event *event,
3176 3177 3178
			struct task_struct *task)
{
	struct perf_output_handle handle;
3179
	struct perf_read_event read_event = {
3180
		.header = {
3181
			.type = PERF_RECORD_READ,
3182
			.misc = 0,
3183
			.size = sizeof(read_event) + perf_event_read_size(event),
3184
		},
3185 3186
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3187
	};
3188
	int ret;
3189

3190
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3191 3192 3193
	if (ret)
		return;

3194
	perf_output_put(&handle, read_event);
3195
	perf_output_read(&handle, event);
3196

3197 3198 3199
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3200
/*
P
Peter Zijlstra 已提交
3201 3202 3203
 * task tracking -- fork/exit
 *
 * enabled by: attr.comm | attr.mmap | attr.task
P
Peter Zijlstra 已提交
3204 3205
 */

P
Peter Zijlstra 已提交
3206
struct perf_task_event {
3207
	struct task_struct		*task;
3208
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3209 3210 3211 3212 3213 3214

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3215 3216
		u32				tid;
		u32				ptid;
3217
		u64				time;
3218
	} event_id;
P
Peter Zijlstra 已提交
3219 3220
};

3221
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3222
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3223 3224
{
	struct perf_output_handle handle;
3225
	int size;
P
Peter Zijlstra 已提交
3226
	struct task_struct *task = task_event->task;
3227 3228
	int ret;

3229 3230
	size  = task_event->event_id.header.size;
	ret = perf_output_begin(&handle, event, size, 0, 0);
P
Peter Zijlstra 已提交
3231 3232 3233 3234

	if (ret)
		return;

3235 3236
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3237

3238 3239
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3240

3241
	task_event->event_id.time = perf_clock();
3242

3243
	perf_output_put(&handle, task_event->event_id);
3244

P
Peter Zijlstra 已提交
3245 3246 3247
	perf_output_end(&handle);
}

3248
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3249
{
3250
	if (event->attr.comm || event->attr.mmap || event->attr.task)
P
Peter Zijlstra 已提交
3251 3252 3253 3254 3255
		return 1;

	return 0;
}

3256
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
3257
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3258
{
3259
	struct perf_event *event;
P
Peter Zijlstra 已提交
3260

3261 3262 3263
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
3264 3265 3266
	}
}

3267
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3268 3269
{
	struct perf_cpu_context *cpuctx;
3270
	struct perf_event_context *ctx = task_event->task_ctx;
P
Peter Zijlstra 已提交
3271

3272
	rcu_read_lock();
P
Peter Zijlstra 已提交
3273
	cpuctx = &get_cpu_var(perf_cpu_context);
3274
	perf_event_task_ctx(&cpuctx->ctx, task_event);
P
Peter Zijlstra 已提交
3275 3276
	put_cpu_var(perf_cpu_context);

3277
	if (!ctx)
3278
		ctx = rcu_dereference(task_event->task->perf_event_ctxp);
P
Peter Zijlstra 已提交
3279
	if (ctx)
3280
		perf_event_task_ctx(ctx, task_event);
P
Peter Zijlstra 已提交
3281 3282 3283
	rcu_read_unlock();
}

3284 3285
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3286
			      int new)
P
Peter Zijlstra 已提交
3287
{
P
Peter Zijlstra 已提交
3288
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
3289

3290 3291 3292
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
3293 3294
		return;

P
Peter Zijlstra 已提交
3295
	task_event = (struct perf_task_event){
3296 3297
		.task	  = task,
		.task_ctx = task_ctx,
3298
		.event_id    = {
P
Peter Zijlstra 已提交
3299
			.header = {
3300
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3301
				.misc = 0,
3302
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
3303
			},
3304 3305
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
3306 3307
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
3308 3309 3310
		},
	};

3311
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
3312 3313
}

3314
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
3315
{
3316
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
3317 3318
}

3319 3320 3321 3322 3323
/*
 * comm tracking
 */

struct perf_comm_event {
3324 3325
	struct task_struct	*task;
	char			*comm;
3326 3327 3328 3329 3330 3331 3332
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
3333
	} event_id;
3334 3335
};

3336
static void perf_event_comm_output(struct perf_event *event,
3337 3338 3339
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
3340 3341
	int size = comm_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3342 3343 3344 3345

	if (ret)
		return;

3346 3347
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3348

3349
	perf_output_put(&handle, comm_event->event_id);
3350 3351 3352 3353 3354
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
	perf_output_end(&handle);
}

3355
static int perf_event_comm_match(struct perf_event *event)
3356
{
3357
	if (event->attr.comm)
3358 3359 3360 3361 3362
		return 1;

	return 0;
}

3363
static void perf_event_comm_ctx(struct perf_event_context *ctx,
3364 3365
				  struct perf_comm_event *comm_event)
{
3366
	struct perf_event *event;
3367

3368 3369 3370
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
3371 3372 3373
	}
}

3374
static void perf_event_comm_event(struct perf_comm_event *comm_event)
3375 3376
{
	struct perf_cpu_context *cpuctx;
3377
	struct perf_event_context *ctx;
3378
	unsigned int size;
3379
	char comm[TASK_COMM_LEN];
3380

3381
	memset(comm, 0, sizeof(comm));
3382
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
3383
	size = ALIGN(strlen(comm)+1, sizeof(u64));
3384 3385 3386 3387

	comm_event->comm = comm;
	comm_event->comm_size = size;

3388
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3389

3390
	rcu_read_lock();
3391
	cpuctx = &get_cpu_var(perf_cpu_context);
3392
	perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3393
	put_cpu_var(perf_cpu_context);
3394 3395 3396 3397 3398

	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3399
	ctx = rcu_dereference(current->perf_event_ctxp);
3400
	if (ctx)
3401
		perf_event_comm_ctx(ctx, comm_event);
3402
	rcu_read_unlock();
3403 3404
}

3405
void perf_event_comm(struct task_struct *task)
3406
{
3407 3408
	struct perf_comm_event comm_event;

3409 3410
	if (task->perf_event_ctxp)
		perf_event_enable_on_exec(task);
3411

3412
	if (!atomic_read(&nr_comm_events))
3413
		return;
3414

3415
	comm_event = (struct perf_comm_event){
3416
		.task	= task,
3417 3418
		/* .comm      */
		/* .comm_size */
3419
		.event_id  = {
3420
			.header = {
3421
				.type = PERF_RECORD_COMM,
3422 3423 3424 3425 3426
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3427 3428 3429
		},
	};

3430
	perf_event_comm_event(&comm_event);
3431 3432
}

3433 3434 3435 3436 3437
/*
 * mmap tracking
 */

struct perf_mmap_event {
3438 3439 3440 3441
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
3442 3443 3444 3445 3446 3447 3448 3449 3450

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
3451
	} event_id;
3452 3453
};

3454
static void perf_event_mmap_output(struct perf_event *event,
3455 3456 3457
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
3458 3459
	int size = mmap_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3460 3461 3462 3463

	if (ret)
		return;

3464 3465
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
3466

3467
	perf_output_put(&handle, mmap_event->event_id);
3468 3469
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
3470
	perf_output_end(&handle);
3471 3472
}

3473
static int perf_event_mmap_match(struct perf_event *event,
3474 3475
				   struct perf_mmap_event *mmap_event)
{
3476
	if (event->attr.mmap)
3477 3478 3479 3480 3481
		return 1;

	return 0;
}

3482
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3483 3484
				  struct perf_mmap_event *mmap_event)
{
3485
	struct perf_event *event;
3486

3487 3488 3489
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_mmap_match(event, mmap_event))
			perf_event_mmap_output(event, mmap_event);
3490 3491 3492
	}
}

3493
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3494 3495
{
	struct perf_cpu_context *cpuctx;
3496
	struct perf_event_context *ctx;
3497 3498
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
3499 3500 3501
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
3502
	const char *name;
3503

3504 3505
	memset(tmp, 0, sizeof(tmp));

3506
	if (file) {
3507 3508 3509 3510 3511 3512
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3513 3514 3515 3516
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
3517
		name = d_path(&file->f_path, buf, PATH_MAX);
3518 3519 3520 3521 3522
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
3523 3524 3525
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
3526
			goto got_name;
3527
		}
3528 3529 3530 3531 3532 3533

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
		}

3534 3535 3536 3537 3538
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
3539
	size = ALIGN(strlen(name)+1, sizeof(u64));
3540 3541 3542 3543

	mmap_event->file_name = name;
	mmap_event->file_size = size;

3544
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3545

3546
	rcu_read_lock();
3547
	cpuctx = &get_cpu_var(perf_cpu_context);
3548
	perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
3549 3550
	put_cpu_var(perf_cpu_context);

3551 3552 3553 3554
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3555
	ctx = rcu_dereference(current->perf_event_ctxp);
3556
	if (ctx)
3557
		perf_event_mmap_ctx(ctx, mmap_event);
3558 3559
	rcu_read_unlock();

3560 3561 3562
	kfree(buf);
}

3563
void __perf_event_mmap(struct vm_area_struct *vma)
3564
{
3565 3566
	struct perf_mmap_event mmap_event;

3567
	if (!atomic_read(&nr_mmap_events))
3568 3569 3570
		return;

	mmap_event = (struct perf_mmap_event){
3571
		.vma	= vma,
3572 3573
		/* .file_name */
		/* .file_size */
3574
		.event_id  = {
3575
			.header = {
3576
				.type = PERF_RECORD_MMAP,
3577 3578 3579 3580 3581
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3582 3583 3584
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
			.pgoff  = vma->vm_pgoff,
3585 3586 3587
		},
	};

3588
	perf_event_mmap_event(&mmap_event);
3589 3590
}

3591 3592 3593 3594
/*
 * IRQ throttle logging
 */

3595
static void perf_log_throttle(struct perf_event *event, int enable)
3596 3597 3598 3599 3600 3601 3602
{
	struct perf_output_handle handle;
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
3603
		u64				id;
3604
		u64				stream_id;
3605 3606
	} throttle_event = {
		.header = {
3607
			.type = PERF_RECORD_THROTTLE,
3608 3609 3610
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
3611
		.time		= perf_clock(),
3612 3613
		.id		= primary_event_id(event),
		.stream_id	= event->id,
3614 3615
	};

3616
	if (enable)
3617
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3618

3619
	ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3620 3621 3622 3623 3624 3625 3626
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
	perf_output_end(&handle);
}

3627
/*
3628
 * Generic event overflow handling, sampling.
3629 3630
 */

3631
static int __perf_event_overflow(struct perf_event *event, int nmi,
3632 3633
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
3634
{
3635 3636
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
3637 3638
	int ret = 0;

3639
	throttle = (throttle && event->pmu->unthrottle != NULL);
3640

3641
	if (!throttle) {
3642
		hwc->interrupts++;
3643
	} else {
3644 3645
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
3646
			if (HZ * hwc->interrupts >
3647
					(u64)sysctl_perf_event_sample_rate) {
3648
				hwc->interrupts = MAX_INTERRUPTS;
3649
				perf_log_throttle(event, 0);
3650 3651 3652 3653
				ret = 1;
			}
		} else {
			/*
3654
			 * Keep re-disabling events even though on the previous
3655
			 * pass we disabled it - just in case we raced with a
3656
			 * sched-in and the event got enabled again:
3657
			 */
3658 3659 3660
			ret = 1;
		}
	}
3661

3662
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
3663
		u64 now = perf_clock();
3664 3665 3666 3667 3668
		s64 delta = now - hwc->freq_stamp;

		hwc->freq_stamp = now;

		if (delta > 0 && delta < TICK_NSEC)
3669
			perf_adjust_period(event, NSEC_PER_SEC / (int)delta);
3670 3671
	}

3672 3673
	/*
	 * XXX event_limit might not quite work as expected on inherited
3674
	 * events
3675 3676
	 */

3677 3678
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
3679
		ret = 1;
3680
		event->pending_kill = POLL_HUP;
3681
		if (nmi) {
3682 3683 3684
			event->pending_disable = 1;
			perf_pending_queue(&event->pending,
					   perf_pending_event);
3685
		} else
3686
			perf_event_disable(event);
3687 3688
	}

3689 3690 3691 3692 3693
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

3694
	return ret;
3695 3696
}

3697
int perf_event_overflow(struct perf_event *event, int nmi,
3698 3699
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
3700
{
3701
	return __perf_event_overflow(event, nmi, 1, data, regs);
3702 3703
}

3704
/*
3705
 * Generic software event infrastructure
3706 3707
 */

3708
/*
3709 3710
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
3711 3712 3713 3714
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

3715
static u64 perf_swevent_set_period(struct perf_event *event)
3716
{
3717
	struct hw_perf_event *hwc = &event->hw;
3718 3719 3720 3721 3722
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
3723 3724

again:
3725 3726 3727
	old = val = atomic64_read(&hwc->period_left);
	if (val < 0)
		return 0;
3728

3729 3730 3731 3732 3733
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
	if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
		goto again;
3734

3735
	return nr;
3736 3737
}

3738
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3739 3740
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
3741
{
3742
	struct hw_perf_event *hwc = &event->hw;
3743
	int throttle = 0;
3744

3745
	data->period = event->hw.last_period;
3746 3747
	if (!overflow)
		overflow = perf_swevent_set_period(event);
3748

3749 3750
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
3751

3752
	for (; overflow; overflow--) {
3753
		if (__perf_event_overflow(event, nmi, throttle,
3754
					    data, regs)) {
3755 3756 3757 3758 3759 3760
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
3761
		throttle = 1;
3762
	}
3763 3764
}

3765
static void perf_swevent_unthrottle(struct perf_event *event)
3766 3767
{
	/*
3768
	 * Nothing to do, we already reset hwc->interrupts.
3769
	 */
3770
}
3771

3772
static void perf_swevent_add(struct perf_event *event, u64 nr,
3773 3774
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
3775
{
3776
	struct hw_perf_event *hwc = &event->hw;
3777

3778
	atomic64_add(nr, &event->count);
3779

3780 3781 3782
	if (!regs)
		return;

3783 3784
	if (!hwc->sample_period)
		return;
3785

3786 3787 3788 3789
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

	if (atomic64_add_negative(nr, &hwc->period_left))
3790
		return;
3791

3792
	perf_swevent_overflow(event, 0, nmi, data, regs);
3793 3794
}

3795
static int perf_swevent_is_counting(struct perf_event *event)
3796
{
3797
	/*
3798
	 * The event is active, we're good!
3799
	 */
3800
	if (event->state == PERF_EVENT_STATE_ACTIVE)
3801 3802
		return 1;

3803
	/*
3804
	 * The event is off/error, not counting.
3805
	 */
3806
	if (event->state != PERF_EVENT_STATE_INACTIVE)
3807 3808 3809
		return 0;

	/*
3810
	 * The event is inactive, if the context is active
3811 3812
	 * we're part of a group that didn't make it on the 'pmu',
	 * not counting.
3813
	 */
3814
	if (event->ctx->is_active)
3815 3816 3817 3818 3819 3820 3821 3822
		return 0;

	/*
	 * We're inactive and the context is too, this means the
	 * task is scheduled out, we're counting events that happen
	 * to us, like migration events.
	 */
	return 1;
3823 3824
}

L
Li Zefan 已提交
3825 3826 3827
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data);

3828
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
3829
				enum perf_type_id type,
L
Li Zefan 已提交
3830 3831 3832
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
3833
{
3834
	if (!perf_swevent_is_counting(event))
3835 3836
		return 0;

3837
	if (event->attr.type != type)
3838
		return 0;
3839
	if (event->attr.config != event_id)
3840 3841
		return 0;

3842
	if (regs) {
3843
		if (event->attr.exclude_user && user_mode(regs))
3844
			return 0;
3845

3846
		if (event->attr.exclude_kernel && !user_mode(regs))
3847 3848
			return 0;
	}
3849

L
Li Zefan 已提交
3850 3851 3852 3853
	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
	    !perf_tp_event_match(event, data))
		return 0;

3854 3855 3856
	return 1;
}

3857
static void perf_swevent_ctx_event(struct perf_event_context *ctx,
3858
				     enum perf_type_id type,
3859
				     u32 event_id, u64 nr, int nmi,
3860 3861
				     struct perf_sample_data *data,
				     struct pt_regs *regs)
3862
{
3863
	struct perf_event *event;
3864

3865
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
L
Li Zefan 已提交
3866
		if (perf_swevent_match(event, type, event_id, data, regs))
3867
			perf_swevent_add(event, nr, nmi, data, regs);
3868 3869 3870
	}
}

3871 3872 3873 3874
/*
 * Must be called with preemption disabled
 */
int perf_swevent_get_recursion_context(int **recursion)
P
Peter Zijlstra 已提交
3875
{
3876 3877
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

P
Peter Zijlstra 已提交
3878
	if (in_nmi())
3879 3880 3881 3882 3883 3884 3885
		*recursion = &cpuctx->recursion[3];
	else if (in_irq())
		*recursion = &cpuctx->recursion[2];
	else if (in_softirq())
		*recursion = &cpuctx->recursion[1];
	else
		*recursion = &cpuctx->recursion[0];
P
Peter Zijlstra 已提交
3886

3887 3888
	if (**recursion)
		return -1;
P
Peter Zijlstra 已提交
3889

3890
	(**recursion)++;
P
Peter Zijlstra 已提交
3891

3892
	return 0;
P
Peter Zijlstra 已提交
3893
}
I
Ingo Molnar 已提交
3894
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
3895

3896
void perf_swevent_put_recursion_context(int *recursion)
3897
{
3898 3899
	(*recursion)--;
}
I
Ingo Molnar 已提交
3900
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
P
Peter Zijlstra 已提交
3901

3902 3903 3904 3905 3906 3907 3908
static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
			       u64 nr, int nmi,
			       struct perf_sample_data *data,
			       struct pt_regs *regs)
{
	struct perf_event_context *ctx;
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3909

3910
	rcu_read_lock();
3911
	perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
3912
				 nr, nmi, data, regs);
3913 3914 3915 3916
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3917
	ctx = rcu_dereference(current->perf_event_ctxp);
3918
	if (ctx)
3919
		perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs);
3920
	rcu_read_unlock();
3921
}
3922

3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
				    u64 nr, int nmi,
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
{
	int *recursion;

	preempt_disable();

	if (perf_swevent_get_recursion_context(&recursion))
		goto out;

	__do_perf_sw_event(type, event_id, nr, nmi, data, regs);
P
Peter Zijlstra 已提交
3936

3937
	perf_swevent_put_recursion_context(recursion);
P
Peter Zijlstra 已提交
3938
out:
3939
	preempt_enable();
3940 3941
}

3942
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
3943
			    struct pt_regs *regs, u64 addr)
3944
{
3945 3946 3947 3948
	struct perf_sample_data data;

	data.addr = addr;
	data.raw  = NULL;
3949

3950
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
3951 3952
}

3953
static void perf_swevent_read(struct perf_event *event)
3954 3955 3956
{
}

3957
static int perf_swevent_enable(struct perf_event *event)
3958
{
3959
	struct hw_perf_event *hwc = &event->hw;
3960 3961 3962

	if (hwc->sample_period) {
		hwc->last_period = hwc->sample_period;
3963
		perf_swevent_set_period(event);
3964
	}
3965 3966 3967
	return 0;
}

3968
static void perf_swevent_disable(struct perf_event *event)
3969 3970 3971
{
}

3972
static const struct pmu perf_ops_generic = {
3973 3974 3975 3976
	.enable		= perf_swevent_enable,
	.disable	= perf_swevent_disable,
	.read		= perf_swevent_read,
	.unthrottle	= perf_swevent_unthrottle,
3977 3978
};

3979
/*
3980
 * hrtimer based swevent callback
3981 3982
 */

3983
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
3984 3985 3986
{
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
3987
	struct pt_regs *regs;
3988
	struct perf_event *event;
3989 3990
	u64 period;

3991 3992
	event	= container_of(hrtimer, struct perf_event, hw.hrtimer);
	event->pmu->read(event);
3993 3994

	data.addr = 0;
3995
	regs = get_irq_regs();
3996 3997 3998 3999
	/*
	 * In case we exclude kernel IPs or are somehow not in interrupt
	 * context, provide the next best thing, the user IP.
	 */
4000 4001
	if ((event->attr.exclude_kernel || !regs) &&
			!event->attr.exclude_user)
4002
		regs = task_pt_regs(current);
4003

4004
	if (regs) {
4005 4006 4007
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
4008 4009
	}

4010
	period = max_t(u64, 10000, event->hw.sample_period);
4011 4012 4013 4014 4015
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));

	return ret;
}

4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051
static void perf_swevent_start_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
	if (hwc->sample_period) {
		u64 period;

		if (hwc->remaining) {
			if (hwc->remaining < 0)
				period = 10000;
			else
				period = hwc->remaining;
			hwc->remaining = 0;
		} else {
			period = max_t(u64, 10000, hwc->sample_period);
		}
		__hrtimer_start_range_ns(&hwc->hrtimer,
				ns_to_ktime(period), 0,
				HRTIMER_MODE_REL, 0);
	}
}

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (hwc->sample_period) {
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
		hwc->remaining = ktime_to_ns(remaining);

		hrtimer_cancel(&hwc->hrtimer);
	}
}

4052
/*
4053
 * Software event: cpu wall time clock
4054 4055
 */

4056
static void cpu_clock_perf_event_update(struct perf_event *event)
4057 4058 4059 4060 4061 4062
{
	int cpu = raw_smp_processor_id();
	s64 prev;
	u64 now;

	now = cpu_clock(cpu);
4063 4064 4065
	prev = atomic64_read(&event->hw.prev_count);
	atomic64_set(&event->hw.prev_count, now);
	atomic64_add(now - prev, &event->count);
4066 4067
}

4068
static int cpu_clock_perf_event_enable(struct perf_event *event)
4069
{
4070
	struct hw_perf_event *hwc = &event->hw;
4071 4072 4073
	int cpu = raw_smp_processor_id();

	atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4074
	perf_swevent_start_hrtimer(event);
4075 4076 4077 4078

	return 0;
}

4079
static void cpu_clock_perf_event_disable(struct perf_event *event)
4080
{
4081
	perf_swevent_cancel_hrtimer(event);
4082
	cpu_clock_perf_event_update(event);
4083 4084
}

4085
static void cpu_clock_perf_event_read(struct perf_event *event)
4086
{
4087
	cpu_clock_perf_event_update(event);
4088 4089
}

4090
static const struct pmu perf_ops_cpu_clock = {
4091 4092 4093
	.enable		= cpu_clock_perf_event_enable,
	.disable	= cpu_clock_perf_event_disable,
	.read		= cpu_clock_perf_event_read,
4094 4095
};

4096
/*
4097
 * Software event: task time clock
4098 4099
 */

4100
static void task_clock_perf_event_update(struct perf_event *event, u64 now)
I
Ingo Molnar 已提交
4101
{
4102
	u64 prev;
I
Ingo Molnar 已提交
4103 4104
	s64 delta;

4105
	prev = atomic64_xchg(&event->hw.prev_count, now);
I
Ingo Molnar 已提交
4106
	delta = now - prev;
4107
	atomic64_add(delta, &event->count);
4108 4109
}

4110
static int task_clock_perf_event_enable(struct perf_event *event)
I
Ingo Molnar 已提交
4111
{
4112
	struct hw_perf_event *hwc = &event->hw;
4113 4114
	u64 now;

4115
	now = event->ctx->time;
4116

4117
	atomic64_set(&hwc->prev_count, now);
4118 4119

	perf_swevent_start_hrtimer(event);
4120 4121

	return 0;
I
Ingo Molnar 已提交
4122 4123
}

4124
static void task_clock_perf_event_disable(struct perf_event *event)
4125
{
4126
	perf_swevent_cancel_hrtimer(event);
4127
	task_clock_perf_event_update(event, event->ctx->time);
4128

4129
}
I
Ingo Molnar 已提交
4130

4131
static void task_clock_perf_event_read(struct perf_event *event)
4132
{
4133 4134 4135
	u64 time;

	if (!in_nmi()) {
4136 4137
		update_context_time(event->ctx);
		time = event->ctx->time;
4138 4139
	} else {
		u64 now = perf_clock();
4140 4141
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
4142 4143
	}

4144
	task_clock_perf_event_update(event, time);
4145 4146
}

4147
static const struct pmu perf_ops_task_clock = {
4148 4149 4150
	.enable		= task_clock_perf_event_enable,
	.disable	= task_clock_perf_event_disable,
	.read		= task_clock_perf_event_read,
4151 4152
};

4153
#ifdef CONFIG_EVENT_PROFILE
L
Li Zefan 已提交
4154

4155
void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4156
			  int entry_size)
4157
{
4158
	struct perf_raw_record raw = {
4159
		.size = entry_size,
4160
		.data = record,
4161 4162
	};

4163
	struct perf_sample_data data = {
4164
		.addr = addr,
4165
		.raw = &raw,
4166
	};
4167

4168 4169 4170 4171
	struct pt_regs *regs = get_irq_regs();

	if (!regs)
		regs = task_pt_regs(current);
4172

4173 4174
	/* Trace events already protected against recursion */
	__do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4175
				&data, regs);
4176
}
4177
EXPORT_SYMBOL_GPL(perf_tp_event);
4178

L
Li Zefan 已提交
4179 4180 4181 4182 4183 4184 4185 4186 4187
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}
4188

4189
static void tp_perf_event_destroy(struct perf_event *event)
4190
{
4191
	ftrace_profile_disable(event->attr.config);
4192 4193
}

4194
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4195
{
4196 4197 4198 4199
	/*
	 * Raw tracepoint data is a severe data leak, only allow root to
	 * have these.
	 */
4200
	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4201
			perf_paranoid_tracepoint_raw() &&
4202 4203 4204
			!capable(CAP_SYS_ADMIN))
		return ERR_PTR(-EPERM);

4205
	if (ftrace_profile_enable(event->attr.config))
4206 4207
		return NULL;

4208
	event->destroy = tp_perf_event_destroy;
4209 4210 4211

	return &perf_ops_generic;
}
L
Li Zefan 已提交
4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

4236
#else
L
Li Zefan 已提交
4237 4238 4239 4240 4241 4242 4243

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	return 1;
}

4244
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4245 4246 4247
{
	return NULL;
}
L
Li Zefan 已提交
4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

#endif /* CONFIG_EVENT_PROFILE */
4259

4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304
#ifdef CONFIG_HAVE_HW_BREAKPOINT
static void bp_perf_event_destroy(struct perf_event *event)
{
	release_bp_slot(event);
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	int err;
	/*
	 * The breakpoint is already filled if we haven't created the counter
	 * through perf syscall
	 * FIXME: manage to get trigerred to NULL if it comes from syscalls
	 */
	if (!bp->callback)
		err = register_perf_hw_breakpoint(bp);
	else
		err = __register_perf_hw_breakpoint(bp);
	if (err)
		return ERR_PTR(err);

	bp->destroy = bp_perf_event_destroy;

	return &perf_ops_bp;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
	/* TODO */
}
#else
static void bp_perf_event_destroy(struct perf_event *event)
{
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	return NULL;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
}
#endif

4305
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4306

4307
static void sw_perf_event_destroy(struct perf_event *event)
4308
{
4309
	u64 event_id = event->attr.config;
4310

4311
	WARN_ON(event->parent);
4312

4313
	atomic_dec(&perf_swevent_enabled[event_id]);
4314 4315
}

4316
static const struct pmu *sw_perf_event_init(struct perf_event *event)
4317
{
4318
	const struct pmu *pmu = NULL;
4319
	u64 event_id = event->attr.config;
4320

4321
	/*
4322
	 * Software events (currently) can't in general distinguish
4323 4324 4325 4326 4327
	 * between user, kernel and hypervisor events.
	 * However, context switches and cpu migrations are considered
	 * to be kernel events, and page faults are never hypervisor
	 * events.
	 */
4328
	switch (event_id) {
4329
	case PERF_COUNT_SW_CPU_CLOCK:
4330
		pmu = &perf_ops_cpu_clock;
4331

4332
		break;
4333
	case PERF_COUNT_SW_TASK_CLOCK:
4334
		/*
4335 4336
		 * If the user instantiates this as a per-cpu event,
		 * use the cpu_clock event instead.
4337
		 */
4338
		if (event->ctx->task)
4339
			pmu = &perf_ops_task_clock;
4340
		else
4341
			pmu = &perf_ops_cpu_clock;
4342

4343
		break;
4344 4345 4346 4347 4348
	case PERF_COUNT_SW_PAGE_FAULTS:
	case PERF_COUNT_SW_PAGE_FAULTS_MIN:
	case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
	case PERF_COUNT_SW_CONTEXT_SWITCHES:
	case PERF_COUNT_SW_CPU_MIGRATIONS:
4349 4350
	case PERF_COUNT_SW_ALIGNMENT_FAULTS:
	case PERF_COUNT_SW_EMULATION_FAULTS:
4351 4352 4353
		if (!event->parent) {
			atomic_inc(&perf_swevent_enabled[event_id]);
			event->destroy = sw_perf_event_destroy;
4354
		}
4355
		pmu = &perf_ops_generic;
4356
		break;
4357
	}
4358

4359
	return pmu;
4360 4361
}

T
Thomas Gleixner 已提交
4362
/*
4363
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
4364
 */
4365 4366
static struct perf_event *
perf_event_alloc(struct perf_event_attr *attr,
4367
		   int cpu,
4368 4369 4370
		   struct perf_event_context *ctx,
		   struct perf_event *group_leader,
		   struct perf_event *parent_event,
4371
		   perf_callback_t callback,
4372
		   gfp_t gfpflags)
T
Thomas Gleixner 已提交
4373
{
4374
	const struct pmu *pmu;
4375 4376
	struct perf_event *event;
	struct hw_perf_event *hwc;
4377
	long err;
T
Thomas Gleixner 已提交
4378

4379 4380
	event = kzalloc(sizeof(*event), gfpflags);
	if (!event)
4381
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
4382

4383
	/*
4384
	 * Single events are their own group leaders, with an
4385 4386 4387
	 * empty sibling list:
	 */
	if (!group_leader)
4388
		group_leader = event;
4389

4390 4391
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
4392

4393 4394 4395 4396
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
T
Thomas Gleixner 已提交
4397

4398
	mutex_init(&event->mmap_mutex);
4399

4400 4401 4402 4403 4404 4405
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->ctx		= ctx;
	event->oncpu		= -1;
4406

4407
	event->parent		= parent_event;
4408

4409 4410
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
4411

4412
	event->state		= PERF_EVENT_STATE_INACTIVE;
4413

4414 4415 4416 4417 4418
	if (!callback && parent_event)
		callback = parent_event->callback;
	
	event->callback	= callback;

4419
	if (attr->disabled)
4420
		event->state = PERF_EVENT_STATE_OFF;
4421

4422
	pmu = NULL;
4423

4424
	hwc = &event->hw;
4425
	hwc->sample_period = attr->sample_period;
4426
	if (attr->freq && attr->sample_freq)
4427
		hwc->sample_period = 1;
4428
	hwc->last_period = hwc->sample_period;
4429 4430

	atomic64_set(&hwc->period_left, hwc->sample_period);
4431

4432
	/*
4433
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
4434
	 */
4435
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4436 4437
		goto done;

4438
	switch (attr->type) {
4439
	case PERF_TYPE_RAW:
4440
	case PERF_TYPE_HARDWARE:
4441
	case PERF_TYPE_HW_CACHE:
4442
		pmu = hw_perf_event_init(event);
4443 4444 4445
		break;

	case PERF_TYPE_SOFTWARE:
4446
		pmu = sw_perf_event_init(event);
4447 4448 4449
		break;

	case PERF_TYPE_TRACEPOINT:
4450
		pmu = tp_perf_event_init(event);
4451
		break;
4452

4453 4454 4455 4456 4457
	case PERF_TYPE_BREAKPOINT:
		pmu = bp_perf_event_init(event);
		break;


4458 4459
	default:
		break;
4460
	}
4461 4462
done:
	err = 0;
4463
	if (!pmu)
4464
		err = -EINVAL;
4465 4466
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
4467

4468
	if (err) {
4469 4470 4471
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
4472
		return ERR_PTR(err);
I
Ingo Molnar 已提交
4473
	}
4474

4475
	event->pmu = pmu;
T
Thomas Gleixner 已提交
4476

4477 4478 4479 4480 4481 4482 4483 4484
	if (!event->parent) {
		atomic_inc(&nr_events);
		if (event->attr.mmap)
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
4485
	}
4486

4487
	return event;
T
Thomas Gleixner 已提交
4488 4489
}

4490 4491
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
4492 4493
{
	u32 size;
4494
	int ret;
4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
4519 4520 4521
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
4522 4523
	 */
	if (size > sizeof(*attr)) {
4524 4525 4526
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
4527

4528 4529
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
4530

4531
		for (; addr < end; addr++) {
4532 4533 4534 4535 4536 4537
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
4538
		size = sizeof(*attr);
4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Li Zefan 已提交
4570
static int perf_event_set_output(struct perf_event *event, int output_fd)
4571
{
4572
	struct perf_event *output_event = NULL;
4573
	struct file *output_file = NULL;
4574
	struct perf_event *old_output;
4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587
	int fput_needed = 0;
	int ret = -EINVAL;

	if (!output_fd)
		goto set;

	output_file = fget_light(output_fd, &fput_needed);
	if (!output_file)
		return -EBADF;

	if (output_file->f_op != &perf_fops)
		goto out;

4588
	output_event = output_file->private_data;
4589 4590

	/* Don't chain output fds */
4591
	if (output_event->output)
4592 4593 4594
		goto out;

	/* Don't set an output fd when we already have an output channel */
4595
	if (event->data)
4596 4597 4598 4599 4600
		goto out;

	atomic_long_inc(&output_file->f_count);

set:
4601 4602 4603 4604
	mutex_lock(&event->mmap_mutex);
	old_output = event->output;
	rcu_assign_pointer(event->output, output_event);
	mutex_unlock(&event->mmap_mutex);
4605 4606 4607 4608

	if (old_output) {
		/*
		 * we need to make sure no existing perf_output_*()
4609
		 * is still referencing this event.
4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620
		 */
		synchronize_rcu();
		fput(old_output->filp);
	}

	ret = 0;
out:
	fput_light(output_file, fput_needed);
	return ret;
}

T
Thomas Gleixner 已提交
4621
/**
4622
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
4623
 *
4624
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
4625
 * @pid:		target pid
I
Ingo Molnar 已提交
4626
 * @cpu:		target cpu
4627
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
4628
 */
4629 4630
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
4631
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
4632
{
4633 4634 4635 4636
	struct perf_event *event, *group_leader;
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
4637 4638
	struct file *group_file = NULL;
	int fput_needed = 0;
4639
	int fput_needed2 = 0;
4640
	int err;
T
Thomas Gleixner 已提交
4641

4642
	/* for future expandability... */
4643
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4644 4645
		return -EINVAL;

4646 4647 4648
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
4649

4650 4651 4652 4653 4654
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

4655
	if (attr.freq) {
4656
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
4657 4658 4659
			return -EINVAL;
	}

4660
	/*
I
Ingo Molnar 已提交
4661 4662 4663 4664 4665 4666 4667
	 * Get the target context (task or percpu):
	 */
	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

	/*
4668
	 * Look up the group leader (we will attach this event to it):
4669 4670
	 */
	group_leader = NULL;
4671
	if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4672
		err = -EINVAL;
4673 4674
		group_file = fget_light(group_fd, &fput_needed);
		if (!group_file)
I
Ingo Molnar 已提交
4675
			goto err_put_context;
4676
		if (group_file->f_op != &perf_fops)
I
Ingo Molnar 已提交
4677
			goto err_put_context;
4678 4679 4680

		group_leader = group_file->private_data;
		/*
I
Ingo Molnar 已提交
4681 4682 4683 4684 4685 4686 4687 4688
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
			goto err_put_context;
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
4689
		 */
I
Ingo Molnar 已提交
4690 4691
		if (group_leader->ctx != ctx)
			goto err_put_context;
4692 4693 4694
		/*
		 * Only a group leader can be exclusive or pinned
		 */
4695
		if (attr.exclusive || attr.pinned)
4696
			goto err_put_context;
4697 4698
	}

4699
	event = perf_event_alloc(&attr, cpu, ctx, group_leader,
4700
				     NULL, NULL, GFP_KERNEL);
4701 4702
	err = PTR_ERR(event);
	if (IS_ERR(event))
T
Thomas Gleixner 已提交
4703 4704
		goto err_put_context;

4705
	err = anon_inode_getfd("[perf_event]", &perf_fops, event, 0);
4706
	if (err < 0)
4707 4708
		goto err_free_put_context;

4709 4710
	event_file = fget_light(err, &fput_needed2);
	if (!event_file)
4711 4712
		goto err_free_put_context;

4713
	if (flags & PERF_FLAG_FD_OUTPUT) {
4714
		err = perf_event_set_output(event, group_fd);
4715 4716
		if (err)
			goto err_fput_free_put_context;
4717 4718
	}

4719
	event->filp = event_file;
4720
	WARN_ON_ONCE(ctx->parent_ctx);
4721
	mutex_lock(&ctx->mutex);
4722
	perf_install_in_context(ctx, event, cpu);
4723
	++ctx->generation;
4724
	mutex_unlock(&ctx->mutex);
4725

4726
	event->owner = current;
4727
	get_task_struct(current);
4728 4729 4730
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
4731

4732
err_fput_free_put_context:
4733
	fput_light(event_file, fput_needed2);
T
Thomas Gleixner 已提交
4734

4735
err_free_put_context:
4736
	if (err < 0)
4737
		kfree(event);
T
Thomas Gleixner 已提交
4738 4739

err_put_context:
4740 4741 4742 4743
	if (err < 0)
		put_ctx(ctx);

	fput_light(group_file, fput_needed);
T
Thomas Gleixner 已提交
4744

4745
	return err;
T
Thomas Gleixner 已提交
4746 4747
}

4748 4749 4750 4751 4752 4753 4754 4755 4756
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
 * @pid: task to profile
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4757
				 pid_t pid, perf_callback_t callback)
4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768
{
	struct perf_event *event;
	struct perf_event_context *ctx;
	int err;

	/*
	 * Get the target context (task or percpu):
	 */

	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
4769
		return NULL;
4770 4771

	event = perf_event_alloc(attr, cpu, ctx, NULL,
4772
				     NULL, callback, GFP_KERNEL);
4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799
	err = PTR_ERR(event);
	if (IS_ERR(event))
		goto err_put_context;

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
	mutex_unlock(&ctx->mutex);

	event->owner = current;
	get_task_struct(current);
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);

	return event;

err_put_context:
	if (err < 0)
		put_ctx(ctx);

	return NULL;
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);

4800
/*
4801
 * inherit a event from parent task to child task:
4802
 */
4803 4804
static struct perf_event *
inherit_event(struct perf_event *parent_event,
4805
	      struct task_struct *parent,
4806
	      struct perf_event_context *parent_ctx,
4807
	      struct task_struct *child,
4808 4809
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
4810
{
4811
	struct perf_event *child_event;
4812

4813
	/*
4814 4815
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
4816 4817 4818
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
4819 4820
	if (parent_event->parent)
		parent_event = parent_event->parent;
4821

4822 4823 4824
	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu, child_ctx,
					   group_leader, parent_event,
4825
					   NULL, GFP_KERNEL);
4826 4827
	if (IS_ERR(child_event))
		return child_event;
4828
	get_ctx(child_ctx);
4829

4830
	/*
4831
	 * Make the child state follow the state of the parent event,
4832
	 * not its attr.disabled bit.  We hold the parent's mutex,
4833
	 * so we won't race with perf_event_{en, dis}able_family.
4834
	 */
4835 4836
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
4837
	else
4838
		child_event->state = PERF_EVENT_STATE_OFF;
4839

4840 4841
	if (parent_event->attr.freq)
		child_event->hw.sample_period = parent_event->hw.sample_period;
4842

4843 4844
	child_event->overflow_handler = parent_event->overflow_handler;

4845 4846 4847
	/*
	 * Link it up in the child's context:
	 */
4848
	add_event_to_ctx(child_event, child_ctx);
4849 4850 4851

	/*
	 * Get a reference to the parent filp - we will fput it
4852
	 * when the child event exits. This is safe to do because
4853 4854 4855
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
4856
	atomic_long_inc(&parent_event->filp->f_count);
4857

4858
	/*
4859
	 * Link this into the parent event's child list
4860
	 */
4861 4862 4863 4864
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4865

4866
	return child_event;
4867 4868
}

4869
static int inherit_group(struct perf_event *parent_event,
4870
	      struct task_struct *parent,
4871
	      struct perf_event_context *parent_ctx,
4872
	      struct task_struct *child,
4873
	      struct perf_event_context *child_ctx)
4874
{
4875 4876 4877
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;
4878

4879
	leader = inherit_event(parent_event, parent, parent_ctx,
4880
				 child, NULL, child_ctx);
4881 4882
	if (IS_ERR(leader))
		return PTR_ERR(leader);
4883 4884
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
4885 4886 4887
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
4888
	}
4889 4890 4891
	return 0;
}

4892
static void sync_child_event(struct perf_event *child_event,
4893
			       struct task_struct *child)
4894
{
4895
	struct perf_event *parent_event = child_event->parent;
4896
	u64 child_val;
4897

4898 4899
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
4900

4901
	child_val = atomic64_read(&child_event->count);
4902 4903 4904 4905

	/*
	 * Add back the child's count to the parent's count:
	 */
4906 4907 4908 4909 4910
	atomic64_add(child_val, &parent_event->count);
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
4911 4912

	/*
4913
	 * Remove this event from the parent's list
4914
	 */
4915 4916 4917 4918
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4919 4920

	/*
4921
	 * Release the parent event, if this was the last
4922 4923
	 * reference to it.
	 */
4924
	fput(parent_event->filp);
4925 4926
}

4927
static void
4928 4929
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
4930
			 struct task_struct *child)
4931
{
4932
	struct perf_event *parent_event;
4933

4934 4935
	update_event_times(child_event);
	perf_event_remove_from_context(child_event);
4936

4937
	parent_event = child_event->parent;
4938
	/*
4939
	 * It can happen that parent exits first, and has events
4940
	 * that are still around due to the child reference. These
4941
	 * events need to be zapped - but otherwise linger.
4942
	 */
4943 4944 4945
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
4946
	}
4947 4948 4949
}

/*
4950
 * When a child task exits, feed back event values to parent events.
4951
 */
4952
void perf_event_exit_task(struct task_struct *child)
4953
{
4954 4955
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
4956
	unsigned long flags;
4957

4958 4959
	if (likely(!child->perf_event_ctxp)) {
		perf_event_task(child, NULL, 0);
4960
		return;
P
Peter Zijlstra 已提交
4961
	}
4962

4963
	local_irq_save(flags);
4964 4965 4966 4967 4968 4969
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
4970 4971
	child_ctx = child->perf_event_ctxp;
	__perf_event_task_sched_out(child_ctx);
4972 4973 4974

	/*
	 * Take the context lock here so that if find_get_context is
4975
	 * reading child->perf_event_ctxp, we wait until it has
4976 4977 4978
	 * incremented the context's refcount before we do put_ctx below.
	 */
	spin_lock(&child_ctx->lock);
4979
	child->perf_event_ctxp = NULL;
4980 4981 4982
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
4983
	 * the events from it.
4984 4985
	 */
	unclone_ctx(child_ctx);
4986
	update_context_time(child_ctx);
P
Peter Zijlstra 已提交
4987 4988 4989
	spin_unlock_irqrestore(&child_ctx->lock, flags);

	/*
4990 4991 4992
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
4993
	 */
4994
	perf_event_task(child, child_ctx, 0);
4995

4996 4997 4998
	/*
	 * We can recurse on the same lock type through:
	 *
4999 5000 5001
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
5002 5003 5004 5005 5006 5007
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
	mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
5008

5009
again:
5010
	list_for_each_entry_safe(child_event, tmp, &child_ctx->group_list,
5011
				 group_entry)
5012
		__perf_event_exit_task(child_event, child_ctx, child);
5013 5014

	/*
5015
	 * If the last event was a group event, it will have appended all
5016 5017 5018
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
5019
	if (!list_empty(&child_ctx->group_list))
5020
		goto again;
5021 5022 5023 5024

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
5025 5026
}

5027 5028 5029 5030
/*
 * free an unexposed, unused context as created by inheritance by
 * init_task below, used by fork() in case of fail.
 */
5031
void perf_event_free_task(struct task_struct *task)
5032
{
5033 5034
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event *event, *tmp;
5035 5036 5037 5038 5039 5040

	if (!ctx)
		return;

	mutex_lock(&ctx->mutex);
again:
5041 5042
	list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry) {
		struct perf_event *parent = event->parent;
5043 5044 5045 5046 5047

		if (WARN_ON_ONCE(!parent))
			continue;

		mutex_lock(&parent->child_mutex);
5048
		list_del_init(&event->child_list);
5049 5050 5051 5052
		mutex_unlock(&parent->child_mutex);

		fput(parent->filp);

5053 5054
		list_del_event(event, ctx);
		free_event(event);
5055 5056
	}

5057
	if (!list_empty(&ctx->group_list))
5058 5059 5060 5061 5062 5063 5064
		goto again;

	mutex_unlock(&ctx->mutex);

	put_ctx(ctx);
}

5065
/*
5066
 * Initialize the perf_event context in task_struct
5067
 */
5068
int perf_event_init_task(struct task_struct *child)
5069
{
5070 5071 5072
	struct perf_event_context *child_ctx, *parent_ctx;
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
5073
	struct task_struct *parent = current;
5074
	int inherited_all = 1;
5075
	int ret = 0;
5076

5077
	child->perf_event_ctxp = NULL;
5078

5079 5080
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);
5081

5082
	if (likely(!parent->perf_event_ctxp))
5083 5084
		return 0;

5085 5086
	/*
	 * This is executed from the parent task context, so inherit
5087
	 * events that have been marked for cloning.
5088
	 * First allocate and initialize a context for the child.
5089 5090
	 */

5091
	child_ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL);
5092
	if (!child_ctx)
5093
		return -ENOMEM;
5094

5095 5096
	__perf_event_init_context(child_ctx, child);
	child->perf_event_ctxp = child_ctx;
5097
	get_task_struct(child);
5098

5099
	/*
5100 5101
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
5102
	 */
5103 5104
	parent_ctx = perf_pin_task_context(parent);

5105 5106 5107 5108 5109 5110 5111
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

5112 5113 5114 5115
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
5116
	mutex_lock(&parent_ctx->mutex);
5117 5118 5119 5120 5121

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
5122
	list_for_each_entry(event, &parent_ctx->group_list, group_entry) {
5123

5124
		if (!event->attr.inherit) {
5125
			inherited_all = 0;
5126
			continue;
5127
		}
5128

5129
		ret = inherit_group(event, parent, parent_ctx,
5130 5131
					     child, child_ctx);
		if (ret) {
5132
			inherited_all = 0;
5133
			break;
5134 5135 5136 5137 5138 5139 5140
		}
	}

	if (inherited_all) {
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
5141 5142
		 * Note that if the parent is a clone, it could get
		 * uncloned at any point, but that doesn't matter
5143
		 * because the list of events and the generation
5144
		 * count can't have changed since we took the mutex.
5145
		 */
5146 5147 5148
		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
5149
			child_ctx->parent_gen = parent_ctx->parent_gen;
5150 5151 5152 5153 5154
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
5155 5156
	}

5157
	mutex_unlock(&parent_ctx->mutex);
5158

5159
	perf_unpin_context(parent_ctx);
5160

5161
	return ret;
5162 5163
}

5164
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
5165
{
5166
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
5167

5168
	cpuctx = &per_cpu(perf_cpu_context, cpu);
5169
	__perf_event_init_context(&cpuctx->ctx, NULL);
T
Thomas Gleixner 已提交
5170

5171
	spin_lock(&perf_resource_lock);
5172
	cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5173
	spin_unlock(&perf_resource_lock);
5174

5175
	hw_perf_event_setup(cpu);
T
Thomas Gleixner 已提交
5176 5177 5178
}

#ifdef CONFIG_HOTPLUG_CPU
5179
static void __perf_event_exit_cpu(void *info)
T
Thomas Gleixner 已提交
5180 5181
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5182 5183
	struct perf_event_context *ctx = &cpuctx->ctx;
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
5184

5185 5186
	list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry)
		__perf_event_remove_from_context(event);
T
Thomas Gleixner 已提交
5187
}
5188
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
5189
{
5190
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5191
	struct perf_event_context *ctx = &cpuctx->ctx;
5192 5193

	mutex_lock(&ctx->mutex);
5194
	smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5195
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
5196 5197
}
#else
5198
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209
#endif

static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action) {

	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5210
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
5211 5212
		break;

5213 5214
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
5215
		hw_perf_event_setup_online(cpu);
5216 5217
		break;

T
Thomas Gleixner 已提交
5218 5219
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
5220
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
5221 5222 5223 5224 5225 5226 5227 5228 5229
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

5230 5231 5232
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
T
Thomas Gleixner 已提交
5233 5234
static struct notifier_block __cpuinitdata perf_cpu_nb = {
	.notifier_call		= perf_cpu_notify,
5235
	.priority		= 20,
T
Thomas Gleixner 已提交
5236 5237
};

5238
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
5239 5240 5241
{
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
			(void *)(long)smp_processor_id());
5242 5243
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
			(void *)(long)smp_processor_id());
T
Thomas Gleixner 已提交
5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263
	register_cpu_notifier(&perf_cpu_nb);
}

static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_reserved_percpu);
}

static ssize_t
perf_set_reserve_percpu(struct sysdev_class *class,
			const char *buf,
			size_t count)
{
	struct perf_cpu_context *cpuctx;
	unsigned long val;
	int err, cpu, mpt;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
5264
	if (val > perf_max_events)
T
Thomas Gleixner 已提交
5265 5266
		return -EINVAL;

5267
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5268 5269 5270 5271
	perf_reserved_percpu = val;
	for_each_online_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
		spin_lock_irq(&cpuctx->ctx.lock);
5272 5273
		mpt = min(perf_max_events - cpuctx->ctx.nr_events,
			  perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
5274 5275 5276
		cpuctx->max_pertask = mpt;
		spin_unlock_irq(&cpuctx->ctx.lock);
	}
5277
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298

	return count;
}

static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_overcommit);
}

static ssize_t
perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
{
	unsigned long val;
	int err;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
	if (val > 1)
		return -EINVAL;

5299
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5300
	perf_overcommit = val;
5301
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327

	return count;
}

static SYSDEV_CLASS_ATTR(
				reserve_percpu,
				0644,
				perf_show_reserve_percpu,
				perf_set_reserve_percpu
			);

static SYSDEV_CLASS_ATTR(
				overcommit,
				0644,
				perf_show_overcommit,
				perf_set_overcommit
			);

static struct attribute *perfclass_attrs[] = {
	&attr_reserve_percpu.attr,
	&attr_overcommit.attr,
	NULL
};

static struct attribute_group perfclass_attr_group = {
	.attrs			= perfclass_attrs,
5328
	.name			= "perf_events",
T
Thomas Gleixner 已提交
5329 5330
};

5331
static int __init perf_event_sysfs_init(void)
T
Thomas Gleixner 已提交
5332 5333 5334 5335
{
	return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
				  &perfclass_attr_group);
}
5336
device_initcall(perf_event_sysfs_init);