perf_event.c 120.9 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
16
#include <linux/file.h>
T
Thomas Gleixner 已提交
17 18
#include <linux/poll.h>
#include <linux/sysfs.h>
19
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
20
#include <linux/percpu.h>
21
#include <linux/ptrace.h>
22
#include <linux/vmstat.h>
23
#include <linux/vmalloc.h>
24 25
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
26 27 28
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
29
#include <linux/kernel_stat.h>
30
#include <linux/perf_event.h>
L
Li Zefan 已提交
31
#include <linux/ftrace_event.h>
32
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
33

34 35
#include <asm/irq_regs.h>

T
Thomas Gleixner 已提交
36
/*
37
 * Each CPU has a list of per CPU events:
T
Thomas Gleixner 已提交
38 39 40
 */
DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);

41
int perf_max_events __read_mostly = 1;
T
Thomas Gleixner 已提交
42 43 44
static int perf_reserved_percpu __read_mostly;
static int perf_overcommit __read_mostly = 1;

45 46 47 48
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
49

50
/*
51
 * perf event paranoia level:
52 53
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
54
 *   1 - disallow cpu events for unpriv
55
 *   2 - disallow kernel profiling for unpriv
56
 */
57
int sysctl_perf_event_paranoid __read_mostly = 1;
58

59 60
static inline bool perf_paranoid_tracepoint_raw(void)
{
61
	return sysctl_perf_event_paranoid > -1;
62 63
}

64 65
static inline bool perf_paranoid_cpu(void)
{
66
	return sysctl_perf_event_paranoid > 0;
67 68 69 70
}

static inline bool perf_paranoid_kernel(void)
{
71
	return sysctl_perf_event_paranoid > 1;
72 73
}

74
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
75 76

/*
77
 * max perf event sample rate
78
 */
79
int sysctl_perf_event_sample_rate __read_mostly = 100000;
80

81
static atomic64_t perf_event_id;
82

T
Thomas Gleixner 已提交
83
/*
84
 * Lock for (sysadmin-configurable) event reservations:
T
Thomas Gleixner 已提交
85
 */
86
static DEFINE_SPINLOCK(perf_resource_lock);
T
Thomas Gleixner 已提交
87 88 89 90

/*
 * Architecture provided APIs - weak aliases:
 */
91
extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
T
Thomas Gleixner 已提交
92
{
93
	return NULL;
T
Thomas Gleixner 已提交
94 95
}

96 97 98
void __weak hw_perf_disable(void)		{ barrier(); }
void __weak hw_perf_enable(void)		{ barrier(); }

99 100
void __weak hw_perf_event_setup(int cpu)	{ barrier(); }
void __weak hw_perf_event_setup_online(int cpu)	{ barrier(); }
101 102

int __weak
103
hw_perf_group_sched_in(struct perf_event *group_leader,
104
	       struct perf_cpu_context *cpuctx,
105
	       struct perf_event_context *ctx, int cpu)
106 107 108
{
	return 0;
}
T
Thomas Gleixner 已提交
109

110
void __weak perf_event_print_debug(void)	{ }
111

112
static DEFINE_PER_CPU(int, perf_disable_count);
113 114 115

void __perf_disable(void)
{
116
	__get_cpu_var(perf_disable_count)++;
117 118 119 120
}

bool __perf_enable(void)
{
121
	return !--__get_cpu_var(perf_disable_count);
122 123 124 125 126 127 128 129 130 131 132 133 134 135
}

void perf_disable(void)
{
	__perf_disable();
	hw_perf_disable();
}

void perf_enable(void)
{
	if (__perf_enable())
		hw_perf_enable();
}

136
static void get_ctx(struct perf_event_context *ctx)
137
{
138
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
139 140
}

141 142
static void free_ctx(struct rcu_head *head)
{
143
	struct perf_event_context *ctx;
144

145
	ctx = container_of(head, struct perf_event_context, rcu_head);
146 147 148
	kfree(ctx);
}

149
static void put_ctx(struct perf_event_context *ctx)
150
{
151 152 153
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
154 155 156
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
157
	}
158 159
}

160
static void unclone_ctx(struct perf_event_context *ctx)
161 162 163 164 165 166 167
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

168
/*
169
 * If we inherit events we want to return the parent event id
170 171
 * to userspace.
 */
172
static u64 primary_event_id(struct perf_event *event)
173
{
174
	u64 id = event->id;
175

176 177
	if (event->parent)
		id = event->parent->id;
178 179 180 181

	return id;
}

182
/*
183
 * Get the perf_event_context for a task and lock it.
184 185 186
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
187
static struct perf_event_context *
188
perf_lock_task_context(struct task_struct *task, unsigned long *flags)
189
{
190
	struct perf_event_context *ctx;
191 192 193

	rcu_read_lock();
 retry:
194
	ctx = rcu_dereference(task->perf_event_ctxp);
195 196 197 198
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
199
		 * perf_event_task_sched_out, though the
200 201 202 203 204 205 206
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
		spin_lock_irqsave(&ctx->lock, *flags);
207
		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
208 209 210
			spin_unlock_irqrestore(&ctx->lock, *flags);
			goto retry;
		}
211 212 213 214 215

		if (!atomic_inc_not_zero(&ctx->refcount)) {
			spin_unlock_irqrestore(&ctx->lock, *flags);
			ctx = NULL;
		}
216 217 218 219 220 221 222 223 224 225
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
226
static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
227
{
228
	struct perf_event_context *ctx;
229 230 231 232 233 234 235 236 237 238
	unsigned long flags;

	ctx = perf_lock_task_context(task, &flags);
	if (ctx) {
		++ctx->pin_count;
		spin_unlock_irqrestore(&ctx->lock, flags);
	}
	return ctx;
}

239
static void perf_unpin_context(struct perf_event_context *ctx)
240 241 242 243 244 245 246 247 248
{
	unsigned long flags;

	spin_lock_irqsave(&ctx->lock, flags);
	--ctx->pin_count;
	spin_unlock_irqrestore(&ctx->lock, flags);
	put_ctx(ctx);
}

249
/*
250
 * Add a event from the lists for its context.
251 252
 * Must be called with ctx->mutex and ctx->lock held.
 */
253
static void
254
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
255
{
256
	struct perf_event *group_leader = event->group_leader;
257 258

	/*
259 260
	 * Depending on whether it is a standalone or sibling event,
	 * add it straight to the context's event list, or to the group
261 262
	 * leader's sibling list:
	 */
263 264
	if (group_leader == event)
		list_add_tail(&event->group_entry, &ctx->group_list);
P
Peter Zijlstra 已提交
265
	else {
266
		list_add_tail(&event->group_entry, &group_leader->sibling_list);
P
Peter Zijlstra 已提交
267 268
		group_leader->nr_siblings++;
	}
P
Peter Zijlstra 已提交
269

270 271 272
	list_add_rcu(&event->event_entry, &ctx->event_list);
	ctx->nr_events++;
	if (event->attr.inherit_stat)
273
		ctx->nr_stat++;
274 275
}

276
/*
277
 * Remove a event from the lists for its context.
278
 * Must be called with ctx->mutex and ctx->lock held.
279
 */
280
static void
281
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
282
{
283
	struct perf_event *sibling, *tmp;
284

285
	if (list_empty(&event->group_entry))
286
		return;
287 288
	ctx->nr_events--;
	if (event->attr.inherit_stat)
289
		ctx->nr_stat--;
290

291 292
	list_del_init(&event->group_entry);
	list_del_rcu(&event->event_entry);
293

294 295
	if (event->group_leader != event)
		event->group_leader->nr_siblings--;
P
Peter Zijlstra 已提交
296

297
	/*
298 299
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
300 301
	 * to the context list directly:
	 */
302
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
303

304
		list_move_tail(&sibling->group_entry, &ctx->group_list);
305 306 307 308
		sibling->group_leader = sibling;
	}
}

309
static void
310
event_sched_out(struct perf_event *event,
311
		  struct perf_cpu_context *cpuctx,
312
		  struct perf_event_context *ctx)
313
{
314
	if (event->state != PERF_EVENT_STATE_ACTIVE)
315 316
		return;

317 318 319 320
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
321
	}
322 323 324
	event->tstamp_stopped = ctx->time;
	event->pmu->disable(event);
	event->oncpu = -1;
325

326
	if (!is_software_event(event))
327 328
		cpuctx->active_oncpu--;
	ctx->nr_active--;
329
	if (event->attr.exclusive || !cpuctx->active_oncpu)
330 331 332
		cpuctx->exclusive = 0;
}

333
static void
334
group_sched_out(struct perf_event *group_event,
335
		struct perf_cpu_context *cpuctx,
336
		struct perf_event_context *ctx)
337
{
338
	struct perf_event *event;
339

340
	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
341 342
		return;

343
	event_sched_out(group_event, cpuctx, ctx);
344 345 346 347

	/*
	 * Schedule out siblings (if any):
	 */
348 349
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
350

351
	if (group_event->attr.exclusive)
352 353 354
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
355
/*
356
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
357
 *
358
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
359 360
 * remove it from the context list.
 */
361
static void __perf_event_remove_from_context(void *info)
T
Thomas Gleixner 已提交
362 363
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
364 365
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
366 367 368 369 370 371

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
	 */
372
	if (ctx->task && cpuctx->task_ctx != ctx)
T
Thomas Gleixner 已提交
373 374
		return;

375
	spin_lock(&ctx->lock);
376 377
	/*
	 * Protect the list operation against NMI by disabling the
378
	 * events on a global level.
379 380
	 */
	perf_disable();
T
Thomas Gleixner 已提交
381

382
	event_sched_out(event, cpuctx, ctx);
383

384
	list_del_event(event, ctx);
T
Thomas Gleixner 已提交
385 386 387

	if (!ctx->task) {
		/*
388
		 * Allow more per task events with respect to the
T
Thomas Gleixner 已提交
389 390 391
		 * reservation:
		 */
		cpuctx->max_pertask =
392 393
			min(perf_max_events - ctx->nr_events,
			    perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
394 395
	}

396
	perf_enable();
397
	spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
398 399 400 401
}


/*
402
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
403
 *
404
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
405
 *
406
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
407
 * call when the task is on a CPU.
408
 *
409 410
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
411 412
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
413
 * When called from perf_event_exit_task, it's OK because the
414
 * context has been detached from its task.
T
Thomas Gleixner 已提交
415
 */
416
static void perf_event_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
417
{
418
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
419 420 421 422
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
423
		 * Per cpu events are removed via an smp call and
T
Thomas Gleixner 已提交
424 425
		 * the removal is always sucessful.
		 */
426 427 428
		smp_call_function_single(event->cpu,
					 __perf_event_remove_from_context,
					 event, 1);
T
Thomas Gleixner 已提交
429 430 431 432
		return;
	}

retry:
433 434
	task_oncpu_function_call(task, __perf_event_remove_from_context,
				 event);
T
Thomas Gleixner 已提交
435 436 437 438 439

	spin_lock_irq(&ctx->lock);
	/*
	 * If the context is active we need to retry the smp call.
	 */
440
	if (ctx->nr_active && !list_empty(&event->group_entry)) {
T
Thomas Gleixner 已提交
441 442 443 444 445 446
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
447
	 * can remove the event safely, if the call above did not
T
Thomas Gleixner 已提交
448 449
	 * succeed.
	 */
450 451
	if (!list_empty(&event->group_entry)) {
		list_del_event(event, ctx);
T
Thomas Gleixner 已提交
452 453 454 455
	}
	spin_unlock_irq(&ctx->lock);
}

456
static inline u64 perf_clock(void)
457
{
458
	return cpu_clock(smp_processor_id());
459 460 461 462 463
}

/*
 * Update the record of the current time in a context.
 */
464
static void update_context_time(struct perf_event_context *ctx)
465
{
466 467 468 469
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
470 471 472
}

/*
473
 * Update the total_time_enabled and total_time_running fields for a event.
474
 */
475
static void update_event_times(struct perf_event *event)
476
{
477
	struct perf_event_context *ctx = event->ctx;
478 479
	u64 run_end;

480 481
	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
482 483
		return;

484
	event->total_time_enabled = ctx->time - event->tstamp_enabled;
485

486 487
	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
488 489 490
	else
		run_end = ctx->time;

491
	event->total_time_running = run_end - event->tstamp_running;
492 493 494
}

/*
495
 * Update total_time_enabled and total_time_running for all events in a group.
496
 */
497
static void update_group_times(struct perf_event *leader)
498
{
499
	struct perf_event *event;
500

501 502 503
	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
504 505
}

506
/*
507
 * Cross CPU call to disable a performance event
508
 */
509
static void __perf_event_disable(void *info)
510
{
511
	struct perf_event *event = info;
512
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
513
	struct perf_event_context *ctx = event->ctx;
514 515

	/*
516 517
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
518
	 */
519
	if (ctx->task && cpuctx->task_ctx != ctx)
520 521
		return;

522
	spin_lock(&ctx->lock);
523 524

	/*
525
	 * If the event is on, turn it off.
526 527
	 * If it is in error state, leave it in error state.
	 */
528
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
529
		update_context_time(ctx);
530 531 532
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
533
		else
534 535
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
536 537
	}

538
	spin_unlock(&ctx->lock);
539 540 541
}

/*
542
 * Disable a event.
543
 *
544 545
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
546
 * remains valid.  This condition is satisifed when called through
547 548 549 550
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
551
 * is the current context on this CPU and preemption is disabled,
552
 * hence we can't get into perf_event_task_sched_out for this context.
553
 */
554
static void perf_event_disable(struct perf_event *event)
555
{
556
	struct perf_event_context *ctx = event->ctx;
557 558 559 560
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
561
		 * Disable the event on the cpu that it's on
562
		 */
563 564
		smp_call_function_single(event->cpu, __perf_event_disable,
					 event, 1);
565 566 567 568
		return;
	}

 retry:
569
	task_oncpu_function_call(task, __perf_event_disable, event);
570 571 572

	spin_lock_irq(&ctx->lock);
	/*
573
	 * If the event is still active, we need to retry the cross-call.
574
	 */
575
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
576 577 578 579 580 581 582 583
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
584 585 586
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
587
	}
588 589 590 591

	spin_unlock_irq(&ctx->lock);
}

592
static int
593
event_sched_in(struct perf_event *event,
594
		 struct perf_cpu_context *cpuctx,
595
		 struct perf_event_context *ctx,
596 597
		 int cpu)
{
598
	if (event->state <= PERF_EVENT_STATE_OFF)
599 600
		return 0;

601 602
	event->state = PERF_EVENT_STATE_ACTIVE;
	event->oncpu = cpu;	/* TODO: put 'cpu' into cpuctx->cpu */
603 604 605 606 607
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

608 609 610
	if (event->pmu->enable(event)) {
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
611 612 613
		return -EAGAIN;
	}

614
	event->tstamp_running += ctx->time - event->tstamp_stopped;
615

616
	if (!is_software_event(event))
617
		cpuctx->active_oncpu++;
618 619
	ctx->nr_active++;

620
	if (event->attr.exclusive)
621 622
		cpuctx->exclusive = 1;

623 624 625
	return 0;
}

626
static int
627
group_sched_in(struct perf_event *group_event,
628
	       struct perf_cpu_context *cpuctx,
629
	       struct perf_event_context *ctx,
630 631
	       int cpu)
{
632
	struct perf_event *event, *partial_group;
633 634
	int ret;

635
	if (group_event->state == PERF_EVENT_STATE_OFF)
636 637
		return 0;

638
	ret = hw_perf_group_sched_in(group_event, cpuctx, ctx, cpu);
639 640 641
	if (ret)
		return ret < 0 ? ret : 0;

642
	if (event_sched_in(group_event, cpuctx, ctx, cpu))
643 644 645 646 647
		return -EAGAIN;

	/*
	 * Schedule in siblings as one group (if any):
	 */
648 649 650
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event_sched_in(event, cpuctx, ctx, cpu)) {
			partial_group = event;
651 652 653 654 655 656 657 658 659 660 661
			goto group_error;
		}
	}

	return 0;

group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
	 */
662 663
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
664
			break;
665
		event_sched_out(event, cpuctx, ctx);
666
	}
667
	event_sched_out(group_event, cpuctx, ctx);
668 669 670 671

	return -EAGAIN;
}

672
/*
673 674
 * Return 1 for a group consisting entirely of software events,
 * 0 if the group contains any hardware events.
675
 */
676
static int is_software_only_group(struct perf_event *leader)
677
{
678
	struct perf_event *event;
679

680
	if (!is_software_event(leader))
681
		return 0;
P
Peter Zijlstra 已提交
682

683 684
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		if (!is_software_event(event))
685
			return 0;
P
Peter Zijlstra 已提交
686

687 688 689 690
	return 1;
}

/*
691
 * Work out whether we can put this event group on the CPU now.
692
 */
693
static int group_can_go_on(struct perf_event *event,
694 695 696 697
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
698
	 * Groups consisting entirely of software events can always go on.
699
	 */
700
	if (is_software_only_group(event))
701 702 703
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
704
	 * events can go on.
705 706 707 708 709
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
710
	 * events on the CPU, it can't go on.
711
	 */
712
	if (event->attr.exclusive && cpuctx->active_oncpu)
713 714 715 716 717 718 719 720
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

721 722
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
723
{
724 725 726 727
	list_add_event(event, ctx);
	event->tstamp_enabled = ctx->time;
	event->tstamp_running = ctx->time;
	event->tstamp_stopped = ctx->time;
728 729
}

T
Thomas Gleixner 已提交
730
/*
731
 * Cross CPU call to install and enable a performance event
732 733
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
734 735 736 737
 */
static void __perf_install_in_context(void *info)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
738 739 740
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
T
Thomas Gleixner 已提交
741
	int cpu = smp_processor_id();
742
	int err;
T
Thomas Gleixner 已提交
743 744 745 746 747

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
748
	 * Or possibly this is the right context but it isn't
749
	 * on this cpu because it had no events.
T
Thomas Gleixner 已提交
750
	 */
751
	if (ctx->task && cpuctx->task_ctx != ctx) {
752
		if (cpuctx->task_ctx || ctx->task != current)
753 754 755
			return;
		cpuctx->task_ctx = ctx;
	}
T
Thomas Gleixner 已提交
756

757
	spin_lock(&ctx->lock);
758
	ctx->is_active = 1;
759
	update_context_time(ctx);
T
Thomas Gleixner 已提交
760 761 762

	/*
	 * Protect the list operation against NMI by disabling the
763
	 * events on a global level. NOP for non NMI based events.
T
Thomas Gleixner 已提交
764
	 */
765
	perf_disable();
T
Thomas Gleixner 已提交
766

767
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
768

769
	/*
770
	 * Don't put the event on if it is disabled or if
771 772
	 * it is in a group and the group isn't on.
	 */
773 774
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
775 776
		goto unlock;

777
	/*
778 779 780
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
781
	 */
782
	if (!group_can_go_on(event, cpuctx, 1))
783 784
		err = -EEXIST;
	else
785
		err = event_sched_in(event, cpuctx, ctx, cpu);
786

787 788
	if (err) {
		/*
789
		 * This event couldn't go on.  If it is in a group
790
		 * then we have to pull the whole group off.
791
		 * If the event group is pinned then put it in error state.
792
		 */
793
		if (leader != event)
794
			group_sched_out(leader, cpuctx, ctx);
795
		if (leader->attr.pinned) {
796
			update_group_times(leader);
797
			leader->state = PERF_EVENT_STATE_ERROR;
798
		}
799
	}
T
Thomas Gleixner 已提交
800

801
	if (!err && !ctx->task && cpuctx->max_pertask)
T
Thomas Gleixner 已提交
802 803
		cpuctx->max_pertask--;

804
 unlock:
805
	perf_enable();
806

807
	spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
808 809 810
}

/*
811
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
812
 *
813 814
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
815
 *
816
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
817 818
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
819 820
 *
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
821 822
 */
static void
823 824
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
825 826 827 828 829 830
			int cpu)
{
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
831
		 * Per cpu events are installed via an smp call and
T
Thomas Gleixner 已提交
832 833 834
		 * the install is always sucessful.
		 */
		smp_call_function_single(cpu, __perf_install_in_context,
835
					 event, 1);
T
Thomas Gleixner 已提交
836 837 838 839 840
		return;
	}

retry:
	task_oncpu_function_call(task, __perf_install_in_context,
841
				 event);
T
Thomas Gleixner 已提交
842 843 844 845 846

	spin_lock_irq(&ctx->lock);
	/*
	 * we need to retry the smp call.
	 */
847
	if (ctx->is_active && list_empty(&event->group_entry)) {
T
Thomas Gleixner 已提交
848 849 850 851 852 853
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
854
	 * can add the event safely, if it the call above did not
T
Thomas Gleixner 已提交
855 856
	 * succeed.
	 */
857 858
	if (list_empty(&event->group_entry))
		add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
859 860 861
	spin_unlock_irq(&ctx->lock);
}

862
/*
863
 * Put a event into inactive state and update time fields.
864 865 866 867 868 869
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
870 871
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
872
{
873
	struct perf_event *sub;
874

875 876 877 878
	event->state = PERF_EVENT_STATE_INACTIVE;
	event->tstamp_enabled = ctx->time - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry)
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
879 880 881 882
			sub->tstamp_enabled =
				ctx->time - sub->total_time_enabled;
}

883
/*
884
 * Cross CPU call to enable a performance event
885
 */
886
static void __perf_event_enable(void *info)
887
{
888
	struct perf_event *event = info;
889
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
890 891
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
892
	int err;
893

894
	/*
895 896
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
897
	 */
898
	if (ctx->task && cpuctx->task_ctx != ctx) {
899
		if (cpuctx->task_ctx || ctx->task != current)
900 901 902
			return;
		cpuctx->task_ctx = ctx;
	}
903

904
	spin_lock(&ctx->lock);
905
	ctx->is_active = 1;
906
	update_context_time(ctx);
907

908
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
909
		goto unlock;
910
	__perf_event_mark_enabled(event, ctx);
911 912

	/*
913
	 * If the event is in a group and isn't the group leader,
914
	 * then don't put it on unless the group is on.
915
	 */
916
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
917
		goto unlock;
918

919
	if (!group_can_go_on(event, cpuctx, 1)) {
920
		err = -EEXIST;
921
	} else {
922
		perf_disable();
923 924
		if (event == leader)
			err = group_sched_in(event, cpuctx, ctx,
925 926
					     smp_processor_id());
		else
927
			err = event_sched_in(event, cpuctx, ctx,
928
					       smp_processor_id());
929
		perf_enable();
930
	}
931 932 933

	if (err) {
		/*
934
		 * If this event can't go on and it's part of a
935 936
		 * group, then the whole group has to come off.
		 */
937
		if (leader != event)
938
			group_sched_out(leader, cpuctx, ctx);
939
		if (leader->attr.pinned) {
940
			update_group_times(leader);
941
			leader->state = PERF_EVENT_STATE_ERROR;
942
		}
943 944 945
	}

 unlock:
946
	spin_unlock(&ctx->lock);
947 948 949
}

/*
950
 * Enable a event.
951
 *
952 953
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
954
 * remains valid.  This condition is satisfied when called through
955 956
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
957
 */
958
static void perf_event_enable(struct perf_event *event)
959
{
960
	struct perf_event_context *ctx = event->ctx;
961 962 963 964
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
965
		 * Enable the event on the cpu that it's on
966
		 */
967 968
		smp_call_function_single(event->cpu, __perf_event_enable,
					 event, 1);
969 970 971 972
		return;
	}

	spin_lock_irq(&ctx->lock);
973
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
974 975 976
		goto out;

	/*
977 978
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
979 980 981 982
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
983 984
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
985 986 987

 retry:
	spin_unlock_irq(&ctx->lock);
988
	task_oncpu_function_call(task, __perf_event_enable, event);
989 990 991 992

	spin_lock_irq(&ctx->lock);

	/*
993
	 * If the context is active and the event is still off,
994 995
	 * we need to retry the cross-call.
	 */
996
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
997 998 999 1000 1001 1002
		goto retry;

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1003 1004
	if (event->state == PERF_EVENT_STATE_OFF)
		__perf_event_mark_enabled(event, ctx);
1005

1006 1007 1008 1009
 out:
	spin_unlock_irq(&ctx->lock);
}

1010
static int perf_event_refresh(struct perf_event *event, int refresh)
1011
{
1012
	/*
1013
	 * not supported on inherited events
1014
	 */
1015
	if (event->attr.inherit)
1016 1017
		return -EINVAL;

1018 1019
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1020 1021

	return 0;
1022 1023
}

1024
void __perf_event_sched_out(struct perf_event_context *ctx,
1025 1026
			      struct perf_cpu_context *cpuctx)
{
1027
	struct perf_event *event;
1028

1029 1030
	spin_lock(&ctx->lock);
	ctx->is_active = 0;
1031
	if (likely(!ctx->nr_events))
1032
		goto out;
1033
	update_context_time(ctx);
1034

1035
	perf_disable();
1036 1037 1038 1039
	if (ctx->nr_active)
		list_for_each_entry(event, &ctx->group_list, group_entry)
			group_sched_out(event, cpuctx, ctx);

1040
	perf_enable();
1041
 out:
1042 1043 1044
	spin_unlock(&ctx->lock);
}

1045 1046 1047
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1048 1049 1050 1051
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1052
 * in them directly with an fd; we can only enable/disable all
1053
 * events via prctl, or enable/disable all events in a family
1054 1055
 * via ioctl, which will have the same effect on both contexts.
 */
1056 1057
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1058 1059
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1060
		&& ctx1->parent_gen == ctx2->parent_gen
1061
		&& !ctx1->pin_count && !ctx2->pin_count;
1062 1063
}

1064 1065
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1066 1067 1068
{
	u64 value;

1069
	if (!event->attr.inherit_stat)
1070 1071 1072
		return;

	/*
1073
	 * Update the event value, we cannot use perf_event_read()
1074 1075
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1076
	 * we know the event must be on the current CPU, therefore we
1077 1078
	 * don't need to use it.
	 */
1079 1080
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1081 1082
		event->pmu->read(event);
		/* fall-through */
1083

1084 1085
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1086 1087 1088 1089 1090 1091 1092
		break;

	default:
		break;
	}

	/*
1093
	 * In order to keep per-task stats reliable we need to flip the event
1094 1095
	 * values when we flip the contexts.
	 */
1096 1097 1098
	value = atomic64_read(&next_event->count);
	value = atomic64_xchg(&event->count, value);
	atomic64_set(&next_event->count, value);
1099

1100 1101
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1102

1103
	/*
1104
	 * Since we swizzled the values, update the user visible data too.
1105
	 */
1106 1107
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1108 1109 1110 1111 1112
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1113 1114
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1115
{
1116
	struct perf_event *event, *next_event;
1117 1118 1119 1120

	if (!ctx->nr_stat)
		return;

1121 1122
	update_context_time(ctx);

1123 1124
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1125

1126 1127
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1128

1129 1130
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1131

1132
		__perf_event_sync_stat(event, next_event);
1133

1134 1135
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1136 1137 1138
	}
}

T
Thomas Gleixner 已提交
1139
/*
1140
 * Called from scheduler to remove the events of the current task,
T
Thomas Gleixner 已提交
1141 1142
 * with interrupts disabled.
 *
1143
 * We stop each event and update the event value in event->count.
T
Thomas Gleixner 已提交
1144
 *
I
Ingo Molnar 已提交
1145
 * This does not protect us against NMI, but disable()
1146 1147 1148
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
T
Thomas Gleixner 已提交
1149
 */
1150
void perf_event_task_sched_out(struct task_struct *task,
1151
				 struct task_struct *next, int cpu)
T
Thomas Gleixner 已提交
1152 1153
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1154 1155 1156
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1157
	struct pt_regs *regs;
1158
	int do_switch = 1;
T
Thomas Gleixner 已提交
1159

1160
	regs = task_pt_regs(task);
1161
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1162

1163
	if (likely(!ctx || !cpuctx->task_ctx))
T
Thomas Gleixner 已提交
1164 1165
		return;

1166 1167
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1168
	next_ctx = next->perf_event_ctxp;
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
		spin_lock(&ctx->lock);
		spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
		if (context_equiv(ctx, next_ctx)) {
1183 1184
			/*
			 * XXX do we need a memory barrier of sorts
1185
			 * wrt to rcu_dereference() of perf_event_ctxp
1186
			 */
1187 1188
			task->perf_event_ctxp = next_ctx;
			next->perf_event_ctxp = ctx;
1189 1190 1191
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1192

1193
			perf_event_sync_stat(ctx, next_ctx);
1194 1195 1196
		}
		spin_unlock(&next_ctx->lock);
		spin_unlock(&ctx->lock);
1197
	}
1198
	rcu_read_unlock();
1199

1200
	if (do_switch) {
1201
		__perf_event_sched_out(ctx, cpuctx);
1202 1203
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1204 1205
}

1206 1207 1208
/*
 * Called with IRQs disabled
 */
1209
static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1210 1211 1212
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

1213 1214
	if (!cpuctx->task_ctx)
		return;
1215 1216 1217 1218

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1219
	__perf_event_sched_out(ctx, cpuctx);
1220 1221 1222
	cpuctx->task_ctx = NULL;
}

1223 1224 1225
/*
 * Called with IRQs disabled
 */
1226
static void perf_event_cpu_sched_out(struct perf_cpu_context *cpuctx)
1227
{
1228
	__perf_event_sched_out(&cpuctx->ctx, cpuctx);
1229 1230
}

1231
static void
1232
__perf_event_sched_in(struct perf_event_context *ctx,
1233
			struct perf_cpu_context *cpuctx, int cpu)
T
Thomas Gleixner 已提交
1234
{
1235
	struct perf_event *event;
1236
	int can_add_hw = 1;
T
Thomas Gleixner 已提交
1237

1238 1239
	spin_lock(&ctx->lock);
	ctx->is_active = 1;
1240
	if (likely(!ctx->nr_events))
1241
		goto out;
T
Thomas Gleixner 已提交
1242

1243
	ctx->timestamp = perf_clock();
1244

1245
	perf_disable();
1246 1247 1248 1249 1250

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
1251 1252 1253
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF ||
		    !event->attr.pinned)
1254
			continue;
1255
		if (event->cpu != -1 && event->cpu != cpu)
1256 1257
			continue;

1258 1259
		if (group_can_go_on(event, cpuctx, 1))
			group_sched_in(event, cpuctx, ctx, cpu);
1260 1261 1262 1263 1264

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1265 1266 1267
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1268
		}
1269 1270
	}

1271
	list_for_each_entry(event, &ctx->group_list, group_entry) {
1272
		/*
1273 1274
		 * Ignore events in OFF or ERROR state, and
		 * ignore pinned events since we did them already.
1275
		 */
1276 1277
		if (event->state <= PERF_EVENT_STATE_OFF ||
		    event->attr.pinned)
1278 1279
			continue;

1280 1281
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1282
		 * of events:
1283
		 */
1284
		if (event->cpu != -1 && event->cpu != cpu)
T
Thomas Gleixner 已提交
1285 1286
			continue;

1287 1288
		if (group_can_go_on(event, cpuctx, can_add_hw))
			if (group_sched_in(event, cpuctx, ctx, cpu))
1289
				can_add_hw = 0;
T
Thomas Gleixner 已提交
1290
	}
1291
	perf_enable();
1292
 out:
T
Thomas Gleixner 已提交
1293
	spin_unlock(&ctx->lock);
1294 1295 1296
}

/*
1297
 * Called from scheduler to add the events of the current task
1298 1299
 * with interrupts disabled.
 *
1300
 * We restore the event value and then enable it.
1301 1302
 *
 * This does not protect us against NMI, but enable()
1303 1304 1305
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
1306
 */
1307
void perf_event_task_sched_in(struct task_struct *task, int cpu)
1308 1309
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1310
	struct perf_event_context *ctx = task->perf_event_ctxp;
1311

1312 1313
	if (likely(!ctx))
		return;
1314 1315
	if (cpuctx->task_ctx == ctx)
		return;
1316
	__perf_event_sched_in(ctx, cpuctx, cpu);
T
Thomas Gleixner 已提交
1317 1318 1319
	cpuctx->task_ctx = ctx;
}

1320
static void perf_event_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1321
{
1322
	struct perf_event_context *ctx = &cpuctx->ctx;
1323

1324
	__perf_event_sched_in(ctx, cpuctx, cpu);
1325 1326
}

1327 1328
#define MAX_INTERRUPTS (~0ULL)

1329
static void perf_log_throttle(struct perf_event *event, int enable);
1330

1331
static void perf_adjust_period(struct perf_event *event, u64 events)
1332
{
1333
	struct hw_perf_event *hwc = &event->hw;
1334 1335 1336 1337
	u64 period, sample_period;
	s64 delta;

	events *= hwc->sample_period;
1338
	period = div64_u64(events, event->attr.sample_freq);
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
}

1351
static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1352
{
1353 1354
	struct perf_event *event;
	struct hw_perf_event *hwc;
1355
	u64 interrupts, freq;
1356 1357

	spin_lock(&ctx->lock);
1358
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1359
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1360 1361
			continue;

1362
		hwc = &event->hw;
1363 1364 1365

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1366

1367
		/*
1368
		 * unthrottle events on the tick
1369
		 */
1370
		if (interrupts == MAX_INTERRUPTS) {
1371 1372 1373
			perf_log_throttle(event, 1);
			event->pmu->unthrottle(event);
			interrupts = 2*sysctl_perf_event_sample_rate/HZ;
1374 1375
		}

1376
		if (!event->attr.freq || !event->attr.sample_freq)
1377 1378
			continue;

1379 1380 1381
		/*
		 * if the specified freq < HZ then we need to skip ticks
		 */
1382 1383
		if (event->attr.sample_freq < HZ) {
			freq = event->attr.sample_freq;
1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396

			hwc->freq_count += freq;
			hwc->freq_interrupts += interrupts;

			if (hwc->freq_count < HZ)
				continue;

			interrupts = hwc->freq_interrupts;
			hwc->freq_interrupts = 0;
			hwc->freq_count -= HZ;
		} else
			freq = HZ;

1397
		perf_adjust_period(event, freq * interrupts);
1398

1399 1400 1401 1402 1403 1404 1405
		/*
		 * In order to avoid being stalled by an (accidental) huge
		 * sample period, force reset the sample period if we didn't
		 * get any events in this freq period.
		 */
		if (!interrupts) {
			perf_disable();
1406
			event->pmu->disable(event);
1407
			atomic64_set(&hwc->period_left, 0);
1408
			event->pmu->enable(event);
1409 1410
			perf_enable();
		}
1411 1412 1413 1414
	}
	spin_unlock(&ctx->lock);
}

1415
/*
1416
 * Round-robin a context's events:
1417
 */
1418
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1419
{
1420
	struct perf_event *event;
T
Thomas Gleixner 已提交
1421

1422
	if (!ctx->nr_events)
T
Thomas Gleixner 已提交
1423 1424 1425 1426
		return;

	spin_lock(&ctx->lock);
	/*
1427
	 * Rotate the first entry last (works just fine for group events too):
T
Thomas Gleixner 已提交
1428
	 */
1429
	perf_disable();
1430 1431
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		list_move_tail(&event->group_entry, &ctx->group_list);
T
Thomas Gleixner 已提交
1432 1433
		break;
	}
1434
	perf_enable();
T
Thomas Gleixner 已提交
1435 1436

	spin_unlock(&ctx->lock);
1437 1438
}

1439
void perf_event_task_tick(struct task_struct *curr, int cpu)
1440
{
1441
	struct perf_cpu_context *cpuctx;
1442
	struct perf_event_context *ctx;
1443

1444
	if (!atomic_read(&nr_events))
1445 1446 1447
		return;

	cpuctx = &per_cpu(perf_cpu_context, cpu);
1448
	ctx = curr->perf_event_ctxp;
1449

1450
	perf_ctx_adjust_freq(&cpuctx->ctx);
1451
	if (ctx)
1452
		perf_ctx_adjust_freq(ctx);
1453

1454
	perf_event_cpu_sched_out(cpuctx);
1455
	if (ctx)
1456
		__perf_event_task_sched_out(ctx);
T
Thomas Gleixner 已提交
1457

1458
	rotate_ctx(&cpuctx->ctx);
1459 1460
	if (ctx)
		rotate_ctx(ctx);
1461

1462
	perf_event_cpu_sched_in(cpuctx, cpu);
1463
	if (ctx)
1464
		perf_event_task_sched_in(curr, cpu);
T
Thomas Gleixner 已提交
1465 1466
}

1467
/*
1468
 * Enable all of a task's events that have been marked enable-on-exec.
1469 1470
 * This expects task == current.
 */
1471
static void perf_event_enable_on_exec(struct task_struct *task)
1472
{
1473 1474
	struct perf_event_context *ctx;
	struct perf_event *event;
1475 1476 1477 1478
	unsigned long flags;
	int enabled = 0;

	local_irq_save(flags);
1479 1480
	ctx = task->perf_event_ctxp;
	if (!ctx || !ctx->nr_events)
1481 1482
		goto out;

1483
	__perf_event_task_sched_out(ctx);
1484 1485 1486

	spin_lock(&ctx->lock);

1487 1488
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		if (!event->attr.enable_on_exec)
1489
			continue;
1490 1491
		event->attr.enable_on_exec = 0;
		if (event->state >= PERF_EVENT_STATE_INACTIVE)
1492
			continue;
1493
		__perf_event_mark_enabled(event, ctx);
1494 1495 1496 1497
		enabled = 1;
	}

	/*
1498
	 * Unclone this context if we enabled any event.
1499
	 */
1500 1501
	if (enabled)
		unclone_ctx(ctx);
1502 1503 1504

	spin_unlock(&ctx->lock);

1505
	perf_event_task_sched_in(task, smp_processor_id());
1506 1507 1508 1509
 out:
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1510
/*
1511
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1512
 */
1513
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1514
{
1515
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1516 1517
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
I
Ingo Molnar 已提交
1518

1519 1520 1521 1522
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1523 1524
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1525 1526 1527 1528
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

P
Peter Zijlstra 已提交
1529
	spin_lock(&ctx->lock);
P
Peter Zijlstra 已提交
1530
	update_context_time(ctx);
1531
	update_event_times(event);
P
Peter Zijlstra 已提交
1532 1533
	spin_unlock(&ctx->lock);

P
Peter Zijlstra 已提交
1534
	event->pmu->read(event);
T
Thomas Gleixner 已提交
1535 1536
}

1537
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1538 1539
{
	/*
1540 1541
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1542
	 */
1543 1544 1545 1546
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
1547 1548 1549 1550 1551
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

		spin_lock_irqsave(&ctx->lock, flags);
		update_context_time(ctx);
1552
		update_event_times(event);
P
Peter Zijlstra 已提交
1553
		spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1554 1555
	}

1556
	return atomic64_read(&event->count);
T
Thomas Gleixner 已提交
1557 1558
}

1559
/*
1560
 * Initialize the perf_event context in a task_struct:
1561 1562
 */
static void
1563
__perf_event_init_context(struct perf_event_context *ctx,
1564 1565 1566 1567 1568
			    struct task_struct *task)
{
	memset(ctx, 0, sizeof(*ctx));
	spin_lock_init(&ctx->lock);
	mutex_init(&ctx->mutex);
1569
	INIT_LIST_HEAD(&ctx->group_list);
1570 1571 1572 1573 1574
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
	ctx->task = task;
}

1575
static struct perf_event_context *find_get_context(pid_t pid, int cpu)
T
Thomas Gleixner 已提交
1576
{
1577
	struct perf_event_context *ctx;
1578
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
1579
	struct task_struct *task;
1580
	unsigned long flags;
1581
	int err;
T
Thomas Gleixner 已提交
1582 1583

	/*
1584
	 * If cpu is not a wildcard then this is a percpu event:
T
Thomas Gleixner 已提交
1585 1586
	 */
	if (cpu != -1) {
1587
		/* Must be root to operate on a CPU event: */
1588
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
1589 1590 1591 1592 1593 1594
			return ERR_PTR(-EACCES);

		if (cpu < 0 || cpu > num_possible_cpus())
			return ERR_PTR(-EINVAL);

		/*
1595
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
1596 1597 1598 1599 1600 1601 1602 1603
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
		if (!cpu_isset(cpu, cpu_online_map))
			return ERR_PTR(-ENODEV);

		cpuctx = &per_cpu(perf_cpu_context, cpu);
		ctx = &cpuctx->ctx;
1604
		get_ctx(ctx);
T
Thomas Gleixner 已提交
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620

		return ctx;
	}

	rcu_read_lock();
	if (!pid)
		task = current;
	else
		task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

1621
	/*
1622
	 * Can't attach events to a dying task.
1623 1624 1625 1626 1627
	 */
	err = -ESRCH;
	if (task->flags & PF_EXITING)
		goto errout;

T
Thomas Gleixner 已提交
1628
	/* Reuse ptrace permission checks for now. */
1629 1630 1631 1632 1633
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

 retry:
1634
	ctx = perf_lock_task_context(task, &flags);
1635
	if (ctx) {
1636
		unclone_ctx(ctx);
1637
		spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1638 1639
	}

1640
	if (!ctx) {
1641
		ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1642 1643 1644
		err = -ENOMEM;
		if (!ctx)
			goto errout;
1645
		__perf_event_init_context(ctx, task);
1646
		get_ctx(ctx);
1647
		if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1648 1649 1650 1651 1652
			/*
			 * We raced with some other task; use
			 * the context they set.
			 */
			kfree(ctx);
1653
			goto retry;
1654
		}
1655
		get_task_struct(task);
1656 1657
	}

1658
	put_task_struct(task);
T
Thomas Gleixner 已提交
1659
	return ctx;
1660 1661 1662 1663

 errout:
	put_task_struct(task);
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
1664 1665
}

L
Li Zefan 已提交
1666 1667
static void perf_event_free_filter(struct perf_event *event);

1668
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
1669
{
1670
	struct perf_event *event;
P
Peter Zijlstra 已提交
1671

1672 1673 1674
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
1675
	perf_event_free_filter(event);
1676
	kfree(event);
P
Peter Zijlstra 已提交
1677 1678
}

1679
static void perf_pending_sync(struct perf_event *event);
1680

1681
static void free_event(struct perf_event *event)
1682
{
1683
	perf_pending_sync(event);
1684

1685 1686 1687 1688 1689 1690 1691 1692
	if (!event->parent) {
		atomic_dec(&nr_events);
		if (event->attr.mmap)
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
1693
	}
1694

1695 1696 1697
	if (event->output) {
		fput(event->output->filp);
		event->output = NULL;
1698 1699
	}

1700 1701
	if (event->destroy)
		event->destroy(event);
1702

1703 1704
	put_ctx(event->ctx);
	call_rcu(&event->rcu_head, free_event_rcu);
1705 1706
}

T
Thomas Gleixner 已提交
1707 1708 1709 1710 1711
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
{
1712 1713
	struct perf_event *event = file->private_data;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1714 1715 1716

	file->private_data = NULL;

1717
	WARN_ON_ONCE(ctx->parent_ctx);
1718
	mutex_lock(&ctx->mutex);
1719
	perf_event_remove_from_context(event);
1720
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
1721

1722 1723 1724 1725
	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);
1726

1727
	free_event(event);
T
Thomas Gleixner 已提交
1728 1729 1730 1731

	return 0;
}

1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751
int perf_event_release_kernel(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;

	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_event_remove_from_context(event);
	mutex_unlock(&ctx->mutex);

	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);

	free_event(event);

	return 0;
}
EXPORT_SYMBOL_GPL(perf_event_release_kernel);

1752
static int perf_event_read_size(struct perf_event *event)
1753 1754 1755 1756 1757
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

1758
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1759 1760
		size += sizeof(u64);

1761
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1762 1763
		size += sizeof(u64);

1764
	if (event->attr.read_format & PERF_FORMAT_ID)
1765 1766
		entry += sizeof(u64);

1767 1768
	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
1769 1770 1771 1772 1773 1774 1775 1776
		size += sizeof(u64);
	}

	size += entry * nr;

	return size;
}

1777
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
1778
{
1779
	struct perf_event *child;
1780 1781
	u64 total = 0;

1782 1783 1784
	*enabled = 0;
	*running = 0;

1785
	mutex_lock(&event->child_mutex);
1786
	total += perf_event_read(event);
1787 1788 1789 1790 1791 1792
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
1793
		total += perf_event_read(child);
1794 1795 1796
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
1797
	mutex_unlock(&event->child_mutex);
1798 1799 1800

	return total;
}
1801
EXPORT_SYMBOL_GPL(perf_event_read_value);
1802

1803
static int perf_event_read_group(struct perf_event *event,
1804 1805
				   u64 read_format, char __user *buf)
{
1806
	struct perf_event *leader = event->group_leader, *sub;
1807 1808
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
1809
	u64 values[5];
1810
	u64 count, enabled, running;
1811

1812
	mutex_lock(&ctx->mutex);
1813
	count = perf_event_read_value(leader, &enabled, &running);
1814 1815

	values[n++] = 1 + leader->nr_siblings;
1816 1817 1818 1819
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1820 1821 1822
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
1823 1824 1825 1826

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
1827
		goto unlock;
1828

1829
	ret = size;
1830

1831
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
1832
		n = 0;
1833

1834
		values[n++] = perf_event_read_value(sub, &enabled, &running);
1835 1836 1837 1838 1839
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

1840 1841 1842 1843
		if (copy_to_user(buf + size, values, size)) {
			ret = -EFAULT;
			goto unlock;
		}
1844 1845

		ret += size;
1846
	}
1847 1848
unlock:
	mutex_unlock(&ctx->mutex);
1849

1850
	return ret;
1851 1852
}

1853
static int perf_event_read_one(struct perf_event *event,
1854 1855
				 u64 read_format, char __user *buf)
{
1856
	u64 enabled, running;
1857 1858 1859
	u64 values[4];
	int n = 0;

1860 1861 1862 1863 1864
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
1865
	if (read_format & PERF_FORMAT_ID)
1866
		values[n++] = primary_event_id(event);
1867 1868 1869 1870 1871 1872 1873

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
1874
/*
1875
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
1876 1877
 */
static ssize_t
1878
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
1879
{
1880
	u64 read_format = event->attr.read_format;
1881
	int ret;
T
Thomas Gleixner 已提交
1882

1883
	/*
1884
	 * Return end-of-file for a read on a event that is in
1885 1886 1887
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
1888
	if (event->state == PERF_EVENT_STATE_ERROR)
1889 1890
		return 0;

1891
	if (count < perf_event_read_size(event))
1892 1893
		return -ENOSPC;

1894
	WARN_ON_ONCE(event->ctx->parent_ctx);
1895
	if (read_format & PERF_FORMAT_GROUP)
1896
		ret = perf_event_read_group(event, read_format, buf);
1897
	else
1898
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
1899

1900
	return ret;
T
Thomas Gleixner 已提交
1901 1902 1903 1904 1905
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
1906
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
1907

1908
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
1909 1910 1911 1912
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
1913
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
1914
	struct perf_mmap_data *data;
1915
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
1916 1917

	rcu_read_lock();
1918
	data = rcu_dereference(event->data);
P
Peter Zijlstra 已提交
1919
	if (data)
1920
		events = atomic_xchg(&data->poll, 0);
P
Peter Zijlstra 已提交
1921
	rcu_read_unlock();
T
Thomas Gleixner 已提交
1922

1923
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
1924 1925 1926 1927

	return events;
}

1928
static void perf_event_reset(struct perf_event *event)
1929
{
1930 1931 1932
	(void)perf_event_read(event);
	atomic64_set(&event->count, 0);
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
1933 1934
}

1935
/*
1936 1937 1938 1939
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
1940
 */
1941 1942
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
1943
{
1944
	struct perf_event *child;
P
Peter Zijlstra 已提交
1945

1946 1947 1948 1949
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
1950
		func(child);
1951
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
1952 1953
}

1954 1955
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
1956
{
1957 1958
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
1959

1960 1961
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
1962
	event = event->group_leader;
1963

1964 1965 1966 1967
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
1968
	mutex_unlock(&ctx->mutex);
1969 1970
}

1971
static int perf_event_period(struct perf_event *event, u64 __user *arg)
1972
{
1973
	struct perf_event_context *ctx = event->ctx;
1974 1975 1976 1977
	unsigned long size;
	int ret = 0;
	u64 value;

1978
	if (!event->attr.sample_period)
1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
		return -EINVAL;

	size = copy_from_user(&value, arg, sizeof(value));
	if (size != sizeof(value))
		return -EFAULT;

	if (!value)
		return -EINVAL;

	spin_lock_irq(&ctx->lock);
1989 1990
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
1991 1992 1993 1994
			ret = -EINVAL;
			goto unlock;
		}

1995
		event->attr.sample_freq = value;
1996
	} else {
1997 1998
		event->attr.sample_period = value;
		event->hw.sample_period = value;
1999 2000 2001 2002 2003 2004 2005
	}
unlock:
	spin_unlock_irq(&ctx->lock);

	return ret;
}

L
Li Zefan 已提交
2006 2007
static int perf_event_set_output(struct perf_event *event, int output_fd);
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2008

2009 2010
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
2011 2012
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2013
	u32 flags = arg;
2014 2015

	switch (cmd) {
2016 2017
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2018
		break;
2019 2020
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2021
		break;
2022 2023
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2024
		break;
P
Peter Zijlstra 已提交
2025

2026 2027
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2028

2029 2030
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2031

2032 2033
	case PERF_EVENT_IOC_SET_OUTPUT:
		return perf_event_set_output(event, arg);
2034

L
Li Zefan 已提交
2035 2036 2037
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2038
	default:
P
Peter Zijlstra 已提交
2039
		return -ENOTTY;
2040
	}
P
Peter Zijlstra 已提交
2041 2042

	if (flags & PERF_IOC_FLAG_GROUP)
2043
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2044
	else
2045
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2046 2047

	return 0;
2048 2049
}

2050
int perf_event_task_enable(void)
2051
{
2052
	struct perf_event *event;
2053

2054 2055 2056 2057
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2058 2059 2060 2061

	return 0;
}

2062
int perf_event_task_disable(void)
2063
{
2064
	struct perf_event *event;
2065

2066 2067 2068 2069
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2070 2071 2072 2073

	return 0;
}

2074 2075
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2076 2077
#endif

2078
static int perf_event_index(struct perf_event *event)
2079
{
2080
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2081 2082
		return 0;

2083
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2084 2085
}

2086 2087 2088 2089 2090
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2091
void perf_event_update_userpage(struct perf_event *event)
2092
{
2093
	struct perf_event_mmap_page *userpg;
2094
	struct perf_mmap_data *data;
2095 2096

	rcu_read_lock();
2097
	data = rcu_dereference(event->data);
2098 2099 2100 2101
	if (!data)
		goto unlock;

	userpg = data->user_page;
2102

2103 2104 2105 2106 2107
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2108
	++userpg->lock;
2109
	barrier();
2110 2111 2112 2113
	userpg->index = perf_event_index(event);
	userpg->offset = atomic64_read(&event->count);
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		userpg->offset -= atomic64_read(&event->hw.prev_count);
2114

2115 2116
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2117

2118 2119
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2120

2121
	barrier();
2122
	++userpg->lock;
2123
	preempt_enable();
2124
unlock:
2125
	rcu_read_unlock();
2126 2127
}

2128
static unsigned long perf_data_size(struct perf_mmap_data *data)
2129
{
2130 2131
	return data->nr_pages << (PAGE_SHIFT + data->data_order);
}
2132

2133
#ifndef CONFIG_PERF_USE_VMALLOC
2134

2135 2136 2137
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2138

2139 2140 2141 2142 2143
static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > data->nr_pages)
		return NULL;
2144

2145 2146
	if (pgoff == 0)
		return virt_to_page(data->user_page);
2147

2148
	return virt_to_page(data->data_pages[pgoff - 1]);
2149 2150
}

2151 2152
static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2153 2154 2155 2156 2157
{
	struct perf_mmap_data *data;
	unsigned long size;
	int i;

2158
	WARN_ON(atomic_read(&event->mmap_count));
2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176

	size = sizeof(struct perf_mmap_data);
	size += nr_pages * sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
	if (!data->user_page)
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
		data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
		if (!data->data_pages[i])
			goto fail_data_pages;
	}

2177
	data->data_order = 0;
2178 2179
	data->nr_pages = nr_pages;

2180
	return data;
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191

fail_data_pages:
	for (i--; i >= 0; i--)
		free_page((unsigned long)data->data_pages[i]);

	free_page((unsigned long)data->user_page);

fail_user_page:
	kfree(data);

fail:
2192
	return NULL;
2193 2194
}

2195 2196
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2197
	struct page *page = virt_to_page((void *)addr);
2198 2199 2200 2201 2202

	page->mapping = NULL;
	__free_page(page);
}

2203
static void perf_mmap_data_free(struct perf_mmap_data *data)
2204 2205 2206
{
	int i;

2207
	perf_mmap_free_page((unsigned long)data->user_page);
2208
	for (i = 0; i < data->nr_pages; i++)
2209
		perf_mmap_free_page((unsigned long)data->data_pages[i]);
2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264
}

#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > (1UL << data->data_order))
		return NULL;

	return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

static void perf_mmap_data_free_work(struct work_struct *work)
{
	struct perf_mmap_data *data;
	void *base;
	int i, nr;

	data = container_of(work, struct perf_mmap_data, work);
	nr = 1 << data->data_order;

	base = data->user_page;
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
}

static void perf_mmap_data_free(struct perf_mmap_data *data)
{
	schedule_work(&data->work);
}

static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
{
	struct perf_mmap_data *data;
	unsigned long size;
	void *all_buf;

	WARN_ON(atomic_read(&event->mmap_count));
2265

2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
	size = sizeof(struct perf_mmap_data);
	size += sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	INIT_WORK(&data->work, perf_mmap_data_free_work);

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

	data->user_page = all_buf;
	data->data_pages[0] = all_buf + PAGE_SIZE;
	data->data_order = ilog2(nr_pages);
	data->nr_pages = 1;

	return data;

fail_all_buf:
	kfree(data);

fail:
	return NULL;
}

#endif

static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
	struct perf_mmap_data *data;
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	data = rcu_dereference(event->data);
	if (!data)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(data, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

static void
perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
{
	long max_size = perf_data_size(data);

	atomic_set(&data->lock, -1);

	if (event->attr.watermark) {
		data->watermark = min_t(long, max_size,
					event->attr.wakeup_watermark);
	}

	if (!data->watermark)
2343
		data->watermark = max_size / 2;
2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354


	rcu_assign_pointer(event->data, data);
}

static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
{
	struct perf_mmap_data *data;

	data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
	perf_mmap_data_free(data);
2355 2356 2357
	kfree(data);
}

2358
static void perf_mmap_data_release(struct perf_event *event)
2359
{
2360
	struct perf_mmap_data *data = event->data;
2361

2362
	WARN_ON(atomic_read(&event->mmap_count));
2363

2364
	rcu_assign_pointer(event->data, NULL);
2365
	call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2366 2367 2368 2369
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
2370
	struct perf_event *event = vma->vm_file->private_data;
2371

2372
	atomic_inc(&event->mmap_count);
2373 2374 2375 2376
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
2377
	struct perf_event *event = vma->vm_file->private_data;
2378

2379 2380
	WARN_ON_ONCE(event->ctx->parent_ctx);
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2381
		unsigned long size = perf_data_size(event->data);
2382 2383
		struct user_struct *user = current_user();

2384
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2385
		vma->vm_mm->locked_vm -= event->data->nr_locked;
2386
		perf_mmap_data_release(event);
2387
		mutex_unlock(&event->mmap_mutex);
2388
	}
2389 2390
}

2391
static const struct vm_operations_struct perf_mmap_vmops = {
2392 2393 2394 2395
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
2396 2397 2398 2399
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
2400
	struct perf_event *event = file->private_data;
2401
	unsigned long user_locked, user_lock_limit;
2402
	struct user_struct *user = current_user();
2403
	unsigned long locked, lock_limit;
2404
	struct perf_mmap_data *data;
2405 2406
	unsigned long vma_size;
	unsigned long nr_pages;
2407
	long user_extra, extra;
2408
	int ret = 0;
2409

2410
	if (!(vma->vm_flags & VM_SHARED))
2411
		return -EINVAL;
2412 2413 2414 2415

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

2416 2417 2418 2419 2420
	/*
	 * If we have data pages ensure they're a power-of-two number, so we
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
2421 2422
		return -EINVAL;

2423
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
2424 2425
		return -EINVAL;

2426 2427
	if (vma->vm_pgoff != 0)
		return -EINVAL;
2428

2429 2430 2431
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
	if (event->output) {
2432 2433 2434 2435
		ret = -EINVAL;
		goto unlock;
	}

2436 2437
	if (atomic_inc_not_zero(&event->mmap_count)) {
		if (nr_pages != event->data->nr_pages)
2438 2439 2440 2441
			ret = -EINVAL;
		goto unlock;
	}

2442
	user_extra = nr_pages + 1;
2443
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
2444 2445 2446 2447 2448 2449

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

2450
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2451

2452 2453 2454
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
2455 2456 2457

	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
	lock_limit >>= PAGE_SHIFT;
2458
	locked = vma->vm_mm->locked_vm + extra;
2459

2460 2461
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
2462 2463 2464
		ret = -EPERM;
		goto unlock;
	}
2465

2466
	WARN_ON(event->data);
2467 2468 2469 2470

	data = perf_mmap_data_alloc(event, nr_pages);
	ret = -ENOMEM;
	if (!data)
2471 2472
		goto unlock;

2473 2474 2475
	ret = 0;
	perf_mmap_data_init(event, data);

2476
	atomic_set(&event->mmap_count, 1);
2477
	atomic_long_add(user_extra, &user->locked_vm);
2478
	vma->vm_mm->locked_vm += extra;
2479
	event->data->nr_locked = extra;
2480
	if (vma->vm_flags & VM_WRITE)
2481
		event->data->writable = 1;
2482

2483
unlock:
2484
	mutex_unlock(&event->mmap_mutex);
2485 2486 2487

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
2488 2489

	return ret;
2490 2491
}

P
Peter Zijlstra 已提交
2492 2493 2494
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
2495
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
2496 2497 2498
	int retval;

	mutex_lock(&inode->i_mutex);
2499
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
2500 2501 2502 2503 2504 2505 2506 2507
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
2508 2509 2510 2511
static const struct file_operations perf_fops = {
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
2512 2513
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
2514
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
2515
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
2516 2517
};

2518
/*
2519
 * Perf event wakeup
2520 2521 2522 2523 2524
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

2525
void perf_event_wakeup(struct perf_event *event)
2526
{
2527
	wake_up_all(&event->waitq);
2528

2529 2530 2531
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
2532
	}
2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543
}

/*
 * Pending wakeups
 *
 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
 *
 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
 * single linked list and use cmpxchg() to add entries lockless.
 */

2544
static void perf_pending_event(struct perf_pending_entry *entry)
2545
{
2546 2547
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
2548

2549 2550 2551
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
2552 2553
	}

2554 2555 2556
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
2557 2558 2559
	}
}

2560
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2561

2562
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2563 2564 2565
	PENDING_TAIL,
};

2566 2567
static void perf_pending_queue(struct perf_pending_entry *entry,
			       void (*func)(struct perf_pending_entry *))
2568
{
2569
	struct perf_pending_entry **head;
2570

2571
	if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2572 2573
		return;

2574 2575 2576
	entry->func = func;

	head = &get_cpu_var(perf_pending_head);
2577 2578

	do {
2579 2580
		entry->next = *head;
	} while (cmpxchg(head, entry->next, entry) != entry->next);
2581

2582
	set_perf_event_pending();
2583

2584
	put_cpu_var(perf_pending_head);
2585 2586 2587 2588
}

static int __perf_pending_run(void)
{
2589
	struct perf_pending_entry *list;
2590 2591
	int nr = 0;

2592
	list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2593
	while (list != PENDING_TAIL) {
2594 2595
		void (*func)(struct perf_pending_entry *);
		struct perf_pending_entry *entry = list;
2596 2597 2598

		list = list->next;

2599 2600
		func = entry->func;
		entry->next = NULL;
2601 2602 2603 2604 2605 2606 2607
		/*
		 * Ensure we observe the unqueue before we issue the wakeup,
		 * so that we won't be waiting forever.
		 * -- see perf_not_pending().
		 */
		smp_wmb();

2608
		func(entry);
2609 2610 2611 2612 2613 2614
		nr++;
	}

	return nr;
}

2615
static inline int perf_not_pending(struct perf_event *event)
2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629
{
	/*
	 * If we flush on whatever cpu we run, there is a chance we don't
	 * need to wait.
	 */
	get_cpu();
	__perf_pending_run();
	put_cpu();

	/*
	 * Ensure we see the proper queue state before going to sleep
	 * so that we do not miss the wakeup. -- see perf_pending_handle()
	 */
	smp_rmb();
2630
	return event->pending.next == NULL;
2631 2632
}

2633
static void perf_pending_sync(struct perf_event *event)
2634
{
2635
	wait_event(event->waitq, perf_not_pending(event));
2636 2637
}

2638
void perf_event_do_pending(void)
2639 2640 2641 2642
{
	__perf_pending_run();
}

2643 2644 2645 2646
/*
 * Callchain support -- arch specific
 */

2647
__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2648 2649 2650 2651
{
	return NULL;
}

2652 2653 2654
/*
 * Output
 */
2655 2656
static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
			      unsigned long offset, unsigned long head)
2657 2658 2659 2660 2661 2662
{
	unsigned long mask;

	if (!data->writable)
		return true;

2663
	mask = perf_data_size(data) - 1;
2664 2665 2666 2667 2668 2669 2670 2671 2672 2673

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

2674
static void perf_output_wakeup(struct perf_output_handle *handle)
2675
{
2676 2677
	atomic_set(&handle->data->poll, POLL_IN);

2678
	if (handle->nmi) {
2679 2680 2681
		handle->event->pending_wakeup = 1;
		perf_pending_queue(&handle->event->pending,
				   perf_pending_event);
2682
	} else
2683
		perf_event_wakeup(handle->event);
2684 2685
}

2686 2687 2688
/*
 * Curious locking construct.
 *
2689 2690
 * We need to ensure a later event_id doesn't publish a head when a former
 * event_id isn't done writing. However since we need to deal with NMIs we
2691 2692 2693 2694 2695 2696
 * cannot fully serialize things.
 *
 * What we do is serialize between CPUs so we only have to deal with NMI
 * nesting on a single CPU.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
2697
 * event_id completes.
2698 2699 2700 2701
 */
static void perf_output_lock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2702
	int cur, cpu = get_cpu();
2703 2704 2705

	handle->locked = 0;

2706 2707 2708 2709 2710 2711 2712 2713
	for (;;) {
		cur = atomic_cmpxchg(&data->lock, -1, cpu);
		if (cur == -1) {
			handle->locked = 1;
			break;
		}
		if (cur == cpu)
			break;
2714 2715

		cpu_relax();
2716
	}
2717 2718 2719 2720 2721
}

static void perf_output_unlock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2722 2723
	unsigned long head;
	int cpu;
2724

2725
	data->done_head = data->head;
2726 2727 2728 2729 2730 2731 2732 2733 2734 2735

	if (!handle->locked)
		goto out;

again:
	/*
	 * The xchg implies a full barrier that ensures all writes are done
	 * before we publish the new head, matched by a rmb() in userspace when
	 * reading this position.
	 */
2736
	while ((head = atomic_long_xchg(&data->done_head, 0)))
2737 2738 2739
		data->user_page->data_head = head;

	/*
2740
	 * NMI can happen here, which means we can miss a done_head update.
2741 2742
	 */

2743
	cpu = atomic_xchg(&data->lock, -1);
2744 2745 2746 2747 2748
	WARN_ON_ONCE(cpu != smp_processor_id());

	/*
	 * Therefore we have to validate we did not indeed do so.
	 */
2749
	if (unlikely(atomic_long_read(&data->done_head))) {
2750 2751 2752
		/*
		 * Since we had it locked, we can lock it again.
		 */
2753
		while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2754 2755 2756 2757 2758
			cpu_relax();

		goto again;
	}

2759
	if (atomic_xchg(&data->wakeup, 0))
2760 2761
		perf_output_wakeup(handle);
out:
2762
	put_cpu();
2763 2764
}

2765 2766
void perf_output_copy(struct perf_output_handle *handle,
		      const void *buf, unsigned int len)
2767 2768
{
	unsigned int pages_mask;
2769
	unsigned long offset;
2770 2771 2772 2773 2774 2775 2776 2777
	unsigned int size;
	void **pages;

	offset		= handle->offset;
	pages_mask	= handle->data->nr_pages - 1;
	pages		= handle->data->data_pages;

	do {
2778 2779
		unsigned long page_offset;
		unsigned long page_size;
2780 2781 2782
		int nr;

		nr	    = (offset >> PAGE_SHIFT) & pages_mask;
2783 2784 2785
		page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
		page_offset = offset & (page_size - 1);
		size	    = min_t(unsigned int, page_size - page_offset, len);
2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802

		memcpy(pages[nr] + page_offset, buf, size);

		len	    -= size;
		buf	    += size;
		offset	    += size;
	} while (len);

	handle->offset = offset;

	/*
	 * Check we didn't copy past our reservation window, taking the
	 * possible unsigned int wrap into account.
	 */
	WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
}

2803
int perf_output_begin(struct perf_output_handle *handle,
2804
		      struct perf_event *event, unsigned int size,
2805
		      int nmi, int sample)
2806
{
2807
	struct perf_event *output_event;
2808
	struct perf_mmap_data *data;
2809
	unsigned long tail, offset, head;
2810 2811 2812 2813 2814 2815
	int have_lost;
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
2816

2817
	rcu_read_lock();
2818
	/*
2819
	 * For inherited events we send all the output towards the parent.
2820
	 */
2821 2822
	if (event->parent)
		event = event->parent;
2823

2824 2825 2826
	output_event = rcu_dereference(event->output);
	if (output_event)
		event = output_event;
2827

2828
	data = rcu_dereference(event->data);
2829 2830 2831
	if (!data)
		goto out;

2832
	handle->data	= data;
2833
	handle->event	= event;
2834 2835
	handle->nmi	= nmi;
	handle->sample	= sample;
2836

2837
	if (!data->nr_pages)
2838
		goto fail;
2839

2840 2841 2842 2843
	have_lost = atomic_read(&data->lost);
	if (have_lost)
		size += sizeof(lost_event);

2844 2845
	perf_output_lock(handle);

2846
	do {
2847 2848 2849 2850 2851 2852 2853
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
		tail = ACCESS_ONCE(data->user_page->data_tail);
		smp_rmb();
2854
		offset = head = atomic_long_read(&data->head);
P
Peter Zijlstra 已提交
2855
		head += size;
2856
		if (unlikely(!perf_output_space(data, tail, offset, head)))
2857
			goto fail;
2858
	} while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2859

2860
	handle->offset	= offset;
2861
	handle->head	= head;
2862

2863
	if (head - tail > data->watermark)
2864
		atomic_set(&data->wakeup, 1);
2865

2866
	if (have_lost) {
2867
		lost_event.header.type = PERF_RECORD_LOST;
2868 2869
		lost_event.header.misc = 0;
		lost_event.header.size = sizeof(lost_event);
2870
		lost_event.id          = event->id;
2871 2872 2873 2874 2875
		lost_event.lost        = atomic_xchg(&data->lost, 0);

		perf_output_put(handle, lost_event);
	}

2876
	return 0;
2877

2878
fail:
2879 2880
	atomic_inc(&data->lost);
	perf_output_unlock(handle);
2881 2882
out:
	rcu_read_unlock();
2883

2884 2885
	return -ENOSPC;
}
2886

2887
void perf_output_end(struct perf_output_handle *handle)
2888
{
2889
	struct perf_event *event = handle->event;
2890 2891
	struct perf_mmap_data *data = handle->data;

2892
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
2893

2894
	if (handle->sample && wakeup_events) {
2895
		int events = atomic_inc_return(&data->events);
P
Peter Zijlstra 已提交
2896
		if (events >= wakeup_events) {
2897
			atomic_sub(wakeup_events, &data->events);
2898
			atomic_set(&data->wakeup, 1);
P
Peter Zijlstra 已提交
2899
		}
2900 2901 2902
	}

	perf_output_unlock(handle);
2903
	rcu_read_unlock();
2904 2905
}

2906
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
2907 2908
{
	/*
2909
	 * only top level events have the pid namespace they were created in
2910
	 */
2911 2912
	if (event->parent)
		event = event->parent;
2913

2914
	return task_tgid_nr_ns(p, event->ns);
2915 2916
}

2917
static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
2918 2919
{
	/*
2920
	 * only top level events have the pid namespace they were created in
2921
	 */
2922 2923
	if (event->parent)
		event = event->parent;
2924

2925
	return task_pid_nr_ns(p, event->ns);
2926 2927
}

2928
static void perf_output_read_one(struct perf_output_handle *handle,
2929
				 struct perf_event *event)
2930
{
2931
	u64 read_format = event->attr.read_format;
2932 2933 2934
	u64 values[4];
	int n = 0;

2935
	values[n++] = atomic64_read(&event->count);
2936
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2937 2938
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2939 2940
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2941 2942
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2943 2944
	}
	if (read_format & PERF_FORMAT_ID)
2945
		values[n++] = primary_event_id(event);
2946 2947 2948 2949 2950

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
2951
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
2952 2953
 */
static void perf_output_read_group(struct perf_output_handle *handle,
2954
			    struct perf_event *event)
2955
{
2956 2957
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = leader->total_time_enabled;

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = leader->total_time_running;

2969
	if (leader != event)
2970 2971 2972 2973
		leader->pmu->read(leader);

	values[n++] = atomic64_read(&leader->count);
	if (read_format & PERF_FORMAT_ID)
2974
		values[n++] = primary_event_id(leader);
2975 2976 2977

	perf_output_copy(handle, values, n * sizeof(u64));

2978
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2979 2980
		n = 0;

2981
		if (sub != event)
2982 2983 2984 2985
			sub->pmu->read(sub);

		values[n++] = atomic64_read(&sub->count);
		if (read_format & PERF_FORMAT_ID)
2986
			values[n++] = primary_event_id(sub);
2987 2988 2989 2990 2991 2992

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

static void perf_output_read(struct perf_output_handle *handle,
2993
			     struct perf_event *event)
2994
{
2995 2996
	if (event->attr.read_format & PERF_FORMAT_GROUP)
		perf_output_read_group(handle, event);
2997
	else
2998
		perf_output_read_one(handle, event);
2999 3000
}

3001 3002 3003
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3004
			struct perf_event *event)
3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3035
		perf_output_read(handle, event);
3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3073
			 struct perf_event *event,
3074
			 struct pt_regs *regs)
3075
{
3076
	u64 sample_type = event->attr.sample_type;
3077

3078
	data->type = sample_type;
3079

3080
	header->type = PERF_RECORD_SAMPLE;
3081 3082 3083 3084
	header->size = sizeof(*header);

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3085

3086
	if (sample_type & PERF_SAMPLE_IP) {
3087 3088 3089
		data->ip = perf_instruction_pointer(regs);

		header->size += sizeof(data->ip);
3090
	}
3091

3092
	if (sample_type & PERF_SAMPLE_TID) {
3093
		/* namespace issues */
3094 3095
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
3096

3097
		header->size += sizeof(data->tid_entry);
3098 3099
	}

3100
	if (sample_type & PERF_SAMPLE_TIME) {
P
Peter Zijlstra 已提交
3101
		data->time = perf_clock();
3102

3103
		header->size += sizeof(data->time);
3104 3105
	}

3106
	if (sample_type & PERF_SAMPLE_ADDR)
3107
		header->size += sizeof(data->addr);
3108

3109
	if (sample_type & PERF_SAMPLE_ID) {
3110
		data->id = primary_event_id(event);
3111

3112 3113 3114 3115
		header->size += sizeof(data->id);
	}

	if (sample_type & PERF_SAMPLE_STREAM_ID) {
3116
		data->stream_id = event->id;
3117 3118 3119

		header->size += sizeof(data->stream_id);
	}
3120

3121
	if (sample_type & PERF_SAMPLE_CPU) {
3122 3123
		data->cpu_entry.cpu		= raw_smp_processor_id();
		data->cpu_entry.reserved	= 0;
3124

3125
		header->size += sizeof(data->cpu_entry);
3126 3127
	}

3128
	if (sample_type & PERF_SAMPLE_PERIOD)
3129
		header->size += sizeof(data->period);
3130

3131
	if (sample_type & PERF_SAMPLE_READ)
3132
		header->size += perf_event_read_size(event);
3133

3134
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3135
		int size = 1;
3136

3137 3138 3139 3140 3141 3142
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3143 3144
	}

3145
	if (sample_type & PERF_SAMPLE_RAW) {
3146 3147 3148 3149 3150 3151 3152 3153
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3154
		header->size += size;
3155
	}
3156
}
3157

3158
static void perf_event_output(struct perf_event *event, int nmi,
3159 3160 3161 3162 3163
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3164

3165
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3166

3167
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3168
		return;
3169

3170
	perf_output_sample(&handle, &header, data, event);
3171

3172
	perf_output_end(&handle);
3173 3174
}

3175
/*
3176
 * read event_id
3177 3178 3179 3180 3181 3182 3183 3184 3185 3186
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3187
perf_event_read_event(struct perf_event *event,
3188 3189 3190
			struct task_struct *task)
{
	struct perf_output_handle handle;
3191
	struct perf_read_event read_event = {
3192
		.header = {
3193
			.type = PERF_RECORD_READ,
3194
			.misc = 0,
3195
			.size = sizeof(read_event) + perf_event_read_size(event),
3196
		},
3197 3198
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3199
	};
3200
	int ret;
3201

3202
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3203 3204 3205
	if (ret)
		return;

3206
	perf_output_put(&handle, read_event);
3207
	perf_output_read(&handle, event);
3208

3209 3210 3211
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3212
/*
P
Peter Zijlstra 已提交
3213 3214 3215
 * task tracking -- fork/exit
 *
 * enabled by: attr.comm | attr.mmap | attr.task
P
Peter Zijlstra 已提交
3216 3217
 */

P
Peter Zijlstra 已提交
3218
struct perf_task_event {
3219
	struct task_struct		*task;
3220
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3221 3222 3223 3224 3225 3226

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3227 3228
		u32				tid;
		u32				ptid;
3229
		u64				time;
3230
	} event_id;
P
Peter Zijlstra 已提交
3231 3232
};

3233
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3234
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3235 3236
{
	struct perf_output_handle handle;
3237
	int size;
P
Peter Zijlstra 已提交
3238
	struct task_struct *task = task_event->task;
3239 3240
	int ret;

3241 3242
	size  = task_event->event_id.header.size;
	ret = perf_output_begin(&handle, event, size, 0, 0);
P
Peter Zijlstra 已提交
3243 3244 3245 3246

	if (ret)
		return;

3247 3248
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3249

3250 3251
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3252

3253
	task_event->event_id.time = perf_clock();
3254

3255
	perf_output_put(&handle, task_event->event_id);
3256

P
Peter Zijlstra 已提交
3257 3258 3259
	perf_output_end(&handle);
}

3260
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3261
{
3262
	if (event->attr.comm || event->attr.mmap || event->attr.task)
P
Peter Zijlstra 已提交
3263 3264 3265 3266 3267
		return 1;

	return 0;
}

3268
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
3269
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3270
{
3271
	struct perf_event *event;
P
Peter Zijlstra 已提交
3272

3273 3274 3275
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
3276 3277 3278
	}
}

3279
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3280 3281
{
	struct perf_cpu_context *cpuctx;
3282
	struct perf_event_context *ctx = task_event->task_ctx;
P
Peter Zijlstra 已提交
3283

3284
	rcu_read_lock();
P
Peter Zijlstra 已提交
3285
	cpuctx = &get_cpu_var(perf_cpu_context);
3286
	perf_event_task_ctx(&cpuctx->ctx, task_event);
P
Peter Zijlstra 已提交
3287 3288
	put_cpu_var(perf_cpu_context);

3289
	if (!ctx)
3290
		ctx = rcu_dereference(task_event->task->perf_event_ctxp);
P
Peter Zijlstra 已提交
3291
	if (ctx)
3292
		perf_event_task_ctx(ctx, task_event);
P
Peter Zijlstra 已提交
3293 3294 3295
	rcu_read_unlock();
}

3296 3297
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3298
			      int new)
P
Peter Zijlstra 已提交
3299
{
P
Peter Zijlstra 已提交
3300
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
3301

3302 3303 3304
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
3305 3306
		return;

P
Peter Zijlstra 已提交
3307
	task_event = (struct perf_task_event){
3308 3309
		.task	  = task,
		.task_ctx = task_ctx,
3310
		.event_id    = {
P
Peter Zijlstra 已提交
3311
			.header = {
3312
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3313
				.misc = 0,
3314
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
3315
			},
3316 3317
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
3318 3319
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
3320 3321 3322
		},
	};

3323
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
3324 3325
}

3326
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
3327
{
3328
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
3329 3330
}

3331 3332 3333 3334 3335
/*
 * comm tracking
 */

struct perf_comm_event {
3336 3337
	struct task_struct	*task;
	char			*comm;
3338 3339 3340 3341 3342 3343 3344
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
3345
	} event_id;
3346 3347
};

3348
static void perf_event_comm_output(struct perf_event *event,
3349 3350 3351
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
3352 3353
	int size = comm_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3354 3355 3356 3357

	if (ret)
		return;

3358 3359
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3360

3361
	perf_output_put(&handle, comm_event->event_id);
3362 3363 3364 3365 3366
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
	perf_output_end(&handle);
}

3367
static int perf_event_comm_match(struct perf_event *event)
3368
{
3369
	if (event->attr.comm)
3370 3371 3372 3373 3374
		return 1;

	return 0;
}

3375
static void perf_event_comm_ctx(struct perf_event_context *ctx,
3376 3377
				  struct perf_comm_event *comm_event)
{
3378
	struct perf_event *event;
3379

3380 3381 3382
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
3383 3384 3385
	}
}

3386
static void perf_event_comm_event(struct perf_comm_event *comm_event)
3387 3388
{
	struct perf_cpu_context *cpuctx;
3389
	struct perf_event_context *ctx;
3390
	unsigned int size;
3391
	char comm[TASK_COMM_LEN];
3392

3393
	memset(comm, 0, sizeof(comm));
3394
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
3395
	size = ALIGN(strlen(comm)+1, sizeof(u64));
3396 3397 3398 3399

	comm_event->comm = comm;
	comm_event->comm_size = size;

3400
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3401

3402
	rcu_read_lock();
3403
	cpuctx = &get_cpu_var(perf_cpu_context);
3404
	perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3405
	put_cpu_var(perf_cpu_context);
3406 3407 3408 3409 3410

	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3411
	ctx = rcu_dereference(current->perf_event_ctxp);
3412
	if (ctx)
3413
		perf_event_comm_ctx(ctx, comm_event);
3414
	rcu_read_unlock();
3415 3416
}

3417
void perf_event_comm(struct task_struct *task)
3418
{
3419 3420
	struct perf_comm_event comm_event;

3421 3422
	if (task->perf_event_ctxp)
		perf_event_enable_on_exec(task);
3423

3424
	if (!atomic_read(&nr_comm_events))
3425
		return;
3426

3427
	comm_event = (struct perf_comm_event){
3428
		.task	= task,
3429 3430
		/* .comm      */
		/* .comm_size */
3431
		.event_id  = {
3432
			.header = {
3433
				.type = PERF_RECORD_COMM,
3434 3435 3436 3437 3438
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3439 3440 3441
		},
	};

3442
	perf_event_comm_event(&comm_event);
3443 3444
}

3445 3446 3447 3448 3449
/*
 * mmap tracking
 */

struct perf_mmap_event {
3450 3451 3452 3453
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
3454 3455 3456 3457 3458 3459 3460 3461 3462

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
3463
	} event_id;
3464 3465
};

3466
static void perf_event_mmap_output(struct perf_event *event,
3467 3468 3469
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
3470 3471
	int size = mmap_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3472 3473 3474 3475

	if (ret)
		return;

3476 3477
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
3478

3479
	perf_output_put(&handle, mmap_event->event_id);
3480 3481
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
3482
	perf_output_end(&handle);
3483 3484
}

3485
static int perf_event_mmap_match(struct perf_event *event,
3486 3487
				   struct perf_mmap_event *mmap_event)
{
3488
	if (event->attr.mmap)
3489 3490 3491 3492 3493
		return 1;

	return 0;
}

3494
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3495 3496
				  struct perf_mmap_event *mmap_event)
{
3497
	struct perf_event *event;
3498

3499 3500 3501
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_mmap_match(event, mmap_event))
			perf_event_mmap_output(event, mmap_event);
3502 3503 3504
	}
}

3505
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3506 3507
{
	struct perf_cpu_context *cpuctx;
3508
	struct perf_event_context *ctx;
3509 3510
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
3511 3512 3513
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
3514
	const char *name;
3515

3516 3517
	memset(tmp, 0, sizeof(tmp));

3518
	if (file) {
3519 3520 3521 3522 3523 3524
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3525 3526 3527 3528
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
3529
		name = d_path(&file->f_path, buf, PATH_MAX);
3530 3531 3532 3533 3534
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
3535 3536 3537
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
3538
			goto got_name;
3539
		}
3540 3541 3542 3543 3544 3545

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
		}

3546 3547 3548 3549 3550
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
3551
	size = ALIGN(strlen(name)+1, sizeof(u64));
3552 3553 3554 3555

	mmap_event->file_name = name;
	mmap_event->file_size = size;

3556
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3557

3558
	rcu_read_lock();
3559
	cpuctx = &get_cpu_var(perf_cpu_context);
3560
	perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
3561 3562
	put_cpu_var(perf_cpu_context);

3563 3564 3565 3566
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3567
	ctx = rcu_dereference(current->perf_event_ctxp);
3568
	if (ctx)
3569
		perf_event_mmap_ctx(ctx, mmap_event);
3570 3571
	rcu_read_unlock();

3572 3573 3574
	kfree(buf);
}

3575
void __perf_event_mmap(struct vm_area_struct *vma)
3576
{
3577 3578
	struct perf_mmap_event mmap_event;

3579
	if (!atomic_read(&nr_mmap_events))
3580 3581 3582
		return;

	mmap_event = (struct perf_mmap_event){
3583
		.vma	= vma,
3584 3585
		/* .file_name */
		/* .file_size */
3586
		.event_id  = {
3587
			.header = {
3588
				.type = PERF_RECORD_MMAP,
3589 3590 3591 3592 3593
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3594 3595 3596
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
			.pgoff  = vma->vm_pgoff,
3597 3598 3599
		},
	};

3600
	perf_event_mmap_event(&mmap_event);
3601 3602
}

3603 3604 3605 3606
/*
 * IRQ throttle logging
 */

3607
static void perf_log_throttle(struct perf_event *event, int enable)
3608 3609 3610 3611 3612 3613 3614
{
	struct perf_output_handle handle;
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
3615
		u64				id;
3616
		u64				stream_id;
3617 3618
	} throttle_event = {
		.header = {
3619
			.type = PERF_RECORD_THROTTLE,
3620 3621 3622
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
3623
		.time		= perf_clock(),
3624 3625
		.id		= primary_event_id(event),
		.stream_id	= event->id,
3626 3627
	};

3628
	if (enable)
3629
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3630

3631
	ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3632 3633 3634 3635 3636 3637 3638
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
	perf_output_end(&handle);
}

3639
/*
3640
 * Generic event overflow handling, sampling.
3641 3642
 */

3643
static int __perf_event_overflow(struct perf_event *event, int nmi,
3644 3645
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
3646
{
3647 3648
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
3649 3650
	int ret = 0;

3651
	throttle = (throttle && event->pmu->unthrottle != NULL);
3652

3653
	if (!throttle) {
3654
		hwc->interrupts++;
3655
	} else {
3656 3657
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
3658
			if (HZ * hwc->interrupts >
3659
					(u64)sysctl_perf_event_sample_rate) {
3660
				hwc->interrupts = MAX_INTERRUPTS;
3661
				perf_log_throttle(event, 0);
3662 3663 3664 3665
				ret = 1;
			}
		} else {
			/*
3666
			 * Keep re-disabling events even though on the previous
3667
			 * pass we disabled it - just in case we raced with a
3668
			 * sched-in and the event got enabled again:
3669
			 */
3670 3671 3672
			ret = 1;
		}
	}
3673

3674
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
3675
		u64 now = perf_clock();
3676 3677 3678 3679 3680
		s64 delta = now - hwc->freq_stamp;

		hwc->freq_stamp = now;

		if (delta > 0 && delta < TICK_NSEC)
3681
			perf_adjust_period(event, NSEC_PER_SEC / (int)delta);
3682 3683
	}

3684 3685
	/*
	 * XXX event_limit might not quite work as expected on inherited
3686
	 * events
3687 3688
	 */

3689 3690
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
3691
		ret = 1;
3692
		event->pending_kill = POLL_HUP;
3693
		if (nmi) {
3694 3695 3696
			event->pending_disable = 1;
			perf_pending_queue(&event->pending,
					   perf_pending_event);
3697
		} else
3698
			perf_event_disable(event);
3699 3700
	}

3701 3702 3703 3704 3705
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

3706
	return ret;
3707 3708
}

3709
int perf_event_overflow(struct perf_event *event, int nmi,
3710 3711
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
3712
{
3713
	return __perf_event_overflow(event, nmi, 1, data, regs);
3714 3715
}

3716
/*
3717
 * Generic software event infrastructure
3718 3719
 */

3720
/*
3721 3722
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
3723 3724 3725 3726
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

3727
static u64 perf_swevent_set_period(struct perf_event *event)
3728
{
3729
	struct hw_perf_event *hwc = &event->hw;
3730 3731 3732 3733 3734
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
3735 3736

again:
3737 3738 3739
	old = val = atomic64_read(&hwc->period_left);
	if (val < 0)
		return 0;
3740

3741 3742 3743 3744 3745
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
	if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
		goto again;
3746

3747
	return nr;
3748 3749
}

3750
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3751 3752
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
3753
{
3754
	struct hw_perf_event *hwc = &event->hw;
3755
	int throttle = 0;
3756

3757
	data->period = event->hw.last_period;
3758 3759
	if (!overflow)
		overflow = perf_swevent_set_period(event);
3760

3761 3762
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
3763

3764
	for (; overflow; overflow--) {
3765
		if (__perf_event_overflow(event, nmi, throttle,
3766
					    data, regs)) {
3767 3768 3769 3770 3771 3772
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
3773
		throttle = 1;
3774
	}
3775 3776
}

3777
static void perf_swevent_unthrottle(struct perf_event *event)
3778 3779
{
	/*
3780
	 * Nothing to do, we already reset hwc->interrupts.
3781
	 */
3782
}
3783

3784
static void perf_swevent_add(struct perf_event *event, u64 nr,
3785 3786
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
3787
{
3788
	struct hw_perf_event *hwc = &event->hw;
3789

3790
	atomic64_add(nr, &event->count);
3791

3792 3793 3794
	if (!regs)
		return;

3795 3796
	if (!hwc->sample_period)
		return;
3797

3798 3799 3800 3801
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

	if (atomic64_add_negative(nr, &hwc->period_left))
3802
		return;
3803

3804
	perf_swevent_overflow(event, 0, nmi, data, regs);
3805 3806
}

3807
static int perf_swevent_is_counting(struct perf_event *event)
3808
{
3809
	/*
3810
	 * The event is active, we're good!
3811
	 */
3812
	if (event->state == PERF_EVENT_STATE_ACTIVE)
3813 3814
		return 1;

3815
	/*
3816
	 * The event is off/error, not counting.
3817
	 */
3818
	if (event->state != PERF_EVENT_STATE_INACTIVE)
3819 3820 3821
		return 0;

	/*
3822
	 * The event is inactive, if the context is active
3823 3824
	 * we're part of a group that didn't make it on the 'pmu',
	 * not counting.
3825
	 */
3826
	if (event->ctx->is_active)
3827 3828 3829 3830 3831 3832 3833 3834
		return 0;

	/*
	 * We're inactive and the context is too, this means the
	 * task is scheduled out, we're counting events that happen
	 * to us, like migration events.
	 */
	return 1;
3835 3836
}

L
Li Zefan 已提交
3837 3838 3839
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data);

3840
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
3841
				enum perf_type_id type,
L
Li Zefan 已提交
3842 3843 3844
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
3845
{
3846
	if (!perf_swevent_is_counting(event))
3847 3848
		return 0;

3849
	if (event->attr.type != type)
3850
		return 0;
3851
	if (event->attr.config != event_id)
3852 3853
		return 0;

3854
	if (regs) {
3855
		if (event->attr.exclude_user && user_mode(regs))
3856
			return 0;
3857

3858
		if (event->attr.exclude_kernel && !user_mode(regs))
3859 3860
			return 0;
	}
3861

L
Li Zefan 已提交
3862 3863 3864 3865
	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
	    !perf_tp_event_match(event, data))
		return 0;

3866 3867 3868
	return 1;
}

3869
static void perf_swevent_ctx_event(struct perf_event_context *ctx,
3870
				     enum perf_type_id type,
3871
				     u32 event_id, u64 nr, int nmi,
3872 3873
				     struct perf_sample_data *data,
				     struct pt_regs *regs)
3874
{
3875
	struct perf_event *event;
3876

3877
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
L
Li Zefan 已提交
3878
		if (perf_swevent_match(event, type, event_id, data, regs))
3879
			perf_swevent_add(event, nr, nmi, data, regs);
3880 3881 3882
	}
}

3883 3884 3885 3886
/*
 * Must be called with preemption disabled
 */
int perf_swevent_get_recursion_context(int **recursion)
P
Peter Zijlstra 已提交
3887
{
3888 3889
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

P
Peter Zijlstra 已提交
3890
	if (in_nmi())
3891 3892 3893 3894 3895 3896 3897
		*recursion = &cpuctx->recursion[3];
	else if (in_irq())
		*recursion = &cpuctx->recursion[2];
	else if (in_softirq())
		*recursion = &cpuctx->recursion[1];
	else
		*recursion = &cpuctx->recursion[0];
P
Peter Zijlstra 已提交
3898

3899 3900
	if (**recursion)
		return -1;
P
Peter Zijlstra 已提交
3901

3902
	(**recursion)++;
P
Peter Zijlstra 已提交
3903

3904
	return 0;
P
Peter Zijlstra 已提交
3905
}
I
Ingo Molnar 已提交
3906
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
3907

3908
void perf_swevent_put_recursion_context(int *recursion)
3909
{
3910 3911
	(*recursion)--;
}
I
Ingo Molnar 已提交
3912
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
P
Peter Zijlstra 已提交
3913

3914 3915 3916 3917 3918 3919 3920
static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
			       u64 nr, int nmi,
			       struct perf_sample_data *data,
			       struct pt_regs *regs)
{
	struct perf_event_context *ctx;
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3921

3922
	rcu_read_lock();
3923
	perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
3924
				 nr, nmi, data, regs);
3925 3926 3927 3928
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3929
	ctx = rcu_dereference(current->perf_event_ctxp);
3930
	if (ctx)
3931
		perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs);
3932
	rcu_read_unlock();
3933
}
3934

3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
				    u64 nr, int nmi,
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
{
	int *recursion;

	preempt_disable();

	if (perf_swevent_get_recursion_context(&recursion))
		goto out;

	__do_perf_sw_event(type, event_id, nr, nmi, data, regs);
P
Peter Zijlstra 已提交
3948

3949
	perf_swevent_put_recursion_context(recursion);
P
Peter Zijlstra 已提交
3950
out:
3951
	preempt_enable();
3952 3953
}

3954
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
3955
			    struct pt_regs *regs, u64 addr)
3956
{
3957 3958 3959 3960
	struct perf_sample_data data;

	data.addr = addr;
	data.raw  = NULL;
3961

3962
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
3963 3964
}

3965
static void perf_swevent_read(struct perf_event *event)
3966 3967 3968
{
}

3969
static int perf_swevent_enable(struct perf_event *event)
3970
{
3971
	struct hw_perf_event *hwc = &event->hw;
3972 3973 3974

	if (hwc->sample_period) {
		hwc->last_period = hwc->sample_period;
3975
		perf_swevent_set_period(event);
3976
	}
3977 3978 3979
	return 0;
}

3980
static void perf_swevent_disable(struct perf_event *event)
3981 3982 3983
{
}

3984
static const struct pmu perf_ops_generic = {
3985 3986 3987 3988
	.enable		= perf_swevent_enable,
	.disable	= perf_swevent_disable,
	.read		= perf_swevent_read,
	.unthrottle	= perf_swevent_unthrottle,
3989 3990
};

3991
/*
3992
 * hrtimer based swevent callback
3993 3994
 */

3995
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
3996 3997 3998
{
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
3999
	struct pt_regs *regs;
4000
	struct perf_event *event;
4001 4002
	u64 period;

4003 4004
	event	= container_of(hrtimer, struct perf_event, hw.hrtimer);
	event->pmu->read(event);
4005 4006

	data.addr = 0;
4007
	regs = get_irq_regs();
4008 4009 4010 4011
	/*
	 * In case we exclude kernel IPs or are somehow not in interrupt
	 * context, provide the next best thing, the user IP.
	 */
4012 4013
	if ((event->attr.exclude_kernel || !regs) &&
			!event->attr.exclude_user)
4014
		regs = task_pt_regs(current);
4015

4016
	if (regs) {
4017 4018 4019
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
4020 4021
	}

4022
	period = max_t(u64, 10000, event->hw.sample_period);
4023 4024 4025 4026 4027
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));

	return ret;
}

4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063
static void perf_swevent_start_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
	if (hwc->sample_period) {
		u64 period;

		if (hwc->remaining) {
			if (hwc->remaining < 0)
				period = 10000;
			else
				period = hwc->remaining;
			hwc->remaining = 0;
		} else {
			period = max_t(u64, 10000, hwc->sample_period);
		}
		__hrtimer_start_range_ns(&hwc->hrtimer,
				ns_to_ktime(period), 0,
				HRTIMER_MODE_REL, 0);
	}
}

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (hwc->sample_period) {
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
		hwc->remaining = ktime_to_ns(remaining);

		hrtimer_cancel(&hwc->hrtimer);
	}
}

4064
/*
4065
 * Software event: cpu wall time clock
4066 4067
 */

4068
static void cpu_clock_perf_event_update(struct perf_event *event)
4069 4070 4071 4072 4073 4074
{
	int cpu = raw_smp_processor_id();
	s64 prev;
	u64 now;

	now = cpu_clock(cpu);
4075 4076 4077
	prev = atomic64_read(&event->hw.prev_count);
	atomic64_set(&event->hw.prev_count, now);
	atomic64_add(now - prev, &event->count);
4078 4079
}

4080
static int cpu_clock_perf_event_enable(struct perf_event *event)
4081
{
4082
	struct hw_perf_event *hwc = &event->hw;
4083 4084 4085
	int cpu = raw_smp_processor_id();

	atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4086
	perf_swevent_start_hrtimer(event);
4087 4088 4089 4090

	return 0;
}

4091
static void cpu_clock_perf_event_disable(struct perf_event *event)
4092
{
4093
	perf_swevent_cancel_hrtimer(event);
4094
	cpu_clock_perf_event_update(event);
4095 4096
}

4097
static void cpu_clock_perf_event_read(struct perf_event *event)
4098
{
4099
	cpu_clock_perf_event_update(event);
4100 4101
}

4102
static const struct pmu perf_ops_cpu_clock = {
4103 4104 4105
	.enable		= cpu_clock_perf_event_enable,
	.disable	= cpu_clock_perf_event_disable,
	.read		= cpu_clock_perf_event_read,
4106 4107
};

4108
/*
4109
 * Software event: task time clock
4110 4111
 */

4112
static void task_clock_perf_event_update(struct perf_event *event, u64 now)
I
Ingo Molnar 已提交
4113
{
4114
	u64 prev;
I
Ingo Molnar 已提交
4115 4116
	s64 delta;

4117
	prev = atomic64_xchg(&event->hw.prev_count, now);
I
Ingo Molnar 已提交
4118
	delta = now - prev;
4119
	atomic64_add(delta, &event->count);
4120 4121
}

4122
static int task_clock_perf_event_enable(struct perf_event *event)
I
Ingo Molnar 已提交
4123
{
4124
	struct hw_perf_event *hwc = &event->hw;
4125 4126
	u64 now;

4127
	now = event->ctx->time;
4128

4129
	atomic64_set(&hwc->prev_count, now);
4130 4131

	perf_swevent_start_hrtimer(event);
4132 4133

	return 0;
I
Ingo Molnar 已提交
4134 4135
}

4136
static void task_clock_perf_event_disable(struct perf_event *event)
4137
{
4138
	perf_swevent_cancel_hrtimer(event);
4139
	task_clock_perf_event_update(event, event->ctx->time);
4140

4141
}
I
Ingo Molnar 已提交
4142

4143
static void task_clock_perf_event_read(struct perf_event *event)
4144
{
4145 4146 4147
	u64 time;

	if (!in_nmi()) {
4148 4149
		update_context_time(event->ctx);
		time = event->ctx->time;
4150 4151
	} else {
		u64 now = perf_clock();
4152 4153
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
4154 4155
	}

4156
	task_clock_perf_event_update(event, time);
4157 4158
}

4159
static const struct pmu perf_ops_task_clock = {
4160 4161 4162
	.enable		= task_clock_perf_event_enable,
	.disable	= task_clock_perf_event_disable,
	.read		= task_clock_perf_event_read,
4163 4164
};

4165
#ifdef CONFIG_EVENT_PROFILE
L
Li Zefan 已提交
4166

4167
void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4168
			  int entry_size)
4169
{
4170
	struct perf_raw_record raw = {
4171
		.size = entry_size,
4172
		.data = record,
4173 4174
	};

4175
	struct perf_sample_data data = {
4176
		.addr = addr,
4177
		.raw = &raw,
4178
	};
4179

4180 4181 4182 4183
	struct pt_regs *regs = get_irq_regs();

	if (!regs)
		regs = task_pt_regs(current);
4184

4185 4186
	/* Trace events already protected against recursion */
	__do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4187
				&data, regs);
4188
}
4189
EXPORT_SYMBOL_GPL(perf_tp_event);
4190

L
Li Zefan 已提交
4191 4192 4193 4194 4195 4196 4197 4198 4199
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}
4200

4201
static void tp_perf_event_destroy(struct perf_event *event)
4202
{
4203
	ftrace_profile_disable(event->attr.config);
4204 4205
}

4206
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4207
{
4208 4209 4210 4211
	/*
	 * Raw tracepoint data is a severe data leak, only allow root to
	 * have these.
	 */
4212
	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4213
			perf_paranoid_tracepoint_raw() &&
4214 4215 4216
			!capable(CAP_SYS_ADMIN))
		return ERR_PTR(-EPERM);

4217
	if (ftrace_profile_enable(event->attr.config))
4218 4219
		return NULL;

4220
	event->destroy = tp_perf_event_destroy;
4221 4222 4223

	return &perf_ops_generic;
}
L
Li Zefan 已提交
4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

4248
#else
L
Li Zefan 已提交
4249 4250 4251 4252 4253 4254 4255

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	return 1;
}

4256
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4257 4258 4259
{
	return NULL;
}
L
Li Zefan 已提交
4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

#endif /* CONFIG_EVENT_PROFILE */
4271

4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316
#ifdef CONFIG_HAVE_HW_BREAKPOINT
static void bp_perf_event_destroy(struct perf_event *event)
{
	release_bp_slot(event);
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	int err;
	/*
	 * The breakpoint is already filled if we haven't created the counter
	 * through perf syscall
	 * FIXME: manage to get trigerred to NULL if it comes from syscalls
	 */
	if (!bp->callback)
		err = register_perf_hw_breakpoint(bp);
	else
		err = __register_perf_hw_breakpoint(bp);
	if (err)
		return ERR_PTR(err);

	bp->destroy = bp_perf_event_destroy;

	return &perf_ops_bp;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
	/* TODO */
}
#else
static void bp_perf_event_destroy(struct perf_event *event)
{
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	return NULL;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
}
#endif

4317
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4318

4319
static void sw_perf_event_destroy(struct perf_event *event)
4320
{
4321
	u64 event_id = event->attr.config;
4322

4323
	WARN_ON(event->parent);
4324

4325
	atomic_dec(&perf_swevent_enabled[event_id]);
4326 4327
}

4328
static const struct pmu *sw_perf_event_init(struct perf_event *event)
4329
{
4330
	const struct pmu *pmu = NULL;
4331
	u64 event_id = event->attr.config;
4332

4333
	/*
4334
	 * Software events (currently) can't in general distinguish
4335 4336 4337 4338 4339
	 * between user, kernel and hypervisor events.
	 * However, context switches and cpu migrations are considered
	 * to be kernel events, and page faults are never hypervisor
	 * events.
	 */
4340
	switch (event_id) {
4341
	case PERF_COUNT_SW_CPU_CLOCK:
4342
		pmu = &perf_ops_cpu_clock;
4343

4344
		break;
4345
	case PERF_COUNT_SW_TASK_CLOCK:
4346
		/*
4347 4348
		 * If the user instantiates this as a per-cpu event,
		 * use the cpu_clock event instead.
4349
		 */
4350
		if (event->ctx->task)
4351
			pmu = &perf_ops_task_clock;
4352
		else
4353
			pmu = &perf_ops_cpu_clock;
4354

4355
		break;
4356 4357 4358 4359 4360
	case PERF_COUNT_SW_PAGE_FAULTS:
	case PERF_COUNT_SW_PAGE_FAULTS_MIN:
	case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
	case PERF_COUNT_SW_CONTEXT_SWITCHES:
	case PERF_COUNT_SW_CPU_MIGRATIONS:
4361 4362
	case PERF_COUNT_SW_ALIGNMENT_FAULTS:
	case PERF_COUNT_SW_EMULATION_FAULTS:
4363 4364 4365
		if (!event->parent) {
			atomic_inc(&perf_swevent_enabled[event_id]);
			event->destroy = sw_perf_event_destroy;
4366
		}
4367
		pmu = &perf_ops_generic;
4368
		break;
4369
	}
4370

4371
	return pmu;
4372 4373
}

T
Thomas Gleixner 已提交
4374
/*
4375
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
4376
 */
4377 4378
static struct perf_event *
perf_event_alloc(struct perf_event_attr *attr,
4379
		   int cpu,
4380 4381 4382
		   struct perf_event_context *ctx,
		   struct perf_event *group_leader,
		   struct perf_event *parent_event,
4383
		   perf_callback_t callback,
4384
		   gfp_t gfpflags)
T
Thomas Gleixner 已提交
4385
{
4386
	const struct pmu *pmu;
4387 4388
	struct perf_event *event;
	struct hw_perf_event *hwc;
4389
	long err;
T
Thomas Gleixner 已提交
4390

4391 4392
	event = kzalloc(sizeof(*event), gfpflags);
	if (!event)
4393
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
4394

4395
	/*
4396
	 * Single events are their own group leaders, with an
4397 4398 4399
	 * empty sibling list:
	 */
	if (!group_leader)
4400
		group_leader = event;
4401

4402 4403
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
4404

4405 4406 4407 4408
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
T
Thomas Gleixner 已提交
4409

4410
	mutex_init(&event->mmap_mutex);
4411

4412 4413 4414 4415 4416 4417
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->ctx		= ctx;
	event->oncpu		= -1;
4418

4419
	event->parent		= parent_event;
4420

4421 4422
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
4423

4424
	event->state		= PERF_EVENT_STATE_INACTIVE;
4425

4426 4427 4428 4429 4430
	if (!callback && parent_event)
		callback = parent_event->callback;
	
	event->callback	= callback;

4431
	if (attr->disabled)
4432
		event->state = PERF_EVENT_STATE_OFF;
4433

4434
	pmu = NULL;
4435

4436
	hwc = &event->hw;
4437
	hwc->sample_period = attr->sample_period;
4438
	if (attr->freq && attr->sample_freq)
4439
		hwc->sample_period = 1;
4440
	hwc->last_period = hwc->sample_period;
4441 4442

	atomic64_set(&hwc->period_left, hwc->sample_period);
4443

4444
	/*
4445
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
4446
	 */
4447
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4448 4449
		goto done;

4450
	switch (attr->type) {
4451
	case PERF_TYPE_RAW:
4452
	case PERF_TYPE_HARDWARE:
4453
	case PERF_TYPE_HW_CACHE:
4454
		pmu = hw_perf_event_init(event);
4455 4456 4457
		break;

	case PERF_TYPE_SOFTWARE:
4458
		pmu = sw_perf_event_init(event);
4459 4460 4461
		break;

	case PERF_TYPE_TRACEPOINT:
4462
		pmu = tp_perf_event_init(event);
4463
		break;
4464

4465 4466 4467 4468 4469
	case PERF_TYPE_BREAKPOINT:
		pmu = bp_perf_event_init(event);
		break;


4470 4471
	default:
		break;
4472
	}
4473 4474
done:
	err = 0;
4475
	if (!pmu)
4476
		err = -EINVAL;
4477 4478
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
4479

4480
	if (err) {
4481 4482 4483
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
4484
		return ERR_PTR(err);
I
Ingo Molnar 已提交
4485
	}
4486

4487
	event->pmu = pmu;
T
Thomas Gleixner 已提交
4488

4489 4490 4491 4492 4493 4494 4495 4496
	if (!event->parent) {
		atomic_inc(&nr_events);
		if (event->attr.mmap)
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
4497
	}
4498

4499
	return event;
T
Thomas Gleixner 已提交
4500 4501
}

4502 4503
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
4504 4505
{
	u32 size;
4506
	int ret;
4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
4531 4532 4533
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
4534 4535
	 */
	if (size > sizeof(*attr)) {
4536 4537 4538
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
4539

4540 4541
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
4542

4543
		for (; addr < end; addr++) {
4544 4545 4546 4547 4548 4549
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
4550
		size = sizeof(*attr);
4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Li Zefan 已提交
4582
static int perf_event_set_output(struct perf_event *event, int output_fd)
4583
{
4584
	struct perf_event *output_event = NULL;
4585
	struct file *output_file = NULL;
4586
	struct perf_event *old_output;
4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599
	int fput_needed = 0;
	int ret = -EINVAL;

	if (!output_fd)
		goto set;

	output_file = fget_light(output_fd, &fput_needed);
	if (!output_file)
		return -EBADF;

	if (output_file->f_op != &perf_fops)
		goto out;

4600
	output_event = output_file->private_data;
4601 4602

	/* Don't chain output fds */
4603
	if (output_event->output)
4604 4605 4606
		goto out;

	/* Don't set an output fd when we already have an output channel */
4607
	if (event->data)
4608 4609 4610 4611 4612
		goto out;

	atomic_long_inc(&output_file->f_count);

set:
4613 4614 4615 4616
	mutex_lock(&event->mmap_mutex);
	old_output = event->output;
	rcu_assign_pointer(event->output, output_event);
	mutex_unlock(&event->mmap_mutex);
4617 4618 4619 4620

	if (old_output) {
		/*
		 * we need to make sure no existing perf_output_*()
4621
		 * is still referencing this event.
4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632
		 */
		synchronize_rcu();
		fput(old_output->filp);
	}

	ret = 0;
out:
	fput_light(output_file, fput_needed);
	return ret;
}

T
Thomas Gleixner 已提交
4633
/**
4634
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
4635
 *
4636
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
4637
 * @pid:		target pid
I
Ingo Molnar 已提交
4638
 * @cpu:		target cpu
4639
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
4640
 */
4641 4642
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
4643
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
4644
{
4645 4646 4647 4648
	struct perf_event *event, *group_leader;
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
4649 4650
	struct file *group_file = NULL;
	int fput_needed = 0;
4651
	int fput_needed2 = 0;
4652
	int err;
T
Thomas Gleixner 已提交
4653

4654
	/* for future expandability... */
4655
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4656 4657
		return -EINVAL;

4658 4659 4660
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
4661

4662 4663 4664 4665 4666
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

4667
	if (attr.freq) {
4668
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
4669 4670 4671
			return -EINVAL;
	}

4672
	/*
I
Ingo Molnar 已提交
4673 4674 4675 4676 4677 4678 4679
	 * Get the target context (task or percpu):
	 */
	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

	/*
4680
	 * Look up the group leader (we will attach this event to it):
4681 4682
	 */
	group_leader = NULL;
4683
	if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4684
		err = -EINVAL;
4685 4686
		group_file = fget_light(group_fd, &fput_needed);
		if (!group_file)
I
Ingo Molnar 已提交
4687
			goto err_put_context;
4688
		if (group_file->f_op != &perf_fops)
I
Ingo Molnar 已提交
4689
			goto err_put_context;
4690 4691 4692

		group_leader = group_file->private_data;
		/*
I
Ingo Molnar 已提交
4693 4694 4695 4696 4697 4698 4699 4700
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
			goto err_put_context;
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
4701
		 */
I
Ingo Molnar 已提交
4702 4703
		if (group_leader->ctx != ctx)
			goto err_put_context;
4704 4705 4706
		/*
		 * Only a group leader can be exclusive or pinned
		 */
4707
		if (attr.exclusive || attr.pinned)
4708
			goto err_put_context;
4709 4710
	}

4711
	event = perf_event_alloc(&attr, cpu, ctx, group_leader,
4712
				     NULL, NULL, GFP_KERNEL);
4713 4714
	err = PTR_ERR(event);
	if (IS_ERR(event))
T
Thomas Gleixner 已提交
4715 4716
		goto err_put_context;

4717
	err = anon_inode_getfd("[perf_event]", &perf_fops, event, 0);
4718
	if (err < 0)
4719 4720
		goto err_free_put_context;

4721 4722
	event_file = fget_light(err, &fput_needed2);
	if (!event_file)
4723 4724
		goto err_free_put_context;

4725
	if (flags & PERF_FLAG_FD_OUTPUT) {
4726
		err = perf_event_set_output(event, group_fd);
4727 4728
		if (err)
			goto err_fput_free_put_context;
4729 4730
	}

4731
	event->filp = event_file;
4732
	WARN_ON_ONCE(ctx->parent_ctx);
4733
	mutex_lock(&ctx->mutex);
4734
	perf_install_in_context(ctx, event, cpu);
4735
	++ctx->generation;
4736
	mutex_unlock(&ctx->mutex);
4737

4738
	event->owner = current;
4739
	get_task_struct(current);
4740 4741 4742
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
4743

4744
err_fput_free_put_context:
4745
	fput_light(event_file, fput_needed2);
T
Thomas Gleixner 已提交
4746

4747
err_free_put_context:
4748
	if (err < 0)
4749
		kfree(event);
T
Thomas Gleixner 已提交
4750 4751

err_put_context:
4752 4753 4754 4755
	if (err < 0)
		put_ctx(ctx);

	fput_light(group_file, fput_needed);
T
Thomas Gleixner 已提交
4756

4757
	return err;
T
Thomas Gleixner 已提交
4758 4759
}

4760 4761 4762 4763 4764 4765 4766 4767 4768
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
 * @pid: task to profile
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4769
				 pid_t pid, perf_callback_t callback)
4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780
{
	struct perf_event *event;
	struct perf_event_context *ctx;
	int err;

	/*
	 * Get the target context (task or percpu):
	 */

	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
4781
		return NULL;
4782 4783

	event = perf_event_alloc(attr, cpu, ctx, NULL,
4784
				     NULL, callback, GFP_KERNEL);
4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811
	err = PTR_ERR(event);
	if (IS_ERR(event))
		goto err_put_context;

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
	mutex_unlock(&ctx->mutex);

	event->owner = current;
	get_task_struct(current);
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);

	return event;

err_put_context:
	if (err < 0)
		put_ctx(ctx);

	return NULL;
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);

4812
/*
4813
 * inherit a event from parent task to child task:
4814
 */
4815 4816
static struct perf_event *
inherit_event(struct perf_event *parent_event,
4817
	      struct task_struct *parent,
4818
	      struct perf_event_context *parent_ctx,
4819
	      struct task_struct *child,
4820 4821
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
4822
{
4823
	struct perf_event *child_event;
4824

4825
	/*
4826 4827
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
4828 4829 4830
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
4831 4832
	if (parent_event->parent)
		parent_event = parent_event->parent;
4833

4834 4835 4836
	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu, child_ctx,
					   group_leader, parent_event,
4837
					   NULL, GFP_KERNEL);
4838 4839
	if (IS_ERR(child_event))
		return child_event;
4840
	get_ctx(child_ctx);
4841

4842
	/*
4843
	 * Make the child state follow the state of the parent event,
4844
	 * not its attr.disabled bit.  We hold the parent's mutex,
4845
	 * so we won't race with perf_event_{en, dis}able_family.
4846
	 */
4847 4848
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
4849
	else
4850
		child_event->state = PERF_EVENT_STATE_OFF;
4851

4852 4853
	if (parent_event->attr.freq)
		child_event->hw.sample_period = parent_event->hw.sample_period;
4854

4855 4856
	child_event->overflow_handler = parent_event->overflow_handler;

4857 4858 4859
	/*
	 * Link it up in the child's context:
	 */
4860
	add_event_to_ctx(child_event, child_ctx);
4861 4862 4863

	/*
	 * Get a reference to the parent filp - we will fput it
4864
	 * when the child event exits. This is safe to do because
4865 4866 4867
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
4868
	atomic_long_inc(&parent_event->filp->f_count);
4869

4870
	/*
4871
	 * Link this into the parent event's child list
4872
	 */
4873 4874 4875 4876
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4877

4878
	return child_event;
4879 4880
}

4881
static int inherit_group(struct perf_event *parent_event,
4882
	      struct task_struct *parent,
4883
	      struct perf_event_context *parent_ctx,
4884
	      struct task_struct *child,
4885
	      struct perf_event_context *child_ctx)
4886
{
4887 4888 4889
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;
4890

4891
	leader = inherit_event(parent_event, parent, parent_ctx,
4892
				 child, NULL, child_ctx);
4893 4894
	if (IS_ERR(leader))
		return PTR_ERR(leader);
4895 4896
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
4897 4898 4899
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
4900
	}
4901 4902 4903
	return 0;
}

4904
static void sync_child_event(struct perf_event *child_event,
4905
			       struct task_struct *child)
4906
{
4907
	struct perf_event *parent_event = child_event->parent;
4908
	u64 child_val;
4909

4910 4911
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
4912

4913
	child_val = atomic64_read(&child_event->count);
4914 4915 4916 4917

	/*
	 * Add back the child's count to the parent's count:
	 */
4918 4919 4920 4921 4922
	atomic64_add(child_val, &parent_event->count);
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
4923 4924

	/*
4925
	 * Remove this event from the parent's list
4926
	 */
4927 4928 4929 4930
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4931 4932

	/*
4933
	 * Release the parent event, if this was the last
4934 4935
	 * reference to it.
	 */
4936
	fput(parent_event->filp);
4937 4938
}

4939
static void
4940 4941
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
4942
			 struct task_struct *child)
4943
{
4944
	struct perf_event *parent_event;
4945

4946 4947
	update_event_times(child_event);
	perf_event_remove_from_context(child_event);
4948

4949
	parent_event = child_event->parent;
4950
	/*
4951
	 * It can happen that parent exits first, and has events
4952
	 * that are still around due to the child reference. These
4953
	 * events need to be zapped - but otherwise linger.
4954
	 */
4955 4956 4957
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
4958
	}
4959 4960 4961
}

/*
4962
 * When a child task exits, feed back event values to parent events.
4963
 */
4964
void perf_event_exit_task(struct task_struct *child)
4965
{
4966 4967
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
4968
	unsigned long flags;
4969

4970 4971
	if (likely(!child->perf_event_ctxp)) {
		perf_event_task(child, NULL, 0);
4972
		return;
P
Peter Zijlstra 已提交
4973
	}
4974

4975
	local_irq_save(flags);
4976 4977 4978 4979 4980 4981
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
4982 4983
	child_ctx = child->perf_event_ctxp;
	__perf_event_task_sched_out(child_ctx);
4984 4985 4986

	/*
	 * Take the context lock here so that if find_get_context is
4987
	 * reading child->perf_event_ctxp, we wait until it has
4988 4989 4990
	 * incremented the context's refcount before we do put_ctx below.
	 */
	spin_lock(&child_ctx->lock);
4991
	child->perf_event_ctxp = NULL;
4992 4993 4994
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
4995
	 * the events from it.
4996 4997
	 */
	unclone_ctx(child_ctx);
P
Peter Zijlstra 已提交
4998 4999 5000
	spin_unlock_irqrestore(&child_ctx->lock, flags);

	/*
5001 5002 5003
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
5004
	 */
5005
	perf_event_task(child, child_ctx, 0);
5006

5007 5008 5009
	/*
	 * We can recurse on the same lock type through:
	 *
5010 5011 5012
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
5013 5014 5015 5016 5017 5018
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
	mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
5019

5020
again:
5021
	list_for_each_entry_safe(child_event, tmp, &child_ctx->group_list,
5022
				 group_entry)
5023
		__perf_event_exit_task(child_event, child_ctx, child);
5024 5025

	/*
5026
	 * If the last event was a group event, it will have appended all
5027 5028 5029
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
5030
	if (!list_empty(&child_ctx->group_list))
5031
		goto again;
5032 5033 5034 5035

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
5036 5037
}

5038 5039 5040 5041
/*
 * free an unexposed, unused context as created by inheritance by
 * init_task below, used by fork() in case of fail.
 */
5042
void perf_event_free_task(struct task_struct *task)
5043
{
5044 5045
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event *event, *tmp;
5046 5047 5048 5049 5050 5051

	if (!ctx)
		return;

	mutex_lock(&ctx->mutex);
again:
5052 5053
	list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry) {
		struct perf_event *parent = event->parent;
5054 5055 5056 5057 5058

		if (WARN_ON_ONCE(!parent))
			continue;

		mutex_lock(&parent->child_mutex);
5059
		list_del_init(&event->child_list);
5060 5061 5062 5063
		mutex_unlock(&parent->child_mutex);

		fput(parent->filp);

5064 5065
		list_del_event(event, ctx);
		free_event(event);
5066 5067
	}

5068
	if (!list_empty(&ctx->group_list))
5069 5070 5071 5072 5073 5074 5075
		goto again;

	mutex_unlock(&ctx->mutex);

	put_ctx(ctx);
}

5076
/*
5077
 * Initialize the perf_event context in task_struct
5078
 */
5079
int perf_event_init_task(struct task_struct *child)
5080
{
5081 5082 5083
	struct perf_event_context *child_ctx, *parent_ctx;
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
5084
	struct task_struct *parent = current;
5085
	int inherited_all = 1;
5086
	int ret = 0;
5087

5088
	child->perf_event_ctxp = NULL;
5089

5090 5091
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);
5092

5093
	if (likely(!parent->perf_event_ctxp))
5094 5095
		return 0;

5096 5097
	/*
	 * This is executed from the parent task context, so inherit
5098
	 * events that have been marked for cloning.
5099
	 * First allocate and initialize a context for the child.
5100 5101
	 */

5102
	child_ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL);
5103
	if (!child_ctx)
5104
		return -ENOMEM;
5105

5106 5107
	__perf_event_init_context(child_ctx, child);
	child->perf_event_ctxp = child_ctx;
5108
	get_task_struct(child);
5109

5110
	/*
5111 5112
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
5113
	 */
5114 5115
	parent_ctx = perf_pin_task_context(parent);

5116 5117 5118 5119 5120 5121 5122
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

5123 5124 5125 5126
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
5127
	mutex_lock(&parent_ctx->mutex);
5128 5129 5130 5131 5132

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
5133
	list_for_each_entry(event, &parent_ctx->group_list, group_entry) {
5134

5135
		if (!event->attr.inherit) {
5136
			inherited_all = 0;
5137
			continue;
5138
		}
5139

5140
		ret = inherit_group(event, parent, parent_ctx,
5141 5142
					     child, child_ctx);
		if (ret) {
5143
			inherited_all = 0;
5144
			break;
5145 5146 5147 5148 5149 5150 5151
		}
	}

	if (inherited_all) {
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
5152 5153
		 * Note that if the parent is a clone, it could get
		 * uncloned at any point, but that doesn't matter
5154
		 * because the list of events and the generation
5155
		 * count can't have changed since we took the mutex.
5156
		 */
5157 5158 5159
		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
5160
			child_ctx->parent_gen = parent_ctx->parent_gen;
5161 5162 5163 5164 5165
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
5166 5167
	}

5168
	mutex_unlock(&parent_ctx->mutex);
5169

5170
	perf_unpin_context(parent_ctx);
5171

5172
	return ret;
5173 5174
}

5175
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
5176
{
5177
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
5178

5179
	cpuctx = &per_cpu(perf_cpu_context, cpu);
5180
	__perf_event_init_context(&cpuctx->ctx, NULL);
T
Thomas Gleixner 已提交
5181

5182
	spin_lock(&perf_resource_lock);
5183
	cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5184
	spin_unlock(&perf_resource_lock);
5185

5186
	hw_perf_event_setup(cpu);
T
Thomas Gleixner 已提交
5187 5188 5189
}

#ifdef CONFIG_HOTPLUG_CPU
5190
static void __perf_event_exit_cpu(void *info)
T
Thomas Gleixner 已提交
5191 5192
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5193 5194
	struct perf_event_context *ctx = &cpuctx->ctx;
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
5195

5196 5197
	list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry)
		__perf_event_remove_from_context(event);
T
Thomas Gleixner 已提交
5198
}
5199
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
5200
{
5201
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5202
	struct perf_event_context *ctx = &cpuctx->ctx;
5203 5204

	mutex_lock(&ctx->mutex);
5205
	smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5206
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
5207 5208
}
#else
5209
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220
#endif

static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action) {

	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5221
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
5222 5223
		break;

5224 5225
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
5226
		hw_perf_event_setup_online(cpu);
5227 5228
		break;

T
Thomas Gleixner 已提交
5229 5230
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
5231
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
5232 5233 5234 5235 5236 5237 5238 5239 5240
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

5241 5242 5243
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
T
Thomas Gleixner 已提交
5244 5245
static struct notifier_block __cpuinitdata perf_cpu_nb = {
	.notifier_call		= perf_cpu_notify,
5246
	.priority		= 20,
T
Thomas Gleixner 已提交
5247 5248
};

5249
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
5250 5251 5252
{
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
			(void *)(long)smp_processor_id());
5253 5254
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
			(void *)(long)smp_processor_id());
T
Thomas Gleixner 已提交
5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274
	register_cpu_notifier(&perf_cpu_nb);
}

static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_reserved_percpu);
}

static ssize_t
perf_set_reserve_percpu(struct sysdev_class *class,
			const char *buf,
			size_t count)
{
	struct perf_cpu_context *cpuctx;
	unsigned long val;
	int err, cpu, mpt;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
5275
	if (val > perf_max_events)
T
Thomas Gleixner 已提交
5276 5277
		return -EINVAL;

5278
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5279 5280 5281 5282
	perf_reserved_percpu = val;
	for_each_online_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
		spin_lock_irq(&cpuctx->ctx.lock);
5283 5284
		mpt = min(perf_max_events - cpuctx->ctx.nr_events,
			  perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
5285 5286 5287
		cpuctx->max_pertask = mpt;
		spin_unlock_irq(&cpuctx->ctx.lock);
	}
5288
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309

	return count;
}

static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_overcommit);
}

static ssize_t
perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
{
	unsigned long val;
	int err;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
	if (val > 1)
		return -EINVAL;

5310
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5311
	perf_overcommit = val;
5312
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338

	return count;
}

static SYSDEV_CLASS_ATTR(
				reserve_percpu,
				0644,
				perf_show_reserve_percpu,
				perf_set_reserve_percpu
			);

static SYSDEV_CLASS_ATTR(
				overcommit,
				0644,
				perf_show_overcommit,
				perf_set_overcommit
			);

static struct attribute *perfclass_attrs[] = {
	&attr_reserve_percpu.attr,
	&attr_overcommit.attr,
	NULL
};

static struct attribute_group perfclass_attr_group = {
	.attrs			= perfclass_attrs,
5339
	.name			= "perf_events",
T
Thomas Gleixner 已提交
5340 5341
};

5342
static int __init perf_event_sysfs_init(void)
T
Thomas Gleixner 已提交
5343 5344 5345 5346
{
	return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
				  &perfclass_attr_group);
}
5347
device_initcall(perf_event_sysfs_init);