perf_event.c 120.3 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4 5 6
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
16
#include <linux/file.h>
T
Thomas Gleixner 已提交
17 18
#include <linux/poll.h>
#include <linux/sysfs.h>
19
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
20
#include <linux/percpu.h>
21
#include <linux/ptrace.h>
22
#include <linux/vmstat.h>
23
#include <linux/vmalloc.h>
24 25
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
26 27 28
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
29
#include <linux/kernel_stat.h>
30
#include <linux/perf_event.h>
L
Li Zefan 已提交
31
#include <linux/ftrace_event.h>
32
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
33

34 35
#include <asm/irq_regs.h>

T
Thomas Gleixner 已提交
36
/*
37
 * Each CPU has a list of per CPU events:
T
Thomas Gleixner 已提交
38 39 40
 */
DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);

41
int perf_max_events __read_mostly = 1;
T
Thomas Gleixner 已提交
42 43 44
static int perf_reserved_percpu __read_mostly;
static int perf_overcommit __read_mostly = 1;

45 46 47 48
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
49

50
/*
51
 * perf event paranoia level:
52 53
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
54
 *   1 - disallow cpu events for unpriv
55
 *   2 - disallow kernel profiling for unpriv
56
 */
57
int sysctl_perf_event_paranoid __read_mostly = 1;
58

59 60
static inline bool perf_paranoid_tracepoint_raw(void)
{
61
	return sysctl_perf_event_paranoid > -1;
62 63
}

64 65
static inline bool perf_paranoid_cpu(void)
{
66
	return sysctl_perf_event_paranoid > 0;
67 68 69 70
}

static inline bool perf_paranoid_kernel(void)
{
71
	return sysctl_perf_event_paranoid > 1;
72 73
}

74
int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */
75 76

/*
77
 * max perf event sample rate
78
 */
79
int sysctl_perf_event_sample_rate __read_mostly = 100000;
80

81
static atomic64_t perf_event_id;
82

T
Thomas Gleixner 已提交
83
/*
84
 * Lock for (sysadmin-configurable) event reservations:
T
Thomas Gleixner 已提交
85
 */
86
static DEFINE_SPINLOCK(perf_resource_lock);
T
Thomas Gleixner 已提交
87 88 89 90

/*
 * Architecture provided APIs - weak aliases:
 */
91
extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
T
Thomas Gleixner 已提交
92
{
93
	return NULL;
T
Thomas Gleixner 已提交
94 95
}

96 97 98
void __weak hw_perf_disable(void)		{ barrier(); }
void __weak hw_perf_enable(void)		{ barrier(); }

99 100
void __weak hw_perf_event_setup(int cpu)	{ barrier(); }
void __weak hw_perf_event_setup_online(int cpu)	{ barrier(); }
101 102

int __weak
103
hw_perf_group_sched_in(struct perf_event *group_leader,
104
	       struct perf_cpu_context *cpuctx,
105
	       struct perf_event_context *ctx, int cpu)
106 107 108
{
	return 0;
}
T
Thomas Gleixner 已提交
109

110
void __weak perf_event_print_debug(void)	{ }
111

112
static DEFINE_PER_CPU(int, perf_disable_count);
113 114 115

void __perf_disable(void)
{
116
	__get_cpu_var(perf_disable_count)++;
117 118 119 120
}

bool __perf_enable(void)
{
121
	return !--__get_cpu_var(perf_disable_count);
122 123 124 125 126 127 128 129 130 131 132 133 134 135
}

void perf_disable(void)
{
	__perf_disable();
	hw_perf_disable();
}

void perf_enable(void)
{
	if (__perf_enable())
		hw_perf_enable();
}

136
static void get_ctx(struct perf_event_context *ctx)
137
{
138
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
139 140
}

141 142
static void free_ctx(struct rcu_head *head)
{
143
	struct perf_event_context *ctx;
144

145
	ctx = container_of(head, struct perf_event_context, rcu_head);
146 147 148
	kfree(ctx);
}

149
static void put_ctx(struct perf_event_context *ctx)
150
{
151 152 153
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
154 155 156
		if (ctx->task)
			put_task_struct(ctx->task);
		call_rcu(&ctx->rcu_head, free_ctx);
157
	}
158 159
}

160
static void unclone_ctx(struct perf_event_context *ctx)
161 162 163 164 165 166 167
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

168
/*
169
 * If we inherit events we want to return the parent event id
170 171
 * to userspace.
 */
172
static u64 primary_event_id(struct perf_event *event)
173
{
174
	u64 id = event->id;
175

176 177
	if (event->parent)
		id = event->parent->id;
178 179 180 181

	return id;
}

182
/*
183
 * Get the perf_event_context for a task and lock it.
184 185 186
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
187
static struct perf_event_context *
188
perf_lock_task_context(struct task_struct *task, unsigned long *flags)
189
{
190
	struct perf_event_context *ctx;
191 192 193

	rcu_read_lock();
 retry:
194
	ctx = rcu_dereference(task->perf_event_ctxp);
195 196 197 198
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
199
		 * perf_event_task_sched_out, though the
200 201 202 203 204 205 206
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
		spin_lock_irqsave(&ctx->lock, *flags);
207
		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
208 209 210
			spin_unlock_irqrestore(&ctx->lock, *flags);
			goto retry;
		}
211 212 213 214 215

		if (!atomic_inc_not_zero(&ctx->refcount)) {
			spin_unlock_irqrestore(&ctx->lock, *flags);
			ctx = NULL;
		}
216 217 218 219 220 221 222 223 224 225
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
226
static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
227
{
228
	struct perf_event_context *ctx;
229 230 231 232 233 234 235 236 237 238
	unsigned long flags;

	ctx = perf_lock_task_context(task, &flags);
	if (ctx) {
		++ctx->pin_count;
		spin_unlock_irqrestore(&ctx->lock, flags);
	}
	return ctx;
}

239
static void perf_unpin_context(struct perf_event_context *ctx)
240 241 242 243 244 245 246 247 248
{
	unsigned long flags;

	spin_lock_irqsave(&ctx->lock, flags);
	--ctx->pin_count;
	spin_unlock_irqrestore(&ctx->lock, flags);
	put_ctx(ctx);
}

249
/*
250
 * Add a event from the lists for its context.
251 252
 * Must be called with ctx->mutex and ctx->lock held.
 */
253
static void
254
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
255
{
256
	struct perf_event *group_leader = event->group_leader;
257 258

	/*
259 260
	 * Depending on whether it is a standalone or sibling event,
	 * add it straight to the context's event list, or to the group
261 262
	 * leader's sibling list:
	 */
263 264
	if (group_leader == event)
		list_add_tail(&event->group_entry, &ctx->group_list);
P
Peter Zijlstra 已提交
265
	else {
266
		list_add_tail(&event->group_entry, &group_leader->sibling_list);
P
Peter Zijlstra 已提交
267 268
		group_leader->nr_siblings++;
	}
P
Peter Zijlstra 已提交
269

270 271 272
	list_add_rcu(&event->event_entry, &ctx->event_list);
	ctx->nr_events++;
	if (event->attr.inherit_stat)
273
		ctx->nr_stat++;
274 275
}

276
/*
277
 * Remove a event from the lists for its context.
278
 * Must be called with ctx->mutex and ctx->lock held.
279
 */
280
static void
281
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
282
{
283
	struct perf_event *sibling, *tmp;
284

285
	if (list_empty(&event->group_entry))
286
		return;
287 288
	ctx->nr_events--;
	if (event->attr.inherit_stat)
289
		ctx->nr_stat--;
290

291 292
	list_del_init(&event->group_entry);
	list_del_rcu(&event->event_entry);
293

294 295
	if (event->group_leader != event)
		event->group_leader->nr_siblings--;
P
Peter Zijlstra 已提交
296

297
	/*
298 299
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
300 301
	 * to the context list directly:
	 */
302
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
303

304
		list_move_tail(&sibling->group_entry, &ctx->group_list);
305 306 307 308
		sibling->group_leader = sibling;
	}
}

309
static void
310
event_sched_out(struct perf_event *event,
311
		  struct perf_cpu_context *cpuctx,
312
		  struct perf_event_context *ctx)
313
{
314
	if (event->state != PERF_EVENT_STATE_ACTIVE)
315 316
		return;

317 318 319 320
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
321
	}
322 323 324
	event->tstamp_stopped = ctx->time;
	event->pmu->disable(event);
	event->oncpu = -1;
325

326
	if (!is_software_event(event))
327 328
		cpuctx->active_oncpu--;
	ctx->nr_active--;
329
	if (event->attr.exclusive || !cpuctx->active_oncpu)
330 331 332
		cpuctx->exclusive = 0;
}

333
static void
334
group_sched_out(struct perf_event *group_event,
335
		struct perf_cpu_context *cpuctx,
336
		struct perf_event_context *ctx)
337
{
338
	struct perf_event *event;
339

340
	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
341 342
		return;

343
	event_sched_out(group_event, cpuctx, ctx);
344 345 346 347

	/*
	 * Schedule out siblings (if any):
	 */
348 349
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
350

351
	if (group_event->attr.exclusive)
352 353 354
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
355
/*
356
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
357
 *
358
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
359 360
 * remove it from the context list.
 */
361
static void __perf_event_remove_from_context(void *info)
T
Thomas Gleixner 已提交
362 363
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
364 365
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
366 367 368 369 370 371

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
	 */
372
	if (ctx->task && cpuctx->task_ctx != ctx)
T
Thomas Gleixner 已提交
373 374
		return;

375
	spin_lock(&ctx->lock);
376 377
	/*
	 * Protect the list operation against NMI by disabling the
378
	 * events on a global level.
379 380
	 */
	perf_disable();
T
Thomas Gleixner 已提交
381

382
	event_sched_out(event, cpuctx, ctx);
383

384
	list_del_event(event, ctx);
T
Thomas Gleixner 已提交
385 386 387

	if (!ctx->task) {
		/*
388
		 * Allow more per task events with respect to the
T
Thomas Gleixner 已提交
389 390 391
		 * reservation:
		 */
		cpuctx->max_pertask =
392 393
			min(perf_max_events - ctx->nr_events,
			    perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
394 395
	}

396
	perf_enable();
397
	spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
398 399 400 401
}


/*
402
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
403
 *
404
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
405
 *
406
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
407
 * call when the task is on a CPU.
408
 *
409 410
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
411 412
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
413
 * When called from perf_event_exit_task, it's OK because the
414
 * context has been detached from its task.
T
Thomas Gleixner 已提交
415
 */
416
static void perf_event_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
417
{
418
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
419 420 421 422
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
423
		 * Per cpu events are removed via an smp call and
T
Thomas Gleixner 已提交
424 425
		 * the removal is always sucessful.
		 */
426 427 428
		smp_call_function_single(event->cpu,
					 __perf_event_remove_from_context,
					 event, 1);
T
Thomas Gleixner 已提交
429 430 431 432
		return;
	}

retry:
433 434
	task_oncpu_function_call(task, __perf_event_remove_from_context,
				 event);
T
Thomas Gleixner 已提交
435 436 437 438 439

	spin_lock_irq(&ctx->lock);
	/*
	 * If the context is active we need to retry the smp call.
	 */
440
	if (ctx->nr_active && !list_empty(&event->group_entry)) {
T
Thomas Gleixner 已提交
441 442 443 444 445 446
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
447
	 * can remove the event safely, if the call above did not
T
Thomas Gleixner 已提交
448 449
	 * succeed.
	 */
450 451
	if (!list_empty(&event->group_entry)) {
		list_del_event(event, ctx);
T
Thomas Gleixner 已提交
452 453 454 455
	}
	spin_unlock_irq(&ctx->lock);
}

456
static inline u64 perf_clock(void)
457
{
458
	return cpu_clock(smp_processor_id());
459 460 461 462 463
}

/*
 * Update the record of the current time in a context.
 */
464
static void update_context_time(struct perf_event_context *ctx)
465
{
466 467 468 469
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
470 471 472
}

/*
473
 * Update the total_time_enabled and total_time_running fields for a event.
474
 */
475
static void update_event_times(struct perf_event *event)
476
{
477
	struct perf_event_context *ctx = event->ctx;
478 479
	u64 run_end;

480 481
	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
482 483
		return;

484
	event->total_time_enabled = ctx->time - event->tstamp_enabled;
485

486 487
	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
488 489 490
	else
		run_end = ctx->time;

491
	event->total_time_running = run_end - event->tstamp_running;
492 493 494
}

/*
495
 * Update total_time_enabled and total_time_running for all events in a group.
496
 */
497
static void update_group_times(struct perf_event *leader)
498
{
499
	struct perf_event *event;
500

501 502 503
	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
504 505
}

506
/*
507
 * Cross CPU call to disable a performance event
508
 */
509
static void __perf_event_disable(void *info)
510
{
511
	struct perf_event *event = info;
512
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
513
	struct perf_event_context *ctx = event->ctx;
514 515

	/*
516 517
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
518
	 */
519
	if (ctx->task && cpuctx->task_ctx != ctx)
520 521
		return;

522
	spin_lock(&ctx->lock);
523 524

	/*
525
	 * If the event is on, turn it off.
526 527
	 * If it is in error state, leave it in error state.
	 */
528
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
529
		update_context_time(ctx);
530 531 532
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
533
		else
534 535
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
536 537
	}

538
	spin_unlock(&ctx->lock);
539 540 541
}

/*
542
 * Disable a event.
543
 *
544 545
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
546
 * remains valid.  This condition is satisifed when called through
547 548 549 550
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
551
 * is the current context on this CPU and preemption is disabled,
552
 * hence we can't get into perf_event_task_sched_out for this context.
553
 */
554
static void perf_event_disable(struct perf_event *event)
555
{
556
	struct perf_event_context *ctx = event->ctx;
557 558 559 560
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
561
		 * Disable the event on the cpu that it's on
562
		 */
563 564
		smp_call_function_single(event->cpu, __perf_event_disable,
					 event, 1);
565 566 567 568
		return;
	}

 retry:
569
	task_oncpu_function_call(task, __perf_event_disable, event);
570 571 572

	spin_lock_irq(&ctx->lock);
	/*
573
	 * If the event is still active, we need to retry the cross-call.
574
	 */
575
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
576 577 578 579 580 581 582 583
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
584 585 586
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
587
	}
588 589 590 591

	spin_unlock_irq(&ctx->lock);
}

592
static int
593
event_sched_in(struct perf_event *event,
594
		 struct perf_cpu_context *cpuctx,
595
		 struct perf_event_context *ctx,
596 597
		 int cpu)
{
598
	if (event->state <= PERF_EVENT_STATE_OFF)
599 600
		return 0;

601 602
	event->state = PERF_EVENT_STATE_ACTIVE;
	event->oncpu = cpu;	/* TODO: put 'cpu' into cpuctx->cpu */
603 604 605 606 607
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

608 609 610
	if (event->pmu->enable(event)) {
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
611 612 613
		return -EAGAIN;
	}

614
	event->tstamp_running += ctx->time - event->tstamp_stopped;
615

616
	if (!is_software_event(event))
617
		cpuctx->active_oncpu++;
618 619
	ctx->nr_active++;

620
	if (event->attr.exclusive)
621 622
		cpuctx->exclusive = 1;

623 624 625
	return 0;
}

626
static int
627
group_sched_in(struct perf_event *group_event,
628
	       struct perf_cpu_context *cpuctx,
629
	       struct perf_event_context *ctx,
630 631
	       int cpu)
{
632
	struct perf_event *event, *partial_group;
633 634
	int ret;

635
	if (group_event->state == PERF_EVENT_STATE_OFF)
636 637
		return 0;

638
	ret = hw_perf_group_sched_in(group_event, cpuctx, ctx, cpu);
639 640 641
	if (ret)
		return ret < 0 ? ret : 0;

642
	if (event_sched_in(group_event, cpuctx, ctx, cpu))
643 644 645 646 647
		return -EAGAIN;

	/*
	 * Schedule in siblings as one group (if any):
	 */
648 649 650
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event_sched_in(event, cpuctx, ctx, cpu)) {
			partial_group = event;
651 652 653 654 655 656 657 658 659 660 661
			goto group_error;
		}
	}

	return 0;

group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
	 */
662 663
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
664
			break;
665
		event_sched_out(event, cpuctx, ctx);
666
	}
667
	event_sched_out(group_event, cpuctx, ctx);
668 669 670 671

	return -EAGAIN;
}

672
/*
673 674
 * Return 1 for a group consisting entirely of software events,
 * 0 if the group contains any hardware events.
675
 */
676
static int is_software_only_group(struct perf_event *leader)
677
{
678
	struct perf_event *event;
679

680
	if (!is_software_event(leader))
681
		return 0;
P
Peter Zijlstra 已提交
682

683 684
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		if (!is_software_event(event))
685
			return 0;
P
Peter Zijlstra 已提交
686

687 688 689 690
	return 1;
}

/*
691
 * Work out whether we can put this event group on the CPU now.
692
 */
693
static int group_can_go_on(struct perf_event *event,
694 695 696 697
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
698
	 * Groups consisting entirely of software events can always go on.
699
	 */
700
	if (is_software_only_group(event))
701 702 703
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
704
	 * events can go on.
705 706 707 708 709
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
710
	 * events on the CPU, it can't go on.
711
	 */
712
	if (event->attr.exclusive && cpuctx->active_oncpu)
713 714 715 716 717 718 719 720
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

721 722
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
723
{
724 725 726 727
	list_add_event(event, ctx);
	event->tstamp_enabled = ctx->time;
	event->tstamp_running = ctx->time;
	event->tstamp_stopped = ctx->time;
728 729
}

T
Thomas Gleixner 已提交
730
/*
731
 * Cross CPU call to install and enable a performance event
732 733
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
734 735 736 737
 */
static void __perf_install_in_context(void *info)
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
738 739 740
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
T
Thomas Gleixner 已提交
741
	int cpu = smp_processor_id();
742
	int err;
T
Thomas Gleixner 已提交
743 744 745 746 747

	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu. If not it has been
	 * scheduled out before the smp call arrived.
748
	 * Or possibly this is the right context but it isn't
749
	 * on this cpu because it had no events.
T
Thomas Gleixner 已提交
750
	 */
751
	if (ctx->task && cpuctx->task_ctx != ctx) {
752
		if (cpuctx->task_ctx || ctx->task != current)
753 754 755
			return;
		cpuctx->task_ctx = ctx;
	}
T
Thomas Gleixner 已提交
756

757
	spin_lock(&ctx->lock);
758
	ctx->is_active = 1;
759
	update_context_time(ctx);
T
Thomas Gleixner 已提交
760 761 762

	/*
	 * Protect the list operation against NMI by disabling the
763
	 * events on a global level. NOP for non NMI based events.
T
Thomas Gleixner 已提交
764
	 */
765
	perf_disable();
T
Thomas Gleixner 已提交
766

767
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
768

769
	/*
770
	 * Don't put the event on if it is disabled or if
771 772
	 * it is in a group and the group isn't on.
	 */
773 774
	if (event->state != PERF_EVENT_STATE_INACTIVE ||
	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
775 776
		goto unlock;

777
	/*
778 779 780
	 * An exclusive event can't go on if there are already active
	 * hardware events, and no hardware event can go on if there
	 * is already an exclusive event on.
781
	 */
782
	if (!group_can_go_on(event, cpuctx, 1))
783 784
		err = -EEXIST;
	else
785
		err = event_sched_in(event, cpuctx, ctx, cpu);
786

787 788
	if (err) {
		/*
789
		 * This event couldn't go on.  If it is in a group
790
		 * then we have to pull the whole group off.
791
		 * If the event group is pinned then put it in error state.
792
		 */
793
		if (leader != event)
794
			group_sched_out(leader, cpuctx, ctx);
795
		if (leader->attr.pinned) {
796
			update_group_times(leader);
797
			leader->state = PERF_EVENT_STATE_ERROR;
798
		}
799
	}
T
Thomas Gleixner 已提交
800

801
	if (!err && !ctx->task && cpuctx->max_pertask)
T
Thomas Gleixner 已提交
802 803
		cpuctx->max_pertask--;

804
 unlock:
805
	perf_enable();
806

807
	spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
808 809 810
}

/*
811
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
812
 *
813 814
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
815
 *
816
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
817 818
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
819 820
 *
 * Must be called with ctx->mutex held.
T
Thomas Gleixner 已提交
821 822
 */
static void
823 824
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
825 826 827 828 829 830
			int cpu)
{
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
831
		 * Per cpu events are installed via an smp call and
T
Thomas Gleixner 已提交
832 833 834
		 * the install is always sucessful.
		 */
		smp_call_function_single(cpu, __perf_install_in_context,
835
					 event, 1);
T
Thomas Gleixner 已提交
836 837 838 839 840
		return;
	}

retry:
	task_oncpu_function_call(task, __perf_install_in_context,
841
				 event);
T
Thomas Gleixner 已提交
842 843 844 845 846

	spin_lock_irq(&ctx->lock);
	/*
	 * we need to retry the smp call.
	 */
847
	if (ctx->is_active && list_empty(&event->group_entry)) {
T
Thomas Gleixner 已提交
848 849 850 851 852 853
		spin_unlock_irq(&ctx->lock);
		goto retry;
	}

	/*
	 * The lock prevents that this context is scheduled in so we
854
	 * can add the event safely, if it the call above did not
T
Thomas Gleixner 已提交
855 856
	 * succeed.
	 */
857 858
	if (list_empty(&event->group_entry))
		add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
859 860 861
	spin_unlock_irq(&ctx->lock);
}

862
/*
863
 * Put a event into inactive state and update time fields.
864 865 866 867 868 869
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
870 871
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
872
{
873
	struct perf_event *sub;
874

875 876 877 878
	event->state = PERF_EVENT_STATE_INACTIVE;
	event->tstamp_enabled = ctx->time - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry)
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
879 880 881 882
			sub->tstamp_enabled =
				ctx->time - sub->total_time_enabled;
}

883
/*
884
 * Cross CPU call to enable a performance event
885
 */
886
static void __perf_event_enable(void *info)
887
{
888
	struct perf_event *event = info;
889
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
890 891
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
892
	int err;
893

894
	/*
895 896
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
897
	 */
898
	if (ctx->task && cpuctx->task_ctx != ctx) {
899
		if (cpuctx->task_ctx || ctx->task != current)
900 901 902
			return;
		cpuctx->task_ctx = ctx;
	}
903

904
	spin_lock(&ctx->lock);
905
	ctx->is_active = 1;
906
	update_context_time(ctx);
907

908
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
909
		goto unlock;
910
	__perf_event_mark_enabled(event, ctx);
911 912

	/*
913
	 * If the event is in a group and isn't the group leader,
914
	 * then don't put it on unless the group is on.
915
	 */
916
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
917
		goto unlock;
918

919
	if (!group_can_go_on(event, cpuctx, 1)) {
920
		err = -EEXIST;
921
	} else {
922
		perf_disable();
923 924
		if (event == leader)
			err = group_sched_in(event, cpuctx, ctx,
925 926
					     smp_processor_id());
		else
927
			err = event_sched_in(event, cpuctx, ctx,
928
					       smp_processor_id());
929
		perf_enable();
930
	}
931 932 933

	if (err) {
		/*
934
		 * If this event can't go on and it's part of a
935 936
		 * group, then the whole group has to come off.
		 */
937
		if (leader != event)
938
			group_sched_out(leader, cpuctx, ctx);
939
		if (leader->attr.pinned) {
940
			update_group_times(leader);
941
			leader->state = PERF_EVENT_STATE_ERROR;
942
		}
943 944 945
	}

 unlock:
946
	spin_unlock(&ctx->lock);
947 948 949
}

/*
950
 * Enable a event.
951
 *
952 953
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
954
 * remains valid.  This condition is satisfied when called through
955 956
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
957
 */
958
static void perf_event_enable(struct perf_event *event)
959
{
960
	struct perf_event_context *ctx = event->ctx;
961 962 963 964
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
965
		 * Enable the event on the cpu that it's on
966
		 */
967 968
		smp_call_function_single(event->cpu, __perf_event_enable,
					 event, 1);
969 970 971 972
		return;
	}

	spin_lock_irq(&ctx->lock);
973
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
974 975 976
		goto out;

	/*
977 978
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
979 980 981 982
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
983 984
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
985 986 987

 retry:
	spin_unlock_irq(&ctx->lock);
988
	task_oncpu_function_call(task, __perf_event_enable, event);
989 990 991 992

	spin_lock_irq(&ctx->lock);

	/*
993
	 * If the context is active and the event is still off,
994 995
	 * we need to retry the cross-call.
	 */
996
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)
997 998 999 1000 1001 1002
		goto retry;

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1003 1004
	if (event->state == PERF_EVENT_STATE_OFF)
		__perf_event_mark_enabled(event, ctx);
1005

1006 1007 1008 1009
 out:
	spin_unlock_irq(&ctx->lock);
}

1010
static int perf_event_refresh(struct perf_event *event, int refresh)
1011
{
1012
	/*
1013
	 * not supported on inherited events
1014
	 */
1015
	if (event->attr.inherit)
1016 1017
		return -EINVAL;

1018 1019
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1020 1021

	return 0;
1022 1023
}

1024
void __perf_event_sched_out(struct perf_event_context *ctx,
1025 1026
			      struct perf_cpu_context *cpuctx)
{
1027
	struct perf_event *event;
1028

1029 1030
	spin_lock(&ctx->lock);
	ctx->is_active = 0;
1031
	if (likely(!ctx->nr_events))
1032
		goto out;
1033
	update_context_time(ctx);
1034

1035
	perf_disable();
1036 1037 1038 1039
	if (ctx->nr_active)
		list_for_each_entry(event, &ctx->group_list, group_entry)
			group_sched_out(event, cpuctx, ctx);

1040
	perf_enable();
1041
 out:
1042 1043 1044
	spin_unlock(&ctx->lock);
}

1045 1046 1047
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1048 1049 1050 1051
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1052
 * in them directly with an fd; we can only enable/disable all
1053
 * events via prctl, or enable/disable all events in a family
1054 1055
 * via ioctl, which will have the same effect on both contexts.
 */
1056 1057
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1058 1059
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1060
		&& ctx1->parent_gen == ctx2->parent_gen
1061
		&& !ctx1->pin_count && !ctx2->pin_count;
1062 1063
}

1064
static void __perf_event_read(void *event);
1065

1066 1067
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1068 1069 1070
{
	u64 value;

1071
	if (!event->attr.inherit_stat)
1072 1073 1074
		return;

	/*
1075
	 * Update the event value, we cannot use perf_event_read()
1076 1077
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1078
	 * we know the event must be on the current CPU, therefore we
1079 1080
	 * don't need to use it.
	 */
1081 1082 1083
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
		__perf_event_read(event);
1084 1085
		break;

1086 1087
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1088 1089 1090 1091 1092 1093 1094
		break;

	default:
		break;
	}

	/*
1095
	 * In order to keep per-task stats reliable we need to flip the event
1096 1097
	 * values when we flip the contexts.
	 */
1098 1099 1100
	value = atomic64_read(&next_event->count);
	value = atomic64_xchg(&event->count, value);
	atomic64_set(&next_event->count, value);
1101

1102 1103
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1104

1105
	/*
1106
	 * Since we swizzled the values, update the user visible data too.
1107
	 */
1108 1109
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1110 1111 1112 1113 1114
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1115 1116
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1117
{
1118
	struct perf_event *event, *next_event;
1119 1120 1121 1122

	if (!ctx->nr_stat)
		return;

1123 1124
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1125

1126 1127
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1128

1129 1130
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1131

1132
		__perf_event_sync_stat(event, next_event);
1133

1134 1135
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1136 1137 1138
	}
}

T
Thomas Gleixner 已提交
1139
/*
1140
 * Called from scheduler to remove the events of the current task,
T
Thomas Gleixner 已提交
1141 1142
 * with interrupts disabled.
 *
1143
 * We stop each event and update the event value in event->count.
T
Thomas Gleixner 已提交
1144
 *
I
Ingo Molnar 已提交
1145
 * This does not protect us against NMI, but disable()
1146 1147 1148
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
T
Thomas Gleixner 已提交
1149
 */
1150
void perf_event_task_sched_out(struct task_struct *task,
1151
				 struct task_struct *next, int cpu)
T
Thomas Gleixner 已提交
1152 1153
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1154 1155 1156
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1157
	struct pt_regs *regs;
1158
	int do_switch = 1;
T
Thomas Gleixner 已提交
1159

1160
	regs = task_pt_regs(task);
1161
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, regs, 0);
1162

1163
	if (likely(!ctx || !cpuctx->task_ctx))
T
Thomas Gleixner 已提交
1164 1165
		return;

1166
	update_context_time(ctx);
1167 1168 1169

	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1170
	next_ctx = next->perf_event_ctxp;
1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
		spin_lock(&ctx->lock);
		spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
		if (context_equiv(ctx, next_ctx)) {
1185 1186
			/*
			 * XXX do we need a memory barrier of sorts
1187
			 * wrt to rcu_dereference() of perf_event_ctxp
1188
			 */
1189 1190
			task->perf_event_ctxp = next_ctx;
			next->perf_event_ctxp = ctx;
1191 1192 1193
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1194

1195
			perf_event_sync_stat(ctx, next_ctx);
1196 1197 1198
		}
		spin_unlock(&next_ctx->lock);
		spin_unlock(&ctx->lock);
1199
	}
1200
	rcu_read_unlock();
1201

1202
	if (do_switch) {
1203
		__perf_event_sched_out(ctx, cpuctx);
1204 1205
		cpuctx->task_ctx = NULL;
	}
T
Thomas Gleixner 已提交
1206 1207
}

1208 1209 1210
/*
 * Called with IRQs disabled
 */
1211
static void __perf_event_task_sched_out(struct perf_event_context *ctx)
1212 1213 1214
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);

1215 1216
	if (!cpuctx->task_ctx)
		return;
1217 1218 1219 1220

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

1221
	__perf_event_sched_out(ctx, cpuctx);
1222 1223 1224
	cpuctx->task_ctx = NULL;
}

1225 1226 1227
/*
 * Called with IRQs disabled
 */
1228
static void perf_event_cpu_sched_out(struct perf_cpu_context *cpuctx)
1229
{
1230
	__perf_event_sched_out(&cpuctx->ctx, cpuctx);
1231 1232
}

1233
static void
1234
__perf_event_sched_in(struct perf_event_context *ctx,
1235
			struct perf_cpu_context *cpuctx, int cpu)
T
Thomas Gleixner 已提交
1236
{
1237
	struct perf_event *event;
1238
	int can_add_hw = 1;
T
Thomas Gleixner 已提交
1239

1240 1241
	spin_lock(&ctx->lock);
	ctx->is_active = 1;
1242
	if (likely(!ctx->nr_events))
1243
		goto out;
T
Thomas Gleixner 已提交
1244

1245
	ctx->timestamp = perf_clock();
1246

1247
	perf_disable();
1248 1249 1250 1251 1252

	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
1253 1254 1255
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF ||
		    !event->attr.pinned)
1256
			continue;
1257
		if (event->cpu != -1 && event->cpu != cpu)
1258 1259
			continue;

1260 1261
		if (group_can_go_on(event, cpuctx, 1))
			group_sched_in(event, cpuctx, ctx, cpu);
1262 1263 1264 1265 1266

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
1267 1268 1269
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
1270
		}
1271 1272
	}

1273
	list_for_each_entry(event, &ctx->group_list, group_entry) {
1274
		/*
1275 1276
		 * Ignore events in OFF or ERROR state, and
		 * ignore pinned events since we did them already.
1277
		 */
1278 1279
		if (event->state <= PERF_EVENT_STATE_OFF ||
		    event->attr.pinned)
1280 1281
			continue;

1282 1283
		/*
		 * Listen to the 'cpu' scheduling filter constraint
1284
		 * of events:
1285
		 */
1286
		if (event->cpu != -1 && event->cpu != cpu)
T
Thomas Gleixner 已提交
1287 1288
			continue;

1289 1290
		if (group_can_go_on(event, cpuctx, can_add_hw))
			if (group_sched_in(event, cpuctx, ctx, cpu))
1291
				can_add_hw = 0;
T
Thomas Gleixner 已提交
1292
	}
1293
	perf_enable();
1294
 out:
T
Thomas Gleixner 已提交
1295
	spin_unlock(&ctx->lock);
1296 1297 1298
}

/*
1299
 * Called from scheduler to add the events of the current task
1300 1301
 * with interrupts disabled.
 *
1302
 * We restore the event value and then enable it.
1303 1304
 *
 * This does not protect us against NMI, but enable()
1305 1306 1307
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
1308
 */
1309
void perf_event_task_sched_in(struct task_struct *task, int cpu)
1310 1311
{
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1312
	struct perf_event_context *ctx = task->perf_event_ctxp;
1313

1314 1315
	if (likely(!ctx))
		return;
1316 1317
	if (cpuctx->task_ctx == ctx)
		return;
1318
	__perf_event_sched_in(ctx, cpuctx, cpu);
T
Thomas Gleixner 已提交
1319 1320 1321
	cpuctx->task_ctx = ctx;
}

1322
static void perf_event_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
1323
{
1324
	struct perf_event_context *ctx = &cpuctx->ctx;
1325

1326
	__perf_event_sched_in(ctx, cpuctx, cpu);
1327 1328
}

1329 1330
#define MAX_INTERRUPTS (~0ULL)

1331
static void perf_log_throttle(struct perf_event *event, int enable);
1332

1333
static void perf_adjust_period(struct perf_event *event, u64 events)
1334
{
1335
	struct hw_perf_event *hwc = &event->hw;
1336 1337 1338 1339
	u64 period, sample_period;
	s64 delta;

	events *= hwc->sample_period;
1340
	period = div64_u64(events, event->attr.sample_freq);
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
}

1353
static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1354
{
1355 1356
	struct perf_event *event;
	struct hw_perf_event *hwc;
1357
	u64 interrupts, freq;
1358 1359

	spin_lock(&ctx->lock);
1360
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
1361
		if (event->state != PERF_EVENT_STATE_ACTIVE)
1362 1363
			continue;

1364
		hwc = &event->hw;
1365 1366 1367

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
1368

1369
		/*
1370
		 * unthrottle events on the tick
1371
		 */
1372
		if (interrupts == MAX_INTERRUPTS) {
1373 1374 1375
			perf_log_throttle(event, 1);
			event->pmu->unthrottle(event);
			interrupts = 2*sysctl_perf_event_sample_rate/HZ;
1376 1377
		}

1378
		if (!event->attr.freq || !event->attr.sample_freq)
1379 1380
			continue;

1381 1382 1383
		/*
		 * if the specified freq < HZ then we need to skip ticks
		 */
1384 1385
		if (event->attr.sample_freq < HZ) {
			freq = event->attr.sample_freq;
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398

			hwc->freq_count += freq;
			hwc->freq_interrupts += interrupts;

			if (hwc->freq_count < HZ)
				continue;

			interrupts = hwc->freq_interrupts;
			hwc->freq_interrupts = 0;
			hwc->freq_count -= HZ;
		} else
			freq = HZ;

1399
		perf_adjust_period(event, freq * interrupts);
1400

1401 1402 1403 1404 1405 1406 1407
		/*
		 * In order to avoid being stalled by an (accidental) huge
		 * sample period, force reset the sample period if we didn't
		 * get any events in this freq period.
		 */
		if (!interrupts) {
			perf_disable();
1408
			event->pmu->disable(event);
1409
			atomic64_set(&hwc->period_left, 0);
1410
			event->pmu->enable(event);
1411 1412
			perf_enable();
		}
1413 1414 1415 1416
	}
	spin_unlock(&ctx->lock);
}

1417
/*
1418
 * Round-robin a context's events:
1419
 */
1420
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
1421
{
1422
	struct perf_event *event;
T
Thomas Gleixner 已提交
1423

1424
	if (!ctx->nr_events)
T
Thomas Gleixner 已提交
1425 1426 1427 1428
		return;

	spin_lock(&ctx->lock);
	/*
1429
	 * Rotate the first entry last (works just fine for group events too):
T
Thomas Gleixner 已提交
1430
	 */
1431
	perf_disable();
1432 1433
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		list_move_tail(&event->group_entry, &ctx->group_list);
T
Thomas Gleixner 已提交
1434 1435
		break;
	}
1436
	perf_enable();
T
Thomas Gleixner 已提交
1437 1438

	spin_unlock(&ctx->lock);
1439 1440
}

1441
void perf_event_task_tick(struct task_struct *curr, int cpu)
1442
{
1443
	struct perf_cpu_context *cpuctx;
1444
	struct perf_event_context *ctx;
1445

1446
	if (!atomic_read(&nr_events))
1447 1448 1449
		return;

	cpuctx = &per_cpu(perf_cpu_context, cpu);
1450
	ctx = curr->perf_event_ctxp;
1451

1452
	perf_ctx_adjust_freq(&cpuctx->ctx);
1453
	if (ctx)
1454
		perf_ctx_adjust_freq(ctx);
1455

1456
	perf_event_cpu_sched_out(cpuctx);
1457
	if (ctx)
1458
		__perf_event_task_sched_out(ctx);
T
Thomas Gleixner 已提交
1459

1460
	rotate_ctx(&cpuctx->ctx);
1461 1462
	if (ctx)
		rotate_ctx(ctx);
1463

1464
	perf_event_cpu_sched_in(cpuctx, cpu);
1465
	if (ctx)
1466
		perf_event_task_sched_in(curr, cpu);
T
Thomas Gleixner 已提交
1467 1468
}

1469
/*
1470
 * Enable all of a task's events that have been marked enable-on-exec.
1471 1472
 * This expects task == current.
 */
1473
static void perf_event_enable_on_exec(struct task_struct *task)
1474
{
1475 1476
	struct perf_event_context *ctx;
	struct perf_event *event;
1477 1478 1479 1480
	unsigned long flags;
	int enabled = 0;

	local_irq_save(flags);
1481 1482
	ctx = task->perf_event_ctxp;
	if (!ctx || !ctx->nr_events)
1483 1484
		goto out;

1485
	__perf_event_task_sched_out(ctx);
1486 1487 1488

	spin_lock(&ctx->lock);

1489 1490
	list_for_each_entry(event, &ctx->group_list, group_entry) {
		if (!event->attr.enable_on_exec)
1491
			continue;
1492 1493
		event->attr.enable_on_exec = 0;
		if (event->state >= PERF_EVENT_STATE_INACTIVE)
1494
			continue;
1495
		__perf_event_mark_enabled(event, ctx);
1496 1497 1498 1499
		enabled = 1;
	}

	/*
1500
	 * Unclone this context if we enabled any event.
1501
	 */
1502 1503
	if (enabled)
		unclone_ctx(ctx);
1504 1505 1506

	spin_unlock(&ctx->lock);

1507
	perf_event_task_sched_in(task, smp_processor_id());
1508 1509 1510 1511
 out:
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
1512
/*
1513
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
1514
 */
1515
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
1516
{
1517
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1518 1519
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
I
Ingo Molnar 已提交
1520
	unsigned long flags;
I
Ingo Molnar 已提交
1521

1522 1523 1524 1525
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
1526 1527
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
1528 1529 1530 1531
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

1532
	local_irq_save(flags);
1533
	if (ctx->is_active)
1534
		update_context_time(ctx);
1535 1536
	event->pmu->read(event);
	update_event_times(event);
1537
	local_irq_restore(flags);
T
Thomas Gleixner 已提交
1538 1539
}

1540
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
1541 1542
{
	/*
1543 1544
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
1545
	 */
1546 1547 1548 1549 1550
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_event_times(event);
T
Thomas Gleixner 已提交
1551 1552
	}

1553
	return atomic64_read(&event->count);
T
Thomas Gleixner 已提交
1554 1555
}

1556
/*
1557
 * Initialize the perf_event context in a task_struct:
1558 1559
 */
static void
1560
__perf_event_init_context(struct perf_event_context *ctx,
1561 1562 1563 1564 1565
			    struct task_struct *task)
{
	memset(ctx, 0, sizeof(*ctx));
	spin_lock_init(&ctx->lock);
	mutex_init(&ctx->mutex);
1566
	INIT_LIST_HEAD(&ctx->group_list);
1567 1568 1569 1570 1571
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
	ctx->task = task;
}

1572
static struct perf_event_context *find_get_context(pid_t pid, int cpu)
T
Thomas Gleixner 已提交
1573
{
1574
	struct perf_event_context *ctx;
1575
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
1576
	struct task_struct *task;
1577
	unsigned long flags;
1578
	int err;
T
Thomas Gleixner 已提交
1579 1580

	/*
1581
	 * If cpu is not a wildcard then this is a percpu event:
T
Thomas Gleixner 已提交
1582 1583
	 */
	if (cpu != -1) {
1584
		/* Must be root to operate on a CPU event: */
1585
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
1586 1587 1588 1589 1590 1591
			return ERR_PTR(-EACCES);

		if (cpu < 0 || cpu > num_possible_cpus())
			return ERR_PTR(-EINVAL);

		/*
1592
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
1593 1594 1595 1596 1597 1598 1599 1600
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
		if (!cpu_isset(cpu, cpu_online_map))
			return ERR_PTR(-ENODEV);

		cpuctx = &per_cpu(perf_cpu_context, cpu);
		ctx = &cpuctx->ctx;
1601
		get_ctx(ctx);
T
Thomas Gleixner 已提交
1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617

		return ctx;
	}

	rcu_read_lock();
	if (!pid)
		task = current;
	else
		task = find_task_by_vpid(pid);
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

1618
	/*
1619
	 * Can't attach events to a dying task.
1620 1621 1622 1623 1624
	 */
	err = -ESRCH;
	if (task->flags & PF_EXITING)
		goto errout;

T
Thomas Gleixner 已提交
1625
	/* Reuse ptrace permission checks for now. */
1626 1627 1628 1629 1630
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

 retry:
1631
	ctx = perf_lock_task_context(task, &flags);
1632
	if (ctx) {
1633
		unclone_ctx(ctx);
1634
		spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
1635 1636
	}

1637
	if (!ctx) {
1638
		ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL);
1639 1640 1641
		err = -ENOMEM;
		if (!ctx)
			goto errout;
1642
		__perf_event_init_context(ctx, task);
1643
		get_ctx(ctx);
1644
		if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
1645 1646 1647 1648 1649
			/*
			 * We raced with some other task; use
			 * the context they set.
			 */
			kfree(ctx);
1650
			goto retry;
1651
		}
1652
		get_task_struct(task);
1653 1654
	}

1655
	put_task_struct(task);
T
Thomas Gleixner 已提交
1656
	return ctx;
1657 1658 1659 1660

 errout:
	put_task_struct(task);
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
1661 1662
}

L
Li Zefan 已提交
1663 1664
static void perf_event_free_filter(struct perf_event *event);

1665
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
1666
{
1667
	struct perf_event *event;
P
Peter Zijlstra 已提交
1668

1669 1670 1671
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
1672
	perf_event_free_filter(event);
1673
	kfree(event);
P
Peter Zijlstra 已提交
1674 1675
}

1676
static void perf_pending_sync(struct perf_event *event);
1677

1678
static void free_event(struct perf_event *event)
1679
{
1680
	perf_pending_sync(event);
1681

1682 1683 1684 1685 1686 1687 1688 1689
	if (!event->parent) {
		atomic_dec(&nr_events);
		if (event->attr.mmap)
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
1690
	}
1691

1692 1693 1694
	if (event->output) {
		fput(event->output->filp);
		event->output = NULL;
1695 1696
	}

1697 1698
	if (event->destroy)
		event->destroy(event);
1699

1700 1701
	put_ctx(event->ctx);
	call_rcu(&event->rcu_head, free_event_rcu);
1702 1703
}

T
Thomas Gleixner 已提交
1704 1705 1706 1707 1708
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
{
1709 1710
	struct perf_event *event = file->private_data;
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1711 1712 1713

	file->private_data = NULL;

1714
	WARN_ON_ONCE(ctx->parent_ctx);
1715
	mutex_lock(&ctx->mutex);
1716
	perf_event_remove_from_context(event);
1717
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
1718

1719 1720 1721 1722
	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);
1723

1724
	free_event(event);
T
Thomas Gleixner 已提交
1725 1726 1727 1728

	return 0;
}

1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748
int perf_event_release_kernel(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;

	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_event_remove_from_context(event);
	mutex_unlock(&ctx->mutex);

	mutex_lock(&event->owner->perf_event_mutex);
	list_del_init(&event->owner_entry);
	mutex_unlock(&event->owner->perf_event_mutex);
	put_task_struct(event->owner);

	free_event(event);

	return 0;
}
EXPORT_SYMBOL_GPL(perf_event_release_kernel);

1749
static int perf_event_read_size(struct perf_event *event)
1750 1751 1752 1753 1754
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

1755
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1756 1757
		size += sizeof(u64);

1758
	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1759 1760
		size += sizeof(u64);

1761
	if (event->attr.read_format & PERF_FORMAT_ID)
1762 1763
		entry += sizeof(u64);

1764 1765
	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
1766 1767 1768 1769 1770 1771 1772 1773
		size += sizeof(u64);
	}

	size += entry * nr;

	return size;
}

1774
u64 perf_event_read_value(struct perf_event *event)
1775
{
1776
	struct perf_event *child;
1777 1778
	u64 total = 0;

1779 1780 1781
	total += perf_event_read(event);
	list_for_each_entry(child, &event->child_list, child_list)
		total += perf_event_read(child);
1782 1783 1784

	return total;
}
1785
EXPORT_SYMBOL_GPL(perf_event_read_value);
1786

1787
static int perf_event_read_entry(struct perf_event *event,
1788 1789 1790 1791 1792
				   u64 read_format, char __user *buf)
{
	int n = 0, count = 0;
	u64 values[2];

1793
	values[n++] = perf_event_read_value(event);
1794
	if (read_format & PERF_FORMAT_ID)
1795
		values[n++] = primary_event_id(event);
1796 1797 1798 1799 1800 1801 1802 1803 1804

	count = n * sizeof(u64);

	if (copy_to_user(buf, values, count))
		return -EFAULT;

	return count;
}

1805
static int perf_event_read_group(struct perf_event *event,
1806 1807
				   u64 read_format, char __user *buf)
{
1808
	struct perf_event *leader = event->group_leader, *sub;
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826
	int n = 0, size = 0, err = -EFAULT;
	u64 values[3];

	values[n++] = 1 + leader->nr_siblings;
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
		values[n++] = leader->total_time_enabled +
			atomic64_read(&leader->child_total_time_enabled);
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
		values[n++] = leader->total_time_running +
			atomic64_read(&leader->child_total_time_running);
	}

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
		return -EFAULT;

1827
	err = perf_event_read_entry(leader, read_format, buf + size);
1828 1829 1830 1831 1832
	if (err < 0)
		return err;

	size += err;

1833
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
1834
		err = perf_event_read_entry(sub, read_format,
1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
				buf + size);
		if (err < 0)
			return err;

		size += err;
	}

	return size;
}

1845
static int perf_event_read_one(struct perf_event *event,
1846 1847 1848 1849 1850
				 u64 read_format, char __user *buf)
{
	u64 values[4];
	int n = 0;

1851
	values[n++] = perf_event_read_value(event);
1852
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1853 1854
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
1855 1856
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1857 1858
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
1859 1860
	}
	if (read_format & PERF_FORMAT_ID)
1861
		values[n++] = primary_event_id(event);
1862 1863 1864 1865 1866 1867 1868

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
1869
/*
1870
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
1871 1872
 */
static ssize_t
1873
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
1874
{
1875
	u64 read_format = event->attr.read_format;
1876
	int ret;
T
Thomas Gleixner 已提交
1877

1878
	/*
1879
	 * Return end-of-file for a read on a event that is in
1880 1881 1882
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
1883
	if (event->state == PERF_EVENT_STATE_ERROR)
1884 1885
		return 0;

1886
	if (count < perf_event_read_size(event))
1887 1888
		return -ENOSPC;

1889 1890
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
1891
	if (read_format & PERF_FORMAT_GROUP)
1892
		ret = perf_event_read_group(event, read_format, buf);
1893
	else
1894 1895
		ret = perf_event_read_one(event, read_format, buf);
	mutex_unlock(&event->child_mutex);
T
Thomas Gleixner 已提交
1896

1897
	return ret;
T
Thomas Gleixner 已提交
1898 1899 1900 1901 1902
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
1903
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
1904

1905
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
1906 1907 1908 1909
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
1910
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
1911
	struct perf_mmap_data *data;
1912
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
1913 1914

	rcu_read_lock();
1915
	data = rcu_dereference(event->data);
P
Peter Zijlstra 已提交
1916
	if (data)
1917
		events = atomic_xchg(&data->poll, 0);
P
Peter Zijlstra 已提交
1918
	rcu_read_unlock();
T
Thomas Gleixner 已提交
1919

1920
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
1921 1922 1923 1924

	return events;
}

1925
static void perf_event_reset(struct perf_event *event)
1926
{
1927 1928 1929
	(void)perf_event_read(event);
	atomic64_set(&event->count, 0);
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
1930 1931
}

1932
/*
1933 1934 1935 1936
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
1937
 */
1938 1939
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
1940
{
1941
	struct perf_event *child;
P
Peter Zijlstra 已提交
1942

1943 1944 1945 1946
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
1947
		func(child);
1948
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
1949 1950
}

1951 1952
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
1953
{
1954 1955
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
1956

1957 1958
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
1959
	event = event->group_leader;
1960

1961 1962 1963 1964
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
1965
	mutex_unlock(&ctx->mutex);
1966 1967
}

1968
static int perf_event_period(struct perf_event *event, u64 __user *arg)
1969
{
1970
	struct perf_event_context *ctx = event->ctx;
1971 1972 1973 1974
	unsigned long size;
	int ret = 0;
	u64 value;

1975
	if (!event->attr.sample_period)
1976 1977 1978 1979 1980 1981 1982 1983 1984 1985
		return -EINVAL;

	size = copy_from_user(&value, arg, sizeof(value));
	if (size != sizeof(value))
		return -EFAULT;

	if (!value)
		return -EINVAL;

	spin_lock_irq(&ctx->lock);
1986 1987
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
1988 1989 1990 1991
			ret = -EINVAL;
			goto unlock;
		}

1992
		event->attr.sample_freq = value;
1993
	} else {
1994 1995
		event->attr.sample_period = value;
		event->hw.sample_period = value;
1996 1997 1998 1999 2000 2001 2002
	}
unlock:
	spin_unlock_irq(&ctx->lock);

	return ret;
}

L
Li Zefan 已提交
2003 2004
static int perf_event_set_output(struct perf_event *event, int output_fd);
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
2005

2006 2007
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
2008 2009
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
2010
	u32 flags = arg;
2011 2012

	switch (cmd) {
2013 2014
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
2015
		break;
2016 2017
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
2018
		break;
2019 2020
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
2021
		break;
P
Peter Zijlstra 已提交
2022

2023 2024
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
2025

2026 2027
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
2028

2029 2030
	case PERF_EVENT_IOC_SET_OUTPUT:
		return perf_event_set_output(event, arg);
2031

L
Li Zefan 已提交
2032 2033 2034
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

2035
	default:
P
Peter Zijlstra 已提交
2036
		return -ENOTTY;
2037
	}
P
Peter Zijlstra 已提交
2038 2039

	if (flags & PERF_IOC_FLAG_GROUP)
2040
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
2041
	else
2042
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
2043 2044

	return 0;
2045 2046
}

2047
int perf_event_task_enable(void)
2048
{
2049
	struct perf_event *event;
2050

2051 2052 2053 2054
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
2055 2056 2057 2058

	return 0;
}

2059
int perf_event_task_disable(void)
2060
{
2061
	struct perf_event *event;
2062

2063 2064 2065 2066
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
2067 2068 2069 2070

	return 0;
}

2071 2072
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
I
Ingo Molnar 已提交
2073 2074
#endif

2075
static int perf_event_index(struct perf_event *event)
2076
{
2077
	if (event->state != PERF_EVENT_STATE_ACTIVE)
2078 2079
		return 0;

2080
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
2081 2082
}

2083 2084 2085 2086 2087
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
2088
void perf_event_update_userpage(struct perf_event *event)
2089
{
2090
	struct perf_event_mmap_page *userpg;
2091
	struct perf_mmap_data *data;
2092 2093

	rcu_read_lock();
2094
	data = rcu_dereference(event->data);
2095 2096 2097 2098
	if (!data)
		goto unlock;

	userpg = data->user_page;
2099

2100 2101 2102 2103 2104
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
2105
	++userpg->lock;
2106
	barrier();
2107 2108 2109 2110
	userpg->index = perf_event_index(event);
	userpg->offset = atomic64_read(&event->count);
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		userpg->offset -= atomic64_read(&event->hw.prev_count);
2111

2112 2113
	userpg->time_enabled = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2114

2115 2116
	userpg->time_running = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2117

2118
	barrier();
2119
	++userpg->lock;
2120
	preempt_enable();
2121
unlock:
2122
	rcu_read_unlock();
2123 2124
}

2125
static unsigned long perf_data_size(struct perf_mmap_data *data)
2126
{
2127 2128
	return data->nr_pages << (PAGE_SHIFT + data->data_order);
}
2129

2130
#ifndef CONFIG_PERF_USE_VMALLOC
2131

2132 2133 2134
/*
 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
 */
2135

2136 2137 2138 2139 2140
static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > data->nr_pages)
		return NULL;
2141

2142 2143
	if (pgoff == 0)
		return virt_to_page(data->user_page);
2144

2145
	return virt_to_page(data->data_pages[pgoff - 1]);
2146 2147
}

2148 2149
static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
2150 2151 2152 2153 2154
{
	struct perf_mmap_data *data;
	unsigned long size;
	int i;

2155
	WARN_ON(atomic_read(&event->mmap_count));
2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173

	size = sizeof(struct perf_mmap_data);
	size += nr_pages * sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
	if (!data->user_page)
		goto fail_user_page;

	for (i = 0; i < nr_pages; i++) {
		data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
		if (!data->data_pages[i])
			goto fail_data_pages;
	}

2174
	data->data_order = 0;
2175 2176
	data->nr_pages = nr_pages;

2177
	return data;
2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188

fail_data_pages:
	for (i--; i >= 0; i--)
		free_page((unsigned long)data->data_pages[i]);

	free_page((unsigned long)data->user_page);

fail_user_page:
	kfree(data);

fail:
2189
	return NULL;
2190 2191
}

2192 2193
static void perf_mmap_free_page(unsigned long addr)
{
K
Kevin Cernekee 已提交
2194
	struct page *page = virt_to_page((void *)addr);
2195 2196 2197 2198 2199

	page->mapping = NULL;
	__free_page(page);
}

2200
static void perf_mmap_data_free(struct perf_mmap_data *data)
2201 2202 2203
{
	int i;

2204
	perf_mmap_free_page((unsigned long)data->user_page);
2205
	for (i = 0; i < data->nr_pages; i++)
2206
		perf_mmap_free_page((unsigned long)data->data_pages[i]);
2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261
}

#else

/*
 * Back perf_mmap() with vmalloc memory.
 *
 * Required for architectures that have d-cache aliasing issues.
 */

static struct page *
perf_mmap_to_page(struct perf_mmap_data *data, unsigned long pgoff)
{
	if (pgoff > (1UL << data->data_order))
		return NULL;

	return vmalloc_to_page((void *)data->user_page + pgoff * PAGE_SIZE);
}

static void perf_mmap_unmark_page(void *addr)
{
	struct page *page = vmalloc_to_page(addr);

	page->mapping = NULL;
}

static void perf_mmap_data_free_work(struct work_struct *work)
{
	struct perf_mmap_data *data;
	void *base;
	int i, nr;

	data = container_of(work, struct perf_mmap_data, work);
	nr = 1 << data->data_order;

	base = data->user_page;
	for (i = 0; i < nr + 1; i++)
		perf_mmap_unmark_page(base + (i * PAGE_SIZE));

	vfree(base);
}

static void perf_mmap_data_free(struct perf_mmap_data *data)
{
	schedule_work(&data->work);
}

static struct perf_mmap_data *
perf_mmap_data_alloc(struct perf_event *event, int nr_pages)
{
	struct perf_mmap_data *data;
	unsigned long size;
	void *all_buf;

	WARN_ON(atomic_read(&event->mmap_count));
2262

2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351
	size = sizeof(struct perf_mmap_data);
	size += sizeof(void *);

	data = kzalloc(size, GFP_KERNEL);
	if (!data)
		goto fail;

	INIT_WORK(&data->work, perf_mmap_data_free_work);

	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
	if (!all_buf)
		goto fail_all_buf;

	data->user_page = all_buf;
	data->data_pages[0] = all_buf + PAGE_SIZE;
	data->data_order = ilog2(nr_pages);
	data->nr_pages = 1;

	return data;

fail_all_buf:
	kfree(data);

fail:
	return NULL;
}

#endif

static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
	struct perf_mmap_data *data;
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
	data = rcu_dereference(event->data);
	if (!data)
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

	vmf->page = perf_mmap_to_page(data, vmf->pgoff);
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

static void
perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data)
{
	long max_size = perf_data_size(data);

	atomic_set(&data->lock, -1);

	if (event->attr.watermark) {
		data->watermark = min_t(long, max_size,
					event->attr.wakeup_watermark);
	}

	if (!data->watermark)
		data->watermark = max_t(long, PAGE_SIZE, max_size / 2);


	rcu_assign_pointer(event->data, data);
}

static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head)
{
	struct perf_mmap_data *data;

	data = container_of(rcu_head, struct perf_mmap_data, rcu_head);
	perf_mmap_data_free(data);
2352 2353 2354
	kfree(data);
}

2355
static void perf_mmap_data_release(struct perf_event *event)
2356
{
2357
	struct perf_mmap_data *data = event->data;
2358

2359
	WARN_ON(atomic_read(&event->mmap_count));
2360

2361
	rcu_assign_pointer(event->data, NULL);
2362
	call_rcu(&data->rcu_head, perf_mmap_data_free_rcu);
2363 2364 2365 2366
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
2367
	struct perf_event *event = vma->vm_file->private_data;
2368

2369
	atomic_inc(&event->mmap_count);
2370 2371 2372 2373
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
2374
	struct perf_event *event = vma->vm_file->private_data;
2375

2376 2377
	WARN_ON_ONCE(event->ctx->parent_ctx);
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
2378
		unsigned long size = perf_data_size(event->data);
2379 2380
		struct user_struct *user = current_user();

2381
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
2382
		vma->vm_mm->locked_vm -= event->data->nr_locked;
2383
		perf_mmap_data_release(event);
2384
		mutex_unlock(&event->mmap_mutex);
2385
	}
2386 2387
}

2388
static const struct vm_operations_struct perf_mmap_vmops = {
2389 2390 2391 2392
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
2393 2394 2395 2396
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
2397
	struct perf_event *event = file->private_data;
2398
	unsigned long user_locked, user_lock_limit;
2399
	struct user_struct *user = current_user();
2400
	unsigned long locked, lock_limit;
2401
	struct perf_mmap_data *data;
2402 2403
	unsigned long vma_size;
	unsigned long nr_pages;
2404
	long user_extra, extra;
2405
	int ret = 0;
2406

2407
	if (!(vma->vm_flags & VM_SHARED))
2408
		return -EINVAL;
2409 2410 2411 2412

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

2413 2414 2415 2416 2417
	/*
	 * If we have data pages ensure they're a power-of-two number, so we
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
2418 2419
		return -EINVAL;

2420
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
2421 2422
		return -EINVAL;

2423 2424
	if (vma->vm_pgoff != 0)
		return -EINVAL;
2425

2426 2427 2428
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
	if (event->output) {
2429 2430 2431 2432
		ret = -EINVAL;
		goto unlock;
	}

2433 2434
	if (atomic_inc_not_zero(&event->mmap_count)) {
		if (nr_pages != event->data->nr_pages)
2435 2436 2437 2438
			ret = -EINVAL;
		goto unlock;
	}

2439
	user_extra = nr_pages + 1;
2440
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
2441 2442 2443 2444 2445 2446

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

2447
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
2448

2449 2450 2451
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
2452 2453 2454

	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
	lock_limit >>= PAGE_SHIFT;
2455
	locked = vma->vm_mm->locked_vm + extra;
2456

2457 2458
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
2459 2460 2461
		ret = -EPERM;
		goto unlock;
	}
2462

2463
	WARN_ON(event->data);
2464 2465 2466 2467

	data = perf_mmap_data_alloc(event, nr_pages);
	ret = -ENOMEM;
	if (!data)
2468 2469
		goto unlock;

2470 2471 2472
	ret = 0;
	perf_mmap_data_init(event, data);

2473
	atomic_set(&event->mmap_count, 1);
2474
	atomic_long_add(user_extra, &user->locked_vm);
2475
	vma->vm_mm->locked_vm += extra;
2476
	event->data->nr_locked = extra;
2477
	if (vma->vm_flags & VM_WRITE)
2478
		event->data->writable = 1;
2479

2480
unlock:
2481
	mutex_unlock(&event->mmap_mutex);
2482 2483 2484

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
2485 2486

	return ret;
2487 2488
}

P
Peter Zijlstra 已提交
2489 2490 2491
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
2492
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
2493 2494 2495
	int retval;

	mutex_lock(&inode->i_mutex);
2496
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
2497 2498 2499 2500 2501 2502 2503 2504
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
2505 2506 2507 2508
static const struct file_operations perf_fops = {
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
2509 2510
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
2511
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
2512
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
2513 2514
};

2515
/*
2516
 * Perf event wakeup
2517 2518 2519 2520 2521
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

2522
void perf_event_wakeup(struct perf_event *event)
2523
{
2524
	wake_up_all(&event->waitq);
2525

2526 2527 2528
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
2529
	}
2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
}

/*
 * Pending wakeups
 *
 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
 *
 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
 * single linked list and use cmpxchg() to add entries lockless.
 */

2541
static void perf_pending_event(struct perf_pending_entry *entry)
2542
{
2543 2544
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
2545

2546 2547 2548
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
2549 2550
	}

2551 2552 2553
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
2554 2555 2556
	}
}

2557
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
2558

2559
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
2560 2561 2562
	PENDING_TAIL,
};

2563 2564
static void perf_pending_queue(struct perf_pending_entry *entry,
			       void (*func)(struct perf_pending_entry *))
2565
{
2566
	struct perf_pending_entry **head;
2567

2568
	if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
2569 2570
		return;

2571 2572 2573
	entry->func = func;

	head = &get_cpu_var(perf_pending_head);
2574 2575

	do {
2576 2577
		entry->next = *head;
	} while (cmpxchg(head, entry->next, entry) != entry->next);
2578

2579
	set_perf_event_pending();
2580

2581
	put_cpu_var(perf_pending_head);
2582 2583 2584 2585
}

static int __perf_pending_run(void)
{
2586
	struct perf_pending_entry *list;
2587 2588
	int nr = 0;

2589
	list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
2590
	while (list != PENDING_TAIL) {
2591 2592
		void (*func)(struct perf_pending_entry *);
		struct perf_pending_entry *entry = list;
2593 2594 2595

		list = list->next;

2596 2597
		func = entry->func;
		entry->next = NULL;
2598 2599 2600 2601 2602 2603 2604
		/*
		 * Ensure we observe the unqueue before we issue the wakeup,
		 * so that we won't be waiting forever.
		 * -- see perf_not_pending().
		 */
		smp_wmb();

2605
		func(entry);
2606 2607 2608 2609 2610 2611
		nr++;
	}

	return nr;
}

2612
static inline int perf_not_pending(struct perf_event *event)
2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626
{
	/*
	 * If we flush on whatever cpu we run, there is a chance we don't
	 * need to wait.
	 */
	get_cpu();
	__perf_pending_run();
	put_cpu();

	/*
	 * Ensure we see the proper queue state before going to sleep
	 * so that we do not miss the wakeup. -- see perf_pending_handle()
	 */
	smp_rmb();
2627
	return event->pending.next == NULL;
2628 2629
}

2630
static void perf_pending_sync(struct perf_event *event)
2631
{
2632
	wait_event(event->waitq, perf_not_pending(event));
2633 2634
}

2635
void perf_event_do_pending(void)
2636 2637 2638 2639
{
	__perf_pending_run();
}

2640 2641 2642 2643
/*
 * Callchain support -- arch specific
 */

2644
__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2645 2646 2647 2648
{
	return NULL;
}

2649 2650 2651
/*
 * Output
 */
2652 2653
static bool perf_output_space(struct perf_mmap_data *data, unsigned long tail,
			      unsigned long offset, unsigned long head)
2654 2655 2656 2657 2658 2659
{
	unsigned long mask;

	if (!data->writable)
		return true;

2660
	mask = perf_data_size(data) - 1;
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670

	offset = (offset - tail) & mask;
	head   = (head   - tail) & mask;

	if ((int)(head - offset) < 0)
		return false;

	return true;
}

2671
static void perf_output_wakeup(struct perf_output_handle *handle)
2672
{
2673 2674
	atomic_set(&handle->data->poll, POLL_IN);

2675
	if (handle->nmi) {
2676 2677 2678
		handle->event->pending_wakeup = 1;
		perf_pending_queue(&handle->event->pending,
				   perf_pending_event);
2679
	} else
2680
		perf_event_wakeup(handle->event);
2681 2682
}

2683 2684 2685
/*
 * Curious locking construct.
 *
2686 2687
 * We need to ensure a later event_id doesn't publish a head when a former
 * event_id isn't done writing. However since we need to deal with NMIs we
2688 2689 2690 2691 2692 2693
 * cannot fully serialize things.
 *
 * What we do is serialize between CPUs so we only have to deal with NMI
 * nesting on a single CPU.
 *
 * We only publish the head (and generate a wakeup) when the outer-most
2694
 * event_id completes.
2695 2696 2697 2698
 */
static void perf_output_lock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2699
	int cur, cpu = get_cpu();
2700 2701 2702

	handle->locked = 0;

2703 2704 2705 2706 2707 2708 2709 2710
	for (;;) {
		cur = atomic_cmpxchg(&data->lock, -1, cpu);
		if (cur == -1) {
			handle->locked = 1;
			break;
		}
		if (cur == cpu)
			break;
2711 2712

		cpu_relax();
2713
	}
2714 2715 2716 2717 2718
}

static void perf_output_unlock(struct perf_output_handle *handle)
{
	struct perf_mmap_data *data = handle->data;
2719 2720
	unsigned long head;
	int cpu;
2721

2722
	data->done_head = data->head;
2723 2724 2725 2726 2727 2728 2729 2730 2731 2732

	if (!handle->locked)
		goto out;

again:
	/*
	 * The xchg implies a full barrier that ensures all writes are done
	 * before we publish the new head, matched by a rmb() in userspace when
	 * reading this position.
	 */
2733
	while ((head = atomic_long_xchg(&data->done_head, 0)))
2734 2735 2736
		data->user_page->data_head = head;

	/*
2737
	 * NMI can happen here, which means we can miss a done_head update.
2738 2739
	 */

2740
	cpu = atomic_xchg(&data->lock, -1);
2741 2742 2743 2744 2745
	WARN_ON_ONCE(cpu != smp_processor_id());

	/*
	 * Therefore we have to validate we did not indeed do so.
	 */
2746
	if (unlikely(atomic_long_read(&data->done_head))) {
2747 2748 2749
		/*
		 * Since we had it locked, we can lock it again.
		 */
2750
		while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
2751 2752 2753 2754 2755
			cpu_relax();

		goto again;
	}

2756
	if (atomic_xchg(&data->wakeup, 0))
2757 2758
		perf_output_wakeup(handle);
out:
2759
	put_cpu();
2760 2761
}

2762 2763
void perf_output_copy(struct perf_output_handle *handle,
		      const void *buf, unsigned int len)
2764 2765
{
	unsigned int pages_mask;
2766
	unsigned long offset;
2767 2768 2769 2770 2771 2772 2773 2774
	unsigned int size;
	void **pages;

	offset		= handle->offset;
	pages_mask	= handle->data->nr_pages - 1;
	pages		= handle->data->data_pages;

	do {
2775 2776
		unsigned long page_offset;
		unsigned long page_size;
2777 2778 2779
		int nr;

		nr	    = (offset >> PAGE_SHIFT) & pages_mask;
2780 2781 2782
		page_size   = 1UL << (handle->data->data_order + PAGE_SHIFT);
		page_offset = offset & (page_size - 1);
		size	    = min_t(unsigned int, page_size - page_offset, len);
2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799

		memcpy(pages[nr] + page_offset, buf, size);

		len	    -= size;
		buf	    += size;
		offset	    += size;
	} while (len);

	handle->offset = offset;

	/*
	 * Check we didn't copy past our reservation window, taking the
	 * possible unsigned int wrap into account.
	 */
	WARN_ON_ONCE(((long)(handle->head - handle->offset)) < 0);
}

2800
int perf_output_begin(struct perf_output_handle *handle,
2801
		      struct perf_event *event, unsigned int size,
2802
		      int nmi, int sample)
2803
{
2804
	struct perf_event *output_event;
2805
	struct perf_mmap_data *data;
2806
	unsigned long tail, offset, head;
2807 2808 2809 2810 2811 2812
	int have_lost;
	struct {
		struct perf_event_header header;
		u64			 id;
		u64			 lost;
	} lost_event;
2813

2814
	rcu_read_lock();
2815
	/*
2816
	 * For inherited events we send all the output towards the parent.
2817
	 */
2818 2819
	if (event->parent)
		event = event->parent;
2820

2821 2822 2823
	output_event = rcu_dereference(event->output);
	if (output_event)
		event = output_event;
2824

2825
	data = rcu_dereference(event->data);
2826 2827 2828
	if (!data)
		goto out;

2829
	handle->data	= data;
2830
	handle->event	= event;
2831 2832
	handle->nmi	= nmi;
	handle->sample	= sample;
2833

2834
	if (!data->nr_pages)
2835
		goto fail;
2836

2837 2838 2839 2840
	have_lost = atomic_read(&data->lost);
	if (have_lost)
		size += sizeof(lost_event);

2841 2842
	perf_output_lock(handle);

2843
	do {
2844 2845 2846 2847 2848 2849 2850
		/*
		 * Userspace could choose to issue a mb() before updating the
		 * tail pointer. So that all reads will be completed before the
		 * write is issued.
		 */
		tail = ACCESS_ONCE(data->user_page->data_tail);
		smp_rmb();
2851
		offset = head = atomic_long_read(&data->head);
P
Peter Zijlstra 已提交
2852
		head += size;
2853
		if (unlikely(!perf_output_space(data, tail, offset, head)))
2854
			goto fail;
2855
	} while (atomic_long_cmpxchg(&data->head, offset, head) != offset);
2856

2857
	handle->offset	= offset;
2858
	handle->head	= head;
2859

2860
	if (head - tail > data->watermark)
2861
		atomic_set(&data->wakeup, 1);
2862

2863
	if (have_lost) {
2864
		lost_event.header.type = PERF_RECORD_LOST;
2865 2866
		lost_event.header.misc = 0;
		lost_event.header.size = sizeof(lost_event);
2867
		lost_event.id          = event->id;
2868 2869 2870 2871 2872
		lost_event.lost        = atomic_xchg(&data->lost, 0);

		perf_output_put(handle, lost_event);
	}

2873
	return 0;
2874

2875
fail:
2876 2877
	atomic_inc(&data->lost);
	perf_output_unlock(handle);
2878 2879
out:
	rcu_read_unlock();
2880

2881 2882
	return -ENOSPC;
}
2883

2884
void perf_output_end(struct perf_output_handle *handle)
2885
{
2886
	struct perf_event *event = handle->event;
2887 2888
	struct perf_mmap_data *data = handle->data;

2889
	int wakeup_events = event->attr.wakeup_events;
P
Peter Zijlstra 已提交
2890

2891
	if (handle->sample && wakeup_events) {
2892
		int events = atomic_inc_return(&data->events);
P
Peter Zijlstra 已提交
2893
		if (events >= wakeup_events) {
2894
			atomic_sub(wakeup_events, &data->events);
2895
			atomic_set(&data->wakeup, 1);
P
Peter Zijlstra 已提交
2896
		}
2897 2898 2899
	}

	perf_output_unlock(handle);
2900
	rcu_read_unlock();
2901 2902
}

2903
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
2904 2905
{
	/*
2906
	 * only top level events have the pid namespace they were created in
2907
	 */
2908 2909
	if (event->parent)
		event = event->parent;
2910

2911
	return task_tgid_nr_ns(p, event->ns);
2912 2913
}

2914
static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
2915 2916
{
	/*
2917
	 * only top level events have the pid namespace they were created in
2918
	 */
2919 2920
	if (event->parent)
		event = event->parent;
2921

2922
	return task_pid_nr_ns(p, event->ns);
2923 2924
}

2925
static void perf_output_read_one(struct perf_output_handle *handle,
2926
				 struct perf_event *event)
2927
{
2928
	u64 read_format = event->attr.read_format;
2929 2930 2931
	u64 values[4];
	int n = 0;

2932
	values[n++] = atomic64_read(&event->count);
2933
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2934 2935
		values[n++] = event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
2936 2937
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2938 2939
		values[n++] = event->total_time_running +
			atomic64_read(&event->child_total_time_running);
2940 2941
	}
	if (read_format & PERF_FORMAT_ID)
2942
		values[n++] = primary_event_id(event);
2943 2944 2945 2946 2947

	perf_output_copy(handle, values, n * sizeof(u64));
}

/*
2948
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
2949 2950
 */
static void perf_output_read_group(struct perf_output_handle *handle,
2951
			    struct perf_event *event)
2952
{
2953 2954
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = leader->total_time_enabled;

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = leader->total_time_running;

2966
	if (leader != event)
2967 2968 2969 2970
		leader->pmu->read(leader);

	values[n++] = atomic64_read(&leader->count);
	if (read_format & PERF_FORMAT_ID)
2971
		values[n++] = primary_event_id(leader);
2972 2973 2974

	perf_output_copy(handle, values, n * sizeof(u64));

2975
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2976 2977
		n = 0;

2978
		if (sub != event)
2979 2980 2981 2982
			sub->pmu->read(sub);

		values[n++] = atomic64_read(&sub->count);
		if (read_format & PERF_FORMAT_ID)
2983
			values[n++] = primary_event_id(sub);
2984 2985 2986 2987 2988 2989

		perf_output_copy(handle, values, n * sizeof(u64));
	}
}

static void perf_output_read(struct perf_output_handle *handle,
2990
			     struct perf_event *event)
2991
{
2992 2993
	if (event->attr.read_format & PERF_FORMAT_GROUP)
		perf_output_read_group(handle, event);
2994
	else
2995
		perf_output_read_one(handle, event);
2996 2997
}

2998 2999 3000
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3001
			struct perf_event *event)
3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3032
		perf_output_read(handle, event);
3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

			perf_output_copy(handle, data->callchain, size);
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
			perf_output_copy(handle, data->raw->data,
					 data->raw->size);
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3070
			 struct perf_event *event,
3071
			 struct pt_regs *regs)
3072
{
3073
	u64 sample_type = event->attr.sample_type;
3074

3075
	data->type = sample_type;
3076

3077
	header->type = PERF_RECORD_SAMPLE;
3078 3079 3080 3081
	header->size = sizeof(*header);

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3082

3083
	if (sample_type & PERF_SAMPLE_IP) {
3084 3085 3086
		data->ip = perf_instruction_pointer(regs);

		header->size += sizeof(data->ip);
3087
	}
3088

3089
	if (sample_type & PERF_SAMPLE_TID) {
3090
		/* namespace issues */
3091 3092
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
3093

3094
		header->size += sizeof(data->tid_entry);
3095 3096
	}

3097
	if (sample_type & PERF_SAMPLE_TIME) {
P
Peter Zijlstra 已提交
3098
		data->time = perf_clock();
3099

3100
		header->size += sizeof(data->time);
3101 3102
	}

3103
	if (sample_type & PERF_SAMPLE_ADDR)
3104
		header->size += sizeof(data->addr);
3105

3106
	if (sample_type & PERF_SAMPLE_ID) {
3107
		data->id = primary_event_id(event);
3108

3109 3110 3111 3112
		header->size += sizeof(data->id);
	}

	if (sample_type & PERF_SAMPLE_STREAM_ID) {
3113
		data->stream_id = event->id;
3114 3115 3116

		header->size += sizeof(data->stream_id);
	}
3117

3118
	if (sample_type & PERF_SAMPLE_CPU) {
3119 3120
		data->cpu_entry.cpu		= raw_smp_processor_id();
		data->cpu_entry.reserved	= 0;
3121

3122
		header->size += sizeof(data->cpu_entry);
3123 3124
	}

3125
	if (sample_type & PERF_SAMPLE_PERIOD)
3126
		header->size += sizeof(data->period);
3127

3128
	if (sample_type & PERF_SAMPLE_READ)
3129
		header->size += perf_event_read_size(event);
3130

3131
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3132
		int size = 1;
3133

3134 3135 3136 3137 3138 3139
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3140 3141
	}

3142
	if (sample_type & PERF_SAMPLE_RAW) {
3143 3144 3145 3146 3147 3148 3149 3150
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3151
		header->size += size;
3152
	}
3153
}
3154

3155
static void perf_event_output(struct perf_event *event, int nmi,
3156 3157 3158 3159 3160
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3161

3162
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
3163

3164
	if (perf_output_begin(&handle, event, header.size, nmi, 1))
3165
		return;
3166

3167
	perf_output_sample(&handle, &header, data, event);
3168

3169
	perf_output_end(&handle);
3170 3171
}

3172
/*
3173
 * read event_id
3174 3175 3176 3177 3178 3179 3180 3181 3182 3183
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3184
perf_event_read_event(struct perf_event *event,
3185 3186 3187
			struct task_struct *task)
{
	struct perf_output_handle handle;
3188
	struct perf_read_event read_event = {
3189
		.header = {
3190
			.type = PERF_RECORD_READ,
3191
			.misc = 0,
3192
			.size = sizeof(read_event) + perf_event_read_size(event),
3193
		},
3194 3195
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3196
	};
3197
	int ret;
3198

3199
	ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
3200 3201 3202
	if (ret)
		return;

3203
	perf_output_put(&handle, read_event);
3204
	perf_output_read(&handle, event);
3205

3206 3207 3208
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
3209
/*
P
Peter Zijlstra 已提交
3210 3211 3212
 * task tracking -- fork/exit
 *
 * enabled by: attr.comm | attr.mmap | attr.task
P
Peter Zijlstra 已提交
3213 3214
 */

P
Peter Zijlstra 已提交
3215
struct perf_task_event {
3216
	struct task_struct		*task;
3217
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
3218 3219 3220 3221 3222 3223

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
3224 3225
		u32				tid;
		u32				ptid;
3226
		u64				time;
3227
	} event_id;
P
Peter Zijlstra 已提交
3228 3229
};

3230
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
3231
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3232 3233
{
	struct perf_output_handle handle;
3234
	int size;
P
Peter Zijlstra 已提交
3235
	struct task_struct *task = task_event->task;
3236 3237
	int ret;

3238 3239
	size  = task_event->event_id.header.size;
	ret = perf_output_begin(&handle, event, size, 0, 0);
P
Peter Zijlstra 已提交
3240 3241 3242 3243

	if (ret)
		return;

3244 3245
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
3246

3247 3248
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
3249

3250
	task_event->event_id.time = perf_clock();
3251

3252
	perf_output_put(&handle, task_event->event_id);
3253

P
Peter Zijlstra 已提交
3254 3255 3256
	perf_output_end(&handle);
}

3257
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
3258
{
3259
	if (event->attr.comm || event->attr.mmap || event->attr.task)
P
Peter Zijlstra 已提交
3260 3261 3262 3263 3264
		return 1;

	return 0;
}

3265
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
3266
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3267
{
3268
	struct perf_event *event;
P
Peter Zijlstra 已提交
3269

3270 3271 3272
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
3273 3274 3275
	}
}

3276
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
3277 3278
{
	struct perf_cpu_context *cpuctx;
3279
	struct perf_event_context *ctx = task_event->task_ctx;
P
Peter Zijlstra 已提交
3280

3281
	rcu_read_lock();
P
Peter Zijlstra 已提交
3282
	cpuctx = &get_cpu_var(perf_cpu_context);
3283
	perf_event_task_ctx(&cpuctx->ctx, task_event);
P
Peter Zijlstra 已提交
3284 3285
	put_cpu_var(perf_cpu_context);

3286
	if (!ctx)
3287
		ctx = rcu_dereference(task_event->task->perf_event_ctxp);
P
Peter Zijlstra 已提交
3288
	if (ctx)
3289
		perf_event_task_ctx(ctx, task_event);
P
Peter Zijlstra 已提交
3290 3291 3292
	rcu_read_unlock();
}

3293 3294
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3295
			      int new)
P
Peter Zijlstra 已提交
3296
{
P
Peter Zijlstra 已提交
3297
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
3298

3299 3300 3301
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
3302 3303
		return;

P
Peter Zijlstra 已提交
3304
	task_event = (struct perf_task_event){
3305 3306
		.task	  = task,
		.task_ctx = task_ctx,
3307
		.event_id    = {
P
Peter Zijlstra 已提交
3308
			.header = {
3309
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
3310
				.misc = 0,
3311
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
3312
			},
3313 3314
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
3315 3316
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
3317 3318 3319
		},
	};

3320
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
3321 3322
}

3323
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
3324
{
3325
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
3326 3327
}

3328 3329 3330 3331 3332
/*
 * comm tracking
 */

struct perf_comm_event {
3333 3334
	struct task_struct	*task;
	char			*comm;
3335 3336 3337 3338 3339 3340 3341
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
3342
	} event_id;
3343 3344
};

3345
static void perf_event_comm_output(struct perf_event *event,
3346 3347 3348
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
3349 3350
	int size = comm_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3351 3352 3353 3354

	if (ret)
		return;

3355 3356
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
3357

3358
	perf_output_put(&handle, comm_event->event_id);
3359 3360 3361 3362 3363
	perf_output_copy(&handle, comm_event->comm,
				   comm_event->comm_size);
	perf_output_end(&handle);
}

3364
static int perf_event_comm_match(struct perf_event *event)
3365
{
3366
	if (event->attr.comm)
3367 3368 3369 3370 3371
		return 1;

	return 0;
}

3372
static void perf_event_comm_ctx(struct perf_event_context *ctx,
3373 3374
				  struct perf_comm_event *comm_event)
{
3375
	struct perf_event *event;
3376 3377 3378 3379 3380

	if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
		return;

	rcu_read_lock();
3381 3382 3383
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
3384 3385 3386 3387
	}
	rcu_read_unlock();
}

3388
static void perf_event_comm_event(struct perf_comm_event *comm_event)
3389 3390
{
	struct perf_cpu_context *cpuctx;
3391
	struct perf_event_context *ctx;
3392
	unsigned int size;
3393
	char comm[TASK_COMM_LEN];
3394

3395 3396
	memset(comm, 0, sizeof(comm));
	strncpy(comm, comm_event->task->comm, sizeof(comm));
3397
	size = ALIGN(strlen(comm)+1, sizeof(u64));
3398 3399 3400 3401

	comm_event->comm = comm;
	comm_event->comm_size = size;

3402
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
3403 3404

	cpuctx = &get_cpu_var(perf_cpu_context);
3405
	perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3406
	put_cpu_var(perf_cpu_context);
3407 3408 3409 3410 3411 3412

	rcu_read_lock();
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3413
	ctx = rcu_dereference(current->perf_event_ctxp);
3414
	if (ctx)
3415
		perf_event_comm_ctx(ctx, comm_event);
3416
	rcu_read_unlock();
3417 3418
}

3419
void perf_event_comm(struct task_struct *task)
3420
{
3421 3422
	struct perf_comm_event comm_event;

3423 3424
	if (task->perf_event_ctxp)
		perf_event_enable_on_exec(task);
3425

3426
	if (!atomic_read(&nr_comm_events))
3427
		return;
3428

3429
	comm_event = (struct perf_comm_event){
3430
		.task	= task,
3431 3432
		/* .comm      */
		/* .comm_size */
3433
		.event_id  = {
3434
			.header = {
3435
				.type = PERF_RECORD_COMM,
3436 3437 3438 3439 3440
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3441 3442 3443
		},
	};

3444
	perf_event_comm_event(&comm_event);
3445 3446
}

3447 3448 3449 3450 3451
/*
 * mmap tracking
 */

struct perf_mmap_event {
3452 3453 3454 3455
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
3456 3457 3458 3459 3460 3461 3462 3463 3464

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
3465
	} event_id;
3466 3467
};

3468
static void perf_event_mmap_output(struct perf_event *event,
3469 3470 3471
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
3472 3473
	int size = mmap_event->event_id.header.size;
	int ret = perf_output_begin(&handle, event, size, 0, 0);
3474 3475 3476 3477

	if (ret)
		return;

3478 3479
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
3480

3481
	perf_output_put(&handle, mmap_event->event_id);
3482 3483
	perf_output_copy(&handle, mmap_event->file_name,
				   mmap_event->file_size);
3484
	perf_output_end(&handle);
3485 3486
}

3487
static int perf_event_mmap_match(struct perf_event *event,
3488 3489
				   struct perf_mmap_event *mmap_event)
{
3490
	if (event->attr.mmap)
3491 3492 3493 3494 3495
		return 1;

	return 0;
}

3496
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
3497 3498
				  struct perf_mmap_event *mmap_event)
{
3499
	struct perf_event *event;
3500 3501 3502 3503 3504

	if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
		return;

	rcu_read_lock();
3505 3506 3507
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_mmap_match(event, mmap_event))
			perf_event_mmap_output(event, mmap_event);
3508 3509 3510 3511
	}
	rcu_read_unlock();
}

3512
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
3513 3514
{
	struct perf_cpu_context *cpuctx;
3515
	struct perf_event_context *ctx;
3516 3517
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
3518 3519 3520
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
3521
	const char *name;
3522

3523 3524
	memset(tmp, 0, sizeof(tmp));

3525
	if (file) {
3526 3527 3528 3529 3530 3531
		/*
		 * d_path works from the end of the buffer backwards, so we
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
3532 3533 3534 3535
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
3536
		name = d_path(&file->f_path, buf, PATH_MAX);
3537 3538 3539 3540 3541
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
3542 3543 3544
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
3545
			goto got_name;
3546
		}
3547 3548 3549 3550 3551 3552

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
		}

3553 3554 3555 3556 3557
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
3558
	size = ALIGN(strlen(name)+1, sizeof(u64));
3559 3560 3561 3562

	mmap_event->file_name = name;
	mmap_event->file_size = size;

3563
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
3564 3565

	cpuctx = &get_cpu_var(perf_cpu_context);
3566
	perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
3567 3568
	put_cpu_var(perf_cpu_context);

3569 3570 3571 3572 3573
	rcu_read_lock();
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3574
	ctx = rcu_dereference(current->perf_event_ctxp);
3575
	if (ctx)
3576
		perf_event_mmap_ctx(ctx, mmap_event);
3577 3578
	rcu_read_unlock();

3579 3580 3581
	kfree(buf);
}

3582
void __perf_event_mmap(struct vm_area_struct *vma)
3583
{
3584 3585
	struct perf_mmap_event mmap_event;

3586
	if (!atomic_read(&nr_mmap_events))
3587 3588 3589
		return;

	mmap_event = (struct perf_mmap_event){
3590
		.vma	= vma,
3591 3592
		/* .file_name */
		/* .file_size */
3593
		.event_id  = {
3594
			.header = {
3595
				.type = PERF_RECORD_MMAP,
3596 3597 3598 3599 3600
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
3601 3602 3603
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
			.pgoff  = vma->vm_pgoff,
3604 3605 3606
		},
	};

3607
	perf_event_mmap_event(&mmap_event);
3608 3609
}

3610 3611 3612 3613
/*
 * IRQ throttle logging
 */

3614
static void perf_log_throttle(struct perf_event *event, int enable)
3615 3616 3617 3618 3619 3620 3621
{
	struct perf_output_handle handle;
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
3622
		u64				id;
3623
		u64				stream_id;
3624 3625
	} throttle_event = {
		.header = {
3626
			.type = PERF_RECORD_THROTTLE,
3627 3628 3629
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
3630
		.time		= perf_clock(),
3631 3632
		.id		= primary_event_id(event),
		.stream_id	= event->id,
3633 3634
	};

3635
	if (enable)
3636
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
3637

3638
	ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
3639 3640 3641 3642 3643 3644 3645
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
	perf_output_end(&handle);
}

3646
/*
3647
 * Generic event overflow handling, sampling.
3648 3649
 */

3650
static int __perf_event_overflow(struct perf_event *event, int nmi,
3651 3652
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
3653
{
3654 3655
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
3656 3657
	int ret = 0;

3658
	throttle = (throttle && event->pmu->unthrottle != NULL);
3659

3660
	if (!throttle) {
3661
		hwc->interrupts++;
3662
	} else {
3663 3664
		if (hwc->interrupts != MAX_INTERRUPTS) {
			hwc->interrupts++;
3665
			if (HZ * hwc->interrupts >
3666
					(u64)sysctl_perf_event_sample_rate) {
3667
				hwc->interrupts = MAX_INTERRUPTS;
3668
				perf_log_throttle(event, 0);
3669 3670 3671 3672
				ret = 1;
			}
		} else {
			/*
3673
			 * Keep re-disabling events even though on the previous
3674
			 * pass we disabled it - just in case we raced with a
3675
			 * sched-in and the event got enabled again:
3676
			 */
3677 3678 3679
			ret = 1;
		}
	}
3680

3681
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
3682
		u64 now = perf_clock();
3683 3684 3685 3686 3687
		s64 delta = now - hwc->freq_stamp;

		hwc->freq_stamp = now;

		if (delta > 0 && delta < TICK_NSEC)
3688
			perf_adjust_period(event, NSEC_PER_SEC / (int)delta);
3689 3690
	}

3691 3692
	/*
	 * XXX event_limit might not quite work as expected on inherited
3693
	 * events
3694 3695
	 */

3696 3697
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
3698
		ret = 1;
3699
		event->pending_kill = POLL_HUP;
3700
		if (nmi) {
3701 3702 3703
			event->pending_disable = 1;
			perf_pending_queue(&event->pending,
					   perf_pending_event);
3704
		} else
3705
			perf_event_disable(event);
3706 3707
	}

3708 3709 3710 3711 3712
	if (event->overflow_handler)
		event->overflow_handler(event, nmi, data, regs);
	else
		perf_event_output(event, nmi, data, regs);

3713
	return ret;
3714 3715
}

3716
int perf_event_overflow(struct perf_event *event, int nmi,
3717 3718
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
3719
{
3720
	return __perf_event_overflow(event, nmi, 1, data, regs);
3721 3722
}

3723
/*
3724
 * Generic software event infrastructure
3725 3726
 */

3727
/*
3728 3729
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
3730 3731 3732 3733
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

3734
static u64 perf_swevent_set_period(struct perf_event *event)
3735
{
3736
	struct hw_perf_event *hwc = &event->hw;
3737 3738 3739 3740 3741
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
3742 3743

again:
3744 3745 3746
	old = val = atomic64_read(&hwc->period_left);
	if (val < 0)
		return 0;
3747

3748 3749 3750 3751 3752
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
	if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
		goto again;
3753

3754
	return nr;
3755 3756
}

3757
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
3758 3759
				    int nmi, struct perf_sample_data *data,
				    struct pt_regs *regs)
3760
{
3761
	struct hw_perf_event *hwc = &event->hw;
3762
	int throttle = 0;
3763

3764
	data->period = event->hw.last_period;
3765 3766
	if (!overflow)
		overflow = perf_swevent_set_period(event);
3767

3768 3769
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
3770

3771
	for (; overflow; overflow--) {
3772
		if (__perf_event_overflow(event, nmi, throttle,
3773
					    data, regs)) {
3774 3775 3776 3777 3778 3779
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
3780
		throttle = 1;
3781
	}
3782 3783
}

3784
static void perf_swevent_unthrottle(struct perf_event *event)
3785 3786
{
	/*
3787
	 * Nothing to do, we already reset hwc->interrupts.
3788
	 */
3789
}
3790

3791
static void perf_swevent_add(struct perf_event *event, u64 nr,
3792 3793
			       int nmi, struct perf_sample_data *data,
			       struct pt_regs *regs)
3794
{
3795
	struct hw_perf_event *hwc = &event->hw;
3796

3797
	atomic64_add(nr, &event->count);
3798

3799 3800 3801
	if (!regs)
		return;

3802 3803
	if (!hwc->sample_period)
		return;
3804

3805 3806 3807 3808
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
		return perf_swevent_overflow(event, 1, nmi, data, regs);

	if (atomic64_add_negative(nr, &hwc->period_left))
3809
		return;
3810

3811
	perf_swevent_overflow(event, 0, nmi, data, regs);
3812 3813
}

3814
static int perf_swevent_is_counting(struct perf_event *event)
3815
{
3816
	/*
3817
	 * The event is active, we're good!
3818
	 */
3819
	if (event->state == PERF_EVENT_STATE_ACTIVE)
3820 3821
		return 1;

3822
	/*
3823
	 * The event is off/error, not counting.
3824
	 */
3825
	if (event->state != PERF_EVENT_STATE_INACTIVE)
3826 3827 3828
		return 0;

	/*
3829
	 * The event is inactive, if the context is active
3830 3831
	 * we're part of a group that didn't make it on the 'pmu',
	 * not counting.
3832
	 */
3833
	if (event->ctx->is_active)
3834 3835 3836 3837 3838 3839 3840 3841
		return 0;

	/*
	 * We're inactive and the context is too, this means the
	 * task is scheduled out, we're counting events that happen
	 * to us, like migration events.
	 */
	return 1;
3842 3843
}

L
Li Zefan 已提交
3844 3845 3846
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data);

3847
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
3848
				enum perf_type_id type,
L
Li Zefan 已提交
3849 3850 3851
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
3852
{
3853
	if (!perf_swevent_is_counting(event))
3854 3855
		return 0;

3856
	if (event->attr.type != type)
3857
		return 0;
3858
	if (event->attr.config != event_id)
3859 3860
		return 0;

3861
	if (regs) {
3862
		if (event->attr.exclude_user && user_mode(regs))
3863
			return 0;
3864

3865
		if (event->attr.exclude_kernel && !user_mode(regs))
3866 3867
			return 0;
	}
3868

L
Li Zefan 已提交
3869 3870 3871 3872
	if (event->attr.type == PERF_TYPE_TRACEPOINT &&
	    !perf_tp_event_match(event, data))
		return 0;

3873 3874 3875
	return 1;
}

3876
static void perf_swevent_ctx_event(struct perf_event_context *ctx,
3877
				     enum perf_type_id type,
3878
				     u32 event_id, u64 nr, int nmi,
3879 3880
				     struct perf_sample_data *data,
				     struct pt_regs *regs)
3881
{
3882
	struct perf_event *event;
3883

3884
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
L
Li Zefan 已提交
3885
		if (perf_swevent_match(event, type, event_id, data, regs))
3886
			perf_swevent_add(event, nr, nmi, data, regs);
3887 3888 3889
	}
}

3890
static int *perf_swevent_recursion_context(struct perf_cpu_context *cpuctx)
P
Peter Zijlstra 已提交
3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903
{
	if (in_nmi())
		return &cpuctx->recursion[3];

	if (in_irq())
		return &cpuctx->recursion[2];

	if (in_softirq())
		return &cpuctx->recursion[1];

	return &cpuctx->recursion[0];
}

3904
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
3905
				    u64 nr, int nmi,
3906 3907
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
3908 3909
{
	struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
3910 3911
	int *recursion = perf_swevent_recursion_context(cpuctx);
	struct perf_event_context *ctx;
P
Peter Zijlstra 已提交
3912 3913 3914 3915 3916 3917

	if (*recursion)
		goto out;

	(*recursion)++;
	barrier();
3918

3919
	rcu_read_lock();
3920
	perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
3921
				 nr, nmi, data, regs);
3922 3923 3924 3925
	/*
	 * doesn't really matter which of the child contexts the
	 * events ends up in.
	 */
3926
	ctx = rcu_dereference(current->perf_event_ctxp);
3927
	if (ctx)
3928
		perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs);
3929
	rcu_read_unlock();
3930

P
Peter Zijlstra 已提交
3931 3932 3933 3934
	barrier();
	(*recursion)--;

out:
3935 3936 3937
	put_cpu_var(perf_cpu_context);
}

3938
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
3939
			    struct pt_regs *regs, u64 addr)
3940
{
3941 3942 3943 3944
	struct perf_sample_data data = {
		.addr = addr,
	};

3945
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi,
3946
				&data, regs);
3947 3948
}

3949
static void perf_swevent_read(struct perf_event *event)
3950 3951 3952
{
}

3953
static int perf_swevent_enable(struct perf_event *event)
3954
{
3955
	struct hw_perf_event *hwc = &event->hw;
3956 3957 3958

	if (hwc->sample_period) {
		hwc->last_period = hwc->sample_period;
3959
		perf_swevent_set_period(event);
3960
	}
3961 3962 3963
	return 0;
}

3964
static void perf_swevent_disable(struct perf_event *event)
3965 3966 3967
{
}

3968
static const struct pmu perf_ops_generic = {
3969 3970 3971 3972
	.enable		= perf_swevent_enable,
	.disable	= perf_swevent_disable,
	.read		= perf_swevent_read,
	.unthrottle	= perf_swevent_unthrottle,
3973 3974
};

3975
/*
3976
 * hrtimer based swevent callback
3977 3978
 */

3979
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
3980 3981 3982
{
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
3983
	struct pt_regs *regs;
3984
	struct perf_event *event;
3985 3986
	u64 period;

3987 3988
	event	= container_of(hrtimer, struct perf_event, hw.hrtimer);
	event->pmu->read(event);
3989 3990

	data.addr = 0;
3991
	regs = get_irq_regs();
3992 3993 3994 3995
	/*
	 * In case we exclude kernel IPs or are somehow not in interrupt
	 * context, provide the next best thing, the user IP.
	 */
3996 3997
	if ((event->attr.exclude_kernel || !regs) &&
			!event->attr.exclude_user)
3998
		regs = task_pt_regs(current);
3999

4000
	if (regs) {
4001 4002 4003
		if (!(event->attr.exclude_idle && current->pid == 0))
			if (perf_event_overflow(event, 0, &data, regs))
				ret = HRTIMER_NORESTART;
4004 4005
	}

4006
	period = max_t(u64, 10000, event->hw.sample_period);
4007 4008 4009 4010 4011
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));

	return ret;
}

4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047
static void perf_swevent_start_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;
	if (hwc->sample_period) {
		u64 period;

		if (hwc->remaining) {
			if (hwc->remaining < 0)
				period = 10000;
			else
				period = hwc->remaining;
			hwc->remaining = 0;
		} else {
			period = max_t(u64, 10000, hwc->sample_period);
		}
		__hrtimer_start_range_ns(&hwc->hrtimer,
				ns_to_ktime(period), 0,
				HRTIMER_MODE_REL, 0);
	}
}

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (hwc->sample_period) {
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
		hwc->remaining = ktime_to_ns(remaining);

		hrtimer_cancel(&hwc->hrtimer);
	}
}

4048
/*
4049
 * Software event: cpu wall time clock
4050 4051
 */

4052
static void cpu_clock_perf_event_update(struct perf_event *event)
4053 4054 4055 4056 4057 4058
{
	int cpu = raw_smp_processor_id();
	s64 prev;
	u64 now;

	now = cpu_clock(cpu);
4059 4060 4061
	prev = atomic64_read(&event->hw.prev_count);
	atomic64_set(&event->hw.prev_count, now);
	atomic64_add(now - prev, &event->count);
4062 4063
}

4064
static int cpu_clock_perf_event_enable(struct perf_event *event)
4065
{
4066
	struct hw_perf_event *hwc = &event->hw;
4067 4068 4069
	int cpu = raw_smp_processor_id();

	atomic64_set(&hwc->prev_count, cpu_clock(cpu));
4070
	perf_swevent_start_hrtimer(event);
4071 4072 4073 4074

	return 0;
}

4075
static void cpu_clock_perf_event_disable(struct perf_event *event)
4076
{
4077
	perf_swevent_cancel_hrtimer(event);
4078
	cpu_clock_perf_event_update(event);
4079 4080
}

4081
static void cpu_clock_perf_event_read(struct perf_event *event)
4082
{
4083
	cpu_clock_perf_event_update(event);
4084 4085
}

4086
static const struct pmu perf_ops_cpu_clock = {
4087 4088 4089
	.enable		= cpu_clock_perf_event_enable,
	.disable	= cpu_clock_perf_event_disable,
	.read		= cpu_clock_perf_event_read,
4090 4091
};

4092
/*
4093
 * Software event: task time clock
4094 4095
 */

4096
static void task_clock_perf_event_update(struct perf_event *event, u64 now)
I
Ingo Molnar 已提交
4097
{
4098
	u64 prev;
I
Ingo Molnar 已提交
4099 4100
	s64 delta;

4101
	prev = atomic64_xchg(&event->hw.prev_count, now);
I
Ingo Molnar 已提交
4102
	delta = now - prev;
4103
	atomic64_add(delta, &event->count);
4104 4105
}

4106
static int task_clock_perf_event_enable(struct perf_event *event)
I
Ingo Molnar 已提交
4107
{
4108
	struct hw_perf_event *hwc = &event->hw;
4109 4110
	u64 now;

4111
	now = event->ctx->time;
4112

4113
	atomic64_set(&hwc->prev_count, now);
4114 4115

	perf_swevent_start_hrtimer(event);
4116 4117

	return 0;
I
Ingo Molnar 已提交
4118 4119
}

4120
static void task_clock_perf_event_disable(struct perf_event *event)
4121
{
4122
	perf_swevent_cancel_hrtimer(event);
4123
	task_clock_perf_event_update(event, event->ctx->time);
4124

4125
}
I
Ingo Molnar 已提交
4126

4127
static void task_clock_perf_event_read(struct perf_event *event)
4128
{
4129 4130 4131
	u64 time;

	if (!in_nmi()) {
4132 4133
		update_context_time(event->ctx);
		time = event->ctx->time;
4134 4135
	} else {
		u64 now = perf_clock();
4136 4137
		u64 delta = now - event->ctx->timestamp;
		time = event->ctx->time + delta;
4138 4139
	}

4140
	task_clock_perf_event_update(event, time);
4141 4142
}

4143
static const struct pmu perf_ops_task_clock = {
4144 4145 4146
	.enable		= task_clock_perf_event_enable,
	.disable	= task_clock_perf_event_disable,
	.read		= task_clock_perf_event_read,
4147 4148
};

4149
#ifdef CONFIG_EVENT_PROFILE
L
Li Zefan 已提交
4150

4151
void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
4152
			  int entry_size)
4153
{
4154
	struct perf_raw_record raw = {
4155
		.size = entry_size,
4156
		.data = record,
4157 4158
	};

4159
	struct perf_sample_data data = {
4160
		.addr = addr,
4161
		.raw = &raw,
4162
	};
4163

4164 4165 4166 4167
	struct pt_regs *regs = get_irq_regs();

	if (!regs)
		regs = task_pt_regs(current);
4168

4169
	do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
4170
				&data, regs);
4171
}
4172
EXPORT_SYMBOL_GPL(perf_tp_event);
4173

L
Li Zefan 已提交
4174 4175 4176 4177 4178 4179 4180 4181 4182
static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}
4183

4184
static void tp_perf_event_destroy(struct perf_event *event)
4185
{
4186
	ftrace_profile_disable(event->attr.config);
4187 4188
}

4189
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4190
{
4191 4192 4193 4194
	/*
	 * Raw tracepoint data is a severe data leak, only allow root to
	 * have these.
	 */
4195
	if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
4196
			perf_paranoid_tracepoint_raw() &&
4197 4198 4199
			!capable(CAP_SYS_ADMIN))
		return ERR_PTR(-EPERM);

4200
	if (ftrace_profile_enable(event->attr.config))
4201 4202
		return NULL;

4203
	event->destroy = tp_perf_event_destroy;
4204 4205 4206

	return &perf_ops_generic;
}
L
Li Zefan 已提交
4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

4231
#else
L
Li Zefan 已提交
4232 4233 4234 4235 4236 4237 4238

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	return 1;
}

4239
static const struct pmu *tp_perf_event_init(struct perf_event *event)
4240 4241 4242
{
	return NULL;
}
L
Li Zefan 已提交
4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

#endif /* CONFIG_EVENT_PROFILE */
4254

4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299
#ifdef CONFIG_HAVE_HW_BREAKPOINT
static void bp_perf_event_destroy(struct perf_event *event)
{
	release_bp_slot(event);
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	int err;
	/*
	 * The breakpoint is already filled if we haven't created the counter
	 * through perf syscall
	 * FIXME: manage to get trigerred to NULL if it comes from syscalls
	 */
	if (!bp->callback)
		err = register_perf_hw_breakpoint(bp);
	else
		err = __register_perf_hw_breakpoint(bp);
	if (err)
		return ERR_PTR(err);

	bp->destroy = bp_perf_event_destroy;

	return &perf_ops_bp;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
	/* TODO */
}
#else
static void bp_perf_event_destroy(struct perf_event *event)
{
}

static const struct pmu *bp_perf_event_init(struct perf_event *bp)
{
	return NULL;
}

void perf_bp_event(struct perf_event *bp, void *regs)
{
}
#endif

4300
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
4301

4302
static void sw_perf_event_destroy(struct perf_event *event)
4303
{
4304
	u64 event_id = event->attr.config;
4305

4306
	WARN_ON(event->parent);
4307

4308
	atomic_dec(&perf_swevent_enabled[event_id]);
4309 4310
}

4311
static const struct pmu *sw_perf_event_init(struct perf_event *event)
4312
{
4313
	const struct pmu *pmu = NULL;
4314
	u64 event_id = event->attr.config;
4315

4316
	/*
4317
	 * Software events (currently) can't in general distinguish
4318 4319 4320 4321 4322
	 * between user, kernel and hypervisor events.
	 * However, context switches and cpu migrations are considered
	 * to be kernel events, and page faults are never hypervisor
	 * events.
	 */
4323
	switch (event_id) {
4324
	case PERF_COUNT_SW_CPU_CLOCK:
4325
		pmu = &perf_ops_cpu_clock;
4326

4327
		break;
4328
	case PERF_COUNT_SW_TASK_CLOCK:
4329
		/*
4330 4331
		 * If the user instantiates this as a per-cpu event,
		 * use the cpu_clock event instead.
4332
		 */
4333
		if (event->ctx->task)
4334
			pmu = &perf_ops_task_clock;
4335
		else
4336
			pmu = &perf_ops_cpu_clock;
4337

4338
		break;
4339 4340 4341 4342 4343
	case PERF_COUNT_SW_PAGE_FAULTS:
	case PERF_COUNT_SW_PAGE_FAULTS_MIN:
	case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
	case PERF_COUNT_SW_CONTEXT_SWITCHES:
	case PERF_COUNT_SW_CPU_MIGRATIONS:
4344 4345
	case PERF_COUNT_SW_ALIGNMENT_FAULTS:
	case PERF_COUNT_SW_EMULATION_FAULTS:
4346 4347 4348
		if (!event->parent) {
			atomic_inc(&perf_swevent_enabled[event_id]);
			event->destroy = sw_perf_event_destroy;
4349
		}
4350
		pmu = &perf_ops_generic;
4351
		break;
4352
	}
4353

4354
	return pmu;
4355 4356
}

T
Thomas Gleixner 已提交
4357
/*
4358
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
4359
 */
4360 4361
static struct perf_event *
perf_event_alloc(struct perf_event_attr *attr,
4362
		   int cpu,
4363 4364 4365
		   struct perf_event_context *ctx,
		   struct perf_event *group_leader,
		   struct perf_event *parent_event,
4366
		   perf_callback_t callback,
4367
		   gfp_t gfpflags)
T
Thomas Gleixner 已提交
4368
{
4369
	const struct pmu *pmu;
4370 4371
	struct perf_event *event;
	struct hw_perf_event *hwc;
4372
	long err;
T
Thomas Gleixner 已提交
4373

4374 4375
	event = kzalloc(sizeof(*event), gfpflags);
	if (!event)
4376
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
4377

4378
	/*
4379
	 * Single events are their own group leaders, with an
4380 4381 4382
	 * empty sibling list:
	 */
	if (!group_leader)
4383
		group_leader = event;
4384

4385 4386
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
4387

4388 4389 4390 4391
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
T
Thomas Gleixner 已提交
4392

4393
	mutex_init(&event->mmap_mutex);
4394

4395 4396 4397 4398 4399 4400
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->ctx		= ctx;
	event->oncpu		= -1;
4401

4402
	event->parent		= parent_event;
4403

4404 4405
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
4406

4407
	event->state		= PERF_EVENT_STATE_INACTIVE;
4408

4409 4410 4411 4412 4413
	if (!callback && parent_event)
		callback = parent_event->callback;
	
	event->callback	= callback;

4414
	if (attr->disabled)
4415
		event->state = PERF_EVENT_STATE_OFF;
4416

4417
	pmu = NULL;
4418

4419
	hwc = &event->hw;
4420
	hwc->sample_period = attr->sample_period;
4421
	if (attr->freq && attr->sample_freq)
4422
		hwc->sample_period = 1;
4423
	hwc->last_period = hwc->sample_period;
4424 4425

	atomic64_set(&hwc->period_left, hwc->sample_period);
4426

4427
	/*
4428
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
4429
	 */
4430
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
4431 4432
		goto done;

4433
	switch (attr->type) {
4434
	case PERF_TYPE_RAW:
4435
	case PERF_TYPE_HARDWARE:
4436
	case PERF_TYPE_HW_CACHE:
4437
		pmu = hw_perf_event_init(event);
4438 4439 4440
		break;

	case PERF_TYPE_SOFTWARE:
4441
		pmu = sw_perf_event_init(event);
4442 4443 4444
		break;

	case PERF_TYPE_TRACEPOINT:
4445
		pmu = tp_perf_event_init(event);
4446
		break;
4447

4448 4449 4450 4451 4452
	case PERF_TYPE_BREAKPOINT:
		pmu = bp_perf_event_init(event);
		break;


4453 4454
	default:
		break;
4455
	}
4456 4457
done:
	err = 0;
4458
	if (!pmu)
4459
		err = -EINVAL;
4460 4461
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
4462

4463
	if (err) {
4464 4465 4466
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
4467
		return ERR_PTR(err);
I
Ingo Molnar 已提交
4468
	}
4469

4470
	event->pmu = pmu;
T
Thomas Gleixner 已提交
4471

4472 4473 4474 4475 4476 4477 4478 4479
	if (!event->parent) {
		atomic_inc(&nr_events);
		if (event->attr.mmap)
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
4480
	}
4481

4482
	return event;
T
Thomas Gleixner 已提交
4483 4484
}

4485 4486
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
4487 4488
{
	u32 size;
4489
	int ret;
4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
4514 4515 4516
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
4517 4518
	 */
	if (size > sizeof(*attr)) {
4519 4520 4521
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
4522

4523 4524
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
4525

4526
		for (; addr < end; addr++) {
4527 4528 4529 4530 4531 4532
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
4533
		size = sizeof(*attr);
4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * If the type exists, the corresponding creation will verify
	 * the attr->config.
	 */
	if (attr->type >= PERF_TYPE_MAX)
		return -EINVAL;

	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Li Zefan 已提交
4565
static int perf_event_set_output(struct perf_event *event, int output_fd)
4566
{
4567
	struct perf_event *output_event = NULL;
4568
	struct file *output_file = NULL;
4569
	struct perf_event *old_output;
4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582
	int fput_needed = 0;
	int ret = -EINVAL;

	if (!output_fd)
		goto set;

	output_file = fget_light(output_fd, &fput_needed);
	if (!output_file)
		return -EBADF;

	if (output_file->f_op != &perf_fops)
		goto out;

4583
	output_event = output_file->private_data;
4584 4585

	/* Don't chain output fds */
4586
	if (output_event->output)
4587 4588 4589
		goto out;

	/* Don't set an output fd when we already have an output channel */
4590
	if (event->data)
4591 4592 4593 4594 4595
		goto out;

	atomic_long_inc(&output_file->f_count);

set:
4596 4597 4598 4599
	mutex_lock(&event->mmap_mutex);
	old_output = event->output;
	rcu_assign_pointer(event->output, output_event);
	mutex_unlock(&event->mmap_mutex);
4600 4601 4602 4603

	if (old_output) {
		/*
		 * we need to make sure no existing perf_output_*()
4604
		 * is still referencing this event.
4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615
		 */
		synchronize_rcu();
		fput(old_output->filp);
	}

	ret = 0;
out:
	fput_light(output_file, fput_needed);
	return ret;
}

T
Thomas Gleixner 已提交
4616
/**
4617
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
4618
 *
4619
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
4620
 * @pid:		target pid
I
Ingo Molnar 已提交
4621
 * @cpu:		target cpu
4622
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
4623
 */
4624 4625
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
4626
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
4627
{
4628 4629 4630 4631
	struct perf_event *event, *group_leader;
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
4632 4633
	struct file *group_file = NULL;
	int fput_needed = 0;
4634
	int fput_needed2 = 0;
4635
	int err;
T
Thomas Gleixner 已提交
4636

4637
	/* for future expandability... */
4638
	if (flags & ~(PERF_FLAG_FD_NO_GROUP | PERF_FLAG_FD_OUTPUT))
4639 4640
		return -EINVAL;

4641 4642 4643
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
4644

4645 4646 4647 4648 4649
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

4650
	if (attr.freq) {
4651
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
4652 4653 4654
			return -EINVAL;
	}

4655
	/*
I
Ingo Molnar 已提交
4656 4657 4658 4659 4660 4661 4662
	 * Get the target context (task or percpu):
	 */
	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

	/*
4663
	 * Look up the group leader (we will attach this event to it):
4664 4665
	 */
	group_leader = NULL;
4666
	if (group_fd != -1 && !(flags & PERF_FLAG_FD_NO_GROUP)) {
4667
		err = -EINVAL;
4668 4669
		group_file = fget_light(group_fd, &fput_needed);
		if (!group_file)
I
Ingo Molnar 已提交
4670
			goto err_put_context;
4671
		if (group_file->f_op != &perf_fops)
I
Ingo Molnar 已提交
4672
			goto err_put_context;
4673 4674 4675

		group_leader = group_file->private_data;
		/*
I
Ingo Molnar 已提交
4676 4677 4678 4679 4680 4681 4682 4683
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
			goto err_put_context;
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
4684
		 */
I
Ingo Molnar 已提交
4685 4686
		if (group_leader->ctx != ctx)
			goto err_put_context;
4687 4688 4689
		/*
		 * Only a group leader can be exclusive or pinned
		 */
4690
		if (attr.exclusive || attr.pinned)
4691
			goto err_put_context;
4692 4693
	}

4694
	event = perf_event_alloc(&attr, cpu, ctx, group_leader,
4695
				     NULL, NULL, GFP_KERNEL);
4696 4697
	err = PTR_ERR(event);
	if (IS_ERR(event))
T
Thomas Gleixner 已提交
4698 4699
		goto err_put_context;

4700
	err = anon_inode_getfd("[perf_event]", &perf_fops, event, 0);
4701
	if (err < 0)
4702 4703
		goto err_free_put_context;

4704 4705
	event_file = fget_light(err, &fput_needed2);
	if (!event_file)
4706 4707
		goto err_free_put_context;

4708
	if (flags & PERF_FLAG_FD_OUTPUT) {
4709
		err = perf_event_set_output(event, group_fd);
4710 4711
		if (err)
			goto err_fput_free_put_context;
4712 4713
	}

4714
	event->filp = event_file;
4715
	WARN_ON_ONCE(ctx->parent_ctx);
4716
	mutex_lock(&ctx->mutex);
4717
	perf_install_in_context(ctx, event, cpu);
4718
	++ctx->generation;
4719
	mutex_unlock(&ctx->mutex);
4720

4721
	event->owner = current;
4722
	get_task_struct(current);
4723 4724 4725
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
4726

4727
err_fput_free_put_context:
4728
	fput_light(event_file, fput_needed2);
T
Thomas Gleixner 已提交
4729

4730
err_free_put_context:
4731
	if (err < 0)
4732
		kfree(event);
T
Thomas Gleixner 已提交
4733 4734

err_put_context:
4735 4736 4737 4738
	if (err < 0)
		put_ctx(ctx);

	fput_light(group_file, fput_needed);
T
Thomas Gleixner 已提交
4739

4740
	return err;
T
Thomas Gleixner 已提交
4741 4742
}

4743 4744 4745 4746 4747 4748 4749 4750 4751
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
 * @pid: task to profile
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
4752
				 pid_t pid, perf_callback_t callback)
4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763
{
	struct perf_event *event;
	struct perf_event_context *ctx;
	int err;

	/*
	 * Get the target context (task or percpu):
	 */

	ctx = find_get_context(pid, cpu);
	if (IS_ERR(ctx))
4764
		return NULL;
4765 4766

	event = perf_event_alloc(attr, cpu, ctx, NULL,
4767
				     NULL, callback, GFP_KERNEL);
4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794
	err = PTR_ERR(event);
	if (IS_ERR(event))
		goto err_put_context;

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
	mutex_unlock(&ctx->mutex);

	event->owner = current;
	get_task_struct(current);
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);

	return event;

err_put_context:
	if (err < 0)
		put_ctx(ctx);

	return NULL;
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);

4795
/*
4796
 * inherit a event from parent task to child task:
4797
 */
4798 4799
static struct perf_event *
inherit_event(struct perf_event *parent_event,
4800
	      struct task_struct *parent,
4801
	      struct perf_event_context *parent_ctx,
4802
	      struct task_struct *child,
4803 4804
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
4805
{
4806
	struct perf_event *child_event;
4807

4808
	/*
4809 4810
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
4811 4812 4813
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
4814 4815
	if (parent_event->parent)
		parent_event = parent_event->parent;
4816

4817 4818 4819
	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu, child_ctx,
					   group_leader, parent_event,
4820
					   NULL, GFP_KERNEL);
4821 4822
	if (IS_ERR(child_event))
		return child_event;
4823
	get_ctx(child_ctx);
4824

4825
	/*
4826
	 * Make the child state follow the state of the parent event,
4827
	 * not its attr.disabled bit.  We hold the parent's mutex,
4828
	 * so we won't race with perf_event_{en, dis}able_family.
4829
	 */
4830 4831
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
4832
	else
4833
		child_event->state = PERF_EVENT_STATE_OFF;
4834

4835 4836
	if (parent_event->attr.freq)
		child_event->hw.sample_period = parent_event->hw.sample_period;
4837

4838 4839
	child_event->overflow_handler = parent_event->overflow_handler;

4840 4841 4842
	/*
	 * Link it up in the child's context:
	 */
4843
	add_event_to_ctx(child_event, child_ctx);
4844 4845 4846

	/*
	 * Get a reference to the parent filp - we will fput it
4847
	 * when the child event exits. This is safe to do because
4848 4849 4850
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
4851
	atomic_long_inc(&parent_event->filp->f_count);
4852

4853
	/*
4854
	 * Link this into the parent event's child list
4855
	 */
4856 4857 4858 4859
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4860

4861
	return child_event;
4862 4863
}

4864
static int inherit_group(struct perf_event *parent_event,
4865
	      struct task_struct *parent,
4866
	      struct perf_event_context *parent_ctx,
4867
	      struct task_struct *child,
4868
	      struct perf_event_context *child_ctx)
4869
{
4870 4871 4872
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;
4873

4874
	leader = inherit_event(parent_event, parent, parent_ctx,
4875
				 child, NULL, child_ctx);
4876 4877
	if (IS_ERR(leader))
		return PTR_ERR(leader);
4878 4879
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
4880 4881 4882
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
4883
	}
4884 4885 4886
	return 0;
}

4887
static void sync_child_event(struct perf_event *child_event,
4888
			       struct task_struct *child)
4889
{
4890
	struct perf_event *parent_event = child_event->parent;
4891
	u64 child_val;
4892

4893 4894
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
4895

4896
	child_val = atomic64_read(&child_event->count);
4897 4898 4899 4900

	/*
	 * Add back the child's count to the parent's count:
	 */
4901 4902 4903 4904 4905
	atomic64_add(child_val, &parent_event->count);
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
4906 4907

	/*
4908
	 * Remove this event from the parent's list
4909
	 */
4910 4911 4912 4913
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
4914 4915

	/*
4916
	 * Release the parent event, if this was the last
4917 4918
	 * reference to it.
	 */
4919
	fput(parent_event->filp);
4920 4921
}

4922
static void
4923 4924
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
4925
			 struct task_struct *child)
4926
{
4927
	struct perf_event *parent_event;
4928

4929 4930
	update_event_times(child_event);
	perf_event_remove_from_context(child_event);
4931

4932
	parent_event = child_event->parent;
4933
	/*
4934
	 * It can happen that parent exits first, and has events
4935
	 * that are still around due to the child reference. These
4936
	 * events need to be zapped - but otherwise linger.
4937
	 */
4938 4939 4940
	if (parent_event) {
		sync_child_event(child_event, child);
		free_event(child_event);
4941
	}
4942 4943 4944
}

/*
4945
 * When a child task exits, feed back event values to parent events.
4946
 */
4947
void perf_event_exit_task(struct task_struct *child)
4948
{
4949 4950
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
4951
	unsigned long flags;
4952

4953 4954
	if (likely(!child->perf_event_ctxp)) {
		perf_event_task(child, NULL, 0);
4955
		return;
P
Peter Zijlstra 已提交
4956
	}
4957

4958
	local_irq_save(flags);
4959 4960 4961 4962 4963 4964
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
4965 4966
	child_ctx = child->perf_event_ctxp;
	__perf_event_task_sched_out(child_ctx);
4967 4968 4969

	/*
	 * Take the context lock here so that if find_get_context is
4970
	 * reading child->perf_event_ctxp, we wait until it has
4971 4972 4973
	 * incremented the context's refcount before we do put_ctx below.
	 */
	spin_lock(&child_ctx->lock);
4974
	child->perf_event_ctxp = NULL;
4975 4976 4977
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
4978
	 * the events from it.
4979 4980
	 */
	unclone_ctx(child_ctx);
P
Peter Zijlstra 已提交
4981 4982 4983
	spin_unlock_irqrestore(&child_ctx->lock, flags);

	/*
4984 4985 4986
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
4987
	 */
4988
	perf_event_task(child, child_ctx, 0);
4989

4990 4991 4992
	/*
	 * We can recurse on the same lock type through:
	 *
4993 4994 4995
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
4996 4997 4998 4999 5000 5001
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
	mutex_lock_nested(&child_ctx->mutex, SINGLE_DEPTH_NESTING);
5002

5003
again:
5004
	list_for_each_entry_safe(child_event, tmp, &child_ctx->group_list,
5005
				 group_entry)
5006
		__perf_event_exit_task(child_event, child_ctx, child);
5007 5008

	/*
5009
	 * If the last event was a group event, it will have appended all
5010 5011 5012
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
5013
	if (!list_empty(&child_ctx->group_list))
5014
		goto again;
5015 5016 5017 5018

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
5019 5020
}

5021 5022 5023 5024
/*
 * free an unexposed, unused context as created by inheritance by
 * init_task below, used by fork() in case of fail.
 */
5025
void perf_event_free_task(struct task_struct *task)
5026
{
5027 5028
	struct perf_event_context *ctx = task->perf_event_ctxp;
	struct perf_event *event, *tmp;
5029 5030 5031 5032 5033 5034

	if (!ctx)
		return;

	mutex_lock(&ctx->mutex);
again:
5035 5036
	list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry) {
		struct perf_event *parent = event->parent;
5037 5038 5039 5040 5041

		if (WARN_ON_ONCE(!parent))
			continue;

		mutex_lock(&parent->child_mutex);
5042
		list_del_init(&event->child_list);
5043 5044 5045 5046
		mutex_unlock(&parent->child_mutex);

		fput(parent->filp);

5047 5048
		list_del_event(event, ctx);
		free_event(event);
5049 5050
	}

5051
	if (!list_empty(&ctx->group_list))
5052 5053 5054 5055 5056 5057 5058
		goto again;

	mutex_unlock(&ctx->mutex);

	put_ctx(ctx);
}

5059
/*
5060
 * Initialize the perf_event context in task_struct
5061
 */
5062
int perf_event_init_task(struct task_struct *child)
5063
{
5064 5065 5066
	struct perf_event_context *child_ctx, *parent_ctx;
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
5067
	struct task_struct *parent = current;
5068
	int inherited_all = 1;
5069
	int ret = 0;
5070

5071
	child->perf_event_ctxp = NULL;
5072

5073 5074
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);
5075

5076
	if (likely(!parent->perf_event_ctxp))
5077 5078
		return 0;

5079 5080
	/*
	 * This is executed from the parent task context, so inherit
5081
	 * events that have been marked for cloning.
5082
	 * First allocate and initialize a context for the child.
5083 5084
	 */

5085
	child_ctx = kmalloc(sizeof(struct perf_event_context), GFP_KERNEL);
5086
	if (!child_ctx)
5087
		return -ENOMEM;
5088

5089 5090
	__perf_event_init_context(child_ctx, child);
	child->perf_event_ctxp = child_ctx;
5091
	get_task_struct(child);
5092

5093
	/*
5094 5095
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
5096
	 */
5097 5098
	parent_ctx = perf_pin_task_context(parent);

5099 5100 5101 5102 5103 5104 5105
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

5106 5107 5108 5109
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
5110
	mutex_lock(&parent_ctx->mutex);
5111 5112 5113 5114 5115

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
5116
	list_for_each_entry(event, &parent_ctx->group_list, group_entry) {
5117

5118
		if (!event->attr.inherit) {
5119
			inherited_all = 0;
5120
			continue;
5121
		}
5122

5123
		ret = inherit_group(event, parent, parent_ctx,
5124 5125
					     child, child_ctx);
		if (ret) {
5126
			inherited_all = 0;
5127
			break;
5128 5129 5130 5131 5132 5133 5134
		}
	}

	if (inherited_all) {
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
5135 5136
		 * Note that if the parent is a clone, it could get
		 * uncloned at any point, but that doesn't matter
5137
		 * because the list of events and the generation
5138
		 * count can't have changed since we took the mutex.
5139
		 */
5140 5141 5142
		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
5143
			child_ctx->parent_gen = parent_ctx->parent_gen;
5144 5145 5146 5147 5148
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
5149 5150
	}

5151
	mutex_unlock(&parent_ctx->mutex);
5152

5153
	perf_unpin_context(parent_ctx);
5154

5155
	return ret;
5156 5157
}

5158
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
5159
{
5160
	struct perf_cpu_context *cpuctx;
T
Thomas Gleixner 已提交
5161

5162
	cpuctx = &per_cpu(perf_cpu_context, cpu);
5163
	__perf_event_init_context(&cpuctx->ctx, NULL);
T
Thomas Gleixner 已提交
5164

5165
	spin_lock(&perf_resource_lock);
5166
	cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
5167
	spin_unlock(&perf_resource_lock);
5168

5169
	hw_perf_event_setup(cpu);
T
Thomas Gleixner 已提交
5170 5171 5172
}

#ifdef CONFIG_HOTPLUG_CPU
5173
static void __perf_event_exit_cpu(void *info)
T
Thomas Gleixner 已提交
5174 5175
{
	struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
5176 5177
	struct perf_event_context *ctx = &cpuctx->ctx;
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
5178

5179 5180
	list_for_each_entry_safe(event, tmp, &ctx->group_list, group_entry)
		__perf_event_remove_from_context(event);
T
Thomas Gleixner 已提交
5181
}
5182
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
5183
{
5184
	struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
5185
	struct perf_event_context *ctx = &cpuctx->ctx;
5186 5187

	mutex_lock(&ctx->mutex);
5188
	smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
5189
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
5190 5191
}
#else
5192
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203
#endif

static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action) {

	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5204
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
5205 5206
		break;

5207 5208
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
5209
		hw_perf_event_setup_online(cpu);
5210 5211
		break;

T
Thomas Gleixner 已提交
5212 5213
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
5214
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
5215 5216 5217 5218 5219 5220 5221 5222 5223
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

5224 5225 5226
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
T
Thomas Gleixner 已提交
5227 5228
static struct notifier_block __cpuinitdata perf_cpu_nb = {
	.notifier_call		= perf_cpu_notify,
5229
	.priority		= 20,
T
Thomas Gleixner 已提交
5230 5231
};

5232
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
5233 5234 5235
{
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
			(void *)(long)smp_processor_id());
5236 5237
	perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
			(void *)(long)smp_processor_id());
T
Thomas Gleixner 已提交
5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257
	register_cpu_notifier(&perf_cpu_nb);
}

static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_reserved_percpu);
}

static ssize_t
perf_set_reserve_percpu(struct sysdev_class *class,
			const char *buf,
			size_t count)
{
	struct perf_cpu_context *cpuctx;
	unsigned long val;
	int err, cpu, mpt;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
5258
	if (val > perf_max_events)
T
Thomas Gleixner 已提交
5259 5260
		return -EINVAL;

5261
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5262 5263 5264 5265
	perf_reserved_percpu = val;
	for_each_online_cpu(cpu) {
		cpuctx = &per_cpu(perf_cpu_context, cpu);
		spin_lock_irq(&cpuctx->ctx.lock);
5266 5267
		mpt = min(perf_max_events - cpuctx->ctx.nr_events,
			  perf_max_events - perf_reserved_percpu);
T
Thomas Gleixner 已提交
5268 5269 5270
		cpuctx->max_pertask = mpt;
		spin_unlock_irq(&cpuctx->ctx.lock);
	}
5271
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292

	return count;
}

static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
{
	return sprintf(buf, "%d\n", perf_overcommit);
}

static ssize_t
perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
{
	unsigned long val;
	int err;

	err = strict_strtoul(buf, 10, &val);
	if (err)
		return err;
	if (val > 1)
		return -EINVAL;

5293
	spin_lock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5294
	perf_overcommit = val;
5295
	spin_unlock(&perf_resource_lock);
T
Thomas Gleixner 已提交
5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321

	return count;
}

static SYSDEV_CLASS_ATTR(
				reserve_percpu,
				0644,
				perf_show_reserve_percpu,
				perf_set_reserve_percpu
			);

static SYSDEV_CLASS_ATTR(
				overcommit,
				0644,
				perf_show_overcommit,
				perf_set_overcommit
			);

static struct attribute *perfclass_attrs[] = {
	&attr_reserve_percpu.attr,
	&attr_overcommit.attr,
	NULL
};

static struct attribute_group perfclass_attr_group = {
	.attrs			= perfclass_attrs,
5322
	.name			= "perf_events",
T
Thomas Gleixner 已提交
5323 5324
};

5325
static int __init perf_event_sysfs_init(void)
T
Thomas Gleixner 已提交
5326 5327 5328 5329
{
	return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
				  &perfclass_attr_group);
}
5330
device_initcall(perf_event_sysfs_init);