core.c 166.6 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events core code:
T
Thomas Gleixner 已提交
3
 *
4
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 6
 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
A
Al Viro 已提交
7
 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
I
Ingo Molnar 已提交
9
 * For licensing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
T
Thomas Gleixner 已提交
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
P
Peter Zijlstra 已提交
16
#include <linux/idr.h>
17
#include <linux/file.h>
T
Thomas Gleixner 已提交
18
#include <linux/poll.h>
19
#include <linux/slab.h>
20
#include <linux/hash.h>
T
Thomas Gleixner 已提交
21
#include <linux/sysfs.h>
22
#include <linux/dcache.h>
T
Thomas Gleixner 已提交
23
#include <linux/percpu.h>
24
#include <linux/ptrace.h>
P
Peter Zijlstra 已提交
25
#include <linux/reboot.h>
26
#include <linux/vmstat.h>
P
Peter Zijlstra 已提交
27
#include <linux/device.h>
28
#include <linux/export.h>
29
#include <linux/vmalloc.h>
30 31
#include <linux/hardirq.h>
#include <linux/rculist.h>
T
Thomas Gleixner 已提交
32 33 34
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
I
Ingo Molnar 已提交
35
#include <linux/kernel_stat.h>
36
#include <linux/perf_event.h>
L
Li Zefan 已提交
37
#include <linux/ftrace_event.h>
38
#include <linux/hw_breakpoint.h>
T
Thomas Gleixner 已提交
39

40 41
#include "internal.h"

42 43
#include <asm/irq_regs.h>

44
struct remote_function_call {
45 46 47 48
	struct task_struct	*p;
	int			(*func)(void *info);
	void			*info;
	int			ret;
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
};

static void remote_function(void *data)
{
	struct remote_function_call *tfc = data;
	struct task_struct *p = tfc->p;

	if (p) {
		tfc->ret = -EAGAIN;
		if (task_cpu(p) != smp_processor_id() || !task_curr(p))
			return;
	}

	tfc->ret = tfc->func(tfc->info);
}

/**
 * task_function_call - call a function on the cpu on which a task runs
 * @p:		the task to evaluate
 * @func:	the function to be called
 * @info:	the function call argument
 *
 * Calls the function @func when the task is currently running. This might
 * be on the current CPU, which just calls the function directly
 *
 * returns: @func return value, or
 *	    -ESRCH  - when the process isn't running
 *	    -EAGAIN - when the process moved away
 */
static int
task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
{
	struct remote_function_call data = {
82 83 84 85
		.p	= p,
		.func	= func,
		.info	= info,
		.ret	= -ESRCH, /* No such (running) process */
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
	};

	if (task_curr(p))
		smp_call_function_single(task_cpu(p), remote_function, &data, 1);

	return data.ret;
}

/**
 * cpu_function_call - call a function on the cpu
 * @func:	the function to be called
 * @info:	the function call argument
 *
 * Calls the function @func on the remote cpu.
 *
 * returns: @func return value or -ENXIO when the cpu is offline
 */
static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
{
	struct remote_function_call data = {
106 107 108 109
		.p	= NULL,
		.func	= func,
		.info	= info,
		.ret	= -ENXIO, /* No such CPU */
110 111 112 113 114 115 116
	};

	smp_call_function_single(cpu, remote_function, &data, 1);

	return data.ret;
}

S
Stephane Eranian 已提交
117 118 119 120
#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
		       PERF_FLAG_FD_OUTPUT  |\
		       PERF_FLAG_PID_CGROUP)

121 122 123 124 125 126 127
/*
 * branch priv levels that need permission checks
 */
#define PERF_SAMPLE_BRANCH_PERM_PLM \
	(PERF_SAMPLE_BRANCH_KERNEL |\
	 PERF_SAMPLE_BRANCH_HV)

128 129 130 131 132 133
enum event_type_t {
	EVENT_FLEXIBLE = 0x1,
	EVENT_PINNED = 0x2,
	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
};

S
Stephane Eranian 已提交
134 135 136 137
/*
 * perf_sched_events : >0 events exist
 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
 */
138
struct static_key_deferred perf_sched_events __read_mostly;
S
Stephane Eranian 已提交
139
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
140
static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
S
Stephane Eranian 已提交
141

142 143 144
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
145

P
Peter Zijlstra 已提交
146 147 148 149
static LIST_HEAD(pmus);
static DEFINE_MUTEX(pmus_lock);
static struct srcu_struct pmus_srcu;

150
/*
151
 * perf event paranoia level:
152 153
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
154
 *   1 - disallow cpu events for unpriv
155
 *   2 - disallow kernel profiling for unpriv
156
 */
157
int sysctl_perf_event_paranoid __read_mostly = 1;
158

159 160
/* Minimum for 512 kiB + 1 user control page */
int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
161 162

/*
163
 * max perf event sample rate
164
 */
P
Peter Zijlstra 已提交
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
#define DEFAULT_MAX_SAMPLE_RATE 100000
int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
static int max_samples_per_tick __read_mostly =
	DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);

int perf_proc_update_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret = proc_dointvec(table, write, buffer, lenp, ppos);

	if (ret || !write)
		return ret;

	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);

	return 0;
}
183

184
static atomic64_t perf_event_id;
185

186 187 188 189
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type);

static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
S
Stephane Eranian 已提交
190 191 192 193 194
			     enum event_type_t event_type,
			     struct task_struct *task);

static void update_context_time(struct perf_event_context *ctx);
static u64 perf_event_time(struct perf_event *event);
195

196 197 198
static void ring_buffer_attach(struct perf_event *event,
			       struct ring_buffer *rb);

199
void __weak perf_event_print_debug(void)	{ }
T
Thomas Gleixner 已提交
200

201
extern __weak const char *perf_pmu_name(void)
T
Thomas Gleixner 已提交
202
{
203
	return "pmu";
T
Thomas Gleixner 已提交
204 205
}

206 207 208 209 210
static inline u64 perf_clock(void)
{
	return local_clock();
}

S
Stephane Eranian 已提交
211 212 213 214 215 216
static inline struct perf_cpu_context *
__get_cpu_context(struct perf_event_context *ctx)
{
	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
}

217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
			  struct perf_event_context *ctx)
{
	raw_spin_lock(&cpuctx->ctx.lock);
	if (ctx)
		raw_spin_lock(&ctx->lock);
}

static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
			    struct perf_event_context *ctx)
{
	if (ctx)
		raw_spin_unlock(&ctx->lock);
	raw_spin_unlock(&cpuctx->ctx.lock);
}

S
Stephane Eranian 已提交
233 234
#ifdef CONFIG_CGROUP_PERF

235 236 237 238 239
/*
 * Must ensure cgroup is pinned (css_get) before calling
 * this function. In other words, we cannot call this function
 * if there is no cgroup event for the current CPU context.
 */
S
Stephane Eranian 已提交
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
static inline struct perf_cgroup *
perf_cgroup_from_task(struct task_struct *task)
{
	return container_of(task_subsys_state(task, perf_subsys_id),
			struct perf_cgroup, css);
}

static inline bool
perf_cgroup_match(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

	return !event->cgrp || event->cgrp == cpuctx->cgrp;
}

256
static inline bool perf_tryget_cgroup(struct perf_event *event)
S
Stephane Eranian 已提交
257
{
258
	return css_tryget(&event->cgrp->css);
S
Stephane Eranian 已提交
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
}

static inline void perf_put_cgroup(struct perf_event *event)
{
	css_put(&event->cgrp->css);
}

static inline void perf_detach_cgroup(struct perf_event *event)
{
	perf_put_cgroup(event);
	event->cgrp = NULL;
}

static inline int is_cgroup_event(struct perf_event *event)
{
	return event->cgrp != NULL;
}

static inline u64 perf_cgroup_event_time(struct perf_event *event)
{
	struct perf_cgroup_info *t;

	t = per_cpu_ptr(event->cgrp->info, event->cpu);
	return t->time;
}

static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
{
	struct perf_cgroup_info *info;
	u64 now;

	now = perf_clock();

	info = this_cpu_ptr(cgrp->info);

	info->time += now - info->timestamp;
	info->timestamp = now;
}

static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
{
	struct perf_cgroup *cgrp_out = cpuctx->cgrp;
	if (cgrp_out)
		__update_cgrp_time(cgrp_out);
}

static inline void update_cgrp_time_from_event(struct perf_event *event)
{
307 308
	struct perf_cgroup *cgrp;

S
Stephane Eranian 已提交
309
	/*
310 311
	 * ensure we access cgroup data only when needed and
	 * when we know the cgroup is pinned (css_get)
S
Stephane Eranian 已提交
312
	 */
313
	if (!is_cgroup_event(event))
S
Stephane Eranian 已提交
314 315
		return;

316 317 318 319 320 321
	cgrp = perf_cgroup_from_task(current);
	/*
	 * Do not update time when cgroup is not active
	 */
	if (cgrp == event->cgrp)
		__update_cgrp_time(event->cgrp);
S
Stephane Eranian 已提交
322 323 324
}

static inline void
325 326
perf_cgroup_set_timestamp(struct task_struct *task,
			  struct perf_event_context *ctx)
S
Stephane Eranian 已提交
327 328 329 330
{
	struct perf_cgroup *cgrp;
	struct perf_cgroup_info *info;

331 332 333 334 335 336
	/*
	 * ctx->lock held by caller
	 * ensure we do not access cgroup data
	 * unless we have the cgroup pinned (css_get)
	 */
	if (!task || !ctx->nr_cgroups)
S
Stephane Eranian 已提交
337 338 339 340
		return;

	cgrp = perf_cgroup_from_task(task);
	info = this_cpu_ptr(cgrp->info);
341
	info->timestamp = ctx->timestamp;
S
Stephane Eranian 已提交
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
}

#define PERF_CGROUP_SWOUT	0x1 /* cgroup switch out every event */
#define PERF_CGROUP_SWIN	0x2 /* cgroup switch in events based on task */

/*
 * reschedule events based on the cgroup constraint of task.
 *
 * mode SWOUT : schedule out everything
 * mode SWIN : schedule in based on cgroup for next
 */
void perf_cgroup_switch(struct task_struct *task, int mode)
{
	struct perf_cpu_context *cpuctx;
	struct pmu *pmu;
	unsigned long flags;

	/*
	 * disable interrupts to avoid geting nr_cgroup
	 * changes via __perf_event_disable(). Also
	 * avoids preemption.
	 */
	local_irq_save(flags);

	/*
	 * we reschedule only in the presence of cgroup
	 * constrained events.
	 */
	rcu_read_lock();

	list_for_each_entry_rcu(pmu, &pmus, entry) {
		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

		/*
		 * perf_cgroup_events says at least one
		 * context on this CPU has cgroup events.
		 *
		 * ctx->nr_cgroups reports the number of cgroup
		 * events for a context.
		 */
		if (cpuctx->ctx.nr_cgroups > 0) {
383 384
			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
			perf_pmu_disable(cpuctx->ctx.pmu);
S
Stephane Eranian 已提交
385 386 387 388 389 390 391 392 393 394 395

			if (mode & PERF_CGROUP_SWOUT) {
				cpu_ctx_sched_out(cpuctx, EVENT_ALL);
				/*
				 * must not be done before ctxswout due
				 * to event_filter_match() in event_sched_out()
				 */
				cpuctx->cgrp = NULL;
			}

			if (mode & PERF_CGROUP_SWIN) {
396
				WARN_ON_ONCE(cpuctx->cgrp);
S
Stephane Eranian 已提交
397 398 399 400 401 402 403
				/* set cgrp before ctxsw in to
				 * allow event_filter_match() to not
				 * have to pass task around
				 */
				cpuctx->cgrp = perf_cgroup_from_task(task);
				cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
			}
404 405
			perf_pmu_enable(cpuctx->ctx.pmu);
			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
S
Stephane Eranian 已提交
406 407 408 409 410 411 412 413
		}
	}

	rcu_read_unlock();

	local_irq_restore(flags);
}

414 415
static inline void perf_cgroup_sched_out(struct task_struct *task,
					 struct task_struct *next)
S
Stephane Eranian 已提交
416
{
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
	struct perf_cgroup *cgrp1;
	struct perf_cgroup *cgrp2 = NULL;

	/*
	 * we come here when we know perf_cgroup_events > 0
	 */
	cgrp1 = perf_cgroup_from_task(task);

	/*
	 * next is NULL when called from perf_event_enable_on_exec()
	 * that will systematically cause a cgroup_switch()
	 */
	if (next)
		cgrp2 = perf_cgroup_from_task(next);

	/*
	 * only schedule out current cgroup events if we know
	 * that we are switching to a different cgroup. Otherwise,
	 * do no touch the cgroup events.
	 */
	if (cgrp1 != cgrp2)
		perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
S
Stephane Eranian 已提交
439 440
}

441 442
static inline void perf_cgroup_sched_in(struct task_struct *prev,
					struct task_struct *task)
S
Stephane Eranian 已提交
443
{
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461
	struct perf_cgroup *cgrp1;
	struct perf_cgroup *cgrp2 = NULL;

	/*
	 * we come here when we know perf_cgroup_events > 0
	 */
	cgrp1 = perf_cgroup_from_task(task);

	/* prev can never be NULL */
	cgrp2 = perf_cgroup_from_task(prev);

	/*
	 * only need to schedule in cgroup events if we are changing
	 * cgroup during ctxsw. Cgroup events were not scheduled
	 * out of ctxsw out if that was not the case.
	 */
	if (cgrp1 != cgrp2)
		perf_cgroup_switch(task, PERF_CGROUP_SWIN);
S
Stephane Eranian 已提交
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477
}

static inline int perf_cgroup_connect(int fd, struct perf_event *event,
				      struct perf_event_attr *attr,
				      struct perf_event *group_leader)
{
	struct perf_cgroup *cgrp;
	struct cgroup_subsys_state *css;
	struct file *file;
	int ret = 0, fput_needed;

	file = fget_light(fd, &fput_needed);
	if (!file)
		return -EBADF;

	css = cgroup_css_from_dir(file, perf_subsys_id);
478 479 480 481
	if (IS_ERR(css)) {
		ret = PTR_ERR(css);
		goto out;
	}
S
Stephane Eranian 已提交
482 483 484 485

	cgrp = container_of(css, struct perf_cgroup, css);
	event->cgrp = cgrp;

486
	/* must be done before we fput() the file */
487 488 489 490 491
	if (!perf_tryget_cgroup(event)) {
		event->cgrp = NULL;
		ret = -ENOENT;
		goto out;
	}
492

S
Stephane Eranian 已提交
493 494 495 496 497 498 499 500 501
	/*
	 * all events in a group must monitor
	 * the same cgroup because a task belongs
	 * to only one perf cgroup at a time
	 */
	if (group_leader && group_leader->cgrp != cgrp) {
		perf_detach_cgroup(event);
		ret = -EINVAL;
	}
502
out:
S
Stephane Eranian 已提交
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
	fput_light(file, fput_needed);
	return ret;
}

static inline void
perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
{
	struct perf_cgroup_info *t;
	t = per_cpu_ptr(event->cgrp->info, event->cpu);
	event->shadow_ctx_time = now - t->timestamp;
}

static inline void
perf_cgroup_defer_enabled(struct perf_event *event)
{
	/*
	 * when the current task's perf cgroup does not match
	 * the event's, we need to remember to call the
	 * perf_mark_enable() function the first time a task with
	 * a matching perf cgroup is scheduled in.
	 */
	if (is_cgroup_event(event) && !perf_cgroup_match(event))
		event->cgrp_defer_enabled = 1;
}

static inline void
perf_cgroup_mark_enabled(struct perf_event *event,
			 struct perf_event_context *ctx)
{
	struct perf_event *sub;
	u64 tstamp = perf_event_time(event);

	if (!event->cgrp_defer_enabled)
		return;

	event->cgrp_defer_enabled = 0;

	event->tstamp_enabled = tstamp - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry) {
		if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
			sub->cgrp_defer_enabled = 0;
		}
	}
}
#else /* !CONFIG_CGROUP_PERF */

static inline bool
perf_cgroup_match(struct perf_event *event)
{
	return true;
}

static inline void perf_detach_cgroup(struct perf_event *event)
{}

static inline int is_cgroup_event(struct perf_event *event)
{
	return 0;
}

static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
{
	return 0;
}

static inline void update_cgrp_time_from_event(struct perf_event *event)
{
}

static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
{
}

577 578
static inline void perf_cgroup_sched_out(struct task_struct *task,
					 struct task_struct *next)
S
Stephane Eranian 已提交
579 580 581
{
}

582 583
static inline void perf_cgroup_sched_in(struct task_struct *prev,
					struct task_struct *task)
S
Stephane Eranian 已提交
584 585 586 587 588 589 590 591 592 593 594
{
}

static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
				      struct perf_event_attr *attr,
				      struct perf_event *group_leader)
{
	return -EINVAL;
}

static inline void
595 596
perf_cgroup_set_timestamp(struct task_struct *task,
			  struct perf_event_context *ctx)
S
Stephane Eranian 已提交
597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
{
}

void
perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
{
}

static inline void
perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
{
}

static inline u64 perf_cgroup_event_time(struct perf_event *event)
{
	return 0;
}

static inline void
perf_cgroup_defer_enabled(struct perf_event *event)
{
}

static inline void
perf_cgroup_mark_enabled(struct perf_event *event,
			 struct perf_event_context *ctx)
{
}
#endif

P
Peter Zijlstra 已提交
627
void perf_pmu_disable(struct pmu *pmu)
628
{
P
Peter Zijlstra 已提交
629 630 631
	int *count = this_cpu_ptr(pmu->pmu_disable_count);
	if (!(*count)++)
		pmu->pmu_disable(pmu);
632 633
}

P
Peter Zijlstra 已提交
634
void perf_pmu_enable(struct pmu *pmu)
635
{
P
Peter Zijlstra 已提交
636 637 638
	int *count = this_cpu_ptr(pmu->pmu_disable_count);
	if (!--(*count))
		pmu->pmu_enable(pmu);
639 640
}

641 642 643 644 645 646 647
static DEFINE_PER_CPU(struct list_head, rotation_list);

/*
 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
 * because they're strictly cpu affine and rotate_start is called with IRQs
 * disabled, while rotate_context is called from IRQ context.
 */
P
Peter Zijlstra 已提交
648
static void perf_pmu_rotate_start(struct pmu *pmu)
649
{
P
Peter Zijlstra 已提交
650
	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
651
	struct list_head *head = &__get_cpu_var(rotation_list);
652

653
	WARN_ON(!irqs_disabled());
654

655 656
	if (list_empty(&cpuctx->rotation_list))
		list_add(&cpuctx->rotation_list, head);
657 658
}

659
static void get_ctx(struct perf_event_context *ctx)
660
{
661
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
662 663
}

664
static void put_ctx(struct perf_event_context *ctx)
665
{
666 667 668
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
669 670
		if (ctx->task)
			put_task_struct(ctx->task);
671
		kfree_rcu(ctx, rcu_head);
672
	}
673 674
}

675
static void unclone_ctx(struct perf_event_context *ctx)
676 677 678 679 680 681 682
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
{
	/*
	 * only top level events have the pid namespace they were created in
	 */
	if (event->parent)
		event = event->parent;

	return task_tgid_nr_ns(p, event->ns);
}

static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
{
	/*
	 * only top level events have the pid namespace they were created in
	 */
	if (event->parent)
		event = event->parent;

	return task_pid_nr_ns(p, event->ns);
}

705
/*
706
 * If we inherit events we want to return the parent event id
707 708
 * to userspace.
 */
709
static u64 primary_event_id(struct perf_event *event)
710
{
711
	u64 id = event->id;
712

713 714
	if (event->parent)
		id = event->parent->id;
715 716 717 718

	return id;
}

719
/*
720
 * Get the perf_event_context for a task and lock it.
721 722 723
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
724
static struct perf_event_context *
P
Peter Zijlstra 已提交
725
perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
726
{
727
	struct perf_event_context *ctx;
728 729

	rcu_read_lock();
P
Peter Zijlstra 已提交
730
retry:
P
Peter Zijlstra 已提交
731
	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
732 733 734 735
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
736
		 * perf_event_task_sched_out, though the
737 738 739 740 741 742
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
743
		raw_spin_lock_irqsave(&ctx->lock, *flags);
P
Peter Zijlstra 已提交
744
		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
745
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
746 747
			goto retry;
		}
748 749

		if (!atomic_inc_not_zero(&ctx->refcount)) {
750
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
751 752
			ctx = NULL;
		}
753 754 755 756 757 758 759 760 761 762
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
P
Peter Zijlstra 已提交
763 764
static struct perf_event_context *
perf_pin_task_context(struct task_struct *task, int ctxn)
765
{
766
	struct perf_event_context *ctx;
767 768
	unsigned long flags;

P
Peter Zijlstra 已提交
769
	ctx = perf_lock_task_context(task, ctxn, &flags);
770 771
	if (ctx) {
		++ctx->pin_count;
772
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
773 774 775 776
	}
	return ctx;
}

777
static void perf_unpin_context(struct perf_event_context *ctx)
778 779 780
{
	unsigned long flags;

781
	raw_spin_lock_irqsave(&ctx->lock, flags);
782
	--ctx->pin_count;
783
	raw_spin_unlock_irqrestore(&ctx->lock, flags);
784 785
}

786 787 788 789 790 791 792 793 794 795 796
/*
 * Update the record of the current time in a context.
 */
static void update_context_time(struct perf_event_context *ctx)
{
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
}

797 798 799
static u64 perf_event_time(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
S
Stephane Eranian 已提交
800 801 802 803

	if (is_cgroup_event(event))
		return perf_cgroup_event_time(event);

804 805 806
	return ctx ? ctx->time : 0;
}

807 808
/*
 * Update the total_time_enabled and total_time_running fields for a event.
809
 * The caller of this function needs to hold the ctx->lock.
810 811 812 813 814 815 816 817 818
 */
static void update_event_times(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	u64 run_end;

	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
		return;
S
Stephane Eranian 已提交
819 820 821 822 823 824 825 826 827 828 829
	/*
	 * in cgroup mode, time_enabled represents
	 * the time the event was enabled AND active
	 * tasks were in the monitored cgroup. This is
	 * independent of the activity of the context as
	 * there may be a mix of cgroup and non-cgroup events.
	 *
	 * That is why we treat cgroup events differently
	 * here.
	 */
	if (is_cgroup_event(event))
830
		run_end = perf_cgroup_event_time(event);
S
Stephane Eranian 已提交
831 832
	else if (ctx->is_active)
		run_end = ctx->time;
833 834 835 836
	else
		run_end = event->tstamp_stopped;

	event->total_time_enabled = run_end - event->tstamp_enabled;
837 838 839 840

	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
	else
841
		run_end = perf_event_time(event);
842 843

	event->total_time_running = run_end - event->tstamp_running;
S
Stephane Eranian 已提交
844

845 846
}

847 848 849 850 851 852 853 854 855 856 857 858
/*
 * Update total_time_enabled and total_time_running for all events in a group.
 */
static void update_group_times(struct perf_event *leader)
{
	struct perf_event *event;

	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
}

859 860 861 862 863 864 865 866 867
static struct list_head *
ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
{
	if (event->attr.pinned)
		return &ctx->pinned_groups;
	else
		return &ctx->flexible_groups;
}

868
/*
869
 * Add a event from the lists for its context.
870 871
 * Must be called with ctx->mutex and ctx->lock held.
 */
872
static void
873
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
874
{
875 876
	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
	event->attach_state |= PERF_ATTACH_CONTEXT;
877 878

	/*
879 880 881
	 * If we're a stand alone event or group leader, we go to the context
	 * list, group events are kept attached to the group so that
	 * perf_group_detach can, at all times, locate all siblings.
882
	 */
883
	if (event->group_leader == event) {
884 885
		struct list_head *list;

886 887 888
		if (is_software_event(event))
			event->group_flags |= PERF_GROUP_SOFTWARE;

889 890
		list = ctx_group_list(event, ctx);
		list_add_tail(&event->group_entry, list);
P
Peter Zijlstra 已提交
891
	}
P
Peter Zijlstra 已提交
892

893
	if (is_cgroup_event(event))
S
Stephane Eranian 已提交
894 895
		ctx->nr_cgroups++;

896 897 898
	if (has_branch_stack(event))
		ctx->nr_branch_stack++;

899
	list_add_rcu(&event->event_entry, &ctx->event_list);
900
	if (!ctx->nr_events)
P
Peter Zijlstra 已提交
901
		perf_pmu_rotate_start(ctx->pmu);
902 903
	ctx->nr_events++;
	if (event->attr.inherit_stat)
904
		ctx->nr_stat++;
905 906
}

907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945
/*
 * Called at perf_event creation and when events are attached/detached from a
 * group.
 */
static void perf_event__read_size(struct perf_event *event)
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		size += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		size += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_ID)
		entry += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
		size += sizeof(u64);
	}

	size += entry * nr;
	event->read_size = size;
}

static void perf_event__header_size(struct perf_event *event)
{
	struct perf_sample_data *data;
	u64 sample_type = event->attr.sample_type;
	u16 size = 0;

	perf_event__read_size(event);

	if (sample_type & PERF_SAMPLE_IP)
		size += sizeof(data->ip);

946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
	if (sample_type & PERF_SAMPLE_ADDR)
		size += sizeof(data->addr);

	if (sample_type & PERF_SAMPLE_PERIOD)
		size += sizeof(data->period);

	if (sample_type & PERF_SAMPLE_READ)
		size += event->read_size;

	event->header_size = size;
}

static void perf_event__id_header_size(struct perf_event *event)
{
	struct perf_sample_data *data;
	u64 sample_type = event->attr.sample_type;
	u16 size = 0;

964 965 966 967 968 969 970 971 972 973 974 975 976 977 978
	if (sample_type & PERF_SAMPLE_TID)
		size += sizeof(data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		size += sizeof(data->time);

	if (sample_type & PERF_SAMPLE_ID)
		size += sizeof(data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		size += sizeof(data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		size += sizeof(data->cpu_entry);

979
	event->id_header_size = size;
980 981
}

982 983
static void perf_group_attach(struct perf_event *event)
{
984
	struct perf_event *group_leader = event->group_leader, *pos;
985

P
Peter Zijlstra 已提交
986 987 988 989 990 991
	/*
	 * We can have double attach due to group movement in perf_event_open.
	 */
	if (event->attach_state & PERF_ATTACH_GROUP)
		return;

992 993 994 995 996 997 998 999 1000 1001 1002
	event->attach_state |= PERF_ATTACH_GROUP;

	if (group_leader == event)
		return;

	if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
			!is_software_event(event))
		group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;

	list_add_tail(&event->group_entry, &group_leader->sibling_list);
	group_leader->nr_siblings++;
1003 1004 1005 1006 1007

	perf_event__header_size(group_leader);

	list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
		perf_event__header_size(pos);
1008 1009
}

1010
/*
1011
 * Remove a event from the lists for its context.
1012
 * Must be called with ctx->mutex and ctx->lock held.
1013
 */
1014
static void
1015
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1016
{
1017
	struct perf_cpu_context *cpuctx;
1018 1019 1020 1021
	/*
	 * We can have double detach due to exit/hot-unplug + close.
	 */
	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1022
		return;
1023 1024 1025

	event->attach_state &= ~PERF_ATTACH_CONTEXT;

1026
	if (is_cgroup_event(event)) {
S
Stephane Eranian 已提交
1027
		ctx->nr_cgroups--;
1028 1029 1030 1031 1032 1033 1034 1035 1036
		cpuctx = __get_cpu_context(ctx);
		/*
		 * if there are no more cgroup events
		 * then cler cgrp to avoid stale pointer
		 * in update_cgrp_time_from_cpuctx()
		 */
		if (!ctx->nr_cgroups)
			cpuctx->cgrp = NULL;
	}
S
Stephane Eranian 已提交
1037

1038 1039 1040
	if (has_branch_stack(event))
		ctx->nr_branch_stack--;

1041 1042
	ctx->nr_events--;
	if (event->attr.inherit_stat)
1043
		ctx->nr_stat--;
1044

1045
	list_del_rcu(&event->event_entry);
1046

1047 1048
	if (event->group_leader == event)
		list_del_init(&event->group_entry);
P
Peter Zijlstra 已提交
1049

1050
	update_group_times(event);
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060

	/*
	 * If event was in error state, then keep it
	 * that way, otherwise bogus counts will be
	 * returned on read(). The only way to get out
	 * of error state is by explicit re-enabling
	 * of the event
	 */
	if (event->state > PERF_EVENT_STATE_OFF)
		event->state = PERF_EVENT_STATE_OFF;
1061 1062
}

1063
static void perf_group_detach(struct perf_event *event)
1064 1065
{
	struct perf_event *sibling, *tmp;
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
	struct list_head *list = NULL;

	/*
	 * We can have double detach due to exit/hot-unplug + close.
	 */
	if (!(event->attach_state & PERF_ATTACH_GROUP))
		return;

	event->attach_state &= ~PERF_ATTACH_GROUP;

	/*
	 * If this is a sibling, remove it from its group.
	 */
	if (event->group_leader != event) {
		list_del_init(&event->group_entry);
		event->group_leader->nr_siblings--;
1082
		goto out;
1083 1084 1085 1086
	}

	if (!list_empty(&event->group_entry))
		list = &event->group_entry;
1087

1088
	/*
1089 1090
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
1091
	 * to whatever list we are on.
1092
	 */
1093
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1094 1095
		if (list)
			list_move_tail(&sibling->group_entry, list);
1096
		sibling->group_leader = sibling;
1097 1098 1099

		/* Inherit group flags from the previous leader */
		sibling->group_flags = event->group_flags;
1100
	}
1101 1102 1103 1104 1105 1106

out:
	perf_event__header_size(event->group_leader);

	list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
		perf_event__header_size(tmp);
1107 1108
}

1109 1110 1111
static inline int
event_filter_match(struct perf_event *event)
{
S
Stephane Eranian 已提交
1112 1113
	return (event->cpu == -1 || event->cpu == smp_processor_id())
	    && perf_cgroup_match(event);
1114 1115
}

1116 1117
static void
event_sched_out(struct perf_event *event,
1118
		  struct perf_cpu_context *cpuctx,
1119
		  struct perf_event_context *ctx)
1120
{
1121
	u64 tstamp = perf_event_time(event);
1122 1123 1124 1125 1126 1127 1128 1129 1130
	u64 delta;
	/*
	 * An event which could not be activated because of
	 * filter mismatch still needs to have its timings
	 * maintained, otherwise bogus information is return
	 * via read() for time_enabled, time_running:
	 */
	if (event->state == PERF_EVENT_STATE_INACTIVE
	    && !event_filter_match(event)) {
S
Stephane Eranian 已提交
1131
		delta = tstamp - event->tstamp_stopped;
1132
		event->tstamp_running += delta;
1133
		event->tstamp_stopped = tstamp;
1134 1135
	}

1136
	if (event->state != PERF_EVENT_STATE_ACTIVE)
1137
		return;
1138

1139 1140 1141 1142
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
1143
	}
1144
	event->tstamp_stopped = tstamp;
P
Peter Zijlstra 已提交
1145
	event->pmu->del(event, 0);
1146
	event->oncpu = -1;
1147

1148
	if (!is_software_event(event))
1149 1150
		cpuctx->active_oncpu--;
	ctx->nr_active--;
1151 1152
	if (event->attr.freq && event->attr.sample_freq)
		ctx->nr_freq--;
1153
	if (event->attr.exclusive || !cpuctx->active_oncpu)
1154 1155 1156
		cpuctx->exclusive = 0;
}

1157
static void
1158
group_sched_out(struct perf_event *group_event,
1159
		struct perf_cpu_context *cpuctx,
1160
		struct perf_event_context *ctx)
1161
{
1162
	struct perf_event *event;
1163
	int state = group_event->state;
1164

1165
	event_sched_out(group_event, cpuctx, ctx);
1166 1167 1168 1169

	/*
	 * Schedule out siblings (if any):
	 */
1170 1171
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
1172

1173
	if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1174 1175 1176
		cpuctx->exclusive = 0;
}

T
Thomas Gleixner 已提交
1177
/*
1178
 * Cross CPU call to remove a performance event
T
Thomas Gleixner 已提交
1179
 *
1180
 * We disable the event on the hardware level first. After that we
T
Thomas Gleixner 已提交
1181 1182
 * remove it from the context list.
 */
1183
static int __perf_remove_from_context(void *info)
T
Thomas Gleixner 已提交
1184
{
1185 1186
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
P
Peter Zijlstra 已提交
1187
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
T
Thomas Gleixner 已提交
1188

1189
	raw_spin_lock(&ctx->lock);
1190 1191
	event_sched_out(event, cpuctx, ctx);
	list_del_event(event, ctx);
1192 1193 1194 1195
	if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
		ctx->is_active = 0;
		cpuctx->task_ctx = NULL;
	}
1196
	raw_spin_unlock(&ctx->lock);
1197 1198

	return 0;
T
Thomas Gleixner 已提交
1199 1200 1201 1202
}


/*
1203
 * Remove the event from a task's (or a CPU's) list of events.
T
Thomas Gleixner 已提交
1204
 *
1205
 * CPU events are removed with a smp call. For task events we only
T
Thomas Gleixner 已提交
1206
 * call when the task is on a CPU.
1207
 *
1208 1209
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
1210 1211
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
1212
 * When called from perf_event_exit_task, it's OK because the
1213
 * context has been detached from its task.
T
Thomas Gleixner 已提交
1214
 */
1215
static void perf_remove_from_context(struct perf_event *event)
T
Thomas Gleixner 已提交
1216
{
1217
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
1218 1219
	struct task_struct *task = ctx->task;

1220 1221
	lockdep_assert_held(&ctx->mutex);

T
Thomas Gleixner 已提交
1222 1223
	if (!task) {
		/*
1224
		 * Per cpu events are removed via an smp call and
1225
		 * the removal is always successful.
T
Thomas Gleixner 已提交
1226
		 */
1227
		cpu_function_call(event->cpu, __perf_remove_from_context, event);
T
Thomas Gleixner 已提交
1228 1229 1230 1231
		return;
	}

retry:
1232 1233
	if (!task_function_call(task, __perf_remove_from_context, event))
		return;
T
Thomas Gleixner 已提交
1234

1235
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1236
	/*
1237 1238
	 * If we failed to find a running task, but find the context active now
	 * that we've acquired the ctx->lock, retry.
T
Thomas Gleixner 已提交
1239
	 */
1240
	if (ctx->is_active) {
1241
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1242 1243 1244 1245
		goto retry;
	}

	/*
1246 1247
	 * Since the task isn't running, its safe to remove the event, us
	 * holding the ctx->lock ensures the task won't get scheduled in.
T
Thomas Gleixner 已提交
1248
	 */
1249
	list_del_event(event, ctx);
1250
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1251 1252
}

1253
/*
1254
 * Cross CPU call to disable a performance event
1255
 */
1256
static int __perf_event_disable(void *info)
1257
{
1258 1259
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
P
Peter Zijlstra 已提交
1260
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1261 1262

	/*
1263 1264
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
1265 1266 1267
	 *
	 * Can trigger due to concurrent perf_event_context_sched_out()
	 * flipping contexts around.
1268
	 */
1269
	if (ctx->task && cpuctx->task_ctx != ctx)
1270
		return -EINVAL;
1271

1272
	raw_spin_lock(&ctx->lock);
1273 1274

	/*
1275
	 * If the event is on, turn it off.
1276 1277
	 * If it is in error state, leave it in error state.
	 */
1278
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1279
		update_context_time(ctx);
S
Stephane Eranian 已提交
1280
		update_cgrp_time_from_event(event);
1281 1282 1283
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
1284
		else
1285 1286
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
1287 1288
	}

1289
	raw_spin_unlock(&ctx->lock);
1290 1291

	return 0;
1292 1293 1294
}

/*
1295
 * Disable a event.
1296
 *
1297 1298
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
1299
 * remains valid.  This condition is satisifed when called through
1300 1301 1302 1303
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
1304
 * is the current context on this CPU and preemption is disabled,
1305
 * hence we can't get into perf_event_task_sched_out for this context.
1306
 */
1307
void perf_event_disable(struct perf_event *event)
1308
{
1309
	struct perf_event_context *ctx = event->ctx;
1310 1311 1312 1313
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
1314
		 * Disable the event on the cpu that it's on
1315
		 */
1316
		cpu_function_call(event->cpu, __perf_event_disable, event);
1317 1318 1319
		return;
	}

P
Peter Zijlstra 已提交
1320
retry:
1321 1322
	if (!task_function_call(task, __perf_event_disable, event))
		return;
1323

1324
	raw_spin_lock_irq(&ctx->lock);
1325
	/*
1326
	 * If the event is still active, we need to retry the cross-call.
1327
	 */
1328
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
1329
		raw_spin_unlock_irq(&ctx->lock);
1330 1331 1332 1333 1334
		/*
		 * Reload the task pointer, it might have been changed by
		 * a concurrent perf_event_context_sched_out().
		 */
		task = ctx->task;
1335 1336 1337 1338 1339 1340 1341
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1342 1343 1344
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
1345
	}
1346
	raw_spin_unlock_irq(&ctx->lock);
1347
}
1348
EXPORT_SYMBOL_GPL(perf_event_disable);
1349

S
Stephane Eranian 已提交
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
static void perf_set_shadow_time(struct perf_event *event,
				 struct perf_event_context *ctx,
				 u64 tstamp)
{
	/*
	 * use the correct time source for the time snapshot
	 *
	 * We could get by without this by leveraging the
	 * fact that to get to this function, the caller
	 * has most likely already called update_context_time()
	 * and update_cgrp_time_xx() and thus both timestamp
	 * are identical (or very close). Given that tstamp is,
	 * already adjusted for cgroup, we could say that:
	 *    tstamp - ctx->timestamp
	 * is equivalent to
	 *    tstamp - cgrp->timestamp.
	 *
	 * Then, in perf_output_read(), the calculation would
	 * work with no changes because:
	 * - event is guaranteed scheduled in
	 * - no scheduled out in between
	 * - thus the timestamp would be the same
	 *
	 * But this is a bit hairy.
	 *
	 * So instead, we have an explicit cgroup call to remain
	 * within the time time source all along. We believe it
	 * is cleaner and simpler to understand.
	 */
	if (is_cgroup_event(event))
		perf_cgroup_set_shadow_time(event, tstamp);
	else
		event->shadow_ctx_time = tstamp - ctx->timestamp;
}

P
Peter Zijlstra 已提交
1385 1386 1387 1388
#define MAX_INTERRUPTS (~0ULL)

static void perf_log_throttle(struct perf_event *event, int enable);

1389
static int
1390
event_sched_in(struct perf_event *event,
1391
		 struct perf_cpu_context *cpuctx,
1392
		 struct perf_event_context *ctx)
1393
{
1394 1395
	u64 tstamp = perf_event_time(event);

1396
	if (event->state <= PERF_EVENT_STATE_OFF)
1397 1398
		return 0;

1399
	event->state = PERF_EVENT_STATE_ACTIVE;
1400
	event->oncpu = smp_processor_id();
P
Peter Zijlstra 已提交
1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411

	/*
	 * Unthrottle events, since we scheduled we might have missed several
	 * ticks already, also for a heavily scheduling task there is little
	 * guarantee it'll get a tick in a timely manner.
	 */
	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
		perf_log_throttle(event, 1);
		event->hw.interrupts = 0;
	}

1412 1413 1414 1415 1416
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

P
Peter Zijlstra 已提交
1417
	if (event->pmu->add(event, PERF_EF_START)) {
1418 1419
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
1420 1421 1422
		return -EAGAIN;
	}

1423
	event->tstamp_running += tstamp - event->tstamp_stopped;
1424

S
Stephane Eranian 已提交
1425
	perf_set_shadow_time(event, ctx, tstamp);
1426

1427
	if (!is_software_event(event))
1428
		cpuctx->active_oncpu++;
1429
	ctx->nr_active++;
1430 1431
	if (event->attr.freq && event->attr.sample_freq)
		ctx->nr_freq++;
1432

1433
	if (event->attr.exclusive)
1434 1435
		cpuctx->exclusive = 1;

1436 1437 1438
	return 0;
}

1439
static int
1440
group_sched_in(struct perf_event *group_event,
1441
	       struct perf_cpu_context *cpuctx,
1442
	       struct perf_event_context *ctx)
1443
{
1444
	struct perf_event *event, *partial_group = NULL;
P
Peter Zijlstra 已提交
1445
	struct pmu *pmu = group_event->pmu;
1446 1447
	u64 now = ctx->time;
	bool simulate = false;
1448

1449
	if (group_event->state == PERF_EVENT_STATE_OFF)
1450 1451
		return 0;

P
Peter Zijlstra 已提交
1452
	pmu->start_txn(pmu);
1453

1454
	if (event_sched_in(group_event, cpuctx, ctx)) {
P
Peter Zijlstra 已提交
1455
		pmu->cancel_txn(pmu);
1456
		return -EAGAIN;
1457
	}
1458 1459 1460 1461

	/*
	 * Schedule in siblings as one group (if any):
	 */
1462
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1463
		if (event_sched_in(event, cpuctx, ctx)) {
1464
			partial_group = event;
1465 1466 1467 1468
			goto group_error;
		}
	}

1469
	if (!pmu->commit_txn(pmu))
1470
		return 0;
1471

1472 1473 1474 1475
group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485
	 * The events up to the failed event are scheduled out normally,
	 * tstamp_stopped will be updated.
	 *
	 * The failed events and the remaining siblings need to have
	 * their timings updated as if they had gone thru event_sched_in()
	 * and event_sched_out(). This is required to get consistent timings
	 * across the group. This also takes care of the case where the group
	 * could never be scheduled by ensuring tstamp_stopped is set to mark
	 * the time the event was actually stopped, such that time delta
	 * calculation in update_event_times() is correct.
1486
	 */
1487 1488
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
1489 1490 1491 1492 1493 1494 1495 1496
			simulate = true;

		if (simulate) {
			event->tstamp_running += now - event->tstamp_stopped;
			event->tstamp_stopped = now;
		} else {
			event_sched_out(event, cpuctx, ctx);
		}
1497
	}
1498
	event_sched_out(group_event, cpuctx, ctx);
1499

P
Peter Zijlstra 已提交
1500
	pmu->cancel_txn(pmu);
1501

1502 1503 1504
	return -EAGAIN;
}

1505
/*
1506
 * Work out whether we can put this event group on the CPU now.
1507
 */
1508
static int group_can_go_on(struct perf_event *event,
1509 1510 1511 1512
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
1513
	 * Groups consisting entirely of software events can always go on.
1514
	 */
1515
	if (event->group_flags & PERF_GROUP_SOFTWARE)
1516 1517 1518
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
1519
	 * events can go on.
1520 1521 1522 1523 1524
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
1525
	 * events on the CPU, it can't go on.
1526
	 */
1527
	if (event->attr.exclusive && cpuctx->active_oncpu)
1528 1529 1530 1531 1532 1533 1534 1535
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

1536 1537
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
1538
{
1539 1540
	u64 tstamp = perf_event_time(event);

1541
	list_add_event(event, ctx);
1542
	perf_group_attach(event);
1543 1544 1545
	event->tstamp_enabled = tstamp;
	event->tstamp_running = tstamp;
	event->tstamp_stopped = tstamp;
1546 1547
}

1548 1549 1550 1551 1552 1553
static void task_ctx_sched_out(struct perf_event_context *ctx);
static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
	     enum event_type_t event_type,
	     struct task_struct *task);
1554

1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
				struct perf_event_context *ctx,
				struct task_struct *task)
{
	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
	if (ctx)
		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
	if (ctx)
		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
}

T
Thomas Gleixner 已提交
1567
/*
1568
 * Cross CPU call to install and enable a performance event
1569 1570
 *
 * Must be called with ctx->mutex held
T
Thomas Gleixner 已提交
1571
 */
1572
static int  __perf_install_in_context(void *info)
T
Thomas Gleixner 已提交
1573
{
1574 1575
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
P
Peter Zijlstra 已提交
1576
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1577 1578 1579
	struct perf_event_context *task_ctx = cpuctx->task_ctx;
	struct task_struct *task = current;

1580
	perf_ctx_lock(cpuctx, task_ctx);
1581
	perf_pmu_disable(cpuctx->ctx.pmu);
T
Thomas Gleixner 已提交
1582 1583

	/*
1584
	 * If there was an active task_ctx schedule it out.
T
Thomas Gleixner 已提交
1585
	 */
1586
	if (task_ctx)
1587
		task_ctx_sched_out(task_ctx);
1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601

	/*
	 * If the context we're installing events in is not the
	 * active task_ctx, flip them.
	 */
	if (ctx->task && task_ctx != ctx) {
		if (task_ctx)
			raw_spin_unlock(&task_ctx->lock);
		raw_spin_lock(&ctx->lock);
		task_ctx = ctx;
	}

	if (task_ctx) {
		cpuctx->task_ctx = task_ctx;
1602 1603
		task = task_ctx->task;
	}
1604

1605
	cpu_ctx_sched_out(cpuctx, EVENT_ALL);
T
Thomas Gleixner 已提交
1606

1607
	update_context_time(ctx);
S
Stephane Eranian 已提交
1608 1609 1610 1611 1612 1613
	/*
	 * update cgrp time only if current cgrp
	 * matches event->cgrp. Must be done before
	 * calling add_event_to_ctx()
	 */
	update_cgrp_time_from_event(event);
T
Thomas Gleixner 已提交
1614

1615
	add_event_to_ctx(event, ctx);
T
Thomas Gleixner 已提交
1616

1617
	/*
1618
	 * Schedule everything back in
1619
	 */
1620
	perf_event_sched_in(cpuctx, task_ctx, task);
1621 1622 1623

	perf_pmu_enable(cpuctx->ctx.pmu);
	perf_ctx_unlock(cpuctx, task_ctx);
1624 1625

	return 0;
T
Thomas Gleixner 已提交
1626 1627 1628
}

/*
1629
 * Attach a performance event to a context
T
Thomas Gleixner 已提交
1630
 *
1631 1632
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
T
Thomas Gleixner 已提交
1633
 *
1634
 * If the event is attached to a task which is on a CPU we use a smp
T
Thomas Gleixner 已提交
1635 1636 1637 1638
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
 */
static void
1639 1640
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
T
Thomas Gleixner 已提交
1641 1642 1643 1644
			int cpu)
{
	struct task_struct *task = ctx->task;

1645 1646
	lockdep_assert_held(&ctx->mutex);

1647 1648
	event->ctx = ctx;

T
Thomas Gleixner 已提交
1649 1650
	if (!task) {
		/*
1651
		 * Per cpu events are installed via an smp call and
1652
		 * the install is always successful.
T
Thomas Gleixner 已提交
1653
		 */
1654
		cpu_function_call(cpu, __perf_install_in_context, event);
T
Thomas Gleixner 已提交
1655 1656 1657 1658
		return;
	}

retry:
1659 1660
	if (!task_function_call(task, __perf_install_in_context, event))
		return;
T
Thomas Gleixner 已提交
1661

1662
	raw_spin_lock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1663
	/*
1664 1665
	 * If we failed to find a running task, but find the context active now
	 * that we've acquired the ctx->lock, retry.
T
Thomas Gleixner 已提交
1666
	 */
1667
	if (ctx->is_active) {
1668
		raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1669 1670 1671 1672
		goto retry;
	}

	/*
1673 1674
	 * Since the task isn't running, its safe to add the event, us holding
	 * the ctx->lock ensures the task won't get scheduled in.
T
Thomas Gleixner 已提交
1675
	 */
1676
	add_event_to_ctx(event, ctx);
1677
	raw_spin_unlock_irq(&ctx->lock);
T
Thomas Gleixner 已提交
1678 1679
}

1680
/*
1681
 * Put a event into inactive state and update time fields.
1682 1683 1684 1685 1686 1687
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
1688
static void __perf_event_mark_enabled(struct perf_event *event)
1689
{
1690
	struct perf_event *sub;
1691
	u64 tstamp = perf_event_time(event);
1692

1693
	event->state = PERF_EVENT_STATE_INACTIVE;
1694
	event->tstamp_enabled = tstamp - event->total_time_enabled;
P
Peter Zijlstra 已提交
1695
	list_for_each_entry(sub, &event->sibling_list, group_entry) {
1696 1697
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
P
Peter Zijlstra 已提交
1698
	}
1699 1700
}

1701
/*
1702
 * Cross CPU call to enable a performance event
1703
 */
1704
static int __perf_event_enable(void *info)
1705
{
1706 1707 1708
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
P
Peter Zijlstra 已提交
1709
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1710
	int err;
1711

1712 1713
	if (WARN_ON_ONCE(!ctx->is_active))
		return -EINVAL;
1714

1715
	raw_spin_lock(&ctx->lock);
1716
	update_context_time(ctx);
1717

1718
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1719
		goto unlock;
S
Stephane Eranian 已提交
1720 1721 1722 1723

	/*
	 * set current task's cgroup time reference point
	 */
1724
	perf_cgroup_set_timestamp(current, ctx);
S
Stephane Eranian 已提交
1725

1726
	__perf_event_mark_enabled(event);
1727

S
Stephane Eranian 已提交
1728 1729 1730
	if (!event_filter_match(event)) {
		if (is_cgroup_event(event))
			perf_cgroup_defer_enabled(event);
1731
		goto unlock;
S
Stephane Eranian 已提交
1732
	}
1733

1734
	/*
1735
	 * If the event is in a group and isn't the group leader,
1736
	 * then don't put it on unless the group is on.
1737
	 */
1738
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1739
		goto unlock;
1740

1741
	if (!group_can_go_on(event, cpuctx, 1)) {
1742
		err = -EEXIST;
1743
	} else {
1744
		if (event == leader)
1745
			err = group_sched_in(event, cpuctx, ctx);
1746
		else
1747
			err = event_sched_in(event, cpuctx, ctx);
1748
	}
1749 1750 1751

	if (err) {
		/*
1752
		 * If this event can't go on and it's part of a
1753 1754
		 * group, then the whole group has to come off.
		 */
1755
		if (leader != event)
1756
			group_sched_out(leader, cpuctx, ctx);
1757
		if (leader->attr.pinned) {
1758
			update_group_times(leader);
1759
			leader->state = PERF_EVENT_STATE_ERROR;
1760
		}
1761 1762
	}

P
Peter Zijlstra 已提交
1763
unlock:
1764
	raw_spin_unlock(&ctx->lock);
1765 1766

	return 0;
1767 1768 1769
}

/*
1770
 * Enable a event.
1771
 *
1772 1773
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
1774
 * remains valid.  This condition is satisfied when called through
1775 1776
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
1777
 */
1778
void perf_event_enable(struct perf_event *event)
1779
{
1780
	struct perf_event_context *ctx = event->ctx;
1781 1782 1783 1784
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
1785
		 * Enable the event on the cpu that it's on
1786
		 */
1787
		cpu_function_call(event->cpu, __perf_event_enable, event);
1788 1789 1790
		return;
	}

1791
	raw_spin_lock_irq(&ctx->lock);
1792
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1793 1794 1795
		goto out;

	/*
1796 1797
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
1798 1799 1800 1801
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
1802 1803
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
1804

P
Peter Zijlstra 已提交
1805
retry:
1806
	if (!ctx->is_active) {
1807
		__perf_event_mark_enabled(event);
1808 1809 1810
		goto out;
	}

1811
	raw_spin_unlock_irq(&ctx->lock);
1812 1813 1814

	if (!task_function_call(task, __perf_event_enable, event))
		return;
1815

1816
	raw_spin_lock_irq(&ctx->lock);
1817 1818

	/*
1819
	 * If the context is active and the event is still off,
1820 1821
	 * we need to retry the cross-call.
	 */
1822 1823 1824 1825 1826 1827
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
		/*
		 * task could have been flipped by a concurrent
		 * perf_event_context_sched_out()
		 */
		task = ctx->task;
1828
		goto retry;
1829
	}
1830

P
Peter Zijlstra 已提交
1831
out:
1832
	raw_spin_unlock_irq(&ctx->lock);
1833
}
1834
EXPORT_SYMBOL_GPL(perf_event_enable);
1835

1836
int perf_event_refresh(struct perf_event *event, int refresh)
1837
{
1838
	/*
1839
	 * not supported on inherited events
1840
	 */
1841
	if (event->attr.inherit || !is_sampling_event(event))
1842 1843
		return -EINVAL;

1844 1845
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1846 1847

	return 0;
1848
}
1849
EXPORT_SYMBOL_GPL(perf_event_refresh);
1850

1851 1852 1853
static void ctx_sched_out(struct perf_event_context *ctx,
			  struct perf_cpu_context *cpuctx,
			  enum event_type_t event_type)
1854
{
1855
	struct perf_event *event;
1856
	int is_active = ctx->is_active;
1857

1858
	ctx->is_active &= ~event_type;
1859
	if (likely(!ctx->nr_events))
1860 1861
		return;

1862
	update_context_time(ctx);
S
Stephane Eranian 已提交
1863
	update_cgrp_time_from_cpuctx(cpuctx);
1864
	if (!ctx->nr_active)
1865
		return;
1866

P
Peter Zijlstra 已提交
1867
	perf_pmu_disable(ctx->pmu);
1868
	if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
1869 1870
		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
			group_sched_out(event, cpuctx, ctx);
P
Peter Zijlstra 已提交
1871
	}
1872

1873
	if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
1874
		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1875
			group_sched_out(event, cpuctx, ctx);
P
Peter Zijlstra 已提交
1876
	}
P
Peter Zijlstra 已提交
1877
	perf_pmu_enable(ctx->pmu);
1878 1879
}

1880 1881 1882
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1883 1884 1885 1886
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1887
 * in them directly with an fd; we can only enable/disable all
1888
 * events via prctl, or enable/disable all events in a family
1889 1890
 * via ioctl, which will have the same effect on both contexts.
 */
1891 1892
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1893 1894
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1895
		&& ctx1->parent_gen == ctx2->parent_gen
1896
		&& !ctx1->pin_count && !ctx2->pin_count;
1897 1898
}

1899 1900
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1901 1902 1903
{
	u64 value;

1904
	if (!event->attr.inherit_stat)
1905 1906 1907
		return;

	/*
1908
	 * Update the event value, we cannot use perf_event_read()
1909 1910
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1911
	 * we know the event must be on the current CPU, therefore we
1912 1913
	 * don't need to use it.
	 */
1914 1915
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1916 1917
		event->pmu->read(event);
		/* fall-through */
1918

1919 1920
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1921 1922 1923 1924 1925 1926 1927
		break;

	default:
		break;
	}

	/*
1928
	 * In order to keep per-task stats reliable we need to flip the event
1929 1930
	 * values when we flip the contexts.
	 */
1931 1932 1933
	value = local64_read(&next_event->count);
	value = local64_xchg(&event->count, value);
	local64_set(&next_event->count, value);
1934

1935 1936
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1937

1938
	/*
1939
	 * Since we swizzled the values, update the user visible data too.
1940
	 */
1941 1942
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1943 1944 1945 1946 1947
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1948 1949
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1950
{
1951
	struct perf_event *event, *next_event;
1952 1953 1954 1955

	if (!ctx->nr_stat)
		return;

1956 1957
	update_context_time(ctx);

1958 1959
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1960

1961 1962
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1963

1964 1965
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1966

1967
		__perf_event_sync_stat(event, next_event);
1968

1969 1970
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1971 1972 1973
	}
}

1974 1975
static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
					 struct task_struct *next)
T
Thomas Gleixner 已提交
1976
{
P
Peter Zijlstra 已提交
1977
	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1978 1979
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
P
Peter Zijlstra 已提交
1980
	struct perf_cpu_context *cpuctx;
1981
	int do_switch = 1;
T
Thomas Gleixner 已提交
1982

P
Peter Zijlstra 已提交
1983 1984
	if (likely(!ctx))
		return;
1985

P
Peter Zijlstra 已提交
1986 1987
	cpuctx = __get_cpu_context(ctx);
	if (!cpuctx->task_ctx)
T
Thomas Gleixner 已提交
1988 1989
		return;

1990 1991
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
P
Peter Zijlstra 已提交
1992
	next_ctx = next->perf_event_ctxp[ctxn];
1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
2004 2005
		raw_spin_lock(&ctx->lock);
		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2006
		if (context_equiv(ctx, next_ctx)) {
2007 2008
			/*
			 * XXX do we need a memory barrier of sorts
2009
			 * wrt to rcu_dereference() of perf_event_ctxp
2010
			 */
P
Peter Zijlstra 已提交
2011 2012
			task->perf_event_ctxp[ctxn] = next_ctx;
			next->perf_event_ctxp[ctxn] = ctx;
2013 2014 2015
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
2016

2017
			perf_event_sync_stat(ctx, next_ctx);
2018
		}
2019 2020
		raw_spin_unlock(&next_ctx->lock);
		raw_spin_unlock(&ctx->lock);
2021
	}
2022
	rcu_read_unlock();
2023

2024
	if (do_switch) {
2025
		raw_spin_lock(&ctx->lock);
2026
		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2027
		cpuctx->task_ctx = NULL;
2028
		raw_spin_unlock(&ctx->lock);
2029
	}
T
Thomas Gleixner 已提交
2030 2031
}

P
Peter Zijlstra 已提交
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045
#define for_each_task_context_nr(ctxn)					\
	for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)

/*
 * Called from scheduler to remove the events of the current task,
 * with interrupts disabled.
 *
 * We stop each event and update the event value in event->count.
 *
 * This does not protect us against NMI, but disable()
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
 */
2046 2047
void __perf_event_task_sched_out(struct task_struct *task,
				 struct task_struct *next)
P
Peter Zijlstra 已提交
2048 2049 2050 2051 2052
{
	int ctxn;

	for_each_task_context_nr(ctxn)
		perf_event_context_sched_out(task, ctxn, next);
S
Stephane Eranian 已提交
2053 2054 2055 2056 2057 2058 2059

	/*
	 * if cgroup events exist on this CPU, then we need
	 * to check if we have to switch out PMU state.
	 * cgroup event are system-wide mode only
	 */
	if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2060
		perf_cgroup_sched_out(task, next);
P
Peter Zijlstra 已提交
2061 2062
}

2063
static void task_ctx_sched_out(struct perf_event_context *ctx)
2064
{
P
Peter Zijlstra 已提交
2065
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2066

2067 2068
	if (!cpuctx->task_ctx)
		return;
2069 2070 2071 2072

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

2073
	ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2074 2075 2076
	cpuctx->task_ctx = NULL;
}

2077 2078 2079 2080 2081 2082 2083
/*
 * Called with IRQs disabled
 */
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type)
{
	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2084 2085
}

2086
static void
2087
ctx_pinned_sched_in(struct perf_event_context *ctx,
2088
		    struct perf_cpu_context *cpuctx)
T
Thomas Gleixner 已提交
2089
{
2090
	struct perf_event *event;
T
Thomas Gleixner 已提交
2091

2092 2093
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF)
2094
			continue;
2095
		if (!event_filter_match(event))
2096 2097
			continue;

S
Stephane Eranian 已提交
2098 2099 2100 2101
		/* may need to reset tstamp_enabled */
		if (is_cgroup_event(event))
			perf_cgroup_mark_enabled(event, ctx);

2102
		if (group_can_go_on(event, cpuctx, 1))
2103
			group_sched_in(event, cpuctx, ctx);
2104 2105 2106 2107 2108

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
2109 2110 2111
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
2112
		}
2113
	}
2114 2115 2116 2117
}

static void
ctx_flexible_sched_in(struct perf_event_context *ctx,
2118
		      struct perf_cpu_context *cpuctx)
2119 2120 2121
{
	struct perf_event *event;
	int can_add_hw = 1;
2122

2123 2124 2125
	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		/* Ignore events in OFF or ERROR state */
		if (event->state <= PERF_EVENT_STATE_OFF)
2126
			continue;
2127 2128
		/*
		 * Listen to the 'cpu' scheduling filter constraint
2129
		 * of events:
2130
		 */
2131
		if (!event_filter_match(event))
T
Thomas Gleixner 已提交
2132 2133
			continue;

S
Stephane Eranian 已提交
2134 2135 2136 2137
		/* may need to reset tstamp_enabled */
		if (is_cgroup_event(event))
			perf_cgroup_mark_enabled(event, ctx);

P
Peter Zijlstra 已提交
2138
		if (group_can_go_on(event, cpuctx, can_add_hw)) {
2139
			if (group_sched_in(event, cpuctx, ctx))
2140
				can_add_hw = 0;
P
Peter Zijlstra 已提交
2141
		}
T
Thomas Gleixner 已提交
2142
	}
2143 2144 2145 2146 2147
}

static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
S
Stephane Eranian 已提交
2148 2149
	     enum event_type_t event_type,
	     struct task_struct *task)
2150
{
S
Stephane Eranian 已提交
2151
	u64 now;
2152
	int is_active = ctx->is_active;
S
Stephane Eranian 已提交
2153

2154
	ctx->is_active |= event_type;
2155
	if (likely(!ctx->nr_events))
2156
		return;
2157

S
Stephane Eranian 已提交
2158 2159
	now = perf_clock();
	ctx->timestamp = now;
2160
	perf_cgroup_set_timestamp(task, ctx);
2161 2162 2163 2164
	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
2165
	if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2166
		ctx_pinned_sched_in(ctx, cpuctx);
2167 2168

	/* Then walk through the lower prio flexible groups */
2169
	if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2170
		ctx_flexible_sched_in(ctx, cpuctx);
2171 2172
}

2173
static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
S
Stephane Eranian 已提交
2174 2175
			     enum event_type_t event_type,
			     struct task_struct *task)
2176 2177 2178
{
	struct perf_event_context *ctx = &cpuctx->ctx;

S
Stephane Eranian 已提交
2179
	ctx_sched_in(ctx, cpuctx, event_type, task);
2180 2181
}

S
Stephane Eranian 已提交
2182 2183
static void perf_event_context_sched_in(struct perf_event_context *ctx,
					struct task_struct *task)
2184
{
P
Peter Zijlstra 已提交
2185
	struct perf_cpu_context *cpuctx;
2186

P
Peter Zijlstra 已提交
2187
	cpuctx = __get_cpu_context(ctx);
2188 2189 2190
	if (cpuctx->task_ctx == ctx)
		return;

2191
	perf_ctx_lock(cpuctx, ctx);
P
Peter Zijlstra 已提交
2192
	perf_pmu_disable(ctx->pmu);
2193 2194 2195 2196 2197 2198 2199
	/*
	 * We want to keep the following priority order:
	 * cpu pinned (that don't need to move), task pinned,
	 * cpu flexible, task flexible.
	 */
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);

2200 2201
	if (ctx->nr_events)
		cpuctx->task_ctx = ctx;
2202

2203 2204
	perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);

2205 2206 2207
	perf_pmu_enable(ctx->pmu);
	perf_ctx_unlock(cpuctx, ctx);

2208 2209 2210 2211
	/*
	 * Since these rotations are per-cpu, we need to ensure the
	 * cpu-context we got scheduled on is actually rotating.
	 */
P
Peter Zijlstra 已提交
2212
	perf_pmu_rotate_start(ctx->pmu);
2213 2214
}

2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274
/*
 * When sampling the branck stack in system-wide, it may be necessary
 * to flush the stack on context switch. This happens when the branch
 * stack does not tag its entries with the pid of the current task.
 * Otherwise it becomes impossible to associate a branch entry with a
 * task. This ambiguity is more likely to appear when the branch stack
 * supports priv level filtering and the user sets it to monitor only
 * at the user level (which could be a useful measurement in system-wide
 * mode). In that case, the risk is high of having a branch stack with
 * branch from multiple tasks. Flushing may mean dropping the existing
 * entries or stashing them somewhere in the PMU specific code layer.
 *
 * This function provides the context switch callback to the lower code
 * layer. It is invoked ONLY when there is at least one system-wide context
 * with at least one active event using taken branch sampling.
 */
static void perf_branch_stack_sched_in(struct task_struct *prev,
				       struct task_struct *task)
{
	struct perf_cpu_context *cpuctx;
	struct pmu *pmu;
	unsigned long flags;

	/* no need to flush branch stack if not changing task */
	if (prev == task)
		return;

	local_irq_save(flags);

	rcu_read_lock();

	list_for_each_entry_rcu(pmu, &pmus, entry) {
		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

		/*
		 * check if the context has at least one
		 * event using PERF_SAMPLE_BRANCH_STACK
		 */
		if (cpuctx->ctx.nr_branch_stack > 0
		    && pmu->flush_branch_stack) {

			pmu = cpuctx->ctx.pmu;

			perf_ctx_lock(cpuctx, cpuctx->task_ctx);

			perf_pmu_disable(pmu);

			pmu->flush_branch_stack();

			perf_pmu_enable(pmu);

			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
		}
	}

	rcu_read_unlock();

	local_irq_restore(flags);
}

P
Peter Zijlstra 已提交
2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285
/*
 * Called from scheduler to add the events of the current task
 * with interrupts disabled.
 *
 * We restore the event value and then enable it.
 *
 * This does not protect us against NMI, but enable()
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
 */
2286 2287
void __perf_event_task_sched_in(struct task_struct *prev,
				struct task_struct *task)
P
Peter Zijlstra 已提交
2288 2289 2290 2291 2292 2293 2294 2295 2296
{
	struct perf_event_context *ctx;
	int ctxn;

	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (likely(!ctx))
			continue;

S
Stephane Eranian 已提交
2297
		perf_event_context_sched_in(ctx, task);
P
Peter Zijlstra 已提交
2298
	}
S
Stephane Eranian 已提交
2299 2300 2301 2302 2303 2304
	/*
	 * if cgroup events exist on this CPU, then we need
	 * to check if we have to switch in PMU state.
	 * cgroup event are system-wide mode only
	 */
	if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2305
		perf_cgroup_sched_in(prev, task);
2306 2307 2308 2309

	/* check for system-wide branch_stack events */
	if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
		perf_branch_stack_sched_in(prev, task);
2310 2311
}

2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
{
	u64 frequency = event->attr.sample_freq;
	u64 sec = NSEC_PER_SEC;
	u64 divisor, dividend;

	int count_fls, nsec_fls, frequency_fls, sec_fls;

	count_fls = fls64(count);
	nsec_fls = fls64(nsec);
	frequency_fls = fls64(frequency);
	sec_fls = 30;

	/*
	 * We got @count in @nsec, with a target of sample_freq HZ
	 * the target period becomes:
	 *
	 *             @count * 10^9
	 * period = -------------------
	 *          @nsec * sample_freq
	 *
	 */

	/*
	 * Reduce accuracy by one bit such that @a and @b converge
	 * to a similar magnitude.
	 */
2339
#define REDUCE_FLS(a, b)		\
2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378
do {					\
	if (a##_fls > b##_fls) {	\
		a >>= 1;		\
		a##_fls--;		\
	} else {			\
		b >>= 1;		\
		b##_fls--;		\
	}				\
} while (0)

	/*
	 * Reduce accuracy until either term fits in a u64, then proceed with
	 * the other, so that finally we can do a u64/u64 division.
	 */
	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
		REDUCE_FLS(nsec, frequency);
		REDUCE_FLS(sec, count);
	}

	if (count_fls + sec_fls > 64) {
		divisor = nsec * frequency;

		while (count_fls + sec_fls > 64) {
			REDUCE_FLS(count, sec);
			divisor >>= 1;
		}

		dividend = count * sec;
	} else {
		dividend = count * sec;

		while (nsec_fls + frequency_fls > 64) {
			REDUCE_FLS(nsec, frequency);
			dividend >>= 1;
		}

		divisor = nsec * frequency;
	}

2379 2380 2381
	if (!divisor)
		return dividend;

2382 2383 2384
	return div64_u64(dividend, divisor);
}

2385 2386 2387
static DEFINE_PER_CPU(int, perf_throttled_count);
static DEFINE_PER_CPU(u64, perf_throttled_seq);

2388
static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2389
{
2390
	struct hw_perf_event *hwc = &event->hw;
2391
	s64 period, sample_period;
2392 2393
	s64 delta;

2394
	period = perf_calculate_period(event, nsec, count);
2395 2396 2397 2398 2399 2400 2401 2402 2403 2404

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
2405

2406
	if (local64_read(&hwc->period_left) > 8*sample_period) {
2407 2408 2409
		if (disable)
			event->pmu->stop(event, PERF_EF_UPDATE);

2410
		local64_set(&hwc->period_left, 0);
2411 2412 2413

		if (disable)
			event->pmu->start(event, PERF_EF_RELOAD);
2414
	}
2415 2416
}

2417 2418 2419 2420 2421 2422 2423
/*
 * combine freq adjustment with unthrottling to avoid two passes over the
 * events. At the same time, make sure, having freq events does not change
 * the rate of unthrottling as that would introduce bias.
 */
static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
					   int needs_unthr)
2424
{
2425 2426
	struct perf_event *event;
	struct hw_perf_event *hwc;
2427
	u64 now, period = TICK_NSEC;
2428
	s64 delta;
2429

2430 2431 2432 2433 2434 2435
	/*
	 * only need to iterate over all events iff:
	 * - context have events in frequency mode (needs freq adjust)
	 * - there are events to unthrottle on this cpu
	 */
	if (!(ctx->nr_freq || needs_unthr))
2436 2437
		return;

2438
	raw_spin_lock(&ctx->lock);
2439
	perf_pmu_disable(ctx->pmu);
2440

2441
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2442
		if (event->state != PERF_EVENT_STATE_ACTIVE)
2443 2444
			continue;

2445
		if (!event_filter_match(event))
2446 2447
			continue;

2448
		hwc = &event->hw;
2449

2450 2451
		if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
			hwc->interrupts = 0;
2452
			perf_log_throttle(event, 1);
P
Peter Zijlstra 已提交
2453
			event->pmu->start(event, 0);
2454 2455
		}

2456
		if (!event->attr.freq || !event->attr.sample_freq)
2457 2458
			continue;

2459 2460 2461 2462 2463
		/*
		 * stop the event and update event->count
		 */
		event->pmu->stop(event, PERF_EF_UPDATE);

2464
		now = local64_read(&event->count);
2465 2466
		delta = now - hwc->freq_count_stamp;
		hwc->freq_count_stamp = now;
2467

2468 2469 2470
		/*
		 * restart the event
		 * reload only if value has changed
2471 2472 2473
		 * we have stopped the event so tell that
		 * to perf_adjust_period() to avoid stopping it
		 * twice.
2474
		 */
2475
		if (delta > 0)
2476
			perf_adjust_period(event, period, delta, false);
2477 2478

		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2479
	}
2480

2481
	perf_pmu_enable(ctx->pmu);
2482
	raw_spin_unlock(&ctx->lock);
2483 2484
}

2485
/*
2486
 * Round-robin a context's events:
2487
 */
2488
static void rotate_ctx(struct perf_event_context *ctx)
T
Thomas Gleixner 已提交
2489
{
2490 2491 2492 2493 2494 2495
	/*
	 * Rotate the first entry last of non-pinned groups. Rotation might be
	 * disabled by the inheritance code.
	 */
	if (!ctx->rotate_disable)
		list_rotate_left(&ctx->flexible_groups);
2496 2497
}

2498
/*
2499 2500 2501
 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
 * because they're strictly cpu affine and rotate_start is called with IRQs
 * disabled, while rotate_context is called from IRQ context.
2502
 */
2503
static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2504
{
P
Peter Zijlstra 已提交
2505
	struct perf_event_context *ctx = NULL;
2506
	int rotate = 0, remove = 1;
2507

2508
	if (cpuctx->ctx.nr_events) {
2509
		remove = 0;
2510 2511 2512
		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
			rotate = 1;
	}
2513

P
Peter Zijlstra 已提交
2514
	ctx = cpuctx->task_ctx;
2515
	if (ctx && ctx->nr_events) {
2516
		remove = 0;
2517 2518 2519
		if (ctx->nr_events != ctx->nr_active)
			rotate = 1;
	}
2520

2521
	if (!rotate)
2522 2523
		goto done;

2524
	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
P
Peter Zijlstra 已提交
2525
	perf_pmu_disable(cpuctx->ctx.pmu);
2526

2527 2528 2529
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
	if (ctx)
		ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
T
Thomas Gleixner 已提交
2530

2531 2532 2533
	rotate_ctx(&cpuctx->ctx);
	if (ctx)
		rotate_ctx(ctx);
2534

2535
	perf_event_sched_in(cpuctx, ctx, current);
2536

2537 2538
	perf_pmu_enable(cpuctx->ctx.pmu);
	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2539
done:
2540 2541 2542 2543 2544 2545 2546 2547
	if (remove)
		list_del_init(&cpuctx->rotation_list);
}

void perf_event_task_tick(void)
{
	struct list_head *head = &__get_cpu_var(rotation_list);
	struct perf_cpu_context *cpuctx, *tmp;
2548 2549
	struct perf_event_context *ctx;
	int throttled;
2550

2551 2552
	WARN_ON(!irqs_disabled());

2553 2554 2555
	__this_cpu_inc(perf_throttled_seq);
	throttled = __this_cpu_xchg(perf_throttled_count, 0);

2556
	list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2557 2558 2559 2560 2561 2562 2563
		ctx = &cpuctx->ctx;
		perf_adjust_freq_unthr_context(ctx, throttled);

		ctx = cpuctx->task_ctx;
		if (ctx)
			perf_adjust_freq_unthr_context(ctx, throttled);

2564 2565 2566 2567
		if (cpuctx->jiffies_interval == 1 ||
				!(jiffies % cpuctx->jiffies_interval))
			perf_rotate_context(cpuctx);
	}
T
Thomas Gleixner 已提交
2568 2569
}

2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

2580
	__perf_event_mark_enabled(event);
2581 2582 2583 2584

	return 1;
}

2585
/*
2586
 * Enable all of a task's events that have been marked enable-on-exec.
2587 2588
 * This expects task == current.
 */
P
Peter Zijlstra 已提交
2589
static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2590
{
2591
	struct perf_event *event;
2592 2593
	unsigned long flags;
	int enabled = 0;
2594
	int ret;
2595 2596

	local_irq_save(flags);
2597
	if (!ctx || !ctx->nr_events)
2598 2599
		goto out;

2600 2601 2602 2603 2604 2605 2606
	/*
	 * We must ctxsw out cgroup events to avoid conflict
	 * when invoking perf_task_event_sched_in() later on
	 * in this function. Otherwise we end up trying to
	 * ctxswin cgroup events which are already scheduled
	 * in.
	 */
2607
	perf_cgroup_sched_out(current, NULL);
2608

2609
	raw_spin_lock(&ctx->lock);
2610
	task_ctx_sched_out(ctx);
2611

2612
	list_for_each_entry(event, &ctx->event_list, event_entry) {
2613 2614 2615
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
2616 2617 2618
	}

	/*
2619
	 * Unclone this context if we enabled any event.
2620
	 */
2621 2622
	if (enabled)
		unclone_ctx(ctx);
2623

2624
	raw_spin_unlock(&ctx->lock);
2625

2626 2627 2628
	/*
	 * Also calls ctxswin for cgroup events, if any:
	 */
S
Stephane Eranian 已提交
2629
	perf_event_context_sched_in(ctx, ctx->task);
P
Peter Zijlstra 已提交
2630
out:
2631 2632 2633
	local_irq_restore(flags);
}

T
Thomas Gleixner 已提交
2634
/*
2635
 * Cross CPU call to read the hardware event
T
Thomas Gleixner 已提交
2636
 */
2637
static void __perf_event_read(void *info)
T
Thomas Gleixner 已提交
2638
{
2639 2640
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
P
Peter Zijlstra 已提交
2641
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
I
Ingo Molnar 已提交
2642

2643 2644 2645 2646
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
2647 2648
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
2649 2650 2651 2652
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

2653
	raw_spin_lock(&ctx->lock);
S
Stephane Eranian 已提交
2654
	if (ctx->is_active) {
2655
		update_context_time(ctx);
S
Stephane Eranian 已提交
2656 2657
		update_cgrp_time_from_event(event);
	}
2658
	update_event_times(event);
2659 2660
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		event->pmu->read(event);
2661
	raw_spin_unlock(&ctx->lock);
T
Thomas Gleixner 已提交
2662 2663
}

P
Peter Zijlstra 已提交
2664 2665
static inline u64 perf_event_count(struct perf_event *event)
{
2666
	return local64_read(&event->count) + atomic64_read(&event->child_count);
P
Peter Zijlstra 已提交
2667 2668
}

2669
static u64 perf_event_read(struct perf_event *event)
T
Thomas Gleixner 已提交
2670 2671
{
	/*
2672 2673
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
T
Thomas Gleixner 已提交
2674
	 */
2675 2676 2677 2678
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
P
Peter Zijlstra 已提交
2679 2680 2681
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

2682
		raw_spin_lock_irqsave(&ctx->lock, flags);
2683 2684 2685 2686 2687
		/*
		 * may read while context is not active
		 * (e.g., thread is blocked), in that case
		 * we cannot update context time
		 */
S
Stephane Eranian 已提交
2688
		if (ctx->is_active) {
2689
			update_context_time(ctx);
S
Stephane Eranian 已提交
2690 2691
			update_cgrp_time_from_event(event);
		}
2692
		update_event_times(event);
2693
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
T
Thomas Gleixner 已提交
2694 2695
	}

P
Peter Zijlstra 已提交
2696
	return perf_event_count(event);
T
Thomas Gleixner 已提交
2697 2698
}

2699
/*
2700
 * Initialize the perf_event context in a task_struct:
2701
 */
2702
static void __perf_event_init_context(struct perf_event_context *ctx)
2703
{
2704
	raw_spin_lock_init(&ctx->lock);
2705
	mutex_init(&ctx->mutex);
2706 2707
	INIT_LIST_HEAD(&ctx->pinned_groups);
	INIT_LIST_HEAD(&ctx->flexible_groups);
2708 2709
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
}

static struct perf_event_context *
alloc_perf_context(struct pmu *pmu, struct task_struct *task)
{
	struct perf_event_context *ctx;

	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
	if (!ctx)
		return NULL;

	__perf_event_init_context(ctx);
	if (task) {
		ctx->task = task;
		get_task_struct(task);
T
Thomas Gleixner 已提交
2725
	}
2726 2727 2728
	ctx->pmu = pmu;

	return ctx;
2729 2730
}

2731 2732 2733 2734 2735
static struct task_struct *
find_lively_task_by_vpid(pid_t vpid)
{
	struct task_struct *task;
	int err;
T
Thomas Gleixner 已提交
2736 2737

	rcu_read_lock();
2738
	if (!vpid)
T
Thomas Gleixner 已提交
2739 2740
		task = current;
	else
2741
		task = find_task_by_vpid(vpid);
T
Thomas Gleixner 已提交
2742 2743 2744 2745 2746 2747 2748 2749
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

	/* Reuse ptrace permission checks for now. */
2750 2751 2752 2753
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

2754 2755 2756 2757 2758 2759 2760
	return task;
errout:
	put_task_struct(task);
	return ERR_PTR(err);

}

2761 2762 2763
/*
 * Returns a matching context with refcount and pincount.
 */
P
Peter Zijlstra 已提交
2764
static struct perf_event_context *
M
Matt Helsley 已提交
2765
find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
T
Thomas Gleixner 已提交
2766
{
2767
	struct perf_event_context *ctx;
2768
	struct perf_cpu_context *cpuctx;
2769
	unsigned long flags;
P
Peter Zijlstra 已提交
2770
	int ctxn, err;
T
Thomas Gleixner 已提交
2771

2772
	if (!task) {
2773
		/* Must be root to operate on a CPU event: */
2774
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
T
Thomas Gleixner 已提交
2775 2776 2777
			return ERR_PTR(-EACCES);

		/*
2778
		 * We could be clever and allow to attach a event to an
T
Thomas Gleixner 已提交
2779 2780 2781
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
2782
		if (!cpu_online(cpu))
T
Thomas Gleixner 已提交
2783 2784
			return ERR_PTR(-ENODEV);

P
Peter Zijlstra 已提交
2785
		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
T
Thomas Gleixner 已提交
2786
		ctx = &cpuctx->ctx;
2787
		get_ctx(ctx);
2788
		++ctx->pin_count;
T
Thomas Gleixner 已提交
2789 2790 2791 2792

		return ctx;
	}

P
Peter Zijlstra 已提交
2793 2794 2795 2796 2797
	err = -EINVAL;
	ctxn = pmu->task_ctx_nr;
	if (ctxn < 0)
		goto errout;

P
Peter Zijlstra 已提交
2798
retry:
P
Peter Zijlstra 已提交
2799
	ctx = perf_lock_task_context(task, ctxn, &flags);
2800
	if (ctx) {
2801
		unclone_ctx(ctx);
2802
		++ctx->pin_count;
2803
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
2804
	} else {
2805
		ctx = alloc_perf_context(pmu, task);
2806 2807 2808
		err = -ENOMEM;
		if (!ctx)
			goto errout;
2809

2810 2811 2812 2813 2814 2815 2816 2817 2818 2819
		err = 0;
		mutex_lock(&task->perf_event_mutex);
		/*
		 * If it has already passed perf_event_exit_task().
		 * we must see PF_EXITING, it takes this mutex too.
		 */
		if (task->flags & PF_EXITING)
			err = -ESRCH;
		else if (task->perf_event_ctxp[ctxn])
			err = -EAGAIN;
2820
		else {
2821
			get_ctx(ctx);
2822
			++ctx->pin_count;
2823
			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2824
		}
2825 2826 2827
		mutex_unlock(&task->perf_event_mutex);

		if (unlikely(err)) {
2828
			put_ctx(ctx);
2829 2830 2831 2832

			if (err == -EAGAIN)
				goto retry;
			goto errout;
2833 2834 2835
		}
	}

T
Thomas Gleixner 已提交
2836
	return ctx;
2837

P
Peter Zijlstra 已提交
2838
errout:
2839
	return ERR_PTR(err);
T
Thomas Gleixner 已提交
2840 2841
}

L
Li Zefan 已提交
2842 2843
static void perf_event_free_filter(struct perf_event *event);

2844
static void free_event_rcu(struct rcu_head *head)
P
Peter Zijlstra 已提交
2845
{
2846
	struct perf_event *event;
P
Peter Zijlstra 已提交
2847

2848 2849 2850
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
L
Li Zefan 已提交
2851
	perf_event_free_filter(event);
2852
	kfree(event);
P
Peter Zijlstra 已提交
2853 2854
}

2855
static void ring_buffer_put(struct ring_buffer *rb);
2856

2857
static void free_event(struct perf_event *event)
2858
{
2859
	irq_work_sync(&event->pending);
2860

2861
	if (!event->parent) {
2862
		if (event->attach_state & PERF_ATTACH_TASK)
2863
			static_key_slow_dec_deferred(&perf_sched_events);
2864
		if (event->attr.mmap || event->attr.mmap_data)
2865 2866 2867 2868 2869
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
2870 2871
		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
			put_callchain_buffers();
2872 2873
		if (is_cgroup_event(event)) {
			atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
2874
			static_key_slow_dec_deferred(&perf_sched_events);
2875
		}
2876 2877 2878 2879 2880 2881 2882 2883

		if (has_branch_stack(event)) {
			static_key_slow_dec_deferred(&perf_sched_events);
			/* is system-wide event */
			if (!(event->attach_state & PERF_ATTACH_TASK))
				atomic_dec(&per_cpu(perf_branch_stack_events,
						    event->cpu));
		}
2884
	}
2885

2886 2887 2888
	if (event->rb) {
		ring_buffer_put(event->rb);
		event->rb = NULL;
2889 2890
	}

S
Stephane Eranian 已提交
2891 2892 2893
	if (is_cgroup_event(event))
		perf_detach_cgroup(event);

2894 2895
	if (event->destroy)
		event->destroy(event);
2896

P
Peter Zijlstra 已提交
2897 2898 2899
	if (event->ctx)
		put_ctx(event->ctx);

2900
	call_rcu(&event->rcu_head, free_event_rcu);
2901 2902
}

2903
int perf_event_release_kernel(struct perf_event *event)
T
Thomas Gleixner 已提交
2904
{
2905
	struct perf_event_context *ctx = event->ctx;
T
Thomas Gleixner 已提交
2906

2907
	WARN_ON_ONCE(ctx->parent_ctx);
2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920
	/*
	 * There are two ways this annotation is useful:
	 *
	 *  1) there is a lock recursion from perf_event_exit_task
	 *     see the comment there.
	 *
	 *  2) there is a lock-inversion with mmap_sem through
	 *     perf_event_read_group(), which takes faults while
	 *     holding ctx->mutex, however this is called after
	 *     the last filedesc died, so there is no possibility
	 *     to trigger the AB-BA case.
	 */
	mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2921
	raw_spin_lock_irq(&ctx->lock);
2922
	perf_group_detach(event);
2923
	raw_spin_unlock_irq(&ctx->lock);
2924
	perf_remove_from_context(event);
2925
	mutex_unlock(&ctx->mutex);
T
Thomas Gleixner 已提交
2926

2927
	free_event(event);
T
Thomas Gleixner 已提交
2928 2929 2930

	return 0;
}
2931
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
T
Thomas Gleixner 已提交
2932

2933 2934 2935 2936
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
2937
{
2938
	struct perf_event *event = file->private_data;
P
Peter Zijlstra 已提交
2939
	struct task_struct *owner;
2940

2941
	file->private_data = NULL;
2942

P
Peter Zijlstra 已提交
2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975
	rcu_read_lock();
	owner = ACCESS_ONCE(event->owner);
	/*
	 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
	 * !owner it means the list deletion is complete and we can indeed
	 * free this event, otherwise we need to serialize on
	 * owner->perf_event_mutex.
	 */
	smp_read_barrier_depends();
	if (owner) {
		/*
		 * Since delayed_put_task_struct() also drops the last
		 * task reference we can safely take a new reference
		 * while holding the rcu_read_lock().
		 */
		get_task_struct(owner);
	}
	rcu_read_unlock();

	if (owner) {
		mutex_lock(&owner->perf_event_mutex);
		/*
		 * We have to re-check the event->owner field, if it is cleared
		 * we raced with perf_event_exit_task(), acquiring the mutex
		 * ensured they're done, and we can proceed with freeing the
		 * event.
		 */
		if (event->owner)
			list_del_init(&event->owner_entry);
		mutex_unlock(&owner->perf_event_mutex);
		put_task_struct(owner);
	}

2976
	return perf_event_release_kernel(event);
2977 2978
}

2979
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2980
{
2981
	struct perf_event *child;
2982 2983
	u64 total = 0;

2984 2985 2986
	*enabled = 0;
	*running = 0;

2987
	mutex_lock(&event->child_mutex);
2988
	total += perf_event_read(event);
2989 2990 2991 2992 2993 2994
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
2995
		total += perf_event_read(child);
2996 2997 2998
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
2999
	mutex_unlock(&event->child_mutex);
3000 3001 3002

	return total;
}
3003
EXPORT_SYMBOL_GPL(perf_event_read_value);
3004

3005
static int perf_event_read_group(struct perf_event *event,
3006 3007
				   u64 read_format, char __user *buf)
{
3008
	struct perf_event *leader = event->group_leader, *sub;
3009 3010
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
3011
	u64 values[5];
3012
	u64 count, enabled, running;
3013

3014
	mutex_lock(&ctx->mutex);
3015
	count = perf_event_read_value(leader, &enabled, &running);
3016 3017

	values[n++] = 1 + leader->nr_siblings;
3018 3019 3020 3021
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
3022 3023 3024
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
3025 3026 3027 3028

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
3029
		goto unlock;
3030

3031
	ret = size;
3032

3033
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3034
		n = 0;
3035

3036
		values[n++] = perf_event_read_value(sub, &enabled, &running);
3037 3038 3039 3040 3041
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

3042
		if (copy_to_user(buf + ret, values, size)) {
3043 3044 3045
			ret = -EFAULT;
			goto unlock;
		}
3046 3047

		ret += size;
3048
	}
3049 3050
unlock:
	mutex_unlock(&ctx->mutex);
3051

3052
	return ret;
3053 3054
}

3055
static int perf_event_read_one(struct perf_event *event,
3056 3057
				 u64 read_format, char __user *buf)
{
3058
	u64 enabled, running;
3059 3060 3061
	u64 values[4];
	int n = 0;

3062 3063 3064 3065 3066
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
3067
	if (read_format & PERF_FORMAT_ID)
3068
		values[n++] = primary_event_id(event);
3069 3070 3071 3072 3073 3074 3075

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

T
Thomas Gleixner 已提交
3076
/*
3077
 * Read the performance event - simple non blocking version for now
T
Thomas Gleixner 已提交
3078 3079
 */
static ssize_t
3080
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
T
Thomas Gleixner 已提交
3081
{
3082
	u64 read_format = event->attr.read_format;
3083
	int ret;
T
Thomas Gleixner 已提交
3084

3085
	/*
3086
	 * Return end-of-file for a read on a event that is in
3087 3088 3089
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
3090
	if (event->state == PERF_EVENT_STATE_ERROR)
3091 3092
		return 0;

3093
	if (count < event->read_size)
3094 3095
		return -ENOSPC;

3096
	WARN_ON_ONCE(event->ctx->parent_ctx);
3097
	if (read_format & PERF_FORMAT_GROUP)
3098
		ret = perf_event_read_group(event, read_format, buf);
3099
	else
3100
		ret = perf_event_read_one(event, read_format, buf);
T
Thomas Gleixner 已提交
3101

3102
	return ret;
T
Thomas Gleixner 已提交
3103 3104 3105 3106 3107
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
3108
	struct perf_event *event = file->private_data;
T
Thomas Gleixner 已提交
3109

3110
	return perf_read_hw(event, buf, count);
T
Thomas Gleixner 已提交
3111 3112 3113 3114
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
3115
	struct perf_event *event = file->private_data;
3116
	struct ring_buffer *rb;
3117
	unsigned int events = POLL_HUP;
P
Peter Zijlstra 已提交
3118

3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135
	/*
	 * Race between perf_event_set_output() and perf_poll(): perf_poll()
	 * grabs the rb reference but perf_event_set_output() overrides it.
	 * Here is the timeline for two threads T1, T2:
	 * t0: T1, rb = rcu_dereference(event->rb)
	 * t1: T2, old_rb = event->rb
	 * t2: T2, event->rb = new rb
	 * t3: T2, ring_buffer_detach(old_rb)
	 * t4: T1, ring_buffer_attach(rb1)
	 * t5: T1, poll_wait(event->waitq)
	 *
	 * To avoid this problem, we grab mmap_mutex in perf_poll()
	 * thereby ensuring that the assignment of the new ring buffer
	 * and the detachment of the old buffer appear atomic to perf_poll()
	 */
	mutex_lock(&event->mmap_mutex);

P
Peter Zijlstra 已提交
3136
	rcu_read_lock();
3137
	rb = rcu_dereference(event->rb);
3138 3139
	if (rb) {
		ring_buffer_attach(event, rb);
3140
		events = atomic_xchg(&rb->poll, 0);
3141
	}
P
Peter Zijlstra 已提交
3142
	rcu_read_unlock();
T
Thomas Gleixner 已提交
3143

3144 3145
	mutex_unlock(&event->mmap_mutex);

3146
	poll_wait(file, &event->waitq, wait);
T
Thomas Gleixner 已提交
3147 3148 3149 3150

	return events;
}

3151
static void perf_event_reset(struct perf_event *event)
3152
{
3153
	(void)perf_event_read(event);
3154
	local64_set(&event->count, 0);
3155
	perf_event_update_userpage(event);
P
Peter Zijlstra 已提交
3156 3157
}

3158
/*
3159 3160 3161 3162
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
3163
 */
3164 3165
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
3166
{
3167
	struct perf_event *child;
P
Peter Zijlstra 已提交
3168

3169 3170 3171 3172
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
P
Peter Zijlstra 已提交
3173
		func(child);
3174
	mutex_unlock(&event->child_mutex);
P
Peter Zijlstra 已提交
3175 3176
}

3177 3178
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
P
Peter Zijlstra 已提交
3179
{
3180 3181
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
P
Peter Zijlstra 已提交
3182

3183 3184
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
3185
	event = event->group_leader;
3186

3187 3188
	perf_event_for_each_child(event, func);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
3189
		perf_event_for_each_child(sibling, func);
3190
	mutex_unlock(&ctx->mutex);
3191 3192
}

3193
static int perf_event_period(struct perf_event *event, u64 __user *arg)
3194
{
3195
	struct perf_event_context *ctx = event->ctx;
3196 3197 3198
	int ret = 0;
	u64 value;

3199
	if (!is_sampling_event(event))
3200 3201
		return -EINVAL;

3202
	if (copy_from_user(&value, arg, sizeof(value)))
3203 3204 3205 3206 3207
		return -EFAULT;

	if (!value)
		return -EINVAL;

3208
	raw_spin_lock_irq(&ctx->lock);
3209 3210
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
3211 3212 3213 3214
			ret = -EINVAL;
			goto unlock;
		}

3215
		event->attr.sample_freq = value;
3216
	} else {
3217 3218
		event->attr.sample_period = value;
		event->hw.sample_period = value;
3219 3220
	}
unlock:
3221
	raw_spin_unlock_irq(&ctx->lock);
3222 3223 3224 3225

	return ret;
}

3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246
static const struct file_operations perf_fops;

static struct perf_event *perf_fget_light(int fd, int *fput_needed)
{
	struct file *file;

	file = fget_light(fd, fput_needed);
	if (!file)
		return ERR_PTR(-EBADF);

	if (file->f_op != &perf_fops) {
		fput_light(file, *fput_needed);
		*fput_needed = 0;
		return ERR_PTR(-EBADF);
	}

	return file->private_data;
}

static int perf_event_set_output(struct perf_event *event,
				 struct perf_event *output_event);
L
Li Zefan 已提交
3247
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3248

3249 3250
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
3251 3252
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
P
Peter Zijlstra 已提交
3253
	u32 flags = arg;
3254 3255

	switch (cmd) {
3256 3257
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
3258
		break;
3259 3260
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
3261
		break;
3262 3263
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
3264
		break;
P
Peter Zijlstra 已提交
3265

3266 3267
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
3268

3269 3270
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
3271

3272
	case PERF_EVENT_IOC_SET_OUTPUT:
3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289
	{
		struct perf_event *output_event = NULL;
		int fput_needed = 0;
		int ret;

		if (arg != -1) {
			output_event = perf_fget_light(arg, &fput_needed);
			if (IS_ERR(output_event))
				return PTR_ERR(output_event);
		}

		ret = perf_event_set_output(event, output_event);
		if (output_event)
			fput_light(output_event->filp, fput_needed);

		return ret;
	}
3290

L
Li Zefan 已提交
3291 3292 3293
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

3294
	default:
P
Peter Zijlstra 已提交
3295
		return -ENOTTY;
3296
	}
P
Peter Zijlstra 已提交
3297 3298

	if (flags & PERF_IOC_FLAG_GROUP)
3299
		perf_event_for_each(event, func);
P
Peter Zijlstra 已提交
3300
	else
3301
		perf_event_for_each_child(event, func);
P
Peter Zijlstra 已提交
3302 3303

	return 0;
3304 3305
}

3306
int perf_event_task_enable(void)
3307
{
3308
	struct perf_event *event;
3309

3310 3311 3312 3313
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
3314 3315 3316 3317

	return 0;
}

3318
int perf_event_task_disable(void)
3319
{
3320
	struct perf_event *event;
3321

3322 3323 3324 3325
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
3326 3327 3328 3329

	return 0;
}

3330
static int perf_event_index(struct perf_event *event)
3331
{
P
Peter Zijlstra 已提交
3332 3333 3334
	if (event->hw.state & PERF_HES_STOPPED)
		return 0;

3335
	if (event->state != PERF_EVENT_STATE_ACTIVE)
3336 3337
		return 0;

3338
	return event->pmu->event_idx(event);
3339 3340
}

3341
static void calc_timer_values(struct perf_event *event,
3342
				u64 *now,
3343 3344
				u64 *enabled,
				u64 *running)
3345
{
3346
	u64 ctx_time;
3347

3348 3349
	*now = perf_clock();
	ctx_time = event->shadow_ctx_time + *now;
3350 3351 3352 3353
	*enabled = ctx_time - event->tstamp_enabled;
	*running = ctx_time - event->tstamp_running;
}

3354
void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3355 3356 3357
{
}

3358 3359 3360 3361 3362
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
3363
void perf_event_update_userpage(struct perf_event *event)
3364
{
3365
	struct perf_event_mmap_page *userpg;
3366
	struct ring_buffer *rb;
3367
	u64 enabled, running, now;
3368 3369

	rcu_read_lock();
3370 3371 3372 3373 3374 3375 3376 3377 3378
	/*
	 * compute total_time_enabled, total_time_running
	 * based on snapshot values taken when the event
	 * was last scheduled in.
	 *
	 * we cannot simply called update_context_time()
	 * because of locking issue as we can be called in
	 * NMI context
	 */
3379
	calc_timer_values(event, &now, &enabled, &running);
3380 3381
	rb = rcu_dereference(event->rb);
	if (!rb)
3382 3383
		goto unlock;

3384
	userpg = rb->user_page;
3385

3386 3387 3388 3389 3390
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
3391
	++userpg->lock;
3392
	barrier();
3393
	userpg->index = perf_event_index(event);
P
Peter Zijlstra 已提交
3394
	userpg->offset = perf_event_count(event);
3395
	if (userpg->index)
3396
		userpg->offset -= local64_read(&event->hw.prev_count);
3397

3398
	userpg->time_enabled = enabled +
3399
			atomic64_read(&event->child_total_time_enabled);
3400

3401
	userpg->time_running = running +
3402
			atomic64_read(&event->child_total_time_running);
3403

3404
	arch_perf_update_userpage(userpg, now);
3405

3406
	barrier();
3407
	++userpg->lock;
3408
	preempt_enable();
3409
unlock:
3410
	rcu_read_unlock();
3411 3412
}

3413 3414 3415
static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
3416
	struct ring_buffer *rb;
3417 3418 3419 3420 3421 3422 3423 3424 3425
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
3426 3427
	rb = rcu_dereference(event->rb);
	if (!rb)
3428 3429 3430 3431 3432
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

3433
	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484
static void ring_buffer_attach(struct perf_event *event,
			       struct ring_buffer *rb)
{
	unsigned long flags;

	if (!list_empty(&event->rb_entry))
		return;

	spin_lock_irqsave(&rb->event_lock, flags);
	if (!list_empty(&event->rb_entry))
		goto unlock;

	list_add(&event->rb_entry, &rb->event_list);
unlock:
	spin_unlock_irqrestore(&rb->event_lock, flags);
}

static void ring_buffer_detach(struct perf_event *event,
			       struct ring_buffer *rb)
{
	unsigned long flags;

	if (list_empty(&event->rb_entry))
		return;

	spin_lock_irqsave(&rb->event_lock, flags);
	list_del_init(&event->rb_entry);
	wake_up_all(&event->waitq);
	spin_unlock_irqrestore(&rb->event_lock, flags);
}

static void ring_buffer_wakeup(struct perf_event *event)
{
	struct ring_buffer *rb;

	rcu_read_lock();
	rb = rcu_dereference(event->rb);
3485 3486 3487 3488
	if (!rb)
		goto unlock;

	list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3489
		wake_up_all(&event->waitq);
3490 3491

unlock:
3492 3493 3494
	rcu_read_unlock();
}

3495
static void rb_free_rcu(struct rcu_head *rcu_head)
3496
{
3497
	struct ring_buffer *rb;
3498

3499 3500
	rb = container_of(rcu_head, struct ring_buffer, rcu_head);
	rb_free(rb);
3501 3502
}

3503
static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3504
{
3505
	struct ring_buffer *rb;
3506

3507
	rcu_read_lock();
3508 3509 3510 3511
	rb = rcu_dereference(event->rb);
	if (rb) {
		if (!atomic_inc_not_zero(&rb->refcount))
			rb = NULL;
3512 3513 3514
	}
	rcu_read_unlock();

3515
	return rb;
3516 3517
}

3518
static void ring_buffer_put(struct ring_buffer *rb)
3519
{
3520 3521 3522
	struct perf_event *event, *n;
	unsigned long flags;

3523
	if (!atomic_dec_and_test(&rb->refcount))
3524
		return;
3525

3526 3527 3528 3529 3530 3531 3532
	spin_lock_irqsave(&rb->event_lock, flags);
	list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
		list_del_init(&event->rb_entry);
		wake_up_all(&event->waitq);
	}
	spin_unlock_irqrestore(&rb->event_lock, flags);

3533
	call_rcu(&rb->rcu_head, rb_free_rcu);
3534 3535 3536 3537
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
3538
	struct perf_event *event = vma->vm_file->private_data;
3539

3540
	atomic_inc(&event->mmap_count);
3541 3542 3543 3544
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
3545
	struct perf_event *event = vma->vm_file->private_data;
3546

3547
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3548
		unsigned long size = perf_data_size(event->rb);
3549
		struct user_struct *user = event->mmap_user;
3550
		struct ring_buffer *rb = event->rb;
3551

3552
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3553
		vma->vm_mm->pinned_vm -= event->mmap_locked;
3554
		rcu_assign_pointer(event->rb, NULL);
3555
		ring_buffer_detach(event, rb);
3556
		mutex_unlock(&event->mmap_mutex);
3557

3558
		ring_buffer_put(rb);
3559
		free_uid(user);
3560
	}
3561 3562
}

3563
static const struct vm_operations_struct perf_mmap_vmops = {
3564 3565 3566 3567
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
3568 3569 3570 3571
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
3572
	struct perf_event *event = file->private_data;
3573
	unsigned long user_locked, user_lock_limit;
3574
	struct user_struct *user = current_user();
3575
	unsigned long locked, lock_limit;
3576
	struct ring_buffer *rb;
3577 3578
	unsigned long vma_size;
	unsigned long nr_pages;
3579
	long user_extra, extra;
3580
	int ret = 0, flags = 0;
3581

3582 3583 3584
	/*
	 * Don't allow mmap() of inherited per-task counters. This would
	 * create a performance issue due to all children writing to the
3585
	 * same rb.
3586 3587 3588 3589
	 */
	if (event->cpu == -1 && event->attr.inherit)
		return -EINVAL;

3590
	if (!(vma->vm_flags & VM_SHARED))
3591
		return -EINVAL;
3592 3593 3594 3595

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

3596
	/*
3597
	 * If we have rb pages ensure they're a power-of-two number, so we
3598 3599 3600
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
3601 3602
		return -EINVAL;

3603
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
3604 3605
		return -EINVAL;

3606 3607
	if (vma->vm_pgoff != 0)
		return -EINVAL;
3608

3609 3610
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
3611 3612 3613
	if (event->rb) {
		if (event->rb->nr_pages == nr_pages)
			atomic_inc(&event->rb->refcount);
3614
		else
3615 3616 3617 3618
			ret = -EINVAL;
		goto unlock;
	}

3619
	user_extra = nr_pages + 1;
3620
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
I
Ingo Molnar 已提交
3621 3622 3623 3624 3625 3626

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

3627
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3628

3629 3630 3631
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
3632

3633
	lock_limit = rlimit(RLIMIT_MEMLOCK);
3634
	lock_limit >>= PAGE_SHIFT;
3635
	locked = vma->vm_mm->pinned_vm + extra;
3636

3637 3638
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
3639 3640 3641
		ret = -EPERM;
		goto unlock;
	}
3642

3643
	WARN_ON(event->rb);
3644

3645
	if (vma->vm_flags & VM_WRITE)
3646
		flags |= RING_BUFFER_WRITABLE;
3647

3648 3649 3650 3651
	rb = rb_alloc(nr_pages, 
		event->attr.watermark ? event->attr.wakeup_watermark : 0,
		event->cpu, flags);

3652
	if (!rb) {
3653
		ret = -ENOMEM;
3654
		goto unlock;
3655
	}
3656
	rcu_assign_pointer(event->rb, rb);
3657

3658 3659 3660
	atomic_long_add(user_extra, &user->locked_vm);
	event->mmap_locked = extra;
	event->mmap_user = get_current_user();
3661
	vma->vm_mm->pinned_vm += event->mmap_locked;
3662

3663 3664
	perf_event_update_userpage(event);

3665
unlock:
3666 3667
	if (!ret)
		atomic_inc(&event->mmap_count);
3668
	mutex_unlock(&event->mmap_mutex);
3669 3670 3671

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
3672 3673

	return ret;
3674 3675
}

P
Peter Zijlstra 已提交
3676 3677 3678
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
3679
	struct perf_event *event = filp->private_data;
P
Peter Zijlstra 已提交
3680 3681 3682
	int retval;

	mutex_lock(&inode->i_mutex);
3683
	retval = fasync_helper(fd, filp, on, &event->fasync);
P
Peter Zijlstra 已提交
3684 3685 3686 3687 3688 3689 3690 3691
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

T
Thomas Gleixner 已提交
3692
static const struct file_operations perf_fops = {
3693
	.llseek			= no_llseek,
T
Thomas Gleixner 已提交
3694 3695 3696
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
3697 3698
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
3699
	.mmap			= perf_mmap,
P
Peter Zijlstra 已提交
3700
	.fasync			= perf_fasync,
T
Thomas Gleixner 已提交
3701 3702
};

3703
/*
3704
 * Perf event wakeup
3705 3706 3707 3708 3709
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

3710
void perf_event_wakeup(struct perf_event *event)
3711
{
3712
	ring_buffer_wakeup(event);
3713

3714 3715 3716
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
3717
	}
3718 3719
}

3720
static void perf_pending_event(struct irq_work *entry)
3721
{
3722 3723
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
3724

3725 3726 3727
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
3728 3729
	}

3730 3731 3732
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
3733 3734 3735
	}
}

3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756
/*
 * We assume there is only KVM supporting the callbacks.
 * Later on, we might change it to a list if there is
 * another virtualization implementation supporting the callbacks.
 */
struct perf_guest_info_callbacks *perf_guest_cbs;

int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = cbs;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);

int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = NULL;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);

3757 3758 3759
static void __perf_event_header__init_id(struct perf_event_header *header,
					 struct perf_sample_data *data,
					 struct perf_event *event)
3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786
{
	u64 sample_type = event->attr.sample_type;

	data->type = sample_type;
	header->size += event->id_header_size;

	if (sample_type & PERF_SAMPLE_TID) {
		/* namespace issues */
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
	}

	if (sample_type & PERF_SAMPLE_TIME)
		data->time = perf_clock();

	if (sample_type & PERF_SAMPLE_ID)
		data->id = primary_event_id(event);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		data->stream_id = event->id;

	if (sample_type & PERF_SAMPLE_CPU) {
		data->cpu_entry.cpu	 = raw_smp_processor_id();
		data->cpu_entry.reserved = 0;
	}
}

3787 3788 3789
void perf_event_header__init_id(struct perf_event_header *header,
				struct perf_sample_data *data,
				struct perf_event *event)
3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815
{
	if (event->attr.sample_id_all)
		__perf_event_header__init_id(header, data, event);
}

static void __perf_event__output_id_sample(struct perf_output_handle *handle,
					   struct perf_sample_data *data)
{
	u64 sample_type = data->type;

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);
}

3816 3817 3818
void perf_event__output_id_sample(struct perf_event *event,
				  struct perf_output_handle *handle,
				  struct perf_sample_data *sample)
3819 3820 3821 3822 3823
{
	if (event->attr.sample_id_all)
		__perf_event__output_id_sample(handle, sample);
}

3824
static void perf_output_read_one(struct perf_output_handle *handle,
3825 3826
				 struct perf_event *event,
				 u64 enabled, u64 running)
3827
{
3828
	u64 read_format = event->attr.read_format;
3829 3830 3831
	u64 values[4];
	int n = 0;

P
Peter Zijlstra 已提交
3832
	values[n++] = perf_event_count(event);
3833
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3834
		values[n++] = enabled +
3835
			atomic64_read(&event->child_total_time_enabled);
3836 3837
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3838
		values[n++] = running +
3839
			atomic64_read(&event->child_total_time_running);
3840 3841
	}
	if (read_format & PERF_FORMAT_ID)
3842
		values[n++] = primary_event_id(event);
3843

3844
	__output_copy(handle, values, n * sizeof(u64));
3845 3846 3847
}

/*
3848
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3849 3850
 */
static void perf_output_read_group(struct perf_output_handle *handle,
3851 3852
			    struct perf_event *event,
			    u64 enabled, u64 running)
3853
{
3854 3855
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
3856 3857 3858 3859 3860 3861
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3862
		values[n++] = enabled;
3863 3864

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3865
		values[n++] = running;
3866

3867
	if (leader != event)
3868 3869
		leader->pmu->read(leader);

P
Peter Zijlstra 已提交
3870
	values[n++] = perf_event_count(leader);
3871
	if (read_format & PERF_FORMAT_ID)
3872
		values[n++] = primary_event_id(leader);
3873

3874
	__output_copy(handle, values, n * sizeof(u64));
3875

3876
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3877 3878
		n = 0;

3879
		if (sub != event)
3880 3881
			sub->pmu->read(sub);

P
Peter Zijlstra 已提交
3882
		values[n++] = perf_event_count(sub);
3883
		if (read_format & PERF_FORMAT_ID)
3884
			values[n++] = primary_event_id(sub);
3885

3886
		__output_copy(handle, values, n * sizeof(u64));
3887 3888 3889
	}
}

3890 3891 3892
#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
				 PERF_FORMAT_TOTAL_TIME_RUNNING)

3893
static void perf_output_read(struct perf_output_handle *handle,
3894
			     struct perf_event *event)
3895
{
3896
	u64 enabled = 0, running = 0, now;
3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
	u64 read_format = event->attr.read_format;

	/*
	 * compute total_time_enabled, total_time_running
	 * based on snapshot values taken when the event
	 * was last scheduled in.
	 *
	 * we cannot simply called update_context_time()
	 * because of locking issue as we are called in
	 * NMI context
	 */
3908
	if (read_format & PERF_FORMAT_TOTAL_TIMES)
3909
		calc_timer_values(event, &now, &enabled, &running);
3910

3911
	if (event->attr.read_format & PERF_FORMAT_GROUP)
3912
		perf_output_read_group(handle, event, enabled, running);
3913
	else
3914
		perf_output_read_one(handle, event, enabled, running);
3915 3916
}

3917 3918 3919
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3920
			struct perf_event *event)
3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3951
		perf_output_read(handle, event);
3952 3953 3954 3955 3956 3957 3958 3959 3960 3961

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

3962
			__output_copy(handle, data->callchain, size);
3963 3964 3965 3966 3967 3968 3969 3970 3971
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
3972 3973
			__output_copy(handle, data->raw->data,
					   data->raw->size);
3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998

	if (!event->attr.watermark) {
		int wakeup_events = event->attr.wakeup_events;

		if (wakeup_events) {
			struct ring_buffer *rb = handle->rb;
			int events = local_inc_return(&rb->events);

			if (events >= wakeup_events) {
				local_sub(wakeup_events, &rb->events);
				local_inc(&rb->wakeup);
			}
		}
	}
3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016

	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
		if (data->br_stack) {
			size_t size;

			size = data->br_stack->nr
			     * sizeof(struct perf_branch_entry);

			perf_output_put(handle, data->br_stack->nr);
			perf_output_copy(handle, data->br_stack->entries, size);
		} else {
			/*
			 * we always store at least the value of nr
			 */
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}
4017 4018 4019 4020
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
4021
			 struct perf_event *event,
4022
			 struct pt_regs *regs)
4023
{
4024
	u64 sample_type = event->attr.sample_type;
4025

4026
	header->type = PERF_RECORD_SAMPLE;
4027
	header->size = sizeof(*header) + event->header_size;
4028 4029 4030

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
4031

4032
	__perf_event_header__init_id(header, data, event);
4033

4034
	if (sample_type & PERF_SAMPLE_IP)
4035 4036
		data->ip = perf_instruction_pointer(regs);

4037
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4038
		int size = 1;
4039

4040 4041 4042 4043 4044 4045
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
4046 4047
	}

4048
	if (sample_type & PERF_SAMPLE_RAW) {
4049 4050 4051 4052 4053 4054 4055 4056
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
4057
		header->size += size;
4058
	}
4059 4060 4061 4062 4063 4064 4065 4066 4067

	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
		int size = sizeof(u64); /* nr */
		if (data->br_stack) {
			size += data->br_stack->nr
			      * sizeof(struct perf_branch_entry);
		}
		header->size += size;
	}
4068
}
4069

4070
static void perf_event_output(struct perf_event *event,
4071 4072 4073 4074 4075
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
4076

4077 4078 4079
	/* protect the callchain buffers */
	rcu_read_lock();

4080
	perf_prepare_sample(&header, data, event, regs);
P
Peter Zijlstra 已提交
4081

4082
	if (perf_output_begin(&handle, event, header.size))
4083
		goto exit;
4084

4085
	perf_output_sample(&handle, &header, data, event);
4086

4087
	perf_output_end(&handle);
4088 4089 4090

exit:
	rcu_read_unlock();
4091 4092
}

4093
/*
4094
 * read event_id
4095 4096 4097 4098 4099 4100 4101 4102 4103 4104
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
4105
perf_event_read_event(struct perf_event *event,
4106 4107 4108
			struct task_struct *task)
{
	struct perf_output_handle handle;
4109
	struct perf_sample_data sample;
4110
	struct perf_read_event read_event = {
4111
		.header = {
4112
			.type = PERF_RECORD_READ,
4113
			.misc = 0,
4114
			.size = sizeof(read_event) + event->read_size,
4115
		},
4116 4117
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
4118
	};
4119
	int ret;
4120

4121
	perf_event_header__init_id(&read_event.header, &sample, event);
4122
	ret = perf_output_begin(&handle, event, read_event.header.size);
4123 4124 4125
	if (ret)
		return;

4126
	perf_output_put(&handle, read_event);
4127
	perf_output_read(&handle, event);
4128
	perf_event__output_id_sample(event, &handle, &sample);
4129

4130 4131 4132
	perf_output_end(&handle);
}

P
Peter Zijlstra 已提交
4133
/*
P
Peter Zijlstra 已提交
4134 4135
 * task tracking -- fork/exit
 *
4136
 * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
P
Peter Zijlstra 已提交
4137 4138
 */

P
Peter Zijlstra 已提交
4139
struct perf_task_event {
4140
	struct task_struct		*task;
4141
	struct perf_event_context	*task_ctx;
P
Peter Zijlstra 已提交
4142 4143 4144 4145 4146 4147

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
P
Peter Zijlstra 已提交
4148 4149
		u32				tid;
		u32				ptid;
4150
		u64				time;
4151
	} event_id;
P
Peter Zijlstra 已提交
4152 4153
};

4154
static void perf_event_task_output(struct perf_event *event,
P
Peter Zijlstra 已提交
4155
				     struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
4156 4157
{
	struct perf_output_handle handle;
4158
	struct perf_sample_data	sample;
P
Peter Zijlstra 已提交
4159
	struct task_struct *task = task_event->task;
4160
	int ret, size = task_event->event_id.header.size;
4161

4162
	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
P
Peter Zijlstra 已提交
4163

4164
	ret = perf_output_begin(&handle, event,
4165
				task_event->event_id.header.size);
4166
	if (ret)
4167
		goto out;
P
Peter Zijlstra 已提交
4168

4169 4170
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
P
Peter Zijlstra 已提交
4171

4172 4173
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
P
Peter Zijlstra 已提交
4174

4175
	perf_output_put(&handle, task_event->event_id);
4176

4177 4178
	perf_event__output_id_sample(event, &handle, &sample);

P
Peter Zijlstra 已提交
4179
	perf_output_end(&handle);
4180 4181
out:
	task_event->event_id.header.size = size;
P
Peter Zijlstra 已提交
4182 4183
}

4184
static int perf_event_task_match(struct perf_event *event)
P
Peter Zijlstra 已提交
4185
{
P
Peter Zijlstra 已提交
4186
	if (event->state < PERF_EVENT_STATE_INACTIVE)
4187 4188
		return 0;

4189
	if (!event_filter_match(event))
4190 4191
		return 0;

4192 4193
	if (event->attr.comm || event->attr.mmap ||
	    event->attr.mmap_data || event->attr.task)
P
Peter Zijlstra 已提交
4194 4195 4196 4197 4198
		return 1;

	return 0;
}

4199
static void perf_event_task_ctx(struct perf_event_context *ctx,
P
Peter Zijlstra 已提交
4200
				  struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
4201
{
4202
	struct perf_event *event;
P
Peter Zijlstra 已提交
4203

4204 4205 4206
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
P
Peter Zijlstra 已提交
4207 4208 4209
	}
}

4210
static void perf_event_task_event(struct perf_task_event *task_event)
P
Peter Zijlstra 已提交
4211 4212
{
	struct perf_cpu_context *cpuctx;
P
Peter Zijlstra 已提交
4213
	struct perf_event_context *ctx;
P
Peter Zijlstra 已提交
4214
	struct pmu *pmu;
P
Peter Zijlstra 已提交
4215
	int ctxn;
P
Peter Zijlstra 已提交
4216

4217
	rcu_read_lock();
P
Peter Zijlstra 已提交
4218
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4219
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4220 4221
		if (cpuctx->active_pmu != pmu)
			goto next;
P
Peter Zijlstra 已提交
4222
		perf_event_task_ctx(&cpuctx->ctx, task_event);
P
Peter Zijlstra 已提交
4223 4224 4225 4226 4227

		ctx = task_event->task_ctx;
		if (!ctx) {
			ctxn = pmu->task_ctx_nr;
			if (ctxn < 0)
4228
				goto next;
P
Peter Zijlstra 已提交
4229 4230 4231 4232
			ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		}
		if (ctx)
			perf_event_task_ctx(ctx, task_event);
4233 4234
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
4235
	}
P
Peter Zijlstra 已提交
4236 4237 4238
	rcu_read_unlock();
}

4239 4240
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
4241
			      int new)
P
Peter Zijlstra 已提交
4242
{
P
Peter Zijlstra 已提交
4243
	struct perf_task_event task_event;
P
Peter Zijlstra 已提交
4244

4245 4246 4247
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
P
Peter Zijlstra 已提交
4248 4249
		return;

P
Peter Zijlstra 已提交
4250
	task_event = (struct perf_task_event){
4251 4252
		.task	  = task,
		.task_ctx = task_ctx,
4253
		.event_id    = {
P
Peter Zijlstra 已提交
4254
			.header = {
4255
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4256
				.misc = 0,
4257
				.size = sizeof(task_event.event_id),
P
Peter Zijlstra 已提交
4258
			},
4259 4260
			/* .pid  */
			/* .ppid */
P
Peter Zijlstra 已提交
4261 4262
			/* .tid  */
			/* .ptid */
P
Peter Zijlstra 已提交
4263
			.time = perf_clock(),
P
Peter Zijlstra 已提交
4264 4265 4266
		},
	};

4267
	perf_event_task_event(&task_event);
P
Peter Zijlstra 已提交
4268 4269
}

4270
void perf_event_fork(struct task_struct *task)
P
Peter Zijlstra 已提交
4271
{
4272
	perf_event_task(task, NULL, 1);
P
Peter Zijlstra 已提交
4273 4274
}

4275 4276 4277 4278 4279
/*
 * comm tracking
 */

struct perf_comm_event {
4280 4281
	struct task_struct	*task;
	char			*comm;
4282 4283 4284 4285 4286 4287 4288
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
4289
	} event_id;
4290 4291
};

4292
static void perf_event_comm_output(struct perf_event *event,
4293 4294 4295
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
4296
	struct perf_sample_data sample;
4297
	int size = comm_event->event_id.header.size;
4298 4299 4300 4301
	int ret;

	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
	ret = perf_output_begin(&handle, event,
4302
				comm_event->event_id.header.size);
4303 4304

	if (ret)
4305
		goto out;
4306

4307 4308
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4309

4310
	perf_output_put(&handle, comm_event->event_id);
4311
	__output_copy(&handle, comm_event->comm,
4312
				   comm_event->comm_size);
4313 4314 4315

	perf_event__output_id_sample(event, &handle, &sample);

4316
	perf_output_end(&handle);
4317 4318
out:
	comm_event->event_id.header.size = size;
4319 4320
}

4321
static int perf_event_comm_match(struct perf_event *event)
4322
{
P
Peter Zijlstra 已提交
4323
	if (event->state < PERF_EVENT_STATE_INACTIVE)
4324 4325
		return 0;

4326
	if (!event_filter_match(event))
4327 4328
		return 0;

4329
	if (event->attr.comm)
4330 4331 4332 4333 4334
		return 1;

	return 0;
}

4335
static void perf_event_comm_ctx(struct perf_event_context *ctx,
4336 4337
				  struct perf_comm_event *comm_event)
{
4338
	struct perf_event *event;
4339

4340 4341 4342
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
4343 4344 4345
	}
}

4346
static void perf_event_comm_event(struct perf_comm_event *comm_event)
4347 4348
{
	struct perf_cpu_context *cpuctx;
4349
	struct perf_event_context *ctx;
4350
	char comm[TASK_COMM_LEN];
4351
	unsigned int size;
P
Peter Zijlstra 已提交
4352
	struct pmu *pmu;
P
Peter Zijlstra 已提交
4353
	int ctxn;
4354

4355
	memset(comm, 0, sizeof(comm));
4356
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
4357
	size = ALIGN(strlen(comm)+1, sizeof(u64));
4358 4359 4360 4361

	comm_event->comm = comm;
	comm_event->comm_size = size;

4362
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4363
	rcu_read_lock();
P
Peter Zijlstra 已提交
4364
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4365
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4366 4367
		if (cpuctx->active_pmu != pmu)
			goto next;
P
Peter Zijlstra 已提交
4368
		perf_event_comm_ctx(&cpuctx->ctx, comm_event);
P
Peter Zijlstra 已提交
4369 4370 4371

		ctxn = pmu->task_ctx_nr;
		if (ctxn < 0)
4372
			goto next;
P
Peter Zijlstra 已提交
4373 4374 4375 4376

		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		if (ctx)
			perf_event_comm_ctx(ctx, comm_event);
4377 4378
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
4379
	}
4380
	rcu_read_unlock();
4381 4382
}

4383
void perf_event_comm(struct task_struct *task)
4384
{
4385
	struct perf_comm_event comm_event;
P
Peter Zijlstra 已提交
4386 4387
	struct perf_event_context *ctx;
	int ctxn;
4388

P
Peter Zijlstra 已提交
4389 4390 4391 4392
	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (!ctx)
			continue;
4393

P
Peter Zijlstra 已提交
4394 4395
		perf_event_enable_on_exec(ctx);
	}
4396

4397
	if (!atomic_read(&nr_comm_events))
4398
		return;
4399

4400
	comm_event = (struct perf_comm_event){
4401
		.task	= task,
4402 4403
		/* .comm      */
		/* .comm_size */
4404
		.event_id  = {
4405
			.header = {
4406
				.type = PERF_RECORD_COMM,
4407 4408 4409 4410 4411
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
4412 4413 4414
		},
	};

4415
	perf_event_comm_event(&comm_event);
4416 4417
}

4418 4419 4420 4421 4422
/*
 * mmap tracking
 */

struct perf_mmap_event {
4423 4424 4425 4426
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
4427 4428 4429 4430 4431 4432 4433 4434 4435

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
4436
	} event_id;
4437 4438
};

4439
static void perf_event_mmap_output(struct perf_event *event,
4440 4441 4442
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
4443
	struct perf_sample_data sample;
4444
	int size = mmap_event->event_id.header.size;
4445
	int ret;
4446

4447 4448
	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
	ret = perf_output_begin(&handle, event,
4449
				mmap_event->event_id.header.size);
4450
	if (ret)
4451
		goto out;
4452

4453 4454
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
4455

4456
	perf_output_put(&handle, mmap_event->event_id);
4457
	__output_copy(&handle, mmap_event->file_name,
4458
				   mmap_event->file_size);
4459 4460 4461

	perf_event__output_id_sample(event, &handle, &sample);

4462
	perf_output_end(&handle);
4463 4464
out:
	mmap_event->event_id.header.size = size;
4465 4466
}

4467
static int perf_event_mmap_match(struct perf_event *event,
4468 4469
				   struct perf_mmap_event *mmap_event,
				   int executable)
4470
{
P
Peter Zijlstra 已提交
4471
	if (event->state < PERF_EVENT_STATE_INACTIVE)
4472 4473
		return 0;

4474
	if (!event_filter_match(event))
4475 4476
		return 0;

4477 4478
	if ((!executable && event->attr.mmap_data) ||
	    (executable && event->attr.mmap))
4479 4480 4481 4482 4483
		return 1;

	return 0;
}

4484
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4485 4486
				  struct perf_mmap_event *mmap_event,
				  int executable)
4487
{
4488
	struct perf_event *event;
4489

4490
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4491
		if (perf_event_mmap_match(event, mmap_event, executable))
4492
			perf_event_mmap_output(event, mmap_event);
4493 4494 4495
	}
}

4496
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4497 4498
{
	struct perf_cpu_context *cpuctx;
4499
	struct perf_event_context *ctx;
4500 4501
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
4502 4503 4504
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
4505
	const char *name;
P
Peter Zijlstra 已提交
4506
	struct pmu *pmu;
P
Peter Zijlstra 已提交
4507
	int ctxn;
4508

4509 4510
	memset(tmp, 0, sizeof(tmp));

4511
	if (file) {
4512
		/*
4513
		 * d_path works from the end of the rb backwards, so we
4514 4515 4516 4517
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4518 4519 4520 4521
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
4522
		name = d_path(&file->f_path, buf, PATH_MAX);
4523 4524 4525 4526 4527
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
4528 4529 4530
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
4531
			goto got_name;
4532
		}
4533 4534 4535 4536

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
4537 4538 4539 4540 4541 4542 4543 4544
		} else if (vma->vm_start <= vma->vm_mm->start_brk &&
				vma->vm_end >= vma->vm_mm->brk) {
			name = strncpy(tmp, "[heap]", sizeof(tmp));
			goto got_name;
		} else if (vma->vm_start <= vma->vm_mm->start_stack &&
				vma->vm_end >= vma->vm_mm->start_stack) {
			name = strncpy(tmp, "[stack]", sizeof(tmp));
			goto got_name;
4545 4546
		}

4547 4548 4549 4550 4551
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
4552
	size = ALIGN(strlen(name)+1, sizeof(u64));
4553 4554 4555 4556

	mmap_event->file_name = name;
	mmap_event->file_size = size;

4557
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4558

4559
	rcu_read_lock();
P
Peter Zijlstra 已提交
4560
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4561
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4562 4563
		if (cpuctx->active_pmu != pmu)
			goto next;
P
Peter Zijlstra 已提交
4564 4565
		perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
					vma->vm_flags & VM_EXEC);
P
Peter Zijlstra 已提交
4566 4567 4568

		ctxn = pmu->task_ctx_nr;
		if (ctxn < 0)
4569
			goto next;
P
Peter Zijlstra 已提交
4570 4571 4572 4573 4574 4575

		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		if (ctx) {
			perf_event_mmap_ctx(ctx, mmap_event,
					vma->vm_flags & VM_EXEC);
		}
4576 4577
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
4578
	}
4579 4580
	rcu_read_unlock();

4581 4582 4583
	kfree(buf);
}

4584
void perf_event_mmap(struct vm_area_struct *vma)
4585
{
4586 4587
	struct perf_mmap_event mmap_event;

4588
	if (!atomic_read(&nr_mmap_events))
4589 4590 4591
		return;

	mmap_event = (struct perf_mmap_event){
4592
		.vma	= vma,
4593 4594
		/* .file_name */
		/* .file_size */
4595
		.event_id  = {
4596
			.header = {
4597
				.type = PERF_RECORD_MMAP,
4598
				.misc = PERF_RECORD_MISC_USER,
4599 4600 4601 4602
				/* .size */
			},
			/* .pid */
			/* .tid */
4603 4604
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
4605
			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4606 4607 4608
		},
	};

4609
	perf_event_mmap_event(&mmap_event);
4610 4611
}

4612 4613 4614 4615
/*
 * IRQ throttle logging
 */

4616
static void perf_log_throttle(struct perf_event *event, int enable)
4617 4618
{
	struct perf_output_handle handle;
4619
	struct perf_sample_data sample;
4620 4621 4622 4623 4624
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
4625
		u64				id;
4626
		u64				stream_id;
4627 4628
	} throttle_event = {
		.header = {
4629
			.type = PERF_RECORD_THROTTLE,
4630 4631 4632
			.misc = 0,
			.size = sizeof(throttle_event),
		},
P
Peter Zijlstra 已提交
4633
		.time		= perf_clock(),
4634 4635
		.id		= primary_event_id(event),
		.stream_id	= event->id,
4636 4637
	};

4638
	if (enable)
4639
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4640

4641 4642 4643
	perf_event_header__init_id(&throttle_event.header, &sample, event);

	ret = perf_output_begin(&handle, event,
4644
				throttle_event.header.size);
4645 4646 4647 4648
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
4649
	perf_event__output_id_sample(event, &handle, &sample);
4650 4651 4652
	perf_output_end(&handle);
}

4653
/*
4654
 * Generic event overflow handling, sampling.
4655 4656
 */

4657
static int __perf_event_overflow(struct perf_event *event,
4658 4659
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
4660
{
4661 4662
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
4663
	u64 seq;
4664 4665
	int ret = 0;

4666 4667 4668 4669 4670 4671 4672
	/*
	 * Non-sampling counters might still use the PMI to fold short
	 * hardware counters, ignore those.
	 */
	if (unlikely(!is_sampling_event(event)))
		return 0;

4673 4674 4675 4676 4677 4678 4679 4680 4681
	seq = __this_cpu_read(perf_throttled_seq);
	if (seq != hwc->interrupts_seq) {
		hwc->interrupts_seq = seq;
		hwc->interrupts = 1;
	} else {
		hwc->interrupts++;
		if (unlikely(throttle
			     && hwc->interrupts >= max_samples_per_tick)) {
			__this_cpu_inc(perf_throttled_count);
P
Peter Zijlstra 已提交
4682 4683
			hwc->interrupts = MAX_INTERRUPTS;
			perf_log_throttle(event, 0);
4684 4685
			ret = 1;
		}
4686
	}
4687

4688
	if (event->attr.freq) {
P
Peter Zijlstra 已提交
4689
		u64 now = perf_clock();
4690
		s64 delta = now - hwc->freq_time_stamp;
4691

4692
		hwc->freq_time_stamp = now;
4693

4694
		if (delta > 0 && delta < 2*TICK_NSEC)
4695
			perf_adjust_period(event, delta, hwc->last_period, true);
4696 4697
	}

4698 4699
	/*
	 * XXX event_limit might not quite work as expected on inherited
4700
	 * events
4701 4702
	 */

4703 4704
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
4705
		ret = 1;
4706
		event->pending_kill = POLL_HUP;
4707 4708
		event->pending_disable = 1;
		irq_work_queue(&event->pending);
4709 4710
	}

4711
	if (event->overflow_handler)
4712
		event->overflow_handler(event, data, regs);
4713
	else
4714
		perf_event_output(event, data, regs);
4715

P
Peter Zijlstra 已提交
4716
	if (event->fasync && event->pending_kill) {
4717 4718
		event->pending_wakeup = 1;
		irq_work_queue(&event->pending);
P
Peter Zijlstra 已提交
4719 4720
	}

4721
	return ret;
4722 4723
}

4724
int perf_event_overflow(struct perf_event *event,
4725 4726
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
4727
{
4728
	return __perf_event_overflow(event, 1, data, regs);
4729 4730
}

4731
/*
4732
 * Generic software event infrastructure
4733 4734
 */

4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745
struct swevent_htable {
	struct swevent_hlist		*swevent_hlist;
	struct mutex			hlist_mutex;
	int				hlist_refcount;

	/* Recursion avoidance in each contexts */
	int				recursion[PERF_NR_CONTEXTS];
};

static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);

4746
/*
4747 4748
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
4749 4750 4751 4752
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

4753
static u64 perf_swevent_set_period(struct perf_event *event)
4754
{
4755
	struct hw_perf_event *hwc = &event->hw;
4756 4757 4758 4759 4760
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
4761 4762

again:
4763
	old = val = local64_read(&hwc->period_left);
4764 4765
	if (val < 0)
		return 0;
4766

4767 4768 4769
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
4770
	if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4771
		goto again;
4772

4773
	return nr;
4774 4775
}

4776
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4777
				    struct perf_sample_data *data,
4778
				    struct pt_regs *regs)
4779
{
4780
	struct hw_perf_event *hwc = &event->hw;
4781
	int throttle = 0;
4782

4783 4784
	if (!overflow)
		overflow = perf_swevent_set_period(event);
4785

4786 4787
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
4788

4789
	for (; overflow; overflow--) {
4790
		if (__perf_event_overflow(event, throttle,
4791
					    data, regs)) {
4792 4793 4794 4795 4796 4797
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
4798
		throttle = 1;
4799
	}
4800 4801
}

P
Peter Zijlstra 已提交
4802
static void perf_swevent_event(struct perf_event *event, u64 nr,
4803
			       struct perf_sample_data *data,
4804
			       struct pt_regs *regs)
4805
{
4806
	struct hw_perf_event *hwc = &event->hw;
4807

4808
	local64_add(nr, &event->count);
4809

4810 4811 4812
	if (!regs)
		return;

4813
	if (!is_sampling_event(event))
4814
		return;
4815

4816 4817 4818 4819 4820 4821
	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
		data->period = nr;
		return perf_swevent_overflow(event, 1, data, regs);
	} else
		data->period = event->hw.last_period;

4822
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4823
		return perf_swevent_overflow(event, 1, data, regs);
4824

4825
	if (local64_add_negative(nr, &hwc->period_left))
4826
		return;
4827

4828
	perf_swevent_overflow(event, 0, data, regs);
4829 4830
}

4831 4832 4833
static int perf_exclude_event(struct perf_event *event,
			      struct pt_regs *regs)
{
P
Peter Zijlstra 已提交
4834
	if (event->hw.state & PERF_HES_STOPPED)
4835
		return 1;
P
Peter Zijlstra 已提交
4836

4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847
	if (regs) {
		if (event->attr.exclude_user && user_mode(regs))
			return 1;

		if (event->attr.exclude_kernel && !user_mode(regs))
			return 1;
	}

	return 0;
}

4848
static int perf_swevent_match(struct perf_event *event,
P
Peter Zijlstra 已提交
4849
				enum perf_type_id type,
L
Li Zefan 已提交
4850 4851 4852
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
4853
{
4854
	if (event->attr.type != type)
4855
		return 0;
4856

4857
	if (event->attr.config != event_id)
4858 4859
		return 0;

4860 4861
	if (perf_exclude_event(event, regs))
		return 0;
4862 4863 4864 4865

	return 1;
}

4866 4867 4868 4869 4870 4871 4872
static inline u64 swevent_hash(u64 type, u32 event_id)
{
	u64 val = event_id | (type << 32);

	return hash_64(val, SWEVENT_HLIST_BITS);
}

4873 4874
static inline struct hlist_head *
__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4875
{
4876 4877 4878 4879
	u64 hash = swevent_hash(type, event_id);

	return &hlist->heads[hash];
}
4880

4881 4882
/* For the read side: events when they trigger */
static inline struct hlist_head *
4883
find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4884 4885
{
	struct swevent_hlist *hlist;
4886

4887
	hlist = rcu_dereference(swhash->swevent_hlist);
4888 4889 4890
	if (!hlist)
		return NULL;

4891 4892 4893 4894 4895
	return __find_swevent_head(hlist, type, event_id);
}

/* For the event head insertion and removal in the hlist */
static inline struct hlist_head *
4896
find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4897 4898 4899 4900 4901 4902 4903 4904 4905 4906
{
	struct swevent_hlist *hlist;
	u32 event_id = event->attr.config;
	u64 type = event->attr.type;

	/*
	 * Event scheduling is always serialized against hlist allocation
	 * and release. Which makes the protected version suitable here.
	 * The context lock guarantees that.
	 */
4907
	hlist = rcu_dereference_protected(swhash->swevent_hlist,
4908 4909 4910 4911 4912
					  lockdep_is_held(&event->ctx->lock));
	if (!hlist)
		return NULL;

	return __find_swevent_head(hlist, type, event_id);
4913 4914 4915
}

static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4916
				    u64 nr,
4917 4918
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
4919
{
4920
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4921
	struct perf_event *event;
4922 4923
	struct hlist_node *node;
	struct hlist_head *head;
4924

4925
	rcu_read_lock();
4926
	head = find_swevent_head_rcu(swhash, type, event_id);
4927 4928 4929 4930
	if (!head)
		goto end;

	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
L
Li Zefan 已提交
4931
		if (perf_swevent_match(event, type, event_id, data, regs))
4932
			perf_swevent_event(event, nr, data, regs);
4933
	}
4934 4935
end:
	rcu_read_unlock();
4936 4937
}

4938
int perf_swevent_get_recursion_context(void)
P
Peter Zijlstra 已提交
4939
{
4940
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
P
Peter Zijlstra 已提交
4941

4942
	return get_recursion_context(swhash->recursion);
P
Peter Zijlstra 已提交
4943
}
I
Ingo Molnar 已提交
4944
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
P
Peter Zijlstra 已提交
4945

4946
inline void perf_swevent_put_recursion_context(int rctx)
4947
{
4948
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4949

4950
	put_recursion_context(swhash->recursion, rctx);
4951
}
4952

4953
void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
4954
{
4955
	struct perf_sample_data data;
4956 4957
	int rctx;

4958
	preempt_disable_notrace();
4959 4960 4961
	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return;
4962

4963
	perf_sample_data_init(&data, addr, 0);
4964

4965
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
4966 4967

	perf_swevent_put_recursion_context(rctx);
4968
	preempt_enable_notrace();
4969 4970
}

4971
static void perf_swevent_read(struct perf_event *event)
4972 4973 4974
{
}

P
Peter Zijlstra 已提交
4975
static int perf_swevent_add(struct perf_event *event, int flags)
4976
{
4977
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4978
	struct hw_perf_event *hwc = &event->hw;
4979 4980
	struct hlist_head *head;

4981
	if (is_sampling_event(event)) {
4982
		hwc->last_period = hwc->sample_period;
4983
		perf_swevent_set_period(event);
4984
	}
4985

P
Peter Zijlstra 已提交
4986 4987
	hwc->state = !(flags & PERF_EF_START);

4988
	head = find_swevent_head(swhash, event);
4989 4990 4991 4992 4993
	if (WARN_ON_ONCE(!head))
		return -EINVAL;

	hlist_add_head_rcu(&event->hlist_entry, head);

4994 4995 4996
	return 0;
}

P
Peter Zijlstra 已提交
4997
static void perf_swevent_del(struct perf_event *event, int flags)
4998
{
4999
	hlist_del_rcu(&event->hlist_entry);
5000 5001
}

P
Peter Zijlstra 已提交
5002
static void perf_swevent_start(struct perf_event *event, int flags)
5003
{
P
Peter Zijlstra 已提交
5004
	event->hw.state = 0;
5005
}
I
Ingo Molnar 已提交
5006

P
Peter Zijlstra 已提交
5007
static void perf_swevent_stop(struct perf_event *event, int flags)
5008
{
P
Peter Zijlstra 已提交
5009
	event->hw.state = PERF_HES_STOPPED;
5010 5011
}

5012 5013
/* Deref the hlist from the update side */
static inline struct swevent_hlist *
5014
swevent_hlist_deref(struct swevent_htable *swhash)
5015
{
5016 5017
	return rcu_dereference_protected(swhash->swevent_hlist,
					 lockdep_is_held(&swhash->hlist_mutex));
5018 5019
}

5020
static void swevent_hlist_release(struct swevent_htable *swhash)
5021
{
5022
	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5023

5024
	if (!hlist)
5025 5026
		return;

5027
	rcu_assign_pointer(swhash->swevent_hlist, NULL);
5028
	kfree_rcu(hlist, rcu_head);
5029 5030 5031 5032
}

static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
{
5033
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5034

5035
	mutex_lock(&swhash->hlist_mutex);
5036

5037 5038
	if (!--swhash->hlist_refcount)
		swevent_hlist_release(swhash);
5039

5040
	mutex_unlock(&swhash->hlist_mutex);
5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057
}

static void swevent_hlist_put(struct perf_event *event)
{
	int cpu;

	if (event->cpu != -1) {
		swevent_hlist_put_cpu(event, event->cpu);
		return;
	}

	for_each_possible_cpu(cpu)
		swevent_hlist_put_cpu(event, cpu);
}

static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
{
5058
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5059 5060
	int err = 0;

5061
	mutex_lock(&swhash->hlist_mutex);
5062

5063
	if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5064 5065 5066 5067 5068 5069 5070
		struct swevent_hlist *hlist;

		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
		if (!hlist) {
			err = -ENOMEM;
			goto exit;
		}
5071
		rcu_assign_pointer(swhash->swevent_hlist, hlist);
5072
	}
5073
	swhash->hlist_refcount++;
P
Peter Zijlstra 已提交
5074
exit:
5075
	mutex_unlock(&swhash->hlist_mutex);
5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098

	return err;
}

static int swevent_hlist_get(struct perf_event *event)
{
	int err;
	int cpu, failed_cpu;

	if (event->cpu != -1)
		return swevent_hlist_get_cpu(event, event->cpu);

	get_online_cpus();
	for_each_possible_cpu(cpu) {
		err = swevent_hlist_get_cpu(event, cpu);
		if (err) {
			failed_cpu = cpu;
			goto fail;
		}
	}
	put_online_cpus();

	return 0;
P
Peter Zijlstra 已提交
5099
fail:
5100 5101 5102 5103 5104 5105 5106 5107 5108 5109
	for_each_possible_cpu(cpu) {
		if (cpu == failed_cpu)
			break;
		swevent_hlist_put_cpu(event, cpu);
	}

	put_online_cpus();
	return err;
}

5110
struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
5111

5112 5113 5114
static void sw_perf_event_destroy(struct perf_event *event)
{
	u64 event_id = event->attr.config;
5115

5116 5117
	WARN_ON(event->parent);

5118
	static_key_slow_dec(&perf_swevent_enabled[event_id]);
5119 5120 5121 5122 5123 5124 5125 5126 5127 5128
	swevent_hlist_put(event);
}

static int perf_swevent_init(struct perf_event *event)
{
	int event_id = event->attr.config;

	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

5129 5130 5131 5132 5133 5134
	/*
	 * no branch sampling for software events
	 */
	if (has_branch_stack(event))
		return -EOPNOTSUPP;

5135 5136 5137 5138 5139 5140 5141 5142 5143
	switch (event_id) {
	case PERF_COUNT_SW_CPU_CLOCK:
	case PERF_COUNT_SW_TASK_CLOCK:
		return -ENOENT;

	default:
		break;
	}

5144
	if (event_id >= PERF_COUNT_SW_MAX)
5145 5146 5147 5148 5149 5150 5151 5152 5153
		return -ENOENT;

	if (!event->parent) {
		int err;

		err = swevent_hlist_get(event);
		if (err)
			return err;

5154
		static_key_slow_inc(&perf_swevent_enabled[event_id]);
5155 5156 5157 5158 5159 5160
		event->destroy = sw_perf_event_destroy;
	}

	return 0;
}

5161 5162 5163 5164 5165
static int perf_swevent_event_idx(struct perf_event *event)
{
	return 0;
}

5166
static struct pmu perf_swevent = {
5167
	.task_ctx_nr	= perf_sw_context,
5168

5169
	.event_init	= perf_swevent_init,
P
Peter Zijlstra 已提交
5170 5171 5172 5173
	.add		= perf_swevent_add,
	.del		= perf_swevent_del,
	.start		= perf_swevent_start,
	.stop		= perf_swevent_stop,
5174
	.read		= perf_swevent_read,
5175 5176

	.event_idx	= perf_swevent_event_idx,
5177 5178
};

5179 5180
#ifdef CONFIG_EVENT_TRACING

5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194
static int perf_tp_filter_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
5195 5196
	if (event->hw.state & PERF_HES_STOPPED)
		return 0;
5197 5198 5199 5200
	/*
	 * All tracepoints are from kernel-space.
	 */
	if (event->attr.exclude_kernel)
5201 5202 5203 5204 5205 5206 5207 5208 5209
		return 0;

	if (!perf_tp_filter_match(event, data))
		return 0;

	return 1;
}

void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5210
		   struct pt_regs *regs, struct hlist_head *head, int rctx)
5211 5212
{
	struct perf_sample_data data;
5213 5214 5215
	struct perf_event *event;
	struct hlist_node *node;

5216 5217 5218 5219 5220
	struct perf_raw_record raw = {
		.size = entry_size,
		.data = record,
	};

5221
	perf_sample_data_init(&data, addr, 0);
5222 5223
	data.raw = &raw;

5224 5225
	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
		if (perf_tp_event_match(event, &data, regs))
5226
			perf_swevent_event(event, count, &data, regs);
5227
	}
5228 5229

	perf_swevent_put_recursion_context(rctx);
5230 5231 5232
}
EXPORT_SYMBOL_GPL(perf_tp_event);

5233
static void tp_perf_event_destroy(struct perf_event *event)
5234
{
5235
	perf_trace_destroy(event);
5236 5237
}

5238
static int perf_tp_event_init(struct perf_event *event)
5239
{
5240 5241
	int err;

5242 5243 5244
	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -ENOENT;

5245 5246 5247 5248 5249 5250
	/*
	 * no branch sampling for tracepoint events
	 */
	if (has_branch_stack(event))
		return -EOPNOTSUPP;

5251 5252
	err = perf_trace_init(event);
	if (err)
5253
		return err;
5254

5255
	event->destroy = tp_perf_event_destroy;
5256

5257 5258 5259 5260
	return 0;
}

static struct pmu perf_tracepoint = {
5261 5262
	.task_ctx_nr	= perf_sw_context,

5263
	.event_init	= perf_tp_event_init,
P
Peter Zijlstra 已提交
5264 5265 5266 5267
	.add		= perf_trace_add,
	.del		= perf_trace_del,
	.start		= perf_swevent_start,
	.stop		= perf_swevent_stop,
5268
	.read		= perf_swevent_read,
5269 5270

	.event_idx	= perf_swevent_event_idx,
5271 5272 5273 5274
};

static inline void perf_tp_register(void)
{
P
Peter Zijlstra 已提交
5275
	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5276
}
L
Li Zefan 已提交
5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

5301
#else
L
Li Zefan 已提交
5302

5303
static inline void perf_tp_register(void)
5304 5305
{
}
L
Li Zefan 已提交
5306 5307 5308 5309 5310 5311 5312 5313 5314 5315

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

5316
#endif /* CONFIG_EVENT_TRACING */
5317

5318
#ifdef CONFIG_HAVE_HW_BREAKPOINT
5319
void perf_bp_event(struct perf_event *bp, void *data)
5320
{
5321 5322 5323
	struct perf_sample_data sample;
	struct pt_regs *regs = data;

5324
	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
5325

P
Peter Zijlstra 已提交
5326
	if (!bp->hw.state && !perf_exclude_event(bp, regs))
5327
		perf_swevent_event(bp, 1, &sample, regs);
5328 5329 5330
}
#endif

5331 5332 5333
/*
 * hrtimer based swevent callback
 */
5334

5335
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5336
{
5337 5338 5339 5340 5341
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
	struct pt_regs *regs;
	struct perf_event *event;
	u64 period;
5342

5343
	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
P
Peter Zijlstra 已提交
5344 5345 5346 5347

	if (event->state != PERF_EVENT_STATE_ACTIVE)
		return HRTIMER_NORESTART;

5348
	event->pmu->read(event);
5349

5350
	perf_sample_data_init(&data, 0, event->hw.last_period);
5351 5352 5353
	regs = get_irq_regs();

	if (regs && !perf_exclude_event(event, regs)) {
5354
		if (!(event->attr.exclude_idle && is_idle_task(current)))
5355
			if (__perf_event_overflow(event, 1, &data, regs))
5356 5357
				ret = HRTIMER_NORESTART;
	}
5358

5359 5360
	period = max_t(u64, 10000, event->hw.sample_period);
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5361

5362
	return ret;
5363 5364
}

5365
static void perf_swevent_start_hrtimer(struct perf_event *event)
5366
{
5367
	struct hw_perf_event *hwc = &event->hw;
5368 5369 5370 5371
	s64 period;

	if (!is_sampling_event(event))
		return;
5372

5373 5374 5375 5376
	period = local64_read(&hwc->period_left);
	if (period) {
		if (period < 0)
			period = 10000;
P
Peter Zijlstra 已提交
5377

5378 5379 5380 5381 5382
		local64_set(&hwc->period_left, 0);
	} else {
		period = max_t(u64, 10000, hwc->sample_period);
	}
	__hrtimer_start_range_ns(&hwc->hrtimer,
5383
				ns_to_ktime(period), 0,
5384
				HRTIMER_MODE_REL_PINNED, 0);
5385
}
5386 5387

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5388
{
5389 5390
	struct hw_perf_event *hwc = &event->hw;

5391
	if (is_sampling_event(event)) {
5392
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
P
Peter Zijlstra 已提交
5393
		local64_set(&hwc->period_left, ktime_to_ns(remaining));
5394 5395 5396

		hrtimer_cancel(&hwc->hrtimer);
	}
5397 5398
}

P
Peter Zijlstra 已提交
5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422
static void perf_swevent_init_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (!is_sampling_event(event))
		return;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;

	/*
	 * Since hrtimers have a fixed rate, we can do a static freq->period
	 * mapping and avoid the whole period adjust feedback stuff.
	 */
	if (event->attr.freq) {
		long freq = event->attr.sample_freq;

		event->attr.sample_period = NSEC_PER_SEC / freq;
		hwc->sample_period = event->attr.sample_period;
		local64_set(&hwc->period_left, hwc->sample_period);
		event->attr.freq = 0;
	}
}

5423 5424 5425 5426 5427
/*
 * Software event: cpu wall time clock
 */

static void cpu_clock_event_update(struct perf_event *event)
5428
{
5429 5430 5431
	s64 prev;
	u64 now;

P
Peter Zijlstra 已提交
5432
	now = local_clock();
5433 5434
	prev = local64_xchg(&event->hw.prev_count, now);
	local64_add(now - prev, &event->count);
5435 5436
}

P
Peter Zijlstra 已提交
5437
static void cpu_clock_event_start(struct perf_event *event, int flags)
5438
{
P
Peter Zijlstra 已提交
5439
	local64_set(&event->hw.prev_count, local_clock());
5440 5441 5442
	perf_swevent_start_hrtimer(event);
}

P
Peter Zijlstra 已提交
5443
static void cpu_clock_event_stop(struct perf_event *event, int flags)
5444
{
5445 5446 5447
	perf_swevent_cancel_hrtimer(event);
	cpu_clock_event_update(event);
}
5448

P
Peter Zijlstra 已提交
5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461
static int cpu_clock_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		cpu_clock_event_start(event, flags);

	return 0;
}

static void cpu_clock_event_del(struct perf_event *event, int flags)
{
	cpu_clock_event_stop(event, flags);
}

5462 5463 5464 5465
static void cpu_clock_event_read(struct perf_event *event)
{
	cpu_clock_event_update(event);
}
5466

5467 5468 5469 5470 5471 5472 5473 5474
static int cpu_clock_event_init(struct perf_event *event)
{
	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
		return -ENOENT;

5475 5476 5477 5478 5479 5480
	/*
	 * no branch sampling for software events
	 */
	if (has_branch_stack(event))
		return -EOPNOTSUPP;

P
Peter Zijlstra 已提交
5481 5482
	perf_swevent_init_hrtimer(event);

5483
	return 0;
5484 5485
}

5486
static struct pmu perf_cpu_clock = {
5487 5488
	.task_ctx_nr	= perf_sw_context,

5489
	.event_init	= cpu_clock_event_init,
P
Peter Zijlstra 已提交
5490 5491 5492 5493
	.add		= cpu_clock_event_add,
	.del		= cpu_clock_event_del,
	.start		= cpu_clock_event_start,
	.stop		= cpu_clock_event_stop,
5494
	.read		= cpu_clock_event_read,
5495 5496

	.event_idx	= perf_swevent_event_idx,
5497 5498 5499 5500 5501 5502 5503
};

/*
 * Software event: task time clock
 */

static void task_clock_event_update(struct perf_event *event, u64 now)
5504
{
5505 5506
	u64 prev;
	s64 delta;
5507

5508 5509 5510 5511
	prev = local64_xchg(&event->hw.prev_count, now);
	delta = now - prev;
	local64_add(delta, &event->count);
}
5512

P
Peter Zijlstra 已提交
5513
static void task_clock_event_start(struct perf_event *event, int flags)
5514
{
P
Peter Zijlstra 已提交
5515
	local64_set(&event->hw.prev_count, event->ctx->time);
5516 5517 5518
	perf_swevent_start_hrtimer(event);
}

P
Peter Zijlstra 已提交
5519
static void task_clock_event_stop(struct perf_event *event, int flags)
5520 5521 5522
{
	perf_swevent_cancel_hrtimer(event);
	task_clock_event_update(event, event->ctx->time);
P
Peter Zijlstra 已提交
5523 5524 5525 5526 5527 5528
}

static int task_clock_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		task_clock_event_start(event, flags);
5529

P
Peter Zijlstra 已提交
5530 5531 5532 5533 5534 5535
	return 0;
}

static void task_clock_event_del(struct perf_event *event, int flags)
{
	task_clock_event_stop(event, PERF_EF_UPDATE);
5536 5537 5538 5539
}

static void task_clock_event_read(struct perf_event *event)
{
5540 5541 5542
	u64 now = perf_clock();
	u64 delta = now - event->ctx->timestamp;
	u64 time = event->ctx->time + delta;
5543 5544 5545 5546 5547

	task_clock_event_update(event, time);
}

static int task_clock_event_init(struct perf_event *event)
L
Li Zefan 已提交
5548
{
5549 5550 5551 5552 5553 5554
	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
		return -ENOENT;

5555 5556 5557 5558 5559 5560
	/*
	 * no branch sampling for software events
	 */
	if (has_branch_stack(event))
		return -EOPNOTSUPP;

P
Peter Zijlstra 已提交
5561 5562
	perf_swevent_init_hrtimer(event);

5563
	return 0;
L
Li Zefan 已提交
5564 5565
}

5566
static struct pmu perf_task_clock = {
5567 5568
	.task_ctx_nr	= perf_sw_context,

5569
	.event_init	= task_clock_event_init,
P
Peter Zijlstra 已提交
5570 5571 5572 5573
	.add		= task_clock_event_add,
	.del		= task_clock_event_del,
	.start		= task_clock_event_start,
	.stop		= task_clock_event_stop,
5574
	.read		= task_clock_event_read,
5575 5576

	.event_idx	= perf_swevent_event_idx,
5577
};
L
Li Zefan 已提交
5578

P
Peter Zijlstra 已提交
5579
static void perf_pmu_nop_void(struct pmu *pmu)
5580 5581
{
}
L
Li Zefan 已提交
5582

P
Peter Zijlstra 已提交
5583
static int perf_pmu_nop_int(struct pmu *pmu)
L
Li Zefan 已提交
5584
{
P
Peter Zijlstra 已提交
5585
	return 0;
L
Li Zefan 已提交
5586 5587
}

P
Peter Zijlstra 已提交
5588
static void perf_pmu_start_txn(struct pmu *pmu)
L
Li Zefan 已提交
5589
{
P
Peter Zijlstra 已提交
5590
	perf_pmu_disable(pmu);
L
Li Zefan 已提交
5591 5592
}

P
Peter Zijlstra 已提交
5593 5594 5595 5596 5597
static int perf_pmu_commit_txn(struct pmu *pmu)
{
	perf_pmu_enable(pmu);
	return 0;
}
5598

P
Peter Zijlstra 已提交
5599
static void perf_pmu_cancel_txn(struct pmu *pmu)
5600
{
P
Peter Zijlstra 已提交
5601
	perf_pmu_enable(pmu);
5602 5603
}

5604 5605 5606 5607 5608
static int perf_event_idx_default(struct perf_event *event)
{
	return event->hw.idx + 1;
}

P
Peter Zijlstra 已提交
5609 5610 5611 5612 5613
/*
 * Ensures all contexts with the same task_ctx_nr have the same
 * pmu_cpu_context too.
 */
static void *find_pmu_context(int ctxn)
5614
{
P
Peter Zijlstra 已提交
5615
	struct pmu *pmu;
5616

P
Peter Zijlstra 已提交
5617 5618
	if (ctxn < 0)
		return NULL;
5619

P
Peter Zijlstra 已提交
5620 5621 5622 5623
	list_for_each_entry(pmu, &pmus, entry) {
		if (pmu->task_ctx_nr == ctxn)
			return pmu->pmu_cpu_context;
	}
5624

P
Peter Zijlstra 已提交
5625
	return NULL;
5626 5627
}

5628
static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5629
{
5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644
	int cpu;

	for_each_possible_cpu(cpu) {
		struct perf_cpu_context *cpuctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		if (cpuctx->active_pmu == old_pmu)
			cpuctx->active_pmu = pmu;
	}
}

static void free_pmu_context(struct pmu *pmu)
{
	struct pmu *i;
5645

P
Peter Zijlstra 已提交
5646
	mutex_lock(&pmus_lock);
5647
	/*
P
Peter Zijlstra 已提交
5648
	 * Like a real lame refcount.
5649
	 */
5650 5651 5652
	list_for_each_entry(i, &pmus, entry) {
		if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
			update_pmu_context(i, pmu);
P
Peter Zijlstra 已提交
5653
			goto out;
5654
		}
P
Peter Zijlstra 已提交
5655
	}
5656

5657
	free_percpu(pmu->pmu_cpu_context);
P
Peter Zijlstra 已提交
5658 5659
out:
	mutex_unlock(&pmus_lock);
5660
}
P
Peter Zijlstra 已提交
5661
static struct idr pmu_idr;
5662

P
Peter Zijlstra 已提交
5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694
static ssize_t
type_show(struct device *dev, struct device_attribute *attr, char *page)
{
	struct pmu *pmu = dev_get_drvdata(dev);

	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
}

static struct device_attribute pmu_dev_attrs[] = {
       __ATTR_RO(type),
       __ATTR_NULL,
};

static int pmu_bus_running;
static struct bus_type pmu_bus = {
	.name		= "event_source",
	.dev_attrs	= pmu_dev_attrs,
};

static void pmu_dev_release(struct device *dev)
{
	kfree(dev);
}

static int pmu_dev_alloc(struct pmu *pmu)
{
	int ret = -ENOMEM;

	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
	if (!pmu->dev)
		goto out;

5695
	pmu->dev->groups = pmu->attr_groups;
P
Peter Zijlstra 已提交
5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715
	device_initialize(pmu->dev);
	ret = dev_set_name(pmu->dev, "%s", pmu->name);
	if (ret)
		goto free_dev;

	dev_set_drvdata(pmu->dev, pmu);
	pmu->dev->bus = &pmu_bus;
	pmu->dev->release = pmu_dev_release;
	ret = device_add(pmu->dev);
	if (ret)
		goto free_dev;

out:
	return ret;

free_dev:
	put_device(pmu->dev);
	goto out;
}

5716
static struct lock_class_key cpuctx_mutex;
5717
static struct lock_class_key cpuctx_lock;
5718

P
Peter Zijlstra 已提交
5719
int perf_pmu_register(struct pmu *pmu, char *name, int type)
5720
{
P
Peter Zijlstra 已提交
5721
	int cpu, ret;
5722

5723
	mutex_lock(&pmus_lock);
P
Peter Zijlstra 已提交
5724 5725 5726 5727
	ret = -ENOMEM;
	pmu->pmu_disable_count = alloc_percpu(int);
	if (!pmu->pmu_disable_count)
		goto unlock;
5728

P
Peter Zijlstra 已提交
5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746
	pmu->type = -1;
	if (!name)
		goto skip_type;
	pmu->name = name;

	if (type < 0) {
		int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
		if (!err)
			goto free_pdc;

		err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
		if (err) {
			ret = err;
			goto free_pdc;
		}
	}
	pmu->type = type;

P
Peter Zijlstra 已提交
5747 5748 5749 5750 5751 5752
	if (pmu_bus_running) {
		ret = pmu_dev_alloc(pmu);
		if (ret)
			goto free_idr;
	}

P
Peter Zijlstra 已提交
5753
skip_type:
P
Peter Zijlstra 已提交
5754 5755 5756
	pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
	if (pmu->pmu_cpu_context)
		goto got_cpu_context;
5757

P
Peter Zijlstra 已提交
5758 5759
	pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
	if (!pmu->pmu_cpu_context)
P
Peter Zijlstra 已提交
5760
		goto free_dev;
5761

P
Peter Zijlstra 已提交
5762 5763 5764 5765
	for_each_possible_cpu(cpu) {
		struct perf_cpu_context *cpuctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5766
		__perf_event_init_context(&cpuctx->ctx);
5767
		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5768
		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
5769
		cpuctx->ctx.type = cpu_context;
P
Peter Zijlstra 已提交
5770
		cpuctx->ctx.pmu = pmu;
5771 5772
		cpuctx->jiffies_interval = 1;
		INIT_LIST_HEAD(&cpuctx->rotation_list);
5773
		cpuctx->active_pmu = pmu;
P
Peter Zijlstra 已提交
5774
	}
5775

P
Peter Zijlstra 已提交
5776
got_cpu_context:
P
Peter Zijlstra 已提交
5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790
	if (!pmu->start_txn) {
		if (pmu->pmu_enable) {
			/*
			 * If we have pmu_enable/pmu_disable calls, install
			 * transaction stubs that use that to try and batch
			 * hardware accesses.
			 */
			pmu->start_txn  = perf_pmu_start_txn;
			pmu->commit_txn = perf_pmu_commit_txn;
			pmu->cancel_txn = perf_pmu_cancel_txn;
		} else {
			pmu->start_txn  = perf_pmu_nop_void;
			pmu->commit_txn = perf_pmu_nop_int;
			pmu->cancel_txn = perf_pmu_nop_void;
5791
		}
5792
	}
5793

P
Peter Zijlstra 已提交
5794 5795 5796 5797 5798
	if (!pmu->pmu_enable) {
		pmu->pmu_enable  = perf_pmu_nop_void;
		pmu->pmu_disable = perf_pmu_nop_void;
	}

5799 5800 5801
	if (!pmu->event_idx)
		pmu->event_idx = perf_event_idx_default;

5802
	list_add_rcu(&pmu->entry, &pmus);
P
Peter Zijlstra 已提交
5803 5804
	ret = 0;
unlock:
5805 5806
	mutex_unlock(&pmus_lock);

P
Peter Zijlstra 已提交
5807
	return ret;
P
Peter Zijlstra 已提交
5808

P
Peter Zijlstra 已提交
5809 5810 5811 5812
free_dev:
	device_del(pmu->dev);
	put_device(pmu->dev);

P
Peter Zijlstra 已提交
5813 5814 5815 5816
free_idr:
	if (pmu->type >= PERF_TYPE_MAX)
		idr_remove(&pmu_idr, pmu->type);

P
Peter Zijlstra 已提交
5817 5818 5819
free_pdc:
	free_percpu(pmu->pmu_disable_count);
	goto unlock;
5820 5821
}

5822
void perf_pmu_unregister(struct pmu *pmu)
5823
{
5824 5825 5826
	mutex_lock(&pmus_lock);
	list_del_rcu(&pmu->entry);
	mutex_unlock(&pmus_lock);
5827

5828
	/*
P
Peter Zijlstra 已提交
5829 5830
	 * We dereference the pmu list under both SRCU and regular RCU, so
	 * synchronize against both of those.
5831
	 */
5832
	synchronize_srcu(&pmus_srcu);
P
Peter Zijlstra 已提交
5833
	synchronize_rcu();
5834

P
Peter Zijlstra 已提交
5835
	free_percpu(pmu->pmu_disable_count);
P
Peter Zijlstra 已提交
5836 5837
	if (pmu->type >= PERF_TYPE_MAX)
		idr_remove(&pmu_idr, pmu->type);
P
Peter Zijlstra 已提交
5838 5839
	device_del(pmu->dev);
	put_device(pmu->dev);
5840
	free_pmu_context(pmu);
5841
}
5842

5843 5844 5845 5846
struct pmu *perf_init_event(struct perf_event *event)
{
	struct pmu *pmu = NULL;
	int idx;
5847
	int ret;
5848 5849

	idx = srcu_read_lock(&pmus_srcu);
P
Peter Zijlstra 已提交
5850 5851 5852 5853

	rcu_read_lock();
	pmu = idr_find(&pmu_idr, event->attr.type);
	rcu_read_unlock();
5854
	if (pmu) {
5855
		event->pmu = pmu;
5856 5857 5858
		ret = pmu->event_init(event);
		if (ret)
			pmu = ERR_PTR(ret);
P
Peter Zijlstra 已提交
5859
		goto unlock;
5860
	}
P
Peter Zijlstra 已提交
5861

5862
	list_for_each_entry_rcu(pmu, &pmus, entry) {
5863
		event->pmu = pmu;
5864
		ret = pmu->event_init(event);
5865
		if (!ret)
P
Peter Zijlstra 已提交
5866
			goto unlock;
5867

5868 5869
		if (ret != -ENOENT) {
			pmu = ERR_PTR(ret);
P
Peter Zijlstra 已提交
5870
			goto unlock;
5871
		}
5872
	}
P
Peter Zijlstra 已提交
5873 5874
	pmu = ERR_PTR(-ENOENT);
unlock:
5875
	srcu_read_unlock(&pmus_srcu, idx);
5876

5877
	return pmu;
5878 5879
}

T
Thomas Gleixner 已提交
5880
/*
5881
 * Allocate and initialize a event structure
T
Thomas Gleixner 已提交
5882
 */
5883
static struct perf_event *
5884
perf_event_alloc(struct perf_event_attr *attr, int cpu,
5885 5886 5887
		 struct task_struct *task,
		 struct perf_event *group_leader,
		 struct perf_event *parent_event,
5888 5889
		 perf_overflow_handler_t overflow_handler,
		 void *context)
T
Thomas Gleixner 已提交
5890
{
P
Peter Zijlstra 已提交
5891
	struct pmu *pmu;
5892 5893
	struct perf_event *event;
	struct hw_perf_event *hwc;
5894
	long err;
T
Thomas Gleixner 已提交
5895

5896 5897 5898 5899 5900
	if ((unsigned)cpu >= nr_cpu_ids) {
		if (!task || cpu != -1)
			return ERR_PTR(-EINVAL);
	}

5901
	event = kzalloc(sizeof(*event), GFP_KERNEL);
5902
	if (!event)
5903
		return ERR_PTR(-ENOMEM);
T
Thomas Gleixner 已提交
5904

5905
	/*
5906
	 * Single events are their own group leaders, with an
5907 5908 5909
	 * empty sibling list:
	 */
	if (!group_leader)
5910
		group_leader = event;
5911

5912 5913
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
5914

5915 5916 5917
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
5918 5919
	INIT_LIST_HEAD(&event->rb_entry);

5920
	init_waitqueue_head(&event->waitq);
5921
	init_irq_work(&event->pending, perf_pending_event);
T
Thomas Gleixner 已提交
5922

5923
	mutex_init(&event->mmap_mutex);
5924

5925 5926 5927 5928 5929
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->oncpu		= -1;
5930

5931
	event->parent		= parent_event;
5932

5933 5934
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
5935

5936
	event->state		= PERF_EVENT_STATE_INACTIVE;
5937

5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948
	if (task) {
		event->attach_state = PERF_ATTACH_TASK;
#ifdef CONFIG_HAVE_HW_BREAKPOINT
		/*
		 * hw_breakpoint is a bit difficult here..
		 */
		if (attr->type == PERF_TYPE_BREAKPOINT)
			event->hw.bp_target = task;
#endif
	}

5949
	if (!overflow_handler && parent_event) {
5950
		overflow_handler = parent_event->overflow_handler;
5951 5952
		context = parent_event->overflow_handler_context;
	}
5953

5954
	event->overflow_handler	= overflow_handler;
5955
	event->overflow_handler_context = context;
5956

5957
	if (attr->disabled)
5958
		event->state = PERF_EVENT_STATE_OFF;
5959

5960
	pmu = NULL;
5961

5962
	hwc = &event->hw;
5963
	hwc->sample_period = attr->sample_period;
5964
	if (attr->freq && attr->sample_freq)
5965
		hwc->sample_period = 1;
5966
	hwc->last_period = hwc->sample_period;
5967

5968
	local64_set(&hwc->period_left, hwc->sample_period);
5969

5970
	/*
5971
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
5972
	 */
5973
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5974 5975
		goto done;

5976
	pmu = perf_init_event(event);
5977

5978 5979
done:
	err = 0;
5980
	if (!pmu)
5981
		err = -EINVAL;
5982 5983
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
5984

5985
	if (err) {
5986 5987 5988
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
5989
		return ERR_PTR(err);
I
Ingo Molnar 已提交
5990
	}
5991

5992
	if (!event->parent) {
5993
		if (event->attach_state & PERF_ATTACH_TASK)
5994
			static_key_slow_inc(&perf_sched_events.key);
5995
		if (event->attr.mmap || event->attr.mmap_data)
5996 5997 5998 5999 6000
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
6001 6002 6003 6004 6005 6006 6007
		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
			err = get_callchain_buffers();
			if (err) {
				free_event(event);
				return ERR_PTR(err);
			}
		}
6008 6009 6010 6011 6012 6013
		if (has_branch_stack(event)) {
			static_key_slow_inc(&perf_sched_events.key);
			if (!(event->attach_state & PERF_ATTACH_TASK))
				atomic_inc(&per_cpu(perf_branch_stack_events,
						    event->cpu));
		}
6014
	}
6015

6016
	return event;
T
Thomas Gleixner 已提交
6017 6018
}

6019 6020
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
6021 6022
{
	u32 size;
6023
	int ret;
6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
6048 6049 6050
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
6051 6052
	 */
	if (size > sizeof(*attr)) {
6053 6054 6055
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
6056

6057 6058
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
6059

6060
		for (; addr < end; addr++) {
6061 6062 6063 6064 6065 6066
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
6067
		size = sizeof(*attr);
6068 6069 6070 6071 6072 6073
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

6074
	if (attr->__reserved_1)
6075 6076 6077 6078 6079 6080 6081 6082
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116
	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
		u64 mask = attr->branch_sample_type;

		/* only using defined bits */
		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
			return -EINVAL;

		/* at least one branch bit must be set */
		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
			return -EINVAL;

		/* kernel level capture: check permissions */
		if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
		    && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;

		/* propagate priv level, when not set for branch */
		if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {

			/* exclude_kernel checked on syscall entry */
			if (!attr->exclude_kernel)
				mask |= PERF_SAMPLE_BRANCH_KERNEL;

			if (!attr->exclude_user)
				mask |= PERF_SAMPLE_BRANCH_USER;

			if (!attr->exclude_hv)
				mask |= PERF_SAMPLE_BRANCH_HV;
			/*
			 * adjust user setting (for HW filter setup)
			 */
			attr->branch_sample_type = mask;
		}
	}
6117 6118 6119 6120 6121 6122 6123 6124 6125
out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

6126 6127
static int
perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
6128
{
6129
	struct ring_buffer *rb = NULL, *old_rb = NULL;
6130 6131
	int ret = -EINVAL;

6132
	if (!output_event)
6133 6134
		goto set;

6135 6136
	/* don't allow circular references */
	if (event == output_event)
6137 6138
		goto out;

6139 6140 6141 6142 6143 6144 6145
	/*
	 * Don't allow cross-cpu buffers
	 */
	if (output_event->cpu != event->cpu)
		goto out;

	/*
6146
	 * If its not a per-cpu rb, it must be the same task.
6147 6148 6149 6150
	 */
	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
		goto out;

6151
set:
6152
	mutex_lock(&event->mmap_mutex);
6153 6154 6155
	/* Can't redirect output if we've got an active mmap() */
	if (atomic_read(&event->mmap_count))
		goto unlock;
6156

6157
	if (output_event) {
6158 6159 6160
		/* get the rb we want to redirect to */
		rb = ring_buffer_get(output_event);
		if (!rb)
6161
			goto unlock;
6162 6163
	}

6164 6165
	old_rb = event->rb;
	rcu_assign_pointer(event->rb, rb);
6166 6167
	if (old_rb)
		ring_buffer_detach(event, old_rb);
6168
	ret = 0;
6169 6170 6171
unlock:
	mutex_unlock(&event->mmap_mutex);

6172 6173
	if (old_rb)
		ring_buffer_put(old_rb);
6174 6175 6176 6177
out:
	return ret;
}

T
Thomas Gleixner 已提交
6178
/**
6179
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
I
Ingo Molnar 已提交
6180
 *
6181
 * @attr_uptr:	event_id type attributes for monitoring/sampling
T
Thomas Gleixner 已提交
6182
 * @pid:		target pid
I
Ingo Molnar 已提交
6183
 * @cpu:		target cpu
6184
 * @group_fd:		group leader event fd
T
Thomas Gleixner 已提交
6185
 */
6186 6187
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
6188
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
T
Thomas Gleixner 已提交
6189
{
6190 6191
	struct perf_event *group_leader = NULL, *output_event = NULL;
	struct perf_event *event, *sibling;
6192 6193 6194
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
6195
	struct file *group_file = NULL;
M
Matt Helsley 已提交
6196
	struct task_struct *task = NULL;
6197
	struct pmu *pmu;
6198
	int event_fd;
6199
	int move_group = 0;
6200
	int fput_needed = 0;
6201
	int err;
T
Thomas Gleixner 已提交
6202

6203
	/* for future expandability... */
S
Stephane Eranian 已提交
6204
	if (flags & ~PERF_FLAG_ALL)
6205 6206
		return -EINVAL;

6207 6208 6209
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
6210

6211 6212 6213 6214 6215
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

6216
	if (attr.freq) {
6217
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
6218 6219 6220
			return -EINVAL;
	}

S
Stephane Eranian 已提交
6221 6222 6223 6224 6225 6226 6227 6228 6229
	/*
	 * In cgroup mode, the pid argument is used to pass the fd
	 * opened to the cgroup directory in cgroupfs. The cpu argument
	 * designates the cpu on which to monitor threads from that
	 * cgroup.
	 */
	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
		return -EINVAL;

6230 6231 6232 6233
	event_fd = get_unused_fd_flags(O_RDWR);
	if (event_fd < 0)
		return event_fd;

6234 6235 6236 6237
	if (group_fd != -1) {
		group_leader = perf_fget_light(group_fd, &fput_needed);
		if (IS_ERR(group_leader)) {
			err = PTR_ERR(group_leader);
6238
			goto err_fd;
6239 6240 6241 6242 6243 6244 6245 6246
		}
		group_file = group_leader->filp;
		if (flags & PERF_FLAG_FD_OUTPUT)
			output_event = group_leader;
		if (flags & PERF_FLAG_FD_NO_GROUP)
			group_leader = NULL;
	}

S
Stephane Eranian 已提交
6247
	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
6248 6249 6250 6251 6252 6253 6254
		task = find_lively_task_by_vpid(pid);
		if (IS_ERR(task)) {
			err = PTR_ERR(task);
			goto err_group_fd;
		}
	}

6255 6256
	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
				 NULL, NULL);
6257 6258
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
6259
		goto err_task;
6260 6261
	}

S
Stephane Eranian 已提交
6262 6263 6264 6265
	if (flags & PERF_FLAG_PID_CGROUP) {
		err = perf_cgroup_connect(pid, event, &attr, group_leader);
		if (err)
			goto err_alloc;
6266 6267 6268 6269 6270 6271
		/*
		 * one more event:
		 * - that has cgroup constraint on event->cpu
		 * - that may need work on context switch
		 */
		atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
6272
		static_key_slow_inc(&perf_sched_events.key);
S
Stephane Eranian 已提交
6273 6274
	}

6275 6276 6277 6278 6279
	/*
	 * Special case software events and allow them to be part of
	 * any hardware group.
	 */
	pmu = event->pmu;
6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302

	if (group_leader &&
	    (is_software_event(event) != is_software_event(group_leader))) {
		if (is_software_event(event)) {
			/*
			 * If event and group_leader are not both a software
			 * event, and event is, then group leader is not.
			 *
			 * Allow the addition of software events to !software
			 * groups, this is safe because software events never
			 * fail to schedule.
			 */
			pmu = group_leader->pmu;
		} else if (is_software_event(group_leader) &&
			   (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
			/*
			 * In case the group is a pure software group, and we
			 * try to add a hardware event, move the whole group to
			 * the hardware context.
			 */
			move_group = 1;
		}
	}
6303 6304 6305 6306

	/*
	 * Get the target context (task or percpu):
	 */
M
Matt Helsley 已提交
6307
	ctx = find_get_context(pmu, task, cpu);
6308 6309
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
6310
		goto err_alloc;
6311 6312
	}

6313 6314 6315 6316 6317
	if (task) {
		put_task_struct(task);
		task = NULL;
	}

I
Ingo Molnar 已提交
6318
	/*
6319
	 * Look up the group leader (we will attach this event to it):
6320
	 */
6321
	if (group_leader) {
6322
		err = -EINVAL;
6323 6324

		/*
I
Ingo Molnar 已提交
6325 6326 6327 6328
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
6329
			goto err_context;
I
Ingo Molnar 已提交
6330 6331 6332
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
6333
		 */
6334 6335 6336 6337 6338 6339 6340 6341
		if (move_group) {
			if (group_leader->ctx->type != ctx->type)
				goto err_context;
		} else {
			if (group_leader->ctx != ctx)
				goto err_context;
		}

6342 6343 6344
		/*
		 * Only a group leader can be exclusive or pinned
		 */
6345
		if (attr.exclusive || attr.pinned)
6346
			goto err_context;
6347 6348 6349 6350 6351
	}

	if (output_event) {
		err = perf_event_set_output(event, output_event);
		if (err)
6352
			goto err_context;
6353
	}
T
Thomas Gleixner 已提交
6354

6355 6356 6357
	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
	if (IS_ERR(event_file)) {
		err = PTR_ERR(event_file);
6358
		goto err_context;
6359
	}
6360

6361 6362 6363 6364
	if (move_group) {
		struct perf_event_context *gctx = group_leader->ctx;

		mutex_lock(&gctx->mutex);
6365
		perf_remove_from_context(group_leader);
6366 6367
		list_for_each_entry(sibling, &group_leader->sibling_list,
				    group_entry) {
6368
			perf_remove_from_context(sibling);
6369 6370 6371 6372
			put_ctx(gctx);
		}
		mutex_unlock(&gctx->mutex);
		put_ctx(gctx);
6373
	}
6374

6375
	event->filp = event_file;
6376
	WARN_ON_ONCE(ctx->parent_ctx);
6377
	mutex_lock(&ctx->mutex);
6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388

	if (move_group) {
		perf_install_in_context(ctx, group_leader, cpu);
		get_ctx(ctx);
		list_for_each_entry(sibling, &group_leader->sibling_list,
				    group_entry) {
			perf_install_in_context(ctx, sibling, cpu);
			get_ctx(ctx);
		}
	}

6389
	perf_install_in_context(ctx, event, cpu);
6390
	++ctx->generation;
6391
	perf_unpin_context(ctx);
6392
	mutex_unlock(&ctx->mutex);
6393

6394
	event->owner = current;
P
Peter Zijlstra 已提交
6395

6396 6397 6398
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
6399

6400 6401 6402 6403
	/*
	 * Precalculate sample_data sizes
	 */
	perf_event__header_size(event);
6404
	perf_event__id_header_size(event);
6405

6406 6407 6408 6409 6410 6411
	/*
	 * Drop the reference on the group_event after placing the
	 * new event on the sibling_list. This ensures destruction
	 * of the group leader will find the pointer to itself in
	 * perf_group_detach().
	 */
6412 6413 6414
	fput_light(group_file, fput_needed);
	fd_install(event_fd, event_file);
	return event_fd;
T
Thomas Gleixner 已提交
6415

6416
err_context:
6417
	perf_unpin_context(ctx);
6418
	put_ctx(ctx);
6419
err_alloc:
6420
	free_event(event);
P
Peter Zijlstra 已提交
6421 6422 6423
err_task:
	if (task)
		put_task_struct(task);
6424
err_group_fd:
6425
	fput_light(group_file, fput_needed);
6426 6427
err_fd:
	put_unused_fd(event_fd);
6428
	return err;
T
Thomas Gleixner 已提交
6429 6430
}

6431 6432 6433 6434 6435
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
M
Matt Helsley 已提交
6436
 * @task: task to profile (NULL for percpu)
6437 6438 6439
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
M
Matt Helsley 已提交
6440
				 struct task_struct *task,
6441 6442
				 perf_overflow_handler_t overflow_handler,
				 void *context)
6443 6444
{
	struct perf_event_context *ctx;
6445
	struct perf_event *event;
6446
	int err;
6447

6448 6449 6450
	/*
	 * Get the target context (task or percpu):
	 */
6451

6452 6453
	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
				 overflow_handler, context);
6454 6455 6456 6457
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
		goto err;
	}
6458

M
Matt Helsley 已提交
6459
	ctx = find_get_context(event->pmu, task, cpu);
6460 6461
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
6462
		goto err_free;
6463
	}
6464 6465 6466 6467 6468 6469

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
6470
	perf_unpin_context(ctx);
6471 6472 6473 6474
	mutex_unlock(&ctx->mutex);

	return event;

6475 6476 6477
err_free:
	free_event(event);
err:
6478
	return ERR_PTR(err);
6479
}
6480
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6481

6482
static void sync_child_event(struct perf_event *child_event,
6483
			       struct task_struct *child)
6484
{
6485
	struct perf_event *parent_event = child_event->parent;
6486
	u64 child_val;
6487

6488 6489
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
6490

P
Peter Zijlstra 已提交
6491
	child_val = perf_event_count(child_event);
6492 6493 6494 6495

	/*
	 * Add back the child's count to the parent's count:
	 */
6496
	atomic64_add(child_val, &parent_event->child_count);
6497 6498 6499 6500
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
6501 6502

	/*
6503
	 * Remove this event from the parent's list
6504
	 */
6505 6506 6507 6508
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
6509 6510

	/*
6511
	 * Release the parent event, if this was the last
6512 6513
	 * reference to it.
	 */
6514
	fput(parent_event->filp);
6515 6516
}

6517
static void
6518 6519
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
6520
			 struct task_struct *child)
6521
{
6522 6523 6524 6525 6526
	if (child_event->parent) {
		raw_spin_lock_irq(&child_ctx->lock);
		perf_group_detach(child_event);
		raw_spin_unlock_irq(&child_ctx->lock);
	}
6527

6528
	perf_remove_from_context(child_event);
6529

6530
	/*
6531
	 * It can happen that the parent exits first, and has events
6532
	 * that are still around due to the child reference. These
6533
	 * events need to be zapped.
6534
	 */
6535
	if (child_event->parent) {
6536 6537
		sync_child_event(child_event, child);
		free_event(child_event);
6538
	}
6539 6540
}

P
Peter Zijlstra 已提交
6541
static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6542
{
6543 6544
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
6545
	unsigned long flags;
6546

P
Peter Zijlstra 已提交
6547
	if (likely(!child->perf_event_ctxp[ctxn])) {
6548
		perf_event_task(child, NULL, 0);
6549
		return;
P
Peter Zijlstra 已提交
6550
	}
6551

6552
	local_irq_save(flags);
6553 6554 6555 6556 6557 6558
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
6559
	child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6560 6561 6562

	/*
	 * Take the context lock here so that if find_get_context is
6563
	 * reading child->perf_event_ctxp, we wait until it has
6564 6565
	 * incremented the context's refcount before we do put_ctx below.
	 */
6566
	raw_spin_lock(&child_ctx->lock);
6567
	task_ctx_sched_out(child_ctx);
P
Peter Zijlstra 已提交
6568
	child->perf_event_ctxp[ctxn] = NULL;
6569 6570 6571
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
6572
	 * the events from it.
6573 6574
	 */
	unclone_ctx(child_ctx);
6575
	update_context_time(child_ctx);
6576
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
6577 6578

	/*
6579 6580 6581
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
P
Peter Zijlstra 已提交
6582
	 */
6583
	perf_event_task(child, child_ctx, 0);
6584

6585 6586 6587
	/*
	 * We can recurse on the same lock type through:
	 *
6588 6589 6590
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
6591 6592 6593 6594 6595
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
6596
	mutex_lock(&child_ctx->mutex);
6597

6598
again:
6599 6600 6601 6602 6603
	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
				 group_entry)
		__perf_event_exit_task(child_event, child_ctx, child);

	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6604
				 group_entry)
6605
		__perf_event_exit_task(child_event, child_ctx, child);
6606 6607

	/*
6608
	 * If the last event was a group event, it will have appended all
6609 6610 6611
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
6612 6613
	if (!list_empty(&child_ctx->pinned_groups) ||
	    !list_empty(&child_ctx->flexible_groups))
6614
		goto again;
6615 6616 6617 6618

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
6619 6620
}

P
Peter Zijlstra 已提交
6621 6622 6623 6624 6625
/*
 * When a child task exits, feed back event values to parent events.
 */
void perf_event_exit_task(struct task_struct *child)
{
P
Peter Zijlstra 已提交
6626
	struct perf_event *event, *tmp;
P
Peter Zijlstra 已提交
6627 6628
	int ctxn;

P
Peter Zijlstra 已提交
6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643
	mutex_lock(&child->perf_event_mutex);
	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
				 owner_entry) {
		list_del_init(&event->owner_entry);

		/*
		 * Ensure the list deletion is visible before we clear
		 * the owner, closes a race against perf_release() where
		 * we need to serialize on the owner->perf_event_mutex.
		 */
		smp_wmb();
		event->owner = NULL;
	}
	mutex_unlock(&child->perf_event_mutex);

P
Peter Zijlstra 已提交
6644 6645 6646 6647
	for_each_task_context_nr(ctxn)
		perf_event_exit_task_context(child, ctxn);
}

6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661
static void perf_free_event(struct perf_event *event,
			    struct perf_event_context *ctx)
{
	struct perf_event *parent = event->parent;

	if (WARN_ON_ONCE(!parent))
		return;

	mutex_lock(&parent->child_mutex);
	list_del_init(&event->child_list);
	mutex_unlock(&parent->child_mutex);

	fput(parent->filp);

6662
	perf_group_detach(event);
6663 6664 6665 6666
	list_del_event(event, ctx);
	free_event(event);
}

6667 6668
/*
 * free an unexposed, unused context as created by inheritance by
P
Peter Zijlstra 已提交
6669
 * perf_event_init_task below, used by fork() in case of fail.
6670
 */
6671
void perf_event_free_task(struct task_struct *task)
6672
{
P
Peter Zijlstra 已提交
6673
	struct perf_event_context *ctx;
6674
	struct perf_event *event, *tmp;
P
Peter Zijlstra 已提交
6675
	int ctxn;
6676

P
Peter Zijlstra 已提交
6677 6678 6679 6680
	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (!ctx)
			continue;
6681

P
Peter Zijlstra 已提交
6682
		mutex_lock(&ctx->mutex);
6683
again:
P
Peter Zijlstra 已提交
6684 6685 6686
		list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
				group_entry)
			perf_free_event(event, ctx);
6687

P
Peter Zijlstra 已提交
6688 6689 6690
		list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
				group_entry)
			perf_free_event(event, ctx);
6691

P
Peter Zijlstra 已提交
6692 6693 6694
		if (!list_empty(&ctx->pinned_groups) ||
				!list_empty(&ctx->flexible_groups))
			goto again;
6695

P
Peter Zijlstra 已提交
6696
		mutex_unlock(&ctx->mutex);
6697

P
Peter Zijlstra 已提交
6698 6699
		put_ctx(ctx);
	}
6700 6701
}

6702 6703 6704 6705 6706 6707 6708 6709
void perf_event_delayed_put(struct task_struct *task)
{
	int ctxn;

	for_each_task_context_nr(ctxn)
		WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
}

P
Peter Zijlstra 已提交
6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721
/*
 * inherit a event from parent task to child task:
 */
static struct perf_event *
inherit_event(struct perf_event *parent_event,
	      struct task_struct *parent,
	      struct perf_event_context *parent_ctx,
	      struct task_struct *child,
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
{
	struct perf_event *child_event;
6722
	unsigned long flags;
P
Peter Zijlstra 已提交
6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734

	/*
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
	if (parent_event->parent)
		parent_event = parent_event->parent;

	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu,
6735
					   child,
P
Peter Zijlstra 已提交
6736
					   group_leader, parent_event,
6737
				           NULL, NULL);
P
Peter Zijlstra 已提交
6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763
	if (IS_ERR(child_event))
		return child_event;
	get_ctx(child_ctx);

	/*
	 * Make the child state follow the state of the parent event,
	 * not its attr.disabled bit.  We hold the parent's mutex,
	 * so we won't race with perf_event_{en, dis}able_family.
	 */
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
	else
		child_event->state = PERF_EVENT_STATE_OFF;

	if (parent_event->attr.freq) {
		u64 sample_period = parent_event->hw.sample_period;
		struct hw_perf_event *hwc = &child_event->hw;

		hwc->sample_period = sample_period;
		hwc->last_period   = sample_period;

		local64_set(&hwc->period_left, sample_period);
	}

	child_event->ctx = child_ctx;
	child_event->overflow_handler = parent_event->overflow_handler;
6764 6765
	child_event->overflow_handler_context
		= parent_event->overflow_handler_context;
P
Peter Zijlstra 已提交
6766

6767 6768 6769 6770
	/*
	 * Precalculate sample_data sizes
	 */
	perf_event__header_size(child_event);
6771
	perf_event__id_header_size(child_event);
6772

P
Peter Zijlstra 已提交
6773 6774 6775
	/*
	 * Link it up in the child's context:
	 */
6776
	raw_spin_lock_irqsave(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
6777
	add_event_to_ctx(child_event, child_ctx);
6778
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
P
Peter Zijlstra 已提交
6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819

	/*
	 * Get a reference to the parent filp - we will fput it
	 * when the child event exits. This is safe to do because
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
	atomic_long_inc(&parent_event->filp->f_count);

	/*
	 * Link this into the parent event's child list
	 */
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);

	return child_event;
}

static int inherit_group(struct perf_event *parent_event,
	      struct task_struct *parent,
	      struct perf_event_context *parent_ctx,
	      struct task_struct *child,
	      struct perf_event_context *child_ctx)
{
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;

	leader = inherit_event(parent_event, parent, parent_ctx,
				 child, NULL, child_ctx);
	if (IS_ERR(leader))
		return PTR_ERR(leader);
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
	}
	return 0;
6820 6821 6822 6823 6824
}

static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
		   struct perf_event_context *parent_ctx,
P
Peter Zijlstra 已提交
6825
		   struct task_struct *child, int ctxn,
6826 6827 6828
		   int *inherited_all)
{
	int ret;
P
Peter Zijlstra 已提交
6829
	struct perf_event_context *child_ctx;
6830 6831 6832 6833

	if (!event->attr.inherit) {
		*inherited_all = 0;
		return 0;
6834 6835
	}

6836
	child_ctx = child->perf_event_ctxp[ctxn];
6837 6838 6839 6840 6841 6842 6843
	if (!child_ctx) {
		/*
		 * This is executed from the parent task context, so
		 * inherit events that have been marked for cloning.
		 * First allocate and initialize a context for the
		 * child.
		 */
6844

6845
		child_ctx = alloc_perf_context(event->pmu, child);
6846 6847
		if (!child_ctx)
			return -ENOMEM;
6848

P
Peter Zijlstra 已提交
6849
		child->perf_event_ctxp[ctxn] = child_ctx;
6850 6851 6852 6853 6854 6855 6856 6857 6858
	}

	ret = inherit_group(event, parent, parent_ctx,
			    child, child_ctx);

	if (ret)
		*inherited_all = 0;

	return ret;
6859 6860
}

6861
/*
6862
 * Initialize the perf_event context in task_struct
6863
 */
P
Peter Zijlstra 已提交
6864
int perf_event_init_context(struct task_struct *child, int ctxn)
6865
{
6866
	struct perf_event_context *child_ctx, *parent_ctx;
6867 6868
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
6869
	struct task_struct *parent = current;
6870
	int inherited_all = 1;
6871
	unsigned long flags;
6872
	int ret = 0;
6873

P
Peter Zijlstra 已提交
6874
	if (likely(!parent->perf_event_ctxp[ctxn]))
6875 6876
		return 0;

6877
	/*
6878 6879
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
6880
	 */
P
Peter Zijlstra 已提交
6881
	parent_ctx = perf_pin_task_context(parent, ctxn);
6882

6883 6884 6885 6886 6887 6888 6889
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

6890 6891 6892 6893
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
6894
	mutex_lock(&parent_ctx->mutex);
6895 6896 6897 6898 6899

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
6900
	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
P
Peter Zijlstra 已提交
6901 6902
		ret = inherit_task_group(event, parent, parent_ctx,
					 child, ctxn, &inherited_all);
6903 6904 6905
		if (ret)
			break;
	}
6906

6907 6908 6909 6910 6911 6912 6913 6914 6915
	/*
	 * We can't hold ctx->lock when iterating the ->flexible_group list due
	 * to allocations, but we need to prevent rotation because
	 * rotate_ctx() will change the list from interrupt context.
	 */
	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
	parent_ctx->rotate_disable = 1;
	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);

6916
	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
P
Peter Zijlstra 已提交
6917 6918
		ret = inherit_task_group(event, parent, parent_ctx,
					 child, ctxn, &inherited_all);
6919
		if (ret)
6920
			break;
6921 6922
	}

6923 6924 6925
	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
	parent_ctx->rotate_disable = 0;

P
Peter Zijlstra 已提交
6926
	child_ctx = child->perf_event_ctxp[ctxn];
6927

6928
	if (child_ctx && inherited_all) {
6929 6930 6931
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
P
Peter Zijlstra 已提交
6932 6933 6934
		 *
		 * Note that if the parent is a clone, the holding of
		 * parent_ctx->lock avoids it from being uncloned.
6935
		 */
P
Peter Zijlstra 已提交
6936
		cloned_ctx = parent_ctx->parent_ctx;
6937 6938
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
6939
			child_ctx->parent_gen = parent_ctx->parent_gen;
6940 6941 6942 6943 6944
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
6945 6946
	}

P
Peter Zijlstra 已提交
6947
	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6948
	mutex_unlock(&parent_ctx->mutex);
6949

6950
	perf_unpin_context(parent_ctx);
6951
	put_ctx(parent_ctx);
6952

6953
	return ret;
6954 6955
}

P
Peter Zijlstra 已提交
6956 6957 6958 6959 6960 6961 6962
/*
 * Initialize the perf_event context in task_struct
 */
int perf_event_init_task(struct task_struct *child)
{
	int ctxn, ret;

6963 6964 6965 6966
	memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);

P
Peter Zijlstra 已提交
6967 6968 6969 6970 6971 6972 6973 6974 6975
	for_each_task_context_nr(ctxn) {
		ret = perf_event_init_context(child, ctxn);
		if (ret)
			return ret;
	}

	return 0;
}

6976 6977
static void __init perf_event_init_all_cpus(void)
{
6978
	struct swevent_htable *swhash;
6979 6980 6981
	int cpu;

	for_each_possible_cpu(cpu) {
6982 6983
		swhash = &per_cpu(swevent_htable, cpu);
		mutex_init(&swhash->hlist_mutex);
6984
		INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
6985 6986 6987
	}
}

6988
static void __cpuinit perf_event_init_cpu(int cpu)
T
Thomas Gleixner 已提交
6989
{
P
Peter Zijlstra 已提交
6990
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
T
Thomas Gleixner 已提交
6991

6992
	mutex_lock(&swhash->hlist_mutex);
6993
	if (swhash->hlist_refcount > 0) {
6994 6995
		struct swevent_hlist *hlist;

6996 6997 6998
		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
		WARN_ON(!hlist);
		rcu_assign_pointer(swhash->swevent_hlist, hlist);
6999
	}
7000
	mutex_unlock(&swhash->hlist_mutex);
T
Thomas Gleixner 已提交
7001 7002
}

P
Peter Zijlstra 已提交
7003
#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
7004
static void perf_pmu_rotate_stop(struct pmu *pmu)
T
Thomas Gleixner 已提交
7005
{
7006 7007 7008 7009 7010 7011 7012
	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

	WARN_ON(!irqs_disabled());

	list_del_init(&cpuctx->rotation_list);
}

P
Peter Zijlstra 已提交
7013
static void __perf_event_exit_context(void *__info)
T
Thomas Gleixner 已提交
7014
{
P
Peter Zijlstra 已提交
7015
	struct perf_event_context *ctx = __info;
7016
	struct perf_event *event, *tmp;
T
Thomas Gleixner 已提交
7017

P
Peter Zijlstra 已提交
7018
	perf_pmu_rotate_stop(ctx->pmu);
7019

7020
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
7021
		__perf_remove_from_context(event);
7022
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
7023
		__perf_remove_from_context(event);
T
Thomas Gleixner 已提交
7024
}
P
Peter Zijlstra 已提交
7025 7026 7027 7028 7029 7030 7031 7032 7033

static void perf_event_exit_cpu_context(int cpu)
{
	struct perf_event_context *ctx;
	struct pmu *pmu;
	int idx;

	idx = srcu_read_lock(&pmus_srcu);
	list_for_each_entry_rcu(pmu, &pmus, entry) {
7034
		ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
P
Peter Zijlstra 已提交
7035 7036 7037 7038 7039 7040 7041 7042

		mutex_lock(&ctx->mutex);
		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
		mutex_unlock(&ctx->mutex);
	}
	srcu_read_unlock(&pmus_srcu, idx);
}

7043
static void perf_event_exit_cpu(int cpu)
T
Thomas Gleixner 已提交
7044
{
7045
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
7046

7047 7048 7049
	mutex_lock(&swhash->hlist_mutex);
	swevent_hlist_release(swhash);
	mutex_unlock(&swhash->hlist_mutex);
7050

P
Peter Zijlstra 已提交
7051
	perf_event_exit_cpu_context(cpu);
T
Thomas Gleixner 已提交
7052 7053
}
#else
7054
static inline void perf_event_exit_cpu(int cpu) { }
T
Thomas Gleixner 已提交
7055 7056
#endif

P
Peter Zijlstra 已提交
7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076
static int
perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
{
	int cpu;

	for_each_online_cpu(cpu)
		perf_event_exit_cpu(cpu);

	return NOTIFY_OK;
}

/*
 * Run the perf reboot notifier at the very last possible moment so that
 * the generic watchdog code runs as long as possible.
 */
static struct notifier_block perf_reboot_notifier = {
	.notifier_call = perf_reboot,
	.priority = INT_MIN,
};

T
Thomas Gleixner 已提交
7077 7078 7079 7080 7081
static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

7082
	switch (action & ~CPU_TASKS_FROZEN) {
T
Thomas Gleixner 已提交
7083 7084

	case CPU_UP_PREPARE:
P
Peter Zijlstra 已提交
7085
	case CPU_DOWN_FAILED:
7086
		perf_event_init_cpu(cpu);
T
Thomas Gleixner 已提交
7087 7088
		break;

P
Peter Zijlstra 已提交
7089
	case CPU_UP_CANCELED:
T
Thomas Gleixner 已提交
7090
	case CPU_DOWN_PREPARE:
7091
		perf_event_exit_cpu(cpu);
T
Thomas Gleixner 已提交
7092 7093 7094 7095 7096 7097 7098 7099 7100
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

7101
void __init perf_event_init(void)
T
Thomas Gleixner 已提交
7102
{
7103 7104
	int ret;

P
Peter Zijlstra 已提交
7105 7106
	idr_init(&pmu_idr);

7107
	perf_event_init_all_cpus();
7108
	init_srcu_struct(&pmus_srcu);
P
Peter Zijlstra 已提交
7109 7110 7111
	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
	perf_pmu_register(&perf_cpu_clock, NULL, -1);
	perf_pmu_register(&perf_task_clock, NULL, -1);
7112 7113
	perf_tp_register();
	perf_cpu_notifier(perf_cpu_notify);
P
Peter Zijlstra 已提交
7114
	register_reboot_notifier(&perf_reboot_notifier);
7115 7116 7117

	ret = init_hw_breakpoint();
	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
7118 7119 7120

	/* do not patch jump label more than once per second */
	jump_label_rate_limit(&perf_sched_events, HZ);
7121 7122 7123 7124 7125 7126 7127

	/*
	 * Build time assertion that we keep the data_head at the intended
	 * location.  IOW, validation we got the __reserved[] size right.
	 */
	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
		     != 1024);
T
Thomas Gleixner 已提交
7128
}
P
Peter Zijlstra 已提交
7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156

static int __init perf_event_sysfs_init(void)
{
	struct pmu *pmu;
	int ret;

	mutex_lock(&pmus_lock);

	ret = bus_register(&pmu_bus);
	if (ret)
		goto unlock;

	list_for_each_entry(pmu, &pmus, entry) {
		if (!pmu->name || pmu->type < 0)
			continue;

		ret = pmu_dev_alloc(pmu);
		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
	}
	pmu_bus_running = 1;
	ret = 0;

unlock:
	mutex_unlock(&pmus_lock);

	return ret;
}
device_initcall(perf_event_sysfs_init);
S
Stephane Eranian 已提交
7157 7158

#ifdef CONFIG_CGROUP_PERF
7159
static struct cgroup_subsys_state *perf_cgroup_create(struct cgroup *cont)
S
Stephane Eranian 已提交
7160 7161 7162
{
	struct perf_cgroup *jc;

7163
	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
S
Stephane Eranian 已提交
7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175
	if (!jc)
		return ERR_PTR(-ENOMEM);

	jc->info = alloc_percpu(struct perf_cgroup_info);
	if (!jc->info) {
		kfree(jc);
		return ERR_PTR(-ENOMEM);
	}

	return &jc->css;
}

7176
static void perf_cgroup_destroy(struct cgroup *cont)
S
Stephane Eranian 已提交
7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191
{
	struct perf_cgroup *jc;
	jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
			  struct perf_cgroup, css);
	free_percpu(jc->info);
	kfree(jc);
}

static int __perf_cgroup_move(void *info)
{
	struct task_struct *task = info;
	perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
	return 0;
}

7192
static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
S
Stephane Eranian 已提交
7193
{
7194 7195 7196 7197
	struct task_struct *task;

	cgroup_taskset_for_each(task, cgrp, tset)
		task_function_call(task, __perf_cgroup_move, task);
S
Stephane Eranian 已提交
7198 7199
}

7200 7201
static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
			     struct task_struct *task)
S
Stephane Eranian 已提交
7202 7203 7204 7205 7206 7207 7208 7209 7210
{
	/*
	 * cgroup_exit() is called in the copy_process() failure path.
	 * Ignore this case since the task hasn't ran yet, this avoids
	 * trying to poke a half freed task state from generic code.
	 */
	if (!(task->flags & PF_EXITING))
		return;

7211
	task_function_call(task, __perf_cgroup_move, task);
S
Stephane Eranian 已提交
7212 7213 7214
}

struct cgroup_subsys perf_subsys = {
7215 7216 7217 7218 7219
	.name		= "perf_event",
	.subsys_id	= perf_subsys_id,
	.create		= perf_cgroup_create,
	.destroy	= perf_cgroup_destroy,
	.exit		= perf_cgroup_exit,
7220
	.attach		= perf_cgroup_attach,
S
Stephane Eranian 已提交
7221 7222
};
#endif /* CONFIG_CGROUP_PERF */