perf_event.h 38.3 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events:
T
Thomas Gleixner 已提交
3
 *
I
Ingo Molnar 已提交
4
 *    Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5 6
 *    Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
 *    Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
T
Thomas Gleixner 已提交
7
 *
I
Ingo Molnar 已提交
8
 * Data type definitions, declarations, prototypes.
T
Thomas Gleixner 已提交
9
 *
I
Ingo Molnar 已提交
10
 *    Started by: Thomas Gleixner and Ingo Molnar
T
Thomas Gleixner 已提交
11
 *
I
Ingo Molnar 已提交
12
 * For licencing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
13
 */
14 15
#ifndef _LINUX_PERF_EVENT_H
#define _LINUX_PERF_EVENT_H
T
Thomas Gleixner 已提交
16

17
#include <uapi/linux/perf_event.h>
T
Thomas Gleixner 已提交
18

I
Ingo Molnar 已提交
19
/*
20
 * Kernel-internal data types and definitions:
I
Ingo Molnar 已提交
21 22
 */

23 24
#ifdef CONFIG_PERF_EVENTS
# include <asm/perf_event.h>
25
# include <asm/local64.h>
26 27
#endif

28
struct perf_guest_info_callbacks {
29 30 31
	int				(*is_in_guest)(void);
	int				(*is_user_mode)(void);
	unsigned long			(*get_guest_ip)(void);
32 33
};

34 35 36 37
#ifdef CONFIG_HAVE_HW_BREAKPOINT
#include <asm/hw_breakpoint.h>
#endif

38 39 40 41 42
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
43
#include <linux/hrtimer.h>
P
Peter Zijlstra 已提交
44
#include <linux/fs.h>
45
#include <linux/pid_namespace.h>
46
#include <linux/workqueue.h>
47
#include <linux/ftrace.h>
48
#include <linux/cpu.h>
49
#include <linux/irq_work.h>
50
#include <linux/static_key.h>
51
#include <linux/jump_label_ratelimit.h>
A
Arun Sharma 已提交
52
#include <linux/atomic.h>
53
#include <linux/sysfs.h>
54
#include <linux/perf_regs.h>
55
#include <linux/workqueue.h>
56
#include <linux/cgroup.h>
57
#include <asm/local.h>
58

59 60
struct perf_callchain_entry {
	__u64				nr;
61
	__u64				ip[0]; /* /proc/sys/kernel/perf_event_max_stack */
62 63
};

64 65 66
struct perf_callchain_entry_ctx {
	struct perf_callchain_entry *entry;
	u32			    max_stack;
67
	u32			    nr;
68 69
	short			    contexts;
	bool			    contexts_maxed;
70 71
};

72 73 74
struct perf_raw_record {
	u32				size;
	void				*data;
75 76
};

77 78 79 80 81 82 83 84 85
/*
 * branch stack layout:
 *  nr: number of taken branches stored in entries[]
 *
 * Note that nr can vary from sample to sample
 * branches (to, from) are stored from most recent
 * to least recent, i.e., entries[0] contains the most
 * recent branch.
 */
86 87 88 89 90
struct perf_branch_stack {
	__u64				nr;
	struct perf_branch_entry	entries[0];
};

91 92
struct task_struct;

93 94 95 96 97 98 99 100 101 102
/*
 * extra PMU register associated with an event
 */
struct hw_perf_event_extra {
	u64		config;	/* register value */
	unsigned int	reg;	/* register address or index */
	int		alloc;	/* extra register already allocated */
	int		idx;	/* index in shared_regs->regs[] */
};

T
Thomas Gleixner 已提交
103
/**
104
 * struct hw_perf_event - performance event hardware details:
T
Thomas Gleixner 已提交
105
 */
106 107
struct hw_perf_event {
#ifdef CONFIG_PERF_EVENTS
108 109
	union {
		struct { /* hardware */
I
Ingo Molnar 已提交
110
			u64		config;
111
			u64		last_tag;
I
Ingo Molnar 已提交
112
			unsigned long	config_base;
113
			unsigned long	event_base;
114
			int		event_base_rdpmc;
I
Ingo Molnar 已提交
115
			int		idx;
116
			int		last_cpu;
117
			int		flags;
118

119
			struct hw_perf_event_extra extra_reg;
120
			struct hw_perf_event_extra branch_reg;
121
		};
122
		struct { /* software */
I
Ingo Molnar 已提交
123
			struct hrtimer	hrtimer;
124
		};
125 126 127 128
		struct { /* tracepoint */
			/* for tp_event->class */
			struct list_head	tp_list;
		};
129 130
		struct { /* intel_cqm */
			int			cqm_state;
131
			u32			cqm_rmid;
132
			int			is_group_event;
133 134 135 136
			struct list_head	cqm_events_entry;
			struct list_head	cqm_groups_entry;
			struct list_head	cqm_group_entry;
		};
137 138 139
		struct { /* itrace */
			int			itrace_started;
		};
140 141 142 143
		struct { /* amd_power */
			u64	pwr_acc;
			u64	ptsc;
		};
144
#ifdef CONFIG_HAVE_HW_BREAKPOINT
145
		struct { /* breakpoint */
146 147 148 149 150
			/*
			 * Crufty hack to avoid the chicken and egg
			 * problem hw_breakpoint has with context
			 * creation and event initalization.
			 */
151 152
			struct arch_hw_breakpoint	info;
			struct list_head		bp_list;
153
		};
154
#endif
155
	};
156 157 158 159
	/*
	 * If the event is a per task event, this will point to the task in
	 * question. See the comment in perf_event_alloc().
	 */
160
	struct task_struct		*target;
161

162 163 164 165 166 167 168 169 170
	/*
	 * PMU would store hardware filter configuration
	 * here.
	 */
	void				*addr_filters;

	/* Last sync'ed generation of filters */
	unsigned long			addr_filters_gen;

171 172 173 174 175 176 177
/*
 * hw_perf_event::state flags; used to track the PERF_EF_* state.
 */
#define PERF_HES_STOPPED	0x01 /* the counter is stopped */
#define PERF_HES_UPTODATE	0x02 /* event->count up-to-date */
#define PERF_HES_ARCH		0x04

P
Peter Zijlstra 已提交
178
	int				state;
179 180 181 182 183

	/*
	 * The last observed hardware counter value, updated with a
	 * local64_cmpxchg() such that pmu::read() can be called nested.
	 */
184
	local64_t			prev_count;
185 186 187 188

	/*
	 * The period to start the next sample with.
	 */
189
	u64				sample_period;
190 191 192 193

	/*
	 * The period we started this sample with.
	 */
194
	u64				last_period;
195 196 197 198 199 200

	/*
	 * However much is left of the current period; note that this is
	 * a full 64bit value and allows for generation of periods longer
	 * than hardware might allow.
	 */
201
	local64_t			period_left;
202 203 204 205 206

	/*
	 * State for throttling the event, see __perf_event_overflow() and
	 * perf_adjust_freq_unthr_context().
	 */
207
	u64                             interrupts_seq;
208
	u64				interrupts;
209

210 211 212 213
	/*
	 * State for freq target events, see __perf_event_overflow() and
	 * perf_adjust_freq_unthr_context().
	 */
214 215
	u64				freq_time_stamp;
	u64				freq_count_stamp;
216
#endif
T
Thomas Gleixner 已提交
217 218
};

219
struct perf_event;
I
Ingo Molnar 已提交
220

221 222 223
/*
 * Common implementation detail of pmu::{start,commit,cancel}_txn
 */
224
#define PERF_PMU_TXN_ADD  0x1		/* txn to add/schedule event on PMU */
225
#define PERF_PMU_TXN_READ 0x2		/* txn to read event group from PMU */
226

227 228 229 230
/**
 * pmu::capabilities flags
 */
#define PERF_PMU_CAP_NO_INTERRUPT		0x01
231
#define PERF_PMU_CAP_NO_NMI			0x02
232
#define PERF_PMU_CAP_AUX_NO_SG			0x04
233
#define PERF_PMU_CAP_AUX_SW_DOUBLEBUF		0x08
234
#define PERF_PMU_CAP_EXCLUSIVE			0x10
235
#define PERF_PMU_CAP_ITRACE			0x20
236
#define PERF_PMU_CAP_HETEROGENEOUS_CPUS		0x40
237

I
Ingo Molnar 已提交
238
/**
239
 * struct pmu - generic performance monitoring unit
I
Ingo Molnar 已提交
240
 */
241
struct pmu {
242 243
	struct list_head		entry;

244
	struct module			*module;
P
Peter Zijlstra 已提交
245
	struct device			*dev;
246
	const struct attribute_group	**attr_groups;
247
	const char			*name;
P
Peter Zijlstra 已提交
248 249
	int				type;

250 251 252 253 254
	/*
	 * various common per-pmu feature flags
	 */
	int				capabilities;

P
Peter Zijlstra 已提交
255 256
	int * __percpu			pmu_disable_count;
	struct perf_cpu_context * __percpu pmu_cpu_context;
257
	atomic_t			exclusive_cnt; /* < 0: cpu; > 0: tsk */
P
Peter Zijlstra 已提交
258
	int				task_ctx_nr;
259
	int				hrtimer_interval_ms;
260

261 262 263
	/* number of address filters this PMU can do */
	unsigned int			nr_addr_filters;

264
	/*
P
Peter Zijlstra 已提交
265 266
	 * Fully disable/enable this PMU, can be used to protect from the PMI
	 * as well as for lazy/batch writing of the MSRs.
267
	 */
P
Peter Zijlstra 已提交
268 269
	void (*pmu_enable)		(struct pmu *pmu); /* optional */
	void (*pmu_disable)		(struct pmu *pmu); /* optional */
270

271
	/*
P
Peter Zijlstra 已提交
272
	 * Try and initialize the event for this PMU.
273 274 275 276 277 278 279 280 281 282 283 284 285
	 *
	 * Returns:
	 *  -ENOENT	-- @event is not for this PMU
	 *
	 *  -ENODEV	-- @event is for this PMU but PMU not present
	 *  -EBUSY	-- @event is for this PMU but PMU temporarily unavailable
	 *  -EINVAL	-- @event is for this PMU but @event is not valid
	 *  -EOPNOTSUPP -- @event is for this PMU, @event is valid, but not supported
	 *  -EACCESS	-- @event is for this PMU, @event is valid, but no privilidges
	 *
	 *  0		-- @event is for this PMU and valid
	 *
	 * Other error return values are allowed.
286
	 */
287 288
	int (*event_init)		(struct perf_event *event);

289 290 291 292 293 294 295
	/*
	 * Notification that the event was mapped or unmapped.  Called
	 * in the context of the mapping task.
	 */
	void (*event_mapped)		(struct perf_event *event); /*optional*/
	void (*event_unmapped)		(struct perf_event *event); /*optional*/

296 297 298 299
	/*
	 * Flags for ->add()/->del()/ ->start()/->stop(). There are
	 * matching hw_perf_event::state flags.
	 */
P
Peter Zijlstra 已提交
300 301 302 303
#define PERF_EF_START	0x01		/* start the counter when adding    */
#define PERF_EF_RELOAD	0x02		/* reload the counter when starting */
#define PERF_EF_UPDATE	0x04		/* update the counter when stopping */

304
	/*
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
	 * Adds/Removes a counter to/from the PMU, can be done inside a
	 * transaction, see the ->*_txn() methods.
	 *
	 * The add/del callbacks will reserve all hardware resources required
	 * to service the event, this includes any counter constraint
	 * scheduling etc.
	 *
	 * Called with IRQs disabled and the PMU disabled on the CPU the event
	 * is on.
	 *
	 * ->add() called without PERF_EF_START should result in the same state
	 *  as ->add() followed by ->stop().
	 *
	 * ->del() must always PERF_EF_UPDATE stop an event. If it calls
	 *  ->stop() that must deal with already being stopped without
	 *  PERF_EF_UPDATE.
P
Peter Zijlstra 已提交
321 322 323 324 325
	 */
	int  (*add)			(struct perf_event *event, int flags);
	void (*del)			(struct perf_event *event, int flags);

	/*
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
	 * Starts/Stops a counter present on the PMU.
	 *
	 * The PMI handler should stop the counter when perf_event_overflow()
	 * returns !0. ->start() will be used to continue.
	 *
	 * Also used to change the sample period.
	 *
	 * Called with IRQs disabled and the PMU disabled on the CPU the event
	 * is on -- will be called from NMI context with the PMU generates
	 * NMIs.
	 *
	 * ->stop() with PERF_EF_UPDATE will read the counter and update
	 *  period/count values like ->read() would.
	 *
	 * ->start() with PERF_EF_RELOAD will reprogram the the counter
	 *  value, must be preceded by a ->stop() with PERF_EF_UPDATE.
P
Peter Zijlstra 已提交
342 343 344 345 346 347
	 */
	void (*start)			(struct perf_event *event, int flags);
	void (*stop)			(struct perf_event *event, int flags);

	/*
	 * Updates the counter value of the event.
348 349 350
	 *
	 * For sampling capable PMUs this will also update the software period
	 * hw_perf_event::period_left field.
P
Peter Zijlstra 已提交
351
	 */
352
	void (*read)			(struct perf_event *event);
353 354

	/*
355 356 357
	 * Group events scheduling is treated as a transaction, add
	 * group events as a whole and perform one schedulability test.
	 * If the test fails, roll back the whole group
P
Peter Zijlstra 已提交
358 359
	 *
	 * Start the transaction, after this ->add() doesn't need to
360
	 * do schedulability tests.
361 362
	 *
	 * Optional.
363
	 */
364
	void (*start_txn)		(struct pmu *pmu, unsigned int txn_flags);
365
	/*
P
Peter Zijlstra 已提交
366
	 * If ->start_txn() disabled the ->add() schedulability test
367 368 369
	 * then ->commit_txn() is required to perform one. On success
	 * the transaction is closed. On error the transaction is kept
	 * open until ->cancel_txn() is called.
370 371
	 *
	 * Optional.
372
	 */
373
	int  (*commit_txn)		(struct pmu *pmu);
374
	/*
P
Peter Zijlstra 已提交
375
	 * Will cancel the transaction, assumes ->del() is called
L
Lucas De Marchi 已提交
376
	 * for each successful ->add() during the transaction.
377 378
	 *
	 * Optional.
379
	 */
380
	void (*cancel_txn)		(struct pmu *pmu);
381 382 383 384 385 386

	/*
	 * Will return the value for perf_event_mmap_page::index for this event,
	 * if no implementation is provided it will default to: event->hw.idx + 1.
	 */
	int (*event_idx)		(struct perf_event *event); /*optional */
387

388 389 390 391 392
	/*
	 * context-switches callback
	 */
	void (*sched_task)		(struct perf_event_context *ctx,
					bool sched_in);
393 394 395 396
	/*
	 * PMU specific data size
	 */
	size_t				task_ctx_size;
397

398 399 400 401 402

	/*
	 * Return the count value for a counter.
	 */
	u64 (*count)			(struct perf_event *event); /*optional*/
403 404 405 406 407 408 409 410 411 412 413 414

	/*
	 * Set up pmu-private data structures for an AUX area
	 */
	void *(*setup_aux)		(int cpu, void **pages,
					 int nr_pages, bool overwrite);
					/* optional */

	/*
	 * Free pmu-private AUX data structures
	 */
	void (*free_aux)		(void *aux); /* optional */
415

416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
	/*
	 * Validate address range filters: make sure the HW supports the
	 * requested configuration and number of filters; return 0 if the
	 * supplied filters are valid, -errno otherwise.
	 *
	 * Runs in the context of the ioctl()ing process and is not serialized
	 * with the rest of the PMU callbacks.
	 */
	int (*addr_filters_validate)	(struct list_head *filters);
					/* optional */

	/*
	 * Synchronize address range filter configuration:
	 * translate hw-agnostic filters into hardware configuration in
	 * event::hw::addr_filters.
	 *
	 * Runs as a part of filter sync sequence that is done in ->start()
	 * callback by calling perf_event_addr_filters_sync().
	 *
	 * May (and should) traverse event::addr_filters::list, for which its
	 * caller provides necessary serialization.
	 */
	void (*addr_filters_sync)	(struct perf_event *event);
					/* optional */

441 442 443 444
	/*
	 * Filter events for PMU-specific reasons.
	 */
	int (*filter_match)		(struct perf_event *event); /* optional */
I
Ingo Molnar 已提交
445 446
};

447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
/**
 * struct perf_addr_filter - address range filter definition
 * @entry:	event's filter list linkage
 * @inode:	object file's inode for file-based filters
 * @offset:	filter range offset
 * @size:	filter range size
 * @range:	1: range, 0: address
 * @filter:	1: filter/start, 0: stop
 *
 * This is a hardware-agnostic filter configuration as specified by the user.
 */
struct perf_addr_filter {
	struct list_head	entry;
	struct inode		*inode;
	unsigned long		offset;
	unsigned long		size;
	unsigned int		range	: 1,
				filter	: 1;
};

/**
 * struct perf_addr_filters_head - container for address range filters
 * @list:	list of filters for this event
 * @lock:	spinlock that serializes accesses to the @list and event's
 *		(and its children's) filter generations.
 *
 * A child event will use parent's @list (and therefore @lock), so they are
 * bundled together; see perf_event_addr_filters().
 */
struct perf_addr_filters_head {
	struct list_head	list;
	raw_spinlock_t		lock;
};

481
/**
482
 * enum perf_event_active_state - the states of a event
483
 */
484
enum perf_event_active_state {
P
Peter Zijlstra 已提交
485
	PERF_EVENT_STATE_DEAD		= -4,
486
	PERF_EVENT_STATE_EXIT		= -3,
I
Ingo Molnar 已提交
487
	PERF_EVENT_STATE_ERROR		= -2,
488 489
	PERF_EVENT_STATE_OFF		= -1,
	PERF_EVENT_STATE_INACTIVE	=  0,
I
Ingo Molnar 已提交
490
	PERF_EVENT_STATE_ACTIVE		=  1,
491 492
};

493
struct file;
494 495
struct perf_sample_data;

496
typedef void (*perf_overflow_handler_t)(struct perf_event *,
497 498 499
					struct perf_sample_data *,
					struct pt_regs *regs);

500
enum perf_group_flag {
501
	PERF_GROUP_SOFTWARE		= 0x1,
502 503
};

504 505
#define SWEVENT_HLIST_BITS		8
#define SWEVENT_HLIST_SIZE		(1 << SWEVENT_HLIST_BITS)
506 507

struct swevent_hlist {
508 509
	struct hlist_head		heads[SWEVENT_HLIST_SIZE];
	struct rcu_head			rcu_head;
510 511
};

512 513
#define PERF_ATTACH_CONTEXT	0x01
#define PERF_ATTACH_GROUP	0x02
514
#define PERF_ATTACH_TASK	0x04
515
#define PERF_ATTACH_TASK_DATA	0x08
516

517
struct perf_cgroup;
518 519
struct ring_buffer;

520 521 522 523 524
struct pmu_event_list {
	raw_spinlock_t		lock;
	struct list_head	list;
};

T
Thomas Gleixner 已提交
525
/**
526
 * struct perf_event - performance event kernel representation:
T
Thomas Gleixner 已提交
527
 */
528 529
struct perf_event {
#ifdef CONFIG_PERF_EVENTS
530 531 532 533 534
	/*
	 * entry onto perf_event_context::event_list;
	 *   modifications require ctx->lock
	 *   RCU safe iterations.
	 */
P
Peter Zijlstra 已提交
535
	struct list_head		event_entry;
536 537 538 539 540 541 542 543 544 545

	/*
	 * XXX: group_entry and sibling_list should be mutually exclusive;
	 * either you're a sibling on a group, or you're the group leader.
	 * Rework the code to always use the same list element.
	 *
	 * Locked for modification by both ctx->mutex and ctx->lock; holding
	 * either sufficies for read.
	 */
	struct list_head		group_entry;
546
	struct list_head		sibling_list;
547 548 549 550 551 552 553 554

	/*
	 * We need storage to track the entries in perf_pmu_migrate_context; we
	 * cannot use the event_entry because of RCU and we want to keep the
	 * group in tact which avoids us using the other two entries.
	 */
	struct list_head		migrate_entry;

555 556
	struct hlist_node		hlist_entry;
	struct list_head		active_entry;
I
Ingo Molnar 已提交
557
	int				nr_siblings;
558
	int				group_flags;
559
	struct perf_event		*group_leader;
P
Peter Zijlstra 已提交
560
	struct pmu			*pmu;
561
	void				*pmu_private;
562

563
	enum perf_event_active_state	state;
564
	unsigned int			attach_state;
565
	local64_t			count;
566
	atomic64_t			child_count;
567

568
	/*
569
	 * These are the total time in nanoseconds that the event
570
	 * has been enabled (i.e. eligible to run, and the task has
571
	 * been scheduled in, if this is a per-task event)
572 573 574
	 * and running (scheduled onto the CPU), respectively.
	 *
	 * They are computed from tstamp_enabled, tstamp_running and
575
	 * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
576 577 578 579 580 581
	 */
	u64				total_time_enabled;
	u64				total_time_running;

	/*
	 * These are timestamps used for computing total_time_enabled
582
	 * and total_time_running when the event is in INACTIVE or
583 584
	 * ACTIVE state, measured in nanoseconds from an arbitrary point
	 * in time.
585 586
	 * tstamp_enabled: the notional time when the event was enabled
	 * tstamp_running: the notional time when the event was scheduled on
587
	 * tstamp_stopped: in INACTIVE state, the notional time when the
588
	 *	event was scheduled off.
589 590 591 592 593
	 */
	u64				tstamp_enabled;
	u64				tstamp_running;
	u64				tstamp_stopped;

594 595 596 597 598 599 600 601 602 603
	/*
	 * timestamp shadows the actual context timing but it can
	 * be safely used in NMI interrupt context. It reflects the
	 * context time as it was when the event was last scheduled in.
	 *
	 * ctx_time already accounts for ctx->timestamp. Therefore to
	 * compute ctx_time for a sample, simply add perf_clock().
	 */
	u64				shadow_ctx_time;

604
	struct perf_event_attr		attr;
605
	u16				header_size;
606
	u16				id_header_size;
607
	u16				read_size;
608
	struct hw_perf_event		hw;
T
Thomas Gleixner 已提交
609

610
	struct perf_event_context	*ctx;
611
	atomic_long_t			refcount;
T
Thomas Gleixner 已提交
612

613 614
	/*
	 * These accumulate total time (in nanoseconds) that children
615
	 * events have been enabled and running, respectively.
616 617 618 619
	 */
	atomic64_t			child_total_time_enabled;
	atomic64_t			child_total_time_running;

T
Thomas Gleixner 已提交
620
	/*
621
	 * Protect attach/detach and child_list:
T
Thomas Gleixner 已提交
622
	 */
623 624
	struct mutex			child_mutex;
	struct list_head		child_list;
625
	struct perf_event		*parent;
T
Thomas Gleixner 已提交
626 627 628 629

	int				oncpu;
	int				cpu;

630 631 632
	struct list_head		owner_entry;
	struct task_struct		*owner;

633 634 635
	/* mmap bits */
	struct mutex			mmap_mutex;
	atomic_t			mmap_count;
P
Peter Zijlstra 已提交
636

637
	struct ring_buffer		*rb;
638
	struct list_head		rb_entry;
639 640
	unsigned long			rcu_batches;
	int				rcu_pending;
641

642
	/* poll related */
T
Thomas Gleixner 已提交
643
	wait_queue_head_t		waitq;
P
Peter Zijlstra 已提交
644
	struct fasync_struct		*fasync;
645 646 647

	/* delayed work for NMIs and such */
	int				pending_wakeup;
648
	int				pending_kill;
649
	int				pending_disable;
650
	struct irq_work			pending;
P
Peter Zijlstra 已提交
651

652 653
	atomic_t			event_limit;

654 655 656 657 658 659
	/* address range filters */
	struct perf_addr_filters_head	addr_filters;
	/* vma address array for file-based filders */
	unsigned long			*addr_filters_offs;
	unsigned long			addr_filters_gen;

660
	void (*destroy)(struct perf_event *);
P
Peter Zijlstra 已提交
661
	struct rcu_head			rcu_head;
662 663

	struct pid_namespace		*ns;
664
	u64				id;
L
Li Zefan 已提交
665

666
	u64				(*clock)(void);
667
	perf_overflow_handler_t		overflow_handler;
668
	void				*overflow_handler_context;
669

670
#ifdef CONFIG_EVENT_TRACING
671
	struct trace_event_call		*tp_event;
L
Li Zefan 已提交
672
	struct event_filter		*filter;
673 674 675
#ifdef CONFIG_FUNCTION_TRACER
	struct ftrace_ops               ftrace_ops;
#endif
676
#endif
L
Li Zefan 已提交
677

S
Stephane Eranian 已提交
678 679 680 681 682
#ifdef CONFIG_CGROUP_PERF
	struct perf_cgroup		*cgrp; /* cgroup event is attach to */
	int				cgrp_defer_enabled;
#endif

683
	struct list_head		sb_list;
L
Li Zefan 已提交
684
#endif /* CONFIG_PERF_EVENTS */
T
Thomas Gleixner 已提交
685 686 687
};

/**
688
 * struct perf_event_context - event context structure
T
Thomas Gleixner 已提交
689
 *
690
 * Used as a container for task events and CPU events as well:
T
Thomas Gleixner 已提交
691
 */
692
struct perf_event_context {
P
Peter Zijlstra 已提交
693
	struct pmu			*pmu;
T
Thomas Gleixner 已提交
694
	/*
695
	 * Protect the states of the events in the list,
696
	 * nr_active, and the list:
T
Thomas Gleixner 已提交
697
	 */
698
	raw_spinlock_t			lock;
699
	/*
700
	 * Protect the list of events.  Locking either mutex or lock
701 702 703
	 * is sufficient to ensure the list doesn't change; to change
	 * the list you need to lock both the mutex and the spinlock.
	 */
I
Ingo Molnar 已提交
704
	struct mutex			mutex;
705

706
	struct list_head		active_ctx_list;
707 708
	struct list_head		pinned_groups;
	struct list_head		flexible_groups;
I
Ingo Molnar 已提交
709
	struct list_head		event_list;
710
	int				nr_events;
I
Ingo Molnar 已提交
711 712
	int				nr_active;
	int				is_active;
713
	int				nr_stat;
714
	int				nr_freq;
715
	int				rotate_disable;
I
Ingo Molnar 已提交
716 717
	atomic_t			refcount;
	struct task_struct		*task;
718 719

	/*
720
	 * Context clock, runs when context enabled.
721
	 */
I
Ingo Molnar 已提交
722 723
	u64				time;
	u64				timestamp;
724 725 726 727 728

	/*
	 * These fields let us detect when two contexts have both
	 * been cloned (inherited) from a common ancestor.
	 */
729
	struct perf_event_context	*parent_ctx;
I
Ingo Molnar 已提交
730 731 732
	u64				parent_gen;
	u64				generation;
	int				pin_count;
733
	int				nr_cgroups;	 /* cgroup evts */
734
	void				*task_ctx_data; /* pmu specific data */
735
	struct rcu_head			rcu_head;
T
Thomas Gleixner 已提交
736 737
};

738 739
/*
 * Number of contexts where an event can trigger:
740
 *	task, softirq, hardirq, nmi.
741 742 743
 */
#define PERF_NR_CONTEXTS	4

T
Thomas Gleixner 已提交
744
/**
745
 * struct perf_event_cpu_context - per cpu event context structure
T
Thomas Gleixner 已提交
746 747
 */
struct perf_cpu_context {
748 749
	struct perf_event_context	ctx;
	struct perf_event_context	*task_ctx;
T
Thomas Gleixner 已提交
750
	int				active_oncpu;
751
	int				exclusive;
P
Peter Zijlstra 已提交
752 753

	raw_spinlock_t			hrtimer_lock;
754 755
	struct hrtimer			hrtimer;
	ktime_t				hrtimer_interval;
P
Peter Zijlstra 已提交
756 757
	unsigned int			hrtimer_active;

758
	struct pmu			*unique_pmu;
S
Stephane Eranian 已提交
759
	struct perf_cgroup		*cgrp;
T
Thomas Gleixner 已提交
760 761
};

762
struct perf_output_handle {
I
Ingo Molnar 已提交
763
	struct perf_event		*event;
764
	struct ring_buffer		*rb;
765
	unsigned long			wakeup;
766
	unsigned long			size;
767 768 769 770
	union {
		void			*addr;
		unsigned long		head;
	};
771
	int				page;
772 773
};

774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795
#ifdef CONFIG_CGROUP_PERF

/*
 * perf_cgroup_info keeps track of time_enabled for a cgroup.
 * This is a per-cpu dynamically allocated data structure.
 */
struct perf_cgroup_info {
	u64				time;
	u64				timestamp;
};

struct perf_cgroup {
	struct cgroup_subsys_state	css;
	struct perf_cgroup_info	__percpu *info;
};

/*
 * Must ensure cgroup is pinned (css_get) before calling
 * this function. In other words, we cannot call this function
 * if there is no cgroup event for the current CPU context.
 */
static inline struct perf_cgroup *
796
perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx)
797
{
798 799 800
	return container_of(task_css_check(task, perf_event_cgrp_id,
					   ctx ? lockdep_is_held(&ctx->lock)
					       : true),
801 802 803 804
			    struct perf_cgroup, css);
}
#endif /* CONFIG_CGROUP_PERF */

805
#ifdef CONFIG_PERF_EVENTS
806

807 808 809 810 811 812 813 814
extern void *perf_aux_output_begin(struct perf_output_handle *handle,
				   struct perf_event *event);
extern void perf_aux_output_end(struct perf_output_handle *handle,
				unsigned long size, bool truncated);
extern int perf_aux_output_skip(struct perf_output_handle *handle,
				unsigned long size);
extern void *perf_get_aux(struct perf_output_handle *handle);

815
extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
816
extern void perf_pmu_unregister(struct pmu *pmu);
I
Ingo Molnar 已提交
817

818
extern int perf_num_counters(void);
819
extern const char *perf_pmu_name(void);
820 821 822 823
extern void __perf_event_task_sched_in(struct task_struct *prev,
				       struct task_struct *task);
extern void __perf_event_task_sched_out(struct task_struct *prev,
					struct task_struct *next);
824 825 826
extern int perf_event_init_task(struct task_struct *child);
extern void perf_event_exit_task(struct task_struct *child);
extern void perf_event_free_task(struct task_struct *task);
827
extern void perf_event_delayed_put(struct task_struct *task);
828
extern struct file *perf_event_get(unsigned int fd);
829
extern const struct perf_event_attr *perf_event_attrs(struct perf_event *event);
830
extern void perf_event_print_debug(void);
P
Peter Zijlstra 已提交
831 832
extern void perf_pmu_disable(struct pmu *pmu);
extern void perf_pmu_enable(struct pmu *pmu);
833 834
extern void perf_sched_cb_dec(struct pmu *pmu);
extern void perf_sched_cb_inc(struct pmu *pmu);
835 836
extern int perf_event_task_disable(void);
extern int perf_event_task_enable(void);
837
extern int perf_event_refresh(struct perf_event *event, int refresh);
838
extern void perf_event_update_userpage(struct perf_event *event);
839 840 841 842
extern int perf_event_release_kernel(struct perf_event *event);
extern struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr,
				int cpu,
M
Matt Helsley 已提交
843
				struct task_struct *task,
844 845
				perf_overflow_handler_t callback,
				void *context);
846 847
extern void perf_pmu_migrate_context(struct pmu *pmu,
				int src_cpu, int dst_cpu);
848
extern u64 perf_event_read_local(struct perf_event *event);
849 850
extern u64 perf_event_read_value(struct perf_event *event,
				 u64 *enabled, u64 *running);
851

852

853
struct perf_sample_data {
854 855 856 857 858 859 860 861 862 863 864
	/*
	 * Fields set by perf_sample_data_init(), group so as to
	 * minimize the cachelines touched.
	 */
	u64				addr;
	struct perf_raw_record		*raw;
	struct perf_branch_stack	*br_stack;
	u64				period;
	u64				weight;
	u64				txn;
	union  perf_mem_data_src	data_src;
865

866 867 868 869 870
	/*
	 * The other fields, optionally {set,used} by
	 * perf_{prepare,output}_sample().
	 */
	u64				type;
871 872 873 874 875 876 877 878 879 880 881 882 883
	u64				ip;
	struct {
		u32	pid;
		u32	tid;
	}				tid_entry;
	u64				time;
	u64				id;
	u64				stream_id;
	struct {
		u32	cpu;
		u32	reserved;
	}				cpu_entry;
	struct perf_callchain_entry	*callchain;
884 885 886 887 888

	/*
	 * regs_user may point to task_pt_regs or to regs_user_copy, depending
	 * on arch details.
	 */
889
	struct perf_regs		regs_user;
890 891
	struct pt_regs			regs_user_copy;

892
	struct perf_regs		regs_intr;
893
	u64				stack_user_size;
894
} ____cacheline_aligned;
895

896 897 898 899 900 901 902
/* default value for data source */
#define PERF_MEM_NA (PERF_MEM_S(OP, NA)   |\
		    PERF_MEM_S(LVL, NA)   |\
		    PERF_MEM_S(SNOOP, NA) |\
		    PERF_MEM_S(LOCK, NA)  |\
		    PERF_MEM_S(TLB, NA))

903 904
static inline void perf_sample_data_init(struct perf_sample_data *data,
					 u64 addr, u64 period)
905
{
906
	/* remaining struct members initialized in perf_prepare_sample() */
907 908
	data->addr = addr;
	data->raw  = NULL;
909
	data->br_stack = NULL;
910
	data->period = period;
A
Andi Kleen 已提交
911
	data->weight = 0;
912
	data->data_src.val = PERF_MEM_NA;
A
Andi Kleen 已提交
913
	data->txn = 0;
914 915
}

916 917 918
extern void perf_output_sample(struct perf_output_handle *handle,
			       struct perf_event_header *header,
			       struct perf_sample_data *data,
919
			       struct perf_event *event);
920 921
extern void perf_prepare_sample(struct perf_event_header *header,
				struct perf_sample_data *data,
922
				struct perf_event *event,
923 924
				struct pt_regs *regs);

925
extern int perf_event_overflow(struct perf_event *event,
926 927
				 struct perf_sample_data *data,
				 struct pt_regs *regs);
928

929 930 931 932 933 934
extern void perf_event_output_forward(struct perf_event *event,
				     struct perf_sample_data *data,
				     struct pt_regs *regs);
extern void perf_event_output_backward(struct perf_event *event,
				       struct perf_sample_data *data,
				       struct pt_regs *regs);
935
extern void perf_event_output(struct perf_event *event,
936 937
			      struct perf_sample_data *data,
			      struct pt_regs *regs);
938

939 940 941
static inline bool
is_default_overflow_handler(struct perf_event *event)
{
942 943 944 945 946
	if (likely(event->overflow_handler == perf_event_output_forward))
		return true;
	if (unlikely(event->overflow_handler == perf_event_output_backward))
		return true;
	return false;
947 948
}

949 950 951 952 953 954 955 956 957
extern void
perf_event_header__init_id(struct perf_event_header *header,
			   struct perf_sample_data *data,
			   struct perf_event *event);
extern void
perf_event__output_id_sample(struct perf_event *event,
			     struct perf_output_handle *handle,
			     struct perf_sample_data *sample);

958 959 960
extern void
perf_log_lost_samples(struct perf_event *event, u64 lost);

961 962 963 964 965
static inline bool is_sampling_event(struct perf_event *event)
{
	return event->attr.sample_period != 0;
}

966
/*
967
 * Return 1 for a software event, 0 for a hardware event
968
 */
969
static inline int is_software_event(struct perf_event *event)
970
{
971
	return event->pmu->task_ctx_nr == perf_sw_context;
972 973
}

974
extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
975

976
extern void ___perf_sw_event(u32, u64, struct pt_regs *, u64);
977
extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
978

979
#ifndef perf_arch_fetch_caller_regs
980
static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
981
#endif
982 983 984 985 986 987 988 989 990

/*
 * Take a snapshot of the regs. Skip ip and frame pointer to
 * the nth caller. We only need a few of the regs:
 * - ip for PERF_SAMPLE_IP
 * - cs for user_mode() tests
 * - bp for callchains
 * - eflags, for future purposes, just in case
 */
991
static inline void perf_fetch_caller_regs(struct pt_regs *regs)
992 993 994
{
	memset(regs, 0, sizeof(*regs));

995
	perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
996 997
}

P
Peter Zijlstra 已提交
998
static __always_inline void
999
perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
1000
{
1001 1002 1003 1004 1005
	if (static_key_false(&perf_swevent_enabled[event_id]))
		__perf_sw_event(event_id, nr, regs, addr);
}

DECLARE_PER_CPU(struct pt_regs, __perf_regs[4]);
P
Peter Zijlstra 已提交
1006

1007 1008 1009 1010 1011 1012 1013 1014
/*
 * 'Special' version for the scheduler, it hard assumes no recursion,
 * which is guaranteed by us not actually scheduling inside other swevents
 * because those disable preemption.
 */
static __always_inline void
perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)
{
1015
	if (static_key_false(&perf_swevent_enabled[event_id])) {
1016 1017 1018 1019
		struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);

		perf_fetch_caller_regs(regs);
		___perf_sw_event(event_id, nr, regs, addr);
1020 1021 1022
	}
}

1023
extern struct static_key_false perf_sched_events;
1024

1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
static __always_inline bool
perf_sw_migrate_enabled(void)
{
	if (static_key_false(&perf_swevent_enabled[PERF_COUNT_SW_CPU_MIGRATIONS]))
		return true;
	return false;
}

static inline void perf_event_task_migrate(struct task_struct *task)
{
	if (perf_sw_migrate_enabled())
		task->sched_migrated = 1;
}

1039
static inline void perf_event_task_sched_in(struct task_struct *prev,
1040
					    struct task_struct *task)
1041
{
1042
	if (static_branch_unlikely(&perf_sched_events))
1043
		__perf_event_task_sched_in(prev, task);
1044 1045 1046 1047 1048 1049 1050 1051

	if (perf_sw_migrate_enabled() && task->sched_migrated) {
		struct pt_regs *regs = this_cpu_ptr(&__perf_regs[0]);

		perf_fetch_caller_regs(regs);
		___perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, regs, 0);
		task->sched_migrated = 0;
	}
1052 1053 1054 1055
}

static inline void perf_event_task_sched_out(struct task_struct *prev,
					     struct task_struct *next)
1056
{
1057
	perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
1058

1059
	if (static_branch_unlikely(&perf_sched_events))
1060
		__perf_event_task_sched_out(prev, next);
1061 1062
}

1063 1064 1065 1066 1067
static inline u64 __perf_event_count(struct perf_event *event)
{
	return local64_read(&event->count) + atomic64_read(&event->child_count);
}

1068
extern void perf_event_mmap(struct vm_area_struct *vma);
1069
extern struct perf_guest_info_callbacks *perf_guest_cbs;
1070 1071
extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
1072

1073
extern void perf_event_exec(void);
1074
extern void perf_event_comm(struct task_struct *tsk, bool exec);
1075
extern void perf_event_fork(struct task_struct *tsk);
1076

1077 1078 1079
/* Callchains */
DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);

1080 1081
extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
1082 1083
extern struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
1084
		   u32 max_stack, bool crosstask, bool add_mark);
1085
extern int get_callchain_buffers(int max_stack);
1086
extern void put_callchain_buffers(void);
1087

1088
extern int sysctl_perf_event_max_stack;
1089
extern int sysctl_perf_event_max_contexts_per_stack;
1090

1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
static inline int perf_callchain_store_context(struct perf_callchain_entry_ctx *ctx, u64 ip)
{
	if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {
		struct perf_callchain_entry *entry = ctx->entry;
		entry->ip[entry->nr++] = ip;
		++ctx->contexts;
		return 0;
	} else {
		ctx->contexts_maxed = true;
		return -1; /* no more room, stop walking the stack */
	}
}
1103

1104
static inline int perf_callchain_store(struct perf_callchain_entry_ctx *ctx, u64 ip)
1105
{
1106
	if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {
1107
		struct perf_callchain_entry *entry = ctx->entry;
1108
		entry->ip[entry->nr++] = ip;
1109
		++ctx->nr;
1110 1111 1112 1113
		return 0;
	} else {
		return -1; /* no more room, stop walking the stack */
	}
1114
}
1115

1116 1117 1118
extern int sysctl_perf_event_paranoid;
extern int sysctl_perf_event_mlock;
extern int sysctl_perf_event_sample_rate;
1119 1120 1121
extern int sysctl_perf_cpu_time_max_percent;

extern void perf_sample_event_took(u64 sample_len_ns);
1122

P
Peter Zijlstra 已提交
1123 1124 1125
extern int perf_proc_update_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos);
1126 1127 1128 1129
extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos);

1130 1131
int perf_event_max_stack_handler(struct ctl_table *table, int write,
				 void __user *buffer, size_t *lenp, loff_t *ppos);
P
Peter Zijlstra 已提交
1132

1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
static inline bool perf_paranoid_tracepoint_raw(void)
{
	return sysctl_perf_event_paranoid > -1;
}

static inline bool perf_paranoid_cpu(void)
{
	return sysctl_perf_event_paranoid > 0;
}

static inline bool perf_paranoid_kernel(void)
{
	return sysctl_perf_event_paranoid > 1;
}

1148
extern void perf_event_init(void);
1149 1150
extern void perf_tp_event(u64 addr, u64 count, void *record,
			  int entry_size, struct pt_regs *regs,
1151 1152
			  struct hlist_head *head, int rctx,
			  struct task_struct *task);
1153
extern void perf_bp_event(struct perf_event *event, void *data);
1154

1155
#ifndef perf_misc_flags
1156 1157 1158
# define perf_misc_flags(regs) \
		(user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
# define perf_instruction_pointer(regs)	instruction_pointer(regs)
1159 1160
#endif

1161 1162 1163
static inline bool has_branch_stack(struct perf_event *event)
{
	return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
1164 1165 1166 1167 1168
}

static inline bool needs_branch_stack(struct perf_event *event)
{
	return event->attr.branch_sample_type != 0;
1169 1170
}

1171 1172 1173 1174 1175
static inline bool has_aux(struct perf_event *event)
{
	return event->pmu->setup_aux;
}

1176 1177 1178 1179 1180
static inline bool is_write_backward(struct perf_event *event)
{
	return !!event->attr.write_backward;
}

1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
static inline bool has_addr_filter(struct perf_event *event)
{
	return event->pmu->nr_addr_filters;
}

/*
 * An inherited event uses parent's filters
 */
static inline struct perf_addr_filters_head *
perf_event_addr_filters(struct perf_event *event)
{
	struct perf_addr_filters_head *ifh = &event->addr_filters;

	if (event->parent)
		ifh = &event->parent->addr_filters;

	return ifh;
}

extern void perf_event_addr_filters_sync(struct perf_event *event);

1202
extern int perf_output_begin(struct perf_output_handle *handle,
1203
			     struct perf_event *event, unsigned int size);
1204 1205 1206 1207 1208 1209 1210
extern int perf_output_begin_forward(struct perf_output_handle *handle,
				    struct perf_event *event,
				    unsigned int size);
extern int perf_output_begin_backward(struct perf_output_handle *handle,
				      struct perf_event *event,
				      unsigned int size);

1211
extern void perf_output_end(struct perf_output_handle *handle);
1212
extern unsigned int perf_output_copy(struct perf_output_handle *handle,
1213
			     const void *buf, unsigned int len);
1214 1215
extern unsigned int perf_output_skip(struct perf_output_handle *handle,
				     unsigned int len);
1216 1217
extern int perf_swevent_get_recursion_context(void);
extern void perf_swevent_put_recursion_context(int rctx);
1218
extern u64 perf_swevent_set_period(struct perf_event *event);
1219 1220
extern void perf_event_enable(struct perf_event *event);
extern void perf_event_disable(struct perf_event *event);
1221
extern void perf_event_disable_local(struct perf_event *event);
1222
extern void perf_event_task_tick(void);
1223
#else /* !CONFIG_PERF_EVENTS: */
1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
static inline void *
perf_aux_output_begin(struct perf_output_handle *handle,
		      struct perf_event *event)				{ return NULL; }
static inline void
perf_aux_output_end(struct perf_output_handle *handle, unsigned long size,
		    bool truncated)					{ }
static inline int
perf_aux_output_skip(struct perf_output_handle *handle,
		     unsigned long size)				{ return -EINVAL; }
static inline void *
perf_get_aux(struct perf_output_handle *handle)				{ return NULL; }
T
Thomas Gleixner 已提交
1235
static inline void
1236 1237
perf_event_task_migrate(struct task_struct *task)			{ }
static inline void
1238 1239 1240 1241 1242
perf_event_task_sched_in(struct task_struct *prev,
			 struct task_struct *task)			{ }
static inline void
perf_event_task_sched_out(struct task_struct *prev,
			  struct task_struct *next)			{ }
1243 1244 1245
static inline int perf_event_init_task(struct task_struct *child)	{ return 0; }
static inline void perf_event_exit_task(struct task_struct *child)	{ }
static inline void perf_event_free_task(struct task_struct *task)	{ }
1246
static inline void perf_event_delayed_put(struct task_struct *task)	{ }
1247
static inline struct file *perf_event_get(unsigned int fd)	{ return ERR_PTR(-EINVAL); }
1248 1249 1250 1251 1252
static inline const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
{
	return ERR_PTR(-EINVAL);
}
static inline u64 perf_event_read_local(struct perf_event *event)	{ return -EINVAL; }
I
Ingo Molnar 已提交
1253 1254 1255
static inline void perf_event_print_debug(void)				{ }
static inline int perf_event_task_disable(void)				{ return -EINVAL; }
static inline int perf_event_task_enable(void)				{ return -EINVAL; }
1256 1257 1258 1259
static inline int perf_event_refresh(struct perf_event *event, int refresh)
{
	return -EINVAL;
}
1260

1261
static inline void
1262
perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)	{ }
1263
static inline void
1264 1265
perf_sw_event_sched(u32 event_id, u64 nr, u64 addr)			{ }
static inline void
1266
perf_bp_event(struct perf_event *event, void *data)			{ }
1267

1268
static inline int perf_register_guest_info_callbacks
1269
(struct perf_guest_info_callbacks *callbacks)				{ return 0; }
1270
static inline int perf_unregister_guest_info_callbacks
1271
(struct perf_guest_info_callbacks *callbacks)				{ return 0; }
1272

I
Ingo Molnar 已提交
1273
static inline void perf_event_mmap(struct vm_area_struct *vma)		{ }
1274
static inline void perf_event_exec(void)				{ }
1275
static inline void perf_event_comm(struct task_struct *tsk, bool exec)	{ }
1276 1277
static inline void perf_event_fork(struct task_struct *tsk)		{ }
static inline void perf_event_init(void)				{ }
1278
static inline int  perf_swevent_get_recursion_context(void)		{ return -1; }
1279
static inline void perf_swevent_put_recursion_context(int rctx)		{ }
1280
static inline u64 perf_swevent_set_period(struct perf_event *event)	{ return 0; }
1281 1282
static inline void perf_event_enable(struct perf_event *event)		{ }
static inline void perf_event_disable(struct perf_event *event)		{ }
1283
static inline int __perf_event_disable(void *info)			{ return -1; }
1284
static inline void perf_event_task_tick(void)				{ }
1285
static inline int perf_event_release_kernel(struct perf_event *event)	{ return 0; }
T
Thomas Gleixner 已提交
1286 1287
#endif

1288 1289 1290
#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
extern void perf_restore_debug_store(void);
#else
1291
static inline void perf_restore_debug_store(void)			{ }
T
Thomas Gleixner 已提交
1292 1293
#endif

1294
#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
1295

1296
/*
1297
 * This has to have a higher priority than migration_notifier in sched/core.c.
1298
 */
1299 1300
#define perf_cpu_notifier(fn)						\
do {									\
1301
	static struct notifier_block fn##_nb =				\
1302
		{ .notifier_call = fn, .priority = CPU_PRI_PERF };	\
1303
	unsigned long cpu = smp_processor_id();				\
1304
	unsigned long flags;						\
1305 1306
									\
	cpu_notifier_register_begin();					\
1307
	fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE,			\
1308
		(void *)(unsigned long)cpu);				\
1309
	local_irq_save(flags);						\
1310
	fn(&fn##_nb, (unsigned long)CPU_STARTING,			\
1311
		(void *)(unsigned long)cpu);				\
1312
	local_irq_restore(flags);					\
1313
	fn(&fn##_nb, (unsigned long)CPU_ONLINE,				\
1314
		(void *)(unsigned long)cpu);				\
1315 1316
	__register_cpu_notifier(&fn##_nb);				\
	cpu_notifier_register_done();					\
1317 1318
} while (0)

1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329
/*
 * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the
 * callback for already online CPUs.
 */
#define __perf_cpu_notifier(fn)						\
do {									\
	static struct notifier_block fn##_nb =				\
		{ .notifier_call = fn, .priority = CPU_PRI_PERF };	\
									\
	__register_cpu_notifier(&fn##_nb);				\
} while (0)
1330

1331 1332 1333
struct perf_pmu_events_attr {
	struct device_attribute attr;
	u64 id;
1334
	const char *event_str;
1335 1336
};

1337 1338 1339
ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
			      char *page);

1340 1341 1342 1343 1344 1345
#define PMU_EVENT_ATTR(_name, _var, _id, _show)				\
static struct perf_pmu_events_attr _var = {				\
	.attr = __ATTR(_name, 0444, _show, NULL),			\
	.id   =  _id,							\
};

1346 1347 1348 1349 1350 1351 1352
#define PMU_EVENT_ATTR_STRING(_name, _var, _str)			    \
static struct perf_pmu_events_attr _var = {				    \
	.attr		= __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
	.id		= 0,						    \
	.event_str	= _str,						    \
};

1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
#define PMU_FORMAT_ATTR(_name, _format)					\
static ssize_t								\
_name##_show(struct device *dev,					\
			       struct device_attribute *attr,		\
			       char *page)				\
{									\
	BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);			\
	return sprintf(page, _format "\n");				\
}									\
									\
static struct device_attribute format_attr_##_name = __ATTR_RO(_name)

1365
#endif /* _LINUX_PERF_EVENT_H */