perf_event.h 23.0 KB
Newer Older
T
Thomas Gleixner 已提交
1
/*
I
Ingo Molnar 已提交
2
 * Performance events:
T
Thomas Gleixner 已提交
3
 *
I
Ingo Molnar 已提交
4
 *    Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5 6
 *    Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
 *    Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
T
Thomas Gleixner 已提交
7
 *
I
Ingo Molnar 已提交
8
 * Data type definitions, declarations, prototypes.
T
Thomas Gleixner 已提交
9
 *
I
Ingo Molnar 已提交
10
 *    Started by: Thomas Gleixner and Ingo Molnar
T
Thomas Gleixner 已提交
11
 *
I
Ingo Molnar 已提交
12
 * For licencing details see kernel-base/COPYING
T
Thomas Gleixner 已提交
13
 */
14 15
#ifndef _LINUX_PERF_EVENT_H
#define _LINUX_PERF_EVENT_H
T
Thomas Gleixner 已提交
16

17
#include <uapi/linux/perf_event.h>
T
Thomas Gleixner 已提交
18

I
Ingo Molnar 已提交
19
/*
20
 * Kernel-internal data types and definitions:
I
Ingo Molnar 已提交
21 22
 */

23
#ifdef CONFIG_PERF_EVENTS
S
Stephane Eranian 已提交
24
# include <linux/cgroup.h>
25
# include <asm/perf_event.h>
26
# include <asm/local64.h>
27 28
#endif

29
struct perf_guest_info_callbacks {
30 31 32
	int				(*is_in_guest)(void);
	int				(*is_user_mode)(void);
	unsigned long			(*get_guest_ip)(void);
33 34
};

35 36 37 38
#ifdef CONFIG_HAVE_HW_BREAKPOINT
#include <asm/hw_breakpoint.h>
#endif

39 40 41 42 43
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
44
#include <linux/hrtimer.h>
P
Peter Zijlstra 已提交
45
#include <linux/fs.h>
46
#include <linux/pid_namespace.h>
47
#include <linux/workqueue.h>
48
#include <linux/ftrace.h>
49
#include <linux/cpu.h>
50
#include <linux/irq_work.h>
51
#include <linux/static_key.h>
A
Arun Sharma 已提交
52
#include <linux/atomic.h>
53
#include <linux/sysfs.h>
54
#include <linux/perf_regs.h>
55
#include <asm/local.h>
56

57 58 59 60 61
struct perf_callchain_entry {
	__u64				nr;
	__u64				ip[PERF_MAX_STACK_DEPTH];
};

62 63 64
struct perf_raw_record {
	u32				size;
	void				*data;
65 66
};

67 68 69 70 71 72 73 74 75 76 77
/*
 * single taken branch record layout:
 *
 *      from: source instruction (may not always be a branch insn)
 *        to: branch target
 *   mispred: branch target was mispredicted
 * predicted: branch target was predicted
 *
 * support for mispred, predicted is optional. In case it
 * is not supported mispred = predicted = 0.
 */
78
struct perf_branch_entry {
79 80 81 82 83
	__u64	from;
	__u64	to;
	__u64	mispred:1,  /* target mispredicted */
		predicted:1,/* target predicted */
		reserved:62;
84 85
};

86 87 88 89 90 91 92 93 94
/*
 * branch stack layout:
 *  nr: number of taken branches stored in entries[]
 *
 * Note that nr can vary from sample to sample
 * branches (to, from) are stored from most recent
 * to least recent, i.e., entries[0] contains the most
 * recent branch.
 */
95 96 97 98 99
struct perf_branch_stack {
	__u64				nr;
	struct perf_branch_entry	entries[0];
};

100 101 102 103 104
struct perf_regs_user {
	__u64		abi;
	struct pt_regs	*regs;
};

105 106
struct task_struct;

107 108 109 110 111 112 113 114 115 116
/*
 * extra PMU register associated with an event
 */
struct hw_perf_event_extra {
	u64		config;	/* register value */
	unsigned int	reg;	/* register address or index */
	int		alloc;	/* extra register already allocated */
	int		idx;	/* index in shared_regs->regs[] */
};

T
Thomas Gleixner 已提交
117
/**
118
 * struct hw_perf_event - performance event hardware details:
T
Thomas Gleixner 已提交
119
 */
120 121
struct hw_perf_event {
#ifdef CONFIG_PERF_EVENTS
122 123
	union {
		struct { /* hardware */
I
Ingo Molnar 已提交
124
			u64		config;
125
			u64		last_tag;
I
Ingo Molnar 已提交
126
			unsigned long	config_base;
127
			unsigned long	event_base;
128
			int		event_base_rdpmc;
I
Ingo Molnar 已提交
129
			int		idx;
130
			int		last_cpu;
131

132
			struct hw_perf_event_extra extra_reg;
133
			struct hw_perf_event_extra branch_reg;
134
		};
135
		struct { /* software */
I
Ingo Molnar 已提交
136
			struct hrtimer	hrtimer;
137
		};
138 139 140 141 142
		struct { /* tracepoint */
			struct task_struct	*tp_target;
			/* for tp_event->class */
			struct list_head	tp_list;
		};
143
#ifdef CONFIG_HAVE_HW_BREAKPOINT
144
		struct { /* breakpoint */
145 146 147 148 149 150
			/*
			 * Crufty hack to avoid the chicken and egg
			 * problem hw_breakpoint has with context
			 * creation and event initalization.
			 */
			struct task_struct		*bp_target;
151 152
			struct arch_hw_breakpoint	info;
			struct list_head		bp_list;
153
		};
154
#endif
155
	};
P
Peter Zijlstra 已提交
156
	int				state;
157
	local64_t			prev_count;
158
	u64				sample_period;
159
	u64				last_period;
160
	local64_t			period_left;
161
	u64                             interrupts_seq;
162
	u64				interrupts;
163

164 165
	u64				freq_time_stamp;
	u64				freq_count_stamp;
166
#endif
T
Thomas Gleixner 已提交
167 168
};

P
Peter Zijlstra 已提交
169 170 171 172 173 174 175
/*
 * hw_perf_event::state flags
 */
#define PERF_HES_STOPPED	0x01 /* the counter is stopped */
#define PERF_HES_UPTODATE	0x02 /* event->count up-to-date */
#define PERF_HES_ARCH		0x04

176
struct perf_event;
I
Ingo Molnar 已提交
177

178 179 180 181
/*
 * Common implementation detail of pmu::{start,commit,cancel}_txn
 */
#define PERF_EVENT_TXN 0x1
182

I
Ingo Molnar 已提交
183
/**
184
 * struct pmu - generic performance monitoring unit
I
Ingo Molnar 已提交
185
 */
186
struct pmu {
187 188
	struct list_head		entry;

P
Peter Zijlstra 已提交
189
	struct device			*dev;
190
	const struct attribute_group	**attr_groups;
P
Peter Zijlstra 已提交
191 192 193
	char				*name;
	int				type;

P
Peter Zijlstra 已提交
194 195
	int * __percpu			pmu_disable_count;
	struct perf_cpu_context * __percpu pmu_cpu_context;
P
Peter Zijlstra 已提交
196
	int				task_ctx_nr;
197 198

	/*
P
Peter Zijlstra 已提交
199 200
	 * Fully disable/enable this PMU, can be used to protect from the PMI
	 * as well as for lazy/batch writing of the MSRs.
201
	 */
P
Peter Zijlstra 已提交
202 203
	void (*pmu_enable)		(struct pmu *pmu); /* optional */
	void (*pmu_disable)		(struct pmu *pmu); /* optional */
204

205
	/*
P
Peter Zijlstra 已提交
206
	 * Try and initialize the event for this PMU.
207
	 * Should return -ENOENT when the @event doesn't match this PMU.
208
	 */
209 210
	int (*event_init)		(struct perf_event *event);

P
Peter Zijlstra 已提交
211 212 213 214
#define PERF_EF_START	0x01		/* start the counter when adding    */
#define PERF_EF_RELOAD	0x02		/* reload the counter when starting */
#define PERF_EF_UPDATE	0x04		/* update the counter when stopping */

215
	/*
P
Peter Zijlstra 已提交
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
	 * Adds/Removes a counter to/from the PMU, can be done inside
	 * a transaction, see the ->*_txn() methods.
	 */
	int  (*add)			(struct perf_event *event, int flags);
	void (*del)			(struct perf_event *event, int flags);

	/*
	 * Starts/Stops a counter present on the PMU. The PMI handler
	 * should stop the counter when perf_event_overflow() returns
	 * !0. ->start() will be used to continue.
	 */
	void (*start)			(struct perf_event *event, int flags);
	void (*stop)			(struct perf_event *event, int flags);

	/*
	 * Updates the counter value of the event.
	 */
233
	void (*read)			(struct perf_event *event);
234 235

	/*
236 237 238
	 * Group events scheduling is treated as a transaction, add
	 * group events as a whole and perform one schedulability test.
	 * If the test fails, roll back the whole group
P
Peter Zijlstra 已提交
239 240
	 *
	 * Start the transaction, after this ->add() doesn't need to
241
	 * do schedulability tests.
242
	 */
243
	void (*start_txn)		(struct pmu *pmu); /* optional */
244
	/*
P
Peter Zijlstra 已提交
245
	 * If ->start_txn() disabled the ->add() schedulability test
246 247 248 249
	 * then ->commit_txn() is required to perform one. On success
	 * the transaction is closed. On error the transaction is kept
	 * open until ->cancel_txn() is called.
	 */
250
	int  (*commit_txn)		(struct pmu *pmu); /* optional */
251
	/*
P
Peter Zijlstra 已提交
252
	 * Will cancel the transaction, assumes ->del() is called
L
Lucas De Marchi 已提交
253
	 * for each successful ->add() during the transaction.
254
	 */
255
	void (*cancel_txn)		(struct pmu *pmu); /* optional */
256 257 258 259 260 261

	/*
	 * Will return the value for perf_event_mmap_page::index for this event,
	 * if no implementation is provided it will default to: event->hw.idx + 1.
	 */
	int (*event_idx)		(struct perf_event *event); /*optional */
262 263 264 265 266

	/*
	 * flush branch stack on context-switches (needed in cpu-wide mode)
	 */
	void (*flush_branch_stack)	(void);
I
Ingo Molnar 已提交
267 268
};

269
/**
270
 * enum perf_event_active_state - the states of a event
271
 */
272
enum perf_event_active_state {
I
Ingo Molnar 已提交
273
	PERF_EVENT_STATE_ERROR		= -2,
274 275
	PERF_EVENT_STATE_OFF		= -1,
	PERF_EVENT_STATE_INACTIVE	=  0,
I
Ingo Molnar 已提交
276
	PERF_EVENT_STATE_ACTIVE		=  1,
277 278
};

279
struct file;
280 281
struct perf_sample_data;

282
typedef void (*perf_overflow_handler_t)(struct perf_event *,
283 284 285
					struct perf_sample_data *,
					struct pt_regs *regs);

286
enum perf_group_flag {
287
	PERF_GROUP_SOFTWARE		= 0x1,
288 289
};

290 291
#define SWEVENT_HLIST_BITS		8
#define SWEVENT_HLIST_SIZE		(1 << SWEVENT_HLIST_BITS)
292 293

struct swevent_hlist {
294 295
	struct hlist_head		heads[SWEVENT_HLIST_SIZE];
	struct rcu_head			rcu_head;
296 297
};

298 299
#define PERF_ATTACH_CONTEXT	0x01
#define PERF_ATTACH_GROUP	0x02
300
#define PERF_ATTACH_TASK	0x04
301

S
Stephane Eranian 已提交
302 303 304 305 306 307
#ifdef CONFIG_CGROUP_PERF
/*
 * perf_cgroup_info keeps track of time_enabled for a cgroup.
 * This is a per-cpu dynamically allocated data structure.
 */
struct perf_cgroup_info {
308 309
	u64				time;
	u64				timestamp;
S
Stephane Eranian 已提交
310 311 312
};

struct perf_cgroup {
313 314
	struct				cgroup_subsys_state css;
	struct				perf_cgroup_info *info;	/* timing info, one per cpu */
S
Stephane Eranian 已提交
315 316 317
};
#endif

318 319
struct ring_buffer;

T
Thomas Gleixner 已提交
320
/**
321
 * struct perf_event - performance event kernel representation:
T
Thomas Gleixner 已提交
322
 */
323 324
struct perf_event {
#ifdef CONFIG_PERF_EVENTS
325
	struct list_head		group_entry;
P
Peter Zijlstra 已提交
326
	struct list_head		event_entry;
327
	struct list_head		sibling_list;
328
	struct hlist_node		hlist_entry;
I
Ingo Molnar 已提交
329
	int				nr_siblings;
330
	int				group_flags;
331
	struct perf_event		*group_leader;
P
Peter Zijlstra 已提交
332
	struct pmu			*pmu;
333

334
	enum perf_event_active_state	state;
335
	unsigned int			attach_state;
336
	local64_t			count;
337
	atomic64_t			child_count;
338

339
	/*
340
	 * These are the total time in nanoseconds that the event
341
	 * has been enabled (i.e. eligible to run, and the task has
342
	 * been scheduled in, if this is a per-task event)
343 344 345
	 * and running (scheduled onto the CPU), respectively.
	 *
	 * They are computed from tstamp_enabled, tstamp_running and
346
	 * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
347 348 349 350 351 352
	 */
	u64				total_time_enabled;
	u64				total_time_running;

	/*
	 * These are timestamps used for computing total_time_enabled
353
	 * and total_time_running when the event is in INACTIVE or
354 355
	 * ACTIVE state, measured in nanoseconds from an arbitrary point
	 * in time.
356 357
	 * tstamp_enabled: the notional time when the event was enabled
	 * tstamp_running: the notional time when the event was scheduled on
358
	 * tstamp_stopped: in INACTIVE state, the notional time when the
359
	 *	event was scheduled off.
360 361 362 363 364
	 */
	u64				tstamp_enabled;
	u64				tstamp_running;
	u64				tstamp_stopped;

365 366 367 368 369 370 371 372 373 374
	/*
	 * timestamp shadows the actual context timing but it can
	 * be safely used in NMI interrupt context. It reflects the
	 * context time as it was when the event was last scheduled in.
	 *
	 * ctx_time already accounts for ctx->timestamp. Therefore to
	 * compute ctx_time for a sample, simply add perf_clock().
	 */
	u64				shadow_ctx_time;

375
	struct perf_event_attr		attr;
376
	u16				header_size;
377
	u16				id_header_size;
378
	u16				read_size;
379
	struct hw_perf_event		hw;
T
Thomas Gleixner 已提交
380

381
	struct perf_event_context	*ctx;
382
	atomic_long_t			refcount;
T
Thomas Gleixner 已提交
383

384 385
	/*
	 * These accumulate total time (in nanoseconds) that children
386
	 * events have been enabled and running, respectively.
387 388 389 390
	 */
	atomic64_t			child_total_time_enabled;
	atomic64_t			child_total_time_running;

T
Thomas Gleixner 已提交
391
	/*
392
	 * Protect attach/detach and child_list:
T
Thomas Gleixner 已提交
393
	 */
394 395
	struct mutex			child_mutex;
	struct list_head		child_list;
396
	struct perf_event		*parent;
T
Thomas Gleixner 已提交
397 398 399 400

	int				oncpu;
	int				cpu;

401 402 403
	struct list_head		owner_entry;
	struct task_struct		*owner;

404 405 406
	/* mmap bits */
	struct mutex			mmap_mutex;
	atomic_t			mmap_count;
407 408
	int				mmap_locked;
	struct user_struct		*mmap_user;
409
	struct ring_buffer		*rb;
410
	struct list_head		rb_entry;
411

412
	/* poll related */
T
Thomas Gleixner 已提交
413
	wait_queue_head_t		waitq;
P
Peter Zijlstra 已提交
414
	struct fasync_struct		*fasync;
415 416 417

	/* delayed work for NMIs and such */
	int				pending_wakeup;
418
	int				pending_kill;
419
	int				pending_disable;
420
	struct irq_work			pending;
P
Peter Zijlstra 已提交
421

422 423
	atomic_t			event_limit;

424
	void (*destroy)(struct perf_event *);
P
Peter Zijlstra 已提交
425
	struct rcu_head			rcu_head;
426 427

	struct pid_namespace		*ns;
428
	u64				id;
L
Li Zefan 已提交
429

430
	perf_overflow_handler_t		overflow_handler;
431
	void				*overflow_handler_context;
432

433
#ifdef CONFIG_EVENT_TRACING
434
	struct ftrace_event_call	*tp_event;
L
Li Zefan 已提交
435
	struct event_filter		*filter;
436 437 438
#ifdef CONFIG_FUNCTION_TRACER
	struct ftrace_ops               ftrace_ops;
#endif
439
#endif
L
Li Zefan 已提交
440

S
Stephane Eranian 已提交
441 442 443 444 445
#ifdef CONFIG_CGROUP_PERF
	struct perf_cgroup		*cgrp; /* cgroup event is attach to */
	int				cgrp_defer_enabled;
#endif

L
Li Zefan 已提交
446
#endif /* CONFIG_PERF_EVENTS */
T
Thomas Gleixner 已提交
447 448
};

449 450 451 452 453
enum perf_event_context_type {
	task_context,
	cpu_context,
};

T
Thomas Gleixner 已提交
454
/**
455
 * struct perf_event_context - event context structure
T
Thomas Gleixner 已提交
456
 *
457
 * Used as a container for task events and CPU events as well:
T
Thomas Gleixner 已提交
458
 */
459
struct perf_event_context {
P
Peter Zijlstra 已提交
460
	struct pmu			*pmu;
461
	enum perf_event_context_type	type;
T
Thomas Gleixner 已提交
462
	/*
463
	 * Protect the states of the events in the list,
464
	 * nr_active, and the list:
T
Thomas Gleixner 已提交
465
	 */
466
	raw_spinlock_t			lock;
467
	/*
468
	 * Protect the list of events.  Locking either mutex or lock
469 470 471
	 * is sufficient to ensure the list doesn't change; to change
	 * the list you need to lock both the mutex and the spinlock.
	 */
I
Ingo Molnar 已提交
472
	struct mutex			mutex;
473

474 475
	struct list_head		pinned_groups;
	struct list_head		flexible_groups;
I
Ingo Molnar 已提交
476
	struct list_head		event_list;
477
	int				nr_events;
I
Ingo Molnar 已提交
478 479
	int				nr_active;
	int				is_active;
480
	int				nr_stat;
481
	int				nr_freq;
482
	int				rotate_disable;
I
Ingo Molnar 已提交
483 484
	atomic_t			refcount;
	struct task_struct		*task;
485 486

	/*
487
	 * Context clock, runs when context enabled.
488
	 */
I
Ingo Molnar 已提交
489 490
	u64				time;
	u64				timestamp;
491 492 493 494 495

	/*
	 * These fields let us detect when two contexts have both
	 * been cloned (inherited) from a common ancestor.
	 */
496
	struct perf_event_context	*parent_ctx;
I
Ingo Molnar 已提交
497 498 499
	u64				parent_gen;
	u64				generation;
	int				pin_count;
500 501
	int				nr_cgroups;	 /* cgroup evts */
	int				nr_branch_stack; /* branch_stack evt */
502
	struct rcu_head			rcu_head;
T
Thomas Gleixner 已提交
503 504
};

505 506
/*
 * Number of contexts where an event can trigger:
507
 *	task, softirq, hardirq, nmi.
508 509 510
 */
#define PERF_NR_CONTEXTS	4

T
Thomas Gleixner 已提交
511
/**
512
 * struct perf_event_cpu_context - per cpu event context structure
T
Thomas Gleixner 已提交
513 514
 */
struct perf_cpu_context {
515 516
	struct perf_event_context	ctx;
	struct perf_event_context	*task_ctx;
T
Thomas Gleixner 已提交
517
	int				active_oncpu;
518
	int				exclusive;
519 520
	struct list_head		rotation_list;
	int				jiffies_interval;
521
	struct pmu			*unique_pmu;
S
Stephane Eranian 已提交
522
	struct perf_cgroup		*cgrp;
T
Thomas Gleixner 已提交
523 524
};

525
struct perf_output_handle {
I
Ingo Molnar 已提交
526
	struct perf_event		*event;
527
	struct ring_buffer		*rb;
528
	unsigned long			wakeup;
529 530 531
	unsigned long			size;
	void				*addr;
	int				page;
532 533
};

534
#ifdef CONFIG_PERF_EVENTS
535

P
Peter Zijlstra 已提交
536
extern int perf_pmu_register(struct pmu *pmu, char *name, int type);
537
extern void perf_pmu_unregister(struct pmu *pmu);
I
Ingo Molnar 已提交
538

539
extern int perf_num_counters(void);
540
extern const char *perf_pmu_name(void);
541 542 543 544
extern void __perf_event_task_sched_in(struct task_struct *prev,
				       struct task_struct *task);
extern void __perf_event_task_sched_out(struct task_struct *prev,
					struct task_struct *next);
545 546 547
extern int perf_event_init_task(struct task_struct *child);
extern void perf_event_exit_task(struct task_struct *child);
extern void perf_event_free_task(struct task_struct *task);
548
extern void perf_event_delayed_put(struct task_struct *task);
549
extern void perf_event_print_debug(void);
P
Peter Zijlstra 已提交
550 551
extern void perf_pmu_disable(struct pmu *pmu);
extern void perf_pmu_enable(struct pmu *pmu);
552 553
extern int perf_event_task_disable(void);
extern int perf_event_task_enable(void);
554
extern int perf_event_refresh(struct perf_event *event, int refresh);
555
extern void perf_event_update_userpage(struct perf_event *event);
556 557 558 559
extern int perf_event_release_kernel(struct perf_event *event);
extern struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr,
				int cpu,
M
Matt Helsley 已提交
560
				struct task_struct *task,
561 562
				perf_overflow_handler_t callback,
				void *context);
563 564
extern void perf_pmu_migrate_context(struct pmu *pmu,
				int src_cpu, int dst_cpu);
565 566
extern u64 perf_event_read_value(struct perf_event *event,
				 u64 *enabled, u64 *running);
567

568

569
struct perf_sample_data {
570 571 572 573 574 575 576 577
	u64				type;

	u64				ip;
	struct {
		u32	pid;
		u32	tid;
	}				tid_entry;
	u64				time;
I
Ingo Molnar 已提交
578
	u64				addr;
579 580 581 582 583 584
	u64				id;
	u64				stream_id;
	struct {
		u32	cpu;
		u32	reserved;
	}				cpu_entry;
I
Ingo Molnar 已提交
585
	u64				period;
586
	struct perf_callchain_entry	*callchain;
587
	struct perf_raw_record		*raw;
588
	struct perf_branch_stack	*br_stack;
589
	struct perf_regs_user		regs_user;
590
	u64				stack_user_size;
591 592
};

593 594
static inline void perf_sample_data_init(struct perf_sample_data *data,
					 u64 addr, u64 period)
595
{
596
	/* remaining struct members initialized in perf_prepare_sample() */
597 598
	data->addr = addr;
	data->raw  = NULL;
599
	data->br_stack = NULL;
600 601 602
	data->period = period;
	data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE;
	data->regs_user.regs = NULL;
603
	data->stack_user_size = 0;
604 605
}

606 607 608
extern void perf_output_sample(struct perf_output_handle *handle,
			       struct perf_event_header *header,
			       struct perf_sample_data *data,
609
			       struct perf_event *event);
610 611
extern void perf_prepare_sample(struct perf_event_header *header,
				struct perf_sample_data *data,
612
				struct perf_event *event,
613 614
				struct pt_regs *regs);

615
extern int perf_event_overflow(struct perf_event *event,
616 617
				 struct perf_sample_data *data,
				 struct pt_regs *regs);
618

619 620 621 622 623
static inline bool is_sampling_event(struct perf_event *event)
{
	return event->attr.sample_period != 0;
}

624
/*
625
 * Return 1 for a software event, 0 for a hardware event
626
 */
627
static inline int is_software_event(struct perf_event *event)
628
{
629
	return event->pmu->task_ctx_nr == perf_sw_context;
630 631
}

632
extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
633

634
extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
635

636
#ifndef perf_arch_fetch_caller_regs
637
static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
638
#endif
639 640 641 642 643 644 645 646 647

/*
 * Take a snapshot of the regs. Skip ip and frame pointer to
 * the nth caller. We only need a few of the regs:
 * - ip for PERF_SAMPLE_IP
 * - cs for user_mode() tests
 * - bp for callchains
 * - eflags, for future purposes, just in case
 */
648
static inline void perf_fetch_caller_regs(struct pt_regs *regs)
649 650 651
{
	memset(regs, 0, sizeof(*regs));

652
	perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
653 654
}

P
Peter Zijlstra 已提交
655
static __always_inline void
656
perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
657
{
P
Peter Zijlstra 已提交
658 659
	struct pt_regs hot_regs;

660
	if (static_key_false(&perf_swevent_enabled[event_id])) {
661 662 663 664
		if (!regs) {
			perf_fetch_caller_regs(&hot_regs);
			regs = &hot_regs;
		}
665
		__perf_sw_event(event_id, nr, regs, addr);
666 667 668
	}
}

669
extern struct static_key_deferred perf_sched_events;
670

671
static inline void perf_event_task_sched_in(struct task_struct *prev,
672
					    struct task_struct *task)
673 674 675 676 677 678 679
{
	if (static_key_false(&perf_sched_events.key))
		__perf_event_task_sched_in(prev, task);
}

static inline void perf_event_task_sched_out(struct task_struct *prev,
					     struct task_struct *next)
680
{
681
	perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
682

683
	if (static_key_false(&perf_sched_events.key))
684
		__perf_event_task_sched_out(prev, next);
685 686
}

687
extern void perf_event_mmap(struct vm_area_struct *vma);
688
extern struct perf_guest_info_callbacks *perf_guest_cbs;
689 690
extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
691

692 693
extern void perf_event_comm(struct task_struct *tsk);
extern void perf_event_fork(struct task_struct *tsk);
694

695 696 697
/* Callchains */
DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);

698 699
extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
700

701
static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
702 703 704 705
{
	if (entry->nr < PERF_MAX_STACK_DEPTH)
		entry->ip[entry->nr++] = ip;
}
706

707 708 709
extern int sysctl_perf_event_paranoid;
extern int sysctl_perf_event_mlock;
extern int sysctl_perf_event_sample_rate;
710

P
Peter Zijlstra 已提交
711 712 713 714
extern int perf_proc_update_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos);

715 716 717 718 719 720 721 722 723 724 725 726 727 728 729
static inline bool perf_paranoid_tracepoint_raw(void)
{
	return sysctl_perf_event_paranoid > -1;
}

static inline bool perf_paranoid_cpu(void)
{
	return sysctl_perf_event_paranoid > 0;
}

static inline bool perf_paranoid_kernel(void)
{
	return sysctl_perf_event_paranoid > 1;
}

730
extern void perf_event_init(void);
731 732
extern void perf_tp_event(u64 addr, u64 count, void *record,
			  int entry_size, struct pt_regs *regs,
733 734
			  struct hlist_head *head, int rctx,
			  struct task_struct *task);
735
extern void perf_bp_event(struct perf_event *event, void *data);
736

737
#ifndef perf_misc_flags
738 739 740
# define perf_misc_flags(regs) \
		(user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
# define perf_instruction_pointer(regs)	instruction_pointer(regs)
741 742
#endif

743 744 745 746 747
static inline bool has_branch_stack(struct perf_event *event)
{
	return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
}

748
extern int perf_output_begin(struct perf_output_handle *handle,
749
			     struct perf_event *event, unsigned int size);
750
extern void perf_output_end(struct perf_output_handle *handle);
751
extern unsigned int perf_output_copy(struct perf_output_handle *handle,
752
			     const void *buf, unsigned int len);
753 754
extern unsigned int perf_output_skip(struct perf_output_handle *handle,
				     unsigned int len);
755 756
extern int perf_swevent_get_recursion_context(void);
extern void perf_swevent_put_recursion_context(int rctx);
757 758
extern void perf_event_enable(struct perf_event *event);
extern void perf_event_disable(struct perf_event *event);
759
extern int __perf_event_disable(void *info);
760
extern void perf_event_task_tick(void);
T
Thomas Gleixner 已提交
761 762
#else
static inline void
763 764 765 766 767
perf_event_task_sched_in(struct task_struct *prev,
			 struct task_struct *task)			{ }
static inline void
perf_event_task_sched_out(struct task_struct *prev,
			  struct task_struct *next)			{ }
768 769 770
static inline int perf_event_init_task(struct task_struct *child)	{ return 0; }
static inline void perf_event_exit_task(struct task_struct *child)	{ }
static inline void perf_event_free_task(struct task_struct *task)	{ }
771
static inline void perf_event_delayed_put(struct task_struct *task)	{ }
I
Ingo Molnar 已提交
772 773 774
static inline void perf_event_print_debug(void)				{ }
static inline int perf_event_task_disable(void)				{ return -EINVAL; }
static inline int perf_event_task_enable(void)				{ return -EINVAL; }
775 776 777 778
static inline int perf_event_refresh(struct perf_event *event, int refresh)
{
	return -EINVAL;
}
779

780
static inline void
781
perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)	{ }
782
static inline void
783
perf_bp_event(struct perf_event *event, void *data)			{ }
784

785
static inline int perf_register_guest_info_callbacks
786
(struct perf_guest_info_callbacks *callbacks)				{ return 0; }
787
static inline int perf_unregister_guest_info_callbacks
788
(struct perf_guest_info_callbacks *callbacks)				{ return 0; }
789

I
Ingo Molnar 已提交
790
static inline void perf_event_mmap(struct vm_area_struct *vma)		{ }
791 792 793
static inline void perf_event_comm(struct task_struct *tsk)		{ }
static inline void perf_event_fork(struct task_struct *tsk)		{ }
static inline void perf_event_init(void)				{ }
794
static inline int  perf_swevent_get_recursion_context(void)		{ return -1; }
795
static inline void perf_swevent_put_recursion_context(int rctx)		{ }
796 797
static inline void perf_event_enable(struct perf_event *event)		{ }
static inline void perf_event_disable(struct perf_event *event)		{ }
798
static inline int __perf_event_disable(void *info)			{ return -1; }
799
static inline void perf_event_task_tick(void)				{ }
T
Thomas Gleixner 已提交
800 801
#endif

802
#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
803

804 805 806
/*
 * This has to have a higher priority than migration_notifier in sched.c.
 */
807 808 809 810
#define perf_cpu_notifier(fn)						\
do {									\
	static struct notifier_block fn##_nb __cpuinitdata =		\
		{ .notifier_call = fn, .priority = CPU_PRI_PERF };	\
811
	unsigned long cpu = smp_processor_id();				\
812
	unsigned long flags;						\
813
	fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE,			\
814
		(void *)(unsigned long)cpu);				\
815
	local_irq_save(flags);						\
816
	fn(&fn##_nb, (unsigned long)CPU_STARTING,			\
817
		(void *)(unsigned long)cpu);				\
818
	local_irq_restore(flags);					\
819
	fn(&fn##_nb, (unsigned long)CPU_ONLINE,				\
820
		(void *)(unsigned long)cpu);				\
821
	register_cpu_notifier(&fn##_nb);				\
822 823
} while (0)

824

825 826 827 828 829 830 831 832 833 834 835
struct perf_pmu_events_attr {
	struct device_attribute attr;
	u64 id;
};

#define PMU_EVENT_ATTR(_name, _var, _id, _show)				\
static struct perf_pmu_events_attr _var = {				\
	.attr = __ATTR(_name, 0444, _show, NULL),			\
	.id   =  _id,							\
};

836 837 838 839 840 841 842 843 844 845 846 847
#define PMU_FORMAT_ATTR(_name, _format)					\
static ssize_t								\
_name##_show(struct device *dev,					\
			       struct device_attribute *attr,		\
			       char *page)				\
{									\
	BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);			\
	return sprintf(page, _format "\n");				\
}									\
									\
static struct device_attribute format_attr_##_name = __ATTR_RO(_name)

848
#endif /* _LINUX_PERF_EVENT_H */