perf_event.c 38.4 KB
Newer Older
I
Ingo Molnar 已提交
1
/*
2
 * Performance events x86 architecture code
I
Ingo Molnar 已提交
3
 *
4 5 6 7 8
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2009 Jaswinder Singh Rajput
 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9
 *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10
 *  Copyright (C) 2009 Google, Inc., Stephane Eranian
I
Ingo Molnar 已提交
11 12 13 14
 *
 *  For licencing details see kernel-base/COPYING
 */

15
#include <linux/perf_event.h>
I
Ingo Molnar 已提交
16 17 18 19
#include <linux/capability.h>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
20
#include <linux/module.h>
I
Ingo Molnar 已提交
21 22
#include <linux/kdebug.h>
#include <linux/sched.h>
23
#include <linux/uaccess.h>
24
#include <linux/slab.h>
25
#include <linux/highmem.h>
26
#include <linux/cpu.h>
27
#include <linux/bitops.h>
I
Ingo Molnar 已提交
28 29

#include <asm/apic.h>
30
#include <asm/stacktrace.h>
P
Peter Zijlstra 已提交
31
#include <asm/nmi.h>
32
#include <asm/compat.h>
I
Ingo Molnar 已提交
33

34 35 36 37 38 39 40 41 42 43 44
#if 0
#undef wrmsrl
#define wrmsrl(msr, val) 					\
do {								\
	trace_printk("wrmsrl(%lx, %lx)\n", (unsigned long)(msr),\
			(unsigned long)(val));			\
	native_write_msr((msr), (u32)((u64)(val)), 		\
			(u32)((u64)(val) >> 32));		\
} while (0)
#endif

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
/*
 * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
 */
static unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
{
	unsigned long offset, addr = (unsigned long)from;
	int type = in_nmi() ? KM_NMI : KM_IRQ0;
	unsigned long size, len = 0;
	struct page *page;
	void *map;
	int ret;

	do {
		ret = __get_user_pages_fast(addr, 1, 0, &page);
		if (!ret)
			break;

		offset = addr & (PAGE_SIZE - 1);
		size = min(PAGE_SIZE - offset, n - len);

		map = kmap_atomic(page, type);
		memcpy(to, map+offset, size);
		kunmap_atomic(map, type);
		put_page(page);

		len  += size;
		to   += size;
		addr += size;

	} while (len < n);

	return len;
}

80
struct event_constraint {
81 82
	union {
		unsigned long	idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
83
		u64		idxmsk64;
84
	};
85 86
	u64	code;
	u64	cmask;
87
	int	weight;
88 89
};

90 91 92 93 94 95 96
struct amd_nb {
	int nb_id;  /* NorthBridge id */
	int refcnt; /* reference count */
	struct perf_event *owners[X86_PMC_IDX_MAX];
	struct event_constraint event_constraints[X86_PMC_IDX_MAX];
};

97 98
#define MAX_LBR_ENTRIES		16

99
struct cpu_hw_events {
100 101 102
	/*
	 * Generic x86 PMC bits
	 */
103
	struct perf_event	*events[X86_PMC_IDX_MAX]; /* in counter order */
104
	unsigned long		active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
105
	int			enabled;
I
Ingo Molnar 已提交
106

107 108
	int			n_events;
	int			n_added;
109
	int			n_txn;
110
	int			assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
111
	u64			tags[X86_PMC_IDX_MAX];
112
	struct perf_event	*event_list[X86_PMC_IDX_MAX]; /* in enabled order */
113

114 115
	unsigned int		group_flag;

116 117 118 119 120 121
	/*
	 * Intel DebugStore bits
	 */
	struct debug_store	*ds;
	u64			pebs_enabled;

122 123 124 125 126 127 128 129
	/*
	 * Intel LBR bits
	 */
	int				lbr_users;
	void				*lbr_context;
	struct perf_branch_stack	lbr_stack;
	struct perf_branch_entry	lbr_entries[MAX_LBR_ENTRIES];

130 131 132
	/*
	 * AMD specific bits
	 */
133
	struct amd_nb		*amd_nb;
134 135
};

136
#define __EVENT_CONSTRAINT(c, n, m, w) {\
137
	{ .idxmsk64 = (n) },		\
138 139
	.code = (c),			\
	.cmask = (m),			\
140
	.weight = (w),			\
141
}
142

143 144 145
#define EVENT_CONSTRAINT(c, n, m)	\
	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n))

146 147 148
/*
 * Constraint on the Event code.
 */
149
#define INTEL_EVENT_CONSTRAINT(c, n)	\
150
	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
151

152 153
/*
 * Constraint on the Event code + UMask + fixed-mask
154 155 156 157 158 159 160 161
 *
 * filter mask to validate fixed counter events.
 * the following filters disqualify for fixed counters:
 *  - inv
 *  - edge
 *  - cnt-mask
 *  The other filters are supported by fixed counters.
 *  The any-thread option is supported starting with v3.
162
 */
163
#define FIXED_EVENT_CONSTRAINT(c, n)	\
164
	EVENT_CONSTRAINT(c, (1ULL << (32+n)), X86_RAW_EVENT_MASK)
165

166 167 168 169 170 171
/*
 * Constraint on the Event code + UMask
 */
#define PEBS_EVENT_CONSTRAINT(c, n)	\
	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)

172 173 174 175
#define EVENT_CONSTRAINT_END		\
	EVENT_CONSTRAINT(0, 0, 0)

#define for_each_event_constraint(e, c)	\
176
	for ((e) = (c); (e)->weight; (e)++)
177

178 179 180 181 182 183 184 185 186 187 188
union perf_capabilities {
	struct {
		u64	lbr_format    : 6;
		u64	pebs_trap     : 1;
		u64	pebs_arch_reg : 1;
		u64	pebs_format   : 4;
		u64	smm_freeze    : 1;
	};
	u64	capabilities;
};

I
Ingo Molnar 已提交
189
/*
190
 * struct x86_pmu - generic x86 pmu
I
Ingo Molnar 已提交
191
 */
192
struct x86_pmu {
193 194 195
	/*
	 * Generic x86 PMC bits
	 */
196 197
	const char	*name;
	int		version;
198
	int		(*handle_irq)(struct pt_regs *);
199
	void		(*disable_all)(void);
200
	void		(*enable_all)(int added);
201 202
	void		(*enable)(struct perf_event *);
	void		(*disable)(struct perf_event *);
203
	int		(*hw_config)(struct perf_event *event);
204
	int		(*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
205 206
	unsigned	eventsel;
	unsigned	perfctr;
207
	u64		(*event_map)(int);
208
	int		max_events;
209 210 211 212
	int		num_counters;
	int		num_counters_fixed;
	int		cntval_bits;
	u64		cntval_mask;
213
	int		apic;
214
	u64		max_period;
215 216 217 218
	struct event_constraint *
			(*get_event_constraints)(struct cpu_hw_events *cpuc,
						 struct perf_event *event);

219 220
	void		(*put_event_constraints)(struct cpu_hw_events *cpuc,
						 struct perf_event *event);
221
	struct event_constraint *event_constraints;
222
	void		(*quirks)(void);
223

224
	int		(*cpu_prepare)(int cpu);
225 226 227
	void		(*cpu_starting)(int cpu);
	void		(*cpu_dying)(int cpu);
	void		(*cpu_dead)(int cpu);
228 229 230 231

	/*
	 * Intel Arch Perfmon v2+
	 */
232 233
	u64			intel_ctrl;
	union perf_capabilities intel_cap;
234 235 236 237 238 239 240 241

	/*
	 * Intel DebugStore bits
	 */
	int		bts, pebs;
	int		pebs_record_size;
	void		(*drain_pebs)(struct pt_regs *regs);
	struct event_constraint *pebs_constraints;
242 243 244 245 246 247

	/*
	 * Intel LBR
	 */
	unsigned long	lbr_tos, lbr_from, lbr_to; /* MSR base regs       */
	int		lbr_nr;			   /* hardware stack size */
248 249
};

250
static struct x86_pmu x86_pmu __read_mostly;
251

252
static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
253 254
	.enabled = 1,
};
I
Ingo Molnar 已提交
255

256
static int x86_perf_event_set_period(struct perf_event *event);
257

258
/*
259
 * Generalized hw caching related hw_event table, filled
260
 * in on a per model basis. A value of 0 means
261 262
 * 'not supported', -1 means 'hw_event makes no sense on
 * this CPU', any other value means the raw hw_event
263 264 265 266 267 268 269 270 271 272
 * ID.
 */

#define C(x) PERF_COUNT_HW_CACHE_##x

static u64 __read_mostly hw_cache_event_ids
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX];

273
/*
274 275
 * Propagate event elapsed time into the generic event.
 * Can only be executed on the CPU where the event is active.
276 277
 * Returns the delta events processed.
 */
278
static u64
279
x86_perf_event_update(struct perf_event *event)
280
{
281
	struct hw_perf_event *hwc = &event->hw;
282
	int shift = 64 - x86_pmu.cntval_bits;
283
	u64 prev_raw_count, new_raw_count;
284
	int idx = hwc->idx;
285
	s64 delta;
286

287 288 289
	if (idx == X86_PMC_IDX_FIXED_BTS)
		return 0;

290
	/*
291
	 * Careful: an NMI might modify the previous event value.
292 293 294
	 *
	 * Our tactic to handle this is to first atomically read and
	 * exchange a new raw count - then add that new-prev delta
295
	 * count to the generic event atomically:
296 297 298
	 */
again:
	prev_raw_count = atomic64_read(&hwc->prev_count);
299
	rdmsrl(hwc->event_base + idx, new_raw_count);
300 301 302 303 304 305 306 307

	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
					new_raw_count) != prev_raw_count)
		goto again;

	/*
	 * Now we have the new raw value and have updated the prev
	 * timestamp already. We can now calculate the elapsed delta
308
	 * (event-)time and add that to the generic event.
309 310
	 *
	 * Careful, not all hw sign-extends above the physical width
311
	 * of the count.
312
	 */
313 314
	delta = (new_raw_count << shift) - (prev_raw_count << shift);
	delta >>= shift;
315

316
	atomic64_add(delta, &event->count);
317
	atomic64_sub(delta, &hwc->period_left);
318 319

	return new_raw_count;
320 321
}

322
static atomic_t active_events;
P
Peter Zijlstra 已提交
323 324
static DEFINE_MUTEX(pmc_reserve_mutex);

325 326
#ifdef CONFIG_X86_LOCAL_APIC

P
Peter Zijlstra 已提交
327 328 329 330 331 332 333
static bool reserve_pmc_hardware(void)
{
	int i;

	if (nmi_watchdog == NMI_LOCAL_APIC)
		disable_lapic_nmi_watchdog();

334
	for (i = 0; i < x86_pmu.num_counters; i++) {
335
		if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
P
Peter Zijlstra 已提交
336 337 338
			goto perfctr_fail;
	}

339
	for (i = 0; i < x86_pmu.num_counters; i++) {
340
		if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
P
Peter Zijlstra 已提交
341 342 343 344 345 346 347
			goto eventsel_fail;
	}

	return true;

eventsel_fail:
	for (i--; i >= 0; i--)
348
		release_evntsel_nmi(x86_pmu.eventsel + i);
P
Peter Zijlstra 已提交
349

350
	i = x86_pmu.num_counters;
P
Peter Zijlstra 已提交
351 352 353

perfctr_fail:
	for (i--; i >= 0; i--)
354
		release_perfctr_nmi(x86_pmu.perfctr + i);
P
Peter Zijlstra 已提交
355 356 357 358 359 360 361 362 363 364 365

	if (nmi_watchdog == NMI_LOCAL_APIC)
		enable_lapic_nmi_watchdog();

	return false;
}

static void release_pmc_hardware(void)
{
	int i;

366
	for (i = 0; i < x86_pmu.num_counters; i++) {
367 368
		release_perfctr_nmi(x86_pmu.perfctr + i);
		release_evntsel_nmi(x86_pmu.eventsel + i);
P
Peter Zijlstra 已提交
369 370 371 372 373 374
	}

	if (nmi_watchdog == NMI_LOCAL_APIC)
		enable_lapic_nmi_watchdog();
}

375 376 377 378 379 380 381
#else

static bool reserve_pmc_hardware(void) { return true; }
static void release_pmc_hardware(void) {}

#endif

382 383
static int reserve_ds_buffers(void);
static void release_ds_buffers(void);
384

385
static void hw_perf_event_destroy(struct perf_event *event)
P
Peter Zijlstra 已提交
386
{
387
	if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
P
Peter Zijlstra 已提交
388
		release_pmc_hardware();
389
		release_ds_buffers();
P
Peter Zijlstra 已提交
390 391 392 393
		mutex_unlock(&pmc_reserve_mutex);
	}
}

394 395 396 397 398
static inline int x86_pmu_initialized(void)
{
	return x86_pmu.handle_irq != NULL;
}

399
static inline int
400
set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
{
	unsigned int cache_type, cache_op, cache_result;
	u64 config, val;

	config = attr->config;

	cache_type = (config >>  0) & 0xff;
	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
		return -EINVAL;

	cache_op = (config >>  8) & 0xff;
	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
		return -EINVAL;

	cache_result = (config >> 16) & 0xff;
	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
		return -EINVAL;

	val = hw_cache_event_ids[cache_type][cache_op][cache_result];

	if (val == 0)
		return -ENOENT;

	if (val == -1)
		return -EINVAL;

	hwc->config |= val;

	return 0;
}

432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
static int x86_setup_perfctr(struct perf_event *event)
{
	struct perf_event_attr *attr = &event->attr;
	struct hw_perf_event *hwc = &event->hw;
	u64 config;

	if (!hwc->sample_period) {
		hwc->sample_period = x86_pmu.max_period;
		hwc->last_period = hwc->sample_period;
		atomic64_set(&hwc->period_left, hwc->sample_period);
	} else {
		/*
		 * If we have a PMU initialized but no APIC
		 * interrupts, we cannot sample hardware
		 * events (user-space has to fall back and
		 * sample via a hrtimer based software event):
		 */
		if (!x86_pmu.apic)
			return -EOPNOTSUPP;
	}

	if (attr->type == PERF_TYPE_RAW)
		return 0;

	if (attr->type == PERF_TYPE_HW_CACHE)
		return set_ext_hw_attr(hwc, attr);

	if (attr->config >= x86_pmu.max_events)
		return -EINVAL;

	/*
	 * The generic map:
	 */
	config = x86_pmu.event_map(attr->config);

	if (config == 0)
		return -ENOENT;

	if (config == -1LL)
		return -EINVAL;

	/*
	 * Branch tracing:
	 */
	if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
	    (hwc->sample_period == 1)) {
		/* BTS is not supported by this architecture. */
		if (!x86_pmu.bts)
			return -EOPNOTSUPP;

		/* BTS is currently only allowed for user-mode. */
		if (!attr->exclude_kernel)
			return -EOPNOTSUPP;
	}

	hwc->config |= config;

	return 0;
}
491

492
static int x86_pmu_hw_config(struct perf_event *event)
493
{
P
Peter Zijlstra 已提交
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
	if (event->attr.precise_ip) {
		int precise = 0;

		/* Support for constant skid */
		if (x86_pmu.pebs)
			precise++;

		/* Support for IP fixup */
		if (x86_pmu.lbr_nr)
			precise++;

		if (event->attr.precise_ip > precise)
			return -EOPNOTSUPP;
	}

509 510 511 512
	/*
	 * Generate PMC IRQs:
	 * (keep 'enabled' bit clear for now)
	 */
513
	event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
514 515 516 517

	/*
	 * Count user and OS events unless requested not to
	 */
518 519 520 521
	if (!event->attr.exclude_user)
		event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
	if (!event->attr.exclude_kernel)
		event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
522

523 524
	if (event->attr.type == PERF_TYPE_RAW)
		event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
525

526
	return x86_setup_perfctr(event);
527 528
}

I
Ingo Molnar 已提交
529
/*
530
 * Setup the hardware configuration for a given attr_type
I
Ingo Molnar 已提交
531
 */
532
static int __hw_perf_event_init(struct perf_event *event)
I
Ingo Molnar 已提交
533
{
P
Peter Zijlstra 已提交
534
	int err;
I
Ingo Molnar 已提交
535

536 537
	if (!x86_pmu_initialized())
		return -ENODEV;
I
Ingo Molnar 已提交
538

P
Peter Zijlstra 已提交
539
	err = 0;
540
	if (!atomic_inc_not_zero(&active_events)) {
P
Peter Zijlstra 已提交
541
		mutex_lock(&pmc_reserve_mutex);
542
		if (atomic_read(&active_events) == 0) {
543 544
			if (!reserve_pmc_hardware())
				err = -EBUSY;
545
			else {
546
				err = reserve_ds_buffers();
547 548 549
				if (err)
					release_pmc_hardware();
			}
550 551
		}
		if (!err)
552
			atomic_inc(&active_events);
P
Peter Zijlstra 已提交
553 554 555 556 557
		mutex_unlock(&pmc_reserve_mutex);
	}
	if (err)
		return err;

558
	event->destroy = hw_perf_event_destroy;
559

560 561 562
	event->hw.idx = -1;
	event->hw.last_cpu = -1;
	event->hw.last_tag = ~0ULL;
563

564
	return x86_pmu.hw_config(event);
565 566
}

567
static void x86_pmu_disable_all(void)
568
{
569
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
570 571
	int idx;

572
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
573 574
		u64 val;

575
		if (!test_bit(idx, cpuc->active_mask))
576
			continue;
577
		rdmsrl(x86_pmu.eventsel + idx, val);
578
		if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
579
			continue;
580
		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
581
		wrmsrl(x86_pmu.eventsel + idx, val);
582 583 584
	}
}

585
void hw_perf_disable(void)
586
{
587 588
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

589
	if (!x86_pmu_initialized())
590
		return;
591

592 593 594 595 596 597
	if (!cpuc->enabled)
		return;

	cpuc->n_added = 0;
	cpuc->enabled = 0;
	barrier();
598 599

	x86_pmu.disable_all();
600
}
I
Ingo Molnar 已提交
601

602
static void x86_pmu_enable_all(int added)
603
{
604
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
605 606
	int idx;

607
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
608
		struct perf_event *event = cpuc->events[idx];
609
		u64 val;
610

611
		if (!test_bit(idx, cpuc->active_mask))
612
			continue;
613

614
		val = event->hw.config;
615
		val |= ARCH_PERFMON_EVENTSEL_ENABLE;
616
		wrmsrl(x86_pmu.eventsel + idx, val);
617 618 619
	}
}

620 621 622 623 624 625 626 627 628
static const struct pmu pmu;

static inline int is_x86_event(struct perf_event *event)
{
	return event->pmu == &pmu;
}

static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
{
629
	struct event_constraint *c, *constraints[X86_PMC_IDX_MAX];
630
	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
631
	int i, j, w, wmax, num = 0;
632 633 634 635 636
	struct hw_perf_event *hwc;

	bitmap_zero(used_mask, X86_PMC_IDX_MAX);

	for (i = 0; i < n; i++) {
637 638
		c = x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
		constraints[i] = c;
639 640
	}

641 642 643
	/*
	 * fastpath, try to reuse previous register
	 */
644
	for (i = 0; i < n; i++) {
645
		hwc = &cpuc->event_list[i]->hw;
646
		c = constraints[i];
647 648 649 650 651 652

		/* never assigned */
		if (hwc->idx == -1)
			break;

		/* constraint still honored */
653
		if (!test_bit(hwc->idx, c->idxmsk))
654 655 656 657 658 659
			break;

		/* not already used */
		if (test_bit(hwc->idx, used_mask))
			break;

P
Peter Zijlstra 已提交
660
		__set_bit(hwc->idx, used_mask);
661 662 663
		if (assign)
			assign[i] = hwc->idx;
	}
664
	if (i == n)
665 666 667 668 669 670 671 672
		goto done;

	/*
	 * begin slow path
	 */

	bitmap_zero(used_mask, X86_PMC_IDX_MAX);

673 674 675 676 677 678 679 680 681
	/*
	 * weight = number of possible counters
	 *
	 * 1    = most constrained, only works on one counter
	 * wmax = least constrained, works on any counter
	 *
	 * assign events to counters starting with most
	 * constrained events.
	 */
682
	wmax = x86_pmu.num_counters;
683 684 685 686 687 688

	/*
	 * when fixed event counters are present,
	 * wmax is incremented by 1 to account
	 * for one more choice
	 */
689
	if (x86_pmu.num_counters_fixed)
690 691
		wmax++;

692
	for (w = 1, num = n; num && w <= wmax; w++) {
693
		/* for each event */
694
		for (i = 0; num && i < n; i++) {
695
			c = constraints[i];
696 697
			hwc = &cpuc->event_list[i]->hw;

698
			if (c->weight != w)
699 700
				continue;

701
			for_each_set_bit(j, c->idxmsk, X86_PMC_IDX_MAX) {
702 703 704 705 706 707 708
				if (!test_bit(j, used_mask))
					break;
			}

			if (j == X86_PMC_IDX_MAX)
				break;

P
Peter Zijlstra 已提交
709
			__set_bit(j, used_mask);
710

711 712 713 714 715
			if (assign)
				assign[i] = j;
			num--;
		}
	}
716
done:
717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
	/*
	 * scheduling failed or is just a simulation,
	 * free resources if necessary
	 */
	if (!assign || num) {
		for (i = 0; i < n; i++) {
			if (x86_pmu.put_event_constraints)
				x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
		}
	}
	return num ? -ENOSPC : 0;
}

/*
 * dogrp: true if must collect siblings events (group)
 * returns total number of events and error code
 */
static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
{
	struct perf_event *event;
	int n, max_count;

739
	max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
740 741 742 743 744 745 746 747 748 749 750 751 752 753 754

	/* current number of events already accepted */
	n = cpuc->n_events;

	if (is_x86_event(leader)) {
		if (n >= max_count)
			return -ENOSPC;
		cpuc->event_list[n] = leader;
		n++;
	}
	if (!dogrp)
		return n;

	list_for_each_entry(event, &leader->sibling_list, group_entry) {
		if (!is_x86_event(event) ||
755
		    event->state <= PERF_EVENT_STATE_OFF)
756 757 758 759 760 761 762 763 764 765 766 767
			continue;

		if (n >= max_count)
			return -ENOSPC;

		cpuc->event_list[n] = event;
		n++;
	}
	return n;
}

static inline void x86_assign_hw_event(struct perf_event *event,
768
				struct cpu_hw_events *cpuc, int i)
769
{
770 771 772 773 774
	struct hw_perf_event *hwc = &event->hw;

	hwc->idx = cpuc->assign[i];
	hwc->last_cpu = smp_processor_id();
	hwc->last_tag = ++cpuc->tags[i];
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792

	if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
		hwc->config_base = 0;
		hwc->event_base	= 0;
	} else if (hwc->idx >= X86_PMC_IDX_FIXED) {
		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
		/*
		 * We set it so that event_base + idx in wrmsr/rdmsr maps to
		 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
		 */
		hwc->event_base =
			MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
	} else {
		hwc->config_base = x86_pmu.eventsel;
		hwc->event_base  = x86_pmu.perfctr;
	}
}

793 794 795 796 797 798 799 800 801
static inline int match_prev_assignment(struct hw_perf_event *hwc,
					struct cpu_hw_events *cpuc,
					int i)
{
	return hwc->idx == cpuc->assign[i] &&
		hwc->last_cpu == smp_processor_id() &&
		hwc->last_tag == cpuc->tags[i];
}

P
Peter Zijlstra 已提交
802
static int x86_pmu_start(struct perf_event *event);
803
static void x86_pmu_stop(struct perf_event *event);
804

805
void hw_perf_enable(void)
806
{
807 808 809
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct perf_event *event;
	struct hw_perf_event *hwc;
810
	int i, added = cpuc->n_added;
811

812
	if (!x86_pmu_initialized())
813
		return;
814 815 816 817

	if (cpuc->enabled)
		return;

818
	if (cpuc->n_added) {
819
		int n_running = cpuc->n_events - cpuc->n_added;
820 821 822 823 824 825 826
		/*
		 * apply assignment obtained either from
		 * hw_perf_group_sched_in() or x86_pmu_enable()
		 *
		 * step1: save events moving to new counters
		 * step2: reprogram moved events into new counters
		 */
827
		for (i = 0; i < n_running; i++) {
828 829 830
			event = cpuc->event_list[i];
			hwc = &event->hw;

831 832 833 834 835 836 837 838
			/*
			 * we can avoid reprogramming counter if:
			 * - assigned same counter as last time
			 * - running on same CPU as last time
			 * - no other event has used the counter since
			 */
			if (hwc->idx == -1 ||
			    match_prev_assignment(hwc, cpuc, i))
839 840
				continue;

841
			x86_pmu_stop(event);
842 843 844 845 846 847
		}

		for (i = 0; i < cpuc->n_events; i++) {
			event = cpuc->event_list[i];
			hwc = &event->hw;

848
			if (!match_prev_assignment(hwc, cpuc, i))
849
				x86_assign_hw_event(event, cpuc, i);
850 851
			else if (i < n_running)
				continue;
852

P
Peter Zijlstra 已提交
853
			x86_pmu_start(event);
854 855 856 857
		}
		cpuc->n_added = 0;
		perf_events_lapic_init();
	}
858 859 860 861

	cpuc->enabled = 1;
	barrier();

862
	x86_pmu.enable_all(added);
863 864
}

865 866
static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
					  u64 enable_mask)
867
{
868
	wrmsrl(hwc->config_base + hwc->idx, hwc->config | enable_mask);
869 870
}

871
static inline void x86_pmu_disable_event(struct perf_event *event)
872
{
873
	struct hw_perf_event *hwc = &event->hw;
874 875

	wrmsrl(hwc->config_base + hwc->idx, hwc->config);
876 877
}

878
static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
I
Ingo Molnar 已提交
879

880 881
/*
 * Set the next IRQ period, based on the hwc->period_left value.
882
 * To be called with the event disabled in hw:
883
 */
884
static int
885
x86_perf_event_set_period(struct perf_event *event)
I
Ingo Molnar 已提交
886
{
887
	struct hw_perf_event *hwc = &event->hw;
888
	s64 left = atomic64_read(&hwc->period_left);
889
	s64 period = hwc->sample_period;
890
	int ret = 0, idx = hwc->idx;
891

892 893 894
	if (idx == X86_PMC_IDX_FIXED_BTS)
		return 0;

895
	/*
896
	 * If we are way outside a reasonable range then just skip forward:
897 898 899 900
	 */
	if (unlikely(left <= -period)) {
		left = period;
		atomic64_set(&hwc->period_left, left);
901
		hwc->last_period = period;
902
		ret = 1;
903 904 905 906 907
	}

	if (unlikely(left <= 0)) {
		left += period;
		atomic64_set(&hwc->period_left, left);
908
		hwc->last_period = period;
909
		ret = 1;
910
	}
911
	/*
912
	 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
913 914 915
	 */
	if (unlikely(left < 2))
		left = 2;
I
Ingo Molnar 已提交
916

917 918 919
	if (left > x86_pmu.max_period)
		left = x86_pmu.max_period;

920
	per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
921 922

	/*
923
	 * The hw event starts counting from this event offset,
924 925
	 * mark it to be able to extra future deltas:
	 */
926
	atomic64_set(&hwc->prev_count, (u64)-left);
927

928
	wrmsrl(hwc->event_base + idx,
929
			(u64)(-left) & x86_pmu.cntval_mask);
930

931
	perf_event_update_userpage(event);
932

933
	return ret;
934 935
}

936
static void x86_pmu_enable_event(struct perf_event *event)
937
{
938
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
939
	if (cpuc->enabled)
940 941
		__x86_pmu_enable_event(&event->hw,
				       ARCH_PERFMON_EVENTSEL_ENABLE);
I
Ingo Molnar 已提交
942 943
}

944
/*
945 946 947 948 949 950 951
 * activate a single event
 *
 * The event is added to the group of enabled events
 * but only if it can be scehduled with existing events.
 *
 * Called with PMU disabled. If successful and return value 1,
 * then guaranteed to call perf_enable() and hw_perf_enable()
952 953 954 955
 */
static int x86_pmu_enable(struct perf_event *event)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
956 957 958
	struct hw_perf_event *hwc;
	int assign[X86_PMC_IDX_MAX];
	int n, n0, ret;
959

960
	hwc = &event->hw;
961

962 963 964 965
	n0 = cpuc->n_events;
	n = collect_events(cpuc, event, false);
	if (n < 0)
		return n;
966

967 968 969 970 971
	/*
	 * If group events scheduling transaction was started,
	 * skip the schedulability test here, it will be peformed
	 * at commit time(->commit_txn) as a whole
	 */
972
	if (cpuc->group_flag & PERF_EVENT_TXN)
973 974
		goto out;

975
	ret = x86_pmu.schedule_events(cpuc, n, assign);
976 977 978 979 980 981 982
	if (ret)
		return ret;
	/*
	 * copy new assignment, now we know it is possible
	 * will be used by hw_perf_enable()
	 */
	memcpy(cpuc->assign, assign, n*sizeof(int));
983

984
out:
985
	cpuc->n_events = n;
986
	cpuc->n_added += n - n0;
987
	cpuc->n_txn += n - n0;
988 989

	return 0;
I
Ingo Molnar 已提交
990 991
}

992 993
static int x86_pmu_start(struct perf_event *event)
{
P
Peter Zijlstra 已提交
994 995 996 997
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int idx = event->hw.idx;

	if (idx == -1)
998 999
		return -EAGAIN;

1000
	x86_perf_event_set_period(event);
P
Peter Zijlstra 已提交
1001 1002
	cpuc->events[idx] = event;
	__set_bit(idx, cpuc->active_mask);
1003
	x86_pmu.enable(event);
P
Peter Zijlstra 已提交
1004
	perf_event_update_userpage(event);
1005 1006 1007 1008

	return 0;
}

1009
static void x86_pmu_unthrottle(struct perf_event *event)
1010
{
1011 1012
	int ret = x86_pmu_start(event);
	WARN_ON_ONCE(ret);
1013 1014
}

1015
void perf_event_print_debug(void)
I
Ingo Molnar 已提交
1016
{
1017
	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1018
	u64 pebs;
1019
	struct cpu_hw_events *cpuc;
1020
	unsigned long flags;
1021 1022
	int cpu, idx;

1023
	if (!x86_pmu.num_counters)
1024
		return;
I
Ingo Molnar 已提交
1025

1026
	local_irq_save(flags);
I
Ingo Molnar 已提交
1027 1028

	cpu = smp_processor_id();
1029
	cpuc = &per_cpu(cpu_hw_events, cpu);
I
Ingo Molnar 已提交
1030

1031
	if (x86_pmu.version >= 2) {
1032 1033 1034 1035
		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1036
		rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1037 1038 1039 1040 1041 1042

		pr_info("\n");
		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1043
		pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
1044
	}
1045
	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
I
Ingo Molnar 已提交
1046

1047
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1048 1049
		rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
		rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
I
Ingo Molnar 已提交
1050

1051
		prev_left = per_cpu(pmc_prev_left[idx], cpu);
I
Ingo Molnar 已提交
1052

1053
		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
I
Ingo Molnar 已提交
1054
			cpu, idx, pmc_ctrl);
1055
		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
I
Ingo Molnar 已提交
1056
			cpu, idx, pmc_count);
1057
		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1058
			cpu, idx, prev_left);
I
Ingo Molnar 已提交
1059
	}
1060
	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1061 1062
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);

1063
		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1064 1065
			cpu, idx, pmc_count);
	}
1066
	local_irq_restore(flags);
I
Ingo Molnar 已提交
1067 1068
}

1069
static void x86_pmu_stop(struct perf_event *event)
I
Ingo Molnar 已提交
1070
{
1071
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1072
	struct hw_perf_event *hwc = &event->hw;
1073
	int idx = hwc->idx;
I
Ingo Molnar 已提交
1074

1075 1076 1077
	if (!__test_and_clear_bit(idx, cpuc->active_mask))
		return;

1078
	x86_pmu.disable(event);
I
Ingo Molnar 已提交
1079

1080
	/*
1081
	 * Drain the remaining delta count out of a event
1082 1083
	 * that we are disabling:
	 */
1084
	x86_perf_event_update(event);
1085

1086
	cpuc->events[idx] = NULL;
1087 1088 1089 1090 1091 1092 1093
}

static void x86_pmu_disable(struct perf_event *event)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int i;

1094 1095 1096 1097 1098
	/*
	 * If we're called during a txn, we don't need to do anything.
	 * The events never got scheduled and ->cancel_txn will truncate
	 * the event_list.
	 */
1099
	if (cpuc->group_flag & PERF_EVENT_TXN)
1100 1101
		return;

1102
	x86_pmu_stop(event);
1103

1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
	for (i = 0; i < cpuc->n_events; i++) {
		if (event == cpuc->event_list[i]) {

			if (x86_pmu.put_event_constraints)
				x86_pmu.put_event_constraints(cpuc, event);

			while (++i < cpuc->n_events)
				cpuc->event_list[i-1] = cpuc->event_list[i];

			--cpuc->n_events;
1114
			break;
1115 1116
		}
	}
1117
	perf_event_update_userpage(event);
I
Ingo Molnar 已提交
1118 1119
}

1120
static int x86_pmu_handle_irq(struct pt_regs *regs)
1121
{
1122
	struct perf_sample_data data;
1123 1124 1125
	struct cpu_hw_events *cpuc;
	struct perf_event *event;
	struct hw_perf_event *hwc;
V
Vince Weaver 已提交
1126
	int idx, handled = 0;
1127 1128
	u64 val;

1129
	perf_sample_data_init(&data, 0);
1130

1131
	cpuc = &__get_cpu_var(cpu_hw_events);
1132

1133
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1134
		if (!test_bit(idx, cpuc->active_mask))
1135
			continue;
1136

1137 1138
		event = cpuc->events[idx];
		hwc = &event->hw;
1139

1140
		val = x86_perf_event_update(event);
1141
		if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1142
			continue;
1143

1144
		/*
1145
		 * event overflow
1146 1147
		 */
		handled		= 1;
1148
		data.period	= event->hw.last_period;
1149

1150
		if (!x86_perf_event_set_period(event))
1151 1152
			continue;

1153
		if (perf_event_overflow(event, 1, &data, regs))
1154
			x86_pmu_stop(event);
1155
	}
1156

1157 1158 1159
	if (handled)
		inc_irq_stat(apic_perf_irqs);

1160 1161
	return handled;
}
1162

1163 1164 1165 1166 1167
void smp_perf_pending_interrupt(struct pt_regs *regs)
{
	irq_enter();
	ack_APIC_irq();
	inc_irq_stat(apic_pending_irqs);
1168
	perf_event_do_pending();
1169 1170 1171
	irq_exit();
}

1172
void set_perf_event_pending(void)
1173
{
1174
#ifdef CONFIG_X86_LOCAL_APIC
1175 1176 1177
	if (!x86_pmu.apic || !x86_pmu_initialized())
		return;

1178
	apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1179
#endif
1180 1181
}

1182
void perf_events_lapic_init(void)
I
Ingo Molnar 已提交
1183
{
1184
	if (!x86_pmu.apic || !x86_pmu_initialized())
I
Ingo Molnar 已提交
1185
		return;
1186

I
Ingo Molnar 已提交
1187
	/*
1188
	 * Always use NMI for PMU
I
Ingo Molnar 已提交
1189
	 */
1190
	apic_write(APIC_LVTPC, APIC_DM_NMI);
I
Ingo Molnar 已提交
1191 1192 1193
}

static int __kprobes
1194
perf_event_nmi_handler(struct notifier_block *self,
I
Ingo Molnar 已提交
1195 1196 1197 1198
			 unsigned long cmd, void *__args)
{
	struct die_args *args = __args;
	struct pt_regs *regs;
1199

1200
	if (!atomic_read(&active_events))
1201 1202
		return NOTIFY_DONE;

1203 1204 1205 1206
	switch (cmd) {
	case DIE_NMI:
	case DIE_NMI_IPI:
		break;
I
Ingo Molnar 已提交
1207

1208
	default:
I
Ingo Molnar 已提交
1209
		return NOTIFY_DONE;
1210
	}
I
Ingo Molnar 已提交
1211 1212 1213 1214

	regs = args->regs;

	apic_write(APIC_LVTPC, APIC_DM_NMI);
1215 1216
	/*
	 * Can't rely on the handled return value to say it was our NMI, two
1217
	 * events could trigger 'simultaneously' raising two back-to-back NMIs.
1218 1219 1220 1221
	 *
	 * If the first NMI handles both, the latter will be empty and daze
	 * the CPU.
	 */
1222
	x86_pmu.handle_irq(regs);
I
Ingo Molnar 已提交
1223

1224
	return NOTIFY_STOP;
I
Ingo Molnar 已提交
1225 1226
}

1227 1228 1229 1230 1231 1232
static __read_mostly struct notifier_block perf_event_nmi_notifier = {
	.notifier_call		= perf_event_nmi_handler,
	.next			= NULL,
	.priority		= 1
};

1233
static struct event_constraint unconstrained;
1234
static struct event_constraint emptyconstraint;
1235 1236

static struct event_constraint *
1237
x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
1238
{
1239
	struct event_constraint *c;
1240 1241 1242

	if (x86_pmu.event_constraints) {
		for_each_event_constraint(c, x86_pmu.event_constraints) {
1243 1244
			if ((event->hw.config & c->cmask) == c->code)
				return c;
1245 1246
		}
	}
1247 1248

	return &unconstrained;
1249 1250
}

1251 1252
#include "perf_event_amd.c"
#include "perf_event_p6.c"
1253
#include "perf_event_p4.c"
1254
#include "perf_event_intel_lbr.c"
1255
#include "perf_event_intel_ds.c"
1256
#include "perf_event_intel.c"
1257

1258 1259 1260 1261
static int __cpuinit
x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;
1262
	int ret = NOTIFY_OK;
1263 1264 1265 1266

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_UP_PREPARE:
		if (x86_pmu.cpu_prepare)
1267
			ret = x86_pmu.cpu_prepare(cpu);
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
		break;

	case CPU_STARTING:
		if (x86_pmu.cpu_starting)
			x86_pmu.cpu_starting(cpu);
		break;

	case CPU_DYING:
		if (x86_pmu.cpu_dying)
			x86_pmu.cpu_dying(cpu);
		break;

1280
	case CPU_UP_CANCELED:
1281 1282 1283 1284 1285 1286 1287 1288 1289
	case CPU_DEAD:
		if (x86_pmu.cpu_dead)
			x86_pmu.cpu_dead(cpu);
		break;

	default:
		break;
	}

1290
	return ret;
1291 1292
}

1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
static void __init pmu_check_apic(void)
{
	if (cpu_has_apic)
		return;

	x86_pmu.apic = 0;
	pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
	pr_info("no hardware sampling interrupt available.\n");
}

1303
void __init init_hw_perf_events(void)
1304
{
1305
	struct event_constraint *c;
1306 1307
	int err;

1308
	pr_info("Performance Events: ");
1309

1310 1311
	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_INTEL:
1312
		err = intel_pmu_init();
1313
		break;
1314
	case X86_VENDOR_AMD:
1315
		err = amd_pmu_init();
1316
		break;
1317 1318
	default:
		return;
1319
	}
1320
	if (err != 0) {
1321
		pr_cont("no PMU driver, software events only.\n");
1322
		return;
1323
	}
1324

1325 1326
	pmu_check_apic();

1327
	pr_cont("%s PMU driver.\n", x86_pmu.name);
1328

1329 1330 1331
	if (x86_pmu.quirks)
		x86_pmu.quirks();

1332
	if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1333
		WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
1334 1335
		     x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
		x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
I
Ingo Molnar 已提交
1336
	}
1337 1338
	x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
	perf_max_events = x86_pmu.num_counters;
I
Ingo Molnar 已提交
1339

1340
	if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1341
		WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
1342 1343
		     x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
		x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1344
	}
1345

1346
	x86_pmu.intel_ctrl |=
1347
		((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
I
Ingo Molnar 已提交
1348

1349 1350
	perf_events_lapic_init();
	register_die_notifier(&perf_event_nmi_notifier);
1351

1352
	unconstrained = (struct event_constraint)
1353 1354
		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
				   0, x86_pmu.num_counters);
1355

1356 1357
	if (x86_pmu.event_constraints) {
		for_each_event_constraint(c, x86_pmu.event_constraints) {
1358
			if (c->cmask != X86_RAW_EVENT_MASK)
1359 1360
				continue;

1361 1362
			c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
			c->weight += x86_pmu.num_counters;
1363 1364 1365
		}
	}

I
Ingo Molnar 已提交
1366
	pr_info("... version:                %d\n",     x86_pmu.version);
1367 1368 1369
	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
	pr_info("... generic registers:      %d\n",     x86_pmu.num_counters);
	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
I
Ingo Molnar 已提交
1370
	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
1371
	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
1372
	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
1373 1374

	perf_cpu_notifier(x86_pmu_notifier);
I
Ingo Molnar 已提交
1375
}
I
Ingo Molnar 已提交
1376

1377
static inline void x86_pmu_read(struct perf_event *event)
1378
{
1379
	x86_perf_event_update(event);
1380 1381
}

1382 1383 1384 1385 1386 1387 1388 1389 1390
/*
 * Start group events scheduling transaction
 * Set the flag to make pmu::enable() not perform the
 * schedulability test, it will be performed at commit time
 */
static void x86_pmu_start_txn(const struct pmu *pmu)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

1391
	cpuc->group_flag |= PERF_EVENT_TXN;
1392
	cpuc->n_txn = 0;
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
}

/*
 * Stop group events scheduling transaction
 * Clear the flag and pmu::enable() will perform the
 * schedulability test.
 */
static void x86_pmu_cancel_txn(const struct pmu *pmu)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

1404
	cpuc->group_flag &= ~PERF_EVENT_TXN;
1405 1406 1407 1408 1409
	/*
	 * Truncate the collected events.
	 */
	cpuc->n_added -= cpuc->n_txn;
	cpuc->n_events -= cpuc->n_txn;
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
}

/*
 * Commit group events scheduling transaction
 * Perform the group schedulability test as a whole
 * Return 0 if success
 */
static int x86_pmu_commit_txn(const struct pmu *pmu)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int assign[X86_PMC_IDX_MAX];
	int n, ret;

	n = cpuc->n_events;

	if (!x86_pmu_initialized())
		return -EAGAIN;

	ret = x86_pmu.schedule_events(cpuc, n, assign);
	if (ret)
		return ret;

	/*
	 * copy new assignment, now we know it is possible
	 * will be used by hw_perf_enable()
	 */
	memcpy(cpuc->assign, assign, n*sizeof(int));

1438
	cpuc->group_flag &= ~PERF_EVENT_TXN;
1439

1440 1441 1442
	return 0;
}

1443 1444 1445
static const struct pmu pmu = {
	.enable		= x86_pmu_enable,
	.disable	= x86_pmu_disable,
1446 1447
	.start		= x86_pmu_start,
	.stop		= x86_pmu_stop,
1448
	.read		= x86_pmu_read,
1449
	.unthrottle	= x86_pmu_unthrottle,
1450 1451 1452
	.start_txn	= x86_pmu_start_txn,
	.cancel_txn	= x86_pmu_cancel_txn,
	.commit_txn	= x86_pmu_commit_txn,
I
Ingo Molnar 已提交
1453 1454
};

1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
/*
 * validate that we can schedule this event
 */
static int validate_event(struct perf_event *event)
{
	struct cpu_hw_events *fake_cpuc;
	struct event_constraint *c;
	int ret = 0;

	fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
	if (!fake_cpuc)
		return -ENOMEM;

	c = x86_pmu.get_event_constraints(fake_cpuc, event);

	if (!c || !c->weight)
		ret = -ENOSPC;

	if (x86_pmu.put_event_constraints)
		x86_pmu.put_event_constraints(fake_cpuc, event);

	kfree(fake_cpuc);

	return ret;
}

1481 1482 1483 1484
/*
 * validate a single event group
 *
 * validation include:
1485 1486 1487
 *	- check events are compatible which each other
 *	- events do not compete for the same counter
 *	- number of events <= number of counters
1488 1489 1490 1491
 *
 * validation ensures the group can be loaded onto the
 * PMU if it was the only group available.
 */
1492 1493
static int validate_group(struct perf_event *event)
{
1494
	struct perf_event *leader = event->group_leader;
1495 1496
	struct cpu_hw_events *fake_cpuc;
	int ret, n;
1497

1498 1499 1500 1501
	ret = -ENOMEM;
	fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
	if (!fake_cpuc)
		goto out;
1502

1503 1504 1505 1506 1507 1508
	/*
	 * the event is not yet connected with its
	 * siblings therefore we must first collect
	 * existing siblings, then add the new event
	 * before we can simulate the scheduling
	 */
1509 1510
	ret = -ENOSPC;
	n = collect_events(fake_cpuc, leader, true);
1511
	if (n < 0)
1512
		goto out_free;
1513

1514 1515
	fake_cpuc->n_events = n;
	n = collect_events(fake_cpuc, event, false);
1516
	if (n < 0)
1517
		goto out_free;
1518

1519
	fake_cpuc->n_events = n;
1520

1521
	ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
1522 1523 1524 1525 1526

out_free:
	kfree(fake_cpuc);
out:
	return ret;
1527 1528
}

1529
const struct pmu *hw_perf_event_init(struct perf_event *event)
I
Ingo Molnar 已提交
1530
{
1531
	const struct pmu *tmp;
I
Ingo Molnar 已提交
1532 1533
	int err;

1534
	err = __hw_perf_event_init(event);
1535
	if (!err) {
1536 1537 1538 1539 1540 1541 1542 1543
		/*
		 * we temporarily connect event to its pmu
		 * such that validate_group() can classify
		 * it as an x86 event using is_x86_event()
		 */
		tmp = event->pmu;
		event->pmu = &pmu;

1544 1545
		if (event->group_leader != event)
			err = validate_group(event);
1546 1547
		else
			err = validate_event(event);
1548 1549

		event->pmu = tmp;
1550
	}
1551
	if (err) {
1552 1553
		if (event->destroy)
			event->destroy(event);
1554
		return ERR_PTR(err);
1555
	}
I
Ingo Molnar 已提交
1556

1557
	return &pmu;
I
Ingo Molnar 已提交
1558
}
1559 1560 1561 1562 1563 1564

/*
 * callchain support
 */

static inline
1565
void callchain_store(struct perf_callchain_entry *entry, u64 ip)
1566
{
1567
	if (entry->nr < PERF_MAX_STACK_DEPTH)
1568 1569 1570
		entry->ip[entry->nr++] = ip;
}

1571 1572
static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587


static void
backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
{
	/* Ignore warnings */
}

static void backtrace_warning(void *data, char *msg)
{
	/* Ignore warnings */
}

static int backtrace_stack(void *data, char *name)
{
1588
	return 0;
1589 1590 1591 1592 1593 1594
}

static void backtrace_address(void *data, unsigned long addr, int reliable)
{
	struct perf_callchain_entry *entry = data;

1595
	callchain_store(entry, addr);
1596 1597 1598 1599 1600 1601 1602
}

static const struct stacktrace_ops backtrace_ops = {
	.warning		= backtrace_warning,
	.warning_symbol		= backtrace_warning_symbol,
	.stack			= backtrace_stack,
	.address		= backtrace_address,
1603
	.walk_stack		= print_context_stack_bp,
1604 1605
};

1606 1607
#include "../dumpstack.h"

1608 1609 1610
static void
perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
1611
	callchain_store(entry, PERF_CONTEXT_KERNEL);
1612
	callchain_store(entry, regs->ip);
1613

1614
	dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
1615 1616
}

1617 1618 1619
#ifdef CONFIG_COMPAT
static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
1620
{
1621 1622 1623
	/* 32-bit process in 64-bit kernel. */
	struct stack_frame_ia32 frame;
	const void __user *fp;
1624

1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
	if (!test_thread_flag(TIF_IA32))
		return 0;

	fp = compat_ptr(regs->bp);
	while (entry->nr < PERF_MAX_STACK_DEPTH) {
		unsigned long bytes;
		frame.next_frame     = 0;
		frame.return_address = 0;

		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
		if (bytes != sizeof(frame))
			break;
1637

1638 1639
		if (fp < compat_ptr(regs->sp))
			break;
1640

1641 1642 1643 1644
		callchain_store(entry, frame.return_address);
		fp = compat_ptr(frame.next_frame);
	}
	return 1;
1645
}
1646 1647 1648 1649 1650 1651 1652
#else
static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
    return 0;
}
#endif
1653 1654 1655 1656 1657 1658 1659

static void
perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
	struct stack_frame frame;
	const void __user *fp;

1660 1661 1662
	if (!user_mode(regs))
		regs = task_pt_regs(current);

1663
	fp = (void __user *)regs->bp;
1664

1665
	callchain_store(entry, PERF_CONTEXT_USER);
1666 1667
	callchain_store(entry, regs->ip);

1668 1669 1670
	if (perf_callchain_user32(regs, entry))
		return;

1671
	while (entry->nr < PERF_MAX_STACK_DEPTH) {
1672
		unsigned long bytes;
1673
		frame.next_frame	     = NULL;
1674 1675
		frame.return_address = 0;

1676 1677
		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
		if (bytes != sizeof(frame))
1678 1679
			break;

1680
		if ((unsigned long)fp < regs->sp)
1681 1682 1683
			break;

		callchain_store(entry, frame.return_address);
1684
		fp = frame.next_frame;
1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
	}
}

static void
perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
	int is_user;

	if (!regs)
		return;

	is_user = user_mode(regs);

	if (is_user && current->state != TASK_RUNNING)
		return;

	if (!is_user)
		perf_callchain_kernel(regs, entry);

	if (current->mm)
		perf_callchain_user(regs, entry);
}

struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
{
	struct perf_callchain_entry *entry;

1712 1713 1714 1715 1716
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
		/* TODO: We don't support guest os callchain now */
		return NULL;
	}

1717
	if (in_nmi())
1718
		entry = &__get_cpu_var(pmc_nmi_entry);
1719
	else
1720
		entry = &__get_cpu_var(pmc_irq_entry);
1721 1722 1723 1724 1725 1726 1727

	entry->nr = 0;

	perf_do_callchain(regs, entry);

	return entry;
}
1728 1729 1730 1731 1732 1733 1734 1735 1736 1737

void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip)
{
	regs->ip = ip;
	/*
	 * perf_arch_fetch_caller_regs adds another call, we need to increment
	 * the skip level
	 */
	regs->bp = rewind_frame_pointer(skip + 1);
	regs->cs = __KERNEL_CS;
P
Peter Zijlstra 已提交
1738 1739 1740 1741 1742
	/*
	 * We abuse bit 3 to pass exact information, see perf_misc_flags
	 * and the comment with PERF_EFLAGS_EXACT.
	 */
	regs->flags = 0;
1743
}
1744 1745 1746 1747

unsigned long perf_instruction_pointer(struct pt_regs *regs)
{
	unsigned long ip;
1748

1749 1750 1751 1752
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
		ip = perf_guest_cbs->get_guest_ip();
	else
		ip = instruction_pointer(regs);
1753

1754 1755 1756 1757 1758 1759
	return ip;
}

unsigned long perf_misc_flags(struct pt_regs *regs)
{
	int misc = 0;
1760

1761
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772
		if (perf_guest_cbs->is_user_mode())
			misc |= PERF_RECORD_MISC_GUEST_USER;
		else
			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
	} else {
		if (user_mode(regs))
			misc |= PERF_RECORD_MISC_USER;
		else
			misc |= PERF_RECORD_MISC_KERNEL;
	}

1773
	if (regs->flags & PERF_EFLAGS_EXACT)
P
Peter Zijlstra 已提交
1774
		misc |= PERF_RECORD_MISC_EXACT_IP;
1775 1776 1777

	return misc;
}