perf_event.c 40.7 KB
Newer Older
I
Ingo Molnar 已提交
1
/*
2
 * Performance events x86 architecture code
I
Ingo Molnar 已提交
3
 *
4 5 6 7 8
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2009 Jaswinder Singh Rajput
 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9
 *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10
 *  Copyright (C) 2009 Google, Inc., Stephane Eranian
I
Ingo Molnar 已提交
11 12 13 14
 *
 *  For licencing details see kernel-base/COPYING
 */

15
#include <linux/perf_event.h>
I
Ingo Molnar 已提交
16 17 18 19
#include <linux/capability.h>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
20
#include <linux/module.h>
I
Ingo Molnar 已提交
21 22
#include <linux/kdebug.h>
#include <linux/sched.h>
23
#include <linux/uaccess.h>
24
#include <linux/slab.h>
25
#include <linux/highmem.h>
26
#include <linux/cpu.h>
27
#include <linux/bitops.h>
I
Ingo Molnar 已提交
28 29

#include <asm/apic.h>
30
#include <asm/stacktrace.h>
P
Peter Zijlstra 已提交
31
#include <asm/nmi.h>
32
#include <asm/compat.h>
I
Ingo Molnar 已提交
33

34 35 36 37 38 39 40 41 42 43 44
#if 0
#undef wrmsrl
#define wrmsrl(msr, val) 					\
do {								\
	trace_printk("wrmsrl(%lx, %lx)\n", (unsigned long)(msr),\
			(unsigned long)(val));			\
	native_write_msr((msr), (u32)((u64)(val)), 		\
			(u32)((u64)(val) >> 32));		\
} while (0)
#endif

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
/*
 * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
 */
static unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
{
	unsigned long offset, addr = (unsigned long)from;
	unsigned long size, len = 0;
	struct page *page;
	void *map;
	int ret;

	do {
		ret = __get_user_pages_fast(addr, 1, 0, &page);
		if (!ret)
			break;

		offset = addr & (PAGE_SIZE - 1);
		size = min(PAGE_SIZE - offset, n - len);

65
		map = kmap_atomic(page);
66
		memcpy(to, map+offset, size);
67
		kunmap_atomic(map);
68 69 70 71 72 73 74 75 76 77 78
		put_page(page);

		len  += size;
		to   += size;
		addr += size;

	} while (len < n);

	return len;
}

79
struct event_constraint {
80 81
	union {
		unsigned long	idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
82
		u64		idxmsk64;
83
	};
84 85
	u64	code;
	u64	cmask;
86
	int	weight;
87 88
};

89 90 91 92 93 94 95
struct amd_nb {
	int nb_id;  /* NorthBridge id */
	int refcnt; /* reference count */
	struct perf_event *owners[X86_PMC_IDX_MAX];
	struct event_constraint event_constraints[X86_PMC_IDX_MAX];
};

96 97
#define MAX_LBR_ENTRIES		16

98
struct cpu_hw_events {
99 100 101
	/*
	 * Generic x86 PMC bits
	 */
102
	struct perf_event	*events[X86_PMC_IDX_MAX]; /* in counter order */
103
	unsigned long		active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
104
	unsigned long		running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
105
	int			enabled;
I
Ingo Molnar 已提交
106

107 108
	int			n_events;
	int			n_added;
109
	int			n_txn;
110
	int			assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
111
	u64			tags[X86_PMC_IDX_MAX];
112
	struct perf_event	*event_list[X86_PMC_IDX_MAX]; /* in enabled order */
113

114 115
	unsigned int		group_flag;

116 117 118 119 120 121
	/*
	 * Intel DebugStore bits
	 */
	struct debug_store	*ds;
	u64			pebs_enabled;

122 123 124 125 126 127 128 129
	/*
	 * Intel LBR bits
	 */
	int				lbr_users;
	void				*lbr_context;
	struct perf_branch_stack	lbr_stack;
	struct perf_branch_entry	lbr_entries[MAX_LBR_ENTRIES];

130 131 132
	/*
	 * AMD specific bits
	 */
133
	struct amd_nb		*amd_nb;
134 135
};

136
#define __EVENT_CONSTRAINT(c, n, m, w) {\
137
	{ .idxmsk64 = (n) },		\
138 139
	.code = (c),			\
	.cmask = (m),			\
140
	.weight = (w),			\
141
}
142

143 144 145
#define EVENT_CONSTRAINT(c, n, m)	\
	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n))

146 147 148
/*
 * Constraint on the Event code.
 */
149
#define INTEL_EVENT_CONSTRAINT(c, n)	\
150
	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
151

152 153
/*
 * Constraint on the Event code + UMask + fixed-mask
154 155 156 157 158 159 160 161
 *
 * filter mask to validate fixed counter events.
 * the following filters disqualify for fixed counters:
 *  - inv
 *  - edge
 *  - cnt-mask
 *  The other filters are supported by fixed counters.
 *  The any-thread option is supported starting with v3.
162
 */
163
#define FIXED_EVENT_CONSTRAINT(c, n)	\
164
	EVENT_CONSTRAINT(c, (1ULL << (32+n)), X86_RAW_EVENT_MASK)
165

166 167 168
/*
 * Constraint on the Event code + UMask
 */
169
#define INTEL_UEVENT_CONSTRAINT(c, n)	\
170
	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
171 172
#define PEBS_EVENT_CONSTRAINT(c, n)	\
	INTEL_UEVENT_CONSTRAINT(c, n)
173

174 175 176 177
#define EVENT_CONSTRAINT_END		\
	EVENT_CONSTRAINT(0, 0, 0)

#define for_each_event_constraint(e, c)	\
178
	for ((e) = (c); (e)->weight; (e)++)
179

180 181 182 183 184 185 186 187 188 189 190
union perf_capabilities {
	struct {
		u64	lbr_format    : 6;
		u64	pebs_trap     : 1;
		u64	pebs_arch_reg : 1;
		u64	pebs_format   : 4;
		u64	smm_freeze    : 1;
	};
	u64	capabilities;
};

I
Ingo Molnar 已提交
191
/*
192
 * struct x86_pmu - generic x86 pmu
I
Ingo Molnar 已提交
193
 */
194
struct x86_pmu {
195 196 197
	/*
	 * Generic x86 PMC bits
	 */
198 199
	const char	*name;
	int		version;
200
	int		(*handle_irq)(struct pt_regs *);
201
	void		(*disable_all)(void);
202
	void		(*enable_all)(int added);
203 204
	void		(*enable)(struct perf_event *);
	void		(*disable)(struct perf_event *);
205
	int		(*hw_config)(struct perf_event *event);
206
	int		(*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
207 208
	unsigned	eventsel;
	unsigned	perfctr;
209
	u64		(*event_map)(int);
210
	int		max_events;
211 212 213 214
	int		num_counters;
	int		num_counters_fixed;
	int		cntval_bits;
	u64		cntval_mask;
215
	int		apic;
216
	u64		max_period;
217 218 219 220
	struct event_constraint *
			(*get_event_constraints)(struct cpu_hw_events *cpuc,
						 struct perf_event *event);

221 222
	void		(*put_event_constraints)(struct cpu_hw_events *cpuc,
						 struct perf_event *event);
223
	struct event_constraint *event_constraints;
224
	void		(*quirks)(void);
225
	int		perfctr_second_write;
226

227
	int		(*cpu_prepare)(int cpu);
228 229 230
	void		(*cpu_starting)(int cpu);
	void		(*cpu_dying)(int cpu);
	void		(*cpu_dead)(int cpu);
231 232 233 234

	/*
	 * Intel Arch Perfmon v2+
	 */
235 236
	u64			intel_ctrl;
	union perf_capabilities intel_cap;
237 238 239 240 241

	/*
	 * Intel DebugStore bits
	 */
	int		bts, pebs;
242
	int		bts_active, pebs_active;
243 244 245
	int		pebs_record_size;
	void		(*drain_pebs)(struct pt_regs *regs);
	struct event_constraint *pebs_constraints;
246 247 248 249 250 251

	/*
	 * Intel LBR
	 */
	unsigned long	lbr_tos, lbr_from, lbr_to; /* MSR base regs       */
	int		lbr_nr;			   /* hardware stack size */
252 253
};

254
static struct x86_pmu x86_pmu __read_mostly;
255

256
static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
257 258
	.enabled = 1,
};
I
Ingo Molnar 已提交
259

260
static int x86_perf_event_set_period(struct perf_event *event);
261

262
/*
263
 * Generalized hw caching related hw_event table, filled
264
 * in on a per model basis. A value of 0 means
265 266
 * 'not supported', -1 means 'hw_event makes no sense on
 * this CPU', any other value means the raw hw_event
267 268 269 270 271 272 273 274 275 276
 * ID.
 */

#define C(x) PERF_COUNT_HW_CACHE_##x

static u64 __read_mostly hw_cache_event_ids
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX];

277
/*
278 279
 * Propagate event elapsed time into the generic event.
 * Can only be executed on the CPU where the event is active.
280 281
 * Returns the delta events processed.
 */
282
static u64
283
x86_perf_event_update(struct perf_event *event)
284
{
285
	struct hw_perf_event *hwc = &event->hw;
286
	int shift = 64 - x86_pmu.cntval_bits;
287
	u64 prev_raw_count, new_raw_count;
288
	int idx = hwc->idx;
289
	s64 delta;
290

291 292 293
	if (idx == X86_PMC_IDX_FIXED_BTS)
		return 0;

294
	/*
295
	 * Careful: an NMI might modify the previous event value.
296 297 298
	 *
	 * Our tactic to handle this is to first atomically read and
	 * exchange a new raw count - then add that new-prev delta
299
	 * count to the generic event atomically:
300 301
	 */
again:
302
	prev_raw_count = local64_read(&hwc->prev_count);
303
	rdmsrl(hwc->event_base, new_raw_count);
304

305
	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
306 307 308 309 310 311
					new_raw_count) != prev_raw_count)
		goto again;

	/*
	 * Now we have the new raw value and have updated the prev
	 * timestamp already. We can now calculate the elapsed delta
312
	 * (event-)time and add that to the generic event.
313 314
	 *
	 * Careful, not all hw sign-extends above the physical width
315
	 * of the count.
316
	 */
317 318
	delta = (new_raw_count << shift) - (prev_raw_count << shift);
	delta >>= shift;
319

320 321
	local64_add(delta, &event->count);
	local64_sub(delta, &hwc->period_left);
322 323

	return new_raw_count;
324 325
}

326 327 328 329 330 331 332 333
/* using X86_FEATURE_PERFCTR_CORE to later implement ALTERNATIVE() here */
static inline int x86_pmu_addr_offset(int index)
{
	if (boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
		return index << 1;
	return index;
}

334 335
static inline unsigned int x86_pmu_config_addr(int index)
{
336
	return x86_pmu.eventsel + x86_pmu_addr_offset(index);
337 338 339 340
}

static inline unsigned int x86_pmu_event_addr(int index)
{
341
	return x86_pmu.perfctr + x86_pmu_addr_offset(index);
342 343
}

344
static atomic_t active_events;
P
Peter Zijlstra 已提交
345 346
static DEFINE_MUTEX(pmc_reserve_mutex);

347 348
#ifdef CONFIG_X86_LOCAL_APIC

P
Peter Zijlstra 已提交
349 350 351 352
static bool reserve_pmc_hardware(void)
{
	int i;

353
	for (i = 0; i < x86_pmu.num_counters; i++) {
354
		if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
P
Peter Zijlstra 已提交
355 356 357
			goto perfctr_fail;
	}

358
	for (i = 0; i < x86_pmu.num_counters; i++) {
359
		if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
P
Peter Zijlstra 已提交
360 361 362 363 364 365 366
			goto eventsel_fail;
	}

	return true;

eventsel_fail:
	for (i--; i >= 0; i--)
367
		release_evntsel_nmi(x86_pmu_config_addr(i));
P
Peter Zijlstra 已提交
368

369
	i = x86_pmu.num_counters;
P
Peter Zijlstra 已提交
370 371 372

perfctr_fail:
	for (i--; i >= 0; i--)
373
		release_perfctr_nmi(x86_pmu_event_addr(i));
P
Peter Zijlstra 已提交
374 375 376 377 378 379 380 381

	return false;
}

static void release_pmc_hardware(void)
{
	int i;

382
	for (i = 0; i < x86_pmu.num_counters; i++) {
383 384
		release_perfctr_nmi(x86_pmu_event_addr(i));
		release_evntsel_nmi(x86_pmu_config_addr(i));
P
Peter Zijlstra 已提交
385 386 387
	}
}

388 389 390 391 392 393 394
#else

static bool reserve_pmc_hardware(void) { return true; }
static void release_pmc_hardware(void) {}

#endif

395 396 397
static bool check_hw_exists(void)
{
	u64 val, val_new = 0;
398
	int i, reg, ret = 0;
399

400 401 402 403 404
	/*
	 * Check to see if the BIOS enabled any of the counters, if so
	 * complain and bail.
	 */
	for (i = 0; i < x86_pmu.num_counters; i++) {
405
		reg = x86_pmu_config_addr(i);
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
		ret = rdmsrl_safe(reg, &val);
		if (ret)
			goto msr_fail;
		if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
			goto bios_fail;
	}

	if (x86_pmu.num_counters_fixed) {
		reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
		ret = rdmsrl_safe(reg, &val);
		if (ret)
			goto msr_fail;
		for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
			if (val & (0x03 << i*4))
				goto bios_fail;
		}
	}

	/*
	 * Now write a value and read it back to see if it matches,
	 * this is needed to detect certain hardware emulators (qemu/kvm)
	 * that don't trap on the MSR access and always return 0s.
	 */
429
	val = 0xabcdUL;
430 431
	ret = checking_wrmsrl(x86_pmu_event_addr(0), val);
	ret |= rdmsrl_safe(x86_pmu_event_addr(0), &val_new);
432
	if (ret || val != val_new)
433
		goto msr_fail;
434 435

	return true;
436 437 438 439 440 441 442 443 444

bios_fail:
	printk(KERN_CONT "Broken BIOS detected, using software events only.\n");
	printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg, val);
	return false;

msr_fail:
	printk(KERN_CONT "Broken PMU hardware detected, using software events only.\n");
	return false;
445 446
}

447
static void reserve_ds_buffers(void);
448
static void release_ds_buffers(void);
449

450
static void hw_perf_event_destroy(struct perf_event *event)
P
Peter Zijlstra 已提交
451
{
452
	if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
P
Peter Zijlstra 已提交
453
		release_pmc_hardware();
454
		release_ds_buffers();
P
Peter Zijlstra 已提交
455 456 457 458
		mutex_unlock(&pmc_reserve_mutex);
	}
}

459 460 461 462 463
static inline int x86_pmu_initialized(void)
{
	return x86_pmu.handle_irq != NULL;
}

464
static inline int
465
set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
{
	unsigned int cache_type, cache_op, cache_result;
	u64 config, val;

	config = attr->config;

	cache_type = (config >>  0) & 0xff;
	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
		return -EINVAL;

	cache_op = (config >>  8) & 0xff;
	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
		return -EINVAL;

	cache_result = (config >> 16) & 0xff;
	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
		return -EINVAL;

	val = hw_cache_event_ids[cache_type][cache_op][cache_result];

	if (val == 0)
		return -ENOENT;

	if (val == -1)
		return -EINVAL;

	hwc->config |= val;

	return 0;
}

497 498 499 500 501 502
static int x86_setup_perfctr(struct perf_event *event)
{
	struct perf_event_attr *attr = &event->attr;
	struct hw_perf_event *hwc = &event->hw;
	u64 config;

503
	if (!is_sampling_event(event)) {
504 505
		hwc->sample_period = x86_pmu.max_period;
		hwc->last_period = hwc->sample_period;
506
		local64_set(&hwc->period_left, hwc->sample_period);
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
	} else {
		/*
		 * If we have a PMU initialized but no APIC
		 * interrupts, we cannot sample hardware
		 * events (user-space has to fall back and
		 * sample via a hrtimer based software event):
		 */
		if (!x86_pmu.apic)
			return -EOPNOTSUPP;
	}

	if (attr->type == PERF_TYPE_RAW)
		return 0;

	if (attr->type == PERF_TYPE_HW_CACHE)
		return set_ext_hw_attr(hwc, attr);

	if (attr->config >= x86_pmu.max_events)
		return -EINVAL;

	/*
	 * The generic map:
	 */
	config = x86_pmu.event_map(attr->config);

	if (config == 0)
		return -ENOENT;

	if (config == -1LL)
		return -EINVAL;

	/*
	 * Branch tracing:
	 */
	if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
	    (hwc->sample_period == 1)) {
		/* BTS is not supported by this architecture. */
544
		if (!x86_pmu.bts_active)
545 546 547 548 549 550 551 552 553 554 555
			return -EOPNOTSUPP;

		/* BTS is currently only allowed for user-mode. */
		if (!attr->exclude_kernel)
			return -EOPNOTSUPP;
	}

	hwc->config |= config;

	return 0;
}
556

557
static int x86_pmu_hw_config(struct perf_event *event)
558
{
P
Peter Zijlstra 已提交
559 560 561 562
	if (event->attr.precise_ip) {
		int precise = 0;

		/* Support for constant skid */
563
		if (x86_pmu.pebs_active) {
P
Peter Zijlstra 已提交
564 565
			precise++;

566 567 568 569
			/* Support for IP fixup */
			if (x86_pmu.lbr_nr)
				precise++;
		}
P
Peter Zijlstra 已提交
570 571 572 573 574

		if (event->attr.precise_ip > precise)
			return -EOPNOTSUPP;
	}

575 576 577 578
	/*
	 * Generate PMC IRQs:
	 * (keep 'enabled' bit clear for now)
	 */
579
	event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
580 581 582 583

	/*
	 * Count user and OS events unless requested not to
	 */
584 585 586 587
	if (!event->attr.exclude_user)
		event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
	if (!event->attr.exclude_kernel)
		event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
588

589 590
	if (event->attr.type == PERF_TYPE_RAW)
		event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
591

592
	return x86_setup_perfctr(event);
593 594
}

I
Ingo Molnar 已提交
595
/*
596
 * Setup the hardware configuration for a given attr_type
I
Ingo Molnar 已提交
597
 */
598
static int __x86_pmu_event_init(struct perf_event *event)
I
Ingo Molnar 已提交
599
{
P
Peter Zijlstra 已提交
600
	int err;
I
Ingo Molnar 已提交
601

602 603
	if (!x86_pmu_initialized())
		return -ENODEV;
I
Ingo Molnar 已提交
604

P
Peter Zijlstra 已提交
605
	err = 0;
606
	if (!atomic_inc_not_zero(&active_events)) {
P
Peter Zijlstra 已提交
607
		mutex_lock(&pmc_reserve_mutex);
608
		if (atomic_read(&active_events) == 0) {
609 610
			if (!reserve_pmc_hardware())
				err = -EBUSY;
611 612
			else
				reserve_ds_buffers();
613 614
		}
		if (!err)
615
			atomic_inc(&active_events);
P
Peter Zijlstra 已提交
616 617 618 619 620
		mutex_unlock(&pmc_reserve_mutex);
	}
	if (err)
		return err;

621
	event->destroy = hw_perf_event_destroy;
622

623 624 625
	event->hw.idx = -1;
	event->hw.last_cpu = -1;
	event->hw.last_tag = ~0ULL;
626

627
	return x86_pmu.hw_config(event);
628 629
}

630
static void x86_pmu_disable_all(void)
631
{
632
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
633 634
	int idx;

635
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
636 637
		u64 val;

638
		if (!test_bit(idx, cpuc->active_mask))
639
			continue;
640
		rdmsrl(x86_pmu_config_addr(idx), val);
641
		if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
642
			continue;
643
		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
644
		wrmsrl(x86_pmu_config_addr(idx), val);
645 646 647
	}
}

P
Peter Zijlstra 已提交
648
static void x86_pmu_disable(struct pmu *pmu)
649
{
650 651
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

652
	if (!x86_pmu_initialized())
653
		return;
654

655 656 657 658 659 660
	if (!cpuc->enabled)
		return;

	cpuc->n_added = 0;
	cpuc->enabled = 0;
	barrier();
661 662

	x86_pmu.disable_all();
663
}
I
Ingo Molnar 已提交
664

665 666 667
static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
					  u64 enable_mask)
{
668
	wrmsrl(hwc->config_base, hwc->config | enable_mask);
669 670
}

671
static void x86_pmu_enable_all(int added)
672
{
673
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
674 675
	int idx;

676
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
677
		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
678

679
		if (!test_bit(idx, cpuc->active_mask))
680
			continue;
681

682
		__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
683 684 685
	}
}

P
Peter Zijlstra 已提交
686
static struct pmu pmu;
687 688 689 690 691 692 693 694

static inline int is_x86_event(struct perf_event *event)
{
	return event->pmu == &pmu;
}

static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
{
695
	struct event_constraint *c, *constraints[X86_PMC_IDX_MAX];
696
	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
697
	int i, j, w, wmax, num = 0;
698 699 700 701 702
	struct hw_perf_event *hwc;

	bitmap_zero(used_mask, X86_PMC_IDX_MAX);

	for (i = 0; i < n; i++) {
703 704
		c = x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
		constraints[i] = c;
705 706
	}

707 708 709
	/*
	 * fastpath, try to reuse previous register
	 */
710
	for (i = 0; i < n; i++) {
711
		hwc = &cpuc->event_list[i]->hw;
712
		c = constraints[i];
713 714 715 716 717 718

		/* never assigned */
		if (hwc->idx == -1)
			break;

		/* constraint still honored */
719
		if (!test_bit(hwc->idx, c->idxmsk))
720 721 722 723 724 725
			break;

		/* not already used */
		if (test_bit(hwc->idx, used_mask))
			break;

P
Peter Zijlstra 已提交
726
		__set_bit(hwc->idx, used_mask);
727 728 729
		if (assign)
			assign[i] = hwc->idx;
	}
730
	if (i == n)
731 732 733 734 735 736 737 738
		goto done;

	/*
	 * begin slow path
	 */

	bitmap_zero(used_mask, X86_PMC_IDX_MAX);

739 740 741 742 743 744 745 746 747
	/*
	 * weight = number of possible counters
	 *
	 * 1    = most constrained, only works on one counter
	 * wmax = least constrained, works on any counter
	 *
	 * assign events to counters starting with most
	 * constrained events.
	 */
748
	wmax = x86_pmu.num_counters;
749 750 751 752 753 754

	/*
	 * when fixed event counters are present,
	 * wmax is incremented by 1 to account
	 * for one more choice
	 */
755
	if (x86_pmu.num_counters_fixed)
756 757
		wmax++;

758
	for (w = 1, num = n; num && w <= wmax; w++) {
759
		/* for each event */
760
		for (i = 0; num && i < n; i++) {
761
			c = constraints[i];
762 763
			hwc = &cpuc->event_list[i]->hw;

764
			if (c->weight != w)
765 766
				continue;

767
			for_each_set_bit(j, c->idxmsk, X86_PMC_IDX_MAX) {
768 769 770 771 772 773 774
				if (!test_bit(j, used_mask))
					break;
			}

			if (j == X86_PMC_IDX_MAX)
				break;

P
Peter Zijlstra 已提交
775
			__set_bit(j, used_mask);
776

777 778 779 780 781
			if (assign)
				assign[i] = j;
			num--;
		}
	}
782
done:
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
	/*
	 * scheduling failed or is just a simulation,
	 * free resources if necessary
	 */
	if (!assign || num) {
		for (i = 0; i < n; i++) {
			if (x86_pmu.put_event_constraints)
				x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
		}
	}
	return num ? -ENOSPC : 0;
}

/*
 * dogrp: true if must collect siblings events (group)
 * returns total number of events and error code
 */
static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
{
	struct perf_event *event;
	int n, max_count;

805
	max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
806 807 808 809 810 811 812 813 814 815 816 817 818 819 820

	/* current number of events already accepted */
	n = cpuc->n_events;

	if (is_x86_event(leader)) {
		if (n >= max_count)
			return -ENOSPC;
		cpuc->event_list[n] = leader;
		n++;
	}
	if (!dogrp)
		return n;

	list_for_each_entry(event, &leader->sibling_list, group_entry) {
		if (!is_x86_event(event) ||
821
		    event->state <= PERF_EVENT_STATE_OFF)
822 823 824 825 826 827 828 829 830 831 832 833
			continue;

		if (n >= max_count)
			return -ENOSPC;

		cpuc->event_list[n] = event;
		n++;
	}
	return n;
}

static inline void x86_assign_hw_event(struct perf_event *event,
834
				struct cpu_hw_events *cpuc, int i)
835
{
836 837 838 839 840
	struct hw_perf_event *hwc = &event->hw;

	hwc->idx = cpuc->assign[i];
	hwc->last_cpu = smp_processor_id();
	hwc->last_tag = ++cpuc->tags[i];
841 842 843 844 845 846

	if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
		hwc->config_base = 0;
		hwc->event_base	= 0;
	} else if (hwc->idx >= X86_PMC_IDX_FIXED) {
		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
847
		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0;
848
	} else {
849 850
		hwc->config_base = x86_pmu_config_addr(hwc->idx);
		hwc->event_base  = x86_pmu_event_addr(hwc->idx);
851 852 853
	}
}

854 855 856 857 858 859 860 861 862
static inline int match_prev_assignment(struct hw_perf_event *hwc,
					struct cpu_hw_events *cpuc,
					int i)
{
	return hwc->idx == cpuc->assign[i] &&
		hwc->last_cpu == smp_processor_id() &&
		hwc->last_tag == cpuc->tags[i];
}

P
Peter Zijlstra 已提交
863 864
static void x86_pmu_start(struct perf_event *event, int flags);
static void x86_pmu_stop(struct perf_event *event, int flags);
865

P
Peter Zijlstra 已提交
866
static void x86_pmu_enable(struct pmu *pmu)
867
{
868 869 870
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct perf_event *event;
	struct hw_perf_event *hwc;
871
	int i, added = cpuc->n_added;
872

873
	if (!x86_pmu_initialized())
874
		return;
875 876 877 878

	if (cpuc->enabled)
		return;

879
	if (cpuc->n_added) {
880
		int n_running = cpuc->n_events - cpuc->n_added;
881 882 883 884 885 886 887
		/*
		 * apply assignment obtained either from
		 * hw_perf_group_sched_in() or x86_pmu_enable()
		 *
		 * step1: save events moving to new counters
		 * step2: reprogram moved events into new counters
		 */
888
		for (i = 0; i < n_running; i++) {
889 890 891
			event = cpuc->event_list[i];
			hwc = &event->hw;

892 893 894 895 896 897 898 899
			/*
			 * we can avoid reprogramming counter if:
			 * - assigned same counter as last time
			 * - running on same CPU as last time
			 * - no other event has used the counter since
			 */
			if (hwc->idx == -1 ||
			    match_prev_assignment(hwc, cpuc, i))
900 901
				continue;

P
Peter Zijlstra 已提交
902 903 904 905 906 907 908 909
			/*
			 * Ensure we don't accidentally enable a stopped
			 * counter simply because we rescheduled.
			 */
			if (hwc->state & PERF_HES_STOPPED)
				hwc->state |= PERF_HES_ARCH;

			x86_pmu_stop(event, PERF_EF_UPDATE);
910 911 912 913 914 915
		}

		for (i = 0; i < cpuc->n_events; i++) {
			event = cpuc->event_list[i];
			hwc = &event->hw;

916
			if (!match_prev_assignment(hwc, cpuc, i))
917
				x86_assign_hw_event(event, cpuc, i);
918 919
			else if (i < n_running)
				continue;
920

P
Peter Zijlstra 已提交
921 922 923 924
			if (hwc->state & PERF_HES_ARCH)
				continue;

			x86_pmu_start(event, PERF_EF_RELOAD);
925 926 927 928
		}
		cpuc->n_added = 0;
		perf_events_lapic_init();
	}
929 930 931 932

	cpuc->enabled = 1;
	barrier();

933
	x86_pmu.enable_all(added);
934 935
}

936
static inline void x86_pmu_disable_event(struct perf_event *event)
937
{
938
	struct hw_perf_event *hwc = &event->hw;
939

940
	wrmsrl(hwc->config_base, hwc->config);
941 942
}

943
static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
I
Ingo Molnar 已提交
944

945 946
/*
 * Set the next IRQ period, based on the hwc->period_left value.
947
 * To be called with the event disabled in hw:
948
 */
949
static int
950
x86_perf_event_set_period(struct perf_event *event)
I
Ingo Molnar 已提交
951
{
952
	struct hw_perf_event *hwc = &event->hw;
953
	s64 left = local64_read(&hwc->period_left);
954
	s64 period = hwc->sample_period;
955
	int ret = 0, idx = hwc->idx;
956

957 958 959
	if (idx == X86_PMC_IDX_FIXED_BTS)
		return 0;

960
	/*
961
	 * If we are way outside a reasonable range then just skip forward:
962 963 964
	 */
	if (unlikely(left <= -period)) {
		left = period;
965
		local64_set(&hwc->period_left, left);
966
		hwc->last_period = period;
967
		ret = 1;
968 969 970 971
	}

	if (unlikely(left <= 0)) {
		left += period;
972
		local64_set(&hwc->period_left, left);
973
		hwc->last_period = period;
974
		ret = 1;
975
	}
976
	/*
977
	 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
978 979 980
	 */
	if (unlikely(left < 2))
		left = 2;
I
Ingo Molnar 已提交
981

982 983 984
	if (left > x86_pmu.max_period)
		left = x86_pmu.max_period;

985
	per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
986 987

	/*
988
	 * The hw event starts counting from this event offset,
989 990
	 * mark it to be able to extra future deltas:
	 */
991
	local64_set(&hwc->prev_count, (u64)-left);
992

993
	wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
994 995 996 997 998 999 1000

	/*
	 * Due to erratum on certan cpu we need
	 * a second write to be sure the register
	 * is updated properly
	 */
	if (x86_pmu.perfctr_second_write) {
1001
		wrmsrl(hwc->event_base,
1002
			(u64)(-left) & x86_pmu.cntval_mask);
1003
	}
1004

1005
	perf_event_update_userpage(event);
1006

1007
	return ret;
1008 1009
}

1010
static void x86_pmu_enable_event(struct perf_event *event)
1011
{
T
Tejun Heo 已提交
1012
	if (__this_cpu_read(cpu_hw_events.enabled))
1013 1014
		__x86_pmu_enable_event(&event->hw,
				       ARCH_PERFMON_EVENTSEL_ENABLE);
I
Ingo Molnar 已提交
1015 1016
}

1017
/*
P
Peter Zijlstra 已提交
1018
 * Add a single event to the PMU.
1019 1020 1021
 *
 * The event is added to the group of enabled events
 * but only if it can be scehduled with existing events.
1022
 */
P
Peter Zijlstra 已提交
1023
static int x86_pmu_add(struct perf_event *event, int flags)
1024 1025
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1026 1027 1028
	struct hw_perf_event *hwc;
	int assign[X86_PMC_IDX_MAX];
	int n, n0, ret;
1029

1030
	hwc = &event->hw;
1031

P
Peter Zijlstra 已提交
1032
	perf_pmu_disable(event->pmu);
1033
	n0 = cpuc->n_events;
1034 1035 1036
	ret = n = collect_events(cpuc, event, false);
	if (ret < 0)
		goto out;
1037

P
Peter Zijlstra 已提交
1038 1039 1040 1041
	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
	if (!(flags & PERF_EF_START))
		hwc->state |= PERF_HES_ARCH;

1042 1043 1044
	/*
	 * If group events scheduling transaction was started,
	 * skip the schedulability test here, it will be peformed
P
Peter Zijlstra 已提交
1045
	 * at commit time (->commit_txn) as a whole
1046
	 */
1047
	if (cpuc->group_flag & PERF_EVENT_TXN)
1048
		goto done_collect;
1049

1050
	ret = x86_pmu.schedule_events(cpuc, n, assign);
1051
	if (ret)
1052
		goto out;
1053 1054 1055 1056 1057
	/*
	 * copy new assignment, now we know it is possible
	 * will be used by hw_perf_enable()
	 */
	memcpy(cpuc->assign, assign, n*sizeof(int));
1058

1059
done_collect:
1060
	cpuc->n_events = n;
1061
	cpuc->n_added += n - n0;
1062
	cpuc->n_txn += n - n0;
1063

1064 1065
	ret = 0;
out:
P
Peter Zijlstra 已提交
1066
	perf_pmu_enable(event->pmu);
1067
	return ret;
I
Ingo Molnar 已提交
1068 1069
}

P
Peter Zijlstra 已提交
1070
static void x86_pmu_start(struct perf_event *event, int flags)
1071
{
P
Peter Zijlstra 已提交
1072 1073 1074
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int idx = event->hw.idx;

P
Peter Zijlstra 已提交
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
		return;

	if (WARN_ON_ONCE(idx == -1))
		return;

	if (flags & PERF_EF_RELOAD) {
		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
		x86_perf_event_set_period(event);
	}

	event->hw.state = 0;
1087

P
Peter Zijlstra 已提交
1088 1089
	cpuc->events[idx] = event;
	__set_bit(idx, cpuc->active_mask);
1090
	__set_bit(idx, cpuc->running);
1091
	x86_pmu.enable(event);
P
Peter Zijlstra 已提交
1092
	perf_event_update_userpage(event);
1093 1094
}

1095
void perf_event_print_debug(void)
I
Ingo Molnar 已提交
1096
{
1097
	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1098
	u64 pebs;
1099
	struct cpu_hw_events *cpuc;
1100
	unsigned long flags;
1101 1102
	int cpu, idx;

1103
	if (!x86_pmu.num_counters)
1104
		return;
I
Ingo Molnar 已提交
1105

1106
	local_irq_save(flags);
I
Ingo Molnar 已提交
1107 1108

	cpu = smp_processor_id();
1109
	cpuc = &per_cpu(cpu_hw_events, cpu);
I
Ingo Molnar 已提交
1110

1111
	if (x86_pmu.version >= 2) {
1112 1113 1114 1115
		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1116
		rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1117 1118 1119 1120 1121 1122

		pr_info("\n");
		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1123
		pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
1124
	}
1125
	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
I
Ingo Molnar 已提交
1126

1127
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1128 1129
		rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
		rdmsrl(x86_pmu_event_addr(idx), pmc_count);
I
Ingo Molnar 已提交
1130

1131
		prev_left = per_cpu(pmc_prev_left[idx], cpu);
I
Ingo Molnar 已提交
1132

1133
		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
I
Ingo Molnar 已提交
1134
			cpu, idx, pmc_ctrl);
1135
		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
I
Ingo Molnar 已提交
1136
			cpu, idx, pmc_count);
1137
		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1138
			cpu, idx, prev_left);
I
Ingo Molnar 已提交
1139
	}
1140
	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1141 1142
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);

1143
		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1144 1145
			cpu, idx, pmc_count);
	}
1146
	local_irq_restore(flags);
I
Ingo Molnar 已提交
1147 1148
}

P
Peter Zijlstra 已提交
1149
static void x86_pmu_stop(struct perf_event *event, int flags)
I
Ingo Molnar 已提交
1150
{
1151
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1152
	struct hw_perf_event *hwc = &event->hw;
I
Ingo Molnar 已提交
1153

P
Peter Zijlstra 已提交
1154 1155 1156 1157 1158 1159
	if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
		x86_pmu.disable(event);
		cpuc->events[hwc->idx] = NULL;
		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
		hwc->state |= PERF_HES_STOPPED;
	}
1160

P
Peter Zijlstra 已提交
1161 1162 1163 1164 1165 1166 1167 1168
	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
		/*
		 * Drain the remaining delta count out of a event
		 * that we are disabling:
		 */
		x86_perf_event_update(event);
		hwc->state |= PERF_HES_UPTODATE;
	}
1169 1170
}

P
Peter Zijlstra 已提交
1171
static void x86_pmu_del(struct perf_event *event, int flags)
1172 1173 1174 1175
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int i;

1176 1177 1178 1179 1180
	/*
	 * If we're called during a txn, we don't need to do anything.
	 * The events never got scheduled and ->cancel_txn will truncate
	 * the event_list.
	 */
1181
	if (cpuc->group_flag & PERF_EVENT_TXN)
1182 1183
		return;

P
Peter Zijlstra 已提交
1184
	x86_pmu_stop(event, PERF_EF_UPDATE);
1185

1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
	for (i = 0; i < cpuc->n_events; i++) {
		if (event == cpuc->event_list[i]) {

			if (x86_pmu.put_event_constraints)
				x86_pmu.put_event_constraints(cpuc, event);

			while (++i < cpuc->n_events)
				cpuc->event_list[i-1] = cpuc->event_list[i];

			--cpuc->n_events;
1196
			break;
1197 1198
		}
	}
1199
	perf_event_update_userpage(event);
I
Ingo Molnar 已提交
1200 1201
}

1202
static int x86_pmu_handle_irq(struct pt_regs *regs)
1203
{
1204
	struct perf_sample_data data;
1205 1206
	struct cpu_hw_events *cpuc;
	struct perf_event *event;
V
Vince Weaver 已提交
1207
	int idx, handled = 0;
1208 1209
	u64 val;

1210
	perf_sample_data_init(&data, 0);
1211

1212
	cpuc = &__get_cpu_var(cpu_hw_events);
1213

1214
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1215 1216 1217 1218 1219 1220 1221 1222
		if (!test_bit(idx, cpuc->active_mask)) {
			/*
			 * Though we deactivated the counter some cpus
			 * might still deliver spurious interrupts still
			 * in flight. Catch them:
			 */
			if (__test_and_clear_bit(idx, cpuc->running))
				handled++;
1223
			continue;
1224
		}
1225

1226
		event = cpuc->events[idx];
1227

1228
		val = x86_perf_event_update(event);
1229
		if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1230
			continue;
1231

1232
		/*
1233
		 * event overflow
1234
		 */
1235
		handled++;
1236
		data.period	= event->hw.last_period;
1237

1238
		if (!x86_perf_event_set_period(event))
1239 1240
			continue;

1241
		if (perf_event_overflow(event, 1, &data, regs))
P
Peter Zijlstra 已提交
1242
			x86_pmu_stop(event, 0);
1243
	}
1244

1245 1246 1247
	if (handled)
		inc_irq_stat(apic_perf_irqs);

1248 1249
	return handled;
}
1250

1251
void perf_events_lapic_init(void)
I
Ingo Molnar 已提交
1252
{
1253
	if (!x86_pmu.apic || !x86_pmu_initialized())
I
Ingo Molnar 已提交
1254
		return;
1255

I
Ingo Molnar 已提交
1256
	/*
1257
	 * Always use NMI for PMU
I
Ingo Molnar 已提交
1258
	 */
1259
	apic_write(APIC_LVTPC, APIC_DM_NMI);
I
Ingo Molnar 已提交
1260 1261
}

1262 1263 1264 1265 1266 1267 1268
struct pmu_nmi_state {
	unsigned int	marked;
	int		handled;
};

static DEFINE_PER_CPU(struct pmu_nmi_state, pmu_nmi);

I
Ingo Molnar 已提交
1269
static int __kprobes
1270
perf_event_nmi_handler(struct notifier_block *self,
I
Ingo Molnar 已提交
1271 1272 1273
			 unsigned long cmd, void *__args)
{
	struct die_args *args = __args;
1274 1275
	unsigned int this_nmi;
	int handled;
1276

1277
	if (!atomic_read(&active_events))
1278 1279
		return NOTIFY_DONE;

1280 1281 1282
	switch (cmd) {
	case DIE_NMI:
		break;
1283 1284
	case DIE_NMIUNKNOWN:
		this_nmi = percpu_read(irq_stat.__nmi_count);
T
Tejun Heo 已提交
1285
		if (this_nmi != __this_cpu_read(pmu_nmi.marked))
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
			/* let the kernel handle the unknown nmi */
			return NOTIFY_DONE;
		/*
		 * This one is a PMU back-to-back nmi. Two events
		 * trigger 'simultaneously' raising two back-to-back
		 * NMIs. If the first NMI handles both, the latter
		 * will be empty and daze the CPU. So, we drop it to
		 * avoid false-positive 'unknown nmi' messages.
		 */
		return NOTIFY_STOP;
1296
	default:
I
Ingo Molnar 已提交
1297
		return NOTIFY_DONE;
1298
	}
I
Ingo Molnar 已提交
1299 1300

	apic_write(APIC_LVTPC, APIC_DM_NMI);
1301 1302 1303 1304 1305 1306 1307 1308

	handled = x86_pmu.handle_irq(args->regs);
	if (!handled)
		return NOTIFY_DONE;

	this_nmi = percpu_read(irq_stat.__nmi_count);
	if ((handled > 1) ||
		/* the next nmi could be a back-to-back nmi */
T
Tejun Heo 已提交
1309 1310
	    ((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
	     (__this_cpu_read(pmu_nmi.handled) > 1))) {
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
		/*
		 * We could have two subsequent back-to-back nmis: The
		 * first handles more than one counter, the 2nd
		 * handles only one counter and the 3rd handles no
		 * counter.
		 *
		 * This is the 2nd nmi because the previous was
		 * handling more than one counter. We will mark the
		 * next (3rd) and then drop it if unhandled.
		 */
T
Tejun Heo 已提交
1321 1322
		__this_cpu_write(pmu_nmi.marked, this_nmi + 1);
		__this_cpu_write(pmu_nmi.handled, handled);
1323
	}
I
Ingo Molnar 已提交
1324

1325
	return NOTIFY_STOP;
I
Ingo Molnar 已提交
1326 1327
}

1328 1329 1330
static __read_mostly struct notifier_block perf_event_nmi_notifier = {
	.notifier_call		= perf_event_nmi_handler,
	.next			= NULL,
1331
	.priority		= NMI_LOCAL_LOW_PRIOR,
1332 1333
};

1334
static struct event_constraint unconstrained;
1335
static struct event_constraint emptyconstraint;
1336 1337

static struct event_constraint *
1338
x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
1339
{
1340
	struct event_constraint *c;
1341 1342 1343

	if (x86_pmu.event_constraints) {
		for_each_event_constraint(c, x86_pmu.event_constraints) {
1344 1345
			if ((event->hw.config & c->cmask) == c->code)
				return c;
1346 1347
		}
	}
1348 1349

	return &unconstrained;
1350 1351
}

1352 1353
#include "perf_event_amd.c"
#include "perf_event_p6.c"
1354
#include "perf_event_p4.c"
1355
#include "perf_event_intel_lbr.c"
1356
#include "perf_event_intel_ds.c"
1357
#include "perf_event_intel.c"
1358

1359 1360 1361 1362
static int __cpuinit
x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;
1363
	int ret = NOTIFY_OK;
1364 1365 1366 1367

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_UP_PREPARE:
		if (x86_pmu.cpu_prepare)
1368
			ret = x86_pmu.cpu_prepare(cpu);
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
		break;

	case CPU_STARTING:
		if (x86_pmu.cpu_starting)
			x86_pmu.cpu_starting(cpu);
		break;

	case CPU_DYING:
		if (x86_pmu.cpu_dying)
			x86_pmu.cpu_dying(cpu);
		break;

1381
	case CPU_UP_CANCELED:
1382 1383 1384 1385 1386 1387 1388 1389 1390
	case CPU_DEAD:
		if (x86_pmu.cpu_dead)
			x86_pmu.cpu_dead(cpu);
		break;

	default:
		break;
	}

1391
	return ret;
1392 1393
}

1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
static void __init pmu_check_apic(void)
{
	if (cpu_has_apic)
		return;

	x86_pmu.apic = 0;
	pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
	pr_info("no hardware sampling interrupt available.\n");
}

1404
static int __init init_hw_perf_events(void)
1405
{
1406
	struct event_constraint *c;
1407 1408
	int err;

1409
	pr_info("Performance Events: ");
1410

1411 1412
	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_INTEL:
1413
		err = intel_pmu_init();
1414
		break;
1415
	case X86_VENDOR_AMD:
1416
		err = amd_pmu_init();
1417
		break;
1418
	default:
1419
		return 0;
1420
	}
1421
	if (err != 0) {
1422
		pr_cont("no PMU driver, software events only.\n");
1423
		return 0;
1424
	}
1425

1426 1427
	pmu_check_apic();

1428
	/* sanity check that the hardware exists or is emulated */
1429
	if (!check_hw_exists())
1430
		return 0;
1431

1432
	pr_cont("%s PMU driver.\n", x86_pmu.name);
1433

1434 1435 1436
	if (x86_pmu.quirks)
		x86_pmu.quirks();

1437
	if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1438
		WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
1439 1440
		     x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
		x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
I
Ingo Molnar 已提交
1441
	}
1442
	x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
I
Ingo Molnar 已提交
1443

1444
	if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1445
		WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
1446 1447
		     x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
		x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1448
	}
1449

1450
	x86_pmu.intel_ctrl |=
1451
		((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
I
Ingo Molnar 已提交
1452

1453 1454
	perf_events_lapic_init();
	register_die_notifier(&perf_event_nmi_notifier);
1455

1456
	unconstrained = (struct event_constraint)
1457 1458
		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
				   0, x86_pmu.num_counters);
1459

1460 1461
	if (x86_pmu.event_constraints) {
		for_each_event_constraint(c, x86_pmu.event_constraints) {
1462
			if (c->cmask != X86_RAW_EVENT_MASK)
1463 1464
				continue;

1465 1466
			c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
			c->weight += x86_pmu.num_counters;
1467 1468 1469
		}
	}

I
Ingo Molnar 已提交
1470
	pr_info("... version:                %d\n",     x86_pmu.version);
1471 1472 1473
	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
	pr_info("... generic registers:      %d\n",     x86_pmu.num_counters);
	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
I
Ingo Molnar 已提交
1474
	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
1475
	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
1476
	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
1477

P
Peter Zijlstra 已提交
1478
	perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1479
	perf_cpu_notifier(x86_pmu_notifier);
1480 1481

	return 0;
I
Ingo Molnar 已提交
1482
}
1483
early_initcall(init_hw_perf_events);
I
Ingo Molnar 已提交
1484

1485
static inline void x86_pmu_read(struct perf_event *event)
1486
{
1487
	x86_perf_event_update(event);
1488 1489
}

1490 1491 1492 1493 1494
/*
 * Start group events scheduling transaction
 * Set the flag to make pmu::enable() not perform the
 * schedulability test, it will be performed at commit time
 */
P
Peter Zijlstra 已提交
1495
static void x86_pmu_start_txn(struct pmu *pmu)
1496
{
P
Peter Zijlstra 已提交
1497
	perf_pmu_disable(pmu);
T
Tejun Heo 已提交
1498 1499
	__this_cpu_or(cpu_hw_events.group_flag, PERF_EVENT_TXN);
	__this_cpu_write(cpu_hw_events.n_txn, 0);
1500 1501 1502 1503 1504 1505 1506
}

/*
 * Stop group events scheduling transaction
 * Clear the flag and pmu::enable() will perform the
 * schedulability test.
 */
P
Peter Zijlstra 已提交
1507
static void x86_pmu_cancel_txn(struct pmu *pmu)
1508
{
T
Tejun Heo 已提交
1509
	__this_cpu_and(cpu_hw_events.group_flag, ~PERF_EVENT_TXN);
1510 1511 1512
	/*
	 * Truncate the collected events.
	 */
T
Tejun Heo 已提交
1513 1514
	__this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
	__this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
P
Peter Zijlstra 已提交
1515
	perf_pmu_enable(pmu);
1516 1517 1518 1519 1520 1521 1522
}

/*
 * Commit group events scheduling transaction
 * Perform the group schedulability test as a whole
 * Return 0 if success
 */
P
Peter Zijlstra 已提交
1523
static int x86_pmu_commit_txn(struct pmu *pmu)
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int assign[X86_PMC_IDX_MAX];
	int n, ret;

	n = cpuc->n_events;

	if (!x86_pmu_initialized())
		return -EAGAIN;

	ret = x86_pmu.schedule_events(cpuc, n, assign);
	if (ret)
		return ret;

	/*
	 * copy new assignment, now we know it is possible
	 * will be used by hw_perf_enable()
	 */
	memcpy(cpuc->assign, assign, n*sizeof(int));

1544
	cpuc->group_flag &= ~PERF_EVENT_TXN;
P
Peter Zijlstra 已提交
1545
	perf_pmu_enable(pmu);
1546 1547 1548
	return 0;
}

1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574
/*
 * validate that we can schedule this event
 */
static int validate_event(struct perf_event *event)
{
	struct cpu_hw_events *fake_cpuc;
	struct event_constraint *c;
	int ret = 0;

	fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
	if (!fake_cpuc)
		return -ENOMEM;

	c = x86_pmu.get_event_constraints(fake_cpuc, event);

	if (!c || !c->weight)
		ret = -ENOSPC;

	if (x86_pmu.put_event_constraints)
		x86_pmu.put_event_constraints(fake_cpuc, event);

	kfree(fake_cpuc);

	return ret;
}

1575 1576 1577 1578
/*
 * validate a single event group
 *
 * validation include:
1579 1580 1581
 *	- check events are compatible which each other
 *	- events do not compete for the same counter
 *	- number of events <= number of counters
1582 1583 1584 1585
 *
 * validation ensures the group can be loaded onto the
 * PMU if it was the only group available.
 */
1586 1587
static int validate_group(struct perf_event *event)
{
1588
	struct perf_event *leader = event->group_leader;
1589 1590
	struct cpu_hw_events *fake_cpuc;
	int ret, n;
1591

1592 1593 1594 1595
	ret = -ENOMEM;
	fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
	if (!fake_cpuc)
		goto out;
1596

1597 1598 1599 1600 1601 1602
	/*
	 * the event is not yet connected with its
	 * siblings therefore we must first collect
	 * existing siblings, then add the new event
	 * before we can simulate the scheduling
	 */
1603 1604
	ret = -ENOSPC;
	n = collect_events(fake_cpuc, leader, true);
1605
	if (n < 0)
1606
		goto out_free;
1607

1608 1609
	fake_cpuc->n_events = n;
	n = collect_events(fake_cpuc, event, false);
1610
	if (n < 0)
1611
		goto out_free;
1612

1613
	fake_cpuc->n_events = n;
1614

1615
	ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
1616 1617 1618 1619 1620

out_free:
	kfree(fake_cpuc);
out:
	return ret;
1621 1622
}

1623
static int x86_pmu_event_init(struct perf_event *event)
I
Ingo Molnar 已提交
1624
{
P
Peter Zijlstra 已提交
1625
	struct pmu *tmp;
I
Ingo Molnar 已提交
1626 1627
	int err;

1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
	switch (event->attr.type) {
	case PERF_TYPE_RAW:
	case PERF_TYPE_HARDWARE:
	case PERF_TYPE_HW_CACHE:
		break;

	default:
		return -ENOENT;
	}

	err = __x86_pmu_event_init(event);
1639
	if (!err) {
1640 1641 1642 1643 1644 1645 1646 1647
		/*
		 * we temporarily connect event to its pmu
		 * such that validate_group() can classify
		 * it as an x86 event using is_x86_event()
		 */
		tmp = event->pmu;
		event->pmu = &pmu;

1648 1649
		if (event->group_leader != event)
			err = validate_group(event);
1650 1651
		else
			err = validate_event(event);
1652 1653

		event->pmu = tmp;
1654
	}
1655
	if (err) {
1656 1657
		if (event->destroy)
			event->destroy(event);
1658
	}
I
Ingo Molnar 已提交
1659

1660
	return err;
I
Ingo Molnar 已提交
1661
}
1662

1663
static struct pmu pmu = {
P
Peter Zijlstra 已提交
1664 1665 1666
	.pmu_enable	= x86_pmu_enable,
	.pmu_disable	= x86_pmu_disable,

1667
	.event_init	= x86_pmu_event_init,
P
Peter Zijlstra 已提交
1668 1669 1670

	.add		= x86_pmu_add,
	.del		= x86_pmu_del,
1671 1672 1673
	.start		= x86_pmu_start,
	.stop		= x86_pmu_stop,
	.read		= x86_pmu_read,
P
Peter Zijlstra 已提交
1674

1675 1676 1677 1678 1679
	.start_txn	= x86_pmu_start_txn,
	.cancel_txn	= x86_pmu_cancel_txn,
	.commit_txn	= x86_pmu_commit_txn,
};

1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696
/*
 * callchain support
 */

static void
backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
{
	/* Ignore warnings */
}

static void backtrace_warning(void *data, char *msg)
{
	/* Ignore warnings */
}

static int backtrace_stack(void *data, char *name)
{
1697
	return 0;
1698 1699 1700 1701 1702 1703
}

static void backtrace_address(void *data, unsigned long addr, int reliable)
{
	struct perf_callchain_entry *entry = data;

1704
	perf_callchain_store(entry, addr);
1705 1706 1707 1708 1709 1710 1711
}

static const struct stacktrace_ops backtrace_ops = {
	.warning		= backtrace_warning,
	.warning_symbol		= backtrace_warning_symbol,
	.stack			= backtrace_stack,
	.address		= backtrace_address,
1712
	.walk_stack		= print_context_stack_bp,
1713 1714
};

1715 1716
void
perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
1717
{
1718 1719
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
		/* TODO: We don't support guest os callchain now */
1720
		return;
1721 1722
	}

1723
	perf_callchain_store(entry, regs->ip);
1724

1725
	dump_trace(NULL, regs, NULL, &backtrace_ops, entry);
1726 1727
}

1728 1729 1730
#ifdef CONFIG_COMPAT
static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
1731
{
1732 1733 1734
	/* 32-bit process in 64-bit kernel. */
	struct stack_frame_ia32 frame;
	const void __user *fp;
1735

1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
	if (!test_thread_flag(TIF_IA32))
		return 0;

	fp = compat_ptr(regs->bp);
	while (entry->nr < PERF_MAX_STACK_DEPTH) {
		unsigned long bytes;
		frame.next_frame     = 0;
		frame.return_address = 0;

		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
		if (bytes != sizeof(frame))
			break;
1748

1749 1750
		if (fp < compat_ptr(regs->sp))
			break;
1751

1752
		perf_callchain_store(entry, frame.return_address);
1753 1754 1755
		fp = compat_ptr(frame.next_frame);
	}
	return 1;
1756
}
1757 1758 1759 1760 1761 1762 1763
#else
static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
    return 0;
}
#endif
1764

1765 1766
void
perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
1767 1768 1769 1770
{
	struct stack_frame frame;
	const void __user *fp;

1771 1772
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
		/* TODO: We don't support guest os callchain now */
1773
		return;
1774
	}
1775

1776
	fp = (void __user *)regs->bp;
1777

1778
	perf_callchain_store(entry, regs->ip);
1779

1780 1781 1782
	if (perf_callchain_user32(regs, entry))
		return;

1783
	while (entry->nr < PERF_MAX_STACK_DEPTH) {
1784
		unsigned long bytes;
1785
		frame.next_frame	     = NULL;
1786 1787
		frame.return_address = 0;

1788 1789
		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
		if (bytes != sizeof(frame))
1790 1791
			break;

1792
		if ((unsigned long)fp < regs->sp)
1793 1794
			break;

1795
		perf_callchain_store(entry, frame.return_address);
1796
		fp = frame.next_frame;
1797 1798 1799
	}
}

1800 1801 1802
unsigned long perf_instruction_pointer(struct pt_regs *regs)
{
	unsigned long ip;
1803

1804 1805 1806 1807
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
		ip = perf_guest_cbs->get_guest_ip();
	else
		ip = instruction_pointer(regs);
1808

1809 1810 1811 1812 1813 1814
	return ip;
}

unsigned long perf_misc_flags(struct pt_regs *regs)
{
	int misc = 0;
1815

1816
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
		if (perf_guest_cbs->is_user_mode())
			misc |= PERF_RECORD_MISC_GUEST_USER;
		else
			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
	} else {
		if (user_mode(regs))
			misc |= PERF_RECORD_MISC_USER;
		else
			misc |= PERF_RECORD_MISC_KERNEL;
	}

1828
	if (regs->flags & PERF_EFLAGS_EXACT)
P
Peter Zijlstra 已提交
1829
		misc |= PERF_RECORD_MISC_EXACT_IP;
1830 1831 1832

	return misc;
}