perf_counter.c 26.8 KB
Newer Older
I
Ingo Molnar 已提交
1 2 3 4 5
/*
 * Performance counter x86 architecture code
 *
 *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
6
 *  Copyright(C) 2009 Jaswinder Singh Rajput
7
 *  Copyright(C) 2009 Advanced Micro Devices, Inc., Robert Richter
I
Ingo Molnar 已提交
8 9 10 11 12 13 14 15 16
 *
 *  For licencing details see kernel-base/COPYING
 */

#include <linux/perf_counter.h>
#include <linux/capability.h>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
17
#include <linux/module.h>
I
Ingo Molnar 已提交
18 19
#include <linux/kdebug.h>
#include <linux/sched.h>
20
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
21 22

#include <asm/apic.h>
23
#include <asm/stacktrace.h>
P
Peter Zijlstra 已提交
24
#include <asm/nmi.h>
I
Ingo Molnar 已提交
25 26

static bool perf_counters_initialized __read_mostly;
27
static u64 perf_counter_mask __read_mostly;
28

I
Ingo Molnar 已提交
29
struct cpu_hw_counters {
30 31
	struct perf_counter	*counters[X86_PMC_IDX_MAX];
	unsigned long		used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
32
	unsigned long		active[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
33
	unsigned long		interrupts;
34 35
	u64			throttle_ctrl;
	int			enabled;
I
Ingo Molnar 已提交
36 37 38
};

/*
39
 * struct x86_pmu - generic x86 pmu
I
Ingo Molnar 已提交
40
 */
41
struct x86_pmu {
42 43
	const char	*name;
	int		version;
44
	int		(*handle_irq)(struct pt_regs *, int);
45
	u64		(*save_disable_all)(void);
46
	void		(*restore_all)(u64);
47
	void		(*enable)(struct hw_perf_counter *, int);
48
	void		(*disable)(struct hw_perf_counter *, int);
49 50
	unsigned	eventsel;
	unsigned	perfctr;
51 52
	u64		(*event_map)(int);
	u64		(*raw_event)(u64);
53
	int		max_events;
54 55 56 57
	int		num_counters;
	int		num_counters_fixed;
	int		counter_bits;
	u64		counter_mask;
58 59
};

60
static struct x86_pmu x86_pmu __read_mostly;
61

62 63 64
static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
	.enabled = 1,
};
I
Ingo Molnar 已提交
65

66 67 68
/*
 * Intel PerfMon v3. Used on Core2 and later.
 */
69
static const u64 intel_perfmon_event_map[] =
I
Ingo Molnar 已提交
70
{
71
  [PERF_COUNT_CPU_CYCLES]		= 0x003c,
I
Ingo Molnar 已提交
72 73 74 75 76
  [PERF_COUNT_INSTRUCTIONS]		= 0x00c0,
  [PERF_COUNT_CACHE_REFERENCES]		= 0x4f2e,
  [PERF_COUNT_CACHE_MISSES]		= 0x412e,
  [PERF_COUNT_BRANCH_INSTRUCTIONS]	= 0x00c4,
  [PERF_COUNT_BRANCH_MISSES]		= 0x00c5,
77
  [PERF_COUNT_BUS_CYCLES]		= 0x013c,
I
Ingo Molnar 已提交
78 79
};

80
static u64 intel_pmu_event_map(int event)
81 82 83
{
	return intel_perfmon_event_map[event];
}
I
Ingo Molnar 已提交
84

85
static u64 intel_pmu_raw_event(u64 event)
86
{
87 88 89
#define CORE_EVNTSEL_EVENT_MASK		0x000000FFULL
#define CORE_EVNTSEL_UNIT_MASK		0x0000FF00ULL
#define CORE_EVNTSEL_COUNTER_MASK	0xFF000000ULL
90 91 92 93 94 95 96 97 98

#define CORE_EVNTSEL_MASK 		\
	(CORE_EVNTSEL_EVENT_MASK |	\
	 CORE_EVNTSEL_UNIT_MASK  |	\
	 CORE_EVNTSEL_COUNTER_MASK)

	return event & CORE_EVNTSEL_MASK;
}

99 100 101
/*
 * AMD Performance Monitor K7 and later.
 */
102
static const u64 amd_perfmon_event_map[] =
103 104 105 106 107 108 109 110 111
{
  [PERF_COUNT_CPU_CYCLES]		= 0x0076,
  [PERF_COUNT_INSTRUCTIONS]		= 0x00c0,
  [PERF_COUNT_CACHE_REFERENCES]		= 0x0080,
  [PERF_COUNT_CACHE_MISSES]		= 0x0081,
  [PERF_COUNT_BRANCH_INSTRUCTIONS]	= 0x00c4,
  [PERF_COUNT_BRANCH_MISSES]		= 0x00c5,
};

112
static u64 amd_pmu_event_map(int event)
113 114 115 116
{
	return amd_perfmon_event_map[event];
}

117
static u64 amd_pmu_raw_event(u64 event)
118
{
119 120 121
#define K7_EVNTSEL_EVENT_MASK	0x7000000FFULL
#define K7_EVNTSEL_UNIT_MASK	0x00000FF00ULL
#define K7_EVNTSEL_COUNTER_MASK	0x0FF000000ULL
122 123 124 125 126 127 128 129 130

#define K7_EVNTSEL_MASK			\
	(K7_EVNTSEL_EVENT_MASK |	\
	 K7_EVNTSEL_UNIT_MASK  |	\
	 K7_EVNTSEL_COUNTER_MASK)

	return event & K7_EVNTSEL_MASK;
}

131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
/*
 * Propagate counter elapsed time into the generic counter.
 * Can only be executed on the CPU where the counter is active.
 * Returns the delta events processed.
 */
static void
x86_perf_counter_update(struct perf_counter *counter,
			struct hw_perf_counter *hwc, int idx)
{
	u64 prev_raw_count, new_raw_count, delta;

	/*
	 * Careful: an NMI might modify the previous counter value.
	 *
	 * Our tactic to handle this is to first atomically read and
	 * exchange a new raw count - then add that new-prev delta
	 * count to the generic counter atomically:
	 */
again:
	prev_raw_count = atomic64_read(&hwc->prev_count);
	rdmsrl(hwc->counter_base + idx, new_raw_count);

	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
					new_raw_count) != prev_raw_count)
		goto again;

	/*
	 * Now we have the new raw value and have updated the prev
	 * timestamp already. We can now calculate the elapsed delta
	 * (counter-)time and add that to the generic counter.
	 *
	 * Careful, not all hw sign-extends above the physical width
	 * of the count, so we do that by clipping the delta to 32 bits:
	 */
	delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);

	atomic64_add(delta, &counter->count);
	atomic64_sub(delta, &hwc->period_left);
}

P
Peter Zijlstra 已提交
171 172 173 174 175 176 177 178 179 180
static atomic_t num_counters;
static DEFINE_MUTEX(pmc_reserve_mutex);

static bool reserve_pmc_hardware(void)
{
	int i;

	if (nmi_watchdog == NMI_LOCAL_APIC)
		disable_lapic_nmi_watchdog();

181
	for (i = 0; i < x86_pmu.num_counters; i++) {
182
		if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
P
Peter Zijlstra 已提交
183 184 185
			goto perfctr_fail;
	}

186
	for (i = 0; i < x86_pmu.num_counters; i++) {
187
		if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
P
Peter Zijlstra 已提交
188 189 190 191 192 193 194
			goto eventsel_fail;
	}

	return true;

eventsel_fail:
	for (i--; i >= 0; i--)
195
		release_evntsel_nmi(x86_pmu.eventsel + i);
P
Peter Zijlstra 已提交
196

197
	i = x86_pmu.num_counters;
P
Peter Zijlstra 已提交
198 199 200

perfctr_fail:
	for (i--; i >= 0; i--)
201
		release_perfctr_nmi(x86_pmu.perfctr + i);
P
Peter Zijlstra 已提交
202 203 204 205 206 207 208 209 210 211 212

	if (nmi_watchdog == NMI_LOCAL_APIC)
		enable_lapic_nmi_watchdog();

	return false;
}

static void release_pmc_hardware(void)
{
	int i;

213
	for (i = 0; i < x86_pmu.num_counters; i++) {
214 215
		release_perfctr_nmi(x86_pmu.perfctr + i);
		release_evntsel_nmi(x86_pmu.eventsel + i);
P
Peter Zijlstra 已提交
216 217 218 219 220 221 222 223 224 225 226 227 228 229
	}

	if (nmi_watchdog == NMI_LOCAL_APIC)
		enable_lapic_nmi_watchdog();
}

static void hw_perf_counter_destroy(struct perf_counter *counter)
{
	if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
		release_pmc_hardware();
		mutex_unlock(&pmc_reserve_mutex);
	}
}

I
Ingo Molnar 已提交
230 231 232
/*
 * Setup the hardware configuration for a given hw_event_type
 */
I
Ingo Molnar 已提交
233
static int __hw_perf_counter_init(struct perf_counter *counter)
I
Ingo Molnar 已提交
234
{
I
Ingo Molnar 已提交
235
	struct perf_counter_hw_event *hw_event = &counter->hw_event;
I
Ingo Molnar 已提交
236
	struct hw_perf_counter *hwc = &counter->hw;
P
Peter Zijlstra 已提交
237
	int err;
I
Ingo Molnar 已提交
238

239 240 241 242
	/* disable temporarily */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
		return -ENOSYS;

I
Ingo Molnar 已提交
243 244 245
	if (unlikely(!perf_counters_initialized))
		return -EINVAL;

P
Peter Zijlstra 已提交
246 247 248 249 250 251 252 253 254 255 256 257
	err = 0;
	if (atomic_inc_not_zero(&num_counters)) {
		mutex_lock(&pmc_reserve_mutex);
		if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
			err = -EBUSY;
		else
			atomic_inc(&num_counters);
		mutex_unlock(&pmc_reserve_mutex);
	}
	if (err)
		return err;

I
Ingo Molnar 已提交
258
	/*
259
	 * Generate PMC IRQs:
I
Ingo Molnar 已提交
260 261
	 * (keep 'enabled' bit clear for now)
	 */
262
	hwc->config = ARCH_PERFMON_EVENTSEL_INT;
I
Ingo Molnar 已提交
263 264

	/*
265
	 * Count user and OS events unless requested not to.
I
Ingo Molnar 已提交
266
	 */
267 268 269
	if (!hw_event->exclude_user)
		hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
	if (!hw_event->exclude_kernel)
I
Ingo Molnar 已提交
270
		hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
271 272 273 274 275 276 277

	/*
	 * If privileged enough, allow NMI events:
	 */
	hwc->nmi = 0;
	if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
		hwc->nmi = 1;
I
Ingo Molnar 已提交
278

I
Ingo Molnar 已提交
279
	hwc->irq_period		= hw_event->irq_period;
I
Ingo Molnar 已提交
280 281 282 283 284
	/*
	 * Intel PMCs cannot be accessed sanely above 32 bit width,
	 * so we install an artificial 1<<31 period regardless of
	 * the generic counter period:
	 */
285 286 287
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
			hwc->irq_period = 0x7FFFFFFF;
I
Ingo Molnar 已提交
288

289
	atomic64_set(&hwc->period_left, hwc->irq_period);
I
Ingo Molnar 已提交
290 291

	/*
292
	 * Raw event type provide the config in the event structure
I
Ingo Molnar 已提交
293
	 */
294
	if (perf_event_raw(hw_event)) {
295
		hwc->config |= x86_pmu.raw_event(perf_event_config(hw_event));
I
Ingo Molnar 已提交
296
	} else {
297
		if (perf_event_id(hw_event) >= x86_pmu.max_events)
I
Ingo Molnar 已提交
298 299 300 301
			return -EINVAL;
		/*
		 * The generic map:
		 */
302
		hwc->config |= x86_pmu.event_map(perf_event_id(hw_event));
I
Ingo Molnar 已提交
303 304
	}

P
Peter Zijlstra 已提交
305 306
	counter->destroy = hw_perf_counter_destroy;

I
Ingo Molnar 已提交
307 308 309
	return 0;
}

310
static u64 intel_pmu_save_disable_all(void)
311 312 313 314
{
	u64 ctrl;

	rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
315
	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
316

317
	return ctrl;
I
Ingo Molnar 已提交
318
}
319

320
static u64 amd_pmu_save_disable_all(void)
321
{
322 323 324 325 326
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	int enabled, idx;

	enabled = cpuc->enabled;
	cpuc->enabled = 0;
327 328
	/*
	 * ensure we write the disable before we start disabling the
329 330
	 * counters proper, so that amd_pmu_enable_counter() does the
	 * right thing.
331
	 */
332
	barrier();
333

334
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
335 336
		u64 val;

337
		if (!test_bit(idx, cpuc->active))
338
			continue;
339
		rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
340 341 342 343
		if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
			continue;
		val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
		wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
344 345
	}

346
	return enabled;
347 348
}

349 350 351 352 353
u64 hw_perf_save_disable(void)
{
	if (unlikely(!perf_counters_initialized))
		return 0;

354
	return x86_pmu.save_disable_all();
355
}
356 357 358
/*
 * Exported because of ACPI idle
 */
359
EXPORT_SYMBOL_GPL(hw_perf_save_disable);
I
Ingo Molnar 已提交
360

361
static void intel_pmu_restore_all(u64 ctrl)
362 363 364 365
{
	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
}

366
static void amd_pmu_restore_all(u64 ctrl)
367
{
368
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
369 370
	int idx;

371 372 373 374 375
	cpuc->enabled = ctrl;
	barrier();
	if (!ctrl)
		return;

376
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
377
		u64 val;
378

379
		if (!test_bit(idx, cpuc->active))
380 381 382 383 384 385
			continue;
		rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
		if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
			continue;
		val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
		wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
386 387 388
	}
}

389 390
void hw_perf_restore(u64 ctrl)
{
391 392 393
	if (unlikely(!perf_counters_initialized))
		return;

394
	x86_pmu.restore_all(ctrl);
395
}
396 397 398
/*
 * Exported because of ACPI idle
 */
399 400
EXPORT_SYMBOL_GPL(hw_perf_restore);

401
static inline u64 intel_pmu_get_status(u64 mask)
402 403 404 405 406
{
	u64 status;

	if (unlikely(!perf_counters_initialized))
		return 0;
407
	rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
408

409
	return status;
410 411
}

412
static inline void intel_pmu_ack_status(u64 ack)
413 414 415 416
{
	wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
}

417
static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
418
{
419
	int err;
420 421 422 423

	if (unlikely(!perf_counters_initialized))
		return;

424 425
	err = checking_wrmsrl(hwc->config_base + idx,
			      hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
426 427
}

428
static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
429
{
430
	int err;
431 432 433 434

	if (unlikely(!perf_counters_initialized))
		return;

435 436
	err = checking_wrmsrl(hwc->config_base + idx,
			      hwc->config);
437 438
}

439
static inline void
440
intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
441 442 443 444 445 446 447 448 449 450 451 452
{
	int idx = __idx - X86_PMC_IDX_FIXED;
	u64 ctrl_val, mask;
	int err;

	mask = 0xfULL << (idx * 4);

	rdmsrl(hwc->config_base, ctrl_val);
	ctrl_val &= ~mask;
	err = checking_wrmsrl(hwc->config_base, ctrl_val);
}

453
static inline void
454
intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
455
{
456 457 458 459 460 461 462 463 464 465 466 467
	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
		intel_pmu_disable_fixed(hwc, idx);
		return;
	}

	x86_pmu_disable_counter(hwc, idx);
}

static inline void
amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
{
	x86_pmu_disable_counter(hwc, idx);
468 469
}

470
static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
I
Ingo Molnar 已提交
471

472 473 474 475 476
/*
 * Set the next IRQ period, based on the hwc->period_left value.
 * To be called with the counter disabled in hw:
 */
static void
477
x86_perf_counter_set_period(struct perf_counter *counter,
478
			     struct hw_perf_counter *hwc, int idx)
I
Ingo Molnar 已提交
479
{
480
	s64 left = atomic64_read(&hwc->period_left);
481
	s64 period = hwc->irq_period;
482
	int err;
483 484 485 486 487 488 489 490 491 492 493 494 495

	/*
	 * If we are way outside a reasoable range then just skip forward:
	 */
	if (unlikely(left <= -period)) {
		left = period;
		atomic64_set(&hwc->period_left, left);
	}

	if (unlikely(left <= 0)) {
		left += period;
		atomic64_set(&hwc->period_left, left);
	}
I
Ingo Molnar 已提交
496

497 498 499 500 501 502
	per_cpu(prev_left[idx], smp_processor_id()) = left;

	/*
	 * The hw counter starts counting from this counter offset,
	 * mark it to be able to extra future deltas:
	 */
503
	atomic64_set(&hwc->prev_count, (u64)-left);
504

505
	err = checking_wrmsrl(hwc->counter_base + idx,
506
			     (u64)(-left) & x86_pmu.counter_mask);
507 508 509
}

static inline void
510
intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
511 512 513 514 515 516
{
	int idx = __idx - X86_PMC_IDX_FIXED;
	u64 ctrl_val, bits, mask;
	int err;

	/*
517 518 519
	 * Enable IRQ generation (0x8),
	 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
	 * if requested:
520
	 */
521 522 523
	bits = 0x8ULL;
	if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
		bits |= 0x2;
524 525 526 527 528 529 530 531 532
	if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
		bits |= 0x1;
	bits <<= (idx * 4);
	mask = 0xfULL << (idx * 4);

	rdmsrl(hwc->config_base, ctrl_val);
	ctrl_val &= ~mask;
	ctrl_val |= bits;
	err = checking_wrmsrl(hwc->config_base, ctrl_val);
533 534
}

535
static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
536
{
537 538 539 540 541 542 543 544 545 546 547 548 549 550
	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
		intel_pmu_enable_fixed(hwc, idx);
		return;
	}

	x86_pmu_enable_counter(hwc, idx);
}

static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);

	if (cpuc->enabled)
		x86_pmu_enable_counter(hwc, idx);
551
	else
552
		x86_pmu_disable_counter(hwc, idx);
I
Ingo Molnar 已提交
553 554
}

555 556
static int
fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
557
{
558 559
	unsigned int event;

560 561 562
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
		return -1;

563 564 565 566 567
	if (unlikely(hwc->nmi))
		return -1;

	event = hwc->config & ARCH_PERFMON_EVENT_MASK;

568
	if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
569
		return X86_PMC_IDX_FIXED_INSTRUCTIONS;
570
	if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
571
		return X86_PMC_IDX_FIXED_CPU_CYCLES;
572
	if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
573 574
		return X86_PMC_IDX_FIXED_BUS_CYCLES;

575 576 577
	return -1;
}

578 579 580
/*
 * Find a PMC slot for the freshly enabled / scheduled in counter:
 */
581
static int x86_pmu_enable(struct perf_counter *counter)
I
Ingo Molnar 已提交
582 583 584
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	struct hw_perf_counter *hwc = &counter->hw;
585
	int idx;
I
Ingo Molnar 已提交
586

587 588 589 590 591 592 593 594
	idx = fixed_mode_idx(counter, hwc);
	if (idx >= 0) {
		/*
		 * Try to get the fixed counter, if that is already taken
		 * then try to get a generic counter:
		 */
		if (test_and_set_bit(idx, cpuc->used))
			goto try_generic;
595

596 597 598 599 600 601 602
		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
		/*
		 * We set it so that counter_base + idx in wrmsr/rdmsr maps to
		 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
		 */
		hwc->counter_base =
			MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
I
Ingo Molnar 已提交
603
		hwc->idx = idx;
604 605 606 607 608
	} else {
		idx = hwc->idx;
		/* Try to get the previous generic counter again */
		if (test_and_set_bit(idx, cpuc->used)) {
try_generic:
609 610 611
			idx = find_first_zero_bit(cpuc->used,
						  x86_pmu.num_counters);
			if (idx == x86_pmu.num_counters)
612 613 614 615 616
				return -EAGAIN;

			set_bit(idx, cpuc->used);
			hwc->idx = idx;
		}
617 618
		hwc->config_base  = x86_pmu.eventsel;
		hwc->counter_base = x86_pmu.perfctr;
I
Ingo Molnar 已提交
619 620 621 622
	}

	perf_counters_lapic_init(hwc->nmi);

623
	x86_pmu.disable(hwc, idx);
I
Ingo Molnar 已提交
624

625
	cpuc->counters[idx] = counter;
626
	set_bit(idx, cpuc->active);
627

628
	x86_perf_counter_set_period(counter, hwc, idx);
629
	x86_pmu.enable(hwc, idx);
630 631

	return 0;
I
Ingo Molnar 已提交
632 633 634 635
}

void perf_counter_print_debug(void)
{
636
	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
637
	struct cpu_hw_counters *cpuc;
638 639
	int cpu, idx;

640
	if (!x86_pmu.num_counters)
641
		return;
I
Ingo Molnar 已提交
642 643 644 645

	local_irq_disable();

	cpu = smp_processor_id();
646
	cpuc = &per_cpu(cpu_hw_counters, cpu);
I
Ingo Molnar 已提交
647

648
	if (x86_pmu.version >= 2) {
649 650 651 652 653 654 655 656 657 658
		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);

		pr_info("\n");
		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
659
	}
660
	pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
I
Ingo Molnar 已提交
661

662
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
663 664
		rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
		rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
I
Ingo Molnar 已提交
665

666
		prev_left = per_cpu(prev_left[idx], cpu);
I
Ingo Molnar 已提交
667

668
		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
I
Ingo Molnar 已提交
669
			cpu, idx, pmc_ctrl);
670
		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
I
Ingo Molnar 已提交
671
			cpu, idx, pmc_count);
672
		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
673
			cpu, idx, prev_left);
I
Ingo Molnar 已提交
674
	}
675
	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
676 677
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);

678
		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
679 680
			cpu, idx, pmc_count);
	}
I
Ingo Molnar 已提交
681 682 683
	local_irq_enable();
}

684
static void x86_pmu_disable(struct perf_counter *counter)
I
Ingo Molnar 已提交
685 686 687
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	struct hw_perf_counter *hwc = &counter->hw;
688
	int idx = hwc->idx;
I
Ingo Molnar 已提交
689

690 691 692 693 694
	/*
	 * Must be done before we disable, otherwise the nmi handler
	 * could reenable again:
	 */
	clear_bit(idx, cpuc->active);
695
	x86_pmu.disable(hwc, idx);
I
Ingo Molnar 已提交
696

697 698 699 700
	/*
	 * Make sure the cleared pointer becomes visible before we
	 * (potentially) free the counter:
	 */
701
	barrier();
I
Ingo Molnar 已提交
702

703 704 705 706 707
	/*
	 * Drain the remaining delta count out of a counter
	 * that we are disabling:
	 */
	x86_perf_counter_update(counter, hwc, idx);
708 709
	cpuc->counters[idx] = NULL;
	clear_bit(idx, cpuc->used);
I
Ingo Molnar 已提交
710 711
}

712
/*
713 714
 * Save and restart an expired counter. Called by NMI contexts,
 * so it has to be careful about preempting normal counter ops:
715
 */
716
static void intel_pmu_save_and_restart(struct perf_counter *counter)
I
Ingo Molnar 已提交
717 718 719 720
{
	struct hw_perf_counter *hwc = &counter->hw;
	int idx = hwc->idx;

721
	x86_perf_counter_update(counter, hwc, idx);
722
	x86_perf_counter_set_period(counter, hwc, idx);
723

724
	if (counter->state == PERF_COUNTER_STATE_ACTIVE)
725
		intel_pmu_enable_counter(hwc, idx);
I
Ingo Molnar 已提交
726 727
}

728 729 730
/*
 * Maximum interrupt frequency of 100KHz per CPU
 */
731
#define PERFMON_MAX_INTERRUPTS (100000/HZ)
732

I
Ingo Molnar 已提交
733 734 735 736
/*
 * This handler is triggered by the local APIC, so the APIC IRQ handling
 * rules apply:
 */
737
static int intel_pmu_handle_irq(struct pt_regs *regs, int nmi)
I
Ingo Molnar 已提交
738 739
{
	int bit, cpu = smp_processor_id();
740
	u64 ack, status;
741
	struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
742
	int ret = 0;
743

744
	cpuc->throttle_ctrl = intel_pmu_save_disable_all();
I
Ingo Molnar 已提交
745

746
	status = intel_pmu_get_status(cpuc->throttle_ctrl);
747 748 749
	if (!status)
		goto out;

750
	ret = 1;
I
Ingo Molnar 已提交
751
again:
752
	inc_irq_stat(apic_perf_irqs);
I
Ingo Molnar 已提交
753
	ack = status;
754
	for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
755
		struct perf_counter *counter = cpuc->counters[bit];
I
Ingo Molnar 已提交
756 757

		clear_bit(bit, (unsigned long *) &status);
758
		if (!test_bit(bit, cpuc->active))
I
Ingo Molnar 已提交
759 760
			continue;

761
		intel_pmu_save_and_restart(counter);
762
		if (perf_counter_overflow(counter, nmi, regs, 0))
763
			intel_pmu_disable_counter(&counter->hw, bit);
I
Ingo Molnar 已提交
764 765
	}

766
	intel_pmu_ack_status(ack);
I
Ingo Molnar 已提交
767 768 769 770

	/*
	 * Repeat if there is more work to be done:
	 */
771
	status = intel_pmu_get_status(cpuc->throttle_ctrl);
I
Ingo Molnar 已提交
772 773
	if (status)
		goto again;
774
out:
I
Ingo Molnar 已提交
775
	/*
776
	 * Restore - do not reenable when global enable is off or throttled:
I
Ingo Molnar 已提交
777
	 */
778
	if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
779
		intel_pmu_restore_all(cpuc->throttle_ctrl);
780 781

	return ret;
782 783
}

784 785
static int amd_pmu_handle_irq(struct pt_regs *regs, int nmi) { return 0; }

786 787 788 789 790 791 792 793 794 795
void perf_counter_unthrottle(void)
{
	struct cpu_hw_counters *cpuc;

	if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
		return;

	if (unlikely(!perf_counters_initialized))
		return;

796
	cpuc = &__get_cpu_var(cpu_hw_counters);
797
	if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
798
		if (printk_ratelimit())
799
			printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
800
		hw_perf_restore(cpuc->throttle_ctrl);
801
	}
802
	cpuc->interrupts = 0;
I
Ingo Molnar 已提交
803 804 805 806 807 808
}

void smp_perf_counter_interrupt(struct pt_regs *regs)
{
	irq_enter();
	apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
809
	ack_APIC_irq();
810
	x86_pmu.handle_irq(regs, 0);
I
Ingo Molnar 已提交
811 812 813
	irq_exit();
}

814 815 816 817 818 819 820 821 822 823 824 825 826 827
void smp_perf_pending_interrupt(struct pt_regs *regs)
{
	irq_enter();
	ack_APIC_irq();
	inc_irq_stat(apic_pending_irqs);
	perf_counter_do_pending();
	irq_exit();
}

void set_perf_counter_pending(void)
{
	apic->send_IPI_self(LOCAL_PENDING_VECTOR);
}

828
void perf_counters_lapic_init(int nmi)
I
Ingo Molnar 已提交
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
{
	u32 apic_val;

	if (!perf_counters_initialized)
		return;
	/*
	 * Enable the performance counter vector in the APIC LVT:
	 */
	apic_val = apic_read(APIC_LVTERR);

	apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
	if (nmi)
		apic_write(APIC_LVTPC, APIC_DM_NMI);
	else
		apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
	apic_write(APIC_LVTERR, apic_val);
}

static int __kprobes
perf_counter_nmi_handler(struct notifier_block *self,
			 unsigned long cmd, void *__args)
{
	struct die_args *args = __args;
	struct pt_regs *regs;
853 854 855 856 857 858
	int ret;

	switch (cmd) {
	case DIE_NMI:
	case DIE_NMI_IPI:
		break;
I
Ingo Molnar 已提交
859

860
	default:
I
Ingo Molnar 已提交
861
		return NOTIFY_DONE;
862
	}
I
Ingo Molnar 已提交
863 864 865 866

	regs = args->regs;

	apic_write(APIC_LVTPC, APIC_DM_NMI);
867
	ret = x86_pmu.handle_irq(regs, 1);
I
Ingo Molnar 已提交
868

869
	return ret ? NOTIFY_STOP : NOTIFY_OK;
I
Ingo Molnar 已提交
870 871 872
}

static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
873 874 875
	.notifier_call		= perf_counter_nmi_handler,
	.next			= NULL,
	.priority		= 1
I
Ingo Molnar 已提交
876 877
};

878
static struct x86_pmu intel_pmu = {
879
	.name			= "Intel",
880
	.handle_irq		= intel_pmu_handle_irq,
881 882 883 884
	.save_disable_all	= intel_pmu_save_disable_all,
	.restore_all		= intel_pmu_restore_all,
	.enable			= intel_pmu_enable_counter,
	.disable		= intel_pmu_disable_counter,
885 886
	.eventsel		= MSR_ARCH_PERFMON_EVENTSEL0,
	.perfctr		= MSR_ARCH_PERFMON_PERFCTR0,
887 888
	.event_map		= intel_pmu_event_map,
	.raw_event		= intel_pmu_raw_event,
889 890 891
	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
};

892
static struct x86_pmu amd_pmu = {
893
	.name			= "AMD",
894
	.handle_irq		= amd_pmu_handle_irq,
895 896 897 898
	.save_disable_all	= amd_pmu_save_disable_all,
	.restore_all		= amd_pmu_restore_all,
	.enable			= amd_pmu_enable_counter,
	.disable		= amd_pmu_disable_counter,
899 900
	.eventsel		= MSR_K7_EVNTSEL0,
	.perfctr		= MSR_K7_PERFCTR0,
901 902
	.event_map		= amd_pmu_event_map,
	.raw_event		= amd_pmu_raw_event,
903
	.max_events		= ARRAY_SIZE(amd_perfmon_event_map),
904 905 906
	.num_counters		= 4,
	.counter_bits		= 48,
	.counter_mask		= (1ULL << 48) - 1,
907 908
};

909
static int intel_pmu_init(void)
I
Ingo Molnar 已提交
910
{
911
	union cpuid10_edx edx;
I
Ingo Molnar 已提交
912
	union cpuid10_eax eax;
913
	unsigned int unused;
914
	unsigned int ebx;
915
	int version;
I
Ingo Molnar 已提交
916

917
	if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
918
		return -ENODEV;
919

I
Ingo Molnar 已提交
920 921 922 923
	/*
	 * Check whether the Architectural PerfMon supports
	 * Branch Misses Retired Event or not.
	 */
924
	cpuid(10, &eax.full, &ebx, &unused, &edx.full);
I
Ingo Molnar 已提交
925
	if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
926
		return -ENODEV;
I
Ingo Molnar 已提交
927

928 929
	version = eax.split.version_id;
	if (version < 2)
930
		return -ENODEV;
931

932
	x86_pmu = intel_pmu;
933
	x86_pmu.version = version;
934 935 936 937
	x86_pmu.num_counters = eax.split.num_counters;
	x86_pmu.num_counters_fixed = edx.split.num_counters_fixed;
	x86_pmu.counter_bits = eax.split.bit_width;
	x86_pmu.counter_mask = (1ULL << eax.split.bit_width) - 1;
938

939
	return 0;
940 941
}

942
static int amd_pmu_init(void)
943
{
944
	x86_pmu = amd_pmu;
945
	return 0;
946 947
}

948 949
void __init init_hw_perf_counters(void)
{
950 951
	int err;

952 953
	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_INTEL:
954
		err = intel_pmu_init();
955
		break;
956
	case X86_VENDOR_AMD:
957
		err = amd_pmu_init();
958
		break;
959 960
	default:
		return;
961
	}
962
	if (err != 0)
963 964
		return;

965 966 967 968
	pr_info("%s Performance Monitoring support detected.\n", x86_pmu.name);
	pr_info("... version:         %d\n", x86_pmu.version);
	pr_info("... bit width:       %d\n", x86_pmu.counter_bits);

969 970 971
	pr_info("... num counters:    %d\n", x86_pmu.num_counters);
	if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
		x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
I
Ingo Molnar 已提交
972
		WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
973
		     x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
I
Ingo Molnar 已提交
974
	}
975 976
	perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
	perf_max_counters = x86_pmu.num_counters;
I
Ingo Molnar 已提交
977

978
	pr_info("... value mask:      %016Lx\n", x86_pmu.counter_mask);
979

980 981
	if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
		x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
982
		WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
983
		     x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
984
	}
985
	pr_info("... fixed counters:  %d\n", x86_pmu.num_counters_fixed);
986

987 988
	perf_counter_mask |=
		((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
I
Ingo Molnar 已提交
989

990
	pr_info("... counter mask:    %016Lx\n", perf_counter_mask);
991 992
	perf_counters_initialized = true;

I
Ingo Molnar 已提交
993 994 995
	perf_counters_lapic_init(0);
	register_die_notifier(&perf_counter_nmi_notifier);
}
I
Ingo Molnar 已提交
996

997
static inline void x86_pmu_read(struct perf_counter *counter)
998 999 1000 1001
{
	x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
}

1002 1003 1004 1005
static const struct pmu pmu = {
	.enable		= x86_pmu_enable,
	.disable	= x86_pmu_disable,
	.read		= x86_pmu_read,
I
Ingo Molnar 已提交
1006 1007
};

1008
const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
I
Ingo Molnar 已提交
1009 1010 1011 1012 1013
{
	int err;

	err = __hw_perf_counter_init(counter);
	if (err)
1014
		return ERR_PTR(err);
I
Ingo Molnar 已提交
1015

1016
	return &pmu;
I
Ingo Molnar 已提交
1017
}
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070

/*
 * callchain support
 */

static inline
void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
{
	if (entry->nr < MAX_STACK_DEPTH)
		entry->ip[entry->nr++] = ip;
}

static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);


static void
backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
{
	/* Ignore warnings */
}

static void backtrace_warning(void *data, char *msg)
{
	/* Ignore warnings */
}

static int backtrace_stack(void *data, char *name)
{
	/* Don't bother with IRQ stacks for now */
	return -1;
}

static void backtrace_address(void *data, unsigned long addr, int reliable)
{
	struct perf_callchain_entry *entry = data;

	if (reliable)
		callchain_store(entry, addr);
}

static const struct stacktrace_ops backtrace_ops = {
	.warning		= backtrace_warning,
	.warning_symbol		= backtrace_warning_symbol,
	.stack			= backtrace_stack,
	.address		= backtrace_address,
};

static void
perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
	unsigned long bp;
	char *stack;
1071
	int nr = entry->nr;
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082

	callchain_store(entry, instruction_pointer(regs));

	stack = ((char *)regs + sizeof(struct pt_regs));
#ifdef CONFIG_FRAME_POINTER
	bp = frame_pointer(regs);
#else
	bp = 0;
#endif

	dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1083 1084

	entry->kernel = entry->nr - nr;
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
}


struct stack_frame {
	const void __user	*next_fp;
	unsigned long		return_address;
};

static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
{
	int ret;

	if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
		return 0;

	ret = 1;
	pagefault_disable();
	if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
		ret = 0;
	pagefault_enable();

	return ret;
}

static void
perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
	struct stack_frame frame;
	const void __user *fp;
1114
	int nr = entry->nr;
1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133

	regs = (struct pt_regs *)current->thread.sp0 - 1;
	fp   = (void __user *)regs->bp;

	callchain_store(entry, regs->ip);

	while (entry->nr < MAX_STACK_DEPTH) {
		frame.next_fp	     = NULL;
		frame.return_address = 0;

		if (!copy_stack_frame(fp, &frame))
			break;

		if ((unsigned long)fp < user_stack_pointer(regs))
			break;

		callchain_store(entry, frame.return_address);
		fp = frame.next_fp;
	}
1134 1135

	entry->user = entry->nr - nr;
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
}

static void
perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
	int is_user;

	if (!regs)
		return;

	is_user = user_mode(regs);

	if (!current || current->pid == 0)
		return;

	if (is_user && current->state != TASK_RUNNING)
		return;

	if (!is_user)
		perf_callchain_kernel(regs, entry);

	if (current->mm)
		perf_callchain_user(regs, entry);
}

struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
{
	struct perf_callchain_entry *entry;

	if (in_nmi())
		entry = &__get_cpu_var(nmi_entry);
	else
		entry = &__get_cpu_var(irq_entry);

	entry->nr = 0;
1171 1172 1173
	entry->hv = 0;
	entry->kernel = 0;
	entry->user = 0;
1174 1175 1176 1177 1178

	perf_do_callchain(regs, entry);

	return entry;
}