perf_event.c 48.9 KB
Newer Older
I
Ingo Molnar 已提交
1
/*
2
 * Performance events x86 architecture code
I
Ingo Molnar 已提交
3
 *
4 5 6 7 8
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2009 Jaswinder Singh Rajput
 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9
 *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10
 *  Copyright (C) 2009 Google, Inc., Stephane Eranian
I
Ingo Molnar 已提交
11 12 13 14
 *
 *  For licencing details see kernel-base/COPYING
 */

15
#include <linux/perf_event.h>
I
Ingo Molnar 已提交
16 17 18 19
#include <linux/capability.h>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
20
#include <linux/module.h>
I
Ingo Molnar 已提交
21 22
#include <linux/kdebug.h>
#include <linux/sched.h>
23
#include <linux/uaccess.h>
24
#include <linux/slab.h>
25
#include <linux/cpu.h>
26
#include <linux/bitops.h>
27
#include <linux/device.h>
I
Ingo Molnar 已提交
28 29

#include <asm/apic.h>
30
#include <asm/stacktrace.h>
P
Peter Zijlstra 已提交
31
#include <asm/nmi.h>
32
#include <asm/smp.h>
33
#include <asm/alternative.h>
34
#include <asm/timer.h>
35 36
#include <asm/desc.h>
#include <asm/ldt.h>
I
Ingo Molnar 已提交
37

38 39 40
#include "perf_event.h"

struct x86_pmu x86_pmu __read_mostly;
41

42
DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
43 44
	.enabled = 1,
};
I
Ingo Molnar 已提交
45

46
u64 __read_mostly hw_cache_event_ids
47 48 49
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX];
50
u64 __read_mostly hw_cache_extra_regs
51 52 53
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX];
54

55
/*
56 57
 * Propagate event elapsed time into the generic event.
 * Can only be executed on the CPU where the event is active.
58 59
 * Returns the delta events processed.
 */
60
u64 x86_perf_event_update(struct perf_event *event)
61
{
62
	struct hw_perf_event *hwc = &event->hw;
63
	int shift = 64 - x86_pmu.cntval_bits;
64
	u64 prev_raw_count, new_raw_count;
65
	int idx = hwc->idx;
66
	s64 delta;
67

68
	if (idx == INTEL_PMC_IDX_FIXED_BTS)
69 70
		return 0;

71
	/*
72
	 * Careful: an NMI might modify the previous event value.
73 74 75
	 *
	 * Our tactic to handle this is to first atomically read and
	 * exchange a new raw count - then add that new-prev delta
76
	 * count to the generic event atomically:
77 78
	 */
again:
79
	prev_raw_count = local64_read(&hwc->prev_count);
80
	rdpmcl(hwc->event_base_rdpmc, new_raw_count);
81

82
	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
83 84 85 86 87 88
					new_raw_count) != prev_raw_count)
		goto again;

	/*
	 * Now we have the new raw value and have updated the prev
	 * timestamp already. We can now calculate the elapsed delta
89
	 * (event-)time and add that to the generic event.
90 91
	 *
	 * Careful, not all hw sign-extends above the physical width
92
	 * of the count.
93
	 */
94 95
	delta = (new_raw_count << shift) - (prev_raw_count << shift);
	delta >>= shift;
96

97 98
	local64_add(delta, &event->count);
	local64_sub(delta, &hwc->period_left);
99 100

	return new_raw_count;
101 102
}

103 104 105 106 107
/*
 * Find and validate any extra registers to set up.
 */
static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
{
108
	struct hw_perf_event_extra *reg;
109 110
	struct extra_reg *er;

111
	reg = &event->hw.extra_reg;
112 113 114 115 116 117 118 119 120

	if (!x86_pmu.extra_regs)
		return 0;

	for (er = x86_pmu.extra_regs; er->msr; er++) {
		if (er->event != (config & er->config_mask))
			continue;
		if (event->attr.config1 & ~er->valid_mask)
			return -EINVAL;
121 122 123 124

		reg->idx = er->idx;
		reg->config = event->attr.config1;
		reg->reg = er->msr;
125 126 127 128 129
		break;
	}
	return 0;
}

130
static atomic_t active_events;
P
Peter Zijlstra 已提交
131 132
static DEFINE_MUTEX(pmc_reserve_mutex);

133 134
#ifdef CONFIG_X86_LOCAL_APIC

P
Peter Zijlstra 已提交
135 136 137 138
static bool reserve_pmc_hardware(void)
{
	int i;

139
	for (i = 0; i < x86_pmu.num_counters; i++) {
140
		if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
P
Peter Zijlstra 已提交
141 142 143
			goto perfctr_fail;
	}

144
	for (i = 0; i < x86_pmu.num_counters; i++) {
145
		if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
P
Peter Zijlstra 已提交
146 147 148 149 150 151 152
			goto eventsel_fail;
	}

	return true;

eventsel_fail:
	for (i--; i >= 0; i--)
153
		release_evntsel_nmi(x86_pmu_config_addr(i));
P
Peter Zijlstra 已提交
154

155
	i = x86_pmu.num_counters;
P
Peter Zijlstra 已提交
156 157 158

perfctr_fail:
	for (i--; i >= 0; i--)
159
		release_perfctr_nmi(x86_pmu_event_addr(i));
P
Peter Zijlstra 已提交
160 161 162 163 164 165 166 167

	return false;
}

static void release_pmc_hardware(void)
{
	int i;

168
	for (i = 0; i < x86_pmu.num_counters; i++) {
169 170
		release_perfctr_nmi(x86_pmu_event_addr(i));
		release_evntsel_nmi(x86_pmu_config_addr(i));
P
Peter Zijlstra 已提交
171 172 173
	}
}

174 175 176 177 178 179 180
#else

static bool reserve_pmc_hardware(void) { return true; }
static void release_pmc_hardware(void) {}

#endif

181 182
static bool check_hw_exists(void)
{
183 184 185
	u64 val, val_fail, val_new= ~0;
	int i, reg, reg_fail, ret = 0;
	int bios_fail = 0;
186

187 188 189 190 191
	/*
	 * Check to see if the BIOS enabled any of the counters, if so
	 * complain and bail.
	 */
	for (i = 0; i < x86_pmu.num_counters; i++) {
192
		reg = x86_pmu_config_addr(i);
193 194 195
		ret = rdmsrl_safe(reg, &val);
		if (ret)
			goto msr_fail;
196 197 198 199 200
		if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
			bios_fail = 1;
			val_fail = val;
			reg_fail = reg;
		}
201 202 203 204 205 206 207 208
	}

	if (x86_pmu.num_counters_fixed) {
		reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
		ret = rdmsrl_safe(reg, &val);
		if (ret)
			goto msr_fail;
		for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
209 210 211 212 213
			if (val & (0x03 << i*4)) {
				bios_fail = 1;
				val_fail = val;
				reg_fail = reg;
			}
214 215 216 217
		}
	}

	/*
218 219 220
	 * Read the current value, change it and read it back to see if it
	 * matches, this is needed to detect certain hardware emulators
	 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
221
	 */
222
	reg = x86_pmu_event_addr(0);
223 224 225
	if (rdmsrl_safe(reg, &val))
		goto msr_fail;
	val ^= 0xffffUL;
226 227
	ret = wrmsrl_safe(reg, val);
	ret |= rdmsrl_safe(reg, &val_new);
228
	if (ret || val != val_new)
229
		goto msr_fail;
230

231 232 233
	/*
	 * We still allow the PMU driver to operate:
	 */
234 235 236 237
	if (bios_fail) {
		printk(KERN_CONT "Broken BIOS detected, complain to your hardware vendor.\n");
		printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg_fail, val_fail);
	}
238 239

	return true;
240 241 242

msr_fail:
	printk(KERN_CONT "Broken PMU hardware detected, using software events only.\n");
243
	printk(KERN_ERR "Failed to access perfctr msr (MSR %x is %Lx)\n", reg, val_new);
244

245
	return false;
246 247
}

248
static void hw_perf_event_destroy(struct perf_event *event)
P
Peter Zijlstra 已提交
249
{
250
	if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
P
Peter Zijlstra 已提交
251
		release_pmc_hardware();
252
		release_ds_buffers();
P
Peter Zijlstra 已提交
253 254 255 256
		mutex_unlock(&pmc_reserve_mutex);
	}
}

257 258 259 260 261
static inline int x86_pmu_initialized(void)
{
	return x86_pmu.handle_irq != NULL;
}

262
static inline int
263
set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
264
{
265
	struct perf_event_attr *attr = &event->attr;
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
	unsigned int cache_type, cache_op, cache_result;
	u64 config, val;

	config = attr->config;

	cache_type = (config >>  0) & 0xff;
	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
		return -EINVAL;

	cache_op = (config >>  8) & 0xff;
	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
		return -EINVAL;

	cache_result = (config >> 16) & 0xff;
	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
		return -EINVAL;

	val = hw_cache_event_ids[cache_type][cache_op][cache_result];

	if (val == 0)
		return -ENOENT;

	if (val == -1)
		return -EINVAL;

	hwc->config |= val;
292 293
	attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
	return x86_pmu_extra_regs(val, event);
294 295
}

296
int x86_setup_perfctr(struct perf_event *event)
297 298 299 300 301
{
	struct perf_event_attr *attr = &event->attr;
	struct hw_perf_event *hwc = &event->hw;
	u64 config;

302
	if (!is_sampling_event(event)) {
303 304
		hwc->sample_period = x86_pmu.max_period;
		hwc->last_period = hwc->sample_period;
305
		local64_set(&hwc->period_left, hwc->sample_period);
306 307 308 309 310 311 312 313 314 315 316 317
	} else {
		/*
		 * If we have a PMU initialized but no APIC
		 * interrupts, we cannot sample hardware
		 * events (user-space has to fall back and
		 * sample via a hrtimer based software event):
		 */
		if (!x86_pmu.apic)
			return -EOPNOTSUPP;
	}

	if (attr->type == PERF_TYPE_RAW)
318
		return x86_pmu_extra_regs(event->attr.config, event);
319 320

	if (attr->type == PERF_TYPE_HW_CACHE)
321
		return set_ext_hw_attr(hwc, event);
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339

	if (attr->config >= x86_pmu.max_events)
		return -EINVAL;

	/*
	 * The generic map:
	 */
	config = x86_pmu.event_map(attr->config);

	if (config == 0)
		return -ENOENT;

	if (config == -1LL)
		return -EINVAL;

	/*
	 * Branch tracing:
	 */
P
Peter Zijlstra 已提交
340 341
	if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
	    !attr->freq && hwc->sample_period == 1) {
342
		/* BTS is not supported by this architecture. */
343
		if (!x86_pmu.bts_active)
344 345 346 347 348 349 350 351 352 353 354
			return -EOPNOTSUPP;

		/* BTS is currently only allowed for user-mode. */
		if (!attr->exclude_kernel)
			return -EOPNOTSUPP;
	}

	hwc->config |= config;

	return 0;
}
355

356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
/*
 * check that branch_sample_type is compatible with
 * settings needed for precise_ip > 1 which implies
 * using the LBR to capture ALL taken branches at the
 * priv levels of the measurement
 */
static inline int precise_br_compat(struct perf_event *event)
{
	u64 m = event->attr.branch_sample_type;
	u64 b = 0;

	/* must capture all branches */
	if (!(m & PERF_SAMPLE_BRANCH_ANY))
		return 0;

	m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;

	if (!event->attr.exclude_user)
		b |= PERF_SAMPLE_BRANCH_USER;

	if (!event->attr.exclude_kernel)
		b |= PERF_SAMPLE_BRANCH_KERNEL;

	/*
	 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
	 */

	return m == b;
}

386
int x86_pmu_hw_config(struct perf_event *event)
387
{
P
Peter Zijlstra 已提交
388 389 390 391
	if (event->attr.precise_ip) {
		int precise = 0;

		/* Support for constant skid */
392
		if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
P
Peter Zijlstra 已提交
393 394
			precise++;

395 396 397 398
			/* Support for IP fixup */
			if (x86_pmu.lbr_nr)
				precise++;
		}
P
Peter Zijlstra 已提交
399 400 401

		if (event->attr.precise_ip > precise)
			return -EOPNOTSUPP;
402 403 404 405
		/*
		 * check that PEBS LBR correction does not conflict with
		 * whatever the user is asking with attr->branch_sample_type
		 */
406 407
		if (event->attr.precise_ip > 1 &&
		    x86_pmu.intel_cap.pebs_format < 2) {
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
			u64 *br_type = &event->attr.branch_sample_type;

			if (has_branch_stack(event)) {
				if (!precise_br_compat(event))
					return -EOPNOTSUPP;

				/* branch_sample_type is compatible */

			} else {
				/*
				 * user did not specify  branch_sample_type
				 *
				 * For PEBS fixups, we capture all
				 * the branches at the priv level of the
				 * event.
				 */
				*br_type = PERF_SAMPLE_BRANCH_ANY;

				if (!event->attr.exclude_user)
					*br_type |= PERF_SAMPLE_BRANCH_USER;

				if (!event->attr.exclude_kernel)
					*br_type |= PERF_SAMPLE_BRANCH_KERNEL;
			}
		}
P
Peter Zijlstra 已提交
433 434
	}

435 436 437 438
	/*
	 * Generate PMC IRQs:
	 * (keep 'enabled' bit clear for now)
	 */
439
	event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
440 441 442 443

	/*
	 * Count user and OS events unless requested not to
	 */
444 445 446 447
	if (!event->attr.exclude_user)
		event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
	if (!event->attr.exclude_kernel)
		event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
448

449 450
	if (event->attr.type == PERF_TYPE_RAW)
		event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
451

452
	return x86_setup_perfctr(event);
453 454
}

I
Ingo Molnar 已提交
455
/*
456
 * Setup the hardware configuration for a given attr_type
I
Ingo Molnar 已提交
457
 */
458
static int __x86_pmu_event_init(struct perf_event *event)
I
Ingo Molnar 已提交
459
{
P
Peter Zijlstra 已提交
460
	int err;
I
Ingo Molnar 已提交
461

462 463
	if (!x86_pmu_initialized())
		return -ENODEV;
I
Ingo Molnar 已提交
464

P
Peter Zijlstra 已提交
465
	err = 0;
466
	if (!atomic_inc_not_zero(&active_events)) {
P
Peter Zijlstra 已提交
467
		mutex_lock(&pmc_reserve_mutex);
468
		if (atomic_read(&active_events) == 0) {
469 470
			if (!reserve_pmc_hardware())
				err = -EBUSY;
471 472
			else
				reserve_ds_buffers();
473 474
		}
		if (!err)
475
			atomic_inc(&active_events);
P
Peter Zijlstra 已提交
476 477 478 479 480
		mutex_unlock(&pmc_reserve_mutex);
	}
	if (err)
		return err;

481
	event->destroy = hw_perf_event_destroy;
482

483 484 485
	event->hw.idx = -1;
	event->hw.last_cpu = -1;
	event->hw.last_tag = ~0ULL;
486

487 488
	/* mark unused */
	event->hw.extra_reg.idx = EXTRA_REG_NONE;
489 490
	event->hw.branch_reg.idx = EXTRA_REG_NONE;

491
	return x86_pmu.hw_config(event);
492 493
}

494
void x86_pmu_disable_all(void)
495
{
496
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
497 498
	int idx;

499
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
500 501
		u64 val;

502
		if (!test_bit(idx, cpuc->active_mask))
503
			continue;
504
		rdmsrl(x86_pmu_config_addr(idx), val);
505
		if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
506
			continue;
507
		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
508
		wrmsrl(x86_pmu_config_addr(idx), val);
509 510 511
	}
}

P
Peter Zijlstra 已提交
512
static void x86_pmu_disable(struct pmu *pmu)
513
{
514 515
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

516
	if (!x86_pmu_initialized())
517
		return;
518

519 520 521 522 523 524
	if (!cpuc->enabled)
		return;

	cpuc->n_added = 0;
	cpuc->enabled = 0;
	barrier();
525 526

	x86_pmu.disable_all();
527
}
I
Ingo Molnar 已提交
528

529
void x86_pmu_enable_all(int added)
530
{
531
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
532 533
	int idx;

534
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
535
		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
536

537
		if (!test_bit(idx, cpuc->active_mask))
538
			continue;
539

540
		__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
541 542 543
	}
}

P
Peter Zijlstra 已提交
544
static struct pmu pmu;
545 546 547 548 549 550

static inline int is_x86_event(struct perf_event *event)
{
	return event->pmu == &pmu;
}

551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
/*
 * Event scheduler state:
 *
 * Assign events iterating over all events and counters, beginning
 * with events with least weights first. Keep the current iterator
 * state in struct sched_state.
 */
struct sched_state {
	int	weight;
	int	event;		/* event index */
	int	counter;	/* counter index */
	int	unassigned;	/* number of events to be assigned left */
	unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
};

566 567 568
/* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
#define	SCHED_STATES_MAX	2

569 570 571
struct perf_sched {
	int			max_weight;
	int			max_events;
572
	struct perf_event	**events;
573
	struct sched_state	state;
574 575
	int			saved_states;
	struct sched_state	saved[SCHED_STATES_MAX];
576 577 578 579 580
};

/*
 * Initialize interator that runs through all events and counters.
 */
581
static void perf_sched_init(struct perf_sched *sched, struct perf_event **events,
582 583 584 585 586 587 588
			    int num, int wmin, int wmax)
{
	int idx;

	memset(sched, 0, sizeof(*sched));
	sched->max_events	= num;
	sched->max_weight	= wmax;
589
	sched->events		= events;
590 591

	for (idx = 0; idx < num; idx++) {
592
		if (events[idx]->hw.constraint->weight == wmin)
593 594 595 596 597 598 599 600
			break;
	}

	sched->state.event	= idx;		/* start with min weight */
	sched->state.weight	= wmin;
	sched->state.unassigned	= num;
}

601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
static void perf_sched_save_state(struct perf_sched *sched)
{
	if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
		return;

	sched->saved[sched->saved_states] = sched->state;
	sched->saved_states++;
}

static bool perf_sched_restore_state(struct perf_sched *sched)
{
	if (!sched->saved_states)
		return false;

	sched->saved_states--;
	sched->state = sched->saved[sched->saved_states];

	/* continue with next counter: */
	clear_bit(sched->state.counter++, sched->state.used);

	return true;
}

624 625 626 627
/*
 * Select a counter for the current event to schedule. Return true on
 * success.
 */
628
static bool __perf_sched_find_counter(struct perf_sched *sched)
629 630 631 632 633 634 635 636 637 638
{
	struct event_constraint *c;
	int idx;

	if (!sched->state.unassigned)
		return false;

	if (sched->state.event >= sched->max_events)
		return false;

639
	c = sched->events[sched->state.event]->hw.constraint;
640
	/* Prefer fixed purpose counters */
641 642
	if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
		idx = INTEL_PMC_IDX_FIXED;
643
		for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
644 645 646 647
			if (!__test_and_set_bit(idx, sched->state.used))
				goto done;
		}
	}
648 649
	/* Grab the first unused counter starting with idx */
	idx = sched->state.counter;
650
	for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
651
		if (!__test_and_set_bit(idx, sched->state.used))
652
			goto done;
653 654
	}

655 656 657 658
	return false;

done:
	sched->state.counter = idx;
659

660 661 662 663 664 665 666 667 668 669 670 671 672
	if (c->overlap)
		perf_sched_save_state(sched);

	return true;
}

static bool perf_sched_find_counter(struct perf_sched *sched)
{
	while (!__perf_sched_find_counter(sched)) {
		if (!perf_sched_restore_state(sched))
			return false;
	}

673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
	return true;
}

/*
 * Go through all unassigned events and find the next one to schedule.
 * Take events with the least weight first. Return true on success.
 */
static bool perf_sched_next_event(struct perf_sched *sched)
{
	struct event_constraint *c;

	if (!sched->state.unassigned || !--sched->state.unassigned)
		return false;

	do {
		/* next event */
		sched->state.event++;
		if (sched->state.event >= sched->max_events) {
			/* next weight */
			sched->state.event = 0;
			sched->state.weight++;
			if (sched->state.weight > sched->max_weight)
				return false;
		}
697
		c = sched->events[sched->state.event]->hw.constraint;
698 699 700 701 702 703 704 705 706 707
	} while (c->weight != sched->state.weight);

	sched->state.counter = 0;	/* start with first counter */

	return true;
}

/*
 * Assign a counter for each event.
 */
708
int perf_assign_events(struct perf_event **events, int n,
Y
Yan, Zheng 已提交
709
			int wmin, int wmax, int *assign)
710 711 712
{
	struct perf_sched sched;

713
	perf_sched_init(&sched, events, n, wmin, wmax);
714 715 716 717 718 719 720 721 722 723 724

	do {
		if (!perf_sched_find_counter(&sched))
			break;	/* failed */
		if (assign)
			assign[sched.state.event] = sched.state.counter;
	} while (perf_sched_next_event(&sched));

	return sched.state.unassigned;
}

725
int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
726
{
727
	struct event_constraint *c;
728
	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
729
	struct perf_event *e;
730
	int i, wmin, wmax, num = 0;
731 732 733 734
	struct hw_perf_event *hwc;

	bitmap_zero(used_mask, X86_PMC_IDX_MAX);

735
	for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
736
		hwc = &cpuc->event_list[i]->hw;
737
		c = x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
738 739
		hwc->constraint = c;

740 741
		wmin = min(wmin, c->weight);
		wmax = max(wmax, c->weight);
742 743
	}

744 745 746
	/*
	 * fastpath, try to reuse previous register
	 */
747
	for (i = 0; i < n; i++) {
748
		hwc = &cpuc->event_list[i]->hw;
749
		c = hwc->constraint;
750 751 752 753 754 755

		/* never assigned */
		if (hwc->idx == -1)
			break;

		/* constraint still honored */
756
		if (!test_bit(hwc->idx, c->idxmsk))
757 758 759 760 761 762
			break;

		/* not already used */
		if (test_bit(hwc->idx, used_mask))
			break;

P
Peter Zijlstra 已提交
763
		__set_bit(hwc->idx, used_mask);
764 765 766 767
		if (assign)
			assign[i] = hwc->idx;
	}

768 769
	/* slow path */
	if (i != n)
770 771
		num = perf_assign_events(cpuc->event_list, n, wmin,
					 wmax, assign);
772

773 774 775 776 777 778 779 780 781 782
	/*
	 * Mark the event as committed, so we do not put_constraint()
	 * in case new events are added and fail scheduling.
	 */
	if (!num && assign) {
		for (i = 0; i < n; i++) {
			e = cpuc->event_list[i];
			e->hw.flags |= PERF_X86_EVENT_COMMITTED;
		}
	}
783 784 785 786 787 788
	/*
	 * scheduling failed or is just a simulation,
	 * free resources if necessary
	 */
	if (!assign || num) {
		for (i = 0; i < n; i++) {
789 790 791 792 793 794 795 796
			e = cpuc->event_list[i];
			/*
			 * do not put_constraint() on comitted events,
			 * because they are good to go
			 */
			if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
				continue;

797
			if (x86_pmu.put_event_constraints)
798
				x86_pmu.put_event_constraints(cpuc, e);
799 800
		}
	}
801
	return num ? -EINVAL : 0;
802 803 804 805 806 807 808 809 810 811 812
}

/*
 * dogrp: true if must collect siblings events (group)
 * returns total number of events and error code
 */
static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
{
	struct perf_event *event;
	int n, max_count;

813
	max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
814 815 816 817 818 819

	/* current number of events already accepted */
	n = cpuc->n_events;

	if (is_x86_event(leader)) {
		if (n >= max_count)
820
			return -EINVAL;
821 822 823 824 825 826 827 828
		cpuc->event_list[n] = leader;
		n++;
	}
	if (!dogrp)
		return n;

	list_for_each_entry(event, &leader->sibling_list, group_entry) {
		if (!is_x86_event(event) ||
829
		    event->state <= PERF_EVENT_STATE_OFF)
830 831 832
			continue;

		if (n >= max_count)
833
			return -EINVAL;
834 835 836 837 838 839 840 841

		cpuc->event_list[n] = event;
		n++;
	}
	return n;
}

static inline void x86_assign_hw_event(struct perf_event *event,
842
				struct cpu_hw_events *cpuc, int i)
843
{
844 845 846 847 848
	struct hw_perf_event *hwc = &event->hw;

	hwc->idx = cpuc->assign[i];
	hwc->last_cpu = smp_processor_id();
	hwc->last_tag = ++cpuc->tags[i];
849

850
	if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
851 852
		hwc->config_base = 0;
		hwc->event_base	= 0;
853
	} else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
854
		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
855 856
		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
		hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
857
	} else {
858 859
		hwc->config_base = x86_pmu_config_addr(hwc->idx);
		hwc->event_base  = x86_pmu_event_addr(hwc->idx);
860
		hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
861 862 863
	}
}

864 865 866 867 868 869 870 871 872
static inline int match_prev_assignment(struct hw_perf_event *hwc,
					struct cpu_hw_events *cpuc,
					int i)
{
	return hwc->idx == cpuc->assign[i] &&
		hwc->last_cpu == smp_processor_id() &&
		hwc->last_tag == cpuc->tags[i];
}

P
Peter Zijlstra 已提交
873
static void x86_pmu_start(struct perf_event *event, int flags);
874

P
Peter Zijlstra 已提交
875
static void x86_pmu_enable(struct pmu *pmu)
876
{
877 878 879
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct perf_event *event;
	struct hw_perf_event *hwc;
880
	int i, added = cpuc->n_added;
881

882
	if (!x86_pmu_initialized())
883
		return;
884 885 886 887

	if (cpuc->enabled)
		return;

888
	if (cpuc->n_added) {
889
		int n_running = cpuc->n_events - cpuc->n_added;
890 891 892 893 894 895
		/*
		 * apply assignment obtained either from
		 * hw_perf_group_sched_in() or x86_pmu_enable()
		 *
		 * step1: save events moving to new counters
		 */
896
		for (i = 0; i < n_running; i++) {
897 898 899
			event = cpuc->event_list[i];
			hwc = &event->hw;

900 901 902 903 904 905 906 907
			/*
			 * we can avoid reprogramming counter if:
			 * - assigned same counter as last time
			 * - running on same CPU as last time
			 * - no other event has used the counter since
			 */
			if (hwc->idx == -1 ||
			    match_prev_assignment(hwc, cpuc, i))
908 909
				continue;

P
Peter Zijlstra 已提交
910 911 912 913 914 915 916 917
			/*
			 * Ensure we don't accidentally enable a stopped
			 * counter simply because we rescheduled.
			 */
			if (hwc->state & PERF_HES_STOPPED)
				hwc->state |= PERF_HES_ARCH;

			x86_pmu_stop(event, PERF_EF_UPDATE);
918 919
		}

920 921 922
		/*
		 * step2: reprogram moved events into new counters
		 */
923 924 925 926
		for (i = 0; i < cpuc->n_events; i++) {
			event = cpuc->event_list[i];
			hwc = &event->hw;

927
			if (!match_prev_assignment(hwc, cpuc, i))
928
				x86_assign_hw_event(event, cpuc, i);
929 930
			else if (i < n_running)
				continue;
931

P
Peter Zijlstra 已提交
932 933 934 935
			if (hwc->state & PERF_HES_ARCH)
				continue;

			x86_pmu_start(event, PERF_EF_RELOAD);
936 937 938 939
		}
		cpuc->n_added = 0;
		perf_events_lapic_init();
	}
940 941 942 943

	cpuc->enabled = 1;
	barrier();

944
	x86_pmu.enable_all(added);
945 946
}

947
static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
I
Ingo Molnar 已提交
948

949 950
/*
 * Set the next IRQ period, based on the hwc->period_left value.
951
 * To be called with the event disabled in hw:
952
 */
953
int x86_perf_event_set_period(struct perf_event *event)
I
Ingo Molnar 已提交
954
{
955
	struct hw_perf_event *hwc = &event->hw;
956
	s64 left = local64_read(&hwc->period_left);
957
	s64 period = hwc->sample_period;
958
	int ret = 0, idx = hwc->idx;
959

960
	if (idx == INTEL_PMC_IDX_FIXED_BTS)
961 962
		return 0;

963
	/*
964
	 * If we are way outside a reasonable range then just skip forward:
965 966 967
	 */
	if (unlikely(left <= -period)) {
		left = period;
968
		local64_set(&hwc->period_left, left);
969
		hwc->last_period = period;
970
		ret = 1;
971 972 973 974
	}

	if (unlikely(left <= 0)) {
		left += period;
975
		local64_set(&hwc->period_left, left);
976
		hwc->last_period = period;
977
		ret = 1;
978
	}
979
	/*
980
	 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
981 982 983
	 */
	if (unlikely(left < 2))
		left = 2;
I
Ingo Molnar 已提交
984

985 986 987
	if (left > x86_pmu.max_period)
		left = x86_pmu.max_period;

988
	per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
989 990

	/*
991
	 * The hw event starts counting from this event offset,
992 993
	 * mark it to be able to extra future deltas:
	 */
994
	local64_set(&hwc->prev_count, (u64)-left);
995

996
	wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
997 998 999 1000 1001 1002 1003

	/*
	 * Due to erratum on certan cpu we need
	 * a second write to be sure the register
	 * is updated properly
	 */
	if (x86_pmu.perfctr_second_write) {
1004
		wrmsrl(hwc->event_base,
1005
			(u64)(-left) & x86_pmu.cntval_mask);
1006
	}
1007

1008
	perf_event_update_userpage(event);
1009

1010
	return ret;
1011 1012
}

1013
void x86_pmu_enable_event(struct perf_event *event)
1014
{
T
Tejun Heo 已提交
1015
	if (__this_cpu_read(cpu_hw_events.enabled))
1016 1017
		__x86_pmu_enable_event(&event->hw,
				       ARCH_PERFMON_EVENTSEL_ENABLE);
I
Ingo Molnar 已提交
1018 1019
}

1020
/*
P
Peter Zijlstra 已提交
1021
 * Add a single event to the PMU.
1022 1023 1024
 *
 * The event is added to the group of enabled events
 * but only if it can be scehduled with existing events.
1025
 */
P
Peter Zijlstra 已提交
1026
static int x86_pmu_add(struct perf_event *event, int flags)
1027 1028
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1029 1030 1031
	struct hw_perf_event *hwc;
	int assign[X86_PMC_IDX_MAX];
	int n, n0, ret;
1032

1033
	hwc = &event->hw;
1034

P
Peter Zijlstra 已提交
1035
	perf_pmu_disable(event->pmu);
1036
	n0 = cpuc->n_events;
1037 1038 1039
	ret = n = collect_events(cpuc, event, false);
	if (ret < 0)
		goto out;
1040

P
Peter Zijlstra 已提交
1041 1042 1043 1044
	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
	if (!(flags & PERF_EF_START))
		hwc->state |= PERF_HES_ARCH;

1045 1046
	/*
	 * If group events scheduling transaction was started,
L
Lucas De Marchi 已提交
1047
	 * skip the schedulability test here, it will be performed
1048
	 * at commit time (->commit_txn) as a whole.
1049
	 */
1050
	if (cpuc->group_flag & PERF_EVENT_TXN)
1051
		goto done_collect;
1052

1053
	ret = x86_pmu.schedule_events(cpuc, n, assign);
1054
	if (ret)
1055
		goto out;
1056 1057 1058 1059 1060
	/*
	 * copy new assignment, now we know it is possible
	 * will be used by hw_perf_enable()
	 */
	memcpy(cpuc->assign, assign, n*sizeof(int));
1061

1062
done_collect:
1063 1064 1065 1066
	/*
	 * Commit the collect_events() state. See x86_pmu_del() and
	 * x86_pmu_*_txn().
	 */
1067
	cpuc->n_events = n;
1068
	cpuc->n_added += n - n0;
1069
	cpuc->n_txn += n - n0;
1070

1071 1072
	ret = 0;
out:
P
Peter Zijlstra 已提交
1073
	perf_pmu_enable(event->pmu);
1074
	return ret;
I
Ingo Molnar 已提交
1075 1076
}

P
Peter Zijlstra 已提交
1077
static void x86_pmu_start(struct perf_event *event, int flags)
1078
{
P
Peter Zijlstra 已提交
1079 1080 1081
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int idx = event->hw.idx;

P
Peter Zijlstra 已提交
1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
		return;

	if (WARN_ON_ONCE(idx == -1))
		return;

	if (flags & PERF_EF_RELOAD) {
		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
		x86_perf_event_set_period(event);
	}

	event->hw.state = 0;
1094

P
Peter Zijlstra 已提交
1095 1096
	cpuc->events[idx] = event;
	__set_bit(idx, cpuc->active_mask);
1097
	__set_bit(idx, cpuc->running);
1098
	x86_pmu.enable(event);
P
Peter Zijlstra 已提交
1099
	perf_event_update_userpage(event);
1100 1101
}

1102
void perf_event_print_debug(void)
I
Ingo Molnar 已提交
1103
{
1104
	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1105
	u64 pebs;
1106
	struct cpu_hw_events *cpuc;
1107
	unsigned long flags;
1108 1109
	int cpu, idx;

1110
	if (!x86_pmu.num_counters)
1111
		return;
I
Ingo Molnar 已提交
1112

1113
	local_irq_save(flags);
I
Ingo Molnar 已提交
1114 1115

	cpu = smp_processor_id();
1116
	cpuc = &per_cpu(cpu_hw_events, cpu);
I
Ingo Molnar 已提交
1117

1118
	if (x86_pmu.version >= 2) {
1119 1120 1121 1122
		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1123
		rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1124 1125 1126 1127 1128 1129

		pr_info("\n");
		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1130
		pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
1131
	}
1132
	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
I
Ingo Molnar 已提交
1133

1134
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1135 1136
		rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
		rdmsrl(x86_pmu_event_addr(idx), pmc_count);
I
Ingo Molnar 已提交
1137

1138
		prev_left = per_cpu(pmc_prev_left[idx], cpu);
I
Ingo Molnar 已提交
1139

1140
		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
I
Ingo Molnar 已提交
1141
			cpu, idx, pmc_ctrl);
1142
		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
I
Ingo Molnar 已提交
1143
			cpu, idx, pmc_count);
1144
		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1145
			cpu, idx, prev_left);
I
Ingo Molnar 已提交
1146
	}
1147
	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1148 1149
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);

1150
		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1151 1152
			cpu, idx, pmc_count);
	}
1153
	local_irq_restore(flags);
I
Ingo Molnar 已提交
1154 1155
}

1156
void x86_pmu_stop(struct perf_event *event, int flags)
I
Ingo Molnar 已提交
1157
{
1158
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1159
	struct hw_perf_event *hwc = &event->hw;
I
Ingo Molnar 已提交
1160

P
Peter Zijlstra 已提交
1161 1162 1163 1164 1165 1166
	if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
		x86_pmu.disable(event);
		cpuc->events[hwc->idx] = NULL;
		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
		hwc->state |= PERF_HES_STOPPED;
	}
1167

P
Peter Zijlstra 已提交
1168 1169 1170 1171 1172 1173 1174 1175
	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
		/*
		 * Drain the remaining delta count out of a event
		 * that we are disabling:
		 */
		x86_perf_event_update(event);
		hwc->state |= PERF_HES_UPTODATE;
	}
1176 1177
}

P
Peter Zijlstra 已提交
1178
static void x86_pmu_del(struct perf_event *event, int flags)
1179 1180 1181 1182
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int i;

1183 1184 1185 1186 1187
	/*
	 * event is descheduled
	 */
	event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;

1188 1189 1190 1191
	/*
	 * If we're called during a txn, we don't need to do anything.
	 * The events never got scheduled and ->cancel_txn will truncate
	 * the event_list.
1192 1193 1194
	 *
	 * XXX assumes any ->del() called during a TXN will only be on
	 * an event added during that same TXN.
1195
	 */
1196
	if (cpuc->group_flag & PERF_EVENT_TXN)
1197 1198
		return;

1199 1200 1201
	/*
	 * Not a TXN, therefore cleanup properly.
	 */
P
Peter Zijlstra 已提交
1202
	x86_pmu_stop(event, PERF_EF_UPDATE);
1203

1204
	for (i = 0; i < cpuc->n_events; i++) {
1205 1206 1207
		if (event == cpuc->event_list[i])
			break;
	}
1208

1209 1210
	if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
		return;
P
Peter Zijlstra 已提交
1211

1212 1213 1214
	/* If we have a newly added event; make sure to decrease n_added. */
	if (i >= cpuc->n_events - cpuc->n_added)
		--cpuc->n_added;
1215

1216 1217 1218 1219 1220 1221 1222
	if (x86_pmu.put_event_constraints)
		x86_pmu.put_event_constraints(cpuc, event);

	/* Delete the array entry. */
	while (++i < cpuc->n_events)
		cpuc->event_list[i-1] = cpuc->event_list[i];
	--cpuc->n_events;
1223

1224
	perf_event_update_userpage(event);
I
Ingo Molnar 已提交
1225 1226
}

1227
int x86_pmu_handle_irq(struct pt_regs *regs)
1228
{
1229
	struct perf_sample_data data;
1230 1231
	struct cpu_hw_events *cpuc;
	struct perf_event *event;
V
Vince Weaver 已提交
1232
	int idx, handled = 0;
1233 1234
	u64 val;

1235
	cpuc = &__get_cpu_var(cpu_hw_events);
1236

1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
	/*
	 * Some chipsets need to unmask the LVTPC in a particular spot
	 * inside the nmi handler.  As a result, the unmasking was pushed
	 * into all the nmi handlers.
	 *
	 * This generic handler doesn't seem to have any issues where the
	 * unmasking occurs so it was left at the top.
	 */
	apic_write(APIC_LVTPC, APIC_DM_NMI);

1247
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1248 1249 1250 1251 1252 1253 1254 1255
		if (!test_bit(idx, cpuc->active_mask)) {
			/*
			 * Though we deactivated the counter some cpus
			 * might still deliver spurious interrupts still
			 * in flight. Catch them:
			 */
			if (__test_and_clear_bit(idx, cpuc->running))
				handled++;
1256
			continue;
1257
		}
1258

1259
		event = cpuc->events[idx];
1260

1261
		val = x86_perf_event_update(event);
1262
		if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1263
			continue;
1264

1265
		/*
1266
		 * event overflow
1267
		 */
1268
		handled++;
1269
		perf_sample_data_init(&data, 0, event->hw.last_period);
1270

1271
		if (!x86_perf_event_set_period(event))
1272 1273
			continue;

1274
		if (perf_event_overflow(event, &data, regs))
P
Peter Zijlstra 已提交
1275
			x86_pmu_stop(event, 0);
1276
	}
1277

1278 1279 1280
	if (handled)
		inc_irq_stat(apic_perf_irqs);

1281 1282
	return handled;
}
1283

1284
void perf_events_lapic_init(void)
I
Ingo Molnar 已提交
1285
{
1286
	if (!x86_pmu.apic || !x86_pmu_initialized())
I
Ingo Molnar 已提交
1287
		return;
1288

I
Ingo Molnar 已提交
1289
	/*
1290
	 * Always use NMI for PMU
I
Ingo Molnar 已提交
1291
	 */
1292
	apic_write(APIC_LVTPC, APIC_DM_NMI);
I
Ingo Molnar 已提交
1293 1294
}

1295
static int
1296
perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
I
Ingo Molnar 已提交
1297
{
1298 1299
	u64 start_clock;
	u64 finish_clock;
P
Peter Zijlstra 已提交
1300
	int ret;
1301

1302
	if (!atomic_read(&active_events))
1303
		return NMI_DONE;
1304

P
Peter Zijlstra 已提交
1305
	start_clock = sched_clock();
1306
	ret = x86_pmu.handle_irq(regs);
P
Peter Zijlstra 已提交
1307
	finish_clock = sched_clock();
1308 1309 1310 1311

	perf_sample_event_took(finish_clock - start_clock);

	return ret;
I
Ingo Molnar 已提交
1312
}
1313
NOKPROBE_SYMBOL(perf_event_nmi_handler);
I
Ingo Molnar 已提交
1314

1315 1316
struct event_constraint emptyconstraint;
struct event_constraint unconstrained;
1317

1318
static int
1319 1320 1321
x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;
1322
	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1323
	int ret = NOTIFY_OK;
1324 1325 1326

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_UP_PREPARE:
1327
		cpuc->kfree_on_online = NULL;
1328
		if (x86_pmu.cpu_prepare)
1329
			ret = x86_pmu.cpu_prepare(cpu);
1330 1331 1332
		break;

	case CPU_STARTING:
1333 1334
		if (x86_pmu.attr_rdpmc)
			set_in_cr4(X86_CR4_PCE);
1335 1336 1337 1338
		if (x86_pmu.cpu_starting)
			x86_pmu.cpu_starting(cpu);
		break;

1339 1340 1341 1342
	case CPU_ONLINE:
		kfree(cpuc->kfree_on_online);
		break;

1343 1344 1345 1346 1347
	case CPU_DYING:
		if (x86_pmu.cpu_dying)
			x86_pmu.cpu_dying(cpu);
		break;

1348
	case CPU_UP_CANCELED:
1349 1350 1351 1352 1353 1354 1355 1356 1357
	case CPU_DEAD:
		if (x86_pmu.cpu_dead)
			x86_pmu.cpu_dead(cpu);
		break;

	default:
		break;
	}

1358
	return ret;
1359 1360
}

1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
static void __init pmu_check_apic(void)
{
	if (cpu_has_apic)
		return;

	x86_pmu.apic = 0;
	pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
	pr_info("no hardware sampling interrupt available.\n");
}

1371 1372 1373 1374 1375
static struct attribute_group x86_pmu_format_group = {
	.name = "format",
	.attrs = NULL,
};

1376 1377 1378 1379 1380 1381
/*
 * Remove all undefined events (x86_pmu.event_map(id) == 0)
 * out of events_attr attributes.
 */
static void __init filter_events(struct attribute **attrs)
{
1382 1383
	struct device_attribute *d;
	struct perf_pmu_events_attr *pmu_attr;
1384 1385 1386
	int i, j;

	for (i = 0; attrs[i]; i++) {
1387 1388 1389 1390 1391
		d = (struct device_attribute *)attrs[i];
		pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
		/* str trumps id */
		if (pmu_attr->event_str)
			continue;
1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
		if (x86_pmu.event_map(i))
			continue;

		for (j = i; attrs[j]; j++)
			attrs[j] = attrs[j + 1];

		/* Check the shifted attr. */
		i--;
	}
}

1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
/* Merge two pointer arrays */
static __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
{
	struct attribute **new;
	int j, i;

	for (j = 0; a[j]; j++)
		;
	for (i = 0; b[i]; i++)
		j++;
	j++;

	new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL);
	if (!new)
		return NULL;

	j = 0;
	for (i = 0; a[i]; i++)
		new[j++] = a[i];
	for (i = 0; b[i]; i++)
		new[j++] = b[i];
	new[j] = NULL;

	return new;
}

1429
ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
1430 1431 1432 1433 1434 1435
			  char *page)
{
	struct perf_pmu_events_attr *pmu_attr = \
		container_of(attr, struct perf_pmu_events_attr, attr);
	u64 config = x86_pmu.event_map(pmu_attr->id);

1436 1437 1438
	/* string trumps id */
	if (pmu_attr->event_str)
		return sprintf(page, "%s", pmu_attr->event_str);
1439

1440 1441
	return x86_pmu.events_sysfs_show(page, config);
}
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455

EVENT_ATTR(cpu-cycles,			CPU_CYCLES		);
EVENT_ATTR(instructions,		INSTRUCTIONS		);
EVENT_ATTR(cache-references,		CACHE_REFERENCES	);
EVENT_ATTR(cache-misses, 		CACHE_MISSES		);
EVENT_ATTR(branch-instructions,		BRANCH_INSTRUCTIONS	);
EVENT_ATTR(branch-misses,		BRANCH_MISSES		);
EVENT_ATTR(bus-cycles,			BUS_CYCLES		);
EVENT_ATTR(stalled-cycles-frontend,	STALLED_CYCLES_FRONTEND	);
EVENT_ATTR(stalled-cycles-backend,	STALLED_CYCLES_BACKEND	);
EVENT_ATTR(ref-cycles,			REF_CPU_CYCLES		);

static struct attribute *empty_attrs;

P
Peter Huewe 已提交
1456
static struct attribute *events_attr[] = {
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
	EVENT_PTR(CPU_CYCLES),
	EVENT_PTR(INSTRUCTIONS),
	EVENT_PTR(CACHE_REFERENCES),
	EVENT_PTR(CACHE_MISSES),
	EVENT_PTR(BRANCH_INSTRUCTIONS),
	EVENT_PTR(BRANCH_MISSES),
	EVENT_PTR(BUS_CYCLES),
	EVENT_PTR(STALLED_CYCLES_FRONTEND),
	EVENT_PTR(STALLED_CYCLES_BACKEND),
	EVENT_PTR(REF_CPU_CYCLES),
	NULL,
};

static struct attribute_group x86_pmu_events_group = {
	.name = "events",
	.attrs = events_attr,
};

1475
ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
{
	u64 umask  = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
	u64 cmask  = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
	bool edge  = (config & ARCH_PERFMON_EVENTSEL_EDGE);
	bool pc    = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
	bool any   = (config & ARCH_PERFMON_EVENTSEL_ANY);
	bool inv   = (config & ARCH_PERFMON_EVENTSEL_INV);
	ssize_t ret;

	/*
	* We have whole page size to spend and just little data
	* to write, so we can safely use sprintf.
	*/
	ret = sprintf(page, "event=0x%02llx", event);

	if (umask)
		ret += sprintf(page + ret, ",umask=0x%02llx", umask);

	if (edge)
		ret += sprintf(page + ret, ",edge");

	if (pc)
		ret += sprintf(page + ret, ",pc");

	if (any)
		ret += sprintf(page + ret, ",any");

	if (inv)
		ret += sprintf(page + ret, ",inv");

	if (cmask)
		ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);

	ret += sprintf(page + ret, "\n");

	return ret;
}

1514
static int __init init_hw_perf_events(void)
1515
{
1516
	struct x86_pmu_quirk *quirk;
1517 1518
	int err;

1519
	pr_info("Performance Events: ");
1520

1521 1522
	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_INTEL:
1523
		err = intel_pmu_init();
1524
		break;
1525
	case X86_VENDOR_AMD:
1526
		err = amd_pmu_init();
1527
		break;
1528
	default:
1529
		err = -ENOTSUPP;
1530
	}
1531
	if (err != 0) {
1532
		pr_cont("no PMU driver, software events only.\n");
1533
		return 0;
1534
	}
1535

1536 1537
	pmu_check_apic();

1538
	/* sanity check that the hardware exists or is emulated */
1539
	if (!check_hw_exists())
1540
		return 0;
1541

1542
	pr_cont("%s PMU driver.\n", x86_pmu.name);
1543

1544 1545
	x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */

1546 1547
	for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
		quirk->func();
1548

1549 1550
	if (!x86_pmu.intel_ctrl)
		x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
I
Ingo Molnar 已提交
1551

1552
	perf_events_lapic_init();
1553
	register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1554

1555
	unconstrained = (struct event_constraint)
1556
		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1557
				   0, x86_pmu.num_counters, 0, 0);
1558

1559
	x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1560

1561 1562 1563
	if (x86_pmu.event_attrs)
		x86_pmu_events_group.attrs = x86_pmu.event_attrs;

1564 1565
	if (!x86_pmu.events_sysfs_show)
		x86_pmu_events_group.attrs = &empty_attrs;
1566 1567
	else
		filter_events(x86_pmu_events_group.attrs);
1568

1569 1570 1571 1572 1573 1574 1575 1576
	if (x86_pmu.cpu_events) {
		struct attribute **tmp;

		tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
		if (!WARN_ON(!tmp))
			x86_pmu_events_group.attrs = tmp;
	}

I
Ingo Molnar 已提交
1577
	pr_info("... version:                %d\n",     x86_pmu.version);
1578 1579 1580
	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
	pr_info("... generic registers:      %d\n",     x86_pmu.num_counters);
	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
I
Ingo Molnar 已提交
1581
	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
1582
	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
1583
	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
1584

P
Peter Zijlstra 已提交
1585
	perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1586
	perf_cpu_notifier(x86_pmu_notifier);
1587 1588

	return 0;
I
Ingo Molnar 已提交
1589
}
1590
early_initcall(init_hw_perf_events);
I
Ingo Molnar 已提交
1591

1592
static inline void x86_pmu_read(struct perf_event *event)
1593
{
1594
	x86_perf_event_update(event);
1595 1596
}

1597 1598 1599 1600 1601
/*
 * Start group events scheduling transaction
 * Set the flag to make pmu::enable() not perform the
 * schedulability test, it will be performed at commit time
 */
P
Peter Zijlstra 已提交
1602
static void x86_pmu_start_txn(struct pmu *pmu)
1603
{
P
Peter Zijlstra 已提交
1604
	perf_pmu_disable(pmu);
T
Tejun Heo 已提交
1605 1606
	__this_cpu_or(cpu_hw_events.group_flag, PERF_EVENT_TXN);
	__this_cpu_write(cpu_hw_events.n_txn, 0);
1607 1608 1609 1610 1611 1612 1613
}

/*
 * Stop group events scheduling transaction
 * Clear the flag and pmu::enable() will perform the
 * schedulability test.
 */
P
Peter Zijlstra 已提交
1614
static void x86_pmu_cancel_txn(struct pmu *pmu)
1615
{
T
Tejun Heo 已提交
1616
	__this_cpu_and(cpu_hw_events.group_flag, ~PERF_EVENT_TXN);
1617
	/*
1618 1619
	 * Truncate collected array by the number of events added in this
	 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
1620
	 */
T
Tejun Heo 已提交
1621 1622
	__this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
	__this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
P
Peter Zijlstra 已提交
1623
	perf_pmu_enable(pmu);
1624 1625 1626 1627 1628 1629
}

/*
 * Commit group events scheduling transaction
 * Perform the group schedulability test as a whole
 * Return 0 if success
1630 1631
 *
 * Does not cancel the transaction on failure; expects the caller to do this.
1632
 */
P
Peter Zijlstra 已提交
1633
static int x86_pmu_commit_txn(struct pmu *pmu)
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int assign[X86_PMC_IDX_MAX];
	int n, ret;

	n = cpuc->n_events;

	if (!x86_pmu_initialized())
		return -EAGAIN;

	ret = x86_pmu.schedule_events(cpuc, n, assign);
	if (ret)
		return ret;

	/*
	 * copy new assignment, now we know it is possible
	 * will be used by hw_perf_enable()
	 */
	memcpy(cpuc->assign, assign, n*sizeof(int));

1654
	cpuc->group_flag &= ~PERF_EVENT_TXN;
P
Peter Zijlstra 已提交
1655
	perf_pmu_enable(pmu);
1656 1657
	return 0;
}
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686
/*
 * a fake_cpuc is used to validate event groups. Due to
 * the extra reg logic, we need to also allocate a fake
 * per_core and per_cpu structure. Otherwise, group events
 * using extra reg may conflict without the kernel being
 * able to catch this when the last event gets added to
 * the group.
 */
static void free_fake_cpuc(struct cpu_hw_events *cpuc)
{
	kfree(cpuc->shared_regs);
	kfree(cpuc);
}

static struct cpu_hw_events *allocate_fake_cpuc(void)
{
	struct cpu_hw_events *cpuc;
	int cpu = raw_smp_processor_id();

	cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
	if (!cpuc)
		return ERR_PTR(-ENOMEM);

	/* only needed, if we have extra_regs */
	if (x86_pmu.extra_regs) {
		cpuc->shared_regs = allocate_shared_regs(cpu);
		if (!cpuc->shared_regs)
			goto error;
	}
1687
	cpuc->is_fake = 1;
1688 1689 1690 1691 1692
	return cpuc;
error:
	free_fake_cpuc(cpuc);
	return ERR_PTR(-ENOMEM);
}
1693

1694 1695 1696 1697 1698 1699 1700 1701 1702
/*
 * validate that we can schedule this event
 */
static int validate_event(struct perf_event *event)
{
	struct cpu_hw_events *fake_cpuc;
	struct event_constraint *c;
	int ret = 0;

1703 1704 1705
	fake_cpuc = allocate_fake_cpuc();
	if (IS_ERR(fake_cpuc))
		return PTR_ERR(fake_cpuc);
1706 1707 1708 1709

	c = x86_pmu.get_event_constraints(fake_cpuc, event);

	if (!c || !c->weight)
1710
		ret = -EINVAL;
1711 1712 1713 1714

	if (x86_pmu.put_event_constraints)
		x86_pmu.put_event_constraints(fake_cpuc, event);

1715
	free_fake_cpuc(fake_cpuc);
1716 1717 1718 1719

	return ret;
}

1720 1721 1722 1723
/*
 * validate a single event group
 *
 * validation include:
1724 1725 1726
 *	- check events are compatible which each other
 *	- events do not compete for the same counter
 *	- number of events <= number of counters
1727 1728 1729 1730
 *
 * validation ensures the group can be loaded onto the
 * PMU if it was the only group available.
 */
1731 1732
static int validate_group(struct perf_event *event)
{
1733
	struct perf_event *leader = event->group_leader;
1734
	struct cpu_hw_events *fake_cpuc;
1735
	int ret = -EINVAL, n;
1736

1737 1738 1739
	fake_cpuc = allocate_fake_cpuc();
	if (IS_ERR(fake_cpuc))
		return PTR_ERR(fake_cpuc);
1740 1741 1742 1743 1744 1745
	/*
	 * the event is not yet connected with its
	 * siblings therefore we must first collect
	 * existing siblings, then add the new event
	 * before we can simulate the scheduling
	 */
1746
	n = collect_events(fake_cpuc, leader, true);
1747
	if (n < 0)
1748
		goto out;
1749

1750 1751
	fake_cpuc->n_events = n;
	n = collect_events(fake_cpuc, event, false);
1752
	if (n < 0)
1753
		goto out;
1754

1755
	fake_cpuc->n_events = n;
1756

1757
	ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
1758 1759

out:
1760
	free_fake_cpuc(fake_cpuc);
1761
	return ret;
1762 1763
}

1764
static int x86_pmu_event_init(struct perf_event *event)
I
Ingo Molnar 已提交
1765
{
P
Peter Zijlstra 已提交
1766
	struct pmu *tmp;
I
Ingo Molnar 已提交
1767 1768
	int err;

1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779
	switch (event->attr.type) {
	case PERF_TYPE_RAW:
	case PERF_TYPE_HARDWARE:
	case PERF_TYPE_HW_CACHE:
		break;

	default:
		return -ENOENT;
	}

	err = __x86_pmu_event_init(event);
1780
	if (!err) {
1781 1782 1783 1784 1785 1786 1787 1788
		/*
		 * we temporarily connect event to its pmu
		 * such that validate_group() can classify
		 * it as an x86 event using is_x86_event()
		 */
		tmp = event->pmu;
		event->pmu = &pmu;

1789 1790
		if (event->group_leader != event)
			err = validate_group(event);
1791 1792
		else
			err = validate_event(event);
1793 1794

		event->pmu = tmp;
1795
	}
1796
	if (err) {
1797 1798
		if (event->destroy)
			event->destroy(event);
1799
	}
I
Ingo Molnar 已提交
1800

1801
	return err;
I
Ingo Molnar 已提交
1802
}
1803

1804 1805 1806 1807
static int x86_pmu_event_idx(struct perf_event *event)
{
	int idx = event->hw.idx;

1808 1809 1810
	if (!x86_pmu.attr_rdpmc)
		return 0;

1811 1812
	if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
		idx -= INTEL_PMC_IDX_FIXED;
1813 1814 1815 1816 1817 1818
		idx |= 1 << 30;
	}

	return idx + 1;
}

1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839
static ssize_t get_attr_rdpmc(struct device *cdev,
			      struct device_attribute *attr,
			      char *buf)
{
	return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
}

static void change_rdpmc(void *info)
{
	bool enable = !!(unsigned long)info;

	if (enable)
		set_in_cr4(X86_CR4_PCE);
	else
		clear_in_cr4(X86_CR4_PCE);
}

static ssize_t set_attr_rdpmc(struct device *cdev,
			      struct device_attribute *attr,
			      const char *buf, size_t count)
{
1840 1841 1842 1843 1844 1845
	unsigned long val;
	ssize_t ret;

	ret = kstrtoul(buf, 0, &val);
	if (ret)
		return ret;
1846 1847 1848

	if (x86_pmu.attr_rdpmc_broken)
		return -ENOTSUPP;
1849 1850 1851

	if (!!val != !!x86_pmu.attr_rdpmc) {
		x86_pmu.attr_rdpmc = !!val;
1852
		on_each_cpu(change_rdpmc, (void *)val, 1);
1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
	}

	return count;
}

static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);

static struct attribute *x86_pmu_attrs[] = {
	&dev_attr_rdpmc.attr,
	NULL,
};

static struct attribute_group x86_pmu_attr_group = {
	.attrs = x86_pmu_attrs,
};

static const struct attribute_group *x86_pmu_attr_groups[] = {
	&x86_pmu_attr_group,
1871
	&x86_pmu_format_group,
1872
	&x86_pmu_events_group,
1873 1874 1875
	NULL,
};

1876 1877 1878 1879 1880 1881
static void x86_pmu_flush_branch_stack(void)
{
	if (x86_pmu.flush_branch_stack)
		x86_pmu.flush_branch_stack();
}

1882 1883 1884 1885 1886 1887 1888
void perf_check_microcode(void)
{
	if (x86_pmu.check_microcode)
		x86_pmu.check_microcode();
}
EXPORT_SYMBOL_GPL(perf_check_microcode);

1889
static struct pmu pmu = {
1890 1891
	.pmu_enable		= x86_pmu_enable,
	.pmu_disable		= x86_pmu_disable,
P
Peter Zijlstra 已提交
1892

1893
	.attr_groups		= x86_pmu_attr_groups,
1894

1895
	.event_init		= x86_pmu_event_init,
P
Peter Zijlstra 已提交
1896

1897 1898 1899 1900 1901
	.add			= x86_pmu_add,
	.del			= x86_pmu_del,
	.start			= x86_pmu_start,
	.stop			= x86_pmu_stop,
	.read			= x86_pmu_read,
P
Peter Zijlstra 已提交
1902

1903 1904 1905
	.start_txn		= x86_pmu_start_txn,
	.cancel_txn		= x86_pmu_cancel_txn,
	.commit_txn		= x86_pmu_commit_txn,
1906

1907
	.event_idx		= x86_pmu_event_idx,
1908
	.flush_branch_stack	= x86_pmu_flush_branch_stack,
1909 1910
};

1911
void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
1912
{
1913 1914
	struct cyc2ns_data *data;

1915 1916 1917
	userpg->cap_user_time = 0;
	userpg->cap_user_time_zero = 0;
	userpg->cap_user_rdpmc = x86_pmu.attr_rdpmc;
1918 1919
	userpg->pmc_width = x86_pmu.cntval_bits;

1920
	if (!sched_clock_stable())
1921 1922
		return;

1923 1924
	data = cyc2ns_read_begin();

1925
	userpg->cap_user_time = 1;
1926 1927 1928
	userpg->time_mult = data->cyc2ns_mul;
	userpg->time_shift = data->cyc2ns_shift;
	userpg->time_offset = data->cyc2ns_offset - now;
1929

1930
	userpg->cap_user_time_zero = 1;
1931 1932 1933
	userpg->time_zero = data->cyc2ns_offset;

	cyc2ns_read_end(data);
1934 1935
}

1936 1937 1938 1939 1940 1941
/*
 * callchain support
 */

static int backtrace_stack(void *data, char *name)
{
1942
	return 0;
1943 1944 1945 1946 1947 1948
}

static void backtrace_address(void *data, unsigned long addr, int reliable)
{
	struct perf_callchain_entry *entry = data;

1949
	perf_callchain_store(entry, addr);
1950 1951 1952 1953 1954
}

static const struct stacktrace_ops backtrace_ops = {
	.stack			= backtrace_stack,
	.address		= backtrace_address,
1955
	.walk_stack		= print_context_stack_bp,
1956 1957
};

1958 1959
void
perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
1960
{
1961 1962
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
		/* TODO: We don't support guest os callchain now */
1963
		return;
1964 1965
	}

1966
	perf_callchain_store(entry, regs->ip);
1967

1968
	dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
1969 1970
}

1971 1972 1973 1974 1975 1976
static inline int
valid_user_frame(const void __user *fp, unsigned long size)
{
	return (__range_not_ok(fp, size, TASK_SIZE) == 0);
}

1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
static unsigned long get_segment_base(unsigned int segment)
{
	struct desc_struct *desc;
	int idx = segment >> 3;

	if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
		if (idx > LDT_ENTRIES)
			return 0;

		if (idx > current->active_mm->context.size)
			return 0;

		desc = current->active_mm->context.ldt;
	} else {
		if (idx > GDT_ENTRIES)
			return 0;

		desc = __this_cpu_ptr(&gdt_page.gdt[0]);
	}

	return get_desc_base(desc + idx);
}

2000
#ifdef CONFIG_COMPAT
H
H. Peter Anvin 已提交
2001 2002 2003

#include <asm/compat.h>

2004 2005
static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
2006
{
2007
	/* 32-bit process in 64-bit kernel. */
2008
	unsigned long ss_base, cs_base;
2009 2010
	struct stack_frame_ia32 frame;
	const void __user *fp;
2011

2012 2013 2014
	if (!test_thread_flag(TIF_IA32))
		return 0;

2015 2016 2017 2018
	cs_base = get_segment_base(regs->cs);
	ss_base = get_segment_base(regs->ss);

	fp = compat_ptr(ss_base + regs->bp);
2019 2020 2021 2022 2023 2024
	while (entry->nr < PERF_MAX_STACK_DEPTH) {
		unsigned long bytes;
		frame.next_frame     = 0;
		frame.return_address = 0;

		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
2025
		if (bytes != 0)
2026
			break;
2027

2028 2029 2030
		if (!valid_user_frame(fp, sizeof(frame)))
			break;

2031 2032
		perf_callchain_store(entry, cs_base + frame.return_address);
		fp = compat_ptr(ss_base + frame.next_frame);
2033 2034
	}
	return 1;
2035
}
2036 2037 2038 2039 2040 2041 2042
#else
static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
{
    return 0;
}
#endif
2043

2044 2045
void
perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
2046 2047 2048 2049
{
	struct stack_frame frame;
	const void __user *fp;

2050 2051
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
		/* TODO: We don't support guest os callchain now */
2052
		return;
2053
	}
2054

2055 2056 2057 2058 2059 2060
	/*
	 * We don't know what to do with VM86 stacks.. ignore them for now.
	 */
	if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
		return;

2061
	fp = (void __user *)regs->bp;
2062

2063
	perf_callchain_store(entry, regs->ip);
2064

2065 2066 2067
	if (!current->mm)
		return;

2068 2069 2070
	if (perf_callchain_user32(regs, entry))
		return;

2071
	while (entry->nr < PERF_MAX_STACK_DEPTH) {
2072
		unsigned long bytes;
2073
		frame.next_frame	     = NULL;
2074 2075
		frame.return_address = 0;

2076
		bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
2077
		if (bytes != 0)
2078 2079
			break;

2080 2081 2082
		if (!valid_user_frame(fp, sizeof(frame)))
			break;

2083
		perf_callchain_store(entry, frame.return_address);
2084
		fp = frame.next_frame;
2085 2086 2087
	}
}

2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101
/*
 * Deal with code segment offsets for the various execution modes:
 *
 *   VM86 - the good olde 16 bit days, where the linear address is
 *          20 bits and we use regs->ip + 0x10 * regs->cs.
 *
 *   IA32 - Where we need to look at GDT/LDT segment descriptor tables
 *          to figure out what the 32bit base address is.
 *
 *    X32 - has TIF_X32 set, but is running in x86_64
 *
 * X86_64 - CS,DS,SS,ES are all zero based.
 */
static unsigned long code_segment_base(struct pt_regs *regs)
2102
{
2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124
	/*
	 * If we are in VM86 mode, add the segment offset to convert to a
	 * linear address.
	 */
	if (regs->flags & X86_VM_MASK)
		return 0x10 * regs->cs;

	/*
	 * For IA32 we look at the GDT/LDT segment base to convert the
	 * effective IP to a linear address.
	 */
#ifdef CONFIG_X86_32
	if (user_mode(regs) && regs->cs != __USER_CS)
		return get_segment_base(regs->cs);
#else
	if (test_thread_flag(TIF_IA32)) {
		if (user_mode(regs) && regs->cs != __USER32_CS)
			return get_segment_base(regs->cs);
	}
#endif
	return 0;
}
2125

2126 2127
unsigned long perf_instruction_pointer(struct pt_regs *regs)
{
2128
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2129
		return perf_guest_cbs->get_guest_ip();
2130

2131
	return regs->ip + code_segment_base(regs);
2132 2133 2134 2135 2136
}

unsigned long perf_misc_flags(struct pt_regs *regs)
{
	int misc = 0;
2137

2138
	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2139 2140 2141 2142 2143
		if (perf_guest_cbs->is_user_mode())
			misc |= PERF_RECORD_MISC_GUEST_USER;
		else
			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
	} else {
2144
		if (user_mode(regs))
2145 2146 2147 2148 2149
			misc |= PERF_RECORD_MISC_USER;
		else
			misc |= PERF_RECORD_MISC_KERNEL;
	}

2150
	if (regs->flags & PERF_EFLAGS_EXACT)
P
Peter Zijlstra 已提交
2151
		misc |= PERF_RECORD_MISC_EXACT_IP;
2152 2153 2154

	return misc;
}
2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166

void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
{
	cap->version		= x86_pmu.version;
	cap->num_counters_gp	= x86_pmu.num_counters;
	cap->num_counters_fixed	= x86_pmu.num_counters_fixed;
	cap->bit_width_gp	= x86_pmu.cntval_bits;
	cap->bit_width_fixed	= x86_pmu.cntval_bits;
	cap->events_mask	= (unsigned int)x86_pmu.events_maskl;
	cap->events_mask_len	= x86_pmu.events_mask_len;
}
EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);