uncore.c 34.2 KB
Newer Older
1
#include <asm/cpu_device_id.h>
2
#include "uncore.h"
3 4

static struct intel_uncore_type *empty_uncore[] = { NULL, };
5 6
struct intel_uncore_type **uncore_msr_uncores = empty_uncore;
struct intel_uncore_type **uncore_pci_uncores = empty_uncore;
7

8 9 10
static bool pcidrv_registered;
struct pci_driver *uncore_pci_driver;
/* pci bus to socket mapping */
11 12
DEFINE_RAW_SPINLOCK(pci2phy_map_lock);
struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head);
13 14
struct pci_extra_dev *uncore_extra_pci_dev;
static int max_packages;
15

16 17 18 19
/* mask of cpus that collect uncore events */
static cpumask_t uncore_cpu_mask;

/* constraint for the fixed counter */
20
static struct event_constraint uncore_constraint_fixed =
21
	EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
22
struct event_constraint uncore_constraint_empty =
23
	EVENT_CONSTRAINT(0, 0, 0);
24

25 26
MODULE_LICENSE("GPL");

27
static int uncore_pcibus_to_physid(struct pci_bus *bus)
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
{
	struct pci2phy_map *map;
	int phys_id = -1;

	raw_spin_lock(&pci2phy_map_lock);
	list_for_each_entry(map, &pci2phy_map_head, list) {
		if (map->segment == pci_domain_nr(bus)) {
			phys_id = map->pbus_to_physid[bus->number];
			break;
		}
	}
	raw_spin_unlock(&pci2phy_map_lock);

	return phys_id;
}

44 45 46 47 48 49 50 51 52 53
static void uncore_free_pcibus_map(void)
{
	struct pci2phy_map *map, *tmp;

	list_for_each_entry_safe(map, tmp, &pci2phy_map_head, list) {
		list_del(&map->list);
		kfree(map);
	}
}

54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
struct pci2phy_map *__find_pci2phy_map(int segment)
{
	struct pci2phy_map *map, *alloc = NULL;
	int i;

	lockdep_assert_held(&pci2phy_map_lock);

lookup:
	list_for_each_entry(map, &pci2phy_map_head, list) {
		if (map->segment == segment)
			goto end;
	}

	if (!alloc) {
		raw_spin_unlock(&pci2phy_map_lock);
		alloc = kmalloc(sizeof(struct pci2phy_map), GFP_KERNEL);
		raw_spin_lock(&pci2phy_map_lock);

		if (!alloc)
			return NULL;

		goto lookup;
	}

	map = alloc;
	alloc = NULL;
	map->segment = segment;
	for (i = 0; i < 256; i++)
		map->pbus_to_physid[i] = -1;
	list_add_tail(&map->list, &pci2phy_map_head);

end:
	kfree(alloc);
	return map;
}

90 91 92 93 94 95 96 97 98
ssize_t uncore_event_show(struct kobject *kobj,
			  struct kobj_attribute *attr, char *buf)
{
	struct uncore_event_desc *event =
		container_of(attr, struct uncore_event_desc, attr);
	return sprintf(buf, "%s", event->config);
}

struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
99
{
100
	return pmu->boxes[topology_logical_package_id(cpu)];
101 102
}

103
u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event)
104 105 106 107 108 109 110 111 112 113 114
{
	u64 count;

	rdmsrl(event->hw.event_base, count);

	return count;
}

/*
 * generic get constraint function for shared match/mask registers.
 */
115
struct event_constraint *
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
{
	struct intel_uncore_extra_reg *er;
	struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
	struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
	unsigned long flags;
	bool ok = false;

	/*
	 * reg->alloc can be set due to existing state, so for fake box we
	 * need to ignore this, otherwise we might fail to allocate proper
	 * fake state for this extra reg constraint.
	 */
	if (reg1->idx == EXTRA_REG_NONE ||
	    (!uncore_box_is_fake(box) && reg1->alloc))
		return NULL;

	er = &box->shared_regs[reg1->idx];
	raw_spin_lock_irqsave(&er->lock, flags);
	if (!atomic_read(&er->ref) ||
	    (er->config1 == reg1->config && er->config2 == reg2->config)) {
		atomic_inc(&er->ref);
		er->config1 = reg1->config;
		er->config2 = reg2->config;
		ok = true;
	}
	raw_spin_unlock_irqrestore(&er->lock, flags);

	if (ok) {
		if (!uncore_box_is_fake(box))
			reg1->alloc = 1;
		return NULL;
	}

150
	return &uncore_constraint_empty;
151 152
}

153
void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
{
	struct intel_uncore_extra_reg *er;
	struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;

	/*
	 * Only put constraint if extra reg was actually allocated. Also
	 * takes care of event which do not use an extra shared reg.
	 *
	 * Also, if this is a fake box we shouldn't touch any event state
	 * (reg->alloc) and we don't care about leaving inconsistent box
	 * state either since it will be thrown out.
	 */
	if (uncore_box_is_fake(box) || !reg1->alloc)
		return;

	er = &box->shared_regs[reg1->idx];
	atomic_dec(&er->ref);
	reg1->alloc = 0;
}

174
u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
175 176 177 178 179 180 181 182 183 184 185 186 187 188
{
	struct intel_uncore_extra_reg *er;
	unsigned long flags;
	u64 config;

	er = &box->shared_regs[idx];

	raw_spin_lock_irqsave(&er->lock, flags);
	config = er->config;
	raw_spin_unlock_irqrestore(&er->lock, flags);

	return config;
}

189 190
static void uncore_assign_hw_event(struct intel_uncore_box *box,
				   struct perf_event *event, int idx)
191 192 193 194 195 196 197
{
	struct hw_perf_event *hwc = &event->hw;

	hwc->idx = idx;
	hwc->last_tag = ++box->tags[idx];

	if (hwc->idx == UNCORE_PMC_IDX_FIXED) {
198 199
		hwc->event_base = uncore_fixed_ctr(box);
		hwc->config_base = uncore_fixed_ctl(box);
200 201 202
		return;
	}

203 204
	hwc->config_base = uncore_event_ctl(box, hwc->idx);
	hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
205 206
}

207
void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event)
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
{
	u64 prev_count, new_count, delta;
	int shift;

	if (event->hw.idx >= UNCORE_PMC_IDX_FIXED)
		shift = 64 - uncore_fixed_ctr_bits(box);
	else
		shift = 64 - uncore_perf_ctr_bits(box);

	/* the hrtimer might modify the previous event value */
again:
	prev_count = local64_read(&event->hw.prev_count);
	new_count = uncore_read_counter(box, event);
	if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
		goto again;

	delta = (new_count << shift) - (prev_count << shift);
	delta >>= shift;

	local64_add(delta, &event->count);
}

/*
 * The overflow interrupt is unavailable for SandyBridge-EP, is broken
 * for SandyBridge. So we use hrtimer to periodically poll the counter
 * to avoid overflow.
 */
static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
{
	struct intel_uncore_box *box;
238
	struct perf_event *event;
239 240 241 242 243 244 245 246 247 248 249 250
	unsigned long flags;
	int bit;

	box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
	if (!box->n_active || box->cpu != smp_processor_id())
		return HRTIMER_NORESTART;
	/*
	 * disable local interrupt to prevent uncore_pmu_event_start/stop
	 * to interrupt the update process
	 */
	local_irq_save(flags);

251 252 253 254 255 256 257 258
	/*
	 * handle boxes with an active event list as opposed to active
	 * counters
	 */
	list_for_each_entry(event, &box->active_list, active_entry) {
		uncore_perf_event_update(box, event);
	}

259 260 261 262 263
	for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
		uncore_perf_event_update(box, box->events[bit]);

	local_irq_restore(flags);

264
	hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration));
265 266 267
	return HRTIMER_RESTART;
}

268
void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
269
{
270 271
	hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration),
		      HRTIMER_MODE_REL_PINNED);
272 273
}

274
void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
275 276 277 278 279 280 281 282 283 284
{
	hrtimer_cancel(&box->hrtimer);
}

static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
{
	hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	box->hrtimer.function = uncore_pmu_hrtimer;
}

285 286
static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,
						 int node)
287
{
288
	int i, size, numshared = type->num_shared_regs ;
289 290
	struct intel_uncore_box *box;

291
	size = sizeof(*box) + numshared * sizeof(struct intel_uncore_extra_reg);
292

293
	box = kzalloc_node(size, GFP_KERNEL, node);
294 295 296
	if (!box)
		return NULL;

297
	for (i = 0; i < numshared; i++)
298 299
		raw_spin_lock_init(&box->shared_regs[i].lock);

300 301
	uncore_pmu_init_hrtimer(box);
	box->cpu = -1;
302 303
	box->pci_phys_id = -1;
	box->pkgid = -1;
304

305 306
	/* set default hrtimer timeout */
	box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL;
307

308
	INIT_LIST_HEAD(&box->active_list);
309

310 311 312
	return box;
}

313 314 315 316 317 318 319 320 321 322 323
/*
 * Using uncore_pmu_event_init pmu event_init callback
 * as a detection point for uncore events.
 */
static int uncore_pmu_event_init(struct perf_event *event);

static bool is_uncore_event(struct perf_event *event)
{
	return event->pmu->event_init == uncore_pmu_event_init;
}

324
static int
325 326
uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader,
		      bool dogrp)
327 328 329 330 331 332 333 334 335 336 337 338
{
	struct perf_event *event;
	int n, max_count;

	max_count = box->pmu->type->num_counters;
	if (box->pmu->type->fixed_ctl)
		max_count++;

	if (box->n_events >= max_count)
		return -EINVAL;

	n = box->n_events;
339 340 341 342 343 344

	if (is_uncore_event(leader)) {
		box->event_list[n] = leader;
		n++;
	}

345 346 347 348
	if (!dogrp)
		return n;

	list_for_each_entry(event, &leader->sibling_list, group_entry) {
349 350
		if (!is_uncore_event(event) ||
		    event->state <= PERF_EVENT_STATE_OFF)
351 352 353 354 355 356 357 358 359 360 361 362
			continue;

		if (n >= max_count)
			return -EINVAL;

		box->event_list[n] = event;
		n++;
	}
	return n;
}

static struct event_constraint *
363
uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
364
{
365
	struct intel_uncore_type *type = box->pmu->type;
366 367
	struct event_constraint *c;

368 369 370 371 372 373
	if (type->ops->get_constraint) {
		c = type->ops->get_constraint(box, event);
		if (c)
			return c;
	}

374
	if (event->attr.config == UNCORE_FIXED_EVENT)
375
		return &uncore_constraint_fixed;
376 377 378 379 380 381 382 383 384 385 386

	if (type->constraints) {
		for_each_event_constraint(c, type->constraints) {
			if ((event->hw.config & c->cmask) == c->code)
				return c;
		}
	}

	return &type->unconstrainted;
}

387 388
static void uncore_put_event_constraint(struct intel_uncore_box *box,
					struct perf_event *event)
389 390 391 392 393
{
	if (box->pmu->type->ops->put_constraint)
		box->pmu->type->ops->put_constraint(box, event);
}

394
static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n)
395 396
{
	unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
397
	struct event_constraint *c;
398
	int i, wmin, wmax, ret = 0;
399 400 401 402 403
	struct hw_perf_event *hwc;

	bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);

	for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
404
		c = uncore_get_event_constraint(box, box->event_list[i]);
405
		box->event_constraint[i] = c;
406 407 408 409 410 411 412
		wmin = min(wmin, c->weight);
		wmax = max(wmax, c->weight);
	}

	/* fastpath, try to reuse previous register */
	for (i = 0; i < n; i++) {
		hwc = &box->event_list[i]->hw;
413
		c = box->event_constraint[i];
414 415 416 417 418 419 420 421 422 423 424 425 426 427

		/* never assigned */
		if (hwc->idx == -1)
			break;

		/* constraint still honored */
		if (!test_bit(hwc->idx, c->idxmsk))
			break;

		/* not already used */
		if (test_bit(hwc->idx, used_mask))
			break;

		__set_bit(hwc->idx, used_mask);
428 429
		if (assign)
			assign[i] = hwc->idx;
430 431
	}
	/* slow path */
432
	if (i != n)
433
		ret = perf_assign_events(box->event_constraint, n,
434
					 wmin, wmax, n, assign);
435 436 437 438 439

	if (!assign || ret) {
		for (i = 0; i < n; i++)
			uncore_put_event_constraint(box, box->event_list[i]);
	}
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
	return ret ? -EINVAL : 0;
}

static void uncore_pmu_event_start(struct perf_event *event, int flags)
{
	struct intel_uncore_box *box = uncore_event_to_box(event);
	int idx = event->hw.idx;

	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
		return;

	if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
		return;

	event->hw.state = 0;
	box->events[idx] = event;
	box->n_active++;
	__set_bit(idx, box->active_mask);

	local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
	uncore_enable_event(box, event);

	if (box->n_active == 1) {
		uncore_enable_box(box);
		uncore_pmu_start_hrtimer(box);
	}
}

static void uncore_pmu_event_stop(struct perf_event *event, int flags)
{
	struct intel_uncore_box *box = uncore_event_to_box(event);
	struct hw_perf_event *hwc = &event->hw;

	if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
		uncore_disable_event(box, event);
		box->n_active--;
		box->events[hwc->idx] = NULL;
		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
		hwc->state |= PERF_HES_STOPPED;

		if (box->n_active == 0) {
			uncore_disable_box(box);
			uncore_pmu_cancel_hrtimer(box);
		}
	}

	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
		/*
		 * Drain the remaining delta count out of a event
		 * that we are disabling:
		 */
		uncore_perf_event_update(box, event);
		hwc->state |= PERF_HES_UPTODATE;
	}
}

static int uncore_pmu_event_add(struct perf_event *event, int flags)
{
	struct intel_uncore_box *box = uncore_event_to_box(event);
	struct hw_perf_event *hwc = &event->hw;
	int assign[UNCORE_PMC_IDX_MAX];
	int i, n, ret;

	if (!box)
		return -ENODEV;

	ret = n = uncore_collect_events(box, event, false);
	if (ret < 0)
		return ret;

	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
	if (!(flags & PERF_EF_START))
		hwc->state |= PERF_HES_ARCH;

	ret = uncore_assign_events(box, assign, n);
	if (ret)
		return ret;

	/* save events moving to new counters */
	for (i = 0; i < box->n_events; i++) {
		event = box->event_list[i];
		hwc = &event->hw;

		if (hwc->idx == assign[i] &&
			hwc->last_tag == box->tags[assign[i]])
			continue;
		/*
		 * Ensure we don't accidentally enable a stopped
		 * counter simply because we rescheduled.
		 */
		if (hwc->state & PERF_HES_STOPPED)
			hwc->state |= PERF_HES_ARCH;

		uncore_pmu_event_stop(event, PERF_EF_UPDATE);
	}

	/* reprogram moved events into new counters */
	for (i = 0; i < n; i++) {
		event = box->event_list[i];
		hwc = &event->hw;

		if (hwc->idx != assign[i] ||
			hwc->last_tag != box->tags[assign[i]])
			uncore_assign_hw_event(box, event, assign[i]);
		else if (i < box->n_events)
			continue;

		if (hwc->state & PERF_HES_ARCH)
			continue;

		uncore_pmu_event_start(event, 0);
	}
	box->n_events = n;

	return 0;
}

static void uncore_pmu_event_del(struct perf_event *event, int flags)
{
	struct intel_uncore_box *box = uncore_event_to_box(event);
	int i;

	uncore_pmu_event_stop(event, PERF_EF_UPDATE);

	for (i = 0; i < box->n_events; i++) {
		if (event == box->event_list[i]) {
566 567
			uncore_put_event_constraint(box, event);

568
			for (++i; i < box->n_events; i++)
569 570 571 572 573 574 575 576 577 578 579
				box->event_list[i - 1] = box->event_list[i];

			--box->n_events;
			break;
		}
	}

	event->hw.idx = -1;
	event->hw.last_tag = ~0ULL;
}

580
void uncore_pmu_event_read(struct perf_event *event)
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
{
	struct intel_uncore_box *box = uncore_event_to_box(event);
	uncore_perf_event_update(box, event);
}

/*
 * validation ensures the group can be loaded onto the
 * PMU if it was the only group available.
 */
static int uncore_validate_group(struct intel_uncore_pmu *pmu,
				struct perf_event *event)
{
	struct perf_event *leader = event->group_leader;
	struct intel_uncore_box *fake_box;
	int ret = -EINVAL, n;

597
	fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE);
598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
	if (!fake_box)
		return -ENOMEM;

	fake_box->pmu = pmu;
	/*
	 * the event is not yet connected with its
	 * siblings therefore we must first collect
	 * existing siblings, then add the new event
	 * before we can simulate the scheduling
	 */
	n = uncore_collect_events(fake_box, leader, true);
	if (n < 0)
		goto out;

	fake_box->n_events = n;
	n = uncore_collect_events(fake_box, event, false);
	if (n < 0)
		goto out;

	fake_box->n_events = n;

619
	ret = uncore_assign_events(fake_box, NULL, n);
620 621 622 623 624
out:
	kfree(fake_box);
	return ret;
}

625
static int uncore_pmu_event_init(struct perf_event *event)
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661
{
	struct intel_uncore_pmu *pmu;
	struct intel_uncore_box *box;
	struct hw_perf_event *hwc = &event->hw;
	int ret;

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	pmu = uncore_event_to_pmu(event);
	/* no device found for this pmu */
	if (pmu->func_id < 0)
		return -ENOENT;

	/*
	 * Uncore PMU does measure at all privilege level all the time.
	 * So it doesn't make sense to specify any exclude bits.
	 */
	if (event->attr.exclude_user || event->attr.exclude_kernel ||
			event->attr.exclude_hv || event->attr.exclude_idle)
		return -EINVAL;

	/* Sampling not supported yet */
	if (hwc->sample_period)
		return -EINVAL;

	/*
	 * Place all uncore events for a particular physical package
	 * onto a single cpu
	 */
	if (event->cpu < 0)
		return -EINVAL;
	box = uncore_pmu_to_box(pmu, event->cpu);
	if (!box || box->cpu < 0)
		return -EINVAL;
	event->cpu = box->cpu;
662
	event->pmu_private = box;
663

664 665 666
	event->hw.idx = -1;
	event->hw.last_tag = ~0ULL;
	event->hw.extra_reg.idx = EXTRA_REG_NONE;
667
	event->hw.branch_reg.idx = EXTRA_REG_NONE;
668

669 670 671 672 673 674 675 676 677 678
	if (event->attr.config == UNCORE_FIXED_EVENT) {
		/* no fixed counter */
		if (!pmu->type->fixed_ctl)
			return -EINVAL;
		/*
		 * if there is only one fixed counter, only the first pmu
		 * can access the fixed counter
		 */
		if (pmu->type->single_fixed && pmu->pmu_idx > 0)
			return -EINVAL;
679 680 681

		/* fixed counters have event field hardcoded to zero */
		hwc->config = 0ULL;
682 683
	} else {
		hwc->config = event->attr.config & pmu->type->event_mask;
684 685 686 687 688
		if (pmu->type->ops->hw_config) {
			ret = pmu->type->ops->hw_config(box, event);
			if (ret)
				return ret;
		}
689 690 691 692 693 694 695 696 697 698
	}

	if (event->group_leader != event)
		ret = uncore_validate_group(pmu, event);
	else
		ret = 0;

	return ret;
}

699 700 701
static ssize_t uncore_get_attr_cpumask(struct device *dev,
				struct device_attribute *attr, char *buf)
{
702
	return cpumap_print_to_pagebuf(true, buf, &uncore_cpu_mask);
703 704 705 706 707 708 709 710 711 712 713 714 715
}

static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);

static struct attribute *uncore_pmu_attrs[] = {
	&dev_attr_cpumask.attr,
	NULL,
};

static struct attribute_group uncore_pmu_attr_group = {
	.attrs = uncore_pmu_attrs,
};

716
static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
717 718 719
{
	int ret;

720 721 722 723 724 725 726 727 728 729 730 731 732 733 734
	if (!pmu->type->pmu) {
		pmu->pmu = (struct pmu) {
			.attr_groups	= pmu->type->attr_groups,
			.task_ctx_nr	= perf_invalid_context,
			.event_init	= uncore_pmu_event_init,
			.add		= uncore_pmu_event_add,
			.del		= uncore_pmu_event_del,
			.start		= uncore_pmu_event_start,
			.stop		= uncore_pmu_event_stop,
			.read		= uncore_pmu_event_read,
		};
	} else {
		pmu->pmu = *pmu->type->pmu;
		pmu->pmu.attr_groups = pmu->type->attr_groups;
	}
735 736 737 738 739 740 741 742 743 744 745 746

	if (pmu->type->num_boxes == 1) {
		if (strlen(pmu->type->name) > 0)
			sprintf(pmu->name, "uncore_%s", pmu->type->name);
		else
			sprintf(pmu->name, "uncore");
	} else {
		sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
			pmu->pmu_idx);
	}

	ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
747 748
	if (!ret)
		pmu->registered = true;
749 750 751
	return ret;
}

752 753 754 755 756 757 758 759
static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu)
{
	if (!pmu->registered)
		return;
	perf_pmu_unregister(&pmu->pmu);
	pmu->registered = false;
}

760
static void __uncore_exit_boxes(struct intel_uncore_type *type, int cpu)
761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
{
	struct intel_uncore_pmu *pmu = type->pmus;
	struct intel_uncore_box *box;
	int i, pkg;

	if (pmu) {
		pkg = topology_physical_package_id(cpu);
		for (i = 0; i < type->num_boxes; i++, pmu++) {
			box = pmu->boxes[pkg];
			if (box)
				uncore_box_exit(box);
		}
	}
}

776
static void uncore_exit_boxes(void *dummy)
777 778 779 780 781 782 783
{
	struct intel_uncore_type **types;

	for (types = uncore_msr_uncores; *types; types++)
		__uncore_exit_boxes(*types++, smp_processor_id());
}

784 785 786 787 788 789 790 791 792
static void uncore_free_boxes(struct intel_uncore_pmu *pmu)
{
	int pkg;

	for (pkg = 0; pkg < max_packages; pkg++)
		kfree(pmu->boxes[pkg]);
	kfree(pmu->boxes);
}

793
static void uncore_type_exit(struct intel_uncore_type *type)
794
{
795
	struct intel_uncore_pmu *pmu = type->pmus;
796 797
	int i;

798 799 800 801
	if (pmu) {
		for (i = 0; i < type->num_boxes; i++, pmu++) {
			uncore_pmu_unregister(pmu);
			uncore_free_boxes(pmu);
802
		}
803 804 805
		kfree(type->pmus);
		type->pmus = NULL;
	}
806 807
	kfree(type->events_group);
	type->events_group = NULL;
808 809
}

810
static void uncore_types_exit(struct intel_uncore_type **types)
811
{
812 813
	for (; *types; types++)
		uncore_type_exit(*types);
814 815
}

816
static int __init uncore_type_init(struct intel_uncore_type *type, bool setid)
817 818
{
	struct intel_uncore_pmu *pmus;
819
	struct attribute_group *attr_group;
820
	struct attribute **attrs;
821
	size_t size;
822 823 824 825 826 827
	int i, j;

	pmus = kzalloc(sizeof(*pmus) * type->num_boxes, GFP_KERNEL);
	if (!pmus)
		return -ENOMEM;

828
	size = max_packages * sizeof(struct intel_uncore_box *);
829 830

	for (i = 0; i < type->num_boxes; i++) {
831 832 833 834 835
		pmus[i].func_id	= setid ? i : -1;
		pmus[i].pmu_idx	= i;
		pmus[i].type	= type;
		pmus[i].boxes	= kzalloc(size, GFP_KERNEL);
		if (!pmus[i].boxes)
836
			return -ENOMEM;
837 838
	}

839 840 841 842 843
	type->pmus = pmus;
	type->unconstrainted = (struct event_constraint)
		__EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
				0, type->num_counters, 0, 0);

844
	if (type->event_descs) {
845
		for (i = 0; type->event_descs[i].attr.attr.name; i++);
846

847 848 849
		attr_group = kzalloc(sizeof(struct attribute *) * (i + 1) +
					sizeof(*attr_group), GFP_KERNEL);
		if (!attr_group)
850
			return -ENOMEM;
851

852 853 854
		attrs = (struct attribute **)(attr_group + 1);
		attr_group->name = "events";
		attr_group->attrs = attrs;
855 856 857 858

		for (j = 0; j < i; j++)
			attrs[j] = &type->event_descs[j].attr.attr;

859
		type->events_group = attr_group;
860 861
	}

862
	type->pmu_group = &uncore_pmu_attr_group;
863 864 865
	return 0;
}

866 867
static int __init
uncore_types_init(struct intel_uncore_type **types, bool setid)
868
{
869
	int ret;
870

871 872
	for (; *types; types++) {
		ret = uncore_type_init(*types, setid);
873
		if (ret)
874
			return ret;
875 876 877 878
	}
	return 0;
}

879 880 881
/*
 * add a pci uncore device
 */
882
static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
883
{
884
	struct intel_uncore_type *type;
885 886
	struct intel_uncore_pmu *pmu;
	struct intel_uncore_box *box;
887
	int phys_id, pkg, ret;
888

889
	phys_id = uncore_pcibus_to_physid(pdev->bus);
890
	if (phys_id < 0)
891 892
		return -ENODEV;

893
	pkg = topology_phys_to_logical_pkg(phys_id);
894
	if (pkg < 0)
895 896
		return -EINVAL;

897
	if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) {
898
		int idx = UNCORE_PCI_DEV_IDX(id->driver_data);
899 900

		uncore_extra_pci_dev[pkg].dev[idx] = pdev;
901 902 903 904
		pci_set_drvdata(pdev, NULL);
		return 0;
	}

905
	type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
906 907 908 909
	/*
	 * for performance monitoring unit with multiple boxes,
	 * each box has a different function id.
	 */
910
	pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)];
911 912 913 914 915 916
	/* Knights Landing uses a common PCI device ID for multiple instances of
	 * an uncore PMU device type. There is only one entry per device type in
	 * the knl_uncore_pci_ids table inspite of multiple devices present for
	 * some device types. Hence PCI device idx would be 0 for all devices.
	 * So increment pmu pointer to point to an unused array element.
	 */
917
	if (boot_cpu_data.x86_model == 87) {
918 919
		while (pmu->func_id >= 0)
			pmu++;
920 921
	}

922 923 924 925 926 927 928
	if (WARN_ON_ONCE(pmu->boxes[pkg] != NULL))
		return -EINVAL;

	box = uncore_alloc_box(type, NUMA_NO_NODE);
	if (!box)
		return -ENOMEM;

929 930 931 932
	if (pmu->func_id < 0)
		pmu->func_id = pdev->devfn;
	else
		WARN_ON_ONCE(pmu->func_id != pdev->devfn);
933

934 935 936
	atomic_inc(&box->refcnt);
	box->pci_phys_id = phys_id;
	box->pkgid = pkg;
937 938
	box->pci_dev = pdev;
	box->pmu = pmu;
939
	uncore_box_init(box);
940 941
	pci_set_drvdata(pdev, box);

942 943
	pmu->boxes[pkg] = box;
	if (atomic_inc_return(&pmu->activeboxes) > 1)
944 945
		return 0;

946
	/* First active box registers the pmu */
947 948 949
	ret = uncore_pmu_register(pmu);
	if (ret) {
		pci_set_drvdata(pdev, NULL);
950
		pmu->boxes[pkg] = NULL;
951
		uncore_box_exit(box);
952 953 954
		kfree(box);
	}
	return ret;
955 956
}

957
static void uncore_pci_remove(struct pci_dev *pdev)
958 959
{
	struct intel_uncore_box *box = pci_get_drvdata(pdev);
960
	struct intel_uncore_pmu *pmu;
961
	int i, phys_id, pkg;
962

963
	phys_id = uncore_pcibus_to_physid(pdev->bus);
964 965
	pkg = topology_phys_to_logical_pkg(phys_id);

966 967 968
	box = pci_get_drvdata(pdev);
	if (!box) {
		for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) {
969 970
			if (uncore_extra_pci_dev[pkg].dev[i] == pdev) {
				uncore_extra_pci_dev[pkg].dev[i] = NULL;
971 972 973 974 975 976
				break;
			}
		}
		WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX);
		return;
	}
977

978
	pmu = box->pmu;
979
	if (WARN_ON_ONCE(phys_id != box->pci_phys_id))
980 981
		return;

982
	pci_set_drvdata(pdev, NULL);
983 984 985
	pmu->boxes[pkg] = NULL;
	if (atomic_dec_return(&pmu->activeboxes) == 0)
		uncore_pmu_unregister(pmu);
986
	uncore_box_exit(box);
987 988 989 990 991
	kfree(box);
}

static int __init uncore_pci_init(void)
{
992
	size_t size;
993 994
	int ret;

995 996 997 998
	size = max_packages * sizeof(struct pci_extra_dev);
	uncore_extra_pci_dev = kzalloc(size, GFP_KERNEL);
	if (!uncore_extra_pci_dev) {
		ret = -ENOMEM;
999
		goto err;
1000 1001 1002 1003 1004
	}

	ret = uncore_types_init(uncore_pci_uncores, false);
	if (ret)
		goto errtype;
1005 1006 1007 1008 1009

	uncore_pci_driver->probe = uncore_pci_probe;
	uncore_pci_driver->remove = uncore_pci_remove;

	ret = pci_register_driver(uncore_pci_driver);
1010
	if (ret)
1011
		goto errtype;
1012 1013 1014

	pcidrv_registered = true;
	return 0;
1015

1016
errtype:
1017
	uncore_types_exit(uncore_pci_uncores);
1018 1019
	kfree(uncore_extra_pci_dev);
	uncore_extra_pci_dev = NULL;
1020
	uncore_free_pcibus_map();
1021 1022
err:
	uncore_pci_uncores = empty_uncore;
1023 1024 1025
	return ret;
}

1026
static void uncore_pci_exit(void)
1027 1028 1029 1030
{
	if (pcidrv_registered) {
		pcidrv_registered = false;
		pci_unregister_driver(uncore_pci_driver);
1031
		uncore_types_exit(uncore_pci_uncores);
1032
		kfree(uncore_extra_pci_dev);
1033
		uncore_free_pcibus_map();
1034 1035 1036
	}
}

1037
static void uncore_cpu_dying(int cpu)
1038
{
1039
	struct intel_uncore_type *type, **types = uncore_msr_uncores;
1040 1041
	struct intel_uncore_pmu *pmu;
	struct intel_uncore_box *box;
1042
	int i, pkg;
1043

1044 1045 1046 1047 1048 1049 1050
	pkg = topology_logical_package_id(cpu);
	for (; *types; types++) {
		type = *types;
		pmu = type->pmus;
		for (i = 0; i < type->num_boxes; i++, pmu++) {
			box = pmu->boxes[pkg];
			if (box && atomic_dec_return(&box->refcnt) == 0)
1051
				uncore_box_exit(box);
1052 1053 1054 1055
		}
	}
}

1056
static void uncore_cpu_starting(int cpu, bool init)
1057
{
1058
	struct intel_uncore_type *type, **types = uncore_msr_uncores;
1059
	struct intel_uncore_pmu *pmu;
1060 1061
	struct intel_uncore_box *box;
	int i, pkg, ncpus = 1;
1062

1063 1064 1065 1066 1067 1068 1069
	if (init) {
		/*
		 * On init we get the number of online cpus in the package
		 * and set refcount for all of them.
		 */
		ncpus = cpumask_weight(topology_core_cpumask(cpu));
	}
1070

1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
	pkg = topology_logical_package_id(cpu);
	for (; *types; types++) {
		type = *types;
		pmu = type->pmus;
		for (i = 0; i < type->num_boxes; i++, pmu++) {
			box = pmu->boxes[pkg];
			if (!box)
				continue;
			/* The first cpu on a package activates the box */
			if (atomic_add_return(ncpus, &box->refcnt) == ncpus)
1081
				uncore_box_init(box);
1082 1083 1084 1085
		}
	}
}

1086
static int uncore_cpu_prepare(int cpu)
1087
{
1088
	struct intel_uncore_type *type, **types = uncore_msr_uncores;
1089 1090
	struct intel_uncore_pmu *pmu;
	struct intel_uncore_box *box;
1091
	int i, pkg;
1092

1093 1094 1095 1096 1097 1098 1099 1100
	pkg = topology_logical_package_id(cpu);
	for (; *types; types++) {
		type = *types;
		pmu = type->pmus;
		for (i = 0; i < type->num_boxes; i++, pmu++) {
			if (pmu->boxes[pkg])
				continue;
			/* First cpu of a package allocates the box */
1101
			box = uncore_alloc_box(type, cpu_to_node(cpu));
1102 1103 1104
			if (!box)
				return -ENOMEM;
			box->pmu = pmu;
1105 1106
			box->pkgid = pkg;
			pmu->boxes[pkg] = box;
1107 1108 1109 1110 1111
		}
	}
	return 0;
}

1112 1113
static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu,
				   int new_cpu)
1114
{
1115
	struct intel_uncore_pmu *pmu = type->pmus;
1116
	struct intel_uncore_box *box;
1117
	int i, pkg;
1118

1119
	pkg = topology_logical_package_id(old_cpu < 0 ? new_cpu : old_cpu);
1120
	for (i = 0; i < type->num_boxes; i++, pmu++) {
1121
		box = pmu->boxes[pkg];
1122 1123
		if (!box)
			continue;
1124

1125 1126 1127 1128
		if (old_cpu < 0) {
			WARN_ON_ONCE(box->cpu != -1);
			box->cpu = new_cpu;
			continue;
1129
		}
1130 1131 1132 1133 1134 1135 1136 1137 1138

		WARN_ON_ONCE(box->cpu != old_cpu);
		box->cpu = -1;
		if (new_cpu < 0)
			continue;

		uncore_pmu_cancel_hrtimer(box);
		perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu);
		box->cpu = new_cpu;
1139 1140 1141
	}
}

1142 1143 1144 1145 1146 1147 1148
static void uncore_change_context(struct intel_uncore_type **uncores,
				  int old_cpu, int new_cpu)
{
	for (; *uncores; uncores++)
		uncore_change_type_ctx(*uncores, old_cpu, new_cpu);
}

1149
static void uncore_event_exit_cpu(int cpu)
1150
{
1151
	int target;
1152

1153
	/* Check if exiting cpu is used for collecting uncore events */
1154 1155 1156
	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
		return;

1157 1158
	/* Find a new cpu to collect uncore events */
	target = cpumask_any_but(topology_core_cpumask(cpu), cpu);
1159

1160 1161
	/* Migrate uncore events to the new target */
	if (target < nr_cpu_ids)
1162
		cpumask_set_cpu(target, &uncore_cpu_mask);
1163 1164
	else
		target = -1;
1165

1166 1167
	uncore_change_context(uncore_msr_uncores, cpu, target);
	uncore_change_context(uncore_pci_uncores, cpu, target);
1168 1169
}

1170
static void uncore_event_init_cpu(int cpu)
1171
{
1172
	int target;
1173

1174 1175 1176 1177 1178 1179 1180
	/*
	 * Check if there is an online cpu in the package
	 * which collects uncore events already.
	 */
	target = cpumask_any_and(&uncore_cpu_mask, topology_core_cpumask(cpu));
	if (target < nr_cpu_ids)
		return;
1181 1182 1183

	cpumask_set_cpu(cpu, &uncore_cpu_mask);

1184 1185
	uncore_change_context(uncore_msr_uncores, -1, cpu);
	uncore_change_context(uncore_pci_uncores, -1, cpu);
1186 1187
}

1188 1189
static int uncore_cpu_notifier(struct notifier_block *self,
			       unsigned long action, void *hcpu)
1190 1191 1192 1193 1194
{
	unsigned int cpu = (long)hcpu;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_UP_PREPARE:
1195 1196
		return notifier_from_errno(uncore_cpu_prepare(cpu));

1197
	case CPU_STARTING:
1198 1199 1200
		uncore_cpu_starting(cpu, false);
	case CPU_DOWN_FAILED:
		uncore_event_init_cpu(cpu);
1201
		break;
1202

1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
	case CPU_UP_CANCELED:
	case CPU_DYING:
		uncore_cpu_dying(cpu);
		break;

	case CPU_DOWN_PREPARE:
		uncore_event_exit_cpu(cpu);
		break;
	}
	return NOTIFY_OK;
}

1215
static struct notifier_block uncore_cpu_nb = {
1216
	.notifier_call	= uncore_cpu_notifier,
1217 1218 1219 1220
	/*
	 * to migrate uncore events, our notifier should be executed
	 * before perf core's notifier.
	 */
1221
	.priority	= CPU_PRI_PERF + 1,
1222 1223
};

1224
static int __init type_pmu_register(struct intel_uncore_type *type)
1225
{
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
	int i, ret;

	for (i = 0; i < type->num_boxes; i++) {
		ret = uncore_pmu_register(&type->pmus[i]);
		if (ret)
			return ret;
	}
	return 0;
}

static int __init uncore_msr_pmus_register(void)
{
	struct intel_uncore_type **types = uncore_msr_uncores;
	int ret;

1241 1242
	for (; *types; types++) {
		ret = type_pmu_register(*types);
1243 1244 1245 1246
		if (ret)
			return ret;
	}
	return 0;
1247 1248 1249 1250
}

static int __init uncore_cpu_init(void)
{
1251
	int ret;
1252

1253
	ret = uncore_types_init(uncore_msr_uncores, true);
1254 1255 1256 1257
	if (ret)
		goto err;

	ret = uncore_msr_pmus_register();
1258
	if (ret)
1259
		goto err;
1260
	return 0;
1261 1262 1263 1264
err:
	uncore_types_exit(uncore_msr_uncores);
	uncore_msr_uncores = empty_uncore;
	return ret;
1265 1266
}

1267
static void __init uncore_cpu_setup(void *dummy)
1268
{
1269
	uncore_cpu_starting(smp_processor_id(), true);
1270 1271
}

1272 1273 1274
/* Lazy to avoid allocation of a few bytes for the normal case */
static __initdata DECLARE_BITMAP(packages, MAX_LOCAL_APIC);

1275
static int __init uncore_cpumask_init(bool msr)
1276
{
1277
	unsigned int cpu;
1278 1279

	for_each_online_cpu(cpu) {
1280 1281
		unsigned int pkg = topology_logical_package_id(cpu);
		int ret;
1282

1283
		if (test_and_set_bit(pkg, packages))
1284
			continue;
1285
		/*
1286 1287 1288
		 * The first online cpu of each package allocates and takes
		 * the refcounts for all other online cpus in that package.
		 * If msrs are not enabled no allocation is required.
1289
		 */
1290 1291 1292 1293 1294
		if (msr) {
			ret = uncore_cpu_prepare(cpu);
			if (ret)
				return ret;
		}
1295
		uncore_event_init_cpu(cpu);
1296
		smp_call_function_single(cpu, uncore_cpu_setup, NULL, 1);
1297
	}
1298
	__register_cpu_notifier(&uncore_cpu_nb);
1299
	return 0;
1300 1301
}

1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
#define X86_UNCORE_MODEL_MATCH(model, init)	\
	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&init }

struct intel_uncore_init_fun {
	void	(*cpu_init)(void);
	int	(*pci_init)(void);
};

static const struct intel_uncore_init_fun nhm_uncore_init __initconst = {
	.cpu_init = nhm_uncore_cpu_init,
};

static const struct intel_uncore_init_fun snb_uncore_init __initconst = {
	.cpu_init = snb_uncore_cpu_init,
	.pci_init = snb_uncore_pci_init,
};

static const struct intel_uncore_init_fun ivb_uncore_init __initconst = {
	.cpu_init = snb_uncore_cpu_init,
	.pci_init = ivb_uncore_pci_init,
};

static const struct intel_uncore_init_fun hsw_uncore_init __initconst = {
	.cpu_init = snb_uncore_cpu_init,
	.pci_init = hsw_uncore_pci_init,
};

static const struct intel_uncore_init_fun bdw_uncore_init __initconst = {
	.cpu_init = snb_uncore_cpu_init,
	.pci_init = bdw_uncore_pci_init,
};

static const struct intel_uncore_init_fun snbep_uncore_init __initconst = {
	.cpu_init = snbep_uncore_cpu_init,
	.pci_init = snbep_uncore_pci_init,
};

static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = {
	.cpu_init = nhmex_uncore_cpu_init,
};

static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = {
	.cpu_init = ivbep_uncore_cpu_init,
	.pci_init = ivbep_uncore_pci_init,
};

static const struct intel_uncore_init_fun hswep_uncore_init __initconst = {
	.cpu_init = hswep_uncore_cpu_init,
	.pci_init = hswep_uncore_pci_init,
};

static const struct intel_uncore_init_fun bdx_uncore_init __initconst = {
	.cpu_init = bdx_uncore_cpu_init,
	.pci_init = bdx_uncore_pci_init,
};

static const struct intel_uncore_init_fun knl_uncore_init __initconst = {
	.cpu_init = knl_uncore_cpu_init,
	.pci_init = knl_uncore_pci_init,
};

static const struct intel_uncore_init_fun skl_uncore_init __initconst = {
	.pci_init = skl_uncore_pci_init,
};

static const struct x86_cpu_id intel_uncore_match[] __initconst = {
	X86_UNCORE_MODEL_MATCH(26, nhm_uncore_init),	/* Nehalem */
	X86_UNCORE_MODEL_MATCH(30, nhm_uncore_init),
	X86_UNCORE_MODEL_MATCH(37, nhm_uncore_init),	/* Westmere */
	X86_UNCORE_MODEL_MATCH(44, nhm_uncore_init),
	X86_UNCORE_MODEL_MATCH(42, snb_uncore_init),	/* Sandy Bridge */
	X86_UNCORE_MODEL_MATCH(58, ivb_uncore_init),	/* Ivy Bridge */
	X86_UNCORE_MODEL_MATCH(60, hsw_uncore_init),	/* Haswell */
	X86_UNCORE_MODEL_MATCH(69, hsw_uncore_init),	/* Haswell Celeron */
	X86_UNCORE_MODEL_MATCH(70, hsw_uncore_init),	/* Haswell */
	X86_UNCORE_MODEL_MATCH(61, bdw_uncore_init),	/* Broadwell */
	X86_UNCORE_MODEL_MATCH(71, bdw_uncore_init),	/* Broadwell */
	X86_UNCORE_MODEL_MATCH(45, snbep_uncore_init),	/* Sandy Bridge-EP */
	X86_UNCORE_MODEL_MATCH(46, nhmex_uncore_init),	/* Nehalem-EX */
	X86_UNCORE_MODEL_MATCH(47, nhmex_uncore_init),	/* Westmere-EX aka. Xeon E7 */
	X86_UNCORE_MODEL_MATCH(62, ivbep_uncore_init),	/* Ivy Bridge-EP */
	X86_UNCORE_MODEL_MATCH(63, hswep_uncore_init),	/* Haswell-EP */
	X86_UNCORE_MODEL_MATCH(79, bdx_uncore_init),	/* BDX-EP */
	X86_UNCORE_MODEL_MATCH(86, bdx_uncore_init),	/* BDX-DE */
	X86_UNCORE_MODEL_MATCH(87, knl_uncore_init),	/* Knights Landing */
	X86_UNCORE_MODEL_MATCH(94, skl_uncore_init),	/* SkyLake */
	{},
};

MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match);

1393 1394
static int __init intel_uncore_init(void)
{
1395 1396 1397
	const struct x86_cpu_id *id;
	struct intel_uncore_init_fun *uncore_init;
	int pret = 0, cret = 0, ret;
1398

1399 1400
	id = x86_match_cpu(intel_uncore_match);
	if (!id)
1401 1402
		return -ENODEV;

1403
	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1404 1405
		return -ENODEV;

1406 1407
	max_packages = topology_max_packages();

1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418
	uncore_init = (struct intel_uncore_init_fun *)id->driver_data;
	if (uncore_init->pci_init) {
		pret = uncore_init->pci_init();
		if (!pret)
			pret = uncore_pci_init();
	}

	if (uncore_init->cpu_init) {
		uncore_init->cpu_init();
		cret = uncore_cpu_init();
	}
1419 1420 1421

	if (cret && pret)
		return -ENODEV;
1422 1423

	cpu_notifier_register_begin();
1424
	ret = uncore_cpumask_init(!cret);
1425
	if (ret)
1426 1427
		goto err;
	cpu_notifier_register_done();
1428
	return 0;
1429

1430
err:
1431 1432
	/* Undo box->init_box() */
	on_each_cpu_mask(&uncore_cpu_mask, uncore_exit_boxes, NULL, 1);
1433 1434
	uncore_types_exit(uncore_msr_uncores);
	uncore_pci_exit();
1435
	cpu_notifier_register_done();
1436 1437
	return ret;
}
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448
module_init(intel_uncore_init);

static void __exit intel_uncore_exit(void)
{
	cpu_notifier_register_begin();
	__unregister_cpu_notifier(&uncore_cpu_nb);
	uncore_types_exit(uncore_msr_uncores);
	uncore_pci_exit();
	cpu_notifier_register_done();
}
module_exit(intel_uncore_exit);