amd.c 21.9 KB
Newer Older
1
#include <linux/export.h>
L
Linus Torvalds 已提交
2
#include <linux/bitops.h>
3
#include <linux/elf.h>
L
Linus Torvalds 已提交
4
#include <linux/mm.h>
Y
Yinghai Lu 已提交
5

A
Alan Cox 已提交
6
#include <linux/io.h>
7
#include <linux/sched.h>
L
Linus Torvalds 已提交
8
#include <asm/processor.h>
9
#include <asm/apic.h>
10
#include <asm/cpu.h>
B
Borislav Petkov 已提交
11
#include <asm/smp.h>
12
#include <asm/pci-direct.h>
L
Linus Torvalds 已提交
13

Y
Yinghai Lu 已提交
14 15 16 17 18
#ifdef CONFIG_X86_64
# include <asm/mmconfig.h>
# include <asm/cacheflush.h>
#endif

L
Linus Torvalds 已提交
19 20
#include "cpu.h"

21 22 23 24 25
static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
{
	u32 gprs[8] = { 0 };
	int err;

26 27
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42

	gprs[1] = msr;
	gprs[7] = 0x9c5a203a;

	err = rdmsr_safe_regs(gprs);

	*p = gprs[0] | ((u64)gprs[2] << 32);

	return err;
}

static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
{
	u32 gprs[8] = { 0 };

43 44
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
45 46 47 48 49 50 51 52 53

	gprs[0] = (u32)val;
	gprs[1] = msr;
	gprs[2] = val >> 32;
	gprs[7] = 0x9c5a203a;

	return wrmsr_safe_regs(gprs);
}

L
Linus Torvalds 已提交
54 55 56 57 58 59
/*
 *	B step AMD K6 before B 9730xxxx have hardware bugs that can cause
 *	misexecution of code under Linux. Owners of such processors should
 *	contact AMD for precise details and a CPU swap.
 *
 *	See	http://www.multimania.com/poulot/k6bug.html
60 61
 *	and	section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6"
 *		(Publication # 21266  Issue Date: August 1998)
L
Linus Torvalds 已提交
62 63 64 65 66
 *
 *	The following test is erm.. interesting. AMD neglected to up
 *	the chip setting when fixing the bug but they also tweaked some
 *	performance at the same time..
 */
67

68 69
extern __visible void vide(void);
__asm__(".globl vide\n\t.align 4\nvide: ret");
L
Linus Torvalds 已提交
70

71
static void init_amd_k5(struct cpuinfo_x86 *c)
72
{
B
Borislav Petkov 已提交
73
#ifdef CONFIG_X86_32
74 75 76 77 78 79 80 81 82 83
/*
 * General Systems BIOSen alias the cpu frequency registers
 * of the Elan at 0x000df000. Unfortuantly, one of the Linux
 * drivers subsequently pokes it, and changes the CPU speed.
 * Workaround : Remove the unneeded alias.
 */
#define CBAR		(0xfffc) /* Configuration Base Address  (32-bit) */
#define CBAR_ENB	(0x80000000)
#define CBAR_KEY	(0X000000CB)
	if (c->x86_model == 9 || c->x86_model == 10) {
A
Alan Cox 已提交
84 85
		if (inl(CBAR) & CBAR_ENB)
			outl(0 | CBAR_KEY, CBAR);
86
	}
B
Borislav Petkov 已提交
87
#endif
88 89
}

90
static void init_amd_k6(struct cpuinfo_x86 *c)
91
{
B
Borislav Petkov 已提交
92
#ifdef CONFIG_X86_32
93
	u32 l, h;
94
	int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126

	if (c->x86_model < 6) {
		/* Based on AMD doc 20734R - June 2000 */
		if (c->x86_model == 0) {
			clear_cpu_cap(c, X86_FEATURE_APIC);
			set_cpu_cap(c, X86_FEATURE_PGE);
		}
		return;
	}

	if (c->x86_model == 6 && c->x86_mask == 1) {
		const int K6_BUG_LOOP = 1000000;
		int n;
		void (*f_vide)(void);
		unsigned long d, d2;

		printk(KERN_INFO "AMD K6 stepping B detected - ");

		/*
		 * It looks like AMD fixed the 2.6.2 bug and improved indirect
		 * calls at the same time.
		 */

		n = K6_BUG_LOOP;
		f_vide = vide;
		rdtscl(d);
		while (n--)
			f_vide();
		rdtscl(d2);
		d = d2-d;

		if (d > 20*K6_BUG_LOOP)
A
Alan Cox 已提交
127 128
			printk(KERN_CONT
				"system stability may be impaired when more than 32 MB are used.\n");
129
		else
A
Alan Cox 已提交
130
			printk(KERN_CONT "probably OK (after B9730xxxx).\n");
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
	}

	/* K6 with old style WHCR */
	if (c->x86_model < 8 ||
	   (c->x86_model == 8 && c->x86_mask < 8)) {
		/* We can only write allocate on the low 508Mb */
		if (mbytes > 508)
			mbytes = 508;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0x0000FFFF) == 0) {
			unsigned long flags;
			l = (1<<0)|((mbytes/4)<<1);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling old style K6 write allocation for %d Mb\n",
				mbytes);
		}
		return;
	}

	if ((c->x86_model == 8 && c->x86_mask > 7) ||
	     c->x86_model == 9 || c->x86_model == 13) {
		/* The more serious chips .. */

		if (mbytes > 4092)
			mbytes = 4092;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0xFFFF0000) == 0) {
			unsigned long flags;
			l = ((mbytes>>2)<<22)|(1<<16);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling new style K6 write allocation for %d Mb\n",
				mbytes);
		}

		return;
	}

	if (c->x86_model == 10) {
		/* AMD Geode LX is model 10 */
		/* placeholder for any needed mods */
		return;
	}
B
Borislav Petkov 已提交
181
#endif
182 183
}

B
Borislav Petkov 已提交
184
static void init_amd_k7(struct cpuinfo_x86 *c)
185
{
B
Borislav Petkov 已提交
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
#ifdef CONFIG_X86_32
	u32 l, h;

	/*
	 * Bit 15 of Athlon specific MSR 15, needs to be 0
	 * to enable SSE on Palomino/Morgan/Barton CPU's.
	 * If the BIOS didn't enable it already, enable it here.
	 */
	if (c->x86_model >= 6 && c->x86_model <= 10) {
		if (!cpu_has(c, X86_FEATURE_XMM)) {
			printk(KERN_INFO "Enabling disabled K7/SSE Support.\n");
			msr_clear_bit(MSR_K7_HWCR, 15);
			set_cpu_cap(c, X86_FEATURE_XMM);
		}
	}

	/*
	 * It's been determined by AMD that Athlons since model 8 stepping 1
	 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
	 * As per AMD technical note 27212 0.2
	 */
	if ((c->x86_model == 8 && c->x86_mask >= 1) || (c->x86_model > 8)) {
		rdmsr(MSR_K7_CLK_CTL, l, h);
		if ((l & 0xfff00000) != 0x20000000) {
			printk(KERN_INFO
			    "CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
					l, ((l & 0x000fffff)|0x20000000));
			wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
		}
	}

	set_cpu_cap(c, X86_FEATURE_K7);

219
	/* calling is from identify_secondary_cpu() ? */
220
	if (!c->cpu_index)
221 222 223 224 225 226 227 228 229
		return;

	/*
	 * Certain Athlons might work (for various values of 'work') in SMP
	 * but they are not certified as MP capable.
	 */
	/* Athlon 660/661 is valid. */
	if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
	    (c->x86_mask == 1)))
230
		return;
231 232 233

	/* Duron 670 is valid */
	if ((c->x86_model == 7) && (c->x86_mask == 0))
234
		return;
235 236 237 238 239 240 241 242 243 244 245

	/*
	 * Athlon 662, Duron 671, and Athlon >model 7 have capability
	 * bit. It's worth noting that the A5 stepping (662) of some
	 * Athlon XP's have the MP bit set.
	 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
	 * more.
	 */
	if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
	    ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
	     (c->x86_model > 7))
B
Borislav Petkov 已提交
246
		if (cpu_has(c, X86_FEATURE_MP))
247
			return;
248 249 250 251 252 253 254 255

	/* If we get here, not a certified SMP capable AMD system. */

	/*
	 * Don't taint if we are running SMP kernel on a single non-MP
	 * approved Athlon
	 */
	WARN_ONCE(1, "WARNING: This combination of AMD"
256
		" processors is not suitable for SMP.\n");
257
	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE);
258
#endif
B
Borislav Petkov 已提交
259
}
260

261
#ifdef CONFIG_NUMA
262 263 264 265
/*
 * To workaround broken NUMA config.  Read the comment in
 * srat_detect_node().
 */
266
static int nearby_node(int apicid)
267 268 269 270
{
	int i, node;

	for (i = apicid - 1; i >= 0; i--) {
271
		node = __apicid_to_node[i];
272 273 274 275
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
276
		node = __apicid_to_node[i];
277 278 279 280 281 282
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	return first_node(node_online_map); /* Shouldn't happen */
}
#endif
283

284
/*
285 286 287
 * Fixup core topology information for
 * (1) AMD multi-node processors
 *     Assumption: Number of cores in each internal node is the same.
288
 * (2) AMD processors supporting compute units
289 290
 */
#ifdef CONFIG_X86_HT
291
static void amd_get_topology(struct cpuinfo_x86 *c)
292
{
293
	u32 nodes, cores_per_cu = 1;
294
	u8 node_id;
295 296
	int cpu = smp_processor_id();

297
	/* get information required for multi-node processors */
A
Andreas Herrmann 已提交
298
	if (cpu_has_topoext) {
299 300 301 302 303 304 305 306 307
		u32 eax, ebx, ecx, edx;

		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
		nodes = ((ecx >> 8) & 7) + 1;
		node_id = ecx & 7;

		/* get compute unit information */
		smp_num_siblings = ((ebx >> 8) & 3) + 1;
		c->compute_unit_id = ebx & 0xff;
308
		cores_per_cu += ((ebx >> 8) & 3);
309
	} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
310 311
		u64 value;

312 313 314 315
		rdmsrl(MSR_FAM10H_NODE_ID, value);
		nodes = ((value >> 3) & 7) + 1;
		node_id = value & 7;
	} else
316 317
		return;

318 319
	/* fixup multi-node processor information */
	if (nodes > 1) {
320
		u32 cores_per_node;
321
		u32 cus_per_node;
322

323 324
		set_cpu_cap(c, X86_FEATURE_AMD_DCM);
		cores_per_node = c->x86_max_cores / nodes;
325
		cus_per_node = cores_per_node / cores_per_cu;
326

327 328
		/* store NodeID, use llc_shared_map to store sibling info */
		per_cpu(cpu_llc_id, cpu) = node_id;
329

330
		/* core id has to be in the [0 .. cores_per_node - 1] range */
331 332
		c->cpu_core_id %= cores_per_node;
		c->compute_unit_id %= cus_per_node;
333
	}
334 335 336
}
#endif

337
/*
338
 * On a AMD dual core setup the lower bits of the APIC id distinguish the cores.
339 340
 * Assumes number of cores is a power of two.
 */
341
static void amd_detect_cmp(struct cpuinfo_x86 *c)
342 343 344
{
#ifdef CONFIG_X86_HT
	unsigned bits;
345
	int cpu = smp_processor_id();
346 347 348 349 350 351

	bits = c->x86_coreid_bits;
	/* Low order bits define the core id (index of core in socket) */
	c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
	/* Convert the initial APIC ID into the socket ID */
	c->phys_proc_id = c->initial_apicid >> bits;
352 353
	/* use socket ID also for last level cache */
	per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
354
	amd_get_topology(c);
355 356 357
#endif
}

358
u16 amd_get_nb_id(int cpu)
359
{
360
	u16 id = 0;
361 362 363 364 365 366 367
#ifdef CONFIG_SMP
	id = per_cpu(cpu_llc_id, cpu);
#endif
	return id;
}
EXPORT_SYMBOL_GPL(amd_get_nb_id);

368
static void srat_detect_node(struct cpuinfo_x86 *c)
369
{
370
#ifdef CONFIG_NUMA
371 372
	int cpu = smp_processor_id();
	int node;
373
	unsigned apicid = c->apicid;
374

375 376 377
	node = numa_cpu_node(cpu);
	if (node == NUMA_NO_NODE)
		node = per_cpu(cpu_llc_id, cpu);
378

379
	/*
380 381 382
	 * On multi-fabric platform (e.g. Numascale NumaChip) a
	 * platform-specific handler needs to be called to fixup some
	 * IDs of the CPU.
383
	 */
384
	if (x86_cpuinit.fixup_cpu_id)
385 386
		x86_cpuinit.fixup_cpu_id(c, node);

387
	if (!node_online(node)) {
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
		/*
		 * Two possibilities here:
		 *
		 * - The CPU is missing memory and no node was created.  In
		 *   that case try picking one from a nearby CPU.
		 *
		 * - The APIC IDs differ from the HyperTransport node IDs
		 *   which the K8 northbridge parsing fills in.  Assume
		 *   they are all increased by a constant offset, but in
		 *   the same order as the HT nodeids.  If that doesn't
		 *   result in a usable node fall back to the path for the
		 *   previous case.
		 *
		 * This workaround operates directly on the mapping between
		 * APIC ID and NUMA node, assuming certain relationship
		 * between APIC ID, HT node ID and NUMA topology.  As going
		 * through CPU mapping may alter the outcome, directly
		 * access __apicid_to_node[].
		 */
407 408 409
		int ht_nodeid = c->initial_apicid;

		if (ht_nodeid >= 0 &&
410 411
		    __apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
			node = __apicid_to_node[ht_nodeid];
412 413 414 415 416 417 418 419
		/* Pick a nearby node */
		if (!node_online(node))
			node = nearby_node(apicid);
	}
	numa_set_node(cpu, node);
#endif
}

420
static void early_init_amd_mc(struct cpuinfo_x86 *c)
421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
{
#ifdef CONFIG_X86_HT
	unsigned bits, ecx;

	/* Multi core CPU? */
	if (c->extended_cpuid_level < 0x80000008)
		return;

	ecx = cpuid_ecx(0x80000008);

	c->x86_max_cores = (ecx & 0xff) + 1;

	/* CPU telling us the core id bits shift? */
	bits = (ecx >> 12) & 0xF;

	/* Otherwise recompute */
	if (bits == 0) {
		while ((1 << bits) < c->x86_max_cores)
			bits++;
	}

	c->x86_coreid_bits = bits;
#endif
}

446
static void bsp_init_amd(struct cpuinfo_x86 *c)
447
{
B
Borislav Petkov 已提交
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467

#ifdef CONFIG_X86_64
	if (c->x86 >= 0xf) {
		unsigned long long tseg;

		/*
		 * Split up direct mapping around the TSEG SMM area.
		 * Don't do it for gbpages because there seems very little
		 * benefit in doing so.
		 */
		if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
			unsigned long pfn = tseg >> PAGE_SHIFT;

			printk(KERN_DEBUG "tseg: %010llx\n", tseg);
			if (pfn_range_is_mapped(pfn, pfn + 1))
				set_memory_4k((unsigned long)__va(tseg), 1);
		}
	}
#endif

468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
	if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {

		if (c->x86 > 0x10 ||
		    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
			u64 val;

			rdmsrl(MSR_K7_HWCR, val);
			if (!(val & BIT(24)))
				printk(KERN_WARNING FW_BUG "TSC doesn't count "
					"with P0 frequency!\n");
		}
	}

	if (c->x86 == 0x15) {
		unsigned long upperbit;
		u32 cpuid, assoc;

		cpuid	 = cpuid_edx(0x80000005);
		assoc	 = cpuid >> 16 & 0xff;
		upperbit = ((cpuid >> 24) << 10) / assoc;

		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
	}
}

494
static void early_init_amd(struct cpuinfo_x86 *c)
495
{
496 497
	early_init_amd_mc(c);

498 499 500 501 502
	/*
	 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
	 * with P/T states and does not stop in deep C-states
	 */
	if (c->x86_power & (1 << 8)) {
503
		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
504
		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
505
		if (!check_tsc_unstable())
506
			set_sched_clock_stable();
507
	}
508

509 510 511
#ifdef CONFIG_X86_64
	set_cpu_cap(c, X86_FEATURE_SYSCALL32);
#else
512
	/*  Set MTRR capability flag if appropriate */
513 514 515 516 517
	if (c->x86 == 5)
		if (c->x86_model == 13 || c->x86_model == 9 ||
		    (c->x86_model == 8 && c->x86_mask >= 8))
			set_cpu_cap(c, X86_FEATURE_K6_MTRR);
#endif
518 519
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
	/* check CPU config space for extended APIC ID */
520
	if (cpu_has_apic && c->x86 >= 0xf) {
521 522 523 524 525 526
		unsigned int val;
		val = read_pci_config(0, 24, 0, 0x68);
		if ((val & ((1 << 17) | (1 << 18))) == ((1 << 17) | (1 << 18)))
			set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
	}
#endif
527

528 529 530 531 532 533 534
	/*
	 * This is only needed to tell the kernel whether to use VMCALL
	 * and VMMCALL.  VMMCALL is never executed except under virt, so
	 * we can set it unconditionally.
	 */
	set_cpu_cap(c, X86_FEATURE_VMMCALL);

535
	/* F16h erratum 793, CVE-2013-6885 */
536 537
	if (c->x86 == 0x16 && c->x86_model <= 0xf)
		msr_set_bit(MSR_AMD64_LS_CFG, 15);
538 539
}

540
static const int amd_erratum_383[];
541
static const int amd_erratum_400[];
542
static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum);
543

B
Borislav Petkov 已提交
544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
static void init_amd_k8(struct cpuinfo_x86 *c)
{
	u32 level;
	u64 value;

	/* On C+ stepping K8 rep microcode works well for copy/memset */
	level = cpuid_eax(1);
	if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);

	/*
	 * Some BIOSes incorrectly force this feature, but only K8 revision D
	 * (model = 0x14) and later actually support it.
	 * (AMD Erratum #110, docId: 25759).
	 */
	if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
		clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
		if (!rdmsrl_amd_safe(0xc001100d, &value)) {
			value &= ~BIT_64(32);
			wrmsrl_amd_safe(0xc001100d, value);
		}
	}

	if (!c->x86_model_id[0])
		strcpy(c->x86_model_id, "Hammer");
569 570 571 572 573 574 575 576 577 578 579

#ifdef CONFIG_SMP
	/*
	 * Disable TLB flush filter by setting HWCR.FFDIS on K8
	 * bit 6 of msr C001_0015
	 *
	 * Errata 63 for SH-B3 steppings
	 * Errata 122 for all steppings (F+ have it disabled by default)
	 */
	msr_set_bit(MSR_K7_HWCR, 6);
#endif
B
Borislav Petkov 已提交
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
}

static void init_amd_gh(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
	/* do this for boot cpu */
	if (c == &boot_cpu_data)
		check_enable_amd_mmconf_dmi();

	fam10h_check_enable_mmcfg();
#endif

	/*
	 * Disable GART TLB Walk Errors on Fam10h. We do this here because this
	 * is always needed when GART is enabled, even in a kernel which has no
	 * MCE support built in. BIOS should disable GartTlbWlk Errors already.
	 * If it doesn't, we do it here as suggested by the BKDG.
	 *
	 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
	 */
	msr_set_bit(MSR_AMD64_MCx_MASK(4), 10);

	/*
	 * On family 10h BIOS may not have properly enabled WC+ support, causing
	 * it to be converted to CD memtype. This may result in performance
	 * degradation for certain nested-paging guests. Prevent this conversion
	 * by clearing bit 24 in MSR_AMD64_BU_CFG2.
	 *
	 * NOTE: we want to use the _safe accessors so as not to #GP kvm
	 * guests on older kvm hosts.
	 */
	msr_clear_bit(MSR_AMD64_BU_CFG2, 24);

	if (cpu_has_amd_erratum(c, amd_erratum_383))
		set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
}

static void init_amd_bd(struct cpuinfo_x86 *c)
{
	u64 value;

	/* re-enable TopologyExtensions if switched off by BIOS */
	if ((c->x86_model >= 0x10) && (c->x86_model <= 0x1f) &&
	    !cpu_has(c, X86_FEATURE_TOPOEXT)) {

		if (msr_set_bit(0xc0011005, 54) > 0) {
			rdmsrl(0xc0011005, value);
			if (value & BIT_64(54)) {
				set_cpu_cap(c, X86_FEATURE_TOPOEXT);
				pr_info(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n");
			}
		}
	}

	/*
	 * The way access filter has a performance penalty on some workloads.
	 * Disable it on the affected CPUs.
	 */
	if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
		if (!rdmsrl_safe(0xc0011021, &value) && !(value & 0x1E)) {
			value |= 0x1E;
			wrmsrl_safe(0xc0011021, value);
		}
	}
}

646
static void init_amd(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
647
{
648
	u32 dummy;
649

650 651
	early_init_amd(c);

652 653
	/*
	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
654
	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
655
	 */
656
	clear_cpu_cap(c, 0*32+31);
657

658
	if (c->x86 >= 0x10)
659
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);
660 661 662

	/* get apicid instead of initial apic id from cpuid */
	c->apicid = hard_smp_processor_id();
663 664 665 666

	/* K6s reports MCEs but don't actually have all the MSRs */
	if (c->x86 < 6)
		clear_cpu_cap(c, X86_FEATURE_MCE);
B
Borislav Petkov 已提交
667 668 669 670 671 672 673 674 675

	switch (c->x86) {
	case 4:    init_amd_k5(c); break;
	case 5:    init_amd_k6(c); break;
	case 6:	   init_amd_k7(c); break;
	case 0xf:  init_amd_k8(c); break;
	case 0x10: init_amd_gh(c); break;
	case 0x15: init_amd_bd(c); break;
	}
676

677
	/* Enable workaround for FXSAVE leak */
678
	if (c->x86 >= 6)
679
		set_cpu_bug(c, X86_BUG_FXSAVE_LEAK);
L
Linus Torvalds 已提交
680

681
	cpu_detect_cache_sizes(c);
682

683
	/* Multi core CPU? */
684
	if (c->extended_cpuid_level >= 0x80000008) {
685
		amd_detect_cmp(c);
686 687
		srat_detect_node(c);
	}
688

689
#ifdef CONFIG_X86_32
690
	detect_ht(c);
691
#endif
692

693
	init_amd_cacheinfo(c);
694

695
	if (c->x86 >= 0xf)
696
		set_cpu_cap(c, X86_FEATURE_K8);
697

698 699
	if (cpu_has_xmm2) {
		/* MFENCE stops RDTSC speculation */
700
		set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
701
	}
702

703 704 705 706 707
	/*
	 * Family 0x12 and above processors have APIC timer
	 * running in deep C states.
	 */
	if (c->x86 > 0x11)
708
		set_cpu_cap(c, X86_FEATURE_ARAT);
709

710
	if (cpu_has_amd_erratum(c, amd_erratum_400))
711 712
		set_cpu_bug(c, X86_BUG_AMD_APIC_C1E);

713
	rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
L
Linus Torvalds 已提交
714 715
}

716
#ifdef CONFIG_X86_32
717
static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
L
Linus Torvalds 已提交
718 719 720
{
	/* AMD errata T13 (order #21922) */
	if ((c->x86 == 6)) {
A
Alan Cox 已提交
721 722
		/* Duron Rev A0 */
		if (c->x86_model == 3 && c->x86_mask == 0)
L
Linus Torvalds 已提交
723
			size = 64;
A
Alan Cox 已提交
724
		/* Tbird rev A1/A2 */
L
Linus Torvalds 已提交
725
		if (c->x86_model == 4 &&
A
Alan Cox 已提交
726
			(c->x86_mask == 0 || c->x86_mask == 1))
L
Linus Torvalds 已提交
727 728 729 730
			size = 256;
	}
	return size;
}
731
#endif
L
Linus Torvalds 已提交
732

733
static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758
{
	u32 ebx, eax, ecx, edx;
	u16 mask = 0xfff;

	if (c->x86 < 0xf)
		return;

	if (c->extended_cpuid_level < 0x80000006)
		return;

	cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

	tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
	tlb_lli_4k[ENTRIES] = ebx & mask;

	/*
	 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
	 * characteristics from the CPUID function 0x80000005 instead.
	 */
	if (c->x86 == 0xf) {
		cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
		mask = 0xff;
	}

	/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
759 760 761
	if (!((eax >> 16) & mask))
		tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;
	else
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
		tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;

	/* a 4M entry uses two 2M entries */
	tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;

	/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
	if (!(eax & mask)) {
		/* Erratum 658 */
		if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
			tlb_lli_2m[ENTRIES] = 1024;
		} else {
			cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
			tlb_lli_2m[ENTRIES] = eax & 0xff;
		}
	} else
		tlb_lli_2m[ENTRIES] = eax & mask;

	tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
}

782
static const struct cpu_dev amd_cpu_dev = {
L
Linus Torvalds 已提交
783
	.c_vendor	= "AMD",
784
	.c_ident	= { "AuthenticAMD" },
785
#ifdef CONFIG_X86_32
786 787
	.legacy_models = {
		{ .family = 4, .model_names =
L
Linus Torvalds 已提交
788 789 790
		  {
			  [3] = "486 DX/2",
			  [7] = "486 DX/2-WB",
791 792
			  [8] = "486 DX/4",
			  [9] = "486 DX/4-WB",
L
Linus Torvalds 已提交
793
			  [14] = "Am5x86-WT",
794
			  [15] = "Am5x86-WB"
L
Linus Torvalds 已提交
795 796 797
		  }
		},
	},
798
	.legacy_cache_size = amd_size_cache,
799
#endif
800
	.c_early_init   = early_init_amd,
801
	.c_detect_tlb	= cpu_detect_tlb_amd,
802
	.c_bsp_init	= bsp_init_amd,
L
Linus Torvalds 已提交
803
	.c_init		= init_amd,
Y
Yinghai Lu 已提交
804
	.c_x86_vendor	= X86_VENDOR_AMD,
L
Linus Torvalds 已提交
805 806
};

Y
Yinghai Lu 已提交
807
cpu_dev_register(amd_cpu_dev);
808 809 810 811 812 813 814 815

/*
 * AMD errata checking
 *
 * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
 * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
 * have an OSVW id assigned, which it takes as first argument. Both take a
 * variable number of family-specific model-stepping ranges created by
816
 * AMD_MODEL_RANGE().
817 818 819 820 821 822 823 824 825
 *
 * Example:
 *
 * const int amd_erratum_319[] =
 *	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
 *			   AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
 *			   AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
 */

826 827 828 829 830 831 832 833 834
#define AMD_LEGACY_ERRATUM(...)		{ -1, __VA_ARGS__, 0 }
#define AMD_OSVW_ERRATUM(osvw_id, ...)	{ osvw_id, __VA_ARGS__, 0 }
#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
	((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
#define AMD_MODEL_RANGE_FAMILY(range)	(((range) >> 24) & 0xff)
#define AMD_MODEL_RANGE_START(range)	(((range) >> 12) & 0xfff)
#define AMD_MODEL_RANGE_END(range)	((range) & 0xfff)

static const int amd_erratum_400[] =
835
	AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
836 837
			    AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));

838
static const int amd_erratum_383[] =
839
	AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
840

841 842

static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
{
	int osvw_id = *erratum++;
	u32 range;
	u32 ms;

	if (osvw_id >= 0 && osvw_id < 65536 &&
	    cpu_has(cpu, X86_FEATURE_OSVW)) {
		u64 osvw_len;

		rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
		if (osvw_id < osvw_len) {
			u64 osvw_bits;

			rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
			    osvw_bits);
			return osvw_bits & (1ULL << (osvw_id & 0x3f));
		}
	}

	/* OSVW unavailable or ID unknown, match family-model-stepping range */
863
	ms = (cpu->x86_model << 4) | cpu->x86_mask;
864 865 866 867 868 869 870 871
	while ((range = *erratum++))
		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
		    (ms >= AMD_MODEL_RANGE_START(range)) &&
		    (ms <= AMD_MODEL_RANGE_END(range)))
			return true;

	return false;
}
872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890

void set_dr_addr_mask(unsigned long mask, int dr)
{
	if (!cpu_has_bpext)
		return;

	switch (dr) {
	case 0:
		wrmsr(MSR_F16H_DR0_ADDR_MASK, mask, 0);
		break;
	case 1:
	case 2:
	case 3:
		wrmsr(MSR_F16H_DR1_ADDR_MASK - 1 + dr, mask, 0);
		break;
	default:
		break;
	}
}