amd.c 21.8 KB
Newer Older
1
#include <linux/export.h>
L
Linus Torvalds 已提交
2
#include <linux/bitops.h>
3
#include <linux/elf.h>
L
Linus Torvalds 已提交
4
#include <linux/mm.h>
Y
Yinghai Lu 已提交
5

A
Alan Cox 已提交
6
#include <linux/io.h>
7
#include <linux/sched.h>
L
Linus Torvalds 已提交
8
#include <asm/processor.h>
9
#include <asm/apic.h>
10
#include <asm/cpu.h>
11
#include <asm/pci-direct.h>
L
Linus Torvalds 已提交
12

Y
Yinghai Lu 已提交
13 14 15 16 17
#ifdef CONFIG_X86_64
# include <asm/mmconfig.h>
# include <asm/cacheflush.h>
#endif

L
Linus Torvalds 已提交
18 19
#include "cpu.h"

20 21 22 23 24
static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
{
	u32 gprs[8] = { 0 };
	int err;

25 26
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41

	gprs[1] = msr;
	gprs[7] = 0x9c5a203a;

	err = rdmsr_safe_regs(gprs);

	*p = gprs[0] | ((u64)gprs[2] << 32);

	return err;
}

static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
{
	u32 gprs[8] = { 0 };

42 43
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
44 45 46 47 48 49 50 51 52

	gprs[0] = (u32)val;
	gprs[1] = msr;
	gprs[2] = val >> 32;
	gprs[7] = 0x9c5a203a;

	return wrmsr_safe_regs(gprs);
}

53
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
54 55 56 57 58 59
/*
 *	B step AMD K6 before B 9730xxxx have hardware bugs that can cause
 *	misexecution of code under Linux. Owners of such processors should
 *	contact AMD for precise details and a CPU swap.
 *
 *	See	http://www.multimania.com/poulot/k6bug.html
60 61
 *	and	section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6"
 *		(Publication # 21266  Issue Date: August 1998)
L
Linus Torvalds 已提交
62 63 64 65 66
 *
 *	The following test is erm.. interesting. AMD neglected to up
 *	the chip setting when fixing the bug but they also tweaked some
 *	performance at the same time..
 */
67

68 69
extern __visible void vide(void);
__asm__(".globl vide\n\t.align 4\nvide: ret");
L
Linus Torvalds 已提交
70

71
static void init_amd_k5(struct cpuinfo_x86 *c)
72 73 74 75 76 77 78 79 80 81 82
{
/*
 * General Systems BIOSen alias the cpu frequency registers
 * of the Elan at 0x000df000. Unfortuantly, one of the Linux
 * drivers subsequently pokes it, and changes the CPU speed.
 * Workaround : Remove the unneeded alias.
 */
#define CBAR		(0xfffc) /* Configuration Base Address  (32-bit) */
#define CBAR_ENB	(0x80000000)
#define CBAR_KEY	(0X000000CB)
	if (c->x86_model == 9 || c->x86_model == 10) {
A
Alan Cox 已提交
83 84
		if (inl(CBAR) & CBAR_ENB)
			outl(0 | CBAR_KEY, CBAR);
85 86 87 88
	}
}


89
static void init_amd_k6(struct cpuinfo_x86 *c)
90 91
{
	u32 l, h;
92
	int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124

	if (c->x86_model < 6) {
		/* Based on AMD doc 20734R - June 2000 */
		if (c->x86_model == 0) {
			clear_cpu_cap(c, X86_FEATURE_APIC);
			set_cpu_cap(c, X86_FEATURE_PGE);
		}
		return;
	}

	if (c->x86_model == 6 && c->x86_mask == 1) {
		const int K6_BUG_LOOP = 1000000;
		int n;
		void (*f_vide)(void);
		unsigned long d, d2;

		printk(KERN_INFO "AMD K6 stepping B detected - ");

		/*
		 * It looks like AMD fixed the 2.6.2 bug and improved indirect
		 * calls at the same time.
		 */

		n = K6_BUG_LOOP;
		f_vide = vide;
		rdtscl(d);
		while (n--)
			f_vide();
		rdtscl(d2);
		d = d2-d;

		if (d > 20*K6_BUG_LOOP)
A
Alan Cox 已提交
125 126
			printk(KERN_CONT
				"system stability may be impaired when more than 32 MB are used.\n");
127
		else
A
Alan Cox 已提交
128
			printk(KERN_CONT "probably OK (after B9730xxxx).\n");
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
	}

	/* K6 with old style WHCR */
	if (c->x86_model < 8 ||
	   (c->x86_model == 8 && c->x86_mask < 8)) {
		/* We can only write allocate on the low 508Mb */
		if (mbytes > 508)
			mbytes = 508;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0x0000FFFF) == 0) {
			unsigned long flags;
			l = (1<<0)|((mbytes/4)<<1);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling old style K6 write allocation for %d Mb\n",
				mbytes);
		}
		return;
	}

	if ((c->x86_model == 8 && c->x86_mask > 7) ||
	     c->x86_model == 9 || c->x86_model == 13) {
		/* The more serious chips .. */

		if (mbytes > 4092)
			mbytes = 4092;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0xFFFF0000) == 0) {
			unsigned long flags;
			l = ((mbytes>>2)<<22)|(1<<16);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling new style K6 write allocation for %d Mb\n",
				mbytes);
		}

		return;
	}

	if (c->x86_model == 10) {
		/* AMD Geode LX is model 10 */
		/* placeholder for any needed mods */
		return;
	}
}

181
static void amd_k7_smp_check(struct cpuinfo_x86 *c)
182 183
{
	/* calling is from identify_secondary_cpu() ? */
184
	if (!c->cpu_index)
185 186 187 188 189 190 191 192 193
		return;

	/*
	 * Certain Athlons might work (for various values of 'work') in SMP
	 * but they are not certified as MP capable.
	 */
	/* Athlon 660/661 is valid. */
	if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
	    (c->x86_mask == 1)))
194
		return;
195 196 197

	/* Duron 670 is valid */
	if ((c->x86_model == 7) && (c->x86_mask == 0))
198
		return;
199 200 201 202 203 204 205 206 207 208 209 210

	/*
	 * Athlon 662, Duron 671, and Athlon >model 7 have capability
	 * bit. It's worth noting that the A5 stepping (662) of some
	 * Athlon XP's have the MP bit set.
	 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
	 * more.
	 */
	if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
	    ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
	     (c->x86_model > 7))
		if (cpu_has_mp)
211
			return;
212 213 214 215 216 217 218 219

	/* If we get here, not a certified SMP capable AMD system. */

	/*
	 * Don't taint if we are running SMP kernel on a single non-MP
	 * approved Athlon
	 */
	WARN_ONCE(1, "WARNING: This combination of AMD"
220
		" processors is not suitable for SMP.\n");
221
	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE);
222 223
}

224
static void init_amd_k7(struct cpuinfo_x86 *c)
225 226 227 228 229 230 231 232 233 234 235
{
	u32 l, h;

	/*
	 * Bit 15 of Athlon specific MSR 15, needs to be 0
	 * to enable SSE on Palomino/Morgan/Barton CPU's.
	 * If the BIOS didn't enable it already, enable it here.
	 */
	if (c->x86_model >= 6 && c->x86_model <= 10) {
		if (!cpu_has(c, X86_FEATURE_XMM)) {
			printk(KERN_INFO "Enabling disabled K7/SSE Support.\n");
236
			msr_clear_bit(MSR_K7_HWCR, 15);
237 238 239 240 241 242 243 244 245 246 247 248
			set_cpu_cap(c, X86_FEATURE_XMM);
		}
	}

	/*
	 * It's been determined by AMD that Athlons since model 8 stepping 1
	 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
	 * As per AMD technical note 27212 0.2
	 */
	if ((c->x86_model == 8 && c->x86_mask >= 1) || (c->x86_model > 8)) {
		rdmsr(MSR_K7_CLK_CTL, l, h);
		if ((l & 0xfff00000) != 0x20000000) {
A
Alan Cox 已提交
249 250 251
			printk(KERN_INFO
			    "CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
					l, ((l & 0x000fffff)|0x20000000));
252 253 254 255 256
			wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
		}
	}

	set_cpu_cap(c, X86_FEATURE_K7);
257 258

	amd_k7_smp_check(c);
259
}
260 261
#endif

262
#ifdef CONFIG_NUMA
263 264 265 266
/*
 * To workaround broken NUMA config.  Read the comment in
 * srat_detect_node().
 */
267
static int nearby_node(int apicid)
268 269 270 271
{
	int i, node;

	for (i = apicid - 1; i >= 0; i--) {
272
		node = __apicid_to_node[i];
273 274 275 276
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
277
		node = __apicid_to_node[i];
278 279 280 281 282 283
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	return first_node(node_online_map); /* Shouldn't happen */
}
#endif
284

285
/*
286 287 288
 * Fixup core topology information for
 * (1) AMD multi-node processors
 *     Assumption: Number of cores in each internal node is the same.
289
 * (2) AMD processors supporting compute units
290 291
 */
#ifdef CONFIG_X86_HT
292
static void amd_get_topology(struct cpuinfo_x86 *c)
293
{
294
	u32 nodes, cores_per_cu = 1;
295
	u8 node_id;
296 297
	int cpu = smp_processor_id();

298
	/* get information required for multi-node processors */
A
Andreas Herrmann 已提交
299
	if (cpu_has_topoext) {
300 301 302 303 304 305 306 307 308
		u32 eax, ebx, ecx, edx;

		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
		nodes = ((ecx >> 8) & 7) + 1;
		node_id = ecx & 7;

		/* get compute unit information */
		smp_num_siblings = ((ebx >> 8) & 3) + 1;
		c->compute_unit_id = ebx & 0xff;
309
		cores_per_cu += ((ebx >> 8) & 3);
310
	} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
311 312
		u64 value;

313 314 315 316
		rdmsrl(MSR_FAM10H_NODE_ID, value);
		nodes = ((value >> 3) & 7) + 1;
		node_id = value & 7;
	} else
317 318
		return;

319 320
	/* fixup multi-node processor information */
	if (nodes > 1) {
321
		u32 cores_per_node;
322
		u32 cus_per_node;
323

324 325
		set_cpu_cap(c, X86_FEATURE_AMD_DCM);
		cores_per_node = c->x86_max_cores / nodes;
326
		cus_per_node = cores_per_node / cores_per_cu;
327

328 329
		/* store NodeID, use llc_shared_map to store sibling info */
		per_cpu(cpu_llc_id, cpu) = node_id;
330

331
		/* core id has to be in the [0 .. cores_per_node - 1] range */
332 333
		c->cpu_core_id %= cores_per_node;
		c->compute_unit_id %= cus_per_node;
334
	}
335 336 337
}
#endif

338
/*
339
 * On a AMD dual core setup the lower bits of the APIC id distinguish the cores.
340 341
 * Assumes number of cores is a power of two.
 */
342
static void amd_detect_cmp(struct cpuinfo_x86 *c)
343 344 345
{
#ifdef CONFIG_X86_HT
	unsigned bits;
346
	int cpu = smp_processor_id();
347 348 349 350 351 352

	bits = c->x86_coreid_bits;
	/* Low order bits define the core id (index of core in socket) */
	c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
	/* Convert the initial APIC ID into the socket ID */
	c->phys_proc_id = c->initial_apicid >> bits;
353 354
	/* use socket ID also for last level cache */
	per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
355
	amd_get_topology(c);
356 357 358
#endif
}

359
u16 amd_get_nb_id(int cpu)
360
{
361
	u16 id = 0;
362 363 364 365 366 367 368
#ifdef CONFIG_SMP
	id = per_cpu(cpu_llc_id, cpu);
#endif
	return id;
}
EXPORT_SYMBOL_GPL(amd_get_nb_id);

369
static void srat_detect_node(struct cpuinfo_x86 *c)
370
{
371
#ifdef CONFIG_NUMA
372 373
	int cpu = smp_processor_id();
	int node;
374
	unsigned apicid = c->apicid;
375

376 377 378
	node = numa_cpu_node(cpu);
	if (node == NUMA_NO_NODE)
		node = per_cpu(cpu_llc_id, cpu);
379

380
	/*
381 382 383
	 * On multi-fabric platform (e.g. Numascale NumaChip) a
	 * platform-specific handler needs to be called to fixup some
	 * IDs of the CPU.
384
	 */
385
	if (x86_cpuinit.fixup_cpu_id)
386 387
		x86_cpuinit.fixup_cpu_id(c, node);

388
	if (!node_online(node)) {
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
		/*
		 * Two possibilities here:
		 *
		 * - The CPU is missing memory and no node was created.  In
		 *   that case try picking one from a nearby CPU.
		 *
		 * - The APIC IDs differ from the HyperTransport node IDs
		 *   which the K8 northbridge parsing fills in.  Assume
		 *   they are all increased by a constant offset, but in
		 *   the same order as the HT nodeids.  If that doesn't
		 *   result in a usable node fall back to the path for the
		 *   previous case.
		 *
		 * This workaround operates directly on the mapping between
		 * APIC ID and NUMA node, assuming certain relationship
		 * between APIC ID, HT node ID and NUMA topology.  As going
		 * through CPU mapping may alter the outcome, directly
		 * access __apicid_to_node[].
		 */
408 409 410
		int ht_nodeid = c->initial_apicid;

		if (ht_nodeid >= 0 &&
411 412
		    __apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
			node = __apicid_to_node[ht_nodeid];
413 414 415 416 417 418 419 420
		/* Pick a nearby node */
		if (!node_online(node))
			node = nearby_node(apicid);
	}
	numa_set_node(cpu, node);
#endif
}

421
static void early_init_amd_mc(struct cpuinfo_x86 *c)
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
{
#ifdef CONFIG_X86_HT
	unsigned bits, ecx;

	/* Multi core CPU? */
	if (c->extended_cpuid_level < 0x80000008)
		return;

	ecx = cpuid_ecx(0x80000008);

	c->x86_max_cores = (ecx & 0xff) + 1;

	/* CPU telling us the core id bits shift? */
	bits = (ecx >> 12) & 0xF;

	/* Otherwise recompute */
	if (bits == 0) {
		while ((1 << bits) < c->x86_max_cores)
			bits++;
	}

	c->x86_coreid_bits = bits;
#endif
}

447
static void bsp_init_amd(struct cpuinfo_x86 *c)
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
{
	if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {

		if (c->x86 > 0x10 ||
		    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
			u64 val;

			rdmsrl(MSR_K7_HWCR, val);
			if (!(val & BIT(24)))
				printk(KERN_WARNING FW_BUG "TSC doesn't count "
					"with P0 frequency!\n");
		}
	}

	if (c->x86 == 0x15) {
		unsigned long upperbit;
		u32 cpuid, assoc;

		cpuid	 = cpuid_edx(0x80000005);
		assoc	 = cpuid >> 16 & 0xff;
		upperbit = ((cpuid >> 24) << 10) / assoc;

		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
	}
}

475
static void early_init_amd(struct cpuinfo_x86 *c)
476
{
477 478
	early_init_amd_mc(c);

479 480 481 482 483
	/*
	 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
	 * with P/T states and does not stop in deep C-states
	 */
	if (c->x86_power & (1 << 8)) {
484
		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
485
		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
486
		if (!check_tsc_unstable())
487
			set_sched_clock_stable();
488
	}
489

490 491 492
#ifdef CONFIG_X86_64
	set_cpu_cap(c, X86_FEATURE_SYSCALL32);
#else
493
	/*  Set MTRR capability flag if appropriate */
494 495 496 497 498
	if (c->x86 == 5)
		if (c->x86_model == 13 || c->x86_model == 9 ||
		    (c->x86_model == 8 && c->x86_mask >= 8))
			set_cpu_cap(c, X86_FEATURE_K6_MTRR);
#endif
499 500
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
	/* check CPU config space for extended APIC ID */
501
	if (cpu_has_apic && c->x86 >= 0xf) {
502 503 504 505 506 507
		unsigned int val;
		val = read_pci_config(0, 24, 0, 0x68);
		if ((val & ((1 << 17) | (1 << 18))) == ((1 << 17) | (1 << 18)))
			set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
	}
#endif
508 509

	/* F16h erratum 793, CVE-2013-6885 */
510 511
	if (c->x86 == 0x16 && c->x86_model <= 0xf)
		msr_set_bit(MSR_AMD64_LS_CFG, 15);
512 513
}

514
static const int amd_erratum_383[];
515
static const int amd_erratum_400[];
516
static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum);
517

518
static void init_amd(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
519
{
520
	u32 dummy;
521
	unsigned long long value;
522

B
Boris Ostrovsky 已提交
523
#ifdef CONFIG_SMP
524 525
	/*
	 * Disable TLB flush filter by setting HWCR.FFDIS on K8
526 527 528 529 530
	 * bit 6 of msr C001_0015
	 *
	 * Errata 63 for SH-B3 steppings
	 * Errata 122 for all steppings (F+ have it disabled by default)
	 */
531 532
	if (c->x86 == 0xf)
		msr_set_bit(MSR_K7_HWCR, 6);
533 534
#endif

535 536
	early_init_amd(c);

537 538
	/*
	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
539
	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
540
	 */
541
	clear_cpu_cap(c, 0*32+31);
542

543 544 545 546 547 548
#ifdef CONFIG_X86_64
	/* On C+ stepping K8 rep microcode works well for copy/memset */
	if (c->x86 == 0xf) {
		u32 level;

		level = cpuid_eax(1);
A
Alan Cox 已提交
549
		if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
550
			set_cpu_cap(c, X86_FEATURE_REP_GOOD);
551 552 553 554

		/*
		 * Some BIOSes incorrectly force this feature, but only K8
		 * revision D (model = 0x14) and later actually support it.
555
		 * (AMD Erratum #110, docId: 25759).
556
		 */
557
		if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
558
			clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
B
Boris Ostrovsky 已提交
559 560 561
			if (!rdmsrl_amd_safe(0xc001100d, &value)) {
				value &= ~(1ULL << 32);
				wrmsrl_amd_safe(0xc001100d, value);
562 563 564
			}
		}

565
	}
566
	if (c->x86 >= 0x10)
567
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);
568 569 570

	/* get apicid instead of initial apic id from cpuid */
	c->apicid = hard_smp_processor_id();
571 572 573 574 575 576 577 578
#else

	/*
	 *	FIXME: We should handle the K5 here. Set up the write
	 *	range and also turn on MSR 83 bits 4 and 31 (write alloc,
	 *	no bus pipeline)
	 */

579 580
	switch (c->x86) {
	case 4:
581 582
		init_amd_k5(c);
		break;
583
	case 5:
584
		init_amd_k6(c);
L
Linus Torvalds 已提交
585
		break;
586 587
	case 6: /* An Athlon/Duron */
		init_amd_k7(c);
L
Linus Torvalds 已提交
588 589
		break;
	}
590 591 592 593

	/* K6s reports MCEs but don't actually have all the MSRs */
	if (c->x86 < 6)
		clear_cpu_cap(c, X86_FEATURE_MCE);
594
#endif
595

596
	/* Enable workaround for FXSAVE leak */
597
	if (c->x86 >= 6)
598
		set_cpu_bug(c, X86_BUG_FXSAVE_LEAK);
L
Linus Torvalds 已提交
599

600 601 602 603 604 605 606 607 608
	if (!c->x86_model_id[0]) {
		switch (c->x86) {
		case 0xf:
			/* Should distinguish Models here, but this is only
			   a fallback anyways. */
			strcpy(c->x86_model_id, "Hammer");
			break;
		}
	}
609

610 611 612 613 614
	/* re-enable TopologyExtensions if switched off by BIOS */
	if ((c->x86 == 0x15) &&
	    (c->x86_model >= 0x10) && (c->x86_model <= 0x1f) &&
	    !cpu_has(c, X86_FEATURE_TOPOEXT)) {

615
		if (msr_set_bit(0xc0011005, 54) > 0) {
B
Boris Ostrovsky 已提交
616
			rdmsrl(0xc0011005, value);
617
			if (value & BIT_64(54)) {
618
				set_cpu_cap(c, X86_FEATURE_TOPOEXT);
619
				pr_info(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n");
620 621 622 623
			}
		}
	}

624 625 626 627 628 629 630
	/*
	 * The way access filter has a performance penalty on some workloads.
	 * Disable it on the affected CPUs.
	 */
	if ((c->x86 == 0x15) &&
	    (c->x86_model >= 0x02) && (c->x86_model < 0x20)) {

B
Boris Ostrovsky 已提交
631 632 633
		if (!rdmsrl_safe(0xc0011021, &value) && !(value & 0x1E)) {
			value |= 0x1E;
			wrmsrl_safe(0xc0011021, value);
634 635 636
		}
	}

637
	cpu_detect_cache_sizes(c);
638

639
	/* Multi core CPU? */
640
	if (c->extended_cpuid_level >= 0x80000008) {
641
		amd_detect_cmp(c);
642 643
		srat_detect_node(c);
	}
644

645
#ifdef CONFIG_X86_32
646
	detect_ht(c);
647
#endif
648

649
	init_amd_cacheinfo(c);
650

651
	if (c->x86 >= 0xf)
652
		set_cpu_cap(c, X86_FEATURE_K8);
653

654 655
	if (cpu_has_xmm2) {
		/* MFENCE stops RDTSC speculation */
656
		set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
657
	}
658 659 660 661 662 663 664 665 666 667

#ifdef CONFIG_X86_64
	if (c->x86 == 0x10) {
		/* do this for boot cpu */
		if (c == &boot_cpu_data)
			check_enable_amd_mmconf_dmi();

		fam10h_check_enable_mmcfg();
	}

668
	if (c == &boot_cpu_data && c->x86 >= 0xf) {
669 670 671 672 673 674 675 676
		unsigned long long tseg;

		/*
		 * Split up direct mapping around the TSEG SMM area.
		 * Don't do it for gbpages because there seems very little
		 * benefit in doing so.
		 */
		if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
677 678
			unsigned long pfn = tseg >> PAGE_SHIFT;

A
Alan Cox 已提交
679
			printk(KERN_DEBUG "tseg: %010llx\n", tseg);
680
			if (pfn_range_is_mapped(pfn, pfn + 1))
A
Alan Cox 已提交
681
				set_memory_4k((unsigned long)__va(tseg), 1);
682 683 684
		}
	}
#endif
685

686 687 688 689 690
	/*
	 * Family 0x12 and above processors have APIC timer
	 * running in deep C states.
	 */
	if (c->x86 > 0x11)
691
		set_cpu_cap(c, X86_FEATURE_ARAT);
692 693 694

	if (c->x86 == 0x10) {
		/*
695 696 697
		 * Disable GART TLB Walk Errors on Fam10h. We do this here
		 * because this is always needed when GART is enabled, even in a
		 * kernel which has no MCE support built in.
698 699
		 * BIOS should disable GartTlbWlk Errors already. If
		 * it doesn't, do it here as suggested by the BKDG.
700 701 702
		 *
		 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
		 */
703
		msr_set_bit(MSR_AMD64_MCx_MASK(4), 10);
704 705 706 707 708 709 710

		/*
		 * On family 10h BIOS may not have properly enabled WC+ support,
		 * causing it to be converted to CD memtype. This may result in
		 * performance degradation for certain nested-paging guests.
		 * Prevent this conversion by clearing bit 24 in
		 * MSR_AMD64_BU_CFG2.
711 712 713
		 *
		 * NOTE: we want to use the _safe accessors so as not to #GP kvm
		 * guests on older kvm hosts.
714
		 */
715
		msr_clear_bit(MSR_AMD64_BU_CFG2, 24);
716

717
		if (cpu_has_amd_erratum(c, amd_erratum_383))
718
			set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
719
	}
720

721
	if (cpu_has_amd_erratum(c, amd_erratum_400))
722 723
		set_cpu_bug(c, X86_BUG_AMD_APIC_C1E);

724
	rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
L
Linus Torvalds 已提交
725 726
}

727
#ifdef CONFIG_X86_32
728
static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
L
Linus Torvalds 已提交
729 730 731
{
	/* AMD errata T13 (order #21922) */
	if ((c->x86 == 6)) {
A
Alan Cox 已提交
732 733
		/* Duron Rev A0 */
		if (c->x86_model == 3 && c->x86_mask == 0)
L
Linus Torvalds 已提交
734
			size = 64;
A
Alan Cox 已提交
735
		/* Tbird rev A1/A2 */
L
Linus Torvalds 已提交
736
		if (c->x86_model == 4 &&
A
Alan Cox 已提交
737
			(c->x86_mask == 0 || c->x86_mask == 1))
L
Linus Torvalds 已提交
738 739 740 741
			size = 256;
	}
	return size;
}
742
#endif
L
Linus Torvalds 已提交
743

744
static void cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c)
745
{
746
	tlb_flushall_shift = 6;
747 748
}

749
static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774
{
	u32 ebx, eax, ecx, edx;
	u16 mask = 0xfff;

	if (c->x86 < 0xf)
		return;

	if (c->extended_cpuid_level < 0x80000006)
		return;

	cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

	tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
	tlb_lli_4k[ENTRIES] = ebx & mask;

	/*
	 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
	 * characteristics from the CPUID function 0x80000005 instead.
	 */
	if (c->x86 == 0xf) {
		cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
		mask = 0xff;
	}

	/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
775 776 777
	if (!((eax >> 16) & mask))
		tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;
	else
778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795
		tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;

	/* a 4M entry uses two 2M entries */
	tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;

	/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
	if (!(eax & mask)) {
		/* Erratum 658 */
		if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
			tlb_lli_2m[ENTRIES] = 1024;
		} else {
			cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
			tlb_lli_2m[ENTRIES] = eax & 0xff;
		}
	} else
		tlb_lli_2m[ENTRIES] = eax & mask;

	tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
796 797

	cpu_set_tlb_flushall_shift(c);
798 799
}

800
static const struct cpu_dev amd_cpu_dev = {
L
Linus Torvalds 已提交
801
	.c_vendor	= "AMD",
802
	.c_ident	= { "AuthenticAMD" },
803
#ifdef CONFIG_X86_32
804 805
	.legacy_models = {
		{ .family = 4, .model_names =
L
Linus Torvalds 已提交
806 807 808
		  {
			  [3] = "486 DX/2",
			  [7] = "486 DX/2-WB",
809 810
			  [8] = "486 DX/4",
			  [9] = "486 DX/4-WB",
L
Linus Torvalds 已提交
811
			  [14] = "Am5x86-WT",
812
			  [15] = "Am5x86-WB"
L
Linus Torvalds 已提交
813 814 815
		  }
		},
	},
816
	.legacy_cache_size = amd_size_cache,
817
#endif
818
	.c_early_init   = early_init_amd,
819
	.c_detect_tlb	= cpu_detect_tlb_amd,
820
	.c_bsp_init	= bsp_init_amd,
L
Linus Torvalds 已提交
821
	.c_init		= init_amd,
Y
Yinghai Lu 已提交
822
	.c_x86_vendor	= X86_VENDOR_AMD,
L
Linus Torvalds 已提交
823 824
};

Y
Yinghai Lu 已提交
825
cpu_dev_register(amd_cpu_dev);
826 827 828 829 830 831 832 833

/*
 * AMD errata checking
 *
 * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
 * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
 * have an OSVW id assigned, which it takes as first argument. Both take a
 * variable number of family-specific model-stepping ranges created by
834
 * AMD_MODEL_RANGE().
835 836 837 838 839 840 841 842 843
 *
 * Example:
 *
 * const int amd_erratum_319[] =
 *	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
 *			   AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
 *			   AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
 */

844 845 846 847 848 849 850 851 852
#define AMD_LEGACY_ERRATUM(...)		{ -1, __VA_ARGS__, 0 }
#define AMD_OSVW_ERRATUM(osvw_id, ...)	{ osvw_id, __VA_ARGS__, 0 }
#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
	((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
#define AMD_MODEL_RANGE_FAMILY(range)	(((range) >> 24) & 0xff)
#define AMD_MODEL_RANGE_START(range)	(((range) >> 12) & 0xfff)
#define AMD_MODEL_RANGE_END(range)	((range) & 0xfff)

static const int amd_erratum_400[] =
853
	AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
854 855
			    AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));

856
static const int amd_erratum_383[] =
857
	AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
858

859 860

static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880
{
	int osvw_id = *erratum++;
	u32 range;
	u32 ms;

	if (osvw_id >= 0 && osvw_id < 65536 &&
	    cpu_has(cpu, X86_FEATURE_OSVW)) {
		u64 osvw_len;

		rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
		if (osvw_id < osvw_len) {
			u64 osvw_bits;

			rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
			    osvw_bits);
			return osvw_bits & (1ULL << (osvw_id & 0x3f));
		}
	}

	/* OSVW unavailable or ID unknown, match family-model-stepping range */
881
	ms = (cpu->x86_model << 4) | cpu->x86_mask;
882 883 884 885 886 887 888 889
	while ((range = *erratum++))
		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
		    (ms >= AMD_MODEL_RANGE_START(range)) &&
		    (ms <= AMD_MODEL_RANGE_END(range)))
			return true;

	return false;
}