amd.c 22.9 KB
Newer Older
1
#include <linux/export.h>
L
Linus Torvalds 已提交
2
#include <linux/bitops.h>
3
#include <linux/elf.h>
L
Linus Torvalds 已提交
4
#include <linux/mm.h>
Y
Yinghai Lu 已提交
5

A
Alan Cox 已提交
6
#include <linux/io.h>
7
#include <linux/sched.h>
8
#include <linux/random.h>
L
Linus Torvalds 已提交
9
#include <asm/processor.h>
10
#include <asm/apic.h>
11
#include <asm/cpu.h>
B
Borislav Petkov 已提交
12
#include <asm/smp.h>
13
#include <asm/pci-direct.h>
L
Linus Torvalds 已提交
14

Y
Yinghai Lu 已提交
15 16 17 18 19
#ifdef CONFIG_X86_64
# include <asm/mmconfig.h>
# include <asm/cacheflush.h>
#endif

L
Linus Torvalds 已提交
20 21
#include "cpu.h"

22 23 24 25 26 27 28
/*
 * nodes_per_socket: Stores the number of nodes per socket.
 * Refer to Fam15h Models 00-0fh BKDG - CPUID Fn8000_001E_ECX
 * Node Identifiers[10:8]
 */
static u32 nodes_per_socket = 1;

29 30 31 32 33
static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
{
	u32 gprs[8] = { 0 };
	int err;

34 35
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50

	gprs[1] = msr;
	gprs[7] = 0x9c5a203a;

	err = rdmsr_safe_regs(gprs);

	*p = gprs[0] | ((u64)gprs[2] << 32);

	return err;
}

static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
{
	u32 gprs[8] = { 0 };

51 52
	WARN_ONCE((boot_cpu_data.x86 != 0xf),
		  "%s should only be used on K8!\n", __func__);
53 54 55 56 57 58 59 60 61

	gprs[0] = (u32)val;
	gprs[1] = msr;
	gprs[2] = val >> 32;
	gprs[7] = 0x9c5a203a;

	return wrmsr_safe_regs(gprs);
}

L
Linus Torvalds 已提交
62 63 64 65 66 67
/*
 *	B step AMD K6 before B 9730xxxx have hardware bugs that can cause
 *	misexecution of code under Linux. Owners of such processors should
 *	contact AMD for precise details and a CPU swap.
 *
 *	See	http://www.multimania.com/poulot/k6bug.html
68 69
 *	and	section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6"
 *		(Publication # 21266  Issue Date: August 1998)
L
Linus Torvalds 已提交
70 71 72 73 74
 *
 *	The following test is erm.. interesting. AMD neglected to up
 *	the chip setting when fixing the bug but they also tweaked some
 *	performance at the same time..
 */
75

76 77
extern __visible void vide(void);
__asm__(".globl vide\n\t.align 4\nvide: ret");
L
Linus Torvalds 已提交
78

79
static void init_amd_k5(struct cpuinfo_x86 *c)
80
{
B
Borislav Petkov 已提交
81
#ifdef CONFIG_X86_32
82 83 84 85 86 87 88 89 90 91
/*
 * General Systems BIOSen alias the cpu frequency registers
 * of the Elan at 0x000df000. Unfortuantly, one of the Linux
 * drivers subsequently pokes it, and changes the CPU speed.
 * Workaround : Remove the unneeded alias.
 */
#define CBAR		(0xfffc) /* Configuration Base Address  (32-bit) */
#define CBAR_ENB	(0x80000000)
#define CBAR_KEY	(0X000000CB)
	if (c->x86_model == 9 || c->x86_model == 10) {
A
Alan Cox 已提交
92 93
		if (inl(CBAR) & CBAR_ENB)
			outl(0 | CBAR_KEY, CBAR);
94
	}
B
Borislav Petkov 已提交
95
#endif
96 97
}

98
static void init_amd_k6(struct cpuinfo_x86 *c)
99
{
B
Borislav Petkov 已提交
100
#ifdef CONFIG_X86_32
101
	u32 l, h;
102
	int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
103 104 105 106 107 108 109 110 111 112 113 114 115 116

	if (c->x86_model < 6) {
		/* Based on AMD doc 20734R - June 2000 */
		if (c->x86_model == 0) {
			clear_cpu_cap(c, X86_FEATURE_APIC);
			set_cpu_cap(c, X86_FEATURE_PGE);
		}
		return;
	}

	if (c->x86_model == 6 && c->x86_mask == 1) {
		const int K6_BUG_LOOP = 1000000;
		int n;
		void (*f_vide)(void);
117
		u64 d, d2;
118 119 120 121 122 123 124 125 126 127

		printk(KERN_INFO "AMD K6 stepping B detected - ");

		/*
		 * It looks like AMD fixed the 2.6.2 bug and improved indirect
		 * calls at the same time.
		 */

		n = K6_BUG_LOOP;
		f_vide = vide;
128
		d = rdtsc();
129 130
		while (n--)
			f_vide();
131
		d2 = rdtsc();
132 133 134
		d = d2-d;

		if (d > 20*K6_BUG_LOOP)
A
Alan Cox 已提交
135 136
			printk(KERN_CONT
				"system stability may be impaired when more than 32 MB are used.\n");
137
		else
A
Alan Cox 已提交
138
			printk(KERN_CONT "probably OK (after B9730xxxx).\n");
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
	}

	/* K6 with old style WHCR */
	if (c->x86_model < 8 ||
	   (c->x86_model == 8 && c->x86_mask < 8)) {
		/* We can only write allocate on the low 508Mb */
		if (mbytes > 508)
			mbytes = 508;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0x0000FFFF) == 0) {
			unsigned long flags;
			l = (1<<0)|((mbytes/4)<<1);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling old style K6 write allocation for %d Mb\n",
				mbytes);
		}
		return;
	}

	if ((c->x86_model == 8 && c->x86_mask > 7) ||
	     c->x86_model == 9 || c->x86_model == 13) {
		/* The more serious chips .. */

		if (mbytes > 4092)
			mbytes = 4092;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0xFFFF0000) == 0) {
			unsigned long flags;
			l = ((mbytes>>2)<<22)|(1<<16);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling new style K6 write allocation for %d Mb\n",
				mbytes);
		}

		return;
	}

	if (c->x86_model == 10) {
		/* AMD Geode LX is model 10 */
		/* placeholder for any needed mods */
		return;
	}
B
Borislav Petkov 已提交
189
#endif
190 191
}

B
Borislav Petkov 已提交
192
static void init_amd_k7(struct cpuinfo_x86 *c)
193
{
B
Borislav Petkov 已提交
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
#ifdef CONFIG_X86_32
	u32 l, h;

	/*
	 * Bit 15 of Athlon specific MSR 15, needs to be 0
	 * to enable SSE on Palomino/Morgan/Barton CPU's.
	 * If the BIOS didn't enable it already, enable it here.
	 */
	if (c->x86_model >= 6 && c->x86_model <= 10) {
		if (!cpu_has(c, X86_FEATURE_XMM)) {
			printk(KERN_INFO "Enabling disabled K7/SSE Support.\n");
			msr_clear_bit(MSR_K7_HWCR, 15);
			set_cpu_cap(c, X86_FEATURE_XMM);
		}
	}

	/*
	 * It's been determined by AMD that Athlons since model 8 stepping 1
	 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
	 * As per AMD technical note 27212 0.2
	 */
	if ((c->x86_model == 8 && c->x86_mask >= 1) || (c->x86_model > 8)) {
		rdmsr(MSR_K7_CLK_CTL, l, h);
		if ((l & 0xfff00000) != 0x20000000) {
			printk(KERN_INFO
			    "CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
					l, ((l & 0x000fffff)|0x20000000));
			wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
		}
	}

	set_cpu_cap(c, X86_FEATURE_K7);

227
	/* calling is from identify_secondary_cpu() ? */
228
	if (!c->cpu_index)
229 230 231 232 233 234 235 236 237
		return;

	/*
	 * Certain Athlons might work (for various values of 'work') in SMP
	 * but they are not certified as MP capable.
	 */
	/* Athlon 660/661 is valid. */
	if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
	    (c->x86_mask == 1)))
238
		return;
239 240 241

	/* Duron 670 is valid */
	if ((c->x86_model == 7) && (c->x86_mask == 0))
242
		return;
243 244 245 246 247 248 249 250 251 252 253

	/*
	 * Athlon 662, Duron 671, and Athlon >model 7 have capability
	 * bit. It's worth noting that the A5 stepping (662) of some
	 * Athlon XP's have the MP bit set.
	 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
	 * more.
	 */
	if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
	    ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
	     (c->x86_model > 7))
B
Borislav Petkov 已提交
254
		if (cpu_has(c, X86_FEATURE_MP))
255
			return;
256 257 258 259 260 261 262 263

	/* If we get here, not a certified SMP capable AMD system. */

	/*
	 * Don't taint if we are running SMP kernel on a single non-MP
	 * approved Athlon
	 */
	WARN_ONCE(1, "WARNING: This combination of AMD"
264
		" processors is not suitable for SMP.\n");
265
	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE);
266
#endif
B
Borislav Petkov 已提交
267
}
268

269
#ifdef CONFIG_NUMA
270 271 272 273
/*
 * To workaround broken NUMA config.  Read the comment in
 * srat_detect_node().
 */
274
static int nearby_node(int apicid)
275 276 277 278
{
	int i, node;

	for (i = apicid - 1; i >= 0; i--) {
279
		node = __apicid_to_node[i];
280 281 282 283
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
284
		node = __apicid_to_node[i];
285 286 287 288 289 290
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	return first_node(node_online_map); /* Shouldn't happen */
}
#endif
291

292
/*
293 294 295
 * Fixup core topology information for
 * (1) AMD multi-node processors
 *     Assumption: Number of cores in each internal node is the same.
296
 * (2) AMD processors supporting compute units
297
 */
B
Borislav Petkov 已提交
298
#ifdef CONFIG_SMP
299
static void amd_get_topology(struct cpuinfo_x86 *c)
300
{
301
	u32 cores_per_cu = 1;
302
	u8 node_id;
303 304
	int cpu = smp_processor_id();

305
	/* get information required for multi-node processors */
A
Andreas Herrmann 已提交
306
	if (cpu_has_topoext) {
307 308 309
		u32 eax, ebx, ecx, edx;

		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
310
		nodes_per_socket = ((ecx >> 8) & 7) + 1;
311 312 313 314 315
		node_id = ecx & 7;

		/* get compute unit information */
		smp_num_siblings = ((ebx >> 8) & 3) + 1;
		c->compute_unit_id = ebx & 0xff;
316
		cores_per_cu += ((ebx >> 8) & 3);
317
	} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
318 319
		u64 value;

320
		rdmsrl(MSR_FAM10H_NODE_ID, value);
321
		nodes_per_socket = ((value >> 3) & 7) + 1;
322 323
		node_id = value & 7;
	} else
324 325
		return;

326
	/* fixup multi-node processor information */
327
	if (nodes_per_socket > 1) {
328
		u32 cores_per_node;
329
		u32 cus_per_node;
330

331
		set_cpu_cap(c, X86_FEATURE_AMD_DCM);
332
		cores_per_node = c->x86_max_cores / nodes_per_socket;
333
		cus_per_node = cores_per_node / cores_per_cu;
334

335 336
		/* store NodeID, use llc_shared_map to store sibling info */
		per_cpu(cpu_llc_id, cpu) = node_id;
337

338
		/* core id has to be in the [0 .. cores_per_node - 1] range */
339 340
		c->cpu_core_id %= cores_per_node;
		c->compute_unit_id %= cus_per_node;
341
	}
342 343 344
}
#endif

345
/*
346
 * On a AMD dual core setup the lower bits of the APIC id distinguish the cores.
347 348
 * Assumes number of cores is a power of two.
 */
349
static void amd_detect_cmp(struct cpuinfo_x86 *c)
350
{
B
Borislav Petkov 已提交
351
#ifdef CONFIG_SMP
352
	unsigned bits;
353
	int cpu = smp_processor_id();
354 355 356 357 358 359

	bits = c->x86_coreid_bits;
	/* Low order bits define the core id (index of core in socket) */
	c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
	/* Convert the initial APIC ID into the socket ID */
	c->phys_proc_id = c->initial_apicid >> bits;
360 361
	/* use socket ID also for last level cache */
	per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
362
	amd_get_topology(c);
363 364 365
#endif
}

366
u16 amd_get_nb_id(int cpu)
367
{
368
	u16 id = 0;
369 370 371 372 373 374 375
#ifdef CONFIG_SMP
	id = per_cpu(cpu_llc_id, cpu);
#endif
	return id;
}
EXPORT_SYMBOL_GPL(amd_get_nb_id);

376 377 378 379 380 381
u32 amd_get_nodes_per_socket(void)
{
	return nodes_per_socket;
}
EXPORT_SYMBOL_GPL(amd_get_nodes_per_socket);

382
static void srat_detect_node(struct cpuinfo_x86 *c)
383
{
384
#ifdef CONFIG_NUMA
385 386
	int cpu = smp_processor_id();
	int node;
387
	unsigned apicid = c->apicid;
388

389 390 391
	node = numa_cpu_node(cpu);
	if (node == NUMA_NO_NODE)
		node = per_cpu(cpu_llc_id, cpu);
392

393
	/*
394 395 396
	 * On multi-fabric platform (e.g. Numascale NumaChip) a
	 * platform-specific handler needs to be called to fixup some
	 * IDs of the CPU.
397
	 */
398
	if (x86_cpuinit.fixup_cpu_id)
399 400
		x86_cpuinit.fixup_cpu_id(c, node);

401
	if (!node_online(node)) {
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
		/*
		 * Two possibilities here:
		 *
		 * - The CPU is missing memory and no node was created.  In
		 *   that case try picking one from a nearby CPU.
		 *
		 * - The APIC IDs differ from the HyperTransport node IDs
		 *   which the K8 northbridge parsing fills in.  Assume
		 *   they are all increased by a constant offset, but in
		 *   the same order as the HT nodeids.  If that doesn't
		 *   result in a usable node fall back to the path for the
		 *   previous case.
		 *
		 * This workaround operates directly on the mapping between
		 * APIC ID and NUMA node, assuming certain relationship
		 * between APIC ID, HT node ID and NUMA topology.  As going
		 * through CPU mapping may alter the outcome, directly
		 * access __apicid_to_node[].
		 */
421 422 423
		int ht_nodeid = c->initial_apicid;

		if (ht_nodeid >= 0 &&
424 425
		    __apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
			node = __apicid_to_node[ht_nodeid];
426 427 428 429 430 431 432 433
		/* Pick a nearby node */
		if (!node_online(node))
			node = nearby_node(apicid);
	}
	numa_set_node(cpu, node);
#endif
}

434
static void early_init_amd_mc(struct cpuinfo_x86 *c)
435
{
B
Borislav Petkov 已提交
436
#ifdef CONFIG_SMP
437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
	unsigned bits, ecx;

	/* Multi core CPU? */
	if (c->extended_cpuid_level < 0x80000008)
		return;

	ecx = cpuid_ecx(0x80000008);

	c->x86_max_cores = (ecx & 0xff) + 1;

	/* CPU telling us the core id bits shift? */
	bits = (ecx >> 12) & 0xF;

	/* Otherwise recompute */
	if (bits == 0) {
		while ((1 << bits) < c->x86_max_cores)
			bits++;
	}

	c->x86_coreid_bits = bits;
#endif
}

460
static void bsp_init_amd(struct cpuinfo_x86 *c)
461
{
B
Borislav Petkov 已提交
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481

#ifdef CONFIG_X86_64
	if (c->x86 >= 0xf) {
		unsigned long long tseg;

		/*
		 * Split up direct mapping around the TSEG SMM area.
		 * Don't do it for gbpages because there seems very little
		 * benefit in doing so.
		 */
		if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
			unsigned long pfn = tseg >> PAGE_SHIFT;

			printk(KERN_DEBUG "tseg: %010llx\n", tseg);
			if (pfn_range_is_mapped(pfn, pfn + 1))
				set_memory_4k((unsigned long)__va(tseg), 1);
		}
	}
#endif

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
	if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {

		if (c->x86 > 0x10 ||
		    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
			u64 val;

			rdmsrl(MSR_K7_HWCR, val);
			if (!(val & BIT(24)))
				printk(KERN_WARNING FW_BUG "TSC doesn't count "
					"with P0 frequency!\n");
		}
	}

	if (c->x86 == 0x15) {
		unsigned long upperbit;
		u32 cpuid, assoc;

		cpuid	 = cpuid_edx(0x80000005);
		assoc	 = cpuid >> 16 & 0xff;
		upperbit = ((cpuid >> 24) << 10) / assoc;

		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
505 506 507

		/* A random value per boot for bit slice [12:upper_bit) */
		va_align.bits = get_random_int() & va_align.mask;
508 509 510
	}
}

511
static void early_init_amd(struct cpuinfo_x86 *c)
512
{
513 514
	early_init_amd_mc(c);

515 516 517 518 519
	/*
	 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
	 * with P/T states and does not stop in deep C-states
	 */
	if (c->x86_power & (1 << 8)) {
520
		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
521
		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
522
		if (!check_tsc_unstable())
523
			set_sched_clock_stable();
524
	}
525

526 527 528
#ifdef CONFIG_X86_64
	set_cpu_cap(c, X86_FEATURE_SYSCALL32);
#else
529
	/*  Set MTRR capability flag if appropriate */
530 531 532 533 534
	if (c->x86 == 5)
		if (c->x86_model == 13 || c->x86_model == 9 ||
		    (c->x86_model == 8 && c->x86_mask >= 8))
			set_cpu_cap(c, X86_FEATURE_K6_MTRR);
#endif
535
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
536 537 538 539 540 541 542 543 544 545
	/*
	 * ApicID can always be treated as an 8-bit value for AMD APIC versions
	 * >= 0x10, but even old K8s came out of reset with version 0x10. So, we
	 * can safely set X86_FEATURE_EXTD_APICID unconditionally for families
	 * after 16h.
	 */
	if (cpu_has_apic && c->x86 > 0x16) {
		set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
	} else if (cpu_has_apic && c->x86 >= 0xf) {
		/* check CPU config space for extended APIC ID */
546 547 548 549 550 551
		unsigned int val;
		val = read_pci_config(0, 24, 0, 0x68);
		if ((val & ((1 << 17) | (1 << 18))) == ((1 << 17) | (1 << 18)))
			set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
	}
#endif
552

553 554 555 556 557 558 559
	/*
	 * This is only needed to tell the kernel whether to use VMCALL
	 * and VMMCALL.  VMMCALL is never executed except under virt, so
	 * we can set it unconditionally.
	 */
	set_cpu_cap(c, X86_FEATURE_VMMCALL);

560
	/* F16h erratum 793, CVE-2013-6885 */
561 562
	if (c->x86 == 0x16 && c->x86_model <= 0xf)
		msr_set_bit(MSR_AMD64_LS_CFG, 15);
563 564
}

565
static const int amd_erratum_383[];
566
static const int amd_erratum_400[];
567
static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum);
568

B
Borislav Petkov 已提交
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
static void init_amd_k8(struct cpuinfo_x86 *c)
{
	u32 level;
	u64 value;

	/* On C+ stepping K8 rep microcode works well for copy/memset */
	level = cpuid_eax(1);
	if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);

	/*
	 * Some BIOSes incorrectly force this feature, but only K8 revision D
	 * (model = 0x14) and later actually support it.
	 * (AMD Erratum #110, docId: 25759).
	 */
	if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
		clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
		if (!rdmsrl_amd_safe(0xc001100d, &value)) {
			value &= ~BIT_64(32);
			wrmsrl_amd_safe(0xc001100d, value);
		}
	}

	if (!c->x86_model_id[0])
		strcpy(c->x86_model_id, "Hammer");
594 595 596 597 598 599 600 601 602 603 604

#ifdef CONFIG_SMP
	/*
	 * Disable TLB flush filter by setting HWCR.FFDIS on K8
	 * bit 6 of msr C001_0015
	 *
	 * Errata 63 for SH-B3 steppings
	 * Errata 122 for all steppings (F+ have it disabled by default)
	 */
	msr_set_bit(MSR_K7_HWCR, 6);
#endif
B
Borislav Petkov 已提交
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
}

static void init_amd_gh(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
	/* do this for boot cpu */
	if (c == &boot_cpu_data)
		check_enable_amd_mmconf_dmi();

	fam10h_check_enable_mmcfg();
#endif

	/*
	 * Disable GART TLB Walk Errors on Fam10h. We do this here because this
	 * is always needed when GART is enabled, even in a kernel which has no
	 * MCE support built in. BIOS should disable GartTlbWlk Errors already.
	 * If it doesn't, we do it here as suggested by the BKDG.
	 *
	 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
	 */
	msr_set_bit(MSR_AMD64_MCx_MASK(4), 10);

	/*
	 * On family 10h BIOS may not have properly enabled WC+ support, causing
	 * it to be converted to CD memtype. This may result in performance
	 * degradation for certain nested-paging guests. Prevent this conversion
	 * by clearing bit 24 in MSR_AMD64_BU_CFG2.
	 *
	 * NOTE: we want to use the _safe accessors so as not to #GP kvm
	 * guests on older kvm hosts.
	 */
	msr_clear_bit(MSR_AMD64_BU_CFG2, 24);

	if (cpu_has_amd_erratum(c, amd_erratum_383))
		set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
}

static void init_amd_bd(struct cpuinfo_x86 *c)
{
	u64 value;

	/* re-enable TopologyExtensions if switched off by BIOS */
	if ((c->x86_model >= 0x10) && (c->x86_model <= 0x1f) &&
	    !cpu_has(c, X86_FEATURE_TOPOEXT)) {

		if (msr_set_bit(0xc0011005, 54) > 0) {
			rdmsrl(0xc0011005, value);
			if (value & BIT_64(54)) {
				set_cpu_cap(c, X86_FEATURE_TOPOEXT);
				pr_info(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n");
			}
		}
	}

	/*
	 * The way access filter has a performance penalty on some workloads.
	 * Disable it on the affected CPUs.
	 */
	if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
		if (!rdmsrl_safe(0xc0011021, &value) && !(value & 0x1E)) {
			value |= 0x1E;
			wrmsrl_safe(0xc0011021, value);
		}
	}
}

671
static void init_amd(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
672
{
673
	u32 dummy;
674

675 676
	early_init_amd(c);

677 678
	/*
	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
679
	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
680
	 */
681
	clear_cpu_cap(c, 0*32+31);
682

683
	if (c->x86 >= 0x10)
684
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);
685 686 687

	/* get apicid instead of initial apic id from cpuid */
	c->apicid = hard_smp_processor_id();
688 689 690 691

	/* K6s reports MCEs but don't actually have all the MSRs */
	if (c->x86 < 6)
		clear_cpu_cap(c, X86_FEATURE_MCE);
B
Borislav Petkov 已提交
692 693 694 695 696 697 698 699 700

	switch (c->x86) {
	case 4:    init_amd_k5(c); break;
	case 5:    init_amd_k6(c); break;
	case 6:	   init_amd_k7(c); break;
	case 0xf:  init_amd_k8(c); break;
	case 0x10: init_amd_gh(c); break;
	case 0x15: init_amd_bd(c); break;
	}
701

702
	/* Enable workaround for FXSAVE leak */
703
	if (c->x86 >= 6)
704
		set_cpu_bug(c, X86_BUG_FXSAVE_LEAK);
L
Linus Torvalds 已提交
705

706
	cpu_detect_cache_sizes(c);
707

708
	/* Multi core CPU? */
709
	if (c->extended_cpuid_level >= 0x80000008) {
710
		amd_detect_cmp(c);
711 712
		srat_detect_node(c);
	}
713

714
#ifdef CONFIG_X86_32
715
	detect_ht(c);
716
#endif
717

718
	init_amd_cacheinfo(c);
719

720
	if (c->x86 >= 0xf)
721
		set_cpu_cap(c, X86_FEATURE_K8);
722

723 724
	if (cpu_has_xmm2) {
		/* MFENCE stops RDTSC speculation */
725
		set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
726
	}
727

728 729 730 731 732
	/*
	 * Family 0x12 and above processors have APIC timer
	 * running in deep C states.
	 */
	if (c->x86 > 0x11)
733
		set_cpu_cap(c, X86_FEATURE_ARAT);
734

735
	if (cpu_has_amd_erratum(c, amd_erratum_400))
736 737
		set_cpu_bug(c, X86_BUG_AMD_APIC_C1E);

738
	rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
739 740 741 742 743

	/* 3DNow or LM implies PREFETCHW */
	if (!cpu_has(c, X86_FEATURE_3DNOWPREFETCH))
		if (cpu_has(c, X86_FEATURE_3DNOW) || cpu_has(c, X86_FEATURE_LM))
			set_cpu_cap(c, X86_FEATURE_3DNOWPREFETCH);
744 745 746

	/* AMD CPUs don't reset SS attributes on SYSRET */
	set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS);
L
Linus Torvalds 已提交
747 748
}

749
#ifdef CONFIG_X86_32
750
static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
L
Linus Torvalds 已提交
751 752 753
{
	/* AMD errata T13 (order #21922) */
	if ((c->x86 == 6)) {
A
Alan Cox 已提交
754 755
		/* Duron Rev A0 */
		if (c->x86_model == 3 && c->x86_mask == 0)
L
Linus Torvalds 已提交
756
			size = 64;
A
Alan Cox 已提交
757
		/* Tbird rev A1/A2 */
L
Linus Torvalds 已提交
758
		if (c->x86_model == 4 &&
A
Alan Cox 已提交
759
			(c->x86_mask == 0 || c->x86_mask == 1))
L
Linus Torvalds 已提交
760 761 762 763
			size = 256;
	}
	return size;
}
764
#endif
L
Linus Torvalds 已提交
765

766
static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
{
	u32 ebx, eax, ecx, edx;
	u16 mask = 0xfff;

	if (c->x86 < 0xf)
		return;

	if (c->extended_cpuid_level < 0x80000006)
		return;

	cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

	tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
	tlb_lli_4k[ENTRIES] = ebx & mask;

	/*
	 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
	 * characteristics from the CPUID function 0x80000005 instead.
	 */
	if (c->x86 == 0xf) {
		cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
		mask = 0xff;
	}

	/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
792 793 794
	if (!((eax >> 16) & mask))
		tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;
	else
795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
		tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;

	/* a 4M entry uses two 2M entries */
	tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;

	/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
	if (!(eax & mask)) {
		/* Erratum 658 */
		if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
			tlb_lli_2m[ENTRIES] = 1024;
		} else {
			cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
			tlb_lli_2m[ENTRIES] = eax & 0xff;
		}
	} else
		tlb_lli_2m[ENTRIES] = eax & mask;

	tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
}

815
static const struct cpu_dev amd_cpu_dev = {
L
Linus Torvalds 已提交
816
	.c_vendor	= "AMD",
817
	.c_ident	= { "AuthenticAMD" },
818
#ifdef CONFIG_X86_32
819 820
	.legacy_models = {
		{ .family = 4, .model_names =
L
Linus Torvalds 已提交
821 822 823
		  {
			  [3] = "486 DX/2",
			  [7] = "486 DX/2-WB",
824 825
			  [8] = "486 DX/4",
			  [9] = "486 DX/4-WB",
L
Linus Torvalds 已提交
826
			  [14] = "Am5x86-WT",
827
			  [15] = "Am5x86-WB"
L
Linus Torvalds 已提交
828 829 830
		  }
		},
	},
831
	.legacy_cache_size = amd_size_cache,
832
#endif
833
	.c_early_init   = early_init_amd,
834
	.c_detect_tlb	= cpu_detect_tlb_amd,
835
	.c_bsp_init	= bsp_init_amd,
L
Linus Torvalds 已提交
836
	.c_init		= init_amd,
Y
Yinghai Lu 已提交
837
	.c_x86_vendor	= X86_VENDOR_AMD,
L
Linus Torvalds 已提交
838 839
};

Y
Yinghai Lu 已提交
840
cpu_dev_register(amd_cpu_dev);
841 842 843 844 845 846 847 848

/*
 * AMD errata checking
 *
 * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
 * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
 * have an OSVW id assigned, which it takes as first argument. Both take a
 * variable number of family-specific model-stepping ranges created by
849
 * AMD_MODEL_RANGE().
850 851 852 853 854 855 856 857 858
 *
 * Example:
 *
 * const int amd_erratum_319[] =
 *	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
 *			   AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
 *			   AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
 */

859 860 861 862 863 864 865 866 867
#define AMD_LEGACY_ERRATUM(...)		{ -1, __VA_ARGS__, 0 }
#define AMD_OSVW_ERRATUM(osvw_id, ...)	{ osvw_id, __VA_ARGS__, 0 }
#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
	((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
#define AMD_MODEL_RANGE_FAMILY(range)	(((range) >> 24) & 0xff)
#define AMD_MODEL_RANGE_START(range)	(((range) >> 12) & 0xfff)
#define AMD_MODEL_RANGE_END(range)	((range) & 0xfff)

static const int amd_erratum_400[] =
868
	AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
869 870
			    AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));

871
static const int amd_erratum_383[] =
872
	AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
873

874 875

static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
{
	int osvw_id = *erratum++;
	u32 range;
	u32 ms;

	if (osvw_id >= 0 && osvw_id < 65536 &&
	    cpu_has(cpu, X86_FEATURE_OSVW)) {
		u64 osvw_len;

		rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
		if (osvw_id < osvw_len) {
			u64 osvw_bits;

			rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
			    osvw_bits);
			return osvw_bits & (1ULL << (osvw_id & 0x3f));
		}
	}

	/* OSVW unavailable or ID unknown, match family-model-stepping range */
896
	ms = (cpu->x86_model << 4) | cpu->x86_mask;
897 898 899 900 901 902 903 904
	while ((range = *erratum++))
		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
		    (ms >= AMD_MODEL_RANGE_START(range)) &&
		    (ms <= AMD_MODEL_RANGE_END(range)))
			return true;

	return false;
}
905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923

void set_dr_addr_mask(unsigned long mask, int dr)
{
	if (!cpu_has_bpext)
		return;

	switch (dr) {
	case 0:
		wrmsr(MSR_F16H_DR0_ADDR_MASK, mask, 0);
		break;
	case 1:
	case 2:
	case 3:
		wrmsr(MSR_F16H_DR1_ADDR_MASK - 1 + dr, mask, 0);
		break;
	default:
		break;
	}
}