amd.c 21.6 KB
Newer Older
1
#include <linux/export.h>
L
Linus Torvalds 已提交
2 3
#include <linux/init.h>
#include <linux/bitops.h>
4
#include <linux/elf.h>
L
Linus Torvalds 已提交
5
#include <linux/mm.h>
Y
Yinghai Lu 已提交
6

A
Alan Cox 已提交
7
#include <linux/io.h>
8
#include <linux/sched.h>
L
Linus Torvalds 已提交
9
#include <asm/processor.h>
10
#include <asm/apic.h>
11
#include <asm/cpu.h>
12
#include <asm/pci-direct.h>
L
Linus Torvalds 已提交
13

Y
Yinghai Lu 已提交
14 15 16 17 18 19
#ifdef CONFIG_X86_64
# include <asm/numa_64.h>
# include <asm/mmconfig.h>
# include <asm/cacheflush.h>
#endif

L
Linus Torvalds 已提交
20 21
#include "cpu.h"

22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
{
	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
	u32 gprs[8] = { 0 };
	int err;

	WARN_ONCE((c->x86 != 0xf), "%s should only be used on K8!\n", __func__);

	gprs[1] = msr;
	gprs[7] = 0x9c5a203a;

	err = rdmsr_safe_regs(gprs);

	*p = gprs[0] | ((u64)gprs[2] << 32);

	return err;
}

static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
{
	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
	u32 gprs[8] = { 0 };

	WARN_ONCE((c->x86 != 0xf), "%s should only be used on K8!\n", __func__);

	gprs[0] = (u32)val;
	gprs[1] = msr;
	gprs[2] = val >> 32;
	gprs[7] = 0x9c5a203a;

	return wrmsr_safe_regs(gprs);
}

55
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
56 57 58 59 60 61
/*
 *	B step AMD K6 before B 9730xxxx have hardware bugs that can cause
 *	misexecution of code under Linux. Owners of such processors should
 *	contact AMD for precise details and a CPU swap.
 *
 *	See	http://www.multimania.com/poulot/k6bug.html
62 63
 *	and	section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6"
 *		(Publication # 21266  Issue Date: August 1998)
L
Linus Torvalds 已提交
64 65 66 67 68
 *
 *	The following test is erm.. interesting. AMD neglected to up
 *	the chip setting when fixing the bug but they also tweaked some
 *	performance at the same time..
 */
69

L
Linus Torvalds 已提交
70 71 72
extern void vide(void);
__asm__(".align 4\nvide: ret");

73 74 75 76 77 78 79 80 81 82 83 84
static void __cpuinit init_amd_k5(struct cpuinfo_x86 *c)
{
/*
 * General Systems BIOSen alias the cpu frequency registers
 * of the Elan at 0x000df000. Unfortuantly, one of the Linux
 * drivers subsequently pokes it, and changes the CPU speed.
 * Workaround : Remove the unneeded alias.
 */
#define CBAR		(0xfffc) /* Configuration Base Address  (32-bit) */
#define CBAR_ENB	(0x80000000)
#define CBAR_KEY	(0X000000CB)
	if (c->x86_model == 9 || c->x86_model == 10) {
A
Alan Cox 已提交
85 86
		if (inl(CBAR) & CBAR_ENB)
			outl(0 | CBAR_KEY, CBAR);
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
	}
}


static void __cpuinit init_amd_k6(struct cpuinfo_x86 *c)
{
	u32 l, h;
	int mbytes = num_physpages >> (20-PAGE_SHIFT);

	if (c->x86_model < 6) {
		/* Based on AMD doc 20734R - June 2000 */
		if (c->x86_model == 0) {
			clear_cpu_cap(c, X86_FEATURE_APIC);
			set_cpu_cap(c, X86_FEATURE_PGE);
		}
		return;
	}

	if (c->x86_model == 6 && c->x86_mask == 1) {
		const int K6_BUG_LOOP = 1000000;
		int n;
		void (*f_vide)(void);
		unsigned long d, d2;

		printk(KERN_INFO "AMD K6 stepping B detected - ");

		/*
		 * It looks like AMD fixed the 2.6.2 bug and improved indirect
		 * calls at the same time.
		 */

		n = K6_BUG_LOOP;
		f_vide = vide;
		rdtscl(d);
		while (n--)
			f_vide();
		rdtscl(d2);
		d = d2-d;

		if (d > 20*K6_BUG_LOOP)
A
Alan Cox 已提交
127 128
			printk(KERN_CONT
				"system stability may be impaired when more than 32 MB are used.\n");
129
		else
A
Alan Cox 已提交
130
			printk(KERN_CONT "probably OK (after B9730xxxx).\n");
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
	}

	/* K6 with old style WHCR */
	if (c->x86_model < 8 ||
	   (c->x86_model == 8 && c->x86_mask < 8)) {
		/* We can only write allocate on the low 508Mb */
		if (mbytes > 508)
			mbytes = 508;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0x0000FFFF) == 0) {
			unsigned long flags;
			l = (1<<0)|((mbytes/4)<<1);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling old style K6 write allocation for %d Mb\n",
				mbytes);
		}
		return;
	}

	if ((c->x86_model == 8 && c->x86_mask > 7) ||
	     c->x86_model == 9 || c->x86_model == 13) {
		/* The more serious chips .. */

		if (mbytes > 4092)
			mbytes = 4092;

		rdmsr(MSR_K6_WHCR, l, h);
		if ((l&0xFFFF0000) == 0) {
			unsigned long flags;
			l = ((mbytes>>2)<<22)|(1<<16);
			local_irq_save(flags);
			wbinvd();
			wrmsr(MSR_K6_WHCR, l, h);
			local_irq_restore(flags);
			printk(KERN_INFO "Enabling new style K6 write allocation for %d Mb\n",
				mbytes);
		}

		return;
	}

	if (c->x86_model == 10) {
		/* AMD Geode LX is model 10 */
		/* placeholder for any needed mods */
		return;
	}
}

183 184 185
static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c)
{
	/* calling is from identify_secondary_cpu() ? */
186
	if (!c->cpu_index)
187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
		return;

	/*
	 * Certain Athlons might work (for various values of 'work') in SMP
	 * but they are not certified as MP capable.
	 */
	/* Athlon 660/661 is valid. */
	if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
	    (c->x86_mask == 1)))
		goto valid_k7;

	/* Duron 670 is valid */
	if ((c->x86_model == 7) && (c->x86_mask == 0))
		goto valid_k7;

	/*
	 * Athlon 662, Duron 671, and Athlon >model 7 have capability
	 * bit. It's worth noting that the A5 stepping (662) of some
	 * Athlon XP's have the MP bit set.
	 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
	 * more.
	 */
	if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
	    ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
	     (c->x86_model > 7))
		if (cpu_has_mp)
			goto valid_k7;

	/* If we get here, not a certified SMP capable AMD system. */

	/*
	 * Don't taint if we are running SMP kernel on a single non-MP
	 * approved Athlon
	 */
	WARN_ONCE(1, "WARNING: This combination of AMD"
222
		" processors is not suitable for SMP.\n");
223 224 225 226 227 228 229
	if (!test_taint(TAINT_UNSAFE_SMP))
		add_taint(TAINT_UNSAFE_SMP);

valid_k7:
	;
}

230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c)
{
	u32 l, h;

	/*
	 * Bit 15 of Athlon specific MSR 15, needs to be 0
	 * to enable SSE on Palomino/Morgan/Barton CPU's.
	 * If the BIOS didn't enable it already, enable it here.
	 */
	if (c->x86_model >= 6 && c->x86_model <= 10) {
		if (!cpu_has(c, X86_FEATURE_XMM)) {
			printk(KERN_INFO "Enabling disabled K7/SSE Support.\n");
			rdmsr(MSR_K7_HWCR, l, h);
			l &= ~0x00008000;
			wrmsr(MSR_K7_HWCR, l, h);
			set_cpu_cap(c, X86_FEATURE_XMM);
		}
	}

	/*
	 * It's been determined by AMD that Athlons since model 8 stepping 1
	 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
	 * As per AMD technical note 27212 0.2
	 */
	if ((c->x86_model == 8 && c->x86_mask >= 1) || (c->x86_model > 8)) {
		rdmsr(MSR_K7_CLK_CTL, l, h);
		if ((l & 0xfff00000) != 0x20000000) {
A
Alan Cox 已提交
257 258 259
			printk(KERN_INFO
			    "CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
					l, ((l & 0x000fffff)|0x20000000));
260 261 262 263 264
			wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
		}
	}

	set_cpu_cap(c, X86_FEATURE_K7);
265 266

	amd_k7_smp_check(c);
267
}
268 269
#endif

270
#ifdef CONFIG_NUMA
271 272 273 274
/*
 * To workaround broken NUMA config.  Read the comment in
 * srat_detect_node().
 */
275 276 277 278 279
static int __cpuinit nearby_node(int apicid)
{
	int i, node;

	for (i = apicid - 1; i >= 0; i--) {
280
		node = __apicid_to_node[i];
281 282 283 284
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
285
		node = __apicid_to_node[i];
286 287 288 289 290 291
		if (node != NUMA_NO_NODE && node_online(node))
			return node;
	}
	return first_node(node_online_map); /* Shouldn't happen */
}
#endif
292

293
/*
294 295 296
 * Fixup core topology information for
 * (1) AMD multi-node processors
 *     Assumption: Number of cores in each internal node is the same.
297
 * (2) AMD processors supporting compute units
298 299
 */
#ifdef CONFIG_X86_HT
300
static void __cpuinit amd_get_topology(struct cpuinfo_x86 *c)
301
{
302
	u32 nodes, cores_per_cu = 1;
303
	u8 node_id;
304 305
	int cpu = smp_processor_id();

306
	/* get information required for multi-node processors */
A
Andreas Herrmann 已提交
307
	if (cpu_has_topoext) {
308 309 310 311 312 313 314 315 316
		u32 eax, ebx, ecx, edx;

		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
		nodes = ((ecx >> 8) & 7) + 1;
		node_id = ecx & 7;

		/* get compute unit information */
		smp_num_siblings = ((ebx >> 8) & 3) + 1;
		c->compute_unit_id = ebx & 0xff;
317
		cores_per_cu += ((ebx >> 8) & 3);
318
	} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
319 320
		u64 value;

321 322 323 324
		rdmsrl(MSR_FAM10H_NODE_ID, value);
		nodes = ((value >> 3) & 7) + 1;
		node_id = value & 7;
	} else
325 326
		return;

327 328
	/* fixup multi-node processor information */
	if (nodes > 1) {
329
		u32 cores_per_node;
330
		u32 cus_per_node;
331

332 333
		set_cpu_cap(c, X86_FEATURE_AMD_DCM);
		cores_per_node = c->x86_max_cores / nodes;
334
		cus_per_node = cores_per_node / cores_per_cu;
335

336 337
		/* store NodeID, use llc_shared_map to store sibling info */
		per_cpu(cpu_llc_id, cpu) = node_id;
338

339
		/* core id has to be in the [0 .. cores_per_node - 1] range */
340 341
		c->cpu_core_id %= cores_per_node;
		c->compute_unit_id %= cus_per_node;
342
	}
343 344 345
}
#endif

346 347 348 349 350 351 352 353
/*
 * On a AMD dual core setup the lower bits of the APIC id distingush the cores.
 * Assumes number of cores is a power of two.
 */
static void __cpuinit amd_detect_cmp(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_HT
	unsigned bits;
354
	int cpu = smp_processor_id();
355 356 357 358 359 360

	bits = c->x86_coreid_bits;
	/* Low order bits define the core id (index of core in socket) */
	c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
	/* Convert the initial APIC ID into the socket ID */
	c->phys_proc_id = c->initial_apicid >> bits;
361 362
	/* use socket ID also for last level cache */
	per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
363
	amd_get_topology(c);
364 365 366
#endif
}

367 368 369 370 371 372 373 374 375 376
int amd_get_nb_id(int cpu)
{
	int id = 0;
#ifdef CONFIG_SMP
	id = per_cpu(cpu_llc_id, cpu);
#endif
	return id;
}
EXPORT_SYMBOL_GPL(amd_get_nb_id);

377 378
static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c)
{
379
#ifdef CONFIG_NUMA
380 381
	int cpu = smp_processor_id();
	int node;
382
	unsigned apicid = c->apicid;
383

384 385 386
	node = numa_cpu_node(cpu);
	if (node == NUMA_NO_NODE)
		node = per_cpu(cpu_llc_id, cpu);
387

388
	/*
389 390 391
	 * On multi-fabric platform (e.g. Numascale NumaChip) a
	 * platform-specific handler needs to be called to fixup some
	 * IDs of the CPU.
392
	 */
393
	if (x86_cpuinit.fixup_cpu_id)
394 395
		x86_cpuinit.fixup_cpu_id(c, node);

396
	if (!node_online(node)) {
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
		/*
		 * Two possibilities here:
		 *
		 * - The CPU is missing memory and no node was created.  In
		 *   that case try picking one from a nearby CPU.
		 *
		 * - The APIC IDs differ from the HyperTransport node IDs
		 *   which the K8 northbridge parsing fills in.  Assume
		 *   they are all increased by a constant offset, but in
		 *   the same order as the HT nodeids.  If that doesn't
		 *   result in a usable node fall back to the path for the
		 *   previous case.
		 *
		 * This workaround operates directly on the mapping between
		 * APIC ID and NUMA node, assuming certain relationship
		 * between APIC ID, HT node ID and NUMA topology.  As going
		 * through CPU mapping may alter the outcome, directly
		 * access __apicid_to_node[].
		 */
416 417 418
		int ht_nodeid = c->initial_apicid;

		if (ht_nodeid >= 0 &&
419 420
		    __apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
			node = __apicid_to_node[ht_nodeid];
421 422 423 424 425 426 427 428
		/* Pick a nearby node */
		if (!node_online(node))
			node = nearby_node(apicid);
	}
	numa_set_node(cpu, node);
#endif
}

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_HT
	unsigned bits, ecx;

	/* Multi core CPU? */
	if (c->extended_cpuid_level < 0x80000008)
		return;

	ecx = cpuid_ecx(0x80000008);

	c->x86_max_cores = (ecx & 0xff) + 1;

	/* CPU telling us the core id bits shift? */
	bits = (ecx >> 12) & 0xF;

	/* Otherwise recompute */
	if (bits == 0) {
		while ((1 << bits) < c->x86_max_cores)
			bits++;
	}

	c->x86_coreid_bits = bits;
#endif
}

455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
static void __cpuinit bsp_init_amd(struct cpuinfo_x86 *c)
{
	if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {

		if (c->x86 > 0x10 ||
		    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
			u64 val;

			rdmsrl(MSR_K7_HWCR, val);
			if (!(val & BIT(24)))
				printk(KERN_WARNING FW_BUG "TSC doesn't count "
					"with P0 frequency!\n");
		}
	}

	if (c->x86 == 0x15) {
		unsigned long upperbit;
		u32 cpuid, assoc;

		cpuid	 = cpuid_edx(0x80000005);
		assoc	 = cpuid >> 16 & 0xff;
		upperbit = ((cpuid >> 24) << 10) / assoc;

		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
	}
}

483
static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
484
{
485 486
	early_init_amd_mc(c);

487 488 489 490 491
	/*
	 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
	 * with P/T states and does not stop in deep C-states
	 */
	if (c->x86_power & (1 << 8)) {
492
		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
493
		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
494 495
		if (!check_tsc_unstable())
			sched_clock_stable = 1;
496
	}
497

498 499 500
#ifdef CONFIG_X86_64
	set_cpu_cap(c, X86_FEATURE_SYSCALL32);
#else
501
	/*  Set MTRR capability flag if appropriate */
502 503 504 505 506
	if (c->x86 == 5)
		if (c->x86_model == 13 || c->x86_model == 9 ||
		    (c->x86_model == 8 && c->x86_mask >= 8))
			set_cpu_cap(c, X86_FEATURE_K6_MTRR);
#endif
507 508
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
	/* check CPU config space for extended APIC ID */
509
	if (cpu_has_apic && c->x86 >= 0xf) {
510 511 512 513 514 515
		unsigned int val;
		val = read_pci_config(0, 24, 0, 0x68);
		if ((val & ((1 << 17) | (1 << 18))) == ((1 << 17) | (1 << 18)))
			set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
	}
#endif
516 517
}

518
static void __cpuinit init_amd(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
519
{
520 521
	u32 dummy;

522
#ifdef CONFIG_SMP
523
	unsigned long long value;
524

525 526
	/*
	 * Disable TLB flush filter by setting HWCR.FFDIS on K8
527 528 529 530 531
	 * bit 6 of msr C001_0015
	 *
	 * Errata 63 for SH-B3 steppings
	 * Errata 122 for all steppings (F+ have it disabled by default)
	 */
532
	if (c->x86 == 0xf) {
533 534 535 536 537 538
		rdmsrl(MSR_K7_HWCR, value);
		value |= 1 << 6;
		wrmsrl(MSR_K7_HWCR, value);
	}
#endif

539 540
	early_init_amd(c);

541 542
	/*
	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
543
	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
544
	 */
545
	clear_cpu_cap(c, 0*32+31);
546

547 548 549 550 551 552
#ifdef CONFIG_X86_64
	/* On C+ stepping K8 rep microcode works well for copy/memset */
	if (c->x86 == 0xf) {
		u32 level;

		level = cpuid_eax(1);
A
Alan Cox 已提交
553
		if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
554
			set_cpu_cap(c, X86_FEATURE_REP_GOOD);
555 556 557 558

		/*
		 * Some BIOSes incorrectly force this feature, but only K8
		 * revision D (model = 0x14) and later actually support it.
559
		 * (AMD Erratum #110, docId: 25759).
560
		 */
561 562 563
		if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
			u64 val;

564
			clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
565 566 567 568 569 570
			if (!rdmsrl_amd_safe(0xc001100d, &val)) {
				val &= ~(1ULL << 32);
				wrmsrl_amd_safe(0xc001100d, val);
			}
		}

571
	}
572
	if (c->x86 >= 0x10)
573
		set_cpu_cap(c, X86_FEATURE_REP_GOOD);
574 575 576

	/* get apicid instead of initial apic id from cpuid */
	c->apicid = hard_smp_processor_id();
577 578 579 580 581 582 583 584
#else

	/*
	 *	FIXME: We should handle the K5 here. Set up the write
	 *	range and also turn on MSR 83 bits 4 and 31 (write alloc,
	 *	no bus pipeline)
	 */

585 586
	switch (c->x86) {
	case 4:
587 588
		init_amd_k5(c);
		break;
589
	case 5:
590
		init_amd_k6(c);
L
Linus Torvalds 已提交
591
		break;
592 593
	case 6: /* An Athlon/Duron */
		init_amd_k7(c);
L
Linus Torvalds 已提交
594 595
		break;
	}
596 597 598 599

	/* K6s reports MCEs but don't actually have all the MSRs */
	if (c->x86 < 6)
		clear_cpu_cap(c, X86_FEATURE_MCE);
600
#endif
601

602
	/* Enable workaround for FXSAVE leak */
603
	if (c->x86 >= 6)
604
		set_cpu_cap(c, X86_FEATURE_FXSAVE_LEAK);
L
Linus Torvalds 已提交
605

606 607 608 609 610 611 612 613 614
	if (!c->x86_model_id[0]) {
		switch (c->x86) {
		case 0xf:
			/* Should distinguish Models here, but this is only
			   a fallback anyways. */
			strcpy(c->x86_model_id, "Hammer");
			break;
		}
	}
615

616 617 618 619 620 621
	/* re-enable TopologyExtensions if switched off by BIOS */
	if ((c->x86 == 0x15) &&
	    (c->x86_model >= 0x10) && (c->x86_model <= 0x1f) &&
	    !cpu_has(c, X86_FEATURE_TOPOEXT)) {
		u64 val;

622
		if (!rdmsrl_safe(0xc0011005, &val)) {
623
			val |= 1ULL << 54;
624
			wrmsrl_safe(0xc0011005, val);
625 626 627 628 629 630 631 632 633
			rdmsrl(0xc0011005, val);
			if (val & (1ULL << 54)) {
				set_cpu_cap(c, X86_FEATURE_TOPOEXT);
				printk(KERN_INFO FW_INFO "CPU: Re-enabling "
				  "disabled Topology Extensions Support\n");
			}
		}
	}

634
	cpu_detect_cache_sizes(c);
635

636
	/* Multi core CPU? */
637
	if (c->extended_cpuid_level >= 0x80000008) {
638
		amd_detect_cmp(c);
639 640
		srat_detect_node(c);
	}
641

642
#ifdef CONFIG_X86_32
643
	detect_ht(c);
644
#endif
645

646
	if (c->extended_cpuid_level >= 0x80000006) {
647
		if (cpuid_edx(0x80000006) & 0xf000)
648 649 650 651
			num_cache_leaves = 4;
		else
			num_cache_leaves = 3;
	}
652

653
	if (c->x86 >= 0xf)
654
		set_cpu_cap(c, X86_FEATURE_K8);
655

656 657
	if (cpu_has_xmm2) {
		/* MFENCE stops RDTSC speculation */
658
		set_cpu_cap(c, X86_FEATURE_MFENCE_RDTSC);
659
	}
660 661 662 663 664 665 666 667 668 669

#ifdef CONFIG_X86_64
	if (c->x86 == 0x10) {
		/* do this for boot cpu */
		if (c == &boot_cpu_data)
			check_enable_amd_mmconf_dmi();

		fam10h_check_enable_mmcfg();
	}

670
	if (c == &boot_cpu_data && c->x86 >= 0xf) {
671 672 673 674 675 676 677 678
		unsigned long long tseg;

		/*
		 * Split up direct mapping around the TSEG SMM area.
		 * Don't do it for gbpages because there seems very little
		 * benefit in doing so.
		 */
		if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) {
A
Alan Cox 已提交
679 680
			printk(KERN_DEBUG "tseg: %010llx\n", tseg);
			if ((tseg>>PMD_SHIFT) <
681
				(max_low_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) ||
A
Alan Cox 已提交
682
				((tseg>>PMD_SHIFT) <
683
				(max_pfn_mapped>>(PMD_SHIFT-PAGE_SHIFT)) &&
A
Alan Cox 已提交
684 685
				(tseg>>PMD_SHIFT) >= (1ULL<<(32 - PMD_SHIFT))))
				set_memory_4k((unsigned long)__va(tseg), 1);
686 687 688
		}
	}
#endif
689

690 691 692 693 694
	/*
	 * Family 0x12 and above processors have APIC timer
	 * running in deep C states.
	 */
	if (c->x86 > 0x11)
695
		set_cpu_cap(c, X86_FEATURE_ARAT);
696 697 698 699 700 701 702 703 704 705 706 707 708 709

	/*
	 * Disable GART TLB Walk Errors on Fam10h. We do this here
	 * because this is always needed when GART is enabled, even in a
	 * kernel which has no MCE support built in.
	 */
	if (c->x86 == 0x10) {
		/*
		 * BIOS should disable GartTlbWlk Errors themself. If
		 * it doesn't do it here as suggested by the BKDG.
		 *
		 * Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
		 */
		u64 mask;
710
		int err;
711

712 713 714
		err = rdmsrl_safe(MSR_AMD64_MCx_MASK(4), &mask);
		if (err == 0) {
			mask |= (1 << 10);
715
			wrmsrl_safe(MSR_AMD64_MCx_MASK(4), mask);
716
		}
717
	}
718 719

	rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
L
Linus Torvalds 已提交
720 721
}

722
#ifdef CONFIG_X86_32
A
Alan Cox 已提交
723 724
static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c,
							unsigned int size)
L
Linus Torvalds 已提交
725 726 727
{
	/* AMD errata T13 (order #21922) */
	if ((c->x86 == 6)) {
A
Alan Cox 已提交
728 729
		/* Duron Rev A0 */
		if (c->x86_model == 3 && c->x86_mask == 0)
L
Linus Torvalds 已提交
730
			size = 64;
A
Alan Cox 已提交
731
		/* Tbird rev A1/A2 */
L
Linus Torvalds 已提交
732
		if (c->x86_model == 4 &&
A
Alan Cox 已提交
733
			(c->x86_mask == 0 || c->x86_mask == 1))
L
Linus Torvalds 已提交
734 735 736 737
			size = 256;
	}
	return size;
}
738
#endif
L
Linus Torvalds 已提交
739

740 741 742 743 744 745 746 747 748 749 750
static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c)
{
	if (!cpu_has_invlpg)
		return;

	tlb_flushall_shift = 5;

	if (c->x86 <= 0x11)
		tlb_flushall_shift = 4;
}

751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
{
	u32 ebx, eax, ecx, edx;
	u16 mask = 0xfff;

	if (c->x86 < 0xf)
		return;

	if (c->extended_cpuid_level < 0x80000006)
		return;

	cpuid(0x80000006, &eax, &ebx, &ecx, &edx);

	tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
	tlb_lli_4k[ENTRIES] = ebx & mask;

	/*
	 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
	 * characteristics from the CPUID function 0x80000005 instead.
	 */
	if (c->x86 == 0xf) {
		cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
		mask = 0xff;
	}

	/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
	if (!((eax >> 16) & mask)) {
		u32 a, b, c, d;

		cpuid(0x80000005, &a, &b, &c, &d);
		tlb_lld_2m[ENTRIES] = (a >> 16) & 0xff;
	} else {
		tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
	}

	/* a 4M entry uses two 2M entries */
	tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;

	/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
	if (!(eax & mask)) {
		/* Erratum 658 */
		if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
			tlb_lli_2m[ENTRIES] = 1024;
		} else {
			cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
			tlb_lli_2m[ENTRIES] = eax & 0xff;
		}
	} else
		tlb_lli_2m[ENTRIES] = eax & mask;

	tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
802 803

	cpu_set_tlb_flushall_shift(c);
804 805
}

806
static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
L
Linus Torvalds 已提交
807
	.c_vendor	= "AMD",
808
	.c_ident	= { "AuthenticAMD" },
809
#ifdef CONFIG_X86_32
L
Linus Torvalds 已提交
810 811 812 813 814
	.c_models = {
		{ .vendor = X86_VENDOR_AMD, .family = 4, .model_names =
		  {
			  [3] = "486 DX/2",
			  [7] = "486 DX/2-WB",
815 816
			  [8] = "486 DX/4",
			  [9] = "486 DX/4-WB",
L
Linus Torvalds 已提交
817
			  [14] = "Am5x86-WT",
818
			  [15] = "Am5x86-WB"
L
Linus Torvalds 已提交
819 820 821
		  }
		},
	},
822 823
	.c_size_cache	= amd_size_cache,
#endif
824
	.c_early_init   = early_init_amd,
825
	.c_detect_tlb	= cpu_detect_tlb_amd,
826
	.c_bsp_init	= bsp_init_amd,
L
Linus Torvalds 已提交
827
	.c_init		= init_amd,
Y
Yinghai Lu 已提交
828
	.c_x86_vendor	= X86_VENDOR_AMD,
L
Linus Torvalds 已提交
829 830
};

Y
Yinghai Lu 已提交
831
cpu_dev_register(amd_cpu_dev);
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850

/*
 * AMD errata checking
 *
 * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
 * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
 * have an OSVW id assigned, which it takes as first argument. Both take a
 * variable number of family-specific model-stepping ranges created by
 * AMD_MODEL_RANGE(). Each erratum also has to be declared as extern const
 * int[] in arch/x86/include/asm/processor.h.
 *
 * Example:
 *
 * const int amd_erratum_319[] =
 *	AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
 *			   AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
 *			   AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
 */

851
const int amd_erratum_400[] =
852
	AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
853
			    AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));
854
EXPORT_SYMBOL_GPL(amd_erratum_400);
855

856 857
const int amd_erratum_383[] =
	AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
858
EXPORT_SYMBOL_GPL(amd_erratum_383);
859

860 861
bool cpu_has_amd_erratum(const int *erratum)
{
862
	struct cpuinfo_x86 *cpu = __this_cpu_ptr(&cpu_info);
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891
	int osvw_id = *erratum++;
	u32 range;
	u32 ms;

	/*
	 * If called early enough that current_cpu_data hasn't been initialized
	 * yet, fall back to boot_cpu_data.
	 */
	if (cpu->x86 == 0)
		cpu = &boot_cpu_data;

	if (cpu->x86_vendor != X86_VENDOR_AMD)
		return false;

	if (osvw_id >= 0 && osvw_id < 65536 &&
	    cpu_has(cpu, X86_FEATURE_OSVW)) {
		u64 osvw_len;

		rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
		if (osvw_id < osvw_len) {
			u64 osvw_bits;

			rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
			    osvw_bits);
			return osvw_bits & (1ULL << (osvw_id & 0x3f));
		}
	}

	/* OSVW unavailable or ID unknown, match family-model-stepping range */
892
	ms = (cpu->x86_model << 4) | cpu->x86_mask;
893 894 895 896 897 898 899 900
	while ((range = *erratum++))
		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
		    (ms >= AMD_MODEL_RANGE_START(range)) &&
		    (ms <= AMD_MODEL_RANGE_END(range)))
			return true;

	return false;
}
901 902

EXPORT_SYMBOL_GPL(cpu_has_amd_erratum);