common.c 38.6 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/export.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
8
#include <linux/ctype.h>
L
Linus Torvalds 已提交
9
#include <linux/delay.h>
10 11
#include <linux/sched.h>
#include <linux/init.h>
12
#include <linux/kprobes.h>
13
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
14
#include <linux/smp.h>
15
#include <linux/io.h>
16
#include <linux/syscore_ops.h>
17 18

#include <asm/stackprotector.h>
19
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
20
#include <asm/mmu_context.h>
21
#include <asm/archrandom.h>
22 23
#include <asm/hypervisor.h>
#include <asm/processor.h>
24
#include <asm/tlbflush.h>
25
#include <asm/debugreg.h>
26
#include <asm/sections.h>
27
#include <asm/vsyscall.h>
A
Alan Cox 已提交
28 29
#include <linux/topology.h>
#include <linux/cpumask.h>
30
#include <asm/pgtable.h>
A
Arun Sharma 已提交
31
#include <linux/atomic.h>
32 33 34 35
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
36
#include <asm/fpu/internal.h>
37
#include <asm/mtrr.h>
A
Alan Cox 已提交
38
#include <linux/numa.h>
39
#include <asm/asm.h>
40
#include <asm/bugs.h>
41
#include <asm/cpu.h>
42
#include <asm/mce.h>
43
#include <asm/msr.h>
44
#include <asm/pat.h>
45 46
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
47 48

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
49
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
50 51 52 53
#endif

#include "cpu.h"

54 55
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
56 57
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
58 59 60 61

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
62
/* correctly size the local cpu masks */
63
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
64 65 66 67 68 69 70
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

71
static void default_init(struct cpuinfo_x86 *c)
72 73
{
#ifdef CONFIG_X86_64
74
	cpu_detect_cache_sizes(c);
75 76 77 78 79 80 81 82 83 84 85 86 87
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

88
static const struct cpu_dev default_cpu = {
89 90 91 92 93
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

94
static const struct cpu_dev *this_cpu = &default_cpu;
95

B
Brian Gerst 已提交
96
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
97
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
98 99 100 101 102
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
103
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
104 105
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
106 107 108 109 110 111
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
112
#else
A
Akinobu Mita 已提交
113 114 115 116
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
117 118 119 120 121
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
122
	/* 32-bit code */
A
Akinobu Mita 已提交
123
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
124
	/* 16-bit code */
A
Akinobu Mita 已提交
125
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
126
	/* 16-bit data */
A
Akinobu Mita 已提交
127
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
128
	/* 16-bit data */
A
Akinobu Mita 已提交
129
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
130
	/* 16-bit data */
A
Akinobu Mita 已提交
131
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
132 133 134 135
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
136
	/* 32-bit code */
A
Akinobu Mita 已提交
137
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
138
	/* 16-bit code */
A
Akinobu Mita 已提交
139
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
140
	/* data */
141
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
142

A
Akinobu Mita 已提交
143 144
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
145
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
146
#endif
B
Brian Gerst 已提交
147
} };
148
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
149

150
static int __init x86_mpx_setup(char *s)
151
{
152
	/* require an exact match without trailing characters */
153 154
	if (strlen(s))
		return 0;
155

156 157 158
	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_MPX))
		return 1;
159

160 161
	setup_clear_cpu_cap(X86_FEATURE_MPX);
	pr_info("nompx: Intel Memory Protection Extensions (MPX) disabled\n");
162 163
	return 1;
}
164
__setup("nompx", x86_mpx_setup);
165

166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
static int __init x86_noinvpcid_setup(char *s)
{
	/* noinvpcid doesn't accept parameters */
	if (s)
		return -EINVAL;

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_INVPCID))
		return 0;

	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
	pr_info("noinvpcid: INVPCID feature disabled\n");
	return 0;
}
early_param("noinvpcid", x86_noinvpcid_setup);

182
#ifdef CONFIG_X86_32
183 184
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
L
Linus Torvalds 已提交
185

186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

205 206 207 208 209 210 211
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
212 213 214 215 216 217 218 219 220 221 222
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

223 224
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
225 226 227 228 229

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
230
int have_cpuid_p(void)
231 232 233 234
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

235
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
236
{
I
Ingo Molnar 已提交
237 238 239 240 241 242 243 244 245 246 247
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

248
	pr_notice("CPU serial number disabled.\n");
I
Ingo Molnar 已提交
249 250 251 252
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
253 254 255 256 257 258 259 260
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
261
#else
262 263 264 265 266 267 268
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
269
#endif
270

271 272
static __init int setup_disable_smep(char *arg)
{
273
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
274 275
	/* Check for things that depend on SMEP being enabled: */
	check_mpx_erratum(&boot_cpu_data);
276 277 278 279
	return 1;
}
__setup("nosmep", setup_disable_smep);

280
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
281
{
282
	if (cpu_has(c, X86_FEATURE_SMEP))
A
Andy Lutomirski 已提交
283
		cr4_set_bits(X86_CR4_SMEP);
284 285
}

286 287
static __init int setup_disable_smap(char *arg)
{
288
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
289 290 291 292
	return 1;
}
__setup("nosmap", setup_disable_smap);

293 294
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
295
	unsigned long eflags = native_save_fl();
296 297 298 299

	/* This should have been cleared long ago */
	BUG_ON(eflags & X86_EFLAGS_AC);

300 301
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
A
Andy Lutomirski 已提交
302
		cr4_set_bits(X86_CR4_SMAP);
303
#else
A
Andy Lutomirski 已提交
304
		cr4_clear_bits(X86_CR4_SMAP);
305 306
#endif
	}
307 308
}

309 310 311 312 313 314 315
/*
 * Protection Keys are not available in 32-bit mode.
 */
static bool pku_disabled;

static __always_inline void setup_pku(struct cpuinfo_x86 *c)
{
316 317 318 319
	/* check the boot processor, plus compile options for PKU: */
	if (!cpu_feature_enabled(X86_FEATURE_PKU))
		return;
	/* checks the actual processor's cpuid bits: */
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
	if (!cpu_has(c, X86_FEATURE_PKU))
		return;
	if (pku_disabled)
		return;

	cr4_set_bits(X86_CR4_PKE);
	/*
	 * Seting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
	 * cpuid bit to be set.  We need to ensure that we
	 * update that bit in this CPU's "cpu_info".
	 */
	get_cpu_cap(c);
}

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
static __init int setup_disable_pku(char *arg)
{
	/*
	 * Do not clear the X86_FEATURE_PKU bit.  All of the
	 * runtime checks are against OSPKE so clearing the
	 * bit does nothing.
	 *
	 * This way, we will see "pku" in cpuinfo, but not
	 * "ospke", which is exactly what we want.  It shows
	 * that the CPU has PKU, but the OS has not enabled it.
	 * This happens to be exactly how a system would look
	 * if we disabled the config option.
	 */
	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
	pku_disabled = true;
	return 1;
}
__setup("nopku", setup_disable_pku);
#endif /* CONFIG_X86_64 */

355 356 357 358 359 360 361 362 363
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
364

365
static const struct cpuid_dependent_feature
366 367 368 369 370 371 372
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

373
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
374 375
{
	const struct cpuid_dependent_feature *df;
376

377
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
378 379 380

		if (!cpu_has(c, df->feature))
			continue;
381 382 383 384 385 386 387
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
388
		if (!((s32)df->level < 0 ?
389
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
390 391 392 393 394 395 396
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

397 398
		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
			x86_cap_flag(df->feature), df->level);
399
	}
400
}
401

402 403 404
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
405 406
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
407 408 409
 */

/* Look up CPU names by table lookup. */
410
static const char *table_lookup_model(struct cpuinfo_x86 *c)
411
{
412 413
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
414 415 416 417 418 419 420

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

421
	info = this_cpu->legacy_models;
422

423
	while (info->family) {
424 425 426 427
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
428
#endif
429 430 431
	return NULL;		/* Not found */
}

432 433
__u32 cpu_caps_cleared[NCAPINTS];
__u32 cpu_caps_set[NCAPINTS];
434

435 436 437 438 439
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
440
	__loadsegment_simple(gs, 0);
441 442
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
443
	load_stack_canary_segment();
444 445
}

I
Ingo Molnar 已提交
446 447 448 449
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
450
void switch_to_new_gdt(int cpu)
451 452 453
{
	struct desc_ptr gdt_descr;

454
	gdt_descr.address = (long)get_cpu_gdt_table(cpu);
455 456
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
457
	/* Reload the per-cpu base */
458 459

	load_percpu_segment(cpu);
460 461
}

462
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
463

464
static void get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
465 466
{
	unsigned int *v;
467
	char *p, *q, *s;
L
Linus Torvalds 已提交
468

469
	if (c->extended_cpuid_level < 0x80000004)
470
		return;
L
Linus Torvalds 已提交
471

I
Ingo Molnar 已提交
472
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
473 474 475 476 477
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
	/* Trim whitespace */
	p = q = s = &c->x86_model_id[0];

	while (*p == ' ')
		p++;

	while (*p) {
		/* Note the last non-whitespace index */
		if (!isspace(*p))
			s = q;

		*q++ = *p++;
	}

	*(s + 1) = '\0';
L
Linus Torvalds 已提交
493 494
}

495
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
496
{
497
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
498

499
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
500 501

	if (n >= 0x80000005) {
502 503
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
504 505 506 507
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
508 509 510 511 512
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

513
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
514
	l2size = ecx >> 16;
515

516 517 518
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
519
	/* do processor-specific cache resizing */
520 521
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
L
Linus Torvalds 已提交
522 523 524 525 526

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

527
	if (l2size == 0)
L
Linus Torvalds 已提交
528
		return;		/* Again, no L2 cache is possible */
529
#endif
L
Linus Torvalds 已提交
530 531 532 533

	c->x86_cache_size = l2size;
}

534 535 536 537 538 539
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
540
u16 __read_mostly tlb_lld_1g[NR_INFO];
541

542
static void cpu_detect_tlb(struct cpuinfo_x86 *c)
543 544 545 546
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

547
	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
548
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
549 550 551 552 553
		tlb_lli_4m[ENTRIES]);

	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
554 555
}

556
void detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
557
{
B
Borislav Petkov 已提交
558
#ifdef CONFIG_SMP
559 560
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
561
	static bool printed;
L
Linus Torvalds 已提交
562

563
	if (!cpu_has(c, X86_FEATURE_HT))
564
		return;
L
Linus Torvalds 已提交
565

566 567
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
568

569 570
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
571

572
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
573

574 575 576
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
577
		pr_info_once("CPU0: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
578 579
		goto out;
	}
580

I
Ingo Molnar 已提交
581 582
	if (smp_num_siblings <= 1)
		goto out;
583

I
Ingo Molnar 已提交
584 585
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
586

I
Ingo Molnar 已提交
587
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
588

I
Ingo Molnar 已提交
589
	index_msb = get_count_order(smp_num_siblings);
590

I
Ingo Molnar 已提交
591
	core_bits = get_count_order(c->x86_max_cores);
592

I
Ingo Molnar 已提交
593 594
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
595

596
out:
597
	if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
598 599 600 601
		pr_info("CPU: Physical Processor ID: %d\n",
			c->phys_proc_id);
		pr_info("CPU: Processor Core ID: %d\n",
			c->cpu_core_id);
602
		printed = 1;
603 604
	}
#endif
605
}
L
Linus Torvalds 已提交
606

607
static void get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
608 609
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
610
	int i;
L
Linus Torvalds 已提交
611 612

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
613 614 615 616 617 618
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
619

Y
Yinghai Lu 已提交
620 621 622
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
623 624
		}
	}
Y
Yinghai Lu 已提交
625

626 627
	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
		    "CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
628

629 630
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
631 632
}

633
void cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
634 635
{
	/* Get vendor name */
636 637 638 639
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
640 641

	c->x86 = 4;
642
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
643 644
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
645

L
Linus Torvalds 已提交
646
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
647 648 649
		c->x86		= x86_family(tfms);
		c->x86_model	= x86_model(tfms);
		c->x86_mask	= x86_stepping(tfms);
I
Ingo Molnar 已提交
650

H
Huang, Ying 已提交
651 652
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
653
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
654
		}
L
Linus Torvalds 已提交
655 656
	}
}
657

658
void get_cpu_cap(struct cpuinfo_x86 *c)
659
{
660
	u32 eax, ebx, ecx, edx;
661

662 663
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
664
		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
665

666 667
		c->x86_capability[CPUID_1_ECX] = ecx;
		c->x86_capability[CPUID_1_EDX] = edx;
668
	}
669

670 671 672 673
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);

674
		c->x86_capability[CPUID_7_0_EBX] = ebx;
675

676
		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
677
		c->x86_capability[CPUID_7_ECX] = ecx;
678 679
	}

680 681 682 683
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

684
		c->x86_capability[CPUID_D_1_EAX] = eax;
685 686
	}

687 688 689 690 691
	/* Additional Intel-defined flags: level 0x0000000F */
	if (c->cpuid_level >= 0x0000000F) {

		/* QoS sub-leaf, EAX=0Fh, ECX=0 */
		cpuid_count(0x0000000F, 0, &eax, &ebx, &ecx, &edx);
692 693
		c->x86_capability[CPUID_F_0_EDX] = edx;

694 695 696 697 698 699
		if (cpu_has(c, X86_FEATURE_CQM_LLC)) {
			/* will be overridden if occupancy monitoring exists */
			c->x86_cache_max_rmid = ebx;

			/* QoS sub-leaf, EAX=0Fh, ECX=1 */
			cpuid_count(0x0000000F, 1, &eax, &ebx, &ecx, &edx);
700 701
			c->x86_capability[CPUID_F_1_EDX] = edx;

702 703 704
			if ((cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC)) ||
			      ((cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL)) ||
			       (cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL)))) {
705 706 707 708 709 710 711 712 713
				c->x86_cache_max_rmid = ecx;
				c->x86_cache_occ_scale = ebx;
			}
		} else {
			c->x86_cache_max_rmid = -1;
			c->x86_cache_occ_scale = -1;
		}
	}

714
	/* AMD-defined flags: level 0x80000001 */
715 716 717 718 719 720
	eax = cpuid_eax(0x80000000);
	c->extended_cpuid_level = eax;

	if ((eax & 0xffff0000) == 0x80000000) {
		if (eax >= 0x80000001) {
			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
721

722 723
			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
			c->x86_capability[CPUID_8000_0001_EDX] = edx;
724 725 726
		}
	}

727 728 729 730 731 732 733
	if (c->extended_cpuid_level >= 0x80000007) {
		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);

		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
		c->x86_power = edx;
	}

734
	if (c->extended_cpuid_level >= 0x80000008) {
735
		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
736 737 738

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
739
		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
740
	}
741 742 743
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
744
#endif
745

746
	if (c->extended_cpuid_level >= 0x8000000a)
747
		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
748

749
	init_scattered_cpuid_features(c);
750
}
L
Linus Torvalds 已提交
751

752
static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
Y
Yinghai Lu 已提交
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

778 779 780 781 782 783 784 785 786
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
787
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
788
{
789 790
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
791 792
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
793
#else
H
Huang, Ying 已提交
794
	c->x86_clflush_size = 32;
795 796
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
797
#endif
798
	c->x86_cache_alignment = c->x86_clflush_size;
799

800
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
801
	c->extended_cpuid_level = 0;
802

Y
Yinghai Lu 已提交
803 804 805 806
	if (!have_cpuid_p())
		identify_cpu_without_cpuid(c);

	/* cyrix could have cpuid enabled via c_identify()*/
807 808 809 810
	if (have_cpuid_p()) {
		cpu_detect(c);
		get_cpu_vendor(c);
		get_cpu_cap(c);
811

812 813
		if (this_cpu->c_early_init)
			this_cpu->c_early_init(c);
814

815 816
		c->cpu_index = 0;
		filter_cpuid_features(c, false);
817

818 819 820
		if (this_cpu->c_bsp_init)
			this_cpu->c_bsp_init(c);
	}
821 822

	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
823
	fpu__init_system(c);
824 825
}

826 827
void __init early_cpu_init(void)
{
828
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
829 830
	int count = 0;

831
#ifdef CONFIG_PROCESSOR_SELECT
832
	pr_info("KERNEL supported cpus:\n");
833 834
#endif

Y
Yinghai Lu 已提交
835
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
836
		const struct cpu_dev *cpudev = *cdev;
837

Y
Yinghai Lu 已提交
838 839 840 841 842
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

843
#ifdef CONFIG_PROCESSOR_SELECT
844 845 846 847 848 849
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
850
				pr_info("  %s %s\n", cpudev->c_vendor,
851 852
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
853
		}
854
#endif
Y
Yinghai Lu 已提交
855
	}
856
	early_identify_cpu(&boot_cpu_data);
857
}
858

859
/*
B
Borislav Petkov 已提交
860 861 862 863 864
 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
 * unfortunately, that's not true in practice because of early VIA
 * chips and (more importantly) broken virtualizers that are not easy
 * to detect. In the latter case it doesn't even *fail* reliably, so
 * probing for it doesn't even work. Disable it completely on 32-bit
865
 * unless we can find a reliable way to detect all the broken cases.
B
Borislav Petkov 已提交
866
 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
867
 */
868
static void detect_nopl(struct cpuinfo_x86 *c)
869
{
B
Borislav Petkov 已提交
870
#ifdef CONFIG_X86_32
871
	clear_cpu_cap(c, X86_FEATURE_NOPL);
B
Borislav Petkov 已提交
872 873
#else
	set_cpu_cap(c, X86_FEATURE_NOPL);
874
#endif
875
}
876

877 878 879
static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
880
	/*
881 882 883 884 885
	 * Empirically, writing zero to a segment selector on AMD does
	 * not clear the base, whereas writing zero to a segment
	 * selector on Intel does clear the base.  Intel's behavior
	 * allows slightly faster context switches in the common case
	 * where GS is unused by the prev and next threads.
886
	 *
887 888 889 890 891 892
	 * Since neither vendor documents this anywhere that I can see,
	 * detect it directly instead of hardcoding the choice by
	 * vendor.
	 *
	 * I've designated AMD's behavior as the "bug" because it's
	 * counterintuitive and less friendly.
893
	 */
894 895 896 897 898 899 900 901 902

	unsigned long old_base, tmp;
	rdmsrl(MSR_FS_BASE, old_base);
	wrmsrl(MSR_FS_BASE, 1);
	loadsegment(fs, 0);
	rdmsrl(MSR_FS_BASE, tmp);
	if (tmp != 0)
		set_cpu_bug(c, X86_BUG_NULL_SEG);
	wrmsrl(MSR_FS_BASE, old_base);
B
Borislav Petkov 已提交
903
#endif
904 905
}

906
static void generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
907
{
Y
Yinghai Lu 已提交
908
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
909

910
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
911
		identify_cpu_without_cpuid(c);
912

Y
Yinghai Lu 已提交
913
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
914
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
915
		return;
L
Linus Torvalds 已提交
916

917
	cpu_detect(c);
L
Linus Torvalds 已提交
918

919
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
920

921
	get_cpu_cap(c);
L
Linus Torvalds 已提交
922

923 924
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
925
#ifdef CONFIG_X86_32
B
Borislav Petkov 已提交
926
# ifdef CONFIG_SMP
927
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
928
# else
929
		c->apicid = c->initial_apicid;
930 931 932
# endif
#endif
		c->phys_proc_id = c->initial_apicid;
933
	}
L
Linus Torvalds 已提交
934

935
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
936

937
	detect_nopl(c);
938 939

	detect_null_seg_behavior(c);
940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964

	/*
	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
	 * systems that run Linux at CPL > 0 may or may not have the
	 * issue, but, even if they have the issue, there's absolutely
	 * nothing we can do about it because we can't use the real IRET
	 * instruction.
	 *
	 * NB: For the time being, only 32-bit kernels support
	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
	 * whether to apply espfix using paravirt hooks.  If any
	 * non-paravirt system ever shows up that does *not* have the
	 * ESPFIX issue, we can change this.
	 */
#ifdef CONFIG_X86_32
# ifdef CONFIG_PARAVIRT
	do {
		extern void native_iret(void);
		if (pv_cpu_ops.iret == native_iret)
			set_cpu_bug(c, X86_BUG_ESPFIX);
	} while (0);
# else
	set_cpu_bug(c, X86_BUG_ESPFIX);
# endif
#endif
L
Linus Torvalds 已提交
965 966
}

967 968 969 970 971 972 973 974 975 976 977 978 979 980
static void x86_init_cache_qos(struct cpuinfo_x86 *c)
{
	/*
	 * The heavy lifting of max_rmid and cache_occ_scale are handled
	 * in get_cpu_cap().  Here we just set the max_rmid for the boot_cpu
	 * in case CQM bits really aren't there in this CPU.
	 */
	if (c != &boot_cpu_data) {
		boot_cpu_data.x86_cache_max_rmid =
			min(boot_cpu_data.x86_cache_max_rmid,
			    c->x86_cache_max_rmid);
	}
}

981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
/*
 * The physical to logical package id mapping is initialized from the
 * acpi/mptables information. Make sure that CPUID actually agrees with
 * that.
 */
static void sanitize_package_id(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_SMP
	unsigned int pkg, apicid, cpu = smp_processor_id();

	apicid = apic->cpu_present_to_apicid(cpu);
	pkg = apicid >> boot_cpu_data.x86_coreid_bits;

	if (apicid != c->initial_apicid) {
		pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x CPUID: %x\n",
		       cpu, apicid, c->initial_apicid);
		c->initial_apicid = apicid;
	}
	if (pkg != c->phys_proc_id) {
		pr_err(FW_BUG "CPU%u: Using firmware package id %u instead of %u\n",
		       cpu, pkg, c->phys_proc_id);
		c->phys_proc_id = pkg;
	}
	c->logical_proc_id = topology_phys_to_logical_pkg(pkg);
#else
	c->logical_proc_id = 0;
#endif
}

L
Linus Torvalds 已提交
1010 1011 1012
/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
1013
static void identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1014 1015 1016 1017 1018 1019 1020 1021 1022
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
1023
	c->x86_max_cores = 1;
1024
	c->x86_coreid_bits = 0;
1025
#ifdef CONFIG_X86_64
1026
	c->x86_clflush_size = 64;
1027 1028
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
1029 1030
#else
	c->cpuid_level = -1;	/* CPUID not detected */
1031
	c->x86_clflush_size = 32;
1032 1033
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
1034 1035
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
1036 1037 1038 1039
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

1040
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
1041 1042
		this_cpu->c_identify(c);

1043
	/* Clear/Set all flags overridden by options, after probe */
1044 1045 1046 1047 1048
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

1049
#ifdef CONFIG_X86_64
1050
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1051 1052
#endif

L
Linus Torvalds 已提交
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

1069 1070 1071 1072
	/* Set up SMEP/SMAP */
	setup_smep(c);
	setup_smap(c);

L
Linus Torvalds 已提交
1073
	/*
I
Ingo Molnar 已提交
1074 1075
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
1076 1077
	 */

1078 1079 1080
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
1081
	/* If the model name is still unset, do table lookup. */
1082
	if (!c->x86_model_id[0]) {
1083
		const char *p;
L
Linus Torvalds 已提交
1084
		p = table_lookup_model(c);
1085
		if (p)
L
Linus Torvalds 已提交
1086 1087 1088 1089
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
1090
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
1091 1092
	}

1093 1094 1095 1096
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

1097
	init_hypervisor(c);
1098
	x86_init_rdrand(c);
1099
	x86_init_cache_qos(c);
1100
	setup_pku(c);
1101 1102

	/*
1103
	 * Clear/Set all flags overridden by options, need do it
1104 1105 1106 1107 1108 1109 1110
	 * before following smp all cpus cap AND.
	 */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

L
Linus Torvalds 已提交
1111 1112 1113 1114 1115 1116
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
1117
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
1118
		/* AND the already accumulated flags with these */
1119
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
1120
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1121 1122 1123 1124

		/* OR, i.e. replicate the bug flags */
		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
L
Linus Torvalds 已提交
1125 1126 1127
	}

	/* Init Machine Check Exception if available. */
1128
	mcheck_cpu_init(c);
1129 1130

	select_idle_routine(c);
1131

1132
#ifdef CONFIG_NUMA
1133 1134
	numa_add_cpu(smp_processor_id());
#endif
1135
	sanitize_package_id(c);
1136
}
S
Shaohua Li 已提交
1137

1138 1139 1140 1141
/*
 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
 * on 32-bit kernels:
 */
1142 1143 1144
#ifdef CONFIG_X86_32
void enable_sep_cpu(void)
{
1145 1146
	struct tss_struct *tss;
	int cpu;
1147

1148 1149 1150
	if (!boot_cpu_has(X86_FEATURE_SEP))
		return;

1151 1152 1153 1154
	cpu = get_cpu();
	tss = &per_cpu(cpu_tss, cpu);

	/*
1155 1156
	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
	 * see the big comment in struct x86_hw_tss's definition.
1157
	 */
1158 1159

	tss->x86_tss.ss1 = __KERNEL_CS;
1160 1161
	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);

1162 1163 1164
	wrmsr(MSR_IA32_SYSENTER_ESP,
	      (unsigned long)tss + offsetofend(struct tss_struct, SYSENTER_stack),
	      0);
1165

1166
	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1167

1168 1169
	put_cpu();
}
1170 1171
#endif

1172 1173 1174
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
1175
#ifdef CONFIG_X86_32
1176
	sysenter_setup();
L
Li Shaohua 已提交
1177
	enable_sep_cpu();
1178
#endif
1179
	cpu_detect_tlb(&boot_cpu_data);
1180
}
S
Shaohua Li 已提交
1181

1182
void identify_secondary_cpu(struct cpuinfo_x86 *c)
1183 1184 1185
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
1186
#ifdef CONFIG_X86_32
1187
	enable_sep_cpu();
1188
#endif
1189
	mtrr_ap_init();
L
Linus Torvalds 已提交
1190 1191
}

A
Andi Kleen 已提交
1192 1193
static __init int setup_noclflush(char *arg)
{
1194
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1195
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
A
Andi Kleen 已提交
1196 1197 1198 1199
	return 1;
}
__setup("noclflush", setup_noclflush);

1200
void print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1201
{
1202
	const char *vendor = NULL;
L
Linus Torvalds 已提交
1203

I
Ingo Molnar 已提交
1204
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
1205
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
1206 1207 1208 1209
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
1210

1211
	if (vendor && !strstr(c->x86_model_id, vendor))
1212
		pr_cont("%s ", vendor);
L
Linus Torvalds 已提交
1213

1214
	if (c->x86_model_id[0])
1215
		pr_cont("%s", c->x86_model_id);
L
Linus Torvalds 已提交
1216
	else
1217
		pr_cont("%d86", c->x86);
L
Linus Torvalds 已提交
1218

1219
	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1220

1221
	if (c->x86_mask || c->cpuid_level >= 0)
1222
		pr_cont(", stepping: 0x%x)\n", c->x86_mask);
L
Linus Torvalds 已提交
1223
	else
1224
		pr_cont(")\n");
L
Linus Torvalds 已提交
1225 1226
}

1227 1228 1229
static __init int setup_disablecpuid(char *arg)
{
	int bit;
I
Ingo Molnar 已提交
1230

1231 1232 1233 1234
	if (get_option(&arg, &bit) && bit < NCAPINTS*32)
		setup_clear_cpu_cap(bit);
	else
		return 0;
I
Ingo Molnar 已提交
1235

1236 1237 1238 1239
	return 1;
}
__setup("clearcpuid=", setup_disablecpuid);

1240
#ifdef CONFIG_X86_64
1241 1242 1243 1244 1245 1246 1247 1248
struct desc_ptr idt_descr __ro_after_init = {
	.size = NR_VECTORS * 16 - 1,
	.address = (unsigned long) idt_table,
};
const struct desc_ptr debug_idt_descr = {
	.size = NR_VECTORS * 16 - 1,
	.address = (unsigned long) debug_idt_table,
};
1249

1250
DEFINE_PER_CPU_FIRST(union irq_stack_union,
1251
		     irq_stack_union) __aligned(PAGE_SIZE) __visible;
I
Ingo Molnar 已提交
1252

1253
/*
1254 1255
 * The following percpu variables are hot.  Align current_task to
 * cacheline size such that they fall in the same cacheline.
1256 1257 1258 1259
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1260

1261
DEFINE_PER_CPU(char *, irq_stack_ptr) =
1262
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE;
1263

1264
DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1265

1266 1267 1268
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);

I
Ingo Molnar 已提交
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

1280
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
1281
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
1282 1283 1284

/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1285
{
1286
	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
1287
	wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
1288 1289

#ifdef CONFIG_IA32_EMULATION
1290
	wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat);
1291
	/*
1292 1293 1294 1295
	 * This only works on Intel CPUs.
	 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
	 * This does not cause SYSENTER to jump to the wrong location, because
	 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
1296 1297 1298
	 */
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
1299
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
1300
#else
1301
	wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
1302
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1303 1304
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1305
#endif
1306

1307 1308
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
1309
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1310
	       X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
L
Linus Torvalds 已提交
1311
}
1312

1313 1314 1315 1316 1317 1318
/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

1319
static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
1320
DEFINE_PER_CPU(int, debug_stack_usage);
1321 1322 1323

int is_debug_stack(unsigned long addr)
{
1324 1325 1326
	return __this_cpu_read(debug_stack_usage) ||
		(addr <= __this_cpu_read(debug_stack_addr) &&
		 addr > (__this_cpu_read(debug_stack_addr) - DEBUG_STKSZ));
1327
}
1328
NOKPROBE_SYMBOL(is_debug_stack);
1329

1330
DEFINE_PER_CPU(u32, debug_idt_ctr);
1331

1332 1333
void debug_stack_set_zero(void)
{
1334 1335
	this_cpu_inc(debug_idt_ctr);
	load_current_idt();
1336
}
1337
NOKPROBE_SYMBOL(debug_stack_set_zero);
1338 1339 1340

void debug_stack_reset(void)
{
1341
	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
1342
		return;
1343 1344
	if (this_cpu_dec_return(debug_idt_ctr) == 0)
		load_current_idt();
1345
}
1346
NOKPROBE_SYMBOL(debug_stack_reset);
1347

I
Ingo Molnar 已提交
1348
#else	/* CONFIG_X86_64 */
1349

1350 1351
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1352 1353
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
1354

1355 1356 1357 1358 1359 1360 1361 1362 1363
/*
 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
 * the top of the kernel stack.  Use an extra percpu variable to track the
 * top of the kernel stack directly.
 */
DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
	(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);

1364
#ifdef CONFIG_CC_STACKPROTECTOR
1365
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1366
#endif
1367

I
Ingo Molnar 已提交
1368
#endif	/* CONFIG_X86_64 */
1369

1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1385

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
#ifdef CONFIG_KGDB
/*
 * Restore debug regs if using kgdbwait and you have a kernel debugger
 * connection established.
 */
static void dbg_restore_debug_regs(void)
{
	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
		arch_kgdb_ops.correct_hw_break();
}
#else /* ! CONFIG_KGDB */
#define dbg_restore_debug_regs()
#endif /* ! CONFIG_KGDB */

1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
static void wait_for_master_cpu(int cpu)
{
#ifdef CONFIG_SMP
	/*
	 * wait for ACK from master CPU before continuing
	 * with AP initialization
	 */
	WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
	while (!cpumask_test_cpu(cpu, cpu_callout_mask))
		cpu_relax();
#endif
}

1413 1414 1415 1416 1417
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1418
 * A lot of state is already set up in PDA init for 64 bit
1419
 */
1420
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1421

1422
void cpu_init(void)
1423
{
1424
	struct orig_ist *oist;
1425
	struct task_struct *me;
I
Ingo Molnar 已提交
1426 1427
	struct tss_struct *t;
	unsigned long v;
1428
	int cpu = raw_smp_processor_id();
1429 1430
	int i;

1431 1432
	wait_for_master_cpu(cpu);

1433 1434 1435 1436 1437 1438
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1439 1440
	if (cpu)
		load_ucode_ap();
1441

1442
	t = &per_cpu(cpu_tss, cpu);
1443
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1444

1445
#ifdef CONFIG_NUMA
1446
	if (this_cpu_read(numa_node) == 0 &&
1447 1448
	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
		set_numa_node(early_cpu_to_node(cpu));
1449
#endif
1450 1451 1452

	me = current;

1453
	pr_debug("Initializing CPU#%d\n", cpu);
1454

A
Andy Lutomirski 已提交
1455
	cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1456 1457 1458 1459 1460 1461

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1462
	switch_to_new_gdt(cpu);
1463 1464
	loadsegment(fs, 0);

1465
	load_current_idt();
1466 1467 1468 1469 1470 1471 1472 1473

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1474
	x86_configure_nx();
1475
	x2apic_setup();
1476 1477 1478 1479

	/*
	 * set up and load the per-CPU TSS
	 */
1480
	if (!oist->ist[0]) {
1481
		char *estacks = per_cpu(exception_stacks, cpu);
I
Ingo Molnar 已提交
1482

1483
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1484
			estacks += exception_stack_sizes[v];
1485
			oist->ist[v] = t->x86_tss.ist[v] =
1486
					(unsigned long)estacks;
1487 1488
			if (v == DEBUG_STACK-1)
				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1489 1490 1491 1492
		}
	}

	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
I
Ingo Molnar 已提交
1493

1494 1495 1496 1497 1498 1499 1500 1501 1502
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

	atomic_inc(&init_mm.mm_count);
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1503
	BUG_ON(me->mm);
1504 1505 1506 1507 1508
	enter_lazy_tlb(&init_mm, me);

	load_sp0(t, &current->thread);
	set_tss_desc(cpu, t);
	load_TR_desc();
1509
	load_mm_ldt(&init_mm);
1510

1511 1512
	clear_all_debug_regs();
	dbg_restore_debug_regs();
1513

I
Ingo Molnar 已提交
1514
	fpu__init_cpu();
1515 1516 1517 1518 1519 1520 1521

	if (is_uv_system())
		uv_cpu_init();
}

#else

1522
void cpu_init(void)
1523
{
1524 1525
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1526
	struct tss_struct *t = &per_cpu(cpu_tss, cpu);
1527
	struct thread_struct *thread = &curr->thread;
1528

1529
	wait_for_master_cpu(cpu);
1530

1531 1532 1533 1534 1535 1536
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1537
	show_ucode_info_early();
1538

1539
	pr_info("Initializing CPU#%d\n", cpu);
1540

1541
	if (cpu_feature_enabled(X86_FEATURE_VME) ||
1542
	    boot_cpu_has(X86_FEATURE_TSC) ||
1543
	    boot_cpu_has(X86_FEATURE_DE))
A
Andy Lutomirski 已提交
1544
		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1545

1546
	load_current_idt();
1547
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1548 1549 1550 1551 1552

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
	atomic_inc(&init_mm.mm_count);
1553
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1554
	BUG_ON(curr->mm);
1555
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1556

1557
	load_sp0(t, thread);
1558
	set_tss_desc(cpu, t);
L
Linus Torvalds 已提交
1559
	load_TR_desc();
1560
	load_mm_ldt(&init_mm);
L
Linus Torvalds 已提交
1561

1562 1563
	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);

1564
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1565 1566
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1567
#endif
L
Linus Torvalds 已提交
1568

1569
	clear_all_debug_regs();
1570
	dbg_restore_debug_regs();
L
Linus Torvalds 已提交
1571

I
Ingo Molnar 已提交
1572
	fpu__init_cpu();
L
Linus Torvalds 已提交
1573
}
1574
#endif
1575

1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
static void bsp_resume(void)
{
	if (this_cpu->c_bsp_resume)
		this_cpu->c_bsp_resume(&boot_cpu_data);
}

static struct syscore_ops cpu_syscore_ops = {
	.resume		= bsp_resume,
};

static int __init init_cpu_syscore(void)
{
	register_syscore_ops(&cpu_syscore_ops);
	return 0;
}
core_initcall(init_cpu_syscore);