common.c 45.0 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/export.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
8
#include <linux/ctype.h>
L
Linus Torvalds 已提交
9
#include <linux/delay.h>
10
#include <linux/sched/mm.h>
11
#include <linux/sched/clock.h>
12
#include <linux/sched/task.h>
13
#include <linux/init.h>
14
#include <linux/kprobes.h>
15
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
16
#include <linux/smp.h>
17
#include <linux/io.h>
18
#include <linux/syscore_ops.h>
19 20

#include <asm/stackprotector.h>
21
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
22
#include <asm/mmu_context.h>
23
#include <asm/archrandom.h>
24 25
#include <asm/hypervisor.h>
#include <asm/processor.h>
26
#include <asm/tlbflush.h>
27
#include <asm/debugreg.h>
28
#include <asm/sections.h>
29
#include <asm/vsyscall.h>
A
Alan Cox 已提交
30 31
#include <linux/topology.h>
#include <linux/cpumask.h>
32
#include <asm/pgtable.h>
A
Arun Sharma 已提交
33
#include <linux/atomic.h>
34 35 36 37
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
38
#include <asm/fpu/internal.h>
39
#include <asm/mtrr.h>
40
#include <asm/hwcap2.h>
A
Alan Cox 已提交
41
#include <linux/numa.h>
42
#include <asm/asm.h>
43
#include <asm/bugs.h>
44
#include <asm/cpu.h>
45
#include <asm/mce.h>
46
#include <asm/msr.h>
47
#include <asm/pat.h>
48 49
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
50 51
#include <asm/intel-family.h>
#include <asm/cpu_device_id.h>
52 53

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
54
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
55 56 57 58
#endif

#include "cpu.h"

59 60
u32 elf_hwcap2 __read_mostly;

61 62
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
63 64
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
65 66 67 68

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
69
/* correctly size the local cpu masks */
70
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
71 72 73 74 75 76 77
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

78
static void default_init(struct cpuinfo_x86 *c)
79 80
{
#ifdef CONFIG_X86_64
81
	cpu_detect_cache_sizes(c);
82 83 84 85 86 87 88 89 90 91 92 93 94
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

95
static const struct cpu_dev default_cpu = {
96 97 98 99 100
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

101
static const struct cpu_dev *this_cpu = &default_cpu;
102

B
Brian Gerst 已提交
103
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
104
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
105 106 107 108 109
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
110
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
111 112
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
113 114 115 116 117 118
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
119
#else
A
Akinobu Mita 已提交
120 121 122 123
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
124 125 126 127 128
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
129
	/* 32-bit code */
A
Akinobu Mita 已提交
130
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
131
	/* 16-bit code */
A
Akinobu Mita 已提交
132
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
133
	/* 16-bit data */
A
Akinobu Mita 已提交
134
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
135
	/* 16-bit data */
A
Akinobu Mita 已提交
136
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
137
	/* 16-bit data */
A
Akinobu Mita 已提交
138
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
139 140 141 142
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
143
	/* 32-bit code */
A
Akinobu Mita 已提交
144
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
145
	/* 16-bit code */
A
Akinobu Mita 已提交
146
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
147
	/* data */
148
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
149

A
Akinobu Mita 已提交
150 151
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
152
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
153
#endif
B
Brian Gerst 已提交
154
} };
155
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
156

157
static int __init x86_mpx_setup(char *s)
158
{
159
	/* require an exact match without trailing characters */
160 161
	if (strlen(s))
		return 0;
162

163 164 165
	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_MPX))
		return 1;
166

167 168
	setup_clear_cpu_cap(X86_FEATURE_MPX);
	pr_info("nompx: Intel Memory Protection Extensions (MPX) disabled\n");
169 170
	return 1;
}
171
__setup("nompx", x86_mpx_setup);
172

173
#ifdef CONFIG_X86_64
174
static int __init x86_nopcid_setup(char *s)
175
{
176 177 178
	/* nopcid doesn't accept parameters */
	if (s)
		return -EINVAL;
179 180 181

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_PCID))
182
		return 0;
183 184 185

	setup_clear_cpu_cap(X86_FEATURE_PCID);
	pr_info("nopcid: PCID feature disabled\n");
186
	return 0;
187
}
188
early_param("nopcid", x86_nopcid_setup);
189 190
#endif

191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
static int __init x86_noinvpcid_setup(char *s)
{
	/* noinvpcid doesn't accept parameters */
	if (s)
		return -EINVAL;

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_INVPCID))
		return 0;

	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
	pr_info("noinvpcid: INVPCID feature disabled\n");
	return 0;
}
early_param("noinvpcid", x86_noinvpcid_setup);

207
#ifdef CONFIG_X86_32
208 209
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
L
Linus Torvalds 已提交
210

211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

230 231 232 233 234 235 236
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
237 238 239 240 241 242 243 244 245 246 247
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

248 249
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
250 251 252 253 254

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
255
int have_cpuid_p(void)
256 257 258 259
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

260
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
261
{
I
Ingo Molnar 已提交
262 263 264 265 266 267 268 269 270 271 272
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

273
	pr_notice("CPU serial number disabled.\n");
I
Ingo Molnar 已提交
274 275 276 277
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
278 279 280 281 282 283 284 285
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
286
#else
287 288 289 290 291 292 293
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
294
#endif
295

296 297
static __init int setup_disable_smep(char *arg)
{
298
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
299 300
	/* Check for things that depend on SMEP being enabled: */
	check_mpx_erratum(&boot_cpu_data);
301 302 303 304
	return 1;
}
__setup("nosmep", setup_disable_smep);

305
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
306
{
307
	if (cpu_has(c, X86_FEATURE_SMEP))
A
Andy Lutomirski 已提交
308
		cr4_set_bits(X86_CR4_SMEP);
309 310
}

311 312
static __init int setup_disable_smap(char *arg)
{
313
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
314 315 316 317
	return 1;
}
__setup("nosmap", setup_disable_smap);

318 319
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
320
	unsigned long eflags = native_save_fl();
321 322 323 324

	/* This should have been cleared long ago */
	BUG_ON(eflags & X86_EFLAGS_AC);

325 326
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
A
Andy Lutomirski 已提交
327
		cr4_set_bits(X86_CR4_SMAP);
328
#else
A
Andy Lutomirski 已提交
329
		cr4_clear_bits(X86_CR4_SMAP);
330 331
#endif
	}
332 333
}

334 335 336 337 338 339 340 341 342 343 344 345
static __always_inline void setup_umip(struct cpuinfo_x86 *c)
{
	/* Check the boot processor, plus build option for UMIP. */
	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
		goto out;

	/* Check the current processor's cpuid bits. */
	if (!cpu_has(c, X86_FEATURE_UMIP))
		goto out;

	cr4_set_bits(X86_CR4_UMIP);

346 347
	pr_info("x86/cpu: Activated the Intel User Mode Instruction Prevention (UMIP) CPU feature\n");

348 349 350 351 352 353 354 355 356 357
	return;

out:
	/*
	 * Make sure UMIP is disabled in case it was enabled in a
	 * previous boot (e.g., via kexec).
	 */
	cr4_clear_bits(X86_CR4_UMIP);
}

358 359 360 361 362 363 364
/*
 * Protection Keys are not available in 32-bit mode.
 */
static bool pku_disabled;

static __always_inline void setup_pku(struct cpuinfo_x86 *c)
{
365 366 367 368
	/* check the boot processor, plus compile options for PKU: */
	if (!cpu_feature_enabled(X86_FEATURE_PKU))
		return;
	/* checks the actual processor's cpuid bits: */
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
	if (!cpu_has(c, X86_FEATURE_PKU))
		return;
	if (pku_disabled)
		return;

	cr4_set_bits(X86_CR4_PKE);
	/*
	 * Seting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
	 * cpuid bit to be set.  We need to ensure that we
	 * update that bit in this CPU's "cpu_info".
	 */
	get_cpu_cap(c);
}

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
static __init int setup_disable_pku(char *arg)
{
	/*
	 * Do not clear the X86_FEATURE_PKU bit.  All of the
	 * runtime checks are against OSPKE so clearing the
	 * bit does nothing.
	 *
	 * This way, we will see "pku" in cpuinfo, but not
	 * "ospke", which is exactly what we want.  It shows
	 * that the CPU has PKU, but the OS has not enabled it.
	 * This happens to be exactly how a system would look
	 * if we disabled the config option.
	 */
	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
	pku_disabled = true;
	return 1;
}
__setup("nopku", setup_disable_pku);
#endif /* CONFIG_X86_64 */

404 405 406 407 408 409 410 411 412
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
413

414
static const struct cpuid_dependent_feature
415 416 417 418 419 420 421
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

422
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
423 424
{
	const struct cpuid_dependent_feature *df;
425

426
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
427 428 429

		if (!cpu_has(c, df->feature))
			continue;
430 431 432 433 434 435 436
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
437
		if (!((s32)df->level < 0 ?
438
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
439 440 441 442 443 444 445
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

446 447
		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
			x86_cap_flag(df->feature), df->level);
448
	}
449
}
450

451 452 453
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
454 455
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
456 457 458
 */

/* Look up CPU names by table lookup. */
459
static const char *table_lookup_model(struct cpuinfo_x86 *c)
460
{
461 462
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
463 464 465 466 467 468 469

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

470
	info = this_cpu->legacy_models;
471

472
	while (info->family) {
473 474 475 476
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
477
#endif
478 479 480
	return NULL;		/* Not found */
}

481 482
__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS];
__u32 cpu_caps_set[NCAPINTS + NBUGINTS];
483

484 485 486 487 488
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
489
	__loadsegment_simple(gs, 0);
490
	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
491
#endif
492
	load_stack_canary_segment();
493 494
}

495 496 497 498 499
#ifdef CONFIG_X86_32
/* The 32-bit entry code needs to find cpu_entry_area. */
DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
#endif

500 501 502 503 504 505 506 507 508 509 510
#ifdef CONFIG_X86_64
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};
511
#endif
512

513 514 515 516 517 518 519 520 521 522 523
/* Load the original GDT from the per-cpu structure */
void load_direct_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
EXPORT_SYMBOL_GPL(load_direct_gdt);

524 525 526 527 528 529 530 531 532
/* Load a fixmap remapping of the per-cpu GDT */
void load_fixmap_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
533
EXPORT_SYMBOL_GPL(load_fixmap_gdt);
534

I
Ingo Molnar 已提交
535 536 537 538
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
539
void switch_to_new_gdt(int cpu)
540
{
541 542
	/* Load the original GDT */
	load_direct_gdt(cpu);
543
	/* Reload the per-cpu base */
544
	load_percpu_segment(cpu);
545 546
}

547
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
548

549
static void get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
550 551
{
	unsigned int *v;
552
	char *p, *q, *s;
L
Linus Torvalds 已提交
553

554
	if (c->extended_cpuid_level < 0x80000004)
555
		return;
L
Linus Torvalds 已提交
556

I
Ingo Molnar 已提交
557
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
558 559 560 561 562
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
	/* Trim whitespace */
	p = q = s = &c->x86_model_id[0];

	while (*p == ' ')
		p++;

	while (*p) {
		/* Note the last non-whitespace index */
		if (!isspace(*p))
			s = q;

		*q++ = *p++;
	}

	*(s + 1) = '\0';
L
Linus Torvalds 已提交
578 579
}

580
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
581
{
582
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
583

584
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
585 586

	if (n >= 0x80000005) {
587 588
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
589 590 591 592
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
593 594 595 596 597
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

598
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
599
	l2size = ecx >> 16;
600

601 602 603
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
604
	/* do processor-specific cache resizing */
605 606
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
L
Linus Torvalds 已提交
607 608 609 610 611

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

612
	if (l2size == 0)
L
Linus Torvalds 已提交
613
		return;		/* Again, no L2 cache is possible */
614
#endif
L
Linus Torvalds 已提交
615 616 617 618

	c->x86_cache_size = l2size;
}

619 620 621 622 623 624
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
625
u16 __read_mostly tlb_lld_1g[NR_INFO];
626

627
static void cpu_detect_tlb(struct cpuinfo_x86 *c)
628 629 630 631
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

632
	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
633
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
634 635 636 637 638
		tlb_lli_4m[ENTRIES]);

	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
639 640
}

641
void detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
642
{
B
Borislav Petkov 已提交
643
#ifdef CONFIG_SMP
644 645
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
646
	static bool printed;
L
Linus Torvalds 已提交
647

648
	if (!cpu_has(c, X86_FEATURE_HT))
649
		return;
L
Linus Torvalds 已提交
650

651 652
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
653

654 655
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
656

657
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
658

659 660 661
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
662
		pr_info_once("CPU0: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
663 664
		goto out;
	}
665

I
Ingo Molnar 已提交
666 667
	if (smp_num_siblings <= 1)
		goto out;
668

I
Ingo Molnar 已提交
669 670
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
671

I
Ingo Molnar 已提交
672
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
673

I
Ingo Molnar 已提交
674
	index_msb = get_count_order(smp_num_siblings);
675

I
Ingo Molnar 已提交
676
	core_bits = get_count_order(c->x86_max_cores);
677

I
Ingo Molnar 已提交
678 679
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
680

681
out:
682
	if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
683 684 685 686
		pr_info("CPU: Physical Processor ID: %d\n",
			c->phys_proc_id);
		pr_info("CPU: Processor Core ID: %d\n",
			c->cpu_core_id);
687
		printed = 1;
688 689
	}
#endif
690
}
L
Linus Torvalds 已提交
691

692
static void get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
693 694
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
695
	int i;
L
Linus Torvalds 已提交
696 697

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
698 699 700 701 702 703
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
704

Y
Yinghai Lu 已提交
705 706 707
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
708 709
		}
	}
Y
Yinghai Lu 已提交
710

711 712
	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
		    "CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
713

714 715
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
716 717
}

718
void cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
719 720
{
	/* Get vendor name */
721 722 723 724
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
725 726

	c->x86 = 4;
727
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
728 729
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
730

L
Linus Torvalds 已提交
731
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
732 733
		c->x86		= x86_family(tfms);
		c->x86_model	= x86_model(tfms);
734
		c->x86_stepping	= x86_stepping(tfms);
I
Ingo Molnar 已提交
735

H
Huang, Ying 已提交
736 737
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
738
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
739
		}
L
Linus Torvalds 已提交
740 741
	}
}
742

743 744 745 746
static void apply_forced_caps(struct cpuinfo_x86 *c)
{
	int i;

747
	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
748 749 750 751 752
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}
}

753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
static void init_speculation_control(struct cpuinfo_x86 *c)
{
	/*
	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
	 * and they also have a different bit for STIBP support. Also,
	 * a hypervisor might have set the individual AMD bits even on
	 * Intel CPUs, for finer-grained selection of what's available.
	 *
	 * We use the AMD bits in 0x8000_0008 EBX as the generic hardware
	 * features, which are visible in /proc/cpuinfo and used by the
	 * kernel. So set those accordingly from the Intel bits.
	 */
	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
		set_cpu_cap(c, X86_FEATURE_IBRS);
		set_cpu_cap(c, X86_FEATURE_IBPB);
	}
	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
		set_cpu_cap(c, X86_FEATURE_STIBP);
}

773
void get_cpu_cap(struct cpuinfo_x86 *c)
774
{
775
	u32 eax, ebx, ecx, edx;
776

777 778
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
779
		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
780

781 782
		c->x86_capability[CPUID_1_ECX] = ecx;
		c->x86_capability[CPUID_1_EDX] = edx;
783
	}
784

785 786 787 788
	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
	if (c->cpuid_level >= 0x00000006)
		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);

789 790 791
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
792
		c->x86_capability[CPUID_7_0_EBX] = ebx;
793
		c->x86_capability[CPUID_7_ECX] = ecx;
794
		c->x86_capability[CPUID_7_EDX] = edx;
795 796
	}

797 798 799 800
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

801
		c->x86_capability[CPUID_D_1_EAX] = eax;
802 803
	}

804 805 806 807 808
	/* Additional Intel-defined flags: level 0x0000000F */
	if (c->cpuid_level >= 0x0000000F) {

		/* QoS sub-leaf, EAX=0Fh, ECX=0 */
		cpuid_count(0x0000000F, 0, &eax, &ebx, &ecx, &edx);
809 810
		c->x86_capability[CPUID_F_0_EDX] = edx;

811 812 813 814 815 816
		if (cpu_has(c, X86_FEATURE_CQM_LLC)) {
			/* will be overridden if occupancy monitoring exists */
			c->x86_cache_max_rmid = ebx;

			/* QoS sub-leaf, EAX=0Fh, ECX=1 */
			cpuid_count(0x0000000F, 1, &eax, &ebx, &ecx, &edx);
817 818
			c->x86_capability[CPUID_F_1_EDX] = edx;

819 820 821
			if ((cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC)) ||
			      ((cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL)) ||
			       (cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL)))) {
822 823 824 825 826 827 828 829 830
				c->x86_cache_max_rmid = ecx;
				c->x86_cache_occ_scale = ebx;
			}
		} else {
			c->x86_cache_max_rmid = -1;
			c->x86_cache_occ_scale = -1;
		}
	}

831
	/* AMD-defined flags: level 0x80000001 */
832 833 834 835 836 837
	eax = cpuid_eax(0x80000000);
	c->extended_cpuid_level = eax;

	if ((eax & 0xffff0000) == 0x80000000) {
		if (eax >= 0x80000001) {
			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
838

839 840
			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
			c->x86_capability[CPUID_8000_0001_EDX] = edx;
841 842 843
		}
	}

844 845 846 847 848 849 850
	if (c->extended_cpuid_level >= 0x80000007) {
		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);

		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
		c->x86_power = edx;
	}

851 852 853 854 855
	if (c->extended_cpuid_level >= 0x80000008) {
		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
	}

856
	if (c->extended_cpuid_level >= 0x8000000a)
857
		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
858

859
	init_scattered_cpuid_features(c);
860
	init_speculation_control(c);
861 862 863 864 865 866 867

	/*
	 * Clear/Set all flags overridden by options, after probe.
	 * This needs to happen each time we re-probe, which may happen
	 * several times during CPU initialization.
	 */
	apply_forced_caps(c);
868
}
L
Linus Torvalds 已提交
869

870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
static void get_cpu_address_sizes(struct cpuinfo_x86 *c)
{
	u32 eax, ebx, ecx, edx;

	if (c->extended_cpuid_level >= 0x80000008) {
		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
	}
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
#endif
}

886
static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
Y
Yinghai Lu 已提交
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

912
static const __initconst struct x86_cpu_id cpu_no_speculation[] = {
913 914 915 916 917 918 919 920 921 922 923 924
	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_CEDARVIEW,	X86_FEATURE_ANY },
	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_CLOVERVIEW,	X86_FEATURE_ANY },
	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_LINCROFT,	X86_FEATURE_ANY },
	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_PENWELL,	X86_FEATURE_ANY },
	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_PINEVIEW,	X86_FEATURE_ANY },
	{ X86_VENDOR_CENTAUR,	5 },
	{ X86_VENDOR_INTEL,	5 },
	{ X86_VENDOR_NSC,	5 },
	{ X86_VENDOR_ANY,	4 },
	{}
};

925
static const __initconst struct x86_cpu_id cpu_no_meltdown[] = {
926 927 928 929
	{ X86_VENDOR_AMD },
	{}
};

930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
static const __initconst struct x86_cpu_id cpu_no_spec_store_bypass[] = {
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_PINEVIEW	},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_LINCROFT	},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_PENWELL		},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_CLOVERVIEW	},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_CEDARVIEW	},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_SILVERMONT1	},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_AIRMONT		},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_SILVERMONT2	},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_ATOM_MERRIFIELD	},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_CORE_YONAH		},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_XEON_PHI_KNL		},
	{ X86_VENDOR_INTEL,	6,	INTEL_FAM6_XEON_PHI_KNM		},
	{ X86_VENDOR_CENTAUR,	5,					},
	{ X86_VENDOR_INTEL,	5,					},
	{ X86_VENDOR_NSC,	5,					},
	{ X86_VENDOR_ANY,	4,					},
	{}
};

950
static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
951 952 953
{
	u64 ia32_cap = 0;

954 955 956
	if (!x86_match_cpu(cpu_no_spec_store_bypass))
		setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);

957 958 959 960 961 962
	if (x86_match_cpu(cpu_no_speculation))
		return;

	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);

963
	if (x86_match_cpu(cpu_no_meltdown))
964
		return;
965 966 967 968 969 970

	if (cpu_has(c, X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);

	/* Rogue Data Cache Load? No! */
	if (ia32_cap & ARCH_CAP_RDCL_NO)
971
		return;
972

973
	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
974 975
}

976 977 978 979 980 981
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
982 983
 * WARNING: this function is only called on the boot CPU.  Don't add code
 * here that is supposed to run on all CPUs.
984
 */
985
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
986
{
987 988
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
989 990
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
991
#else
H
Huang, Ying 已提交
992
	c->x86_clflush_size = 32;
993 994
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
995
#endif
996
	c->x86_cache_alignment = c->x86_clflush_size;
997

998
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
999
	c->extended_cpuid_level = 0;
1000

Y
Yinghai Lu 已提交
1001
	/* cyrix could have cpuid enabled via c_identify()*/
1002 1003 1004 1005
	if (have_cpuid_p()) {
		cpu_detect(c);
		get_cpu_vendor(c);
		get_cpu_cap(c);
1006
		get_cpu_address_sizes(c);
B
Borislav Petkov 已提交
1007
		setup_force_cpu_cap(X86_FEATURE_CPUID);
1008

1009 1010
		if (this_cpu->c_early_init)
			this_cpu->c_early_init(c);
1011

1012 1013
		c->cpu_index = 0;
		filter_cpuid_features(c, false);
1014

1015 1016
		if (this_cpu->c_bsp_init)
			this_cpu->c_bsp_init(c);
B
Borislav Petkov 已提交
1017 1018 1019
	} else {
		identify_cpu_without_cpuid(c);
		setup_clear_cpu_cap(X86_FEATURE_CPUID);
1020
	}
1021 1022

	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1023

1024
	cpu_set_bug_bits(c);
1025

1026
	fpu__init_system(c);
1027 1028 1029 1030 1031 1032 1033 1034

#ifdef CONFIG_X86_32
	/*
	 * Regardless of whether PCID is enumerated, the SDM says
	 * that it can't be enabled in 32-bit mode.
	 */
	setup_clear_cpu_cap(X86_FEATURE_PCID);
#endif
1035 1036
}

1037 1038
void __init early_cpu_init(void)
{
1039
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
1040 1041
	int count = 0;

1042
#ifdef CONFIG_PROCESSOR_SELECT
1043
	pr_info("KERNEL supported cpus:\n");
1044 1045
#endif

Y
Yinghai Lu 已提交
1046
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1047
		const struct cpu_dev *cpudev = *cdev;
1048

Y
Yinghai Lu 已提交
1049 1050 1051 1052 1053
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

1054
#ifdef CONFIG_PROCESSOR_SELECT
1055 1056 1057 1058 1059 1060
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
1061
				pr_info("  %s %s\n", cpudev->c_vendor,
1062 1063
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
1064
		}
1065
#endif
Y
Yinghai Lu 已提交
1066
	}
1067
	early_identify_cpu(&boot_cpu_data);
1068
}
1069

1070
/*
B
Borislav Petkov 已提交
1071 1072 1073 1074 1075
 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
 * unfortunately, that's not true in practice because of early VIA
 * chips and (more importantly) broken virtualizers that are not easy
 * to detect. In the latter case it doesn't even *fail* reliably, so
 * probing for it doesn't even work. Disable it completely on 32-bit
1076
 * unless we can find a reliable way to detect all the broken cases.
B
Borislav Petkov 已提交
1077
 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1078
 */
1079
static void detect_nopl(struct cpuinfo_x86 *c)
1080
{
B
Borislav Petkov 已提交
1081
#ifdef CONFIG_X86_32
1082
	clear_cpu_cap(c, X86_FEATURE_NOPL);
B
Borislav Petkov 已提交
1083 1084
#else
	set_cpu_cap(c, X86_FEATURE_NOPL);
1085
#endif
1086
}
1087

1088 1089 1090
static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
1091
	/*
1092 1093 1094 1095 1096
	 * Empirically, writing zero to a segment selector on AMD does
	 * not clear the base, whereas writing zero to a segment
	 * selector on Intel does clear the base.  Intel's behavior
	 * allows slightly faster context switches in the common case
	 * where GS is unused by the prev and next threads.
1097
	 *
1098 1099 1100 1101 1102 1103
	 * Since neither vendor documents this anywhere that I can see,
	 * detect it directly instead of hardcoding the choice by
	 * vendor.
	 *
	 * I've designated AMD's behavior as the "bug" because it's
	 * counterintuitive and less friendly.
1104
	 */
1105 1106 1107 1108 1109 1110 1111 1112 1113

	unsigned long old_base, tmp;
	rdmsrl(MSR_FS_BASE, old_base);
	wrmsrl(MSR_FS_BASE, 1);
	loadsegment(fs, 0);
	rdmsrl(MSR_FS_BASE, tmp);
	if (tmp != 0)
		set_cpu_bug(c, X86_BUG_NULL_SEG);
	wrmsrl(MSR_FS_BASE, old_base);
B
Borislav Petkov 已提交
1114
#endif
1115 1116
}

1117
static void generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1118
{
Y
Yinghai Lu 已提交
1119
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
1120

1121
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
1122
		identify_cpu_without_cpuid(c);
1123

Y
Yinghai Lu 已提交
1124
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
1125
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
1126
		return;
L
Linus Torvalds 已提交
1127

1128
	cpu_detect(c);
L
Linus Torvalds 已提交
1129

1130
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
1131

1132
	get_cpu_cap(c);
L
Linus Torvalds 已提交
1133

1134 1135
	get_cpu_address_sizes(c);

1136 1137
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1138
#ifdef CONFIG_X86_32
B
Borislav Petkov 已提交
1139
# ifdef CONFIG_SMP
1140
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1141
# else
1142
		c->apicid = c->initial_apicid;
1143 1144 1145
# endif
#endif
		c->phys_proc_id = c->initial_apicid;
1146
	}
L
Linus Torvalds 已提交
1147

1148
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
1149

1150
	detect_nopl(c);
1151 1152

	detect_null_seg_behavior(c);
1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177

	/*
	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
	 * systems that run Linux at CPL > 0 may or may not have the
	 * issue, but, even if they have the issue, there's absolutely
	 * nothing we can do about it because we can't use the real IRET
	 * instruction.
	 *
	 * NB: For the time being, only 32-bit kernels support
	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
	 * whether to apply espfix using paravirt hooks.  If any
	 * non-paravirt system ever shows up that does *not* have the
	 * ESPFIX issue, we can change this.
	 */
#ifdef CONFIG_X86_32
# ifdef CONFIG_PARAVIRT
	do {
		extern void native_iret(void);
		if (pv_cpu_ops.iret == native_iret)
			set_cpu_bug(c, X86_BUG_ESPFIX);
	} while (0);
# else
	set_cpu_bug(c, X86_BUG_ESPFIX);
# endif
#endif
L
Linus Torvalds 已提交
1178 1179
}

1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193
static void x86_init_cache_qos(struct cpuinfo_x86 *c)
{
	/*
	 * The heavy lifting of max_rmid and cache_occ_scale are handled
	 * in get_cpu_cap().  Here we just set the max_rmid for the boot_cpu
	 * in case CQM bits really aren't there in this CPU.
	 */
	if (c != &boot_cpu_data) {
		boot_cpu_data.x86_cache_max_rmid =
			min(boot_cpu_data.x86_cache_max_rmid,
			    c->x86_cache_max_rmid);
	}
}

1194
/*
1195 1196
 * Validate that ACPI/mptables have the same information about the
 * effective APIC id and update the package map.
1197
 */
1198
static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1199 1200
{
#ifdef CONFIG_SMP
1201
	unsigned int apicid, cpu = smp_processor_id();
1202 1203 1204

	apicid = apic->cpu_present_to_apicid(cpu);

1205 1206
	if (apicid != c->apicid) {
		pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1207 1208
		       cpu, apicid, c->initial_apicid);
	}
1209
	BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1210 1211 1212 1213 1214
#else
	c->logical_proc_id = 0;
#endif
}

L
Linus Torvalds 已提交
1215 1216 1217
/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
1218
static void identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1219 1220 1221 1222
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
1223
	c->x86_cache_size = 0;
L
Linus Torvalds 已提交
1224
	c->x86_vendor = X86_VENDOR_UNKNOWN;
1225
	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
L
Linus Torvalds 已提交
1226 1227
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
1228
	c->x86_max_cores = 1;
1229
	c->x86_coreid_bits = 0;
1230
	c->cu_id = 0xff;
1231
#ifdef CONFIG_X86_64
1232
	c->x86_clflush_size = 64;
1233 1234
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
1235 1236
#else
	c->cpuid_level = -1;	/* CPUID not detected */
1237
	c->x86_clflush_size = 32;
1238 1239
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
1240 1241
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
1242 1243 1244 1245
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

1246
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
1247 1248
		this_cpu->c_identify(c);

1249
	/* Clear/Set all flags overridden by options, after probe */
1250
	apply_forced_caps(c);
1251

1252
#ifdef CONFIG_X86_64
1253
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1254 1255
#endif

L
Linus Torvalds 已提交
1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

1272
	/* Set up SMEP/SMAP/UMIP */
1273 1274
	setup_smep(c);
	setup_smap(c);
1275
	setup_umip(c);
1276

L
Linus Torvalds 已提交
1277
	/*
I
Ingo Molnar 已提交
1278 1279
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
1280 1281
	 */

1282 1283 1284
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
1285
	/* If the model name is still unset, do table lookup. */
1286
	if (!c->x86_model_id[0]) {
1287
		const char *p;
L
Linus Torvalds 已提交
1288
		p = table_lookup_model(c);
1289
		if (p)
L
Linus Torvalds 已提交
1290 1291 1292 1293
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
1294
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
1295 1296
	}

1297 1298 1299 1300
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

1301
	x86_init_rdrand(c);
1302
	x86_init_cache_qos(c);
1303
	setup_pku(c);
1304 1305

	/*
1306
	 * Clear/Set all flags overridden by options, need do it
1307 1308
	 * before following smp all cpus cap AND.
	 */
1309
	apply_forced_caps(c);
1310

L
Linus Torvalds 已提交
1311 1312 1313 1314 1315 1316
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
1317
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
1318
		/* AND the already accumulated flags with these */
1319
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
1320
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1321 1322 1323 1324

		/* OR, i.e. replicate the bug flags */
		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
L
Linus Torvalds 已提交
1325 1326 1327
	}

	/* Init Machine Check Exception if available. */
1328
	mcheck_cpu_init(c);
1329 1330

	select_idle_routine(c);
1331

1332
#ifdef CONFIG_NUMA
1333 1334
	numa_add_cpu(smp_processor_id());
#endif
1335
}
S
Shaohua Li 已提交
1336

1337 1338 1339 1340
/*
 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
 * on 32-bit kernels:
 */
1341 1342 1343
#ifdef CONFIG_X86_32
void enable_sep_cpu(void)
{
1344 1345
	struct tss_struct *tss;
	int cpu;
1346

1347 1348 1349
	if (!boot_cpu_has(X86_FEATURE_SEP))
		return;

1350
	cpu = get_cpu();
1351
	tss = &per_cpu(cpu_tss_rw, cpu);
1352 1353

	/*
1354 1355
	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
	 * see the big comment in struct x86_hw_tss's definition.
1356
	 */
1357 1358

	tss->x86_tss.ss1 = __KERNEL_CS;
1359
	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1360
	wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1361
	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1362

1363 1364
	put_cpu();
}
1365 1366
#endif

1367 1368 1369
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
1370
#ifdef CONFIG_X86_32
1371
	sysenter_setup();
L
Li Shaohua 已提交
1372
	enable_sep_cpu();
1373
#endif
1374
	cpu_detect_tlb(&boot_cpu_data);
1375
}
S
Shaohua Li 已提交
1376

1377
void identify_secondary_cpu(struct cpuinfo_x86 *c)
1378 1379 1380
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
1381
#ifdef CONFIG_X86_32
1382
	enable_sep_cpu();
1383
#endif
1384
	mtrr_ap_init();
1385
	validate_apic_and_package_id(c);
L
Linus Torvalds 已提交
1386 1387
}

A
Andi Kleen 已提交
1388 1389
static __init int setup_noclflush(char *arg)
{
1390
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1391
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
A
Andi Kleen 已提交
1392 1393 1394 1395
	return 1;
}
__setup("noclflush", setup_noclflush);

1396
void print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1397
{
1398
	const char *vendor = NULL;
L
Linus Torvalds 已提交
1399

I
Ingo Molnar 已提交
1400
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
1401
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
1402 1403 1404 1405
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
1406

1407
	if (vendor && !strstr(c->x86_model_id, vendor))
1408
		pr_cont("%s ", vendor);
L
Linus Torvalds 已提交
1409

1410
	if (c->x86_model_id[0])
1411
		pr_cont("%s", c->x86_model_id);
L
Linus Torvalds 已提交
1412
	else
1413
		pr_cont("%d86", c->x86);
L
Linus Torvalds 已提交
1414

1415
	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1416

1417 1418
	if (c->x86_stepping || c->cpuid_level >= 0)
		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
L
Linus Torvalds 已提交
1419
	else
1420
		pr_cont(")\n");
L
Linus Torvalds 已提交
1421 1422
}

1423 1424 1425 1426 1427 1428
/*
 * clearcpuid= was already parsed in fpu__init_parse_early_param.
 * But we need to keep a dummy __setup around otherwise it would
 * show up as an environment variable for init.
 */
static __init int setup_clearcpuid(char *arg)
1429 1430 1431
{
	return 1;
}
1432
__setup("clearcpuid=", setup_clearcpuid);
1433

1434
#ifdef CONFIG_X86_64
1435
DEFINE_PER_CPU_FIRST(union irq_stack_union,
1436
		     irq_stack_union) __aligned(PAGE_SIZE) __visible;
1437
EXPORT_PER_CPU_SYMBOL_GPL(irq_stack_union);
I
Ingo Molnar 已提交
1438

1439
/*
1440 1441
 * The following percpu variables are hot.  Align current_task to
 * cacheline size such that they fall in the same cacheline.
1442 1443 1444 1445
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1446

1447
DEFINE_PER_CPU(char *, irq_stack_ptr) =
1448
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE;
1449

1450
DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1451

1452 1453 1454
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);

1455 1456
/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1457
{
1458 1459 1460
	extern char _entry_trampoline[];
	extern char entry_SYSCALL_64_trampoline[];

1461
	int cpu = smp_processor_id();
1462 1463 1464
	unsigned long SYSCALL64_entry_trampoline =
		(unsigned long)get_cpu_entry_area(cpu)->entry_trampoline +
		(entry_SYSCALL_64_trampoline - _entry_trampoline);
1465

1466
	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
1467 1468 1469 1470
	if (static_cpu_has(X86_FEATURE_PTI))
		wrmsrl(MSR_LSTAR, SYSCALL64_entry_trampoline);
	else
		wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
1471 1472

#ifdef CONFIG_IA32_EMULATION
1473
	wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat);
1474
	/*
1475 1476 1477 1478
	 * This only works on Intel CPUs.
	 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
	 * This does not cause SYSENTER to jump to the wrong location, because
	 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
1479 1480
	 */
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
1481
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1));
1482
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
1483
#else
1484
	wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
1485
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1486 1487
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1488
#endif
1489

1490 1491
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
1492
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1493
	       X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
L
Linus Torvalds 已提交
1494
}
1495

1496 1497 1498 1499 1500 1501
/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

1502
static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
1503
DEFINE_PER_CPU(int, debug_stack_usage);
1504 1505 1506

int is_debug_stack(unsigned long addr)
{
1507 1508 1509
	return __this_cpu_read(debug_stack_usage) ||
		(addr <= __this_cpu_read(debug_stack_addr) &&
		 addr > (__this_cpu_read(debug_stack_addr) - DEBUG_STKSZ));
1510
}
1511
NOKPROBE_SYMBOL(is_debug_stack);
1512

1513
DEFINE_PER_CPU(u32, debug_idt_ctr);
1514

1515 1516
void debug_stack_set_zero(void)
{
1517 1518
	this_cpu_inc(debug_idt_ctr);
	load_current_idt();
1519
}
1520
NOKPROBE_SYMBOL(debug_stack_set_zero);
1521 1522 1523

void debug_stack_reset(void)
{
1524
	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
1525
		return;
1526 1527
	if (this_cpu_dec_return(debug_idt_ctr) == 0)
		load_current_idt();
1528
}
1529
NOKPROBE_SYMBOL(debug_stack_reset);
1530

I
Ingo Molnar 已提交
1531
#else	/* CONFIG_X86_64 */
1532

1533 1534
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1535 1536
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
1537

1538 1539 1540 1541 1542 1543 1544 1545 1546
/*
 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
 * the top of the kernel stack.  Use an extra percpu variable to track the
 * top of the kernel stack directly.
 */
DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
	(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);

1547
#ifdef CONFIG_CC_STACKPROTECTOR
1548
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1549
#endif
1550

I
Ingo Molnar 已提交
1551
#endif	/* CONFIG_X86_64 */
1552

1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1568

1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
#ifdef CONFIG_KGDB
/*
 * Restore debug regs if using kgdbwait and you have a kernel debugger
 * connection established.
 */
static void dbg_restore_debug_regs(void)
{
	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
		arch_kgdb_ops.correct_hw_break();
}
#else /* ! CONFIG_KGDB */
#define dbg_restore_debug_regs()
#endif /* ! CONFIG_KGDB */

1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
static void wait_for_master_cpu(int cpu)
{
#ifdef CONFIG_SMP
	/*
	 * wait for ACK from master CPU before continuing
	 * with AP initialization
	 */
	WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
	while (!cpumask_test_cpu(cpu, cpu_callout_mask))
		cpu_relax();
#endif
}

1596 1597 1598 1599 1600
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1601
 * A lot of state is already set up in PDA init for 64 bit
1602
 */
1603
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1604

1605
void cpu_init(void)
1606
{
1607
	struct orig_ist *oist;
1608
	struct task_struct *me;
I
Ingo Molnar 已提交
1609 1610
	struct tss_struct *t;
	unsigned long v;
1611
	int cpu = raw_smp_processor_id();
1612 1613
	int i;

1614 1615
	wait_for_master_cpu(cpu);

1616 1617 1618 1619 1620 1621
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1622 1623
	if (cpu)
		load_ucode_ap();
1624

1625
	t = &per_cpu(cpu_tss_rw, cpu);
1626
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1627

1628
#ifdef CONFIG_NUMA
1629
	if (this_cpu_read(numa_node) == 0 &&
1630 1631
	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
		set_numa_node(early_cpu_to_node(cpu));
1632
#endif
1633 1634 1635

	me = current;

1636
	pr_debug("Initializing CPU#%d\n", cpu);
1637

A
Andy Lutomirski 已提交
1638
	cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1639 1640 1641 1642 1643 1644

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1645
	switch_to_new_gdt(cpu);
1646 1647
	loadsegment(fs, 0);

1648
	load_current_idt();
1649 1650 1651 1652 1653 1654 1655 1656

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1657
	x86_configure_nx();
1658
	x2apic_setup();
1659 1660 1661 1662

	/*
	 * set up and load the per-CPU TSS
	 */
1663
	if (!oist->ist[0]) {
1664
		char *estacks = get_cpu_entry_area(cpu)->exception_stacks;
I
Ingo Molnar 已提交
1665

1666
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1667
			estacks += exception_stack_sizes[v];
1668
			oist->ist[v] = t->x86_tss.ist[v] =
1669
					(unsigned long)estacks;
1670 1671
			if (v == DEBUG_STACK-1)
				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1672 1673 1674
		}
	}

1675
	t->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
I
Ingo Molnar 已提交
1676

1677 1678 1679 1680 1681 1682 1683
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

V
Vegard Nossum 已提交
1684
	mmgrab(&init_mm);
1685
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1686
	BUG_ON(me->mm);
1687
	initialize_tlbstate_and_flush();
1688 1689
	enter_lazy_tlb(&init_mm, me);

1690
	/*
1691 1692
	 * Initialize the TSS.  sp0 points to the entry trampoline stack
	 * regardless of what task is running.
1693
	 */
1694
	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
1695
	load_TR_desc();
1696
	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
1697

1698
	load_mm_ldt(&init_mm);
1699

1700 1701
	clear_all_debug_regs();
	dbg_restore_debug_regs();
1702

I
Ingo Molnar 已提交
1703
	fpu__init_cpu();
1704 1705 1706

	if (is_uv_system())
		uv_cpu_init();
1707 1708

	load_fixmap_gdt(cpu);
1709 1710 1711 1712
}

#else

1713
void cpu_init(void)
1714
{
1715 1716
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1717
	struct tss_struct *t = &per_cpu(cpu_tss_rw, cpu);
1718

1719
	wait_for_master_cpu(cpu);
1720

1721 1722 1723 1724 1725 1726
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1727
	show_ucode_info_early();
1728

1729
	pr_info("Initializing CPU#%d\n", cpu);
1730

1731
	if (cpu_feature_enabled(X86_FEATURE_VME) ||
1732
	    boot_cpu_has(X86_FEATURE_TSC) ||
1733
	    boot_cpu_has(X86_FEATURE_DE))
A
Andy Lutomirski 已提交
1734
		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1735

1736
	load_current_idt();
1737
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1738 1739 1740 1741

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
V
Vegard Nossum 已提交
1742
	mmgrab(&init_mm);
1743
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1744
	BUG_ON(curr->mm);
1745
	initialize_tlbstate_and_flush();
1746
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1747

1748 1749 1750 1751
	/*
	 * Initialize the TSS.  Don't bother initializing sp0, as the initial
	 * task never enters user mode.
	 */
1752
	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
L
Linus Torvalds 已提交
1753
	load_TR_desc();
1754

1755
	load_mm_ldt(&init_mm);
L
Linus Torvalds 已提交
1756

1757
	t->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
1758

1759
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1760 1761
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1762
#endif
L
Linus Torvalds 已提交
1763

1764
	clear_all_debug_regs();
1765
	dbg_restore_debug_regs();
L
Linus Torvalds 已提交
1766

I
Ingo Molnar 已提交
1767
	fpu__init_cpu();
1768 1769

	load_fixmap_gdt(cpu);
L
Linus Torvalds 已提交
1770
}
1771
#endif
1772

1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788
static void bsp_resume(void)
{
	if (this_cpu->c_bsp_resume)
		this_cpu->c_bsp_resume(&boot_cpu_data);
}

static struct syscore_ops cpu_syscore_ops = {
	.resume		= bsp_resume,
};

static int __init init_cpu_syscore(void)
{
	register_syscore_ops(&cpu_syscore_ops);
	return 0;
}
core_initcall(init_cpu_syscore);
1789 1790 1791 1792 1793 1794 1795 1796

/*
 * The microcode loader calls this upon late microcode load to recheck features,
 * only when microcode has been updated. Caller holds microcode_mutex and CPU
 * hotplug lock.
 */
void microcode_check(void)
{
1797 1798
	struct cpuinfo_x86 info;

1799
	perf_check_microcode();
1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817

	/* Reload CPUID max function as it might've changed. */
	info.cpuid_level = cpuid_eax(0);

	/*
	 * Copy all capability leafs to pick up the synthetic ones so that
	 * memcmp() below doesn't fail on that. The ones coming from CPUID will
	 * get overwritten in get_cpu_cap().
	 */
	memcpy(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability));

	get_cpu_cap(&info);

	if (!memcmp(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability)))
		return;

	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
1818
}