common.c 47.6 KB
Newer Older
1 2 3
/* cpu_feature_enabled() cannot be used this early */
#define USE_EARLY_PGTABLE_L5

4
#include <linux/bootmem.h>
5
#include <linux/linkage.h>
6
#include <linux/bitops.h>
7
#include <linux/kernel.h>
8
#include <linux/export.h>
9 10
#include <linux/percpu.h>
#include <linux/string.h>
11
#include <linux/ctype.h>
L
Linus Torvalds 已提交
12
#include <linux/delay.h>
13
#include <linux/sched/mm.h>
14
#include <linux/sched/clock.h>
15
#include <linux/sched/task.h>
16
#include <linux/init.h>
17
#include <linux/kprobes.h>
18
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
19
#include <linux/smp.h>
20
#include <linux/io.h>
21
#include <linux/syscore_ops.h>
22 23

#include <asm/stackprotector.h>
24
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
25
#include <asm/mmu_context.h>
26
#include <asm/archrandom.h>
27 28
#include <asm/hypervisor.h>
#include <asm/processor.h>
29
#include <asm/tlbflush.h>
30
#include <asm/debugreg.h>
31
#include <asm/sections.h>
32
#include <asm/vsyscall.h>
A
Alan Cox 已提交
33 34
#include <linux/topology.h>
#include <linux/cpumask.h>
35
#include <asm/pgtable.h>
A
Arun Sharma 已提交
36
#include <linux/atomic.h>
37 38 39 40
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
41
#include <asm/fpu/internal.h>
42
#include <asm/mtrr.h>
43
#include <asm/hwcap2.h>
A
Alan Cox 已提交
44
#include <linux/numa.h>
45
#include <asm/asm.h>
46
#include <asm/bugs.h>
47
#include <asm/cpu.h>
48
#include <asm/mce.h>
49
#include <asm/msr.h>
50
#include <asm/pat.h>
51 52
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
53 54
#include <asm/intel-family.h>
#include <asm/cpu_device_id.h>
55 56

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
57
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
58 59 60 61
#endif

#include "cpu.h"

62 63
u32 elf_hwcap2 __read_mostly;

64 65
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
66 67
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
68 69 70 71

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

72 73 74 75 76 77 78
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;

B
Brian Gerst 已提交
79
/* correctly size the local cpu masks */
80
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
81 82 83 84 85 86 87
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

88
static void default_init(struct cpuinfo_x86 *c)
89 90
{
#ifdef CONFIG_X86_64
91
	cpu_detect_cache_sizes(c);
92 93 94 95 96 97 98 99 100 101 102 103 104
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

105
static const struct cpu_dev default_cpu = {
106 107 108 109 110
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

111
static const struct cpu_dev *this_cpu = &default_cpu;
112

B
Brian Gerst 已提交
113
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
114
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
115 116 117 118 119
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
120
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
121 122
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
123 124 125 126 127 128
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
129
#else
A
Akinobu Mita 已提交
130 131 132 133
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
134 135 136 137 138
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
139
	/* 32-bit code */
A
Akinobu Mita 已提交
140
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
141
	/* 16-bit code */
A
Akinobu Mita 已提交
142
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
143
	/* 16-bit data */
A
Akinobu Mita 已提交
144
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
145
	/* 16-bit data */
A
Akinobu Mita 已提交
146
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
147
	/* 16-bit data */
A
Akinobu Mita 已提交
148
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
149 150 151 152
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
153
	/* 32-bit code */
A
Akinobu Mita 已提交
154
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
155
	/* 16-bit code */
A
Akinobu Mita 已提交
156
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
157
	/* data */
158
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
159

A
Akinobu Mita 已提交
160 161
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
162
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
163
#endif
B
Brian Gerst 已提交
164
} };
165
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
166

167
static int __init x86_mpx_setup(char *s)
168
{
169
	/* require an exact match without trailing characters */
170 171
	if (strlen(s))
		return 0;
172

173 174 175
	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_MPX))
		return 1;
176

177 178
	setup_clear_cpu_cap(X86_FEATURE_MPX);
	pr_info("nompx: Intel Memory Protection Extensions (MPX) disabled\n");
179 180
	return 1;
}
181
__setup("nompx", x86_mpx_setup);
182

183
#ifdef CONFIG_X86_64
184
static int __init x86_nopcid_setup(char *s)
185
{
186 187 188
	/* nopcid doesn't accept parameters */
	if (s)
		return -EINVAL;
189 190 191

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_PCID))
192
		return 0;
193 194 195

	setup_clear_cpu_cap(X86_FEATURE_PCID);
	pr_info("nopcid: PCID feature disabled\n");
196
	return 0;
197
}
198
early_param("nopcid", x86_nopcid_setup);
199 200
#endif

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
static int __init x86_noinvpcid_setup(char *s)
{
	/* noinvpcid doesn't accept parameters */
	if (s)
		return -EINVAL;

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_INVPCID))
		return 0;

	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
	pr_info("noinvpcid: INVPCID feature disabled\n");
	return 0;
}
early_param("noinvpcid", x86_noinvpcid_setup);

217
#ifdef CONFIG_X86_32
218 219
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
L
Linus Torvalds 已提交
220

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

240 241 242 243 244 245 246
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
247 248 249 250 251 252 253 254 255 256 257
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

258 259
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
260 261 262 263 264

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
265
int have_cpuid_p(void)
266 267 268 269
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

270
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
271
{
I
Ingo Molnar 已提交
272 273 274 275 276 277 278 279 280 281 282
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

283
	pr_notice("CPU serial number disabled.\n");
I
Ingo Molnar 已提交
284 285 286 287
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
288 289 290 291 292 293 294 295
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
296
#else
297 298 299 300 301 302 303
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
304
#endif
305

306 307
static __init int setup_disable_smep(char *arg)
{
308
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
309 310
	/* Check for things that depend on SMEP being enabled: */
	check_mpx_erratum(&boot_cpu_data);
311 312 313 314
	return 1;
}
__setup("nosmep", setup_disable_smep);

315
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
316
{
317
	if (cpu_has(c, X86_FEATURE_SMEP))
A
Andy Lutomirski 已提交
318
		cr4_set_bits(X86_CR4_SMEP);
319 320
}

321 322
static __init int setup_disable_smap(char *arg)
{
323
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
324 325 326 327
	return 1;
}
__setup("nosmap", setup_disable_smap);

328 329
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
330
	unsigned long eflags = native_save_fl();
331 332 333 334

	/* This should have been cleared long ago */
	BUG_ON(eflags & X86_EFLAGS_AC);

335 336
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
A
Andy Lutomirski 已提交
337
		cr4_set_bits(X86_CR4_SMAP);
338
#else
A
Andy Lutomirski 已提交
339
		cr4_clear_bits(X86_CR4_SMAP);
340 341
#endif
	}
342 343
}

344 345 346 347 348 349 350 351 352 353 354 355
static __always_inline void setup_umip(struct cpuinfo_x86 *c)
{
	/* Check the boot processor, plus build option for UMIP. */
	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
		goto out;

	/* Check the current processor's cpuid bits. */
	if (!cpu_has(c, X86_FEATURE_UMIP))
		goto out;

	cr4_set_bits(X86_CR4_UMIP);

356 357
	pr_info("x86/cpu: Activated the Intel User Mode Instruction Prevention (UMIP) CPU feature\n");

358 359 360 361 362 363 364 365 366 367
	return;

out:
	/*
	 * Make sure UMIP is disabled in case it was enabled in a
	 * previous boot (e.g., via kexec).
	 */
	cr4_clear_bits(X86_CR4_UMIP);
}

368 369 370 371 372 373 374
/*
 * Protection Keys are not available in 32-bit mode.
 */
static bool pku_disabled;

static __always_inline void setup_pku(struct cpuinfo_x86 *c)
{
375 376 377 378
	/* check the boot processor, plus compile options for PKU: */
	if (!cpu_feature_enabled(X86_FEATURE_PKU))
		return;
	/* checks the actual processor's cpuid bits: */
379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
	if (!cpu_has(c, X86_FEATURE_PKU))
		return;
	if (pku_disabled)
		return;

	cr4_set_bits(X86_CR4_PKE);
	/*
	 * Seting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
	 * cpuid bit to be set.  We need to ensure that we
	 * update that bit in this CPU's "cpu_info".
	 */
	get_cpu_cap(c);
}

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
static __init int setup_disable_pku(char *arg)
{
	/*
	 * Do not clear the X86_FEATURE_PKU bit.  All of the
	 * runtime checks are against OSPKE so clearing the
	 * bit does nothing.
	 *
	 * This way, we will see "pku" in cpuinfo, but not
	 * "ospke", which is exactly what we want.  It shows
	 * that the CPU has PKU, but the OS has not enabled it.
	 * This happens to be exactly how a system would look
	 * if we disabled the config option.
	 */
	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
	pku_disabled = true;
	return 1;
}
__setup("nopku", setup_disable_pku);
#endif /* CONFIG_X86_64 */

414 415 416 417 418 419 420 421 422
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
423

424
static const struct cpuid_dependent_feature
425 426 427 428 429 430 431
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

432
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
433 434
{
	const struct cpuid_dependent_feature *df;
435

436
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
437 438 439

		if (!cpu_has(c, df->feature))
			continue;
440 441 442 443 444 445 446
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
447
		if (!((s32)df->level < 0 ?
448
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
449 450 451 452 453 454 455
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

456 457
		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
			x86_cap_flag(df->feature), df->level);
458
	}
459
}
460

461 462 463
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
464 465
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
466 467 468
 */

/* Look up CPU names by table lookup. */
469
static const char *table_lookup_model(struct cpuinfo_x86 *c)
470
{
471 472
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
473 474 475 476 477 478 479

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

480
	info = this_cpu->legacy_models;
481

482
	while (info->family) {
483 484 485 486
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
487
#endif
488 489 490
	return NULL;		/* Not found */
}

491 492
__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS];
__u32 cpu_caps_set[NCAPINTS + NBUGINTS];
493

494 495 496 497 498
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
499
	__loadsegment_simple(gs, 0);
500
	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
501
#endif
502
	load_stack_canary_segment();
503 504
}

505 506 507 508 509
#ifdef CONFIG_X86_32
/* The 32-bit entry code needs to find cpu_entry_area. */
DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
#endif

510 511 512 513 514 515 516 517 518 519 520
#ifdef CONFIG_X86_64
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};
521
#endif
522

523 524 525 526 527 528 529 530 531 532 533
/* Load the original GDT from the per-cpu structure */
void load_direct_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
EXPORT_SYMBOL_GPL(load_direct_gdt);

534 535 536 537 538 539 540 541 542
/* Load a fixmap remapping of the per-cpu GDT */
void load_fixmap_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
543
EXPORT_SYMBOL_GPL(load_fixmap_gdt);
544

I
Ingo Molnar 已提交
545 546 547 548
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
549
void switch_to_new_gdt(int cpu)
550
{
551 552
	/* Load the original GDT */
	load_direct_gdt(cpu);
553
	/* Reload the per-cpu base */
554
	load_percpu_segment(cpu);
555 556
}

557
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
558

559
static void get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
560 561
{
	unsigned int *v;
562
	char *p, *q, *s;
L
Linus Torvalds 已提交
563

564
	if (c->extended_cpuid_level < 0x80000004)
565
		return;
L
Linus Torvalds 已提交
566

I
Ingo Molnar 已提交
567
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
568 569 570 571 572
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	/* Trim whitespace */
	p = q = s = &c->x86_model_id[0];

	while (*p == ' ')
		p++;

	while (*p) {
		/* Note the last non-whitespace index */
		if (!isspace(*p))
			s = q;

		*q++ = *p++;
	}

	*(s + 1) = '\0';
L
Linus Torvalds 已提交
588 589
}

590
void detect_num_cpu_cores(struct cpuinfo_x86 *c)
591 592 593
{
	unsigned int eax, ebx, ecx, edx;

594
	c->x86_max_cores = 1;
595
	if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
596
		return;
597 598 599

	cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
	if (eax & 0x1f)
600
		c->x86_max_cores = (eax >> 26) + 1;
601 602
}

603
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
604
{
605
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
606

607
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
608 609

	if (n >= 0x80000005) {
610 611
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
612 613 614 615
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
616 617 618 619 620
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

621
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
622
	l2size = ecx >> 16;
623

624 625 626
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
627
	/* do processor-specific cache resizing */
628 629
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
L
Linus Torvalds 已提交
630 631 632 633 634

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

635
	if (l2size == 0)
L
Linus Torvalds 已提交
636
		return;		/* Again, no L2 cache is possible */
637
#endif
L
Linus Torvalds 已提交
638 639 640 641

	c->x86_cache_size = l2size;
}

642 643 644 645 646 647
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
648
u16 __read_mostly tlb_lld_1g[NR_INFO];
649

650
static void cpu_detect_tlb(struct cpuinfo_x86 *c)
651 652 653 654
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

655
	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
656
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
657 658 659 660 661
		tlb_lli_4m[ENTRIES]);

	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
662 663
}

664
int detect_ht_early(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
665
{
B
Borislav Petkov 已提交
666
#ifdef CONFIG_SMP
667
	u32 eax, ebx, ecx, edx;
L
Linus Torvalds 已提交
668

669
	if (!cpu_has(c, X86_FEATURE_HT))
670
		return -1;
L
Linus Torvalds 已提交
671

672
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
673
		return -1;
L
Linus Torvalds 已提交
674

675
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
676
		return -1;
L
Linus Torvalds 已提交
677

678
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
679

680
	smp_num_siblings = (ebx & 0xff0000) >> 16;
681
	if (smp_num_siblings == 1)
682
		pr_info_once("CPU0: Hyper-Threading is disabled\n");
683 684 685
#endif
	return 0;
}
686

687 688 689 690
void detect_ht(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_SMP
	int index_msb, core_bits;
691

692
	if (detect_ht_early(c) < 0)
693
		return;
694

I
Ingo Molnar 已提交
695 696
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
697

I
Ingo Molnar 已提交
698
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
699

I
Ingo Molnar 已提交
700
	index_msb = get_count_order(smp_num_siblings);
701

I
Ingo Molnar 已提交
702
	core_bits = get_count_order(c->x86_max_cores);
703

I
Ingo Molnar 已提交
704 705
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
706
#endif
707
}
L
Linus Torvalds 已提交
708

709
static void get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
710 711
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
712
	int i;
L
Linus Torvalds 已提交
713 714

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
715 716 717 718 719 720
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
721

Y
Yinghai Lu 已提交
722 723 724
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
725 726
		}
	}
Y
Yinghai Lu 已提交
727

728 729
	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
		    "CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
730

731 732
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
733 734
}

735
void cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
736 737
{
	/* Get vendor name */
738 739 740 741
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
742 743

	c->x86 = 4;
744
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
745 746
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
747

L
Linus Torvalds 已提交
748
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
749 750
		c->x86		= x86_family(tfms);
		c->x86_model	= x86_model(tfms);
751
		c->x86_stepping	= x86_stepping(tfms);
I
Ingo Molnar 已提交
752

H
Huang, Ying 已提交
753 754
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
755
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
756
		}
L
Linus Torvalds 已提交
757 758
	}
}
759

760 761 762 763
static void apply_forced_caps(struct cpuinfo_x86 *c)
{
	int i;

764
	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
765 766 767 768 769
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}
}

770 771 772 773 774 775 776 777 778 779 780
static void init_speculation_control(struct cpuinfo_x86 *c)
{
	/*
	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
	 * and they also have a different bit for STIBP support. Also,
	 * a hypervisor might have set the individual AMD bits even on
	 * Intel CPUs, for finer-grained selection of what's available.
	 */
	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
		set_cpu_cap(c, X86_FEATURE_IBRS);
		set_cpu_cap(c, X86_FEATURE_IBPB);
781
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
782
	}
783

784 785
	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
		set_cpu_cap(c, X86_FEATURE_STIBP);
786

787 788
	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
	    cpu_has(c, X86_FEATURE_VIRT_SSBD))
789 790
		set_cpu_cap(c, X86_FEATURE_SSBD);

791
	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
792
		set_cpu_cap(c, X86_FEATURE_IBRS);
793 794
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
	}
795 796 797 798

	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
		set_cpu_cap(c, X86_FEATURE_IBPB);

799
	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
800
		set_cpu_cap(c, X86_FEATURE_STIBP);
801 802
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
	}
803 804 805 806 807 808

	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
		set_cpu_cap(c, X86_FEATURE_SSBD);
		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
		clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
	}
809 810
}

811
void get_cpu_cap(struct cpuinfo_x86 *c)
812
{
813
	u32 eax, ebx, ecx, edx;
814

815 816
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
817
		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
818

819 820
		c->x86_capability[CPUID_1_ECX] = ecx;
		c->x86_capability[CPUID_1_EDX] = edx;
821
	}
822

823 824 825 826
	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
	if (c->cpuid_level >= 0x00000006)
		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);

827 828 829
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
830
		c->x86_capability[CPUID_7_0_EBX] = ebx;
831
		c->x86_capability[CPUID_7_ECX] = ecx;
832
		c->x86_capability[CPUID_7_EDX] = edx;
833 834
	}

835 836 837 838
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

839
		c->x86_capability[CPUID_D_1_EAX] = eax;
840 841
	}

842 843 844 845 846
	/* Additional Intel-defined flags: level 0x0000000F */
	if (c->cpuid_level >= 0x0000000F) {

		/* QoS sub-leaf, EAX=0Fh, ECX=0 */
		cpuid_count(0x0000000F, 0, &eax, &ebx, &ecx, &edx);
847 848
		c->x86_capability[CPUID_F_0_EDX] = edx;

849 850 851 852 853 854
		if (cpu_has(c, X86_FEATURE_CQM_LLC)) {
			/* will be overridden if occupancy monitoring exists */
			c->x86_cache_max_rmid = ebx;

			/* QoS sub-leaf, EAX=0Fh, ECX=1 */
			cpuid_count(0x0000000F, 1, &eax, &ebx, &ecx, &edx);
855 856
			c->x86_capability[CPUID_F_1_EDX] = edx;

857 858 859
			if ((cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC)) ||
			      ((cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL)) ||
			       (cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL)))) {
860 861 862 863 864 865 866 867 868
				c->x86_cache_max_rmid = ecx;
				c->x86_cache_occ_scale = ebx;
			}
		} else {
			c->x86_cache_max_rmid = -1;
			c->x86_cache_occ_scale = -1;
		}
	}

869
	/* AMD-defined flags: level 0x80000001 */
870 871 872 873 874 875
	eax = cpuid_eax(0x80000000);
	c->extended_cpuid_level = eax;

	if ((eax & 0xffff0000) == 0x80000000) {
		if (eax >= 0x80000001) {
			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
876

877 878
			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
			c->x86_capability[CPUID_8000_0001_EDX] = edx;
879 880 881
		}
	}

882 883 884 885 886 887 888
	if (c->extended_cpuid_level >= 0x80000007) {
		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);

		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
		c->x86_power = edx;
	}

889 890 891 892 893
	if (c->extended_cpuid_level >= 0x80000008) {
		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
	}

894
	if (c->extended_cpuid_level >= 0x8000000a)
895
		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
896

897
	init_scattered_cpuid_features(c);
898
	init_speculation_control(c);
899 900 901 902 903 904 905

	/*
	 * Clear/Set all flags overridden by options, after probe.
	 * This needs to happen each time we re-probe, which may happen
	 * several times during CPU initialization.
	 */
	apply_forced_caps(c);
906
}
L
Linus Torvalds 已提交
907

908
void get_cpu_address_sizes(struct cpuinfo_x86 *c)
909 910 911 912 913 914 915 916 917 918 919 920 921
{
	u32 eax, ebx, ecx, edx;

	if (c->extended_cpuid_level >= 0x80000008) {
		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
	}
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
#endif
922
	c->x86_cache_bits = c->x86_phys_bits;
923 924
}

925
static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
Y
Yinghai Lu 已提交
926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

951 952 953 954
#define NO_SPECULATION	BIT(0)
#define NO_MELTDOWN	BIT(1)
#define NO_SSB		BIT(2)
#define NO_L1TF		BIT(3)
955
#define NO_MDS		BIT(4)
956
#define MSBDS_ONLY	BIT(5)
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972

#define VULNWL(_vendor, _family, _model, _whitelist)	\
	{ X86_VENDOR_##_vendor, _family, _model, X86_FEATURE_ANY, _whitelist }

#define VULNWL_INTEL(model, whitelist)		\
	VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)

#define VULNWL_AMD(family, whitelist)		\
	VULNWL(AMD, family, X86_MODEL_ANY, whitelist)

static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
	VULNWL(ANY,	4, X86_MODEL_ANY,	NO_SPECULATION),
	VULNWL(CENTAUR,	5, X86_MODEL_ANY,	NO_SPECULATION),
	VULNWL(INTEL,	5, X86_MODEL_ANY,	NO_SPECULATION),
	VULNWL(NSC,	5, X86_MODEL_ANY,	NO_SPECULATION),

973
	/* Intel Family 6 */
974 975 976 977 978 979
	VULNWL_INTEL(ATOM_SALTWELL,		NO_SPECULATION),
	VULNWL_INTEL(ATOM_SALTWELL_TABLET,	NO_SPECULATION),
	VULNWL_INTEL(ATOM_SALTWELL_MID,		NO_SPECULATION),
	VULNWL_INTEL(ATOM_BONNELL,		NO_SPECULATION),
	VULNWL_INTEL(ATOM_BONNELL_MID,		NO_SPECULATION),

980 981 982 983 984 985
	VULNWL_INTEL(ATOM_SILVERMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY),
	VULNWL_INTEL(ATOM_SILVERMONT_X,		NO_SSB | NO_L1TF | MSBDS_ONLY),
	VULNWL_INTEL(ATOM_SILVERMONT_MID,	NO_SSB | NO_L1TF | MSBDS_ONLY),
	VULNWL_INTEL(ATOM_AIRMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY),
	VULNWL_INTEL(XEON_PHI_KNL,		NO_SSB | NO_L1TF | MSBDS_ONLY),
	VULNWL_INTEL(XEON_PHI_KNM,		NO_SSB | NO_L1TF | MSBDS_ONLY),
986 987 988

	VULNWL_INTEL(CORE_YONAH,		NO_SSB),

989
	VULNWL_INTEL(ATOM_AIRMONT_MID,		NO_L1TF | MSBDS_ONLY),
990

991 992 993 994 995 996 997 998 999
	VULNWL_INTEL(ATOM_GOLDMONT,		NO_MDS | NO_L1TF),
	VULNWL_INTEL(ATOM_GOLDMONT_X,		NO_MDS | NO_L1TF),
	VULNWL_INTEL(ATOM_GOLDMONT_PLUS,	NO_MDS | NO_L1TF),

	/* AMD Family 0xf - 0x12 */
	VULNWL_AMD(0x0f,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS),
	VULNWL_AMD(0x10,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS),
	VULNWL_AMD(0x11,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS),
	VULNWL_AMD(0x12,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS),
1000 1001

	/* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1002
	VULNWL_AMD(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS),
1003 1004 1005
	{}
};

1006 1007 1008
static bool __init cpu_matches(unsigned long which)
{
	const struct x86_cpu_id *m = x86_match_cpu(cpu_vuln_whitelist);
1009

1010 1011
	return m && !!(m->driver_data & which);
}
1012

1013
static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1014 1015 1016
{
	u64 ia32_cap = 0;

1017
	if (cpu_matches(NO_SPECULATION))
1018 1019 1020 1021 1022
		return;

	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);

1023 1024 1025
	if (cpu_has(c, X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);

1026
	if (!cpu_matches(NO_SSB) && !(ia32_cap & ARCH_CAP_SSB_NO) &&
1027
	   !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1028 1029
		setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);

1030 1031 1032
	if (ia32_cap & ARCH_CAP_IBRS_ALL)
		setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);

1033
	if (!cpu_matches(NO_MDS) && !(ia32_cap & ARCH_CAP_MDS_NO)) {
1034
		setup_force_cpu_bug(X86_BUG_MDS);
1035 1036 1037
		if (cpu_matches(MSBDS_ONLY))
			setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
	}
1038

1039
	if (cpu_matches(NO_MELTDOWN))
1040
		return;
1041 1042 1043

	/* Rogue Data Cache Load? No! */
	if (ia32_cap & ARCH_CAP_RDCL_NO)
1044
		return;
1045

1046
	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1047

1048
	if (cpu_matches(NO_L1TF))
1049 1050 1051
		return;

	setup_force_cpu_bug(X86_BUG_L1TF);
1052 1053
}

1054 1055 1056 1057 1058 1059 1060 1061 1062
/*
 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
 * unfortunately, that's not true in practice because of early VIA
 * chips and (more importantly) broken virtualizers that are not easy
 * to detect. In the latter case it doesn't even *fail* reliably, so
 * probing for it doesn't even work. Disable it completely on 32-bit
 * unless we can find a reliable way to detect all the broken cases.
 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
 */
1063
static void detect_nopl(void)
1064 1065
{
#ifdef CONFIG_X86_32
1066
	setup_clear_cpu_cap(X86_FEATURE_NOPL);
1067
#else
1068
	setup_force_cpu_cap(X86_FEATURE_NOPL);
1069 1070 1071
#endif
}

1072 1073 1074 1075 1076 1077
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
1078 1079
 * WARNING: this function is only called on the boot CPU.  Don't add code
 * here that is supposed to run on all CPUs.
1080
 */
1081
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1082
{
1083 1084
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
1085 1086
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
1087
#else
H
Huang, Ying 已提交
1088
	c->x86_clflush_size = 32;
1089 1090
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
1091
#endif
1092
	c->x86_cache_alignment = c->x86_clflush_size;
1093

1094
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
1095
	c->extended_cpuid_level = 0;
1096

Y
Yinghai Lu 已提交
1097
	/* cyrix could have cpuid enabled via c_identify()*/
1098 1099 1100 1101
	if (have_cpuid_p()) {
		cpu_detect(c);
		get_cpu_vendor(c);
		get_cpu_cap(c);
1102
		get_cpu_address_sizes(c);
B
Borislav Petkov 已提交
1103
		setup_force_cpu_cap(X86_FEATURE_CPUID);
1104

1105 1106
		if (this_cpu->c_early_init)
			this_cpu->c_early_init(c);
1107

1108 1109
		c->cpu_index = 0;
		filter_cpuid_features(c, false);
1110

1111 1112
		if (this_cpu->c_bsp_init)
			this_cpu->c_bsp_init(c);
B
Borislav Petkov 已提交
1113 1114 1115
	} else {
		identify_cpu_without_cpuid(c);
		setup_clear_cpu_cap(X86_FEATURE_CPUID);
1116
	}
1117 1118

	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1119

1120
	cpu_set_bug_bits(c);
1121

1122
	fpu__init_system(c);
1123 1124 1125 1126 1127 1128 1129 1130

#ifdef CONFIG_X86_32
	/*
	 * Regardless of whether PCID is enumerated, the SDM says
	 * that it can't be enabled in 32-bit mode.
	 */
	setup_clear_cpu_cap(X86_FEATURE_PCID);
#endif
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145

	/*
	 * Later in the boot process pgtable_l5_enabled() relies on
	 * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
	 * enabled by this point we need to clear the feature bit to avoid
	 * false-positives at the later stage.
	 *
	 * pgtable_l5_enabled() can be false here for several reasons:
	 *  - 5-level paging is disabled compile-time;
	 *  - it's 32-bit kernel;
	 *  - machine doesn't support 5-level paging;
	 *  - user specified 'no5lvl' in kernel command line.
	 */
	if (!pgtable_l5_enabled())
		setup_clear_cpu_cap(X86_FEATURE_LA57);
1146

1147
	detect_nopl();
1148 1149
}

1150 1151
void __init early_cpu_init(void)
{
1152
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
1153 1154
	int count = 0;

1155
#ifdef CONFIG_PROCESSOR_SELECT
1156
	pr_info("KERNEL supported cpus:\n");
1157 1158
#endif

Y
Yinghai Lu 已提交
1159
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1160
		const struct cpu_dev *cpudev = *cdev;
1161

Y
Yinghai Lu 已提交
1162 1163 1164 1165 1166
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

1167
#ifdef CONFIG_PROCESSOR_SELECT
1168 1169 1170 1171 1172 1173
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
1174
				pr_info("  %s %s\n", cpudev->c_vendor,
1175 1176
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
1177
		}
1178
#endif
Y
Yinghai Lu 已提交
1179
	}
1180
	early_identify_cpu(&boot_cpu_data);
1181
}
1182

1183 1184 1185
static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
1186
	/*
1187 1188 1189 1190 1191
	 * Empirically, writing zero to a segment selector on AMD does
	 * not clear the base, whereas writing zero to a segment
	 * selector on Intel does clear the base.  Intel's behavior
	 * allows slightly faster context switches in the common case
	 * where GS is unused by the prev and next threads.
1192
	 *
1193 1194 1195 1196 1197 1198
	 * Since neither vendor documents this anywhere that I can see,
	 * detect it directly instead of hardcoding the choice by
	 * vendor.
	 *
	 * I've designated AMD's behavior as the "bug" because it's
	 * counterintuitive and less friendly.
1199
	 */
1200 1201 1202 1203 1204 1205 1206 1207 1208

	unsigned long old_base, tmp;
	rdmsrl(MSR_FS_BASE, old_base);
	wrmsrl(MSR_FS_BASE, 1);
	loadsegment(fs, 0);
	rdmsrl(MSR_FS_BASE, tmp);
	if (tmp != 0)
		set_cpu_bug(c, X86_BUG_NULL_SEG);
	wrmsrl(MSR_FS_BASE, old_base);
B
Borislav Petkov 已提交
1209
#endif
1210 1211
}

1212
static void generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1213
{
Y
Yinghai Lu 已提交
1214
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
1215

1216
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
1217
		identify_cpu_without_cpuid(c);
1218

Y
Yinghai Lu 已提交
1219
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
1220
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
1221
		return;
L
Linus Torvalds 已提交
1222

1223
	cpu_detect(c);
L
Linus Torvalds 已提交
1224

1225
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
1226

1227
	get_cpu_cap(c);
L
Linus Torvalds 已提交
1228

1229 1230
	get_cpu_address_sizes(c);

1231 1232
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1233
#ifdef CONFIG_X86_32
B
Borislav Petkov 已提交
1234
# ifdef CONFIG_SMP
1235
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1236
# else
1237
		c->apicid = c->initial_apicid;
1238 1239 1240
# endif
#endif
		c->phys_proc_id = c->initial_apicid;
1241
	}
L
Linus Torvalds 已提交
1242

1243
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
1244

1245
	detect_null_seg_behavior(c);
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270

	/*
	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
	 * systems that run Linux at CPL > 0 may or may not have the
	 * issue, but, even if they have the issue, there's absolutely
	 * nothing we can do about it because we can't use the real IRET
	 * instruction.
	 *
	 * NB: For the time being, only 32-bit kernels support
	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
	 * whether to apply espfix using paravirt hooks.  If any
	 * non-paravirt system ever shows up that does *not* have the
	 * ESPFIX issue, we can change this.
	 */
#ifdef CONFIG_X86_32
# ifdef CONFIG_PARAVIRT
	do {
		extern void native_iret(void);
		if (pv_cpu_ops.iret == native_iret)
			set_cpu_bug(c, X86_BUG_ESPFIX);
	} while (0);
# else
	set_cpu_bug(c, X86_BUG_ESPFIX);
# endif
#endif
L
Linus Torvalds 已提交
1271 1272
}

1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
static void x86_init_cache_qos(struct cpuinfo_x86 *c)
{
	/*
	 * The heavy lifting of max_rmid and cache_occ_scale are handled
	 * in get_cpu_cap().  Here we just set the max_rmid for the boot_cpu
	 * in case CQM bits really aren't there in this CPU.
	 */
	if (c != &boot_cpu_data) {
		boot_cpu_data.x86_cache_max_rmid =
			min(boot_cpu_data.x86_cache_max_rmid,
			    c->x86_cache_max_rmid);
	}
}

1287
/*
1288 1289
 * Validate that ACPI/mptables have the same information about the
 * effective APIC id and update the package map.
1290
 */
1291
static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1292 1293
{
#ifdef CONFIG_SMP
1294
	unsigned int apicid, cpu = smp_processor_id();
1295 1296 1297

	apicid = apic->cpu_present_to_apicid(cpu);

1298 1299
	if (apicid != c->apicid) {
		pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1300 1301
		       cpu, apicid, c->initial_apicid);
	}
1302
	BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1303 1304 1305 1306 1307
#else
	c->logical_proc_id = 0;
#endif
}

L
Linus Torvalds 已提交
1308 1309 1310
/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
1311
static void identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1312 1313 1314 1315
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
1316
	c->x86_cache_size = 0;
L
Linus Torvalds 已提交
1317
	c->x86_vendor = X86_VENDOR_UNKNOWN;
1318
	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
L
Linus Torvalds 已提交
1319 1320
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
1321
	c->x86_max_cores = 1;
1322
	c->x86_coreid_bits = 0;
1323
	c->cu_id = 0xff;
1324
#ifdef CONFIG_X86_64
1325
	c->x86_clflush_size = 64;
1326 1327
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
1328 1329
#else
	c->cpuid_level = -1;	/* CPUID not detected */
1330
	c->x86_clflush_size = 32;
1331 1332
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
1333 1334
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
1335 1336 1337 1338
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

1339
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
1340 1341
		this_cpu->c_identify(c);

1342
	/* Clear/Set all flags overridden by options, after probe */
1343
	apply_forced_caps(c);
1344

1345
#ifdef CONFIG_X86_64
1346
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1347 1348
#endif

L
Linus Torvalds 已提交
1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

1365
	/* Set up SMEP/SMAP/UMIP */
1366 1367
	setup_smep(c);
	setup_smap(c);
1368
	setup_umip(c);
1369

L
Linus Torvalds 已提交
1370
	/*
I
Ingo Molnar 已提交
1371 1372
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
1373 1374
	 */

1375 1376 1377
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
1378
	/* If the model name is still unset, do table lookup. */
1379
	if (!c->x86_model_id[0]) {
1380
		const char *p;
L
Linus Torvalds 已提交
1381
		p = table_lookup_model(c);
1382
		if (p)
L
Linus Torvalds 已提交
1383 1384 1385 1386
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
1387
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
1388 1389
	}

1390 1391 1392 1393
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

1394
	x86_init_rdrand(c);
1395
	x86_init_cache_qos(c);
1396
	setup_pku(c);
1397 1398

	/*
1399
	 * Clear/Set all flags overridden by options, need do it
1400 1401
	 * before following smp all cpus cap AND.
	 */
1402
	apply_forced_caps(c);
1403

L
Linus Torvalds 已提交
1404 1405 1406 1407 1408 1409
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
1410
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
1411
		/* AND the already accumulated flags with these */
1412
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
1413
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1414 1415 1416 1417

		/* OR, i.e. replicate the bug flags */
		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
L
Linus Torvalds 已提交
1418 1419 1420
	}

	/* Init Machine Check Exception if available. */
1421
	mcheck_cpu_init(c);
1422 1423

	select_idle_routine(c);
1424

1425
#ifdef CONFIG_NUMA
1426 1427
	numa_add_cpu(smp_processor_id());
#endif
1428
}
S
Shaohua Li 已提交
1429

1430 1431 1432 1433
/*
 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
 * on 32-bit kernels:
 */
1434 1435 1436
#ifdef CONFIG_X86_32
void enable_sep_cpu(void)
{
1437 1438
	struct tss_struct *tss;
	int cpu;
1439

1440 1441 1442
	if (!boot_cpu_has(X86_FEATURE_SEP))
		return;

1443
	cpu = get_cpu();
1444
	tss = &per_cpu(cpu_tss_rw, cpu);
1445 1446

	/*
1447 1448
	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
	 * see the big comment in struct x86_hw_tss's definition.
1449
	 */
1450 1451

	tss->x86_tss.ss1 = __KERNEL_CS;
1452
	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1453
	wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1454
	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1455

1456 1457
	put_cpu();
}
1458 1459
#endif

1460 1461 1462
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
1463
#ifdef CONFIG_X86_32
1464
	sysenter_setup();
L
Li Shaohua 已提交
1465
	enable_sep_cpu();
1466
#endif
1467
	cpu_detect_tlb(&boot_cpu_data);
1468
}
S
Shaohua Li 已提交
1469

1470
void identify_secondary_cpu(struct cpuinfo_x86 *c)
1471 1472 1473
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
1474
#ifdef CONFIG_X86_32
1475
	enable_sep_cpu();
1476
#endif
1477
	mtrr_ap_init();
1478
	validate_apic_and_package_id(c);
1479
	x86_spec_ctrl_setup_ap();
L
Linus Torvalds 已提交
1480 1481
}

A
Andi Kleen 已提交
1482 1483
static __init int setup_noclflush(char *arg)
{
1484
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1485
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
A
Andi Kleen 已提交
1486 1487 1488 1489
	return 1;
}
__setup("noclflush", setup_noclflush);

1490
void print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1491
{
1492
	const char *vendor = NULL;
L
Linus Torvalds 已提交
1493

I
Ingo Molnar 已提交
1494
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
1495
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
1496 1497 1498 1499
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
1500

1501
	if (vendor && !strstr(c->x86_model_id, vendor))
1502
		pr_cont("%s ", vendor);
L
Linus Torvalds 已提交
1503

1504
	if (c->x86_model_id[0])
1505
		pr_cont("%s", c->x86_model_id);
L
Linus Torvalds 已提交
1506
	else
1507
		pr_cont("%d86", c->x86);
L
Linus Torvalds 已提交
1508

1509
	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1510

1511 1512
	if (c->x86_stepping || c->cpuid_level >= 0)
		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
L
Linus Torvalds 已提交
1513
	else
1514
		pr_cont(")\n");
L
Linus Torvalds 已提交
1515 1516
}

1517 1518 1519 1520 1521 1522
/*
 * clearcpuid= was already parsed in fpu__init_parse_early_param.
 * But we need to keep a dummy __setup around otherwise it would
 * show up as an environment variable for init.
 */
static __init int setup_clearcpuid(char *arg)
1523 1524 1525
{
	return 1;
}
1526
__setup("clearcpuid=", setup_clearcpuid);
1527

1528
#ifdef CONFIG_X86_64
1529
DEFINE_PER_CPU_FIRST(union irq_stack_union,
1530
		     irq_stack_union) __aligned(PAGE_SIZE) __visible;
1531
EXPORT_PER_CPU_SYMBOL_GPL(irq_stack_union);
I
Ingo Molnar 已提交
1532

1533
/*
1534 1535
 * The following percpu variables are hot.  Align current_task to
 * cacheline size such that they fall in the same cacheline.
1536 1537 1538 1539
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1540

1541
DEFINE_PER_CPU(char *, irq_stack_ptr) =
1542
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE;
1543

1544
DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1545

1546 1547 1548
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);

1549 1550
/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1551
{
1552 1553 1554
	extern char _entry_trampoline[];
	extern char entry_SYSCALL_64_trampoline[];

1555
	int cpu = smp_processor_id();
1556 1557 1558
	unsigned long SYSCALL64_entry_trampoline =
		(unsigned long)get_cpu_entry_area(cpu)->entry_trampoline +
		(entry_SYSCALL_64_trampoline - _entry_trampoline);
1559

1560
	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
1561 1562 1563 1564
	if (static_cpu_has(X86_FEATURE_PTI))
		wrmsrl(MSR_LSTAR, SYSCALL64_entry_trampoline);
	else
		wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
1565 1566

#ifdef CONFIG_IA32_EMULATION
1567
	wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat);
1568
	/*
1569 1570 1571 1572
	 * This only works on Intel CPUs.
	 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
	 * This does not cause SYSENTER to jump to the wrong location, because
	 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
1573 1574
	 */
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
1575
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1));
1576
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
1577
#else
1578
	wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
1579
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1580 1581
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1582
#endif
1583

1584 1585
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
1586
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1587
	       X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
L
Linus Torvalds 已提交
1588
}
1589

1590 1591 1592 1593 1594 1595
/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

1596
static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
1597
DEFINE_PER_CPU(int, debug_stack_usage);
1598 1599 1600

int is_debug_stack(unsigned long addr)
{
1601 1602 1603
	return __this_cpu_read(debug_stack_usage) ||
		(addr <= __this_cpu_read(debug_stack_addr) &&
		 addr > (__this_cpu_read(debug_stack_addr) - DEBUG_STKSZ));
1604
}
1605
NOKPROBE_SYMBOL(is_debug_stack);
1606

1607
DEFINE_PER_CPU(u32, debug_idt_ctr);
1608

1609 1610
void debug_stack_set_zero(void)
{
1611 1612
	this_cpu_inc(debug_idt_ctr);
	load_current_idt();
1613
}
1614
NOKPROBE_SYMBOL(debug_stack_set_zero);
1615 1616 1617

void debug_stack_reset(void)
{
1618
	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
1619
		return;
1620 1621
	if (this_cpu_dec_return(debug_idt_ctr) == 0)
		load_current_idt();
1622
}
1623
NOKPROBE_SYMBOL(debug_stack_reset);
1624

I
Ingo Molnar 已提交
1625
#else	/* CONFIG_X86_64 */
1626

1627 1628
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1629 1630
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
1631

1632 1633 1634 1635 1636 1637 1638 1639 1640
/*
 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
 * the top of the kernel stack.  Use an extra percpu variable to track the
 * top of the kernel stack directly.
 */
DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
	(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);

1641
#ifdef CONFIG_STACKPROTECTOR
1642
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1643
#endif
1644

I
Ingo Molnar 已提交
1645
#endif	/* CONFIG_X86_64 */
1646

1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1662

1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
#ifdef CONFIG_KGDB
/*
 * Restore debug regs if using kgdbwait and you have a kernel debugger
 * connection established.
 */
static void dbg_restore_debug_regs(void)
{
	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
		arch_kgdb_ops.correct_hw_break();
}
#else /* ! CONFIG_KGDB */
#define dbg_restore_debug_regs()
#endif /* ! CONFIG_KGDB */

1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
static void wait_for_master_cpu(int cpu)
{
#ifdef CONFIG_SMP
	/*
	 * wait for ACK from master CPU before continuing
	 * with AP initialization
	 */
	WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
	while (!cpumask_test_cpu(cpu, cpu_callout_mask))
		cpu_relax();
#endif
}

1690 1691 1692 1693 1694
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1695
 * A lot of state is already set up in PDA init for 64 bit
1696
 */
1697
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1698

1699
void cpu_init(void)
1700
{
1701
	struct orig_ist *oist;
1702
	struct task_struct *me;
I
Ingo Molnar 已提交
1703 1704
	struct tss_struct *t;
	unsigned long v;
1705
	int cpu = raw_smp_processor_id();
1706 1707
	int i;

1708 1709
	wait_for_master_cpu(cpu);

1710 1711 1712 1713 1714 1715
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1716 1717
	if (cpu)
		load_ucode_ap();
1718

1719
	t = &per_cpu(cpu_tss_rw, cpu);
1720
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1721

1722
#ifdef CONFIG_NUMA
1723
	if (this_cpu_read(numa_node) == 0 &&
1724 1725
	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
		set_numa_node(early_cpu_to_node(cpu));
1726
#endif
1727 1728 1729

	me = current;

1730
	pr_debug("Initializing CPU#%d\n", cpu);
1731

A
Andy Lutomirski 已提交
1732
	cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1733 1734 1735 1736 1737 1738

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1739
	switch_to_new_gdt(cpu);
1740 1741
	loadsegment(fs, 0);

1742
	load_current_idt();
1743 1744 1745 1746 1747 1748 1749 1750

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1751
	x86_configure_nx();
1752
	x2apic_setup();
1753 1754 1755 1756

	/*
	 * set up and load the per-CPU TSS
	 */
1757
	if (!oist->ist[0]) {
1758
		char *estacks = get_cpu_entry_area(cpu)->exception_stacks;
I
Ingo Molnar 已提交
1759

1760
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1761
			estacks += exception_stack_sizes[v];
1762
			oist->ist[v] = t->x86_tss.ist[v] =
1763
					(unsigned long)estacks;
1764 1765
			if (v == DEBUG_STACK-1)
				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1766 1767 1768
		}
	}

1769
	t->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
I
Ingo Molnar 已提交
1770

1771 1772 1773 1774 1775 1776 1777
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

V
Vegard Nossum 已提交
1778
	mmgrab(&init_mm);
1779
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1780
	BUG_ON(me->mm);
1781
	initialize_tlbstate_and_flush();
1782 1783
	enter_lazy_tlb(&init_mm, me);

1784
	/*
1785 1786
	 * Initialize the TSS.  sp0 points to the entry trampoline stack
	 * regardless of what task is running.
1787
	 */
1788
	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
1789
	load_TR_desc();
1790
	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
1791

1792
	load_mm_ldt(&init_mm);
1793

1794 1795
	clear_all_debug_regs();
	dbg_restore_debug_regs();
1796

I
Ingo Molnar 已提交
1797
	fpu__init_cpu();
1798 1799 1800

	if (is_uv_system())
		uv_cpu_init();
1801 1802

	load_fixmap_gdt(cpu);
1803 1804 1805 1806
}

#else

1807
void cpu_init(void)
1808
{
1809 1810
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1811
	struct tss_struct *t = &per_cpu(cpu_tss_rw, cpu);
1812

1813
	wait_for_master_cpu(cpu);
1814

1815 1816 1817 1818 1819 1820
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1821
	show_ucode_info_early();
1822

1823
	pr_info("Initializing CPU#%d\n", cpu);
1824

1825
	if (cpu_feature_enabled(X86_FEATURE_VME) ||
1826
	    boot_cpu_has(X86_FEATURE_TSC) ||
1827
	    boot_cpu_has(X86_FEATURE_DE))
A
Andy Lutomirski 已提交
1828
		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1829

1830
	load_current_idt();
1831
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1832 1833 1834 1835

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
V
Vegard Nossum 已提交
1836
	mmgrab(&init_mm);
1837
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1838
	BUG_ON(curr->mm);
1839
	initialize_tlbstate_and_flush();
1840
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1841

1842
	/*
1843 1844
	 * Initialize the TSS.  sp0 points to the entry trampoline stack
	 * regardless of what task is running.
1845
	 */
1846
	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
L
Linus Torvalds 已提交
1847
	load_TR_desc();
1848
	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
1849

1850
	load_mm_ldt(&init_mm);
L
Linus Torvalds 已提交
1851

1852
	t->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
1853

1854
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1855 1856
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1857
#endif
L
Linus Torvalds 已提交
1858

1859
	clear_all_debug_regs();
1860
	dbg_restore_debug_regs();
L
Linus Torvalds 已提交
1861

I
Ingo Molnar 已提交
1862
	fpu__init_cpu();
1863 1864

	load_fixmap_gdt(cpu);
L
Linus Torvalds 已提交
1865
}
1866
#endif
1867

1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
static void bsp_resume(void)
{
	if (this_cpu->c_bsp_resume)
		this_cpu->c_bsp_resume(&boot_cpu_data);
}

static struct syscore_ops cpu_syscore_ops = {
	.resume		= bsp_resume,
};

static int __init init_cpu_syscore(void)
{
	register_syscore_ops(&cpu_syscore_ops);
	return 0;
}
core_initcall(init_cpu_syscore);
1884 1885 1886 1887 1888 1889 1890 1891

/*
 * The microcode loader calls this upon late microcode load to recheck features,
 * only when microcode has been updated. Caller holds microcode_mutex and CPU
 * hotplug lock.
 */
void microcode_check(void)
{
1892 1893
	struct cpuinfo_x86 info;

1894
	perf_check_microcode();
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912

	/* Reload CPUID max function as it might've changed. */
	info.cpuid_level = cpuid_eax(0);

	/*
	 * Copy all capability leafs to pick up the synthetic ones so that
	 * memcmp() below doesn't fail on that. The ones coming from CPUID will
	 * get overwritten in get_cpu_cap().
	 */
	memcpy(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability));

	get_cpu_cap(&info);

	if (!memcmp(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability)))
		return;

	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
1913
}