common.c 39.7 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/export.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
8
#include <linux/ctype.h>
L
Linus Torvalds 已提交
9
#include <linux/delay.h>
10
#include <linux/sched/mm.h>
11
#include <linux/sched/clock.h>
12
#include <linux/sched/task.h>
13
#include <linux/init.h>
14
#include <linux/kprobes.h>
15
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
16
#include <linux/smp.h>
17
#include <linux/io.h>
18
#include <linux/syscore_ops.h>
19 20

#include <asm/stackprotector.h>
21
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
22
#include <asm/mmu_context.h>
23
#include <asm/archrandom.h>
24 25
#include <asm/hypervisor.h>
#include <asm/processor.h>
26
#include <asm/tlbflush.h>
27
#include <asm/debugreg.h>
28
#include <asm/sections.h>
29
#include <asm/vsyscall.h>
A
Alan Cox 已提交
30 31
#include <linux/topology.h>
#include <linux/cpumask.h>
32
#include <asm/pgtable.h>
A
Arun Sharma 已提交
33
#include <linux/atomic.h>
34 35 36 37
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
38
#include <asm/fpu/internal.h>
39
#include <asm/mtrr.h>
40
#include <asm/hwcap2.h>
A
Alan Cox 已提交
41
#include <linux/numa.h>
42
#include <asm/asm.h>
43
#include <asm/bugs.h>
44
#include <asm/cpu.h>
45
#include <asm/mce.h>
46
#include <asm/msr.h>
47
#include <asm/pat.h>
48 49
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
50 51

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
52
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
53 54 55 56
#endif

#include "cpu.h"

57 58
u32 elf_hwcap2 __read_mostly;

59 60
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
61 62
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
63 64 65 66

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
67
/* correctly size the local cpu masks */
68
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
69 70 71 72 73 74 75
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

76
static void default_init(struct cpuinfo_x86 *c)
77 78
{
#ifdef CONFIG_X86_64
79
	cpu_detect_cache_sizes(c);
80 81 82 83 84 85 86 87 88 89 90 91 92
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

93
static const struct cpu_dev default_cpu = {
94 95 96 97 98
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

99
static const struct cpu_dev *this_cpu = &default_cpu;
100

B
Brian Gerst 已提交
101
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
102
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
103 104 105 106 107
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
108
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
109 110
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
111 112 113 114 115 116
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
117
#else
A
Akinobu Mita 已提交
118 119 120 121
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
122 123 124 125 126
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
127
	/* 32-bit code */
A
Akinobu Mita 已提交
128
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
129
	/* 16-bit code */
A
Akinobu Mita 已提交
130
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
131
	/* 16-bit data */
A
Akinobu Mita 已提交
132
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
133
	/* 16-bit data */
A
Akinobu Mita 已提交
134
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
135
	/* 16-bit data */
A
Akinobu Mita 已提交
136
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
137 138 139 140
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
141
	/* 32-bit code */
A
Akinobu Mita 已提交
142
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
143
	/* 16-bit code */
A
Akinobu Mita 已提交
144
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
145
	/* data */
146
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
147

A
Akinobu Mita 已提交
148 149
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
150
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
151
#endif
B
Brian Gerst 已提交
152
} };
153
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
154

155
static int __init x86_mpx_setup(char *s)
156
{
157
	/* require an exact match without trailing characters */
158 159
	if (strlen(s))
		return 0;
160

161 162 163
	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_MPX))
		return 1;
164

165 166
	setup_clear_cpu_cap(X86_FEATURE_MPX);
	pr_info("nompx: Intel Memory Protection Extensions (MPX) disabled\n");
167 168
	return 1;
}
169
__setup("nompx", x86_mpx_setup);
170

171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
static int __init x86_noinvpcid_setup(char *s)
{
	/* noinvpcid doesn't accept parameters */
	if (s)
		return -EINVAL;

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_INVPCID))
		return 0;

	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
	pr_info("noinvpcid: INVPCID feature disabled\n");
	return 0;
}
early_param("noinvpcid", x86_noinvpcid_setup);

187
#ifdef CONFIG_X86_32
188 189
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
L
Linus Torvalds 已提交
190

191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

210 211 212 213 214 215 216
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
217 218 219 220 221 222 223 224 225 226 227
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

228 229
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
230 231 232 233 234

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
235
int have_cpuid_p(void)
236 237 238 239
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

240
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
241
{
I
Ingo Molnar 已提交
242 243 244 245 246 247 248 249 250 251 252
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

253
	pr_notice("CPU serial number disabled.\n");
I
Ingo Molnar 已提交
254 255 256 257
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
258 259 260 261 262 263 264 265
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
266
#else
267 268 269 270 271 272 273
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
274
#endif
275

276 277
static __init int setup_disable_smep(char *arg)
{
278
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
279 280
	/* Check for things that depend on SMEP being enabled: */
	check_mpx_erratum(&boot_cpu_data);
281 282 283 284
	return 1;
}
__setup("nosmep", setup_disable_smep);

285
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
286
{
287
	if (cpu_has(c, X86_FEATURE_SMEP))
A
Andy Lutomirski 已提交
288
		cr4_set_bits(X86_CR4_SMEP);
289 290
}

291 292
static __init int setup_disable_smap(char *arg)
{
293
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
294 295 296 297
	return 1;
}
__setup("nosmap", setup_disable_smap);

298 299
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
300
	unsigned long eflags = native_save_fl();
301 302 303 304

	/* This should have been cleared long ago */
	BUG_ON(eflags & X86_EFLAGS_AC);

305 306
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
A
Andy Lutomirski 已提交
307
		cr4_set_bits(X86_CR4_SMAP);
308
#else
A
Andy Lutomirski 已提交
309
		cr4_clear_bits(X86_CR4_SMAP);
310 311
#endif
	}
312 313
}

314 315 316 317 318 319 320
/*
 * Protection Keys are not available in 32-bit mode.
 */
static bool pku_disabled;

static __always_inline void setup_pku(struct cpuinfo_x86 *c)
{
321 322 323 324
	/* check the boot processor, plus compile options for PKU: */
	if (!cpu_feature_enabled(X86_FEATURE_PKU))
		return;
	/* checks the actual processor's cpuid bits: */
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
	if (!cpu_has(c, X86_FEATURE_PKU))
		return;
	if (pku_disabled)
		return;

	cr4_set_bits(X86_CR4_PKE);
	/*
	 * Seting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
	 * cpuid bit to be set.  We need to ensure that we
	 * update that bit in this CPU's "cpu_info".
	 */
	get_cpu_cap(c);
}

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
static __init int setup_disable_pku(char *arg)
{
	/*
	 * Do not clear the X86_FEATURE_PKU bit.  All of the
	 * runtime checks are against OSPKE so clearing the
	 * bit does nothing.
	 *
	 * This way, we will see "pku" in cpuinfo, but not
	 * "ospke", which is exactly what we want.  It shows
	 * that the CPU has PKU, but the OS has not enabled it.
	 * This happens to be exactly how a system would look
	 * if we disabled the config option.
	 */
	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
	pku_disabled = true;
	return 1;
}
__setup("nopku", setup_disable_pku);
#endif /* CONFIG_X86_64 */

360 361 362 363 364 365 366 367 368
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
369

370
static const struct cpuid_dependent_feature
371 372 373 374 375 376 377
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

378
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
379 380
{
	const struct cpuid_dependent_feature *df;
381

382
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
383 384 385

		if (!cpu_has(c, df->feature))
			continue;
386 387 388 389 390 391 392
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
393
		if (!((s32)df->level < 0 ?
394
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
395 396 397 398 399 400 401
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

402 403
		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
			x86_cap_flag(df->feature), df->level);
404
	}
405
}
406

407 408 409
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
410 411
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
412 413 414
 */

/* Look up CPU names by table lookup. */
415
static const char *table_lookup_model(struct cpuinfo_x86 *c)
416
{
417 418
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
419 420 421 422 423 424 425

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

426
	info = this_cpu->legacy_models;
427

428
	while (info->family) {
429 430 431 432
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
433
#endif
434 435 436
	return NULL;		/* Not found */
}

437 438
__u32 cpu_caps_cleared[NCAPINTS];
__u32 cpu_caps_set[NCAPINTS];
439

440 441 442 443 444
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
445
	__loadsegment_simple(gs, 0);
446 447
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
448
	load_stack_canary_segment();
449 450
}

451 452 453 454 455 456 457
/*
 * On 64-bit the GDT remapping is read-only.
 * A global is used for Xen to change the default when required.
 */
#ifdef CONFIG_X86_64
pgprot_t pg_fixmap_gdt_flags = PAGE_KERNEL_RO;
#else
458
pgprot_t pg_fixmap_gdt_flags = PAGE_KERNEL;
459
#endif
460 461 462 463

/* Setup the fixmap mapping only once per-processor */
static inline void setup_fixmap_gdt(int cpu)
{
464 465
	__set_fixmap(get_cpu_gdt_ro_index(cpu), get_cpu_gdt_paddr(cpu),
		     pg_fixmap_gdt_flags);
466 467
}

468 469 470 471 472 473 474 475 476 477 478
/* Load the original GDT from the per-cpu structure */
void load_direct_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
EXPORT_SYMBOL_GPL(load_direct_gdt);

479 480 481 482 483 484 485 486 487
/* Load a fixmap remapping of the per-cpu GDT */
void load_fixmap_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
488
EXPORT_SYMBOL_GPL(load_fixmap_gdt);
489

I
Ingo Molnar 已提交
490 491 492 493
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
494
void switch_to_new_gdt(int cpu)
495
{
496 497
	/* Load the original GDT */
	load_direct_gdt(cpu);
498
	/* Reload the per-cpu base */
499
	load_percpu_segment(cpu);
500 501
}

502
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
503

504
static void get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
505 506
{
	unsigned int *v;
507
	char *p, *q, *s;
L
Linus Torvalds 已提交
508

509
	if (c->extended_cpuid_level < 0x80000004)
510
		return;
L
Linus Torvalds 已提交
511

I
Ingo Molnar 已提交
512
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
513 514 515 516 517
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
	/* Trim whitespace */
	p = q = s = &c->x86_model_id[0];

	while (*p == ' ')
		p++;

	while (*p) {
		/* Note the last non-whitespace index */
		if (!isspace(*p))
			s = q;

		*q++ = *p++;
	}

	*(s + 1) = '\0';
L
Linus Torvalds 已提交
533 534
}

535
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
536
{
537
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
538

539
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
540 541

	if (n >= 0x80000005) {
542 543
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
544 545 546 547
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
548 549 550 551 552
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

553
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
554
	l2size = ecx >> 16;
555

556 557 558
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
559
	/* do processor-specific cache resizing */
560 561
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
L
Linus Torvalds 已提交
562 563 564 565 566

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

567
	if (l2size == 0)
L
Linus Torvalds 已提交
568
		return;		/* Again, no L2 cache is possible */
569
#endif
L
Linus Torvalds 已提交
570 571 572 573

	c->x86_cache_size = l2size;
}

574 575 576 577 578 579
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
580
u16 __read_mostly tlb_lld_1g[NR_INFO];
581

582
static void cpu_detect_tlb(struct cpuinfo_x86 *c)
583 584 585 586
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

587
	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
588
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
589 590 591 592 593
		tlb_lli_4m[ENTRIES]);

	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
594 595
}

596
void detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
597
{
B
Borislav Petkov 已提交
598
#ifdef CONFIG_SMP
599 600
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
601
	static bool printed;
L
Linus Torvalds 已提交
602

603
	if (!cpu_has(c, X86_FEATURE_HT))
604
		return;
L
Linus Torvalds 已提交
605

606 607
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
608

609 610
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
611

612
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
613

614 615 616
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
617
		pr_info_once("CPU0: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
618 619
		goto out;
	}
620

I
Ingo Molnar 已提交
621 622
	if (smp_num_siblings <= 1)
		goto out;
623

I
Ingo Molnar 已提交
624 625
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
626

I
Ingo Molnar 已提交
627
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
628

I
Ingo Molnar 已提交
629
	index_msb = get_count_order(smp_num_siblings);
630

I
Ingo Molnar 已提交
631
	core_bits = get_count_order(c->x86_max_cores);
632

I
Ingo Molnar 已提交
633 634
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
635

636
out:
637
	if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
638 639 640 641
		pr_info("CPU: Physical Processor ID: %d\n",
			c->phys_proc_id);
		pr_info("CPU: Processor Core ID: %d\n",
			c->cpu_core_id);
642
		printed = 1;
643 644
	}
#endif
645
}
L
Linus Torvalds 已提交
646

647
static void get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
648 649
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
650
	int i;
L
Linus Torvalds 已提交
651 652

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
653 654 655 656 657 658
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
659

Y
Yinghai Lu 已提交
660 661 662
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
663 664
		}
	}
Y
Yinghai Lu 已提交
665

666 667
	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
		    "CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
668

669 670
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
671 672
}

673
void cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
674 675
{
	/* Get vendor name */
676 677 678 679
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
680 681

	c->x86 = 4;
682
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
683 684
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
685

L
Linus Torvalds 已提交
686
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
687 688 689
		c->x86		= x86_family(tfms);
		c->x86_model	= x86_model(tfms);
		c->x86_mask	= x86_stepping(tfms);
I
Ingo Molnar 已提交
690

H
Huang, Ying 已提交
691 692
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
693
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
694
		}
L
Linus Torvalds 已提交
695 696
	}
}
697

698 699 700 701 702 703 704 705 706 707
static void apply_forced_caps(struct cpuinfo_x86 *c)
{
	int i;

	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}
}

708
void get_cpu_cap(struct cpuinfo_x86 *c)
709
{
710
	u32 eax, ebx, ecx, edx;
711

712 713
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
714
		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
715

716 717
		c->x86_capability[CPUID_1_ECX] = ecx;
		c->x86_capability[CPUID_1_EDX] = edx;
718
	}
719

720 721 722 723
	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
	if (c->cpuid_level >= 0x00000006)
		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);

724 725 726
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
727
		c->x86_capability[CPUID_7_0_EBX] = ebx;
728
		c->x86_capability[CPUID_7_ECX] = ecx;
729 730
	}

731 732 733 734
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

735
		c->x86_capability[CPUID_D_1_EAX] = eax;
736 737
	}

738 739 740 741 742
	/* Additional Intel-defined flags: level 0x0000000F */
	if (c->cpuid_level >= 0x0000000F) {

		/* QoS sub-leaf, EAX=0Fh, ECX=0 */
		cpuid_count(0x0000000F, 0, &eax, &ebx, &ecx, &edx);
743 744
		c->x86_capability[CPUID_F_0_EDX] = edx;

745 746 747 748 749 750
		if (cpu_has(c, X86_FEATURE_CQM_LLC)) {
			/* will be overridden if occupancy monitoring exists */
			c->x86_cache_max_rmid = ebx;

			/* QoS sub-leaf, EAX=0Fh, ECX=1 */
			cpuid_count(0x0000000F, 1, &eax, &ebx, &ecx, &edx);
751 752
			c->x86_capability[CPUID_F_1_EDX] = edx;

753 754 755
			if ((cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC)) ||
			      ((cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL)) ||
			       (cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL)))) {
756 757 758 759 760 761 762 763 764
				c->x86_cache_max_rmid = ecx;
				c->x86_cache_occ_scale = ebx;
			}
		} else {
			c->x86_cache_max_rmid = -1;
			c->x86_cache_occ_scale = -1;
		}
	}

765
	/* AMD-defined flags: level 0x80000001 */
766 767 768 769 770 771
	eax = cpuid_eax(0x80000000);
	c->extended_cpuid_level = eax;

	if ((eax & 0xffff0000) == 0x80000000) {
		if (eax >= 0x80000001) {
			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
772

773 774
			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
			c->x86_capability[CPUID_8000_0001_EDX] = edx;
775 776 777
		}
	}

778 779 780 781 782 783 784
	if (c->extended_cpuid_level >= 0x80000007) {
		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);

		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
		c->x86_power = edx;
	}

785
	if (c->extended_cpuid_level >= 0x80000008) {
786
		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
787 788 789

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
790
		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
791
	}
792 793 794
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
795
#endif
796

797
	if (c->extended_cpuid_level >= 0x8000000a)
798
		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
799

800
	init_scattered_cpuid_features(c);
801 802 803 804 805 806 807

	/*
	 * Clear/Set all flags overridden by options, after probe.
	 * This needs to happen each time we re-probe, which may happen
	 * several times during CPU initialization.
	 */
	apply_forced_caps(c);
808
}
L
Linus Torvalds 已提交
809

810
static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
Y
Yinghai Lu 已提交
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

836 837 838 839 840 841 842 843 844
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
845
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
846
{
847 848
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
849 850
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
851
#else
H
Huang, Ying 已提交
852
	c->x86_clflush_size = 32;
853 854
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
855
#endif
856
	c->x86_cache_alignment = c->x86_clflush_size;
857

858
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
859
	c->extended_cpuid_level = 0;
860

Y
Yinghai Lu 已提交
861
	/* cyrix could have cpuid enabled via c_identify()*/
862 863 864 865
	if (have_cpuid_p()) {
		cpu_detect(c);
		get_cpu_vendor(c);
		get_cpu_cap(c);
B
Borislav Petkov 已提交
866
		setup_force_cpu_cap(X86_FEATURE_CPUID);
867

868 869
		if (this_cpu->c_early_init)
			this_cpu->c_early_init(c);
870

871 872
		c->cpu_index = 0;
		filter_cpuid_features(c, false);
873

874 875
		if (this_cpu->c_bsp_init)
			this_cpu->c_bsp_init(c);
B
Borislav Petkov 已提交
876 877 878
	} else {
		identify_cpu_without_cpuid(c);
		setup_clear_cpu_cap(X86_FEATURE_CPUID);
879
	}
880 881

	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
882
	fpu__init_system(c);
883 884
}

885 886
void __init early_cpu_init(void)
{
887
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
888 889
	int count = 0;

890
#ifdef CONFIG_PROCESSOR_SELECT
891
	pr_info("KERNEL supported cpus:\n");
892 893
#endif

Y
Yinghai Lu 已提交
894
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
895
		const struct cpu_dev *cpudev = *cdev;
896

Y
Yinghai Lu 已提交
897 898 899 900 901
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

902
#ifdef CONFIG_PROCESSOR_SELECT
903 904 905 906 907 908
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
909
				pr_info("  %s %s\n", cpudev->c_vendor,
910 911
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
912
		}
913
#endif
Y
Yinghai Lu 已提交
914
	}
915
	early_identify_cpu(&boot_cpu_data);
916
}
917

918
/*
B
Borislav Petkov 已提交
919 920 921 922 923
 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
 * unfortunately, that's not true in practice because of early VIA
 * chips and (more importantly) broken virtualizers that are not easy
 * to detect. In the latter case it doesn't even *fail* reliably, so
 * probing for it doesn't even work. Disable it completely on 32-bit
924
 * unless we can find a reliable way to detect all the broken cases.
B
Borislav Petkov 已提交
925
 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
926
 */
927
static void detect_nopl(struct cpuinfo_x86 *c)
928
{
B
Borislav Petkov 已提交
929
#ifdef CONFIG_X86_32
930
	clear_cpu_cap(c, X86_FEATURE_NOPL);
B
Borislav Petkov 已提交
931 932
#else
	set_cpu_cap(c, X86_FEATURE_NOPL);
933
#endif
934
}
935

936 937 938
static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
939
	/*
940 941 942 943 944
	 * Empirically, writing zero to a segment selector on AMD does
	 * not clear the base, whereas writing zero to a segment
	 * selector on Intel does clear the base.  Intel's behavior
	 * allows slightly faster context switches in the common case
	 * where GS is unused by the prev and next threads.
945
	 *
946 947 948 949 950 951
	 * Since neither vendor documents this anywhere that I can see,
	 * detect it directly instead of hardcoding the choice by
	 * vendor.
	 *
	 * I've designated AMD's behavior as the "bug" because it's
	 * counterintuitive and less friendly.
952
	 */
953 954 955 956 957 958 959 960 961

	unsigned long old_base, tmp;
	rdmsrl(MSR_FS_BASE, old_base);
	wrmsrl(MSR_FS_BASE, 1);
	loadsegment(fs, 0);
	rdmsrl(MSR_FS_BASE, tmp);
	if (tmp != 0)
		set_cpu_bug(c, X86_BUG_NULL_SEG);
	wrmsrl(MSR_FS_BASE, old_base);
B
Borislav Petkov 已提交
962
#endif
963 964
}

965
static void generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
966
{
Y
Yinghai Lu 已提交
967
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
968

969
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
970
		identify_cpu_without_cpuid(c);
971

Y
Yinghai Lu 已提交
972
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
973
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
974
		return;
L
Linus Torvalds 已提交
975

976
	cpu_detect(c);
L
Linus Torvalds 已提交
977

978
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
979

980
	get_cpu_cap(c);
L
Linus Torvalds 已提交
981

982 983
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
984
#ifdef CONFIG_X86_32
B
Borislav Petkov 已提交
985
# ifdef CONFIG_SMP
986
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
987
# else
988
		c->apicid = c->initial_apicid;
989 990 991
# endif
#endif
		c->phys_proc_id = c->initial_apicid;
992
	}
L
Linus Torvalds 已提交
993

994
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
995

996
	detect_nopl(c);
997 998

	detect_null_seg_behavior(c);
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023

	/*
	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
	 * systems that run Linux at CPL > 0 may or may not have the
	 * issue, but, even if they have the issue, there's absolutely
	 * nothing we can do about it because we can't use the real IRET
	 * instruction.
	 *
	 * NB: For the time being, only 32-bit kernels support
	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
	 * whether to apply espfix using paravirt hooks.  If any
	 * non-paravirt system ever shows up that does *not* have the
	 * ESPFIX issue, we can change this.
	 */
#ifdef CONFIG_X86_32
# ifdef CONFIG_PARAVIRT
	do {
		extern void native_iret(void);
		if (pv_cpu_ops.iret == native_iret)
			set_cpu_bug(c, X86_BUG_ESPFIX);
	} while (0);
# else
	set_cpu_bug(c, X86_BUG_ESPFIX);
# endif
#endif
L
Linus Torvalds 已提交
1024 1025
}

1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
static void x86_init_cache_qos(struct cpuinfo_x86 *c)
{
	/*
	 * The heavy lifting of max_rmid and cache_occ_scale are handled
	 * in get_cpu_cap().  Here we just set the max_rmid for the boot_cpu
	 * in case CQM bits really aren't there in this CPU.
	 */
	if (c != &boot_cpu_data) {
		boot_cpu_data.x86_cache_max_rmid =
			min(boot_cpu_data.x86_cache_max_rmid,
			    c->x86_cache_max_rmid);
	}
}

1040
/*
1041 1042
 * Validate that ACPI/mptables have the same information about the
 * effective APIC id and update the package map.
1043
 */
1044
static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1045 1046
{
#ifdef CONFIG_SMP
1047
	unsigned int apicid, cpu = smp_processor_id();
1048 1049 1050

	apicid = apic->cpu_present_to_apicid(cpu);

1051 1052
	if (apicid != c->apicid) {
		pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1053 1054
		       cpu, apicid, c->initial_apicid);
	}
1055
	BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1056 1057 1058 1059 1060
#else
	c->logical_proc_id = 0;
#endif
}

L
Linus Torvalds 已提交
1061 1062 1063
/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
1064
static void identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1065 1066 1067 1068 1069 1070 1071 1072 1073
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
1074
	c->x86_max_cores = 1;
1075
	c->x86_coreid_bits = 0;
1076
	c->cu_id = 0xff;
1077
#ifdef CONFIG_X86_64
1078
	c->x86_clflush_size = 64;
1079 1080
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
1081 1082
#else
	c->cpuid_level = -1;	/* CPUID not detected */
1083
	c->x86_clflush_size = 32;
1084 1085
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
1086 1087
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
1088 1089 1090 1091
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

1092
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
1093 1094
		this_cpu->c_identify(c);

1095
	/* Clear/Set all flags overridden by options, after probe */
1096
	apply_forced_caps(c);
1097

1098
#ifdef CONFIG_X86_64
1099
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1100 1101
#endif

L
Linus Torvalds 已提交
1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

1118 1119 1120 1121
	/* Set up SMEP/SMAP */
	setup_smep(c);
	setup_smap(c);

L
Linus Torvalds 已提交
1122
	/*
I
Ingo Molnar 已提交
1123 1124
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
1125 1126
	 */

1127 1128 1129
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
1130
	/* If the model name is still unset, do table lookup. */
1131
	if (!c->x86_model_id[0]) {
1132
		const char *p;
L
Linus Torvalds 已提交
1133
		p = table_lookup_model(c);
1134
		if (p)
L
Linus Torvalds 已提交
1135 1136 1137 1138
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
1139
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
1140 1141
	}

1142 1143 1144 1145
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

1146
	init_hypervisor(c);
1147
	x86_init_rdrand(c);
1148
	x86_init_cache_qos(c);
1149
	setup_pku(c);
1150 1151

	/*
1152
	 * Clear/Set all flags overridden by options, need do it
1153 1154
	 * before following smp all cpus cap AND.
	 */
1155
	apply_forced_caps(c);
1156

L
Linus Torvalds 已提交
1157 1158 1159 1160 1161 1162
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
1163
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
1164
		/* AND the already accumulated flags with these */
1165
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
1166
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1167 1168 1169 1170

		/* OR, i.e. replicate the bug flags */
		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
L
Linus Torvalds 已提交
1171 1172 1173
	}

	/* Init Machine Check Exception if available. */
1174
	mcheck_cpu_init(c);
1175 1176

	select_idle_routine(c);
1177

1178
#ifdef CONFIG_NUMA
1179 1180
	numa_add_cpu(smp_processor_id());
#endif
1181
}
S
Shaohua Li 已提交
1182

1183 1184 1185 1186
/*
 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
 * on 32-bit kernels:
 */
1187 1188 1189
#ifdef CONFIG_X86_32
void enable_sep_cpu(void)
{
1190 1191
	struct tss_struct *tss;
	int cpu;
1192

1193 1194 1195
	if (!boot_cpu_has(X86_FEATURE_SEP))
		return;

1196 1197 1198 1199
	cpu = get_cpu();
	tss = &per_cpu(cpu_tss, cpu);

	/*
1200 1201
	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
	 * see the big comment in struct x86_hw_tss's definition.
1202
	 */
1203 1204

	tss->x86_tss.ss1 = __KERNEL_CS;
1205 1206
	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);

1207 1208 1209
	wrmsr(MSR_IA32_SYSENTER_ESP,
	      (unsigned long)tss + offsetofend(struct tss_struct, SYSENTER_stack),
	      0);
1210

1211
	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1212

1213 1214
	put_cpu();
}
1215 1216
#endif

1217 1218 1219
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
1220
#ifdef CONFIG_X86_32
1221
	sysenter_setup();
L
Li Shaohua 已提交
1222
	enable_sep_cpu();
1223
#endif
1224
	cpu_detect_tlb(&boot_cpu_data);
1225
}
S
Shaohua Li 已提交
1226

1227
void identify_secondary_cpu(struct cpuinfo_x86 *c)
1228 1229 1230
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
1231
#ifdef CONFIG_X86_32
1232
	enable_sep_cpu();
1233
#endif
1234
	mtrr_ap_init();
1235
	validate_apic_and_package_id(c);
L
Linus Torvalds 已提交
1236 1237
}

A
Andi Kleen 已提交
1238 1239
static __init int setup_noclflush(char *arg)
{
1240
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1241
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
A
Andi Kleen 已提交
1242 1243 1244 1245
	return 1;
}
__setup("noclflush", setup_noclflush);

1246
void print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1247
{
1248
	const char *vendor = NULL;
L
Linus Torvalds 已提交
1249

I
Ingo Molnar 已提交
1250
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
1251
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
1252 1253 1254 1255
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
1256

1257
	if (vendor && !strstr(c->x86_model_id, vendor))
1258
		pr_cont("%s ", vendor);
L
Linus Torvalds 已提交
1259

1260
	if (c->x86_model_id[0])
1261
		pr_cont("%s", c->x86_model_id);
L
Linus Torvalds 已提交
1262
	else
1263
		pr_cont("%d86", c->x86);
L
Linus Torvalds 已提交
1264

1265
	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1266

1267
	if (c->x86_mask || c->cpuid_level >= 0)
1268
		pr_cont(", stepping: 0x%x)\n", c->x86_mask);
L
Linus Torvalds 已提交
1269
	else
1270
		pr_cont(")\n");
L
Linus Torvalds 已提交
1271 1272
}

1273 1274 1275
static __init int setup_disablecpuid(char *arg)
{
	int bit;
I
Ingo Molnar 已提交
1276

1277
	if (get_option(&arg, &bit) && bit >= 0 && bit < NCAPINTS * 32)
1278 1279 1280
		setup_clear_cpu_cap(bit);
	else
		return 0;
I
Ingo Molnar 已提交
1281

1282 1283 1284 1285
	return 1;
}
__setup("clearcpuid=", setup_disablecpuid);

1286
#ifdef CONFIG_X86_64
1287 1288 1289 1290 1291 1292 1293 1294
struct desc_ptr idt_descr __ro_after_init = {
	.size = NR_VECTORS * 16 - 1,
	.address = (unsigned long) idt_table,
};
const struct desc_ptr debug_idt_descr = {
	.size = NR_VECTORS * 16 - 1,
	.address = (unsigned long) debug_idt_table,
};
1295

1296
DEFINE_PER_CPU_FIRST(union irq_stack_union,
1297
		     irq_stack_union) __aligned(PAGE_SIZE) __visible;
I
Ingo Molnar 已提交
1298

1299
/*
1300 1301
 * The following percpu variables are hot.  Align current_task to
 * cacheline size such that they fall in the same cacheline.
1302 1303 1304 1305
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1306

1307
DEFINE_PER_CPU(char *, irq_stack_ptr) =
1308
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE;
1309

1310
DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1311

1312 1313 1314
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);

I
Ingo Molnar 已提交
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

1326
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
1327
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
1328 1329 1330

/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1331
{
1332
	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
1333
	wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
1334 1335

#ifdef CONFIG_IA32_EMULATION
1336
	wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat);
1337
	/*
1338 1339 1340 1341
	 * This only works on Intel CPUs.
	 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
	 * This does not cause SYSENTER to jump to the wrong location, because
	 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
1342 1343 1344
	 */
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
1345
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
1346
#else
1347
	wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
1348
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1349 1350
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1351
#endif
1352

1353 1354
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
1355
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1356
	       X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
L
Linus Torvalds 已提交
1357
}
1358

1359 1360 1361 1362 1363 1364
/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

1365
static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
1366
DEFINE_PER_CPU(int, debug_stack_usage);
1367 1368 1369

int is_debug_stack(unsigned long addr)
{
1370 1371 1372
	return __this_cpu_read(debug_stack_usage) ||
		(addr <= __this_cpu_read(debug_stack_addr) &&
		 addr > (__this_cpu_read(debug_stack_addr) - DEBUG_STKSZ));
1373
}
1374
NOKPROBE_SYMBOL(is_debug_stack);
1375

1376
DEFINE_PER_CPU(u32, debug_idt_ctr);
1377

1378 1379
void debug_stack_set_zero(void)
{
1380 1381
	this_cpu_inc(debug_idt_ctr);
	load_current_idt();
1382
}
1383
NOKPROBE_SYMBOL(debug_stack_set_zero);
1384 1385 1386

void debug_stack_reset(void)
{
1387
	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
1388
		return;
1389 1390
	if (this_cpu_dec_return(debug_idt_ctr) == 0)
		load_current_idt();
1391
}
1392
NOKPROBE_SYMBOL(debug_stack_reset);
1393

I
Ingo Molnar 已提交
1394
#else	/* CONFIG_X86_64 */
1395

1396 1397
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1398 1399
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
1400

1401 1402 1403 1404 1405 1406 1407 1408 1409
/*
 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
 * the top of the kernel stack.  Use an extra percpu variable to track the
 * top of the kernel stack directly.
 */
DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
	(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);

1410
#ifdef CONFIG_CC_STACKPROTECTOR
1411
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1412
#endif
1413

I
Ingo Molnar 已提交
1414
#endif	/* CONFIG_X86_64 */
1415

1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1431

1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
#ifdef CONFIG_KGDB
/*
 * Restore debug regs if using kgdbwait and you have a kernel debugger
 * connection established.
 */
static void dbg_restore_debug_regs(void)
{
	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
		arch_kgdb_ops.correct_hw_break();
}
#else /* ! CONFIG_KGDB */
#define dbg_restore_debug_regs()
#endif /* ! CONFIG_KGDB */

1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
static void wait_for_master_cpu(int cpu)
{
#ifdef CONFIG_SMP
	/*
	 * wait for ACK from master CPU before continuing
	 * with AP initialization
	 */
	WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
	while (!cpumask_test_cpu(cpu, cpu_callout_mask))
		cpu_relax();
#endif
}

1459 1460 1461 1462 1463
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1464
 * A lot of state is already set up in PDA init for 64 bit
1465
 */
1466
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1467

1468
void cpu_init(void)
1469
{
1470
	struct orig_ist *oist;
1471
	struct task_struct *me;
I
Ingo Molnar 已提交
1472 1473
	struct tss_struct *t;
	unsigned long v;
1474
	int cpu = raw_smp_processor_id();
1475 1476
	int i;

1477 1478
	wait_for_master_cpu(cpu);

1479 1480 1481 1482 1483 1484
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1485 1486
	if (cpu)
		load_ucode_ap();
1487

1488
	t = &per_cpu(cpu_tss, cpu);
1489
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1490

1491
#ifdef CONFIG_NUMA
1492
	if (this_cpu_read(numa_node) == 0 &&
1493 1494
	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
		set_numa_node(early_cpu_to_node(cpu));
1495
#endif
1496 1497 1498

	me = current;

1499
	pr_debug("Initializing CPU#%d\n", cpu);
1500

A
Andy Lutomirski 已提交
1501
	cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1502 1503 1504 1505 1506 1507

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1508
	switch_to_new_gdt(cpu);
1509 1510
	loadsegment(fs, 0);

1511
	load_current_idt();
1512 1513 1514 1515 1516 1517 1518 1519

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1520
	x86_configure_nx();
1521
	x2apic_setup();
1522 1523 1524 1525

	/*
	 * set up and load the per-CPU TSS
	 */
1526
	if (!oist->ist[0]) {
1527
		char *estacks = per_cpu(exception_stacks, cpu);
I
Ingo Molnar 已提交
1528

1529
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1530
			estacks += exception_stack_sizes[v];
1531
			oist->ist[v] = t->x86_tss.ist[v] =
1532
					(unsigned long)estacks;
1533 1534
			if (v == DEBUG_STACK-1)
				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1535 1536 1537 1538
		}
	}

	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
I
Ingo Molnar 已提交
1539

1540 1541 1542 1543 1544 1545 1546
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

V
Vegard Nossum 已提交
1547
	mmgrab(&init_mm);
1548
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1549
	BUG_ON(me->mm);
1550 1551 1552 1553 1554
	enter_lazy_tlb(&init_mm, me);

	load_sp0(t, &current->thread);
	set_tss_desc(cpu, t);
	load_TR_desc();
1555
	load_mm_ldt(&init_mm);
1556

1557 1558
	clear_all_debug_regs();
	dbg_restore_debug_regs();
1559

I
Ingo Molnar 已提交
1560
	fpu__init_cpu();
1561 1562 1563

	if (is_uv_system())
		uv_cpu_init();
1564 1565 1566

	setup_fixmap_gdt(cpu);
	load_fixmap_gdt(cpu);
1567 1568 1569 1570
}

#else

1571
void cpu_init(void)
1572
{
1573 1574
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1575
	struct tss_struct *t = &per_cpu(cpu_tss, cpu);
1576
	struct thread_struct *thread = &curr->thread;
1577

1578
	wait_for_master_cpu(cpu);
1579

1580 1581 1582 1583 1584 1585
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1586
	show_ucode_info_early();
1587

1588
	pr_info("Initializing CPU#%d\n", cpu);
1589

1590
	if (cpu_feature_enabled(X86_FEATURE_VME) ||
1591
	    boot_cpu_has(X86_FEATURE_TSC) ||
1592
	    boot_cpu_has(X86_FEATURE_DE))
A
Andy Lutomirski 已提交
1593
		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1594

1595
	load_current_idt();
1596
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1597 1598 1599 1600

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
V
Vegard Nossum 已提交
1601
	mmgrab(&init_mm);
1602
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1603
	BUG_ON(curr->mm);
1604
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1605

1606
	load_sp0(t, thread);
1607
	set_tss_desc(cpu, t);
L
Linus Torvalds 已提交
1608
	load_TR_desc();
1609
	load_mm_ldt(&init_mm);
L
Linus Torvalds 已提交
1610

1611 1612
	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);

1613
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1614 1615
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1616
#endif
L
Linus Torvalds 已提交
1617

1618
	clear_all_debug_regs();
1619
	dbg_restore_debug_regs();
L
Linus Torvalds 已提交
1620

I
Ingo Molnar 已提交
1621
	fpu__init_cpu();
1622 1623 1624

	setup_fixmap_gdt(cpu);
	load_fixmap_gdt(cpu);
L
Linus Torvalds 已提交
1625
}
1626
#endif
1627

1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
static void bsp_resume(void)
{
	if (this_cpu->c_bsp_resume)
		this_cpu->c_bsp_resume(&boot_cpu_data);
}

static struct syscore_ops cpu_syscore_ops = {
	.resume		= bsp_resume,
};

static int __init init_cpu_syscore(void)
{
	register_syscore_ops(&cpu_syscore_ops);
	return 0;
}
core_initcall(init_cpu_syscore);