common.c 43.1 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/export.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
8
#include <linux/ctype.h>
L
Linus Torvalds 已提交
9
#include <linux/delay.h>
10
#include <linux/sched/mm.h>
11
#include <linux/sched/clock.h>
12
#include <linux/sched/task.h>
13
#include <linux/init.h>
14
#include <linux/kprobes.h>
15
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
16
#include <linux/smp.h>
17
#include <linux/io.h>
18
#include <linux/syscore_ops.h>
19 20

#include <asm/stackprotector.h>
21
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
22
#include <asm/mmu_context.h>
23
#include <asm/archrandom.h>
24 25
#include <asm/hypervisor.h>
#include <asm/processor.h>
26
#include <asm/tlbflush.h>
27
#include <asm/debugreg.h>
28
#include <asm/sections.h>
29
#include <asm/vsyscall.h>
A
Alan Cox 已提交
30 31
#include <linux/topology.h>
#include <linux/cpumask.h>
32
#include <asm/pgtable.h>
A
Arun Sharma 已提交
33
#include <linux/atomic.h>
34 35 36 37
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
38
#include <asm/fpu/internal.h>
39
#include <asm/mtrr.h>
40
#include <asm/hwcap2.h>
A
Alan Cox 已提交
41
#include <linux/numa.h>
42
#include <asm/asm.h>
43
#include <asm/bugs.h>
44
#include <asm/cpu.h>
45
#include <asm/mce.h>
46
#include <asm/msr.h>
47
#include <asm/pat.h>
48 49
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
50 51

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
52
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
53 54 55 56
#endif

#include "cpu.h"

57 58
u32 elf_hwcap2 __read_mostly;

59 60
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
61 62
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
63 64 65 66

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
67
/* correctly size the local cpu masks */
68
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
69 70 71 72 73 74 75
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

76
static void default_init(struct cpuinfo_x86 *c)
77 78
{
#ifdef CONFIG_X86_64
79
	cpu_detect_cache_sizes(c);
80 81 82 83 84 85 86 87 88 89 90 91 92
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

93
static const struct cpu_dev default_cpu = {
94 95 96 97 98
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

99
static const struct cpu_dev *this_cpu = &default_cpu;
100

B
Brian Gerst 已提交
101
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
102
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
103 104 105 106 107
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
108
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
109 110
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
111 112 113 114 115 116
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
117
#else
A
Akinobu Mita 已提交
118 119 120 121
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
122 123 124 125 126
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
127
	/* 32-bit code */
A
Akinobu Mita 已提交
128
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
129
	/* 16-bit code */
A
Akinobu Mita 已提交
130
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
131
	/* 16-bit data */
A
Akinobu Mita 已提交
132
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
133
	/* 16-bit data */
A
Akinobu Mita 已提交
134
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
135
	/* 16-bit data */
A
Akinobu Mita 已提交
136
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
137 138 139 140
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
141
	/* 32-bit code */
A
Akinobu Mita 已提交
142
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
143
	/* 16-bit code */
A
Akinobu Mita 已提交
144
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
145
	/* data */
146
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
147

A
Akinobu Mita 已提交
148 149
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
150
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
151
#endif
B
Brian Gerst 已提交
152
} };
153
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
154

155
static int __init x86_mpx_setup(char *s)
156
{
157
	/* require an exact match without trailing characters */
158 159
	if (strlen(s))
		return 0;
160

161 162 163
	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_MPX))
		return 1;
164

165 166
	setup_clear_cpu_cap(X86_FEATURE_MPX);
	pr_info("nompx: Intel Memory Protection Extensions (MPX) disabled\n");
167 168
	return 1;
}
169
__setup("nompx", x86_mpx_setup);
170

171
#ifdef CONFIG_X86_64
172
static int __init x86_nopcid_setup(char *s)
173
{
174 175 176
	/* nopcid doesn't accept parameters */
	if (s)
		return -EINVAL;
177 178 179

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_PCID))
180
		return 0;
181 182 183

	setup_clear_cpu_cap(X86_FEATURE_PCID);
	pr_info("nopcid: PCID feature disabled\n");
184
	return 0;
185
}
186
early_param("nopcid", x86_nopcid_setup);
187 188
#endif

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
static int __init x86_noinvpcid_setup(char *s)
{
	/* noinvpcid doesn't accept parameters */
	if (s)
		return -EINVAL;

	/* do not emit a message if the feature is not present */
	if (!boot_cpu_has(X86_FEATURE_INVPCID))
		return 0;

	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
	pr_info("noinvpcid: INVPCID feature disabled\n");
	return 0;
}
early_param("noinvpcid", x86_noinvpcid_setup);

205
#ifdef CONFIG_X86_32
206 207
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
L
Linus Torvalds 已提交
208

209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

228 229 230 231 232 233 234
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
235 236 237 238 239 240 241 242 243 244 245
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

246 247
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
248 249 250 251 252

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
253
int have_cpuid_p(void)
254 255 256 257
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

258
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
259
{
I
Ingo Molnar 已提交
260 261 262 263 264 265 266 267 268 269 270
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

271
	pr_notice("CPU serial number disabled.\n");
I
Ingo Molnar 已提交
272 273 274 275
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
276 277 278 279 280 281 282 283
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
284
#else
285 286 287 288 289 290 291
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
292
#endif
293

294 295
static __init int setup_disable_smep(char *arg)
{
296
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
297 298
	/* Check for things that depend on SMEP being enabled: */
	check_mpx_erratum(&boot_cpu_data);
299 300 301 302
	return 1;
}
__setup("nosmep", setup_disable_smep);

303
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
304
{
305
	if (cpu_has(c, X86_FEATURE_SMEP))
A
Andy Lutomirski 已提交
306
		cr4_set_bits(X86_CR4_SMEP);
307 308
}

309 310
static __init int setup_disable_smap(char *arg)
{
311
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
312 313 314 315
	return 1;
}
__setup("nosmap", setup_disable_smap);

316 317
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
318
	unsigned long eflags = native_save_fl();
319 320 321 322

	/* This should have been cleared long ago */
	BUG_ON(eflags & X86_EFLAGS_AC);

323 324
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
A
Andy Lutomirski 已提交
325
		cr4_set_bits(X86_CR4_SMAP);
326
#else
A
Andy Lutomirski 已提交
327
		cr4_clear_bits(X86_CR4_SMAP);
328 329
#endif
	}
330 331
}

332 333 334 335 336 337 338
/*
 * Protection Keys are not available in 32-bit mode.
 */
static bool pku_disabled;

static __always_inline void setup_pku(struct cpuinfo_x86 *c)
{
339 340 341 342
	/* check the boot processor, plus compile options for PKU: */
	if (!cpu_feature_enabled(X86_FEATURE_PKU))
		return;
	/* checks the actual processor's cpuid bits: */
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
	if (!cpu_has(c, X86_FEATURE_PKU))
		return;
	if (pku_disabled)
		return;

	cr4_set_bits(X86_CR4_PKE);
	/*
	 * Seting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
	 * cpuid bit to be set.  We need to ensure that we
	 * update that bit in this CPU's "cpu_info".
	 */
	get_cpu_cap(c);
}

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
static __init int setup_disable_pku(char *arg)
{
	/*
	 * Do not clear the X86_FEATURE_PKU bit.  All of the
	 * runtime checks are against OSPKE so clearing the
	 * bit does nothing.
	 *
	 * This way, we will see "pku" in cpuinfo, but not
	 * "ospke", which is exactly what we want.  It shows
	 * that the CPU has PKU, but the OS has not enabled it.
	 * This happens to be exactly how a system would look
	 * if we disabled the config option.
	 */
	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
	pku_disabled = true;
	return 1;
}
__setup("nopku", setup_disable_pku);
#endif /* CONFIG_X86_64 */

378 379 380 381 382 383 384 385 386
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
387

388
static const struct cpuid_dependent_feature
389 390 391 392 393 394 395
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

396
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
397 398
{
	const struct cpuid_dependent_feature *df;
399

400
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
401 402 403

		if (!cpu_has(c, df->feature))
			continue;
404 405 406 407 408 409 410
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
411
		if (!((s32)df->level < 0 ?
412
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
413 414 415 416 417 418 419
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

420 421
		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
			x86_cap_flag(df->feature), df->level);
422
	}
423
}
424

425 426 427
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
428 429
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
430 431 432
 */

/* Look up CPU names by table lookup. */
433
static const char *table_lookup_model(struct cpuinfo_x86 *c)
434
{
435 436
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
437 438 439 440 441 442 443

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

444
	info = this_cpu->legacy_models;
445

446
	while (info->family) {
447 448 449 450
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
451
#endif
452 453 454
	return NULL;		/* Not found */
}

455 456
__u32 cpu_caps_cleared[NCAPINTS];
__u32 cpu_caps_set[NCAPINTS];
457

458 459 460 461 462
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
463
	__loadsegment_simple(gs, 0);
464 465
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
466
	load_stack_canary_segment();
467 468
}

469 470 471 472 473
#ifdef CONFIG_X86_32
/* The 32-bit entry code needs to find cpu_entry_area. */
DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
#endif

474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
#ifdef CONFIG_X86_64
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
#endif

static void __init
set_percpu_fixmap_pages(int idx, void *ptr, int pages, pgprot_t prot)
{
	for ( ; pages; pages--, idx--, ptr += PAGE_SIZE)
		__set_fixmap(idx, per_cpu_ptr_to_phys(ptr), prot);
}

497
/* Setup the fixmap mappings only once per-processor */
498
static void __init setup_cpu_entry_area(int cpu)
499
{
500
#ifdef CONFIG_X86_64
501 502
	extern char _entry_trampoline[];

503
	/* On 64-bit systems, we use a read-only fixmap GDT. */
504
	pgprot_t gdt_prot = PAGE_KERNEL_RO;
505
#else
506 507 508 509 510 511 512 513 514
	/*
	 * On native 32-bit systems, the GDT cannot be read-only because
	 * our double fault handler uses a task gate, and entering through
	 * a task gate needs to change an available TSS to busy.  If the GDT
	 * is read-only, that will triple fault.
	 *
	 * On Xen PV, the GDT must be read-only because the hypervisor requires
	 * it.
	 */
515
	pgprot_t gdt_prot = boot_cpu_has(X86_FEATURE_XENPV) ?
516
		PAGE_KERNEL_RO : PAGE_KERNEL;
517
#endif
518

519
	__set_fixmap(get_cpu_entry_area_index(cpu, gdt), get_cpu_gdt_paddr(cpu), gdt_prot);
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539

	/*
	 * The Intel SDM says (Volume 3, 7.2.1):
	 *
	 *  Avoid placing a page boundary in the part of the TSS that the
	 *  processor reads during a task switch (the first 104 bytes). The
	 *  processor may not correctly perform address translations if a
	 *  boundary occurs in this area. During a task switch, the processor
	 *  reads and writes into the first 104 bytes of each TSS (using
	 *  contiguous physical addresses beginning with the physical address
	 *  of the first byte of the TSS). So, after TSS access begins, if
	 *  part of the 104 bytes is not physically contiguous, the processor
	 *  will access incorrect information without generating a page-fault
	 *  exception.
	 *
	 * There are also a lot of errata involving the TSS spanning a page
	 * boundary.  Assert that we're not doing that.
	 */
	BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^
		      offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK);
540 541 542 543 544
	BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0);
	set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, tss),
				&per_cpu(cpu_tss, cpu),
				sizeof(struct tss_struct) / PAGE_SIZE,
				PAGE_KERNEL);
545

546
#ifdef CONFIG_X86_32
547
	per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu);
548
#endif
549 550

#ifdef CONFIG_X86_64
551 552 553 554 555 556 557 558
	BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
	BUILD_BUG_ON(sizeof(exception_stacks) !=
		     sizeof(((struct cpu_entry_area *)0)->exception_stacks));
	set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, exception_stacks),
				&per_cpu(exception_stacks, cpu),
				sizeof(exception_stacks) / PAGE_SIZE,
				PAGE_KERNEL);

559 560 561
	__set_fixmap(get_cpu_entry_area_index(cpu, entry_trampoline),
		     __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX);
#endif
562 563
}

564 565 566 567 568 569 570 571
void __init setup_cpu_entry_areas(void)
{
	unsigned int cpu;

	for_each_possible_cpu(cpu)
		setup_cpu_entry_area(cpu);
}

572 573 574 575 576 577 578 579 580 581 582
/* Load the original GDT from the per-cpu structure */
void load_direct_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
EXPORT_SYMBOL_GPL(load_direct_gdt);

583 584 585 586 587 588 589 590 591
/* Load a fixmap remapping of the per-cpu GDT */
void load_fixmap_gdt(int cpu)
{
	struct desc_ptr gdt_descr;

	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
}
592
EXPORT_SYMBOL_GPL(load_fixmap_gdt);
593

I
Ingo Molnar 已提交
594 595 596 597
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
598
void switch_to_new_gdt(int cpu)
599
{
600 601
	/* Load the original GDT */
	load_direct_gdt(cpu);
602
	/* Reload the per-cpu base */
603
	load_percpu_segment(cpu);
604 605
}

606
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
607

608
static void get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
609 610
{
	unsigned int *v;
611
	char *p, *q, *s;
L
Linus Torvalds 已提交
612

613
	if (c->extended_cpuid_level < 0x80000004)
614
		return;
L
Linus Torvalds 已提交
615

I
Ingo Molnar 已提交
616
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
617 618 619 620 621
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
	/* Trim whitespace */
	p = q = s = &c->x86_model_id[0];

	while (*p == ' ')
		p++;

	while (*p) {
		/* Note the last non-whitespace index */
		if (!isspace(*p))
			s = q;

		*q++ = *p++;
	}

	*(s + 1) = '\0';
L
Linus Torvalds 已提交
637 638
}

639
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
640
{
641
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
642

643
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
644 645

	if (n >= 0x80000005) {
646 647
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
648 649 650 651
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
652 653 654 655 656
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

657
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
658
	l2size = ecx >> 16;
659

660 661 662
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
663
	/* do processor-specific cache resizing */
664 665
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
L
Linus Torvalds 已提交
666 667 668 669 670

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

671
	if (l2size == 0)
L
Linus Torvalds 已提交
672
		return;		/* Again, no L2 cache is possible */
673
#endif
L
Linus Torvalds 已提交
674 675 676 677

	c->x86_cache_size = l2size;
}

678 679 680 681 682 683
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
684
u16 __read_mostly tlb_lld_1g[NR_INFO];
685

686
static void cpu_detect_tlb(struct cpuinfo_x86 *c)
687 688 689 690
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

691
	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
692
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
693 694 695 696 697
		tlb_lli_4m[ENTRIES]);

	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
698 699
}

700
void detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
701
{
B
Borislav Petkov 已提交
702
#ifdef CONFIG_SMP
703 704
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
705
	static bool printed;
L
Linus Torvalds 已提交
706

707
	if (!cpu_has(c, X86_FEATURE_HT))
708
		return;
L
Linus Torvalds 已提交
709

710 711
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
712

713 714
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
715

716
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
717

718 719 720
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
721
		pr_info_once("CPU0: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
722 723
		goto out;
	}
724

I
Ingo Molnar 已提交
725 726
	if (smp_num_siblings <= 1)
		goto out;
727

I
Ingo Molnar 已提交
728 729
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
730

I
Ingo Molnar 已提交
731
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
732

I
Ingo Molnar 已提交
733
	index_msb = get_count_order(smp_num_siblings);
734

I
Ingo Molnar 已提交
735
	core_bits = get_count_order(c->x86_max_cores);
736

I
Ingo Molnar 已提交
737 738
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
739

740
out:
741
	if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
742 743 744 745
		pr_info("CPU: Physical Processor ID: %d\n",
			c->phys_proc_id);
		pr_info("CPU: Processor Core ID: %d\n",
			c->cpu_core_id);
746
		printed = 1;
747 748
	}
#endif
749
}
L
Linus Torvalds 已提交
750

751
static void get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
752 753
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
754
	int i;
L
Linus Torvalds 已提交
755 756

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
757 758 759 760 761 762
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
763

Y
Yinghai Lu 已提交
764 765 766
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
767 768
		}
	}
Y
Yinghai Lu 已提交
769

770 771
	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
		    "CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
772

773 774
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
775 776
}

777
void cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
778 779
{
	/* Get vendor name */
780 781 782 783
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
784 785

	c->x86 = 4;
786
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
787 788
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
789

L
Linus Torvalds 已提交
790
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
791 792 793
		c->x86		= x86_family(tfms);
		c->x86_model	= x86_model(tfms);
		c->x86_mask	= x86_stepping(tfms);
I
Ingo Molnar 已提交
794

H
Huang, Ying 已提交
795 796
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
797
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
798
		}
L
Linus Torvalds 已提交
799 800
	}
}
801

802 803 804 805 806 807 808 809 810 811
static void apply_forced_caps(struct cpuinfo_x86 *c)
{
	int i;

	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}
}

812
void get_cpu_cap(struct cpuinfo_x86 *c)
813
{
814
	u32 eax, ebx, ecx, edx;
815

816 817
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
818
		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
819

820 821
		c->x86_capability[CPUID_1_ECX] = ecx;
		c->x86_capability[CPUID_1_EDX] = edx;
822
	}
823

824 825 826 827
	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
	if (c->cpuid_level >= 0x00000006)
		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);

828 829 830
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
831
		c->x86_capability[CPUID_7_0_EBX] = ebx;
832
		c->x86_capability[CPUID_7_ECX] = ecx;
833 834
	}

835 836 837 838
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

839
		c->x86_capability[CPUID_D_1_EAX] = eax;
840 841
	}

842 843 844 845 846
	/* Additional Intel-defined flags: level 0x0000000F */
	if (c->cpuid_level >= 0x0000000F) {

		/* QoS sub-leaf, EAX=0Fh, ECX=0 */
		cpuid_count(0x0000000F, 0, &eax, &ebx, &ecx, &edx);
847 848
		c->x86_capability[CPUID_F_0_EDX] = edx;

849 850 851 852 853 854
		if (cpu_has(c, X86_FEATURE_CQM_LLC)) {
			/* will be overridden if occupancy monitoring exists */
			c->x86_cache_max_rmid = ebx;

			/* QoS sub-leaf, EAX=0Fh, ECX=1 */
			cpuid_count(0x0000000F, 1, &eax, &ebx, &ecx, &edx);
855 856
			c->x86_capability[CPUID_F_1_EDX] = edx;

857 858 859
			if ((cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC)) ||
			      ((cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL)) ||
			       (cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL)))) {
860 861 862 863 864 865 866 867 868
				c->x86_cache_max_rmid = ecx;
				c->x86_cache_occ_scale = ebx;
			}
		} else {
			c->x86_cache_max_rmid = -1;
			c->x86_cache_occ_scale = -1;
		}
	}

869
	/* AMD-defined flags: level 0x80000001 */
870 871 872 873 874 875
	eax = cpuid_eax(0x80000000);
	c->extended_cpuid_level = eax;

	if ((eax & 0xffff0000) == 0x80000000) {
		if (eax >= 0x80000001) {
			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
I
Ingo Molnar 已提交
876

877 878
			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
			c->x86_capability[CPUID_8000_0001_EDX] = edx;
879 880 881
		}
	}

882 883 884 885 886 887 888
	if (c->extended_cpuid_level >= 0x80000007) {
		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);

		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
		c->x86_power = edx;
	}

889
	if (c->extended_cpuid_level >= 0x80000008) {
890
		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
891 892 893

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
894
		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
895
	}
896 897 898
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
899
#endif
900

901
	if (c->extended_cpuid_level >= 0x8000000a)
902
		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
903

904
	init_scattered_cpuid_features(c);
905 906 907 908 909 910 911

	/*
	 * Clear/Set all flags overridden by options, after probe.
	 * This needs to happen each time we re-probe, which may happen
	 * several times during CPU initialization.
	 */
	apply_forced_caps(c);
912
}
L
Linus Torvalds 已提交
913

914
static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
Y
Yinghai Lu 已提交
915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

940 941 942 943 944 945 946 947 948
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
949
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
950
{
951 952
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
953 954
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
955
#else
H
Huang, Ying 已提交
956
	c->x86_clflush_size = 32;
957 958
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
959
#endif
960
	c->x86_cache_alignment = c->x86_clflush_size;
961

962
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
963
	c->extended_cpuid_level = 0;
964

Y
Yinghai Lu 已提交
965
	/* cyrix could have cpuid enabled via c_identify()*/
966 967 968 969
	if (have_cpuid_p()) {
		cpu_detect(c);
		get_cpu_vendor(c);
		get_cpu_cap(c);
B
Borislav Petkov 已提交
970
		setup_force_cpu_cap(X86_FEATURE_CPUID);
971

972 973
		if (this_cpu->c_early_init)
			this_cpu->c_early_init(c);
974

975 976
		c->cpu_index = 0;
		filter_cpuid_features(c, false);
977

978 979
		if (this_cpu->c_bsp_init)
			this_cpu->c_bsp_init(c);
B
Borislav Petkov 已提交
980 981 982
	} else {
		identify_cpu_without_cpuid(c);
		setup_clear_cpu_cap(X86_FEATURE_CPUID);
983
	}
984 985

	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
986
	fpu__init_system(c);
987 988 989 990 991 992 993 994

#ifdef CONFIG_X86_32
	/*
	 * Regardless of whether PCID is enumerated, the SDM says
	 * that it can't be enabled in 32-bit mode.
	 */
	setup_clear_cpu_cap(X86_FEATURE_PCID);
#endif
995 996
}

997 998
void __init early_cpu_init(void)
{
999
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
1000 1001
	int count = 0;

1002
#ifdef CONFIG_PROCESSOR_SELECT
1003
	pr_info("KERNEL supported cpus:\n");
1004 1005
#endif

Y
Yinghai Lu 已提交
1006
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1007
		const struct cpu_dev *cpudev = *cdev;
1008

Y
Yinghai Lu 已提交
1009 1010 1011 1012 1013
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

1014
#ifdef CONFIG_PROCESSOR_SELECT
1015 1016 1017 1018 1019 1020
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
1021
				pr_info("  %s %s\n", cpudev->c_vendor,
1022 1023
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
1024
		}
1025
#endif
Y
Yinghai Lu 已提交
1026
	}
1027
	early_identify_cpu(&boot_cpu_data);
1028
}
1029

1030
/*
B
Borislav Petkov 已提交
1031 1032 1033 1034 1035
 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
 * unfortunately, that's not true in practice because of early VIA
 * chips and (more importantly) broken virtualizers that are not easy
 * to detect. In the latter case it doesn't even *fail* reliably, so
 * probing for it doesn't even work. Disable it completely on 32-bit
1036
 * unless we can find a reliable way to detect all the broken cases.
B
Borislav Petkov 已提交
1037
 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1038
 */
1039
static void detect_nopl(struct cpuinfo_x86 *c)
1040
{
B
Borislav Petkov 已提交
1041
#ifdef CONFIG_X86_32
1042
	clear_cpu_cap(c, X86_FEATURE_NOPL);
B
Borislav Petkov 已提交
1043 1044
#else
	set_cpu_cap(c, X86_FEATURE_NOPL);
1045
#endif
1046
}
1047

1048 1049 1050
static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
1051
	/*
1052 1053 1054 1055 1056
	 * Empirically, writing zero to a segment selector on AMD does
	 * not clear the base, whereas writing zero to a segment
	 * selector on Intel does clear the base.  Intel's behavior
	 * allows slightly faster context switches in the common case
	 * where GS is unused by the prev and next threads.
1057
	 *
1058 1059 1060 1061 1062 1063
	 * Since neither vendor documents this anywhere that I can see,
	 * detect it directly instead of hardcoding the choice by
	 * vendor.
	 *
	 * I've designated AMD's behavior as the "bug" because it's
	 * counterintuitive and less friendly.
1064
	 */
1065 1066 1067 1068 1069 1070 1071 1072 1073

	unsigned long old_base, tmp;
	rdmsrl(MSR_FS_BASE, old_base);
	wrmsrl(MSR_FS_BASE, 1);
	loadsegment(fs, 0);
	rdmsrl(MSR_FS_BASE, tmp);
	if (tmp != 0)
		set_cpu_bug(c, X86_BUG_NULL_SEG);
	wrmsrl(MSR_FS_BASE, old_base);
B
Borislav Petkov 已提交
1074
#endif
1075 1076
}

1077
static void generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1078
{
Y
Yinghai Lu 已提交
1079
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
1080

1081
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
1082
		identify_cpu_without_cpuid(c);
1083

Y
Yinghai Lu 已提交
1084
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
1085
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
1086
		return;
L
Linus Torvalds 已提交
1087

1088
	cpu_detect(c);
L
Linus Torvalds 已提交
1089

1090
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
1091

1092
	get_cpu_cap(c);
L
Linus Torvalds 已提交
1093

1094 1095
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1096
#ifdef CONFIG_X86_32
B
Borislav Petkov 已提交
1097
# ifdef CONFIG_SMP
1098
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1099
# else
1100
		c->apicid = c->initial_apicid;
1101 1102 1103
# endif
#endif
		c->phys_proc_id = c->initial_apicid;
1104
	}
L
Linus Torvalds 已提交
1105

1106
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
1107

1108
	detect_nopl(c);
1109 1110

	detect_null_seg_behavior(c);
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135

	/*
	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
	 * systems that run Linux at CPL > 0 may or may not have the
	 * issue, but, even if they have the issue, there's absolutely
	 * nothing we can do about it because we can't use the real IRET
	 * instruction.
	 *
	 * NB: For the time being, only 32-bit kernels support
	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
	 * whether to apply espfix using paravirt hooks.  If any
	 * non-paravirt system ever shows up that does *not* have the
	 * ESPFIX issue, we can change this.
	 */
#ifdef CONFIG_X86_32
# ifdef CONFIG_PARAVIRT
	do {
		extern void native_iret(void);
		if (pv_cpu_ops.iret == native_iret)
			set_cpu_bug(c, X86_BUG_ESPFIX);
	} while (0);
# else
	set_cpu_bug(c, X86_BUG_ESPFIX);
# endif
#endif
L
Linus Torvalds 已提交
1136 1137
}

1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
static void x86_init_cache_qos(struct cpuinfo_x86 *c)
{
	/*
	 * The heavy lifting of max_rmid and cache_occ_scale are handled
	 * in get_cpu_cap().  Here we just set the max_rmid for the boot_cpu
	 * in case CQM bits really aren't there in this CPU.
	 */
	if (c != &boot_cpu_data) {
		boot_cpu_data.x86_cache_max_rmid =
			min(boot_cpu_data.x86_cache_max_rmid,
			    c->x86_cache_max_rmid);
	}
}

1152
/*
1153 1154
 * Validate that ACPI/mptables have the same information about the
 * effective APIC id and update the package map.
1155
 */
1156
static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1157 1158
{
#ifdef CONFIG_SMP
1159
	unsigned int apicid, cpu = smp_processor_id();
1160 1161 1162

	apicid = apic->cpu_present_to_apicid(cpu);

1163 1164
	if (apicid != c->apicid) {
		pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1165 1166
		       cpu, apicid, c->initial_apicid);
	}
1167
	BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1168 1169 1170 1171 1172
#else
	c->logical_proc_id = 0;
#endif
}

L
Linus Torvalds 已提交
1173 1174 1175
/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
1176
static void identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1177 1178 1179 1180 1181 1182 1183 1184 1185
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
1186
	c->x86_max_cores = 1;
1187
	c->x86_coreid_bits = 0;
1188
	c->cu_id = 0xff;
1189
#ifdef CONFIG_X86_64
1190
	c->x86_clflush_size = 64;
1191 1192
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
1193 1194
#else
	c->cpuid_level = -1;	/* CPUID not detected */
1195
	c->x86_clflush_size = 32;
1196 1197
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
1198 1199
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
1200 1201 1202 1203
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

1204
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
1205 1206
		this_cpu->c_identify(c);

1207
	/* Clear/Set all flags overridden by options, after probe */
1208
	apply_forced_caps(c);
1209

1210
#ifdef CONFIG_X86_64
1211
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1212 1213
#endif

L
Linus Torvalds 已提交
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

1230 1231 1232 1233
	/* Set up SMEP/SMAP */
	setup_smep(c);
	setup_smap(c);

L
Linus Torvalds 已提交
1234
	/*
I
Ingo Molnar 已提交
1235 1236
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
1237 1238
	 */

1239 1240 1241
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
1242
	/* If the model name is still unset, do table lookup. */
1243
	if (!c->x86_model_id[0]) {
1244
		const char *p;
L
Linus Torvalds 已提交
1245
		p = table_lookup_model(c);
1246
		if (p)
L
Linus Torvalds 已提交
1247 1248 1249 1250
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
1251
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
1252 1253
	}

1254 1255 1256 1257
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

1258
	x86_init_rdrand(c);
1259
	x86_init_cache_qos(c);
1260
	setup_pku(c);
1261 1262

	/*
1263
	 * Clear/Set all flags overridden by options, need do it
1264 1265
	 * before following smp all cpus cap AND.
	 */
1266
	apply_forced_caps(c);
1267

L
Linus Torvalds 已提交
1268 1269 1270 1271 1272 1273
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
1274
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
1275
		/* AND the already accumulated flags with these */
1276
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
1277
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1278 1279 1280 1281

		/* OR, i.e. replicate the bug flags */
		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
L
Linus Torvalds 已提交
1282 1283 1284
	}

	/* Init Machine Check Exception if available. */
1285
	mcheck_cpu_init(c);
1286 1287

	select_idle_routine(c);
1288

1289
#ifdef CONFIG_NUMA
1290 1291
	numa_add_cpu(smp_processor_id());
#endif
1292
}
S
Shaohua Li 已提交
1293

1294 1295 1296 1297
/*
 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
 * on 32-bit kernels:
 */
1298 1299 1300
#ifdef CONFIG_X86_32
void enable_sep_cpu(void)
{
1301 1302
	struct tss_struct *tss;
	int cpu;
1303

1304 1305 1306
	if (!boot_cpu_has(X86_FEATURE_SEP))
		return;

1307 1308 1309 1310
	cpu = get_cpu();
	tss = &per_cpu(cpu_tss, cpu);

	/*
1311 1312
	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
	 * see the big comment in struct x86_hw_tss's definition.
1313
	 */
1314 1315

	tss->x86_tss.ss1 = __KERNEL_CS;
1316
	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1317
	wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_SYSENTER_stack(cpu) + 1), 0);
1318
	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1319

1320 1321
	put_cpu();
}
1322 1323
#endif

1324 1325 1326
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
1327
#ifdef CONFIG_X86_32
1328
	sysenter_setup();
L
Li Shaohua 已提交
1329
	enable_sep_cpu();
1330
#endif
1331
	cpu_detect_tlb(&boot_cpu_data);
1332
}
S
Shaohua Li 已提交
1333

1334
void identify_secondary_cpu(struct cpuinfo_x86 *c)
1335 1336 1337
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
1338
#ifdef CONFIG_X86_32
1339
	enable_sep_cpu();
1340
#endif
1341
	mtrr_ap_init();
1342
	validate_apic_and_package_id(c);
L
Linus Torvalds 已提交
1343 1344
}

A
Andi Kleen 已提交
1345 1346
static __init int setup_noclflush(char *arg)
{
1347
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1348
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
A
Andi Kleen 已提交
1349 1350 1351 1352
	return 1;
}
__setup("noclflush", setup_noclflush);

1353
void print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1354
{
1355
	const char *vendor = NULL;
L
Linus Torvalds 已提交
1356

I
Ingo Molnar 已提交
1357
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
1358
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
1359 1360 1361 1362
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
1363

1364
	if (vendor && !strstr(c->x86_model_id, vendor))
1365
		pr_cont("%s ", vendor);
L
Linus Torvalds 已提交
1366

1367
	if (c->x86_model_id[0])
1368
		pr_cont("%s", c->x86_model_id);
L
Linus Torvalds 已提交
1369
	else
1370
		pr_cont("%d86", c->x86);
L
Linus Torvalds 已提交
1371

1372
	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1373

1374
	if (c->x86_mask || c->cpuid_level >= 0)
1375
		pr_cont(", stepping: 0x%x)\n", c->x86_mask);
L
Linus Torvalds 已提交
1376
	else
1377
		pr_cont(")\n");
L
Linus Torvalds 已提交
1378 1379
}

1380 1381 1382 1383 1384 1385
/*
 * clearcpuid= was already parsed in fpu__init_parse_early_param.
 * But we need to keep a dummy __setup around otherwise it would
 * show up as an environment variable for init.
 */
static __init int setup_clearcpuid(char *arg)
1386 1387 1388
{
	return 1;
}
1389
__setup("clearcpuid=", setup_clearcpuid);
1390

1391
#ifdef CONFIG_X86_64
1392
DEFINE_PER_CPU_FIRST(union irq_stack_union,
1393
		     irq_stack_union) __aligned(PAGE_SIZE) __visible;
I
Ingo Molnar 已提交
1394

1395
/*
1396 1397
 * The following percpu variables are hot.  Align current_task to
 * cacheline size such that they fall in the same cacheline.
1398 1399 1400 1401
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1402

1403
DEFINE_PER_CPU(char *, irq_stack_ptr) =
1404
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE;
1405

1406
DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1407

1408 1409 1410
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);

1411 1412
/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1413
{
1414 1415 1416
	extern char _entry_trampoline[];
	extern char entry_SYSCALL_64_trampoline[];

1417
	int cpu = smp_processor_id();
1418 1419 1420
	unsigned long SYSCALL64_entry_trampoline =
		(unsigned long)get_cpu_entry_area(cpu)->entry_trampoline +
		(entry_SYSCALL_64_trampoline - _entry_trampoline);
1421

1422
	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
1423
	wrmsrl(MSR_LSTAR, SYSCALL64_entry_trampoline);
1424 1425

#ifdef CONFIG_IA32_EMULATION
1426
	wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat);
1427
	/*
1428 1429 1430 1431
	 * This only works on Intel CPUs.
	 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
	 * This does not cause SYSENTER to jump to the wrong location, because
	 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
1432 1433
	 */
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
1434
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_SYSENTER_stack(cpu) + 1));
1435
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
1436
#else
1437
	wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
1438
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1439 1440
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1441
#endif
1442

1443 1444
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
1445
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1446
	       X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
L
Linus Torvalds 已提交
1447
}
1448

1449 1450 1451 1452 1453 1454
/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

1455
static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
1456
DEFINE_PER_CPU(int, debug_stack_usage);
1457 1458 1459

int is_debug_stack(unsigned long addr)
{
1460 1461 1462
	return __this_cpu_read(debug_stack_usage) ||
		(addr <= __this_cpu_read(debug_stack_addr) &&
		 addr > (__this_cpu_read(debug_stack_addr) - DEBUG_STKSZ));
1463
}
1464
NOKPROBE_SYMBOL(is_debug_stack);
1465

1466
DEFINE_PER_CPU(u32, debug_idt_ctr);
1467

1468 1469
void debug_stack_set_zero(void)
{
1470 1471
	this_cpu_inc(debug_idt_ctr);
	load_current_idt();
1472
}
1473
NOKPROBE_SYMBOL(debug_stack_set_zero);
1474 1475 1476

void debug_stack_reset(void)
{
1477
	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
1478
		return;
1479 1480
	if (this_cpu_dec_return(debug_idt_ctr) == 0)
		load_current_idt();
1481
}
1482
NOKPROBE_SYMBOL(debug_stack_reset);
1483

I
Ingo Molnar 已提交
1484
#else	/* CONFIG_X86_64 */
1485

1486 1487
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1488 1489
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
1490

1491 1492 1493 1494 1495 1496 1497 1498 1499
/*
 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
 * the top of the kernel stack.  Use an extra percpu variable to track the
 * top of the kernel stack directly.
 */
DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
	(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);

1500
#ifdef CONFIG_CC_STACKPROTECTOR
1501
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1502
#endif
1503

I
Ingo Molnar 已提交
1504
#endif	/* CONFIG_X86_64 */
1505

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1521

1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
#ifdef CONFIG_KGDB
/*
 * Restore debug regs if using kgdbwait and you have a kernel debugger
 * connection established.
 */
static void dbg_restore_debug_regs(void)
{
	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
		arch_kgdb_ops.correct_hw_break();
}
#else /* ! CONFIG_KGDB */
#define dbg_restore_debug_regs()
#endif /* ! CONFIG_KGDB */

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
static void wait_for_master_cpu(int cpu)
{
#ifdef CONFIG_SMP
	/*
	 * wait for ACK from master CPU before continuing
	 * with AP initialization
	 */
	WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
	while (!cpumask_test_cpu(cpu, cpu_callout_mask))
		cpu_relax();
#endif
}

1549 1550 1551 1552 1553
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1554
 * A lot of state is already set up in PDA init for 64 bit
1555
 */
1556
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1557

1558
void cpu_init(void)
1559
{
1560
	struct orig_ist *oist;
1561
	struct task_struct *me;
I
Ingo Molnar 已提交
1562 1563
	struct tss_struct *t;
	unsigned long v;
1564
	int cpu = raw_smp_processor_id();
1565 1566
	int i;

1567 1568
	wait_for_master_cpu(cpu);

1569 1570 1571 1572 1573 1574
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1575 1576
	if (cpu)
		load_ucode_ap();
1577

1578
	t = &per_cpu(cpu_tss, cpu);
1579
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1580

1581
#ifdef CONFIG_NUMA
1582
	if (this_cpu_read(numa_node) == 0 &&
1583 1584
	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
		set_numa_node(early_cpu_to_node(cpu));
1585
#endif
1586 1587 1588

	me = current;

1589
	pr_debug("Initializing CPU#%d\n", cpu);
1590

A
Andy Lutomirski 已提交
1591
	cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1592 1593 1594 1595 1596 1597

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1598
	switch_to_new_gdt(cpu);
1599 1600
	loadsegment(fs, 0);

1601
	load_current_idt();
1602 1603 1604 1605 1606 1607 1608 1609

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1610
	x86_configure_nx();
1611
	x2apic_setup();
1612 1613 1614 1615

	/*
	 * set up and load the per-CPU TSS
	 */
1616
	if (!oist->ist[0]) {
1617
		char *estacks = get_cpu_entry_area(cpu)->exception_stacks;
I
Ingo Molnar 已提交
1618

1619
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1620
			estacks += exception_stack_sizes[v];
1621
			oist->ist[v] = t->x86_tss.ist[v] =
1622
					(unsigned long)estacks;
1623 1624
			if (v == DEBUG_STACK-1)
				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1625 1626 1627
		}
	}

1628
	t->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
I
Ingo Molnar 已提交
1629

1630 1631 1632 1633 1634 1635 1636
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

V
Vegard Nossum 已提交
1637
	mmgrab(&init_mm);
1638
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1639
	BUG_ON(me->mm);
1640
	initialize_tlbstate_and_flush();
1641 1642
	enter_lazy_tlb(&init_mm, me);

1643
	/*
1644 1645
	 * Initialize the TSS.  sp0 points to the entry trampoline stack
	 * regardless of what task is running.
1646
	 */
1647
	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
1648
	load_TR_desc();
1649
	load_sp0((unsigned long)(cpu_SYSENTER_stack(cpu) + 1));
1650

1651
	load_mm_ldt(&init_mm);
1652

1653 1654
	clear_all_debug_regs();
	dbg_restore_debug_regs();
1655

I
Ingo Molnar 已提交
1656
	fpu__init_cpu();
1657 1658 1659

	if (is_uv_system())
		uv_cpu_init();
1660 1661

	load_fixmap_gdt(cpu);
1662 1663 1664 1665
}

#else

1666
void cpu_init(void)
1667
{
1668 1669
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1670
	struct tss_struct *t = &per_cpu(cpu_tss, cpu);
1671

1672
	wait_for_master_cpu(cpu);
1673

1674 1675 1676 1677 1678 1679
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1680
	show_ucode_info_early();
1681

1682
	pr_info("Initializing CPU#%d\n", cpu);
1683

1684
	if (cpu_feature_enabled(X86_FEATURE_VME) ||
1685
	    boot_cpu_has(X86_FEATURE_TSC) ||
1686
	    boot_cpu_has(X86_FEATURE_DE))
A
Andy Lutomirski 已提交
1687
		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1688

1689
	load_current_idt();
1690
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1691 1692 1693 1694

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
V
Vegard Nossum 已提交
1695
	mmgrab(&init_mm);
1696
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1697
	BUG_ON(curr->mm);
1698
	initialize_tlbstate_and_flush();
1699
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1700

1701 1702 1703 1704
	/*
	 * Initialize the TSS.  Don't bother initializing sp0, as the initial
	 * task never enters user mode.
	 */
1705
	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
L
Linus Torvalds 已提交
1706
	load_TR_desc();
1707

1708
	load_mm_ldt(&init_mm);
L
Linus Torvalds 已提交
1709

1710
	t->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET;
1711

1712
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1713 1714
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1715
#endif
L
Linus Torvalds 已提交
1716

1717
	clear_all_debug_regs();
1718
	dbg_restore_debug_regs();
L
Linus Torvalds 已提交
1719

I
Ingo Molnar 已提交
1720
	fpu__init_cpu();
1721 1722

	load_fixmap_gdt(cpu);
L
Linus Torvalds 已提交
1723
}
1724
#endif
1725

1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741
static void bsp_resume(void)
{
	if (this_cpu->c_bsp_resume)
		this_cpu->c_bsp_resume(&boot_cpu_data);
}

static struct syscore_ops cpu_syscore_ops = {
	.resume		= bsp_resume,
};

static int __init init_cpu_syscore(void)
{
	register_syscore_ops(&cpu_syscore_ops);
	return 0;
}
core_initcall(init_cpu_syscore);