common.c 33.1 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/module.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
L
Linus Torvalds 已提交
8
#include <linux/delay.h>
9 10 11
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
12
#include <linux/smp.h>
13 14 15
#include <linux/io.h>

#include <asm/stackprotector.h>
16
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
17
#include <asm/mmu_context.h>
18
#include <asm/archrandom.h>
19 20
#include <asm/hypervisor.h>
#include <asm/processor.h>
21
#include <asm/debugreg.h>
22
#include <asm/sections.h>
A
Alan Cox 已提交
23 24
#include <linux/topology.h>
#include <linux/cpumask.h>
25
#include <asm/pgtable.h>
A
Arun Sharma 已提交
26
#include <linux/atomic.h>
27 28 29 30 31
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/i387.h>
32
#include <asm/fpu-internal.h>
33
#include <asm/mtrr.h>
A
Alan Cox 已提交
34
#include <linux/numa.h>
35 36
#include <asm/asm.h>
#include <asm/cpu.h>
37
#include <asm/mce.h>
38
#include <asm/msr.h>
39
#include <asm/pat.h>
40 41
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
42 43

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
44
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
45 46 47 48
#endif

#include "cpu.h"

49 50
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
51 52
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
53 54 55 56

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
57
/* correctly size the local cpu masks */
58
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
59 60 61 62 63 64 65
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

66
static void default_init(struct cpuinfo_x86 *c)
67 68
{
#ifdef CONFIG_X86_64
69
	cpu_detect_cache_sizes(c);
70 71 72 73 74 75 76 77 78 79 80 81 82
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

83
static const struct cpu_dev default_cpu = {
84 85 86 87 88
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

89
static const struct cpu_dev *this_cpu = &default_cpu;
90

B
Brian Gerst 已提交
91
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
92
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
93 94 95 96 97
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
98
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
99 100
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
101 102 103 104 105 106
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
107
#else
A
Akinobu Mita 已提交
108 109 110 111
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
112 113 114 115 116
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
117
	/* 32-bit code */
A
Akinobu Mita 已提交
118
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
119
	/* 16-bit code */
A
Akinobu Mita 已提交
120
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
121
	/* 16-bit data */
A
Akinobu Mita 已提交
122
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
123
	/* 16-bit data */
A
Akinobu Mita 已提交
124
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
125
	/* 16-bit data */
A
Akinobu Mita 已提交
126
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
127 128 129 130
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
131
	/* 32-bit code */
A
Akinobu Mita 已提交
132
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
133
	/* 16-bit code */
A
Akinobu Mita 已提交
134
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
135
	/* data */
136
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
137

A
Akinobu Mita 已提交
138 139
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
140
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
141
#endif
B
Brian Gerst 已提交
142
} };
143
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
144

145 146 147
static int __init x86_xsave_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
148
	setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
149 150
	setup_clear_cpu_cap(X86_FEATURE_AVX);
	setup_clear_cpu_cap(X86_FEATURE_AVX2);
151 152 153 154
	return 1;
}
__setup("noxsave", x86_xsave_setup);

155 156 157 158 159 160 161
static int __init x86_xsaveopt_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
	return 1;
}
__setup("noxsaveopt", x86_xsaveopt_setup);

162
#ifdef CONFIG_X86_32
163 164
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
L
Linus Torvalds 已提交
165

166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_fxsr_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_FXSR);
	setup_clear_cpu_cap(X86_FEATURE_XMM);
	return 1;
}
__setup("nofxsr", x86_fxsr_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

193 194 195 196 197 198 199
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
200 201 202 203 204 205 206 207 208 209 210
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

211 212
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
213 214 215 216 217

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
218
int have_cpuid_p(void)
219 220 221 222
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

223
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
224
{
I
Ingo Molnar 已提交
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

	printk(KERN_NOTICE "CPU serial number disabled.\n");
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
241 242 243 244 245 246 247 248
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
249
#else
250 251 252 253 254 255 256
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
257
#endif
258

259 260
static __init int setup_disable_smep(char *arg)
{
261
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
262 263 264 265
	return 1;
}
__setup("nosmep", setup_disable_smep);

266
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
267
{
268 269
	if (cpu_has(c, X86_FEATURE_SMEP))
		set_in_cr4(X86_CR4_SMEP);
270 271
}

272 273
static __init int setup_disable_smap(char *arg)
{
274
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
275 276 277 278
	return 1;
}
__setup("nosmap", setup_disable_smap);

279 280 281 282 283 284 285 286
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
	unsigned long eflags;

	/* This should have been cleared long ago */
	raw_local_save_flags(eflags);
	BUG_ON(eflags & X86_EFLAGS_AC);

287 288
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
289
		set_in_cr4(X86_CR4_SMAP);
290 291 292 293
#else
		clear_in_cr4(X86_CR4_SMAP);
#endif
	}
294 295
}

296 297 298 299 300 301 302 303 304
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
305

306
static const struct cpuid_dependent_feature
307 308 309 310 311 312 313
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

314
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
315 316
{
	const struct cpuid_dependent_feature *df;
317

318
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
319 320 321

		if (!cpu_has(c, df->feature))
			continue;
322 323 324 325 326 327 328
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
329
		if (!((s32)df->level < 0 ?
330
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
331 332 333 334 335 336 337 338 339 340
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

		printk(KERN_WARNING
		       "CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
				x86_cap_flags[df->feature], df->level);
341
	}
342
}
343

344 345 346
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
347 348
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
349 350 351
 */

/* Look up CPU names by table lookup. */
352
static const char *table_lookup_model(struct cpuinfo_x86 *c)
353
{
354 355
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
356 357 358 359 360 361 362

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

363
	info = this_cpu->legacy_models;
364

365
	while (info->family) {
366 367 368 369
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
370
#endif
371 372 373
	return NULL;		/* Not found */
}

374 375
__u32 cpu_caps_cleared[NCAPINTS];
__u32 cpu_caps_set[NCAPINTS];
376

377 378 379 380 381 382 383 384
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
	loadsegment(gs, 0);
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
385
	load_stack_canary_segment();
386 387
}

I
Ingo Molnar 已提交
388 389 390 391
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
392
void switch_to_new_gdt(int cpu)
393 394 395
{
	struct desc_ptr gdt_descr;

396
	gdt_descr.address = (long)get_cpu_gdt_table(cpu);
397 398
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
399
	/* Reload the per-cpu base */
400 401

	load_percpu_segment(cpu);
402 403
}

404
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
405

406
static void get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
407 408 409 410
{
	unsigned int *v;
	char *p, *q;

411
	if (c->extended_cpuid_level < 0x80000004)
412
		return;
L
Linus Torvalds 已提交
413

I
Ingo Molnar 已提交
414
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
415 416 417 418 419
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

I
Ingo Molnar 已提交
420 421 422 423
	/*
	 * Intel chips right-justify this string for some dumb reason;
	 * undo that brain damage:
	 */
L
Linus Torvalds 已提交
424
	p = q = &c->x86_model_id[0];
425
	while (*p == ' ')
426
		p++;
427
	if (p != q) {
428 429 430 431
		while (*p)
			*q++ = *p++;
		while (q <= &c->x86_model_id[48])
			*q++ = '\0';	/* Zero-pad the rest */
L
Linus Torvalds 已提交
432 433 434
	}
}

435
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
436
{
437
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
438

439
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
440 441

	if (n >= 0x80000005) {
442 443
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
444 445 446 447
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
448 449 450 451 452
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

453
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
454
	l2size = ecx >> 16;
455

456 457 458
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
459
	/* do processor-specific cache resizing */
460 461
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
L
Linus Torvalds 已提交
462 463 464 465 466

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

467
	if (l2size == 0)
L
Linus Torvalds 已提交
468
		return;		/* Again, no L2 cache is possible */
469
#endif
L
Linus Torvalds 已提交
470 471 472 473

	c->x86_cache_size = l2size;
}

474 475 476 477 478 479
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
480
u16 __read_mostly tlb_lld_1g[NR_INFO];
481

482 483 484 485 486 487 488 489
/*
 * tlb_flushall_shift shows the balance point in replacing cr3 write
 * with multiple 'invlpg'. It will do this replacement when
 *   flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
 * If tlb_flushall_shift is -1, means the replacement will be disabled.
 */
s8  __read_mostly tlb_flushall_shift = -1;

490
void cpu_detect_tlb(struct cpuinfo_x86 *c)
491 492 493 494
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

495 496
	printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n"
		"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n"
497
		"tlb_flushall_shift: %d\n",
498 499
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
		tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
500
		tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
501
		tlb_lld_1g[ENTRIES], tlb_flushall_shift);
502 503
}

504
void detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
505
{
506
#ifdef CONFIG_X86_HT
507 508
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
509
	static bool printed;
L
Linus Torvalds 已提交
510

511
	if (!cpu_has(c, X86_FEATURE_HT))
512
		return;
L
Linus Torvalds 已提交
513

514 515
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
516

517 518
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
519

520
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
521

522 523 524
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
525
		printk_once(KERN_INFO "CPU0: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
526 527
		goto out;
	}
528

I
Ingo Molnar 已提交
529 530
	if (smp_num_siblings <= 1)
		goto out;
531

I
Ingo Molnar 已提交
532 533
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
534

I
Ingo Molnar 已提交
535
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
536

I
Ingo Molnar 已提交
537
	index_msb = get_count_order(smp_num_siblings);
538

I
Ingo Molnar 已提交
539
	core_bits = get_count_order(c->x86_max_cores);
540

I
Ingo Molnar 已提交
541 542
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
543

544
out:
545
	if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
546 547 548 549
		printk(KERN_INFO  "CPU: Physical Processor ID: %d\n",
		       c->phys_proc_id);
		printk(KERN_INFO  "CPU: Processor Core ID: %d\n",
		       c->cpu_core_id);
550
		printed = 1;
551 552
	}
#endif
553
}
L
Linus Torvalds 已提交
554

555
static void get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
556 557
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
558
	int i;
L
Linus Torvalds 已提交
559 560

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
561 562 563 564 565 566
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
567

Y
Yinghai Lu 已提交
568 569 570
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
571 572
		}
	}
Y
Yinghai Lu 已提交
573

574 575 576
	printk_once(KERN_ERR
			"CPU: vendor_id '%s' unknown, using generic init.\n" \
			"CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
577

578 579
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
580 581
}

582
void cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
583 584
{
	/* Get vendor name */
585 586 587 588
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
589 590

	c->x86 = 4;
591
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
592 593
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
594

L
Linus Torvalds 已提交
595
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
596 597 598
		c->x86 = (tfms >> 8) & 0xf;
		c->x86_model = (tfms >> 4) & 0xf;
		c->x86_mask = tfms & 0xf;
I
Ingo Molnar 已提交
599

600
		if (c->x86 == 0xf)
L
Linus Torvalds 已提交
601
			c->x86 += (tfms >> 20) & 0xff;
602
		if (c->x86 >= 0x6)
603
			c->x86_model += ((tfms >> 16) & 0xf) << 4;
I
Ingo Molnar 已提交
604

H
Huang, Ying 已提交
605 606
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
607
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
608
		}
L
Linus Torvalds 已提交
609 610
	}
}
611

612
void get_cpu_cap(struct cpuinfo_x86 *c)
613 614
{
	u32 tfms, xlvl;
615
	u32 ebx;
616

617 618 619
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
		u32 capability, excap;
I
Ingo Molnar 已提交
620

621 622 623 624
		cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
		c->x86_capability[0] = capability;
		c->x86_capability[4] = excap;
	}
625

626 627 628 629 630 631
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		u32 eax, ebx, ecx, edx;

		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);

632
		c->x86_capability[9] = ebx;
633 634
	}

635 636 637 638 639 640 641 642 643
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		u32 eax, ebx, ecx, edx;

		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

		c->x86_capability[10] = eax;
	}

644 645 646
	/* AMD-defined flags: level 0x80000001 */
	xlvl = cpuid_eax(0x80000000);
	c->extended_cpuid_level = xlvl;
I
Ingo Molnar 已提交
647

648 649 650 651
	if ((xlvl & 0xffff0000) == 0x80000000) {
		if (xlvl >= 0x80000001) {
			c->x86_capability[1] = cpuid_edx(0x80000001);
			c->x86_capability[6] = cpuid_ecx(0x80000001);
652 653 654
		}
	}

655 656 657 658 659
	if (c->extended_cpuid_level >= 0x80000008) {
		u32 eax = cpuid_eax(0x80000008);

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
660
	}
661 662 663
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
664
#endif
665 666 667

	if (c->extended_cpuid_level >= 0x80000007)
		c->x86_power = cpuid_edx(0x80000007);
668

669
	init_scattered_cpuid_features(c);
670
}
L
Linus Torvalds 已提交
671

672
static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
Y
Yinghai Lu 已提交
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

698 699 700 701 702 703 704 705 706
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
707
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
708
{
709 710
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
711 712
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
713
#else
H
Huang, Ying 已提交
714
	c->x86_clflush_size = 32;
715 716
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
717
#endif
718
	c->x86_cache_alignment = c->x86_clflush_size;
719

720
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
721
	c->extended_cpuid_level = 0;
722

Y
Yinghai Lu 已提交
723 724 725 726
	if (!have_cpuid_p())
		identify_cpu_without_cpuid(c);

	/* cyrix could have cpuid enabled via c_identify()*/
727 728 729 730
	if (!have_cpuid_p())
		return;

	cpu_detect(c);
731 732
	get_cpu_vendor(c);
	get_cpu_cap(c);
733
	fpu_detect(c);
734

Y
Yinghai Lu 已提交
735 736
	if (this_cpu->c_early_init)
		this_cpu->c_early_init(c);
737

738
	c->cpu_index = 0;
739
	filter_cpuid_features(c, false);
740

B
Borislav Petkov 已提交
741 742
	if (this_cpu->c_bsp_init)
		this_cpu->c_bsp_init(c);
743 744

	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
745 746
}

747 748
void __init early_cpu_init(void)
{
749
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
750 751
	int count = 0;

752
#ifdef CONFIG_PROCESSOR_SELECT
753
	printk(KERN_INFO "KERNEL supported cpus:\n");
754 755
#endif

Y
Yinghai Lu 已提交
756
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
757
		const struct cpu_dev *cpudev = *cdev;
758

Y
Yinghai Lu 已提交
759 760 761 762 763
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

764
#ifdef CONFIG_PROCESSOR_SELECT
765 766 767 768 769 770 771 772 773
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
				printk(KERN_INFO "  %s %s\n", cpudev->c_vendor,
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
774
		}
775
#endif
Y
Yinghai Lu 已提交
776
	}
777
	early_identify_cpu(&boot_cpu_data);
778
}
779

780
/*
B
Borislav Petkov 已提交
781 782 783 784 785
 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
 * unfortunately, that's not true in practice because of early VIA
 * chips and (more importantly) broken virtualizers that are not easy
 * to detect. In the latter case it doesn't even *fail* reliably, so
 * probing for it doesn't even work. Disable it completely on 32-bit
786
 * unless we can find a reliable way to detect all the broken cases.
B
Borislav Petkov 已提交
787
 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
788
 */
789
static void detect_nopl(struct cpuinfo_x86 *c)
790
{
B
Borislav Petkov 已提交
791
#ifdef CONFIG_X86_32
792
	clear_cpu_cap(c, X86_FEATURE_NOPL);
B
Borislav Petkov 已提交
793 794 795
#else
	set_cpu_cap(c, X86_FEATURE_NOPL);
#endif
796 797
}

798
static void generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
799
{
Y
Yinghai Lu 已提交
800
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
801

802
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
803
		identify_cpu_without_cpuid(c);
804

Y
Yinghai Lu 已提交
805
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
806
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
807
		return;
L
Linus Torvalds 已提交
808

809
	cpu_detect(c);
L
Linus Torvalds 已提交
810

811
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
812

813
	get_cpu_cap(c);
L
Linus Torvalds 已提交
814

815 816
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
817 818
#ifdef CONFIG_X86_32
# ifdef CONFIG_X86_HT
819
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
820
# else
821
		c->apicid = c->initial_apicid;
822 823 824
# endif
#endif
		c->phys_proc_id = c->initial_apicid;
825
	}
L
Linus Torvalds 已提交
826

827
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
828

829
	detect_nopl(c);
L
Linus Torvalds 已提交
830 831 832 833 834
}

/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
835
static void identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
836 837 838 839 840 841 842 843 844
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
845
	c->x86_max_cores = 1;
846
	c->x86_coreid_bits = 0;
847
#ifdef CONFIG_X86_64
848
	c->x86_clflush_size = 64;
849 850
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
851 852
#else
	c->cpuid_level = -1;	/* CPUID not detected */
853
	c->x86_clflush_size = 32;
854 855
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
856 857
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
858 859 860 861
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

862
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
863 864
		this_cpu->c_identify(c);

865 866 867 868 869 870
	/* Clear/Set all flags overriden by options, after probe */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

871
#ifdef CONFIG_X86_64
872
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
873 874
#endif

L
Linus Torvalds 已提交
875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

891 892 893 894
	/* Set up SMEP/SMAP */
	setup_smep(c);
	setup_smap(c);

L
Linus Torvalds 已提交
895
	/*
I
Ingo Molnar 已提交
896 897
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
898 899
	 */

900 901 902
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
903
	/* If the model name is still unset, do table lookup. */
904
	if (!c->x86_model_id[0]) {
905
		const char *p;
L
Linus Torvalds 已提交
906
		p = table_lookup_model(c);
907
		if (p)
L
Linus Torvalds 已提交
908 909 910 911
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
912
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
913 914
	}

915 916 917 918
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

919
	init_hypervisor(c);
920
	x86_init_rdrand(c);
921 922 923 924 925 926 927 928 929 930

	/*
	 * Clear/Set all flags overriden by options, need do it
	 * before following smp all cpus cap AND.
	 */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

L
Linus Torvalds 已提交
931 932 933 934 935 936
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
937
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
938
		/* AND the already accumulated flags with these */
939
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
940
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
941 942 943 944

		/* OR, i.e. replicate the bug flags */
		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
L
Linus Torvalds 已提交
945 946 947
	}

	/* Init Machine Check Exception if available. */
948
	mcheck_cpu_init(c);
949 950

	select_idle_routine(c);
951

952
#ifdef CONFIG_NUMA
953 954
	numa_add_cpu(smp_processor_id());
#endif
955
}
S
Shaohua Li 已提交
956

957 958 959 960 961 962 963 964 965 966
#ifdef CONFIG_X86_64
static void vgetcpu_set_mode(void)
{
	if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
		vgetcpu_mode = VGETCPU_RDTSCP;
	else
		vgetcpu_mode = VGETCPU_LSL;
}
#endif

967 968 969
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
970
	init_amd_e400_c1e_mask();
971
#ifdef CONFIG_X86_32
972
	sysenter_setup();
L
Li Shaohua 已提交
973
	enable_sep_cpu();
974 975
#else
	vgetcpu_set_mode();
976
#endif
977
	cpu_detect_tlb(&boot_cpu_data);
978
}
S
Shaohua Li 已提交
979

980
void identify_secondary_cpu(struct cpuinfo_x86 *c)
981 982 983
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
984
#ifdef CONFIG_X86_32
985
	enable_sep_cpu();
986
#endif
987
	mtrr_ap_init();
L
Linus Torvalds 已提交
988 989
}

990
struct msr_range {
I
Ingo Molnar 已提交
991 992
	unsigned	min;
	unsigned	max;
993
};
L
Linus Torvalds 已提交
994

995
static const struct msr_range msr_range_array[] = {
996 997 998 999 1000
	{ 0x00000000, 0x00000418},
	{ 0xc0000000, 0xc000040b},
	{ 0xc0010000, 0xc0010142},
	{ 0xc0011000, 0xc001103b},
};
L
Linus Torvalds 已提交
1001

1002
static void __print_cpu_msr(void)
1003
{
I
Ingo Molnar 已提交
1004
	unsigned index_min, index_max;
1005 1006 1007 1008 1009 1010 1011
	unsigned index;
	u64 val;
	int i;

	for (i = 0; i < ARRAY_SIZE(msr_range_array); i++) {
		index_min = msr_range_array[i].min;
		index_max = msr_range_array[i].max;
I
Ingo Molnar 已提交
1012

1013
		for (index = index_min; index < index_max; index++) {
1014
			if (rdmsrl_safe(index, &val))
1015 1016
				continue;
			printk(KERN_INFO " MSR%08x: %016llx\n", index, val);
L
Linus Torvalds 已提交
1017
		}
1018 1019
	}
}
1020

1021
static int show_msr;
I
Ingo Molnar 已提交
1022

1023 1024 1025
static __init int setup_show_msr(char *arg)
{
	int num;
1026

1027
	get_option(&arg, &num);
1028

1029 1030 1031
	if (num > 0)
		show_msr = num;
	return 1;
L
Linus Torvalds 已提交
1032
}
1033
__setup("show_msr=", setup_show_msr);
L
Linus Torvalds 已提交
1034

A
Andi Kleen 已提交
1035 1036
static __init int setup_noclflush(char *arg)
{
1037
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1038
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
A
Andi Kleen 已提交
1039 1040 1041 1042
	return 1;
}
__setup("noclflush", setup_noclflush);

1043
void print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1044
{
1045
	const char *vendor = NULL;
L
Linus Torvalds 已提交
1046

I
Ingo Molnar 已提交
1047
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
1048
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
1049 1050 1051 1052
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
1053

1054
	if (vendor && !strstr(c->x86_model_id, vendor))
1055
		printk(KERN_CONT "%s ", vendor);
L
Linus Torvalds 已提交
1056

1057
	if (c->x86_model_id[0])
1058
		printk(KERN_CONT "%s", strim(c->x86_model_id));
L
Linus Torvalds 已提交
1059
	else
1060
		printk(KERN_CONT "%d86", c->x86);
L
Linus Torvalds 已提交
1061

1062 1063
	printk(KERN_CONT " (fam: %02x, model: %02x", c->x86, c->x86_model);

1064
	if (c->x86_mask || c->cpuid_level >= 0)
1065
		printk(KERN_CONT ", stepping: %02x)\n", c->x86_mask);
L
Linus Torvalds 已提交
1066
	else
1067
		printk(KERN_CONT ")\n");
1068

1069
	print_cpu_msr(c);
1070 1071
}

1072
void print_cpu_msr(struct cpuinfo_x86 *c)
1073
{
1074
	if (c->cpu_index < show_msr)
1075
		__print_cpu_msr();
L
Linus Torvalds 已提交
1076 1077
}

1078 1079 1080
static __init int setup_disablecpuid(char *arg)
{
	int bit;
I
Ingo Molnar 已提交
1081

1082 1083 1084 1085
	if (get_option(&arg, &bit) && bit < NCAPINTS*32)
		setup_clear_cpu_cap(bit);
	else
		return 0;
I
Ingo Molnar 已提交
1086

1087 1088 1089 1090
	return 1;
}
__setup("clearcpuid=", setup_disablecpuid);

1091 1092 1093 1094
DEFINE_PER_CPU(unsigned long, kernel_stack) =
	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(kernel_stack);

1095
#ifdef CONFIG_X86_64
1096
struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
1097 1098
struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1,
				    (unsigned long) debug_idt_table };
1099

1100
DEFINE_PER_CPU_FIRST(union irq_stack_union,
1101
		     irq_stack_union) __aligned(PAGE_SIZE) __visible;
I
Ingo Molnar 已提交
1102

1103 1104 1105 1106 1107 1108 1109
/*
 * The following four percpu variables are hot.  Align current_task to
 * cacheline size such that all four fall in the same cacheline.
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1110

1111 1112 1113
DEFINE_PER_CPU(char *, irq_stack_ptr) =
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;

1114
DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1115

1116 1117 1118
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);

1119 1120
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);

I
Ingo Molnar 已提交
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

1132
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
1133
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
1134 1135 1136

/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1137
{
1138 1139 1140 1141 1142 1143 1144 1145
	/*
	 * LSTAR and STAR live in a bit strange symbiosis.
	 * They both write to the same internal register. STAR allows to
	 * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip.
	 */
	wrmsrl(MSR_STAR,  ((u64)__USER32_CS)<<48  | ((u64)__KERNEL_CS)<<32);
	wrmsrl(MSR_LSTAR, system_call);
	wrmsrl(MSR_CSTAR, ignore_sysret);
1146

1147 1148 1149
#ifdef CONFIG_IA32_EMULATION
	syscall32_cpu_init();
#endif
1150

1151 1152
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
1153 1154
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
	       X86_EFLAGS_IOPL|X86_EFLAGS_AC);
L
Linus Torvalds 已提交
1155
}
1156

1157 1158 1159 1160 1161 1162
/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

1163
static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
1164
DEFINE_PER_CPU(int, debug_stack_usage);
1165 1166 1167

int is_debug_stack(unsigned long addr)
{
1168 1169 1170
	return __get_cpu_var(debug_stack_usage) ||
		(addr <= __get_cpu_var(debug_stack_addr) &&
		 addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ));
1171 1172
}

1173
DEFINE_PER_CPU(u32, debug_idt_ctr);
1174

1175 1176
void debug_stack_set_zero(void)
{
1177 1178
	this_cpu_inc(debug_idt_ctr);
	load_current_idt();
1179 1180 1181 1182
}

void debug_stack_reset(void)
{
1183
	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
1184
		return;
1185 1186
	if (this_cpu_dec_return(debug_idt_ctr) == 0)
		load_current_idt();
1187 1188
}

I
Ingo Molnar 已提交
1189
#else	/* CONFIG_X86_64 */
1190

1191 1192
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1193 1194
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
1195
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
1196

1197
#ifdef CONFIG_CC_STACKPROTECTOR
1198
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1199
#endif
1200

I
Ingo Molnar 已提交
1201
#endif	/* CONFIG_X86_64 */
1202

1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1218

1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232
#ifdef CONFIG_KGDB
/*
 * Restore debug regs if using kgdbwait and you have a kernel debugger
 * connection established.
 */
static void dbg_restore_debug_regs(void)
{
	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
		arch_kgdb_ops.correct_hw_break();
}
#else /* ! CONFIG_KGDB */
#define dbg_restore_debug_regs()
#endif /* ! CONFIG_KGDB */

1233 1234 1235 1236 1237
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1238
 * A lot of state is already set up in PDA init for 64 bit
1239
 */
1240
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1241

1242
void cpu_init(void)
1243
{
1244
	struct orig_ist *oist;
1245
	struct task_struct *me;
I
Ingo Molnar 已提交
1246 1247 1248
	struct tss_struct *t;
	unsigned long v;
	int cpu;
1249 1250
	int i;

1251 1252 1253 1254 1255 1256
	/*
	 * Load microcode on this cpu if a valid microcode is available.
	 * This is early microcode loading procedure.
	 */
	load_ucode_ap();

I
Ingo Molnar 已提交
1257 1258
	cpu = stack_smp_processor_id();
	t = &per_cpu(init_tss, cpu);
1259
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1260

1261
#ifdef CONFIG_NUMA
1262
	if (this_cpu_read(numa_node) == 0 &&
1263 1264
	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
		set_numa_node(early_cpu_to_node(cpu));
1265
#endif
1266 1267 1268

	me = current;

1269
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask))
1270 1271
		panic("CPU#%d already initialized!\n", cpu);

1272
	pr_debug("Initializing CPU#%d\n", cpu);
1273 1274 1275 1276 1277 1278 1279 1280

	clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1281
	switch_to_new_gdt(cpu);
1282 1283
	loadsegment(fs, 0);

1284
	load_current_idt();
1285 1286 1287 1288 1289 1290 1291 1292

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1293
	x86_configure_nx();
1294
	enable_x2apic();
1295 1296 1297 1298

	/*
	 * set up and load the per-CPU TSS
	 */
1299
	if (!oist->ist[0]) {
1300
		char *estacks = per_cpu(exception_stacks, cpu);
I
Ingo Molnar 已提交
1301

1302
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1303
			estacks += exception_stack_sizes[v];
1304
			oist->ist[v] = t->x86_tss.ist[v] =
1305
					(unsigned long)estacks;
1306 1307
			if (v == DEBUG_STACK-1)
				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1308 1309 1310 1311
		}
	}

	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
I
Ingo Molnar 已提交
1312

1313 1314 1315 1316 1317 1318 1319 1320 1321
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

	atomic_inc(&init_mm.mm_count);
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1322
	BUG_ON(me->mm);
1323 1324 1325 1326 1327 1328 1329
	enter_lazy_tlb(&init_mm, me);

	load_sp0(t, &current->thread);
	set_tss_desc(cpu, t);
	load_TR_desc();
	load_LDT(&init_mm.context);

1330 1331
	clear_all_debug_regs();
	dbg_restore_debug_regs();
1332 1333 1334 1335 1336 1337 1338 1339 1340

	fpu_init();

	if (is_uv_system())
		uv_cpu_init();
}

#else

1341
void cpu_init(void)
1342
{
1343 1344
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1345
	struct tss_struct *t = &per_cpu(init_tss, cpu);
1346
	struct thread_struct *thread = &curr->thread;
1347

1348 1349
	show_ucode_info_early();

1350
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) {
1351
		printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
1352 1353
		for (;;)
			local_irq_enable();
1354 1355 1356 1357 1358 1359 1360
	}

	printk(KERN_INFO "Initializing CPU#%d\n", cpu);

	if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
		clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

1361
	load_current_idt();
1362
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1363 1364 1365 1366 1367

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
	atomic_inc(&init_mm.mm_count);
1368
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1369
	BUG_ON(curr->mm);
1370
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1371

1372
	load_sp0(t, thread);
1373
	set_tss_desc(cpu, t);
L
Linus Torvalds 已提交
1374 1375 1376
	load_TR_desc();
	load_LDT(&init_mm.context);

1377 1378
	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);

1379
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1380 1381
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1382
#endif
L
Linus Torvalds 已提交
1383

1384
	clear_all_debug_regs();
1385
	dbg_restore_debug_regs();
L
Linus Torvalds 已提交
1386

1387
	fpu_init();
L
Linus Torvalds 已提交
1388
}
1389
#endif
1390 1391 1392 1393 1394 1395 1396 1397

#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
void warn_pre_alternatives(void)
{
	WARN(1, "You're using static_cpu_has before alternatives have run!\n");
}
EXPORT_SYMBOL_GPL(warn_pre_alternatives);
#endif
1398 1399 1400 1401 1402 1403

inline bool __static_cpu_has_safe(u16 bit)
{
	return boot_cpu_has(bit);
}
EXPORT_SYMBOL_GPL(__static_cpu_has_safe);