common.c 35.8 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/module.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
L
Linus Torvalds 已提交
8
#include <linux/delay.h>
9 10
#include <linux/sched.h>
#include <linux/init.h>
11
#include <linux/kprobes.h>
12
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
13
#include <linux/smp.h>
14 15 16
#include <linux/io.h>

#include <asm/stackprotector.h>
17
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
18
#include <asm/mmu_context.h>
19
#include <asm/archrandom.h>
20 21
#include <asm/hypervisor.h>
#include <asm/processor.h>
22
#include <asm/tlbflush.h>
23
#include <asm/debugreg.h>
24
#include <asm/sections.h>
25
#include <asm/vsyscall.h>
A
Alan Cox 已提交
26 27
#include <linux/topology.h>
#include <linux/cpumask.h>
28
#include <asm/pgtable.h>
A
Arun Sharma 已提交
29
#include <linux/atomic.h>
30 31 32 33 34
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/i387.h>
35
#include <asm/fpu-internal.h>
36
#include <asm/mtrr.h>
A
Alan Cox 已提交
37
#include <linux/numa.h>
38 39
#include <asm/asm.h>
#include <asm/cpu.h>
40
#include <asm/mce.h>
41
#include <asm/msr.h>
42
#include <asm/pat.h>
43 44
#include <asm/microcode.h>
#include <asm/microcode_intel.h>
45 46

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
47
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
48 49 50 51
#endif

#include "cpu.h"

52 53
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
54 55
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
56 57 58 59

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
60
/* correctly size the local cpu masks */
61
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
62 63 64 65 66 67 68
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

69
static void default_init(struct cpuinfo_x86 *c)
70 71
{
#ifdef CONFIG_X86_64
72
	cpu_detect_cache_sizes(c);
73 74 75 76 77 78 79 80 81 82 83 84 85
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

86
static const struct cpu_dev default_cpu = {
87 88 89 90 91
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

92
static const struct cpu_dev *this_cpu = &default_cpu;
93

B
Brian Gerst 已提交
94
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
95
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
96 97 98 99 100
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
101
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
102 103
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
104 105 106 107 108 109
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
110
#else
A
Akinobu Mita 已提交
111 112 113 114
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
115 116 117 118 119
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
120
	/* 32-bit code */
A
Akinobu Mita 已提交
121
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
122
	/* 16-bit code */
A
Akinobu Mita 已提交
123
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
124
	/* 16-bit data */
A
Akinobu Mita 已提交
125
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
126
	/* 16-bit data */
A
Akinobu Mita 已提交
127
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
128
	/* 16-bit data */
A
Akinobu Mita 已提交
129
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
130 131 132 133
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
134
	/* 32-bit code */
A
Akinobu Mita 已提交
135
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
136
	/* 16-bit code */
A
Akinobu Mita 已提交
137
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
138
	/* data */
139
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
140

A
Akinobu Mita 已提交
141 142
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
143
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
144
#endif
B
Brian Gerst 已提交
145
} };
146
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
147

148 149
static int __init x86_xsave_setup(char *s)
{
150 151
	if (strlen(s))
		return 0;
152
	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
153
	setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
154
	setup_clear_cpu_cap(X86_FEATURE_XSAVES);
155 156
	setup_clear_cpu_cap(X86_FEATURE_AVX);
	setup_clear_cpu_cap(X86_FEATURE_AVX2);
157 158 159 160
	return 1;
}
__setup("noxsave", x86_xsave_setup);

161 162 163 164 165 166 167
static int __init x86_xsaveopt_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
	return 1;
}
__setup("noxsaveopt", x86_xsaveopt_setup);

168 169 170 171 172 173 174
static int __init x86_xsaves_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_XSAVES);
	return 1;
}
__setup("noxsaves", x86_xsaves_setup);

175
#ifdef CONFIG_X86_32
176 177
static int cachesize_override = -1;
static int disable_x86_serial_nr = 1;
L
Linus Torvalds 已提交
178

179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_fxsr_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_FXSR);
	setup_clear_cpu_cap(X86_FEATURE_XMM);
	return 1;
}
__setup("nofxsr", x86_fxsr_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

206 207 208 209 210 211 212
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
213 214 215 216 217 218 219 220 221 222 223
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

224 225
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
226 227 228 229 230

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
231
int have_cpuid_p(void)
232 233 234 235
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

236
static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
237
{
I
Ingo Molnar 已提交
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

	printk(KERN_NOTICE "CPU serial number disabled.\n");
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
254 255 256 257 258 259 260 261
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
262
#else
263 264 265 266 267 268 269
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
270
#endif
271

272 273
static __init int setup_disable_smep(char *arg)
{
274
	setup_clear_cpu_cap(X86_FEATURE_SMEP);
275 276 277 278
	return 1;
}
__setup("nosmep", setup_disable_smep);

279
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
280
{
281
	if (cpu_has(c, X86_FEATURE_SMEP))
A
Andy Lutomirski 已提交
282
		cr4_set_bits(X86_CR4_SMEP);
283 284
}

285 286
static __init int setup_disable_smap(char *arg)
{
287
	setup_clear_cpu_cap(X86_FEATURE_SMAP);
288 289 290 291
	return 1;
}
__setup("nosmap", setup_disable_smap);

292 293 294 295 296 297 298 299
static __always_inline void setup_smap(struct cpuinfo_x86 *c)
{
	unsigned long eflags;

	/* This should have been cleared long ago */
	raw_local_save_flags(eflags);
	BUG_ON(eflags & X86_EFLAGS_AC);

300 301
	if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP
A
Andy Lutomirski 已提交
302
		cr4_set_bits(X86_CR4_SMAP);
303
#else
A
Andy Lutomirski 已提交
304
		cr4_clear_bits(X86_CR4_SMAP);
305 306
#endif
	}
307 308
}

309 310 311 312 313 314 315 316 317
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
318

319
static const struct cpuid_dependent_feature
320 321 322 323 324 325 326
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

327
static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
328 329
{
	const struct cpuid_dependent_feature *df;
330

331
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
332 333 334

		if (!cpu_has(c, df->feature))
			continue;
335 336 337 338 339 340 341
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
342
		if (!((s32)df->level < 0 ?
343
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
344 345 346 347 348 349 350 351
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

		printk(KERN_WARNING
352 353
		       "CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
				x86_cap_flag(df->feature), df->level);
354
	}
355
}
356

357 358 359
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
360 361
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
362 363 364
 */

/* Look up CPU names by table lookup. */
365
static const char *table_lookup_model(struct cpuinfo_x86 *c)
366
{
367 368
#ifdef CONFIG_X86_32
	const struct legacy_cpu_model_info *info;
369 370 371 372 373 374 375

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

376
	info = this_cpu->legacy_models;
377

378
	while (info->family) {
379 380 381 382
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
383
#endif
384 385 386
	return NULL;		/* Not found */
}

387 388
__u32 cpu_caps_cleared[NCAPINTS];
__u32 cpu_caps_set[NCAPINTS];
389

390 391 392 393 394 395 396 397
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
	loadsegment(gs, 0);
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
398
	load_stack_canary_segment();
399 400
}

I
Ingo Molnar 已提交
401 402 403 404
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
405
void switch_to_new_gdt(int cpu)
406 407 408
{
	struct desc_ptr gdt_descr;

409
	gdt_descr.address = (long)get_cpu_gdt_table(cpu);
410 411
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
412
	/* Reload the per-cpu base */
413 414

	load_percpu_segment(cpu);
415 416
}

417
static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
418

419
static void get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
420 421 422 423
{
	unsigned int *v;
	char *p, *q;

424
	if (c->extended_cpuid_level < 0x80000004)
425
		return;
L
Linus Torvalds 已提交
426

I
Ingo Molnar 已提交
427
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
428 429 430 431 432
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

I
Ingo Molnar 已提交
433 434 435 436
	/*
	 * Intel chips right-justify this string for some dumb reason;
	 * undo that brain damage:
	 */
L
Linus Torvalds 已提交
437
	p = q = &c->x86_model_id[0];
438
	while (*p == ' ')
439
		p++;
440
	if (p != q) {
441 442 443 444
		while (*p)
			*q++ = *p++;
		while (q <= &c->x86_model_id[48])
			*q++ = '\0';	/* Zero-pad the rest */
L
Linus Torvalds 已提交
445 446 447
	}
}

448
void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
449
{
450
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
451

452
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
453 454

	if (n >= 0x80000005) {
455 456
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
457 458 459 460
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
461 462 463 464 465
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

466
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
467
	l2size = ecx >> 16;
468

469 470 471
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
472
	/* do processor-specific cache resizing */
473 474
	if (this_cpu->legacy_cache_size)
		l2size = this_cpu->legacy_cache_size(c, l2size);
L
Linus Torvalds 已提交
475 476 477 478 479

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

480
	if (l2size == 0)
L
Linus Torvalds 已提交
481
		return;		/* Again, no L2 cache is possible */
482
#endif
L
Linus Torvalds 已提交
483 484 485 486

	c->x86_cache_size = l2size;
}

487 488 489 490 491 492
u16 __read_mostly tlb_lli_4k[NR_INFO];
u16 __read_mostly tlb_lli_2m[NR_INFO];
u16 __read_mostly tlb_lli_4m[NR_INFO];
u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
493
u16 __read_mostly tlb_lld_1g[NR_INFO];
494

495
static void cpu_detect_tlb(struct cpuinfo_x86 *c)
496 497 498 499
{
	if (this_cpu->c_detect_tlb)
		this_cpu->c_detect_tlb(c);

500
	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
501
		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
502 503 504 505 506
		tlb_lli_4m[ENTRIES]);

	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
507 508
}

509
void detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
510
{
511
#ifdef CONFIG_X86_HT
512 513
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
514
	static bool printed;
L
Linus Torvalds 已提交
515

516
	if (!cpu_has(c, X86_FEATURE_HT))
517
		return;
L
Linus Torvalds 已提交
518

519 520
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
521

522 523
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
524

525
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
526

527 528 529
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
530
		printk_once(KERN_INFO "CPU0: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
531 532
		goto out;
	}
533

I
Ingo Molnar 已提交
534 535
	if (smp_num_siblings <= 1)
		goto out;
536

I
Ingo Molnar 已提交
537 538
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
539

I
Ingo Molnar 已提交
540
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
541

I
Ingo Molnar 已提交
542
	index_msb = get_count_order(smp_num_siblings);
543

I
Ingo Molnar 已提交
544
	core_bits = get_count_order(c->x86_max_cores);
545

I
Ingo Molnar 已提交
546 547
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
548

549
out:
550
	if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
551 552 553 554
		printk(KERN_INFO  "CPU: Physical Processor ID: %d\n",
		       c->phys_proc_id);
		printk(KERN_INFO  "CPU: Processor Core ID: %d\n",
		       c->cpu_core_id);
555
		printed = 1;
556 557
	}
#endif
558
}
L
Linus Torvalds 已提交
559

560
static void get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
561 562
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
563
	int i;
L
Linus Torvalds 已提交
564 565

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
566 567 568 569 570 571
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
572

Y
Yinghai Lu 已提交
573 574 575
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
576 577
		}
	}
Y
Yinghai Lu 已提交
578

579 580 581
	printk_once(KERN_ERR
			"CPU: vendor_id '%s' unknown, using generic init.\n" \
			"CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
582

583 584
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
585 586
}

587
void cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
588 589
{
	/* Get vendor name */
590 591 592 593
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
594 595

	c->x86 = 4;
596
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
597 598
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
599

L
Linus Torvalds 已提交
600
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
601 602 603
		c->x86 = (tfms >> 8) & 0xf;
		c->x86_model = (tfms >> 4) & 0xf;
		c->x86_mask = tfms & 0xf;
I
Ingo Molnar 已提交
604

605
		if (c->x86 == 0xf)
L
Linus Torvalds 已提交
606
			c->x86 += (tfms >> 20) & 0xff;
607
		if (c->x86 >= 0x6)
608
			c->x86_model += ((tfms >> 16) & 0xf) << 4;
I
Ingo Molnar 已提交
609

H
Huang, Ying 已提交
610 611
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
612
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
613
		}
L
Linus Torvalds 已提交
614 615
	}
}
616

617
void get_cpu_cap(struct cpuinfo_x86 *c)
618 619
{
	u32 tfms, xlvl;
620
	u32 ebx;
621

622 623 624
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
		u32 capability, excap;
I
Ingo Molnar 已提交
625

626 627 628 629
		cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
		c->x86_capability[0] = capability;
		c->x86_capability[4] = excap;
	}
630

631 632 633 634 635 636
	/* Additional Intel-defined flags: level 0x00000007 */
	if (c->cpuid_level >= 0x00000007) {
		u32 eax, ebx, ecx, edx;

		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);

637
		c->x86_capability[9] = ebx;
638 639
	}

640 641 642 643 644 645 646 647 648
	/* Extended state features: level 0x0000000d */
	if (c->cpuid_level >= 0x0000000d) {
		u32 eax, ebx, ecx, edx;

		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);

		c->x86_capability[10] = eax;
	}

649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
	/* Additional Intel-defined flags: level 0x0000000F */
	if (c->cpuid_level >= 0x0000000F) {
		u32 eax, ebx, ecx, edx;

		/* QoS sub-leaf, EAX=0Fh, ECX=0 */
		cpuid_count(0x0000000F, 0, &eax, &ebx, &ecx, &edx);
		c->x86_capability[11] = edx;
		if (cpu_has(c, X86_FEATURE_CQM_LLC)) {
			/* will be overridden if occupancy monitoring exists */
			c->x86_cache_max_rmid = ebx;

			/* QoS sub-leaf, EAX=0Fh, ECX=1 */
			cpuid_count(0x0000000F, 1, &eax, &ebx, &ecx, &edx);
			c->x86_capability[12] = edx;
			if (cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC)) {
				c->x86_cache_max_rmid = ecx;
				c->x86_cache_occ_scale = ebx;
			}
		} else {
			c->x86_cache_max_rmid = -1;
			c->x86_cache_occ_scale = -1;
		}
	}

673 674 675
	/* AMD-defined flags: level 0x80000001 */
	xlvl = cpuid_eax(0x80000000);
	c->extended_cpuid_level = xlvl;
I
Ingo Molnar 已提交
676

677 678 679 680
	if ((xlvl & 0xffff0000) == 0x80000000) {
		if (xlvl >= 0x80000001) {
			c->x86_capability[1] = cpuid_edx(0x80000001);
			c->x86_capability[6] = cpuid_ecx(0x80000001);
681 682 683
		}
	}

684 685 686 687 688
	if (c->extended_cpuid_level >= 0x80000008) {
		u32 eax = cpuid_eax(0x80000008);

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
689
	}
690 691 692
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
693
#endif
694 695 696

	if (c->extended_cpuid_level >= 0x80000007)
		c->x86_power = cpuid_edx(0x80000007);
697

698
	init_scattered_cpuid_features(c);
699
}
L
Linus Torvalds 已提交
700

701
static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
Y
Yinghai Lu 已提交
702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

727 728 729 730 731 732 733 734 735
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
736
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
737
{
738 739
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
740 741
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
742
#else
H
Huang, Ying 已提交
743
	c->x86_clflush_size = 32;
744 745
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
746
#endif
747
	c->x86_cache_alignment = c->x86_clflush_size;
748

749
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
750
	c->extended_cpuid_level = 0;
751

Y
Yinghai Lu 已提交
752 753 754 755
	if (!have_cpuid_p())
		identify_cpu_without_cpuid(c);

	/* cyrix could have cpuid enabled via c_identify()*/
756 757 758 759
	if (!have_cpuid_p())
		return;

	cpu_detect(c);
760 761
	get_cpu_vendor(c);
	get_cpu_cap(c);
762
	fpu_detect(c);
763

Y
Yinghai Lu 已提交
764 765
	if (this_cpu->c_early_init)
		this_cpu->c_early_init(c);
766

767
	c->cpu_index = 0;
768
	filter_cpuid_features(c, false);
769

B
Borislav Petkov 已提交
770 771
	if (this_cpu->c_bsp_init)
		this_cpu->c_bsp_init(c);
772 773

	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
774 775
}

776 777
void __init early_cpu_init(void)
{
778
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
779 780
	int count = 0;

781
#ifdef CONFIG_PROCESSOR_SELECT
782
	printk(KERN_INFO "KERNEL supported cpus:\n");
783 784
#endif

Y
Yinghai Lu 已提交
785
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
786
		const struct cpu_dev *cpudev = *cdev;
787

Y
Yinghai Lu 已提交
788 789 790 791 792
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

793
#ifdef CONFIG_PROCESSOR_SELECT
794 795 796 797 798 799 800 801 802
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
				printk(KERN_INFO "  %s %s\n", cpudev->c_vendor,
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
803
		}
804
#endif
Y
Yinghai Lu 已提交
805
	}
806
	early_identify_cpu(&boot_cpu_data);
807
}
808

809
/*
B
Borislav Petkov 已提交
810 811 812 813 814
 * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
 * unfortunately, that's not true in practice because of early VIA
 * chips and (more importantly) broken virtualizers that are not easy
 * to detect. In the latter case it doesn't even *fail* reliably, so
 * probing for it doesn't even work. Disable it completely on 32-bit
815
 * unless we can find a reliable way to detect all the broken cases.
B
Borislav Petkov 已提交
816
 * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
817
 */
818
static void detect_nopl(struct cpuinfo_x86 *c)
819
{
B
Borislav Petkov 已提交
820
#ifdef CONFIG_X86_32
821
	clear_cpu_cap(c, X86_FEATURE_NOPL);
B
Borislav Petkov 已提交
822 823 824
#else
	set_cpu_cap(c, X86_FEATURE_NOPL);
#endif
825 826
}

827
static void generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
828
{
Y
Yinghai Lu 已提交
829
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
830

831
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
832
		identify_cpu_without_cpuid(c);
833

Y
Yinghai Lu 已提交
834
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
835
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
836
		return;
L
Linus Torvalds 已提交
837

838
	cpu_detect(c);
L
Linus Torvalds 已提交
839

840
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
841

842
	get_cpu_cap(c);
L
Linus Torvalds 已提交
843

844 845
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
846 847
#ifdef CONFIG_X86_32
# ifdef CONFIG_X86_HT
848
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
849
# else
850
		c->apicid = c->initial_apicid;
851 852 853
# endif
#endif
		c->phys_proc_id = c->initial_apicid;
854
	}
L
Linus Torvalds 已提交
855

856
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
857

858
	detect_nopl(c);
L
Linus Torvalds 已提交
859 860
}

861 862 863 864 865 866 867 868 869 870 871 872 873 874
static void x86_init_cache_qos(struct cpuinfo_x86 *c)
{
	/*
	 * The heavy lifting of max_rmid and cache_occ_scale are handled
	 * in get_cpu_cap().  Here we just set the max_rmid for the boot_cpu
	 * in case CQM bits really aren't there in this CPU.
	 */
	if (c != &boot_cpu_data) {
		boot_cpu_data.x86_cache_max_rmid =
			min(boot_cpu_data.x86_cache_max_rmid,
			    c->x86_cache_max_rmid);
	}
}

L
Linus Torvalds 已提交
875 876 877
/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
878
static void identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
879 880 881 882 883 884 885 886 887
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
888
	c->x86_max_cores = 1;
889
	c->x86_coreid_bits = 0;
890
#ifdef CONFIG_X86_64
891
	c->x86_clflush_size = 64;
892 893
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
894 895
#else
	c->cpuid_level = -1;	/* CPUID not detected */
896
	c->x86_clflush_size = 32;
897 898
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
899 900
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
901 902 903 904
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

905
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
906 907
		this_cpu->c_identify(c);

908 909 910 911 912 913
	/* Clear/Set all flags overriden by options, after probe */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

914
#ifdef CONFIG_X86_64
915
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
916 917
#endif

L
Linus Torvalds 已提交
918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

934 935 936 937
	/* Set up SMEP/SMAP */
	setup_smep(c);
	setup_smap(c);

L
Linus Torvalds 已提交
938
	/*
I
Ingo Molnar 已提交
939 940
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
941 942
	 */

943 944 945
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
946
	/* If the model name is still unset, do table lookup. */
947
	if (!c->x86_model_id[0]) {
948
		const char *p;
L
Linus Torvalds 已提交
949
		p = table_lookup_model(c);
950
		if (p)
L
Linus Torvalds 已提交
951 952 953 954
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
955
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
956 957
	}

958 959 960 961
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

962
	init_hypervisor(c);
963
	x86_init_rdrand(c);
964
	x86_init_cache_qos(c);
965 966 967 968 969 970 971 972 973 974

	/*
	 * Clear/Set all flags overriden by options, need do it
	 * before following smp all cpus cap AND.
	 */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

L
Linus Torvalds 已提交
975 976 977 978 979 980
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
981
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
982
		/* AND the already accumulated flags with these */
983
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
984
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
985 986 987 988

		/* OR, i.e. replicate the bug flags */
		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
L
Linus Torvalds 已提交
989 990 991
	}

	/* Init Machine Check Exception if available. */
992
	mcheck_cpu_init(c);
993 994

	select_idle_routine(c);
995

996
#ifdef CONFIG_NUMA
997 998
	numa_add_cpu(smp_processor_id());
#endif
999
}
S
Shaohua Li 已提交
1000

1001 1002 1003 1004
/*
 * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
 * on 32-bit kernels:
 */
1005 1006 1007
#ifdef CONFIG_X86_32
void enable_sep_cpu(void)
{
1008 1009
	struct tss_struct *tss;
	int cpu;
1010

1011 1012 1013 1014 1015 1016 1017
	cpu = get_cpu();
	tss = &per_cpu(cpu_tss, cpu);

	if (!boot_cpu_has(X86_FEATURE_SEP))
		goto out;

	/*
1018 1019
	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
	 * see the big comment in struct x86_hw_tss's definition.
1020
	 */
1021 1022

	tss->x86_tss.ss1 = __KERNEL_CS;
1023 1024
	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);

1025 1026 1027
	wrmsr(MSR_IA32_SYSENTER_ESP,
	      (unsigned long)tss + offsetofend(struct tss_struct, SYSENTER_stack),
	      0);
1028 1029 1030 1031

	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)ia32_sysenter_target, 0);

out:
1032 1033
	put_cpu();
}
1034 1035
#endif

1036 1037 1038
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
1039
	init_amd_e400_c1e_mask();
1040
#ifdef CONFIG_X86_32
1041
	sysenter_setup();
L
Li Shaohua 已提交
1042
	enable_sep_cpu();
1043
#endif
1044
	cpu_detect_tlb(&boot_cpu_data);
1045
}
S
Shaohua Li 已提交
1046

1047
void identify_secondary_cpu(struct cpuinfo_x86 *c)
1048 1049 1050
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
1051
#ifdef CONFIG_X86_32
1052
	enable_sep_cpu();
1053
#endif
1054
	mtrr_ap_init();
L
Linus Torvalds 已提交
1055 1056
}

1057
struct msr_range {
I
Ingo Molnar 已提交
1058 1059
	unsigned	min;
	unsigned	max;
1060
};
L
Linus Torvalds 已提交
1061

1062
static const struct msr_range msr_range_array[] = {
1063 1064 1065 1066 1067
	{ 0x00000000, 0x00000418},
	{ 0xc0000000, 0xc000040b},
	{ 0xc0010000, 0xc0010142},
	{ 0xc0011000, 0xc001103b},
};
L
Linus Torvalds 已提交
1068

1069
static void __print_cpu_msr(void)
1070
{
I
Ingo Molnar 已提交
1071
	unsigned index_min, index_max;
1072 1073 1074 1075 1076 1077 1078
	unsigned index;
	u64 val;
	int i;

	for (i = 0; i < ARRAY_SIZE(msr_range_array); i++) {
		index_min = msr_range_array[i].min;
		index_max = msr_range_array[i].max;
I
Ingo Molnar 已提交
1079

1080
		for (index = index_min; index < index_max; index++) {
1081
			if (rdmsrl_safe(index, &val))
1082 1083
				continue;
			printk(KERN_INFO " MSR%08x: %016llx\n", index, val);
L
Linus Torvalds 已提交
1084
		}
1085 1086
	}
}
1087

1088
static int show_msr;
I
Ingo Molnar 已提交
1089

1090 1091 1092
static __init int setup_show_msr(char *arg)
{
	int num;
1093

1094
	get_option(&arg, &num);
1095

1096 1097 1098
	if (num > 0)
		show_msr = num;
	return 1;
L
Linus Torvalds 已提交
1099
}
1100
__setup("show_msr=", setup_show_msr);
L
Linus Torvalds 已提交
1101

A
Andi Kleen 已提交
1102 1103
static __init int setup_noclflush(char *arg)
{
1104
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1105
	setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
A
Andi Kleen 已提交
1106 1107 1108 1109
	return 1;
}
__setup("noclflush", setup_noclflush);

1110
void print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1111
{
1112
	const char *vendor = NULL;
L
Linus Torvalds 已提交
1113

I
Ingo Molnar 已提交
1114
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
1115
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
1116 1117 1118 1119
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
1120

1121
	if (vendor && !strstr(c->x86_model_id, vendor))
1122
		printk(KERN_CONT "%s ", vendor);
L
Linus Torvalds 已提交
1123

1124
	if (c->x86_model_id[0])
1125
		printk(KERN_CONT "%s", strim(c->x86_model_id));
L
Linus Torvalds 已提交
1126
	else
1127
		printk(KERN_CONT "%d86", c->x86);
L
Linus Torvalds 已提交
1128

1129 1130
	printk(KERN_CONT " (fam: %02x, model: %02x", c->x86, c->x86_model);

1131
	if (c->x86_mask || c->cpuid_level >= 0)
1132
		printk(KERN_CONT ", stepping: %02x)\n", c->x86_mask);
L
Linus Torvalds 已提交
1133
	else
1134
		printk(KERN_CONT ")\n");
1135

1136
	print_cpu_msr(c);
1137 1138
}

1139
void print_cpu_msr(struct cpuinfo_x86 *c)
1140
{
1141
	if (c->cpu_index < show_msr)
1142
		__print_cpu_msr();
L
Linus Torvalds 已提交
1143 1144
}

1145 1146 1147
static __init int setup_disablecpuid(char *arg)
{
	int bit;
I
Ingo Molnar 已提交
1148

1149 1150 1151 1152
	if (get_option(&arg, &bit) && bit < NCAPINTS*32)
		setup_clear_cpu_cap(bit);
	else
		return 0;
I
Ingo Molnar 已提交
1153

1154 1155 1156 1157
	return 1;
}
__setup("clearcpuid=", setup_disablecpuid);

1158
#ifdef CONFIG_X86_64
1159
struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
1160 1161
struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1,
				    (unsigned long) debug_idt_table };
1162

1163
DEFINE_PER_CPU_FIRST(union irq_stack_union,
1164
		     irq_stack_union) __aligned(PAGE_SIZE) __visible;
I
Ingo Molnar 已提交
1165

1166
/*
1167 1168
 * The following percpu variables are hot.  Align current_task to
 * cacheline size such that they fall in the same cacheline.
1169 1170 1171 1172
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1173

1174 1175 1176
DEFINE_PER_CPU(char *, irq_stack_ptr) =
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;

1177
DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1178

1179 1180 1181
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);

1182 1183
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);

I
Ingo Molnar 已提交
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

1195
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
1196
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
1197 1198 1199

/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1200
{
1201 1202 1203 1204 1205 1206 1207
	/*
	 * LSTAR and STAR live in a bit strange symbiosis.
	 * They both write to the same internal register. STAR allows to
	 * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip.
	 */
	wrmsrl(MSR_STAR,  ((u64)__USER32_CS)<<48  | ((u64)__KERNEL_CS)<<32);
	wrmsrl(MSR_LSTAR, system_call);
1208 1209

#ifdef CONFIG_IA32_EMULATION
1210 1211
	wrmsrl(MSR_CSTAR, ia32_cstar_target);
	/*
1212 1213 1214 1215
	 * This only works on Intel CPUs.
	 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
	 * This does not cause SYSENTER to jump to the wrong location, because
	 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
1216 1217 1218 1219
	 */
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)ia32_sysenter_target);
1220 1221
#else
	wrmsrl(MSR_CSTAR, ignore_sysret);
1222
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1223 1224
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1225
#endif
1226

1227 1228
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
1229
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1230
	       X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT);
L
Linus Torvalds 已提交
1231
}
1232

1233 1234 1235 1236 1237 1238
/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

1239
static DEFINE_PER_CPU(unsigned long, debug_stack_addr);
1240
DEFINE_PER_CPU(int, debug_stack_usage);
1241 1242 1243

int is_debug_stack(unsigned long addr)
{
1244 1245 1246
	return __this_cpu_read(debug_stack_usage) ||
		(addr <= __this_cpu_read(debug_stack_addr) &&
		 addr > (__this_cpu_read(debug_stack_addr) - DEBUG_STKSZ));
1247
}
1248
NOKPROBE_SYMBOL(is_debug_stack);
1249

1250
DEFINE_PER_CPU(u32, debug_idt_ctr);
1251

1252 1253
void debug_stack_set_zero(void)
{
1254 1255
	this_cpu_inc(debug_idt_ctr);
	load_current_idt();
1256
}
1257
NOKPROBE_SYMBOL(debug_stack_set_zero);
1258 1259 1260

void debug_stack_reset(void)
{
1261
	if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
1262
		return;
1263 1264
	if (this_cpu_dec_return(debug_idt_ctr) == 0)
		load_current_idt();
1265
}
1266
NOKPROBE_SYMBOL(debug_stack_reset);
1267

I
Ingo Molnar 已提交
1268
#else	/* CONFIG_X86_64 */
1269

1270 1271
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
1272 1273
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
1274
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
1275

1276 1277 1278 1279 1280 1281 1282 1283 1284
/*
 * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
 * the top of the kernel stack.  Use an extra percpu variable to track the
 * top of the kernel stack directly.
 */
DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
	(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);

1285
#ifdef CONFIG_CC_STACKPROTECTOR
1286
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1287
#endif
1288

I
Ingo Molnar 已提交
1289
#endif	/* CONFIG_X86_64 */
1290

1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1306

1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
#ifdef CONFIG_KGDB
/*
 * Restore debug regs if using kgdbwait and you have a kernel debugger
 * connection established.
 */
static void dbg_restore_debug_regs(void)
{
	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
		arch_kgdb_ops.correct_hw_break();
}
#else /* ! CONFIG_KGDB */
#define dbg_restore_debug_regs()
#endif /* ! CONFIG_KGDB */

1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
static void wait_for_master_cpu(int cpu)
{
#ifdef CONFIG_SMP
	/*
	 * wait for ACK from master CPU before continuing
	 * with AP initialization
	 */
	WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
	while (!cpumask_test_cpu(cpu, cpu_callout_mask))
		cpu_relax();
#endif
}

1334 1335 1336 1337 1338
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1339
 * A lot of state is already set up in PDA init for 64 bit
1340
 */
1341
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1342

1343
void cpu_init(void)
1344
{
1345
	struct orig_ist *oist;
1346
	struct task_struct *me;
I
Ingo Molnar 已提交
1347 1348
	struct tss_struct *t;
	unsigned long v;
1349
	int cpu = stack_smp_processor_id();
1350 1351
	int i;

1352 1353
	wait_for_master_cpu(cpu);

1354 1355 1356 1357 1358 1359
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1360 1361 1362 1363 1364 1365
	/*
	 * Load microcode on this cpu if a valid microcode is available.
	 * This is early microcode loading procedure.
	 */
	load_ucode_ap();

1366
	t = &per_cpu(cpu_tss, cpu);
1367
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1368

1369
#ifdef CONFIG_NUMA
1370
	if (this_cpu_read(numa_node) == 0 &&
1371 1372
	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
		set_numa_node(early_cpu_to_node(cpu));
1373
#endif
1374 1375 1376

	me = current;

1377
	pr_debug("Initializing CPU#%d\n", cpu);
1378

A
Andy Lutomirski 已提交
1379
	cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1380 1381 1382 1383 1384 1385

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1386
	switch_to_new_gdt(cpu);
1387 1388
	loadsegment(fs, 0);

1389
	load_current_idt();
1390 1391 1392 1393 1394 1395 1396 1397

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1398
	x86_configure_nx();
1399
	x2apic_setup();
1400 1401 1402 1403

	/*
	 * set up and load the per-CPU TSS
	 */
1404
	if (!oist->ist[0]) {
1405
		char *estacks = per_cpu(exception_stacks, cpu);
I
Ingo Molnar 已提交
1406

1407
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1408
			estacks += exception_stack_sizes[v];
1409
			oist->ist[v] = t->x86_tss.ist[v] =
1410
					(unsigned long)estacks;
1411 1412
			if (v == DEBUG_STACK-1)
				per_cpu(debug_stack_addr, cpu) = (unsigned long)estacks;
1413 1414 1415 1416
		}
	}

	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
I
Ingo Molnar 已提交
1417

1418 1419 1420 1421 1422 1423 1424 1425 1426
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

	atomic_inc(&init_mm.mm_count);
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1427
	BUG_ON(me->mm);
1428 1429 1430 1431 1432 1433 1434
	enter_lazy_tlb(&init_mm, me);

	load_sp0(t, &current->thread);
	set_tss_desc(cpu, t);
	load_TR_desc();
	load_LDT(&init_mm.context);

1435 1436
	clear_all_debug_regs();
	dbg_restore_debug_regs();
1437 1438 1439 1440 1441 1442 1443 1444 1445

	fpu_init();

	if (is_uv_system())
		uv_cpu_init();
}

#else

1446
void cpu_init(void)
1447
{
1448 1449
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1450
	struct tss_struct *t = &per_cpu(cpu_tss, cpu);
1451
	struct thread_struct *thread = &curr->thread;
1452

1453
	wait_for_master_cpu(cpu);
1454

1455 1456 1457 1458 1459 1460
	/*
	 * Initialize the CR4 shadow before doing anything that could
	 * try to read it.
	 */
	cr4_init_shadow();

1461
	show_ucode_info_early();
1462 1463 1464

	printk(KERN_INFO "Initializing CPU#%d\n", cpu);

D
Dave Hansen 已提交
1465
	if (cpu_feature_enabled(X86_FEATURE_VME) || cpu_has_tsc || cpu_has_de)
A
Andy Lutomirski 已提交
1466
		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1467

1468
	load_current_idt();
1469
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1470 1471 1472 1473 1474

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
	atomic_inc(&init_mm.mm_count);
1475
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1476
	BUG_ON(curr->mm);
1477
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1478

1479
	load_sp0(t, thread);
1480
	set_tss_desc(cpu, t);
L
Linus Torvalds 已提交
1481 1482 1483
	load_TR_desc();
	load_LDT(&init_mm.context);

1484 1485
	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);

1486
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1487 1488
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1489
#endif
L
Linus Torvalds 已提交
1490

1491
	clear_all_debug_regs();
1492
	dbg_restore_debug_regs();
L
Linus Torvalds 已提交
1493

1494
	fpu_init();
L
Linus Torvalds 已提交
1495
}
1496
#endif
1497 1498 1499 1500 1501 1502 1503 1504

#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
void warn_pre_alternatives(void)
{
	WARN(1, "You're using static_cpu_has before alternatives have run!\n");
}
EXPORT_SYMBOL_GPL(warn_pre_alternatives);
#endif
1505 1506 1507 1508 1509 1510

inline bool __static_cpu_has_safe(u16 bit)
{
	return boot_cpu_has(bit);
}
EXPORT_SYMBOL_GPL(__static_cpu_has_safe);