common.c 29.6 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/module.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
L
Linus Torvalds 已提交
8
#include <linux/delay.h>
9 10 11
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
12
#include <linux/smp.h>
13 14 15
#include <linux/io.h>

#include <asm/stackprotector.h>
16
#include <asm/perf_event.h>
L
Linus Torvalds 已提交
17
#include <asm/mmu_context.h>
18 19 20
#include <asm/hypervisor.h>
#include <asm/processor.h>
#include <asm/sections.h>
A
Alan Cox 已提交
21 22
#include <linux/topology.h>
#include <linux/cpumask.h>
23 24 25 26 27 28 29
#include <asm/pgtable.h>
#include <asm/atomic.h>
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/i387.h>
30
#include <asm/mtrr.h>
A
Alan Cox 已提交
31
#include <linux/numa.h>
32 33
#include <asm/asm.h>
#include <asm/cpu.h>
34
#include <asm/mce.h>
35
#include <asm/msr.h>
36
#include <asm/pat.h>
37 38

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
39
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
40 41 42 43
#endif

#include "cpu.h"

44 45
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
46 47
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
48 49 50 51

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
52
/* correctly size the local cpu masks */
53
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
54 55 56 57 58 59 60
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

61 62 63
static void __cpuinit default_init(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_64
64
	cpu_detect_cache_sizes(c);
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
#else
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
#endif
}

static const struct cpu_dev __cpuinitconst default_cpu = {
	.c_init		= default_init,
	.c_vendor	= "Unknown",
	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
};

static const struct cpu_dev *this_cpu __cpuinitdata = &default_cpu;
85

B
Brian Gerst 已提交
86
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
87
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
88 89 90 91 92
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
93
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
94 95
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
A
Akinobu Mita 已提交
96 97 98 99 100 101
	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
Y
Yinghai Lu 已提交
102
#else
A
Akinobu Mita 已提交
103 104 105 106
	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
107 108 109 110 111
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
112
	/* 32-bit code */
A
Akinobu Mita 已提交
113
	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
114
	/* 16-bit code */
A
Akinobu Mita 已提交
115
	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
116
	/* 16-bit data */
A
Akinobu Mita 已提交
117
	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
118
	/* 16-bit data */
A
Akinobu Mita 已提交
119
	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
120
	/* 16-bit data */
A
Akinobu Mita 已提交
121
	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
122 123 124 125
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
126
	/* 32-bit code */
A
Akinobu Mita 已提交
127
	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
128
	/* 16-bit code */
A
Akinobu Mita 已提交
129
	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
130
	/* data */
131
	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
132

A
Akinobu Mita 已提交
133 134
	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
135
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
136
#endif
B
Brian Gerst 已提交
137
} };
138
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
139

140 141 142 143 144 145 146
static int __init x86_xsave_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
	return 1;
}
__setup("noxsave", x86_xsave_setup);

147
#ifdef CONFIG_X86_32
148 149
static int cachesize_override __cpuinitdata = -1;
static int disable_x86_serial_nr __cpuinitdata = 1;
L
Linus Torvalds 已提交
150

151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_fxsr_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_FXSR);
	setup_clear_cpu_cap(X86_FEATURE_XMM);
	return 1;
}
__setup("nofxsr", x86_fxsr_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

178 179 180 181 182 183 184
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
185 186 187 188 189 190 191 192 193 194 195
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

196 197
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
198 199 200 201 202 203 204 205 206 207 208 209

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
static int __cpuinit have_cpuid_p(void)
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

static void __cpuinit squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
I
Ingo Molnar 已提交
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

	printk(KERN_NOTICE "CPU serial number disabled.\n");
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
226 227 228 229 230 231 232 233
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
234
#else
235 236 237 238
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
239 240 241 242 243
/* Probe for the CPUID instruction */
static inline int have_cpuid_p(void)
{
	return 1;
}
244 245 246
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
247
#endif
248

249 250 251 252 253 254 255 256 257
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
258

259 260 261 262 263 264 265 266 267 268 269
static const struct cpuid_dependent_feature __cpuinitconst
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
{
	const struct cpuid_dependent_feature *df;
270

271
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
272 273 274

		if (!cpu_has(c, df->feature))
			continue;
275 276 277 278 279 280 281
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
282
		if (!((s32)df->level < 0 ?
283
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
284 285 286 287 288 289 290 291 292 293
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

		printk(KERN_WARNING
		       "CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
				x86_cap_flags[df->feature], df->level);
294
	}
295
}
296

297 298 299
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
300 301
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
302 303 304
 */

/* Look up CPU names by table lookup. */
305
static const char *__cpuinit table_lookup_model(struct cpuinfo_x86 *c)
306
{
307
	const struct cpu_model_info *info;
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

	info = this_cpu->c_models;

	while (info && info->family) {
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
	return NULL;		/* Not found */
}

325 326
__u32 cpu_caps_cleared[NCAPINTS] __cpuinitdata;
__u32 cpu_caps_set[NCAPINTS] __cpuinitdata;
327

328 329 330 331 332 333 334 335
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
	loadsegment(gs, 0);
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
336
	load_stack_canary_segment();
337 338
}

I
Ingo Molnar 已提交
339 340 341 342
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
343
void switch_to_new_gdt(int cpu)
344 345 346
{
	struct desc_ptr gdt_descr;

347
	gdt_descr.address = (long)get_cpu_gdt_table(cpu);
348 349
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
350
	/* Reload the per-cpu base */
351 352

	load_percpu_segment(cpu);
353 354
}

355
static const struct cpu_dev *__cpuinitdata cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
356

357
static void __cpuinit get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
358 359 360 361
{
	unsigned int *v;
	char *p, *q;

362
	if (c->extended_cpuid_level < 0x80000004)
363
		return;
L
Linus Torvalds 已提交
364

I
Ingo Molnar 已提交
365
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
366 367 368 369 370
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

I
Ingo Molnar 已提交
371 372 373 374
	/*
	 * Intel chips right-justify this string for some dumb reason;
	 * undo that brain damage:
	 */
L
Linus Torvalds 已提交
375
	p = q = &c->x86_model_id[0];
376
	while (*p == ' ')
377
		p++;
378
	if (p != q) {
379 380 381 382
		while (*p)
			*q++ = *p++;
		while (q <= &c->x86_model_id[48])
			*q++ = '\0';	/* Zero-pad the rest */
L
Linus Torvalds 已提交
383 384 385
	}
}

386
void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
387
{
388
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
389

390
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
391 392

	if (n >= 0x80000005) {
393 394
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
395 396 397 398
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
399 400 401 402 403
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

404
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
405
	l2size = ecx >> 16;
406

407 408 409
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
410 411
	/* do processor-specific cache resizing */
	if (this_cpu->c_size_cache)
412
		l2size = this_cpu->c_size_cache(c, l2size);
L
Linus Torvalds 已提交
413 414 415 416 417

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

418
	if (l2size == 0)
L
Linus Torvalds 已提交
419
		return;		/* Again, no L2 cache is possible */
420
#endif
L
Linus Torvalds 已提交
421 422 423 424

	c->x86_cache_size = l2size;
}

425
void __cpuinit detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
426
{
427
#ifdef CONFIG_X86_HT
428 429
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
430
	static bool printed;
L
Linus Torvalds 已提交
431

432
	if (!cpu_has(c, X86_FEATURE_HT))
433
		return;
L
Linus Torvalds 已提交
434

435 436
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
437

438 439
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
440

441
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
442

443 444 445
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
446
		printk_once(KERN_INFO "CPU0: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
447 448
		goto out;
	}
449

I
Ingo Molnar 已提交
450 451
	if (smp_num_siblings <= 1)
		goto out;
452

I
Ingo Molnar 已提交
453 454 455 456 457 458
	if (smp_num_siblings > nr_cpu_ids) {
		pr_warning("CPU: Unsupported number of siblings %d",
			   smp_num_siblings);
		smp_num_siblings = 1;
		return;
	}
459

I
Ingo Molnar 已提交
460 461
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
462

I
Ingo Molnar 已提交
463
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
464

I
Ingo Molnar 已提交
465
	index_msb = get_count_order(smp_num_siblings);
466

I
Ingo Molnar 已提交
467
	core_bits = get_count_order(c->x86_max_cores);
468

I
Ingo Molnar 已提交
469 470
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
471

472
out:
473
	if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) {
474 475 476 477
		printk(KERN_INFO  "CPU: Physical Processor ID: %d\n",
		       c->phys_proc_id);
		printk(KERN_INFO  "CPU: Processor Core ID: %d\n",
		       c->cpu_core_id);
478
		printed = 1;
479 480
	}
#endif
481
}
L
Linus Torvalds 已提交
482

483
static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
484 485
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
486
	int i;
L
Linus Torvalds 已提交
487 488

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
489 490 491 492 493 494
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
495

Y
Yinghai Lu 已提交
496 497 498
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
499 500
		}
	}
Y
Yinghai Lu 已提交
501

502 503 504
	printk_once(KERN_ERR
			"CPU: vendor_id '%s' unknown, using generic init.\n" \
			"CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
505

506 507
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
508 509
}

510
void __cpuinit cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
511 512
{
	/* Get vendor name */
513 514 515 516
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
517 518

	c->x86 = 4;
519
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
520 521
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
522

L
Linus Torvalds 已提交
523
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
524 525 526
		c->x86 = (tfms >> 8) & 0xf;
		c->x86_model = (tfms >> 4) & 0xf;
		c->x86_mask = tfms & 0xf;
I
Ingo Molnar 已提交
527

528
		if (c->x86 == 0xf)
L
Linus Torvalds 已提交
529
			c->x86 += (tfms >> 20) & 0xff;
530
		if (c->x86 >= 0x6)
531
			c->x86_model += ((tfms >> 16) & 0xf) << 4;
I
Ingo Molnar 已提交
532

H
Huang, Ying 已提交
533 534
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
535
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
536
		}
L
Linus Torvalds 已提交
537 538
	}
}
539 540

static void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c)
541 542
{
	u32 tfms, xlvl;
543
	u32 ebx;
544

545 546 547
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
		u32 capability, excap;
I
Ingo Molnar 已提交
548

549 550 551 552
		cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
		c->x86_capability[0] = capability;
		c->x86_capability[4] = excap;
	}
553

554 555 556
	/* AMD-defined flags: level 0x80000001 */
	xlvl = cpuid_eax(0x80000000);
	c->extended_cpuid_level = xlvl;
I
Ingo Molnar 已提交
557

558 559 560 561
	if ((xlvl & 0xffff0000) == 0x80000000) {
		if (xlvl >= 0x80000001) {
			c->x86_capability[1] = cpuid_edx(0x80000001);
			c->x86_capability[6] = cpuid_ecx(0x80000001);
562 563 564
		}
	}

565 566 567 568 569
	if (c->extended_cpuid_level >= 0x80000008) {
		u32 eax = cpuid_eax(0x80000008);

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
570
	}
571 572 573
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
574
#endif
575 576 577

	if (c->extended_cpuid_level >= 0x80000007)
		c->x86_power = cpuid_edx(0x80000007);
578 579

}
L
Linus Torvalds 已提交
580

Y
Yinghai Lu 已提交
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
static void __cpuinit identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

607 608 609 610 611 612 613 614 615
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
616
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
617
{
618 619
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
620 621
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
622
#else
H
Huang, Ying 已提交
623
	c->x86_clflush_size = 32;
624 625
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
626
#endif
627
	c->x86_cache_alignment = c->x86_clflush_size;
628

629
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
630
	c->extended_cpuid_level = 0;
631

Y
Yinghai Lu 已提交
632 633 634 635
	if (!have_cpuid_p())
		identify_cpu_without_cpuid(c);

	/* cyrix could have cpuid enabled via c_identify()*/
636 637 638 639 640
	if (!have_cpuid_p())
		return;

	cpu_detect(c);

641
	get_cpu_vendor(c);
642

643
	get_cpu_cap(c);
644

Y
Yinghai Lu 已提交
645 646
	if (this_cpu->c_early_init)
		this_cpu->c_early_init(c);
647

I
Ingo Molnar 已提交
648
#ifdef CONFIG_SMP
649
	c->cpu_index = boot_cpu_id;
I
Ingo Molnar 已提交
650
#endif
651
	filter_cpuid_features(c, false);
652 653
}

654 655
void __init early_cpu_init(void)
{
656
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
657 658
	int count = 0;

659
#ifdef PROCESSOR_SELECT
660
	printk(KERN_INFO "KERNEL supported cpus:\n");
661 662
#endif

Y
Yinghai Lu 已提交
663
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
664
		const struct cpu_dev *cpudev = *cdev;
665

Y
Yinghai Lu 已提交
666 667 668 669 670
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

671 672 673 674 675 676 677 678 679 680
#ifdef PROCESSOR_SELECT
		{
			unsigned int j;

			for (j = 0; j < 2; j++) {
				if (!cpudev->c_ident[j])
					continue;
				printk(KERN_INFO "  %s %s\n", cpudev->c_vendor,
					cpudev->c_ident[j]);
			}
Y
Yinghai Lu 已提交
681
		}
682
#endif
Y
Yinghai Lu 已提交
683
	}
684
	early_identify_cpu(&boot_cpu_data);
685
}
686

687 688
/*
 * The NOPL instruction is supposed to exist on all CPUs with
689
 * family >= 6; unfortunately, that's not true in practice because
690
 * of early VIA chips and (more importantly) broken virtualizers that
691 692 693
 * are not easy to detect.  In the latter case it doesn't even *fail*
 * reliably, so probing for it doesn't even work.  Disable it completely
 * unless we can find a reliable way to detect all the broken cases.
694 695 696 697
 */
static void __cpuinit detect_nopl(struct cpuinfo_x86 *c)
{
	clear_cpu_cap(c, X86_FEATURE_NOPL);
698 699
}

700
static void __cpuinit generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
701
{
Y
Yinghai Lu 已提交
702
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
703

704
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
705
		identify_cpu_without_cpuid(c);
706

Y
Yinghai Lu 已提交
707
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
708
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
709
		return;
L
Linus Torvalds 已提交
710

711
	cpu_detect(c);
L
Linus Torvalds 已提交
712

713
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
714

715
	get_cpu_cap(c);
L
Linus Torvalds 已提交
716

717 718
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
719 720
#ifdef CONFIG_X86_32
# ifdef CONFIG_X86_HT
721
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
722
# else
723
		c->apicid = c->initial_apicid;
724 725
# endif
#endif
L
Linus Torvalds 已提交
726

727 728
#ifdef CONFIG_X86_HT
		c->phys_proc_id = c->initial_apicid;
729
#endif
730
	}
L
Linus Torvalds 已提交
731

732
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
733

734 735
	init_scattered_cpuid_features(c);
	detect_nopl(c);
L
Linus Torvalds 已提交
736 737 738 739 740
}

/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
Y
Yinghai Lu 已提交
741
static void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
742 743 744 745 746 747 748 749 750
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
751
	c->x86_max_cores = 1;
752
	c->x86_coreid_bits = 0;
753
#ifdef CONFIG_X86_64
754
	c->x86_clflush_size = 64;
755 756
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
757 758
#else
	c->cpuid_level = -1;	/* CPUID not detected */
759
	c->x86_clflush_size = 32;
760 761
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
762 763
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
764 765 766 767
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

768
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
769 770
		this_cpu->c_identify(c);

771 772 773 774 775 776
	/* Clear/Set all flags overriden by options, after probe */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

777
#ifdef CONFIG_X86_64
778
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
779 780
#endif

L
Linus Torvalds 已提交
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

	/*
I
Ingo Molnar 已提交
798 799
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
800 801
	 */

802 803 804
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
805
	/* If the model name is still unset, do table lookup. */
806
	if (!c->x86_model_id[0]) {
807
		const char *p;
L
Linus Torvalds 已提交
808
		p = table_lookup_model(c);
809
		if (p)
L
Linus Torvalds 已提交
810 811 812 813
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
814
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
815 816
	}

817 818 819 820
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

821
	init_hypervisor(c);
822 823 824 825 826 827 828 829 830 831

	/*
	 * Clear/Set all flags overriden by options, need do it
	 * before following smp all cpus cap AND.
	 */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

L
Linus Torvalds 已提交
832 833 834 835 836 837
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
838
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
839
		/* AND the already accumulated flags with these */
840
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
841 842 843 844
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
	}

	/* Init Machine Check Exception if available. */
845
	mcheck_cpu_init(c);
846 847

	select_idle_routine(c);
848 849 850 851

#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
	numa_add_cpu(smp_processor_id());
#endif
852
}
S
Shaohua Li 已提交
853

854 855 856 857 858 859 860 861 862 863
#ifdef CONFIG_X86_64
static void vgetcpu_set_mode(void)
{
	if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
		vgetcpu_mode = VGETCPU_RDTSCP;
	else
		vgetcpu_mode = VGETCPU_LSL;
}
#endif

864 865 866
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
867
	init_c1e_mask();
868
#ifdef CONFIG_X86_32
869
	sysenter_setup();
L
Li Shaohua 已提交
870
	enable_sep_cpu();
871 872
#else
	vgetcpu_set_mode();
873
#endif
874
	init_hw_perf_events();
875
}
S
Shaohua Li 已提交
876

877 878 879 880
void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
881
#ifdef CONFIG_X86_32
882
	enable_sep_cpu();
883
#endif
884
	mtrr_ap_init();
L
Linus Torvalds 已提交
885 886
}

887
struct msr_range {
I
Ingo Molnar 已提交
888 889
	unsigned	min;
	unsigned	max;
890
};
L
Linus Torvalds 已提交
891

892
static const struct msr_range msr_range_array[] __cpuinitconst = {
893 894 895 896 897
	{ 0x00000000, 0x00000418},
	{ 0xc0000000, 0xc000040b},
	{ 0xc0010000, 0xc0010142},
	{ 0xc0011000, 0xc001103b},
};
L
Linus Torvalds 已提交
898

899 900
static void __cpuinit print_cpu_msr(void)
{
I
Ingo Molnar 已提交
901
	unsigned index_min, index_max;
902 903 904 905 906 907 908
	unsigned index;
	u64 val;
	int i;

	for (i = 0; i < ARRAY_SIZE(msr_range_array); i++) {
		index_min = msr_range_array[i].min;
		index_max = msr_range_array[i].max;
I
Ingo Molnar 已提交
909

910 911 912 913
		for (index = index_min; index < index_max; index++) {
			if (rdmsrl_amd_safe(index, &val))
				continue;
			printk(KERN_INFO " MSR%08x: %016llx\n", index, val);
L
Linus Torvalds 已提交
914
		}
915 916
	}
}
917

918
static int show_msr __cpuinitdata;
I
Ingo Molnar 已提交
919

920 921 922
static __init int setup_show_msr(char *arg)
{
	int num;
923

924
	get_option(&arg, &num);
925

926 927 928
	if (num > 0)
		show_msr = num;
	return 1;
L
Linus Torvalds 已提交
929
}
930
__setup("show_msr=", setup_show_msr);
L
Linus Torvalds 已提交
931

A
Andi Kleen 已提交
932 933 934 935 936 937 938
static __init int setup_noclflush(char *arg)
{
	setup_clear_cpu_cap(X86_FEATURE_CLFLSH);
	return 1;
}
__setup("noclflush", setup_noclflush);

939
void __cpuinit print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
940
{
941
	const char *vendor = NULL;
L
Linus Torvalds 已提交
942

I
Ingo Molnar 已提交
943
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
944
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
945 946 947 948
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
949

950
	if (vendor && !strstr(c->x86_model_id, vendor))
951
		printk(KERN_CONT "%s ", vendor);
L
Linus Torvalds 已提交
952

953 954
	if (c->x86_model_id[0])
		printk(KERN_CONT "%s", c->x86_model_id);
L
Linus Torvalds 已提交
955
	else
956
		printk(KERN_CONT "%d86", c->x86);
L
Linus Torvalds 已提交
957

958
	if (c->x86_mask || c->cpuid_level >= 0)
959
		printk(KERN_CONT " stepping %02x\n", c->x86_mask);
L
Linus Torvalds 已提交
960
	else
961
		printk(KERN_CONT "\n");
962 963 964 965 966 967 968 969

#ifdef CONFIG_SMP
	if (c->cpu_index < show_msr)
		print_cpu_msr();
#else
	if (show_msr)
		print_cpu_msr();
#endif
L
Linus Torvalds 已提交
970 971
}

972 973 974
static __init int setup_disablecpuid(char *arg)
{
	int bit;
I
Ingo Molnar 已提交
975

976 977 978 979
	if (get_option(&arg, &bit) && bit < NCAPINTS*32)
		setup_clear_cpu_cap(bit);
	else
		return 0;
I
Ingo Molnar 已提交
980

981 982 983 984
	return 1;
}
__setup("clearcpuid=", setup_disablecpuid);

985
#ifdef CONFIG_X86_64
986
struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
987

988 989
DEFINE_PER_CPU_FIRST(union irq_stack_union,
		     irq_stack_union) __aligned(PAGE_SIZE);
I
Ingo Molnar 已提交
990

991 992 993 994 995 996 997
/*
 * The following four percpu variables are hot.  Align current_task to
 * cacheline size such that all four fall in the same cacheline.
 */
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
	&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
998

999 1000 1001 1002
DEFINE_PER_CPU(unsigned long, kernel_stack) =
	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(kernel_stack);

1003 1004 1005
DEFINE_PER_CPU(char *, irq_stack_ptr) =
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;

1006
DEFINE_PER_CPU(unsigned int, irq_count) = -1;
1007

I
Ingo Molnar 已提交
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

1019
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
1020
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
1021 1022 1023

/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1024
{
1025 1026 1027 1028 1029 1030 1031 1032
	/*
	 * LSTAR and STAR live in a bit strange symbiosis.
	 * They both write to the same internal register. STAR allows to
	 * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip.
	 */
	wrmsrl(MSR_STAR,  ((u64)__USER32_CS)<<48  | ((u64)__KERNEL_CS)<<32);
	wrmsrl(MSR_LSTAR, system_call);
	wrmsrl(MSR_CSTAR, ignore_sysret);
1033

1034 1035 1036
#ifdef CONFIG_IA32_EMULATION
	syscall32_cpu_init();
#endif
1037

1038 1039 1040
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL);
L
Linus Torvalds 已提交
1041
}
1042

1043 1044 1045 1046 1047 1048 1049 1050
unsigned long kernel_eflags;

/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

I
Ingo Molnar 已提交
1051
#else	/* CONFIG_X86_64 */
1052

1053 1054 1055
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);

1056
#ifdef CONFIG_CC_STACKPROTECTOR
1057
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
1058
#endif
1059

1060
/* Make sure %fs and %gs are initialized properly in idle threads */
A
Adrian Bunk 已提交
1061
struct pt_regs * __cpuinit idle_regs(struct pt_regs *regs)
1062 1063
{
	memset(regs, 0, sizeof(struct pt_regs));
1064
	regs->fs = __KERNEL_PERCPU;
1065
	regs->gs = __KERNEL_STACK_CANARY;
I
Ingo Molnar 已提交
1066

1067 1068
	return regs;
}
I
Ingo Molnar 已提交
1069
#endif	/* CONFIG_X86_64 */
1070

1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1086

1087 1088 1089 1090 1091
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1092
 * A lot of state is already set up in PDA init for 64 bit
1093
 */
1094
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1095

1096 1097
void __cpuinit cpu_init(void)
{
1098
	struct orig_ist *oist;
1099
	struct task_struct *me;
I
Ingo Molnar 已提交
1100 1101 1102
	struct tss_struct *t;
	unsigned long v;
	int cpu;
1103 1104
	int i;

I
Ingo Molnar 已提交
1105 1106
	cpu = stack_smp_processor_id();
	t = &per_cpu(init_tss, cpu);
1107
	oist = &per_cpu(orig_ist, cpu);
I
Ingo Molnar 已提交
1108

1109 1110 1111 1112 1113
#ifdef CONFIG_NUMA
	if (cpu != 0 && percpu_read(node_number) == 0 &&
	    cpu_to_node(cpu) != NUMA_NO_NODE)
		percpu_write(node_number, cpu_to_node(cpu));
#endif
1114 1115 1116

	me = current;

1117
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask))
1118 1119
		panic("CPU#%d already initialized!\n", cpu);

1120
	pr_debug("Initializing CPU#%d\n", cpu);
1121 1122 1123 1124 1125 1126 1127 1128

	clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1129
	switch_to_new_gdt(cpu);
1130 1131
	loadsegment(fs, 0);

1132 1133 1134 1135 1136 1137 1138 1139 1140
	load_idt((const struct desc_ptr *)&idt_descr);

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

1141
	x86_configure_nx();
Y
Yinghai Lu 已提交
1142
	if (cpu != 0)
1143 1144 1145 1146 1147
		enable_x2apic();

	/*
	 * set up and load the per-CPU TSS
	 */
1148
	if (!oist->ist[0]) {
1149
		char *estacks = per_cpu(exception_stacks, cpu);
I
Ingo Molnar 已提交
1150

1151
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1152
			estacks += exception_stack_sizes[v];
1153
			oist->ist[v] = t->x86_tss.ist[v] =
1154 1155 1156 1157 1158
					(unsigned long)estacks;
		}
	}

	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
I
Ingo Molnar 已提交
1159

1160 1161 1162 1163 1164 1165 1166 1167 1168
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

	atomic_inc(&init_mm.mm_count);
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1169
	BUG_ON(me->mm);
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
	enter_lazy_tlb(&init_mm, me);

	load_sp0(t, &current->thread);
	set_tss_desc(cpu, t);
	load_TR_desc();
	load_LDT(&init_mm.context);

#ifdef CONFIG_KGDB
	/*
	 * If the kgdb is connected no debug regs should be altered.  This
	 * is only applicable when KGDB and a KGDB I/O module are built
	 * into the kernel and you are using early debugging with
	 * kgdbwait. KGDB will control the kernel HW breakpoint registers.
	 */
	if (kgdb_connected && arch_kgdb_ops.correct_hw_break)
		arch_kgdb_ops.correct_hw_break();
1186
	else
1187
#endif
1188
		clear_all_debug_regs();
1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199

	fpu_init();

	raw_local_save_flags(kernel_eflags);

	if (is_uv_system())
		uv_cpu_init();
}

#else

1200
void __cpuinit cpu_init(void)
1201
{
1202 1203
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1204
	struct tss_struct *t = &per_cpu(init_tss, cpu);
1205
	struct thread_struct *thread = &curr->thread;
1206

1207
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) {
1208
		printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
1209 1210
		for (;;)
			local_irq_enable();
1211 1212 1213 1214 1215 1216 1217
	}

	printk(KERN_INFO "Initializing CPU#%d\n", cpu);

	if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
		clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

1218
	load_idt(&idt_descr);
1219
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1220 1221 1222 1223 1224

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
	atomic_inc(&init_mm.mm_count);
1225
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1226
	BUG_ON(curr->mm);
1227
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1228

1229
	load_sp0(t, thread);
1230
	set_tss_desc(cpu, t);
L
Linus Torvalds 已提交
1231 1232 1233
	load_TR_desc();
	load_LDT(&init_mm.context);

1234 1235
	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);

1236
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1237 1238
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1239
#endif
L
Linus Torvalds 已提交
1240

1241
	clear_all_debug_regs();
L
Linus Torvalds 已提交
1242 1243 1244 1245

	/*
	 * Force FPU initialization:
	 */
A
Avi Kivity 已提交
1246
	current_thread_info()->status = 0;
L
Linus Torvalds 已提交
1247 1248
	clear_used_math();
	mxcsr_feature_mask_init();
1249 1250 1251 1252

	/*
	 * Boot processor to setup the FP and extended state context info.
	 */
1253
	if (smp_processor_id() == boot_cpu_id)
1254 1255 1256
		init_thread_xstate();

	xsave_init();
L
Linus Torvalds 已提交
1257
}
1258
#endif