common.c 29.4 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/module.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
L
Linus Torvalds 已提交
8
#include <linux/delay.h>
9 10 11
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
12
#include <linux/smp.h>
13 14 15
#include <linux/io.h>

#include <asm/stackprotector.h>
16
#include <asm/perf_counter.h>
L
Linus Torvalds 已提交
17
#include <asm/mmu_context.h>
18 19 20
#include <asm/hypervisor.h>
#include <asm/processor.h>
#include <asm/sections.h>
I
Ingo Molnar 已提交
21
#include <asm/topology.h>
22 23 24 25 26 27 28 29
#include <asm/cpumask.h>
#include <asm/pgtable.h>
#include <asm/atomic.h>
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/i387.h>
30
#include <asm/mtrr.h>
31 32 33
#include <asm/numa.h>
#include <asm/asm.h>
#include <asm/cpu.h>
34
#include <asm/mce.h>
35
#include <asm/msr.h>
36
#include <asm/pat.h>
I
Ingo Molnar 已提交
37
#include <asm/smp.h>
38 39

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
40
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
41 42 43 44
#endif

#include "cpu.h"

45 46
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
47 48
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
49 50 51 52

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
53
/* correctly size the local cpu masks */
54
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
55 56 57 58 59 60 61
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

62
static const struct cpu_dev *this_cpu __cpuinitdata;
63

B
Brian Gerst 已提交
64
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
65
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
66 67 68 69 70
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
71
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
72 73
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
I
Ingo Molnar 已提交
74 75 76 77 78 79
	[GDT_ENTRY_KERNEL32_CS]		= { { { 0x0000ffff, 0x00cf9b00 } } },
	[GDT_ENTRY_KERNEL_CS]		= { { { 0x0000ffff, 0x00af9b00 } } },
	[GDT_ENTRY_KERNEL_DS]		= { { { 0x0000ffff, 0x00cf9300 } } },
	[GDT_ENTRY_DEFAULT_USER32_CS]	= { { { 0x0000ffff, 0x00cffb00 } } },
	[GDT_ENTRY_DEFAULT_USER_DS]	= { { { 0x0000ffff, 0x00cff300 } } },
	[GDT_ENTRY_DEFAULT_USER_CS]	= { { { 0x0000ffff, 0x00affb00 } } },
Y
Yinghai Lu 已提交
80
#else
I
Ingo Molnar 已提交
81 82 83 84
	[GDT_ENTRY_KERNEL_CS]		= { { { 0x0000ffff, 0x00cf9a00 } } },
	[GDT_ENTRY_KERNEL_DS]		= { { { 0x0000ffff, 0x00cf9200 } } },
	[GDT_ENTRY_DEFAULT_USER_CS]	= { { { 0x0000ffff, 0x00cffa00 } } },
	[GDT_ENTRY_DEFAULT_USER_DS]	= { { { 0x0000ffff, 0x00cff200 } } },
85 86 87 88 89
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
90
	/* 32-bit code */
I
Ingo Molnar 已提交
91
	[GDT_ENTRY_PNPBIOS_CS32]	= { { { 0x0000ffff, 0x00409a00 } } },
92
	/* 16-bit code */
I
Ingo Molnar 已提交
93
	[GDT_ENTRY_PNPBIOS_CS16]	= { { { 0x0000ffff, 0x00009a00 } } },
94
	/* 16-bit data */
I
Ingo Molnar 已提交
95
	[GDT_ENTRY_PNPBIOS_DS]		= { { { 0x0000ffff, 0x00009200 } } },
96
	/* 16-bit data */
I
Ingo Molnar 已提交
97
	[GDT_ENTRY_PNPBIOS_TS1]		= { { { 0x00000000, 0x00009200 } } },
98
	/* 16-bit data */
I
Ingo Molnar 已提交
99
	[GDT_ENTRY_PNPBIOS_TS2]		= { { { 0x00000000, 0x00009200 } } },
100 101 102 103
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
104
	/* 32-bit code */
I
Ingo Molnar 已提交
105
	[GDT_ENTRY_APMBIOS_BASE]	= { { { 0x0000ffff, 0x00409a00 } } },
106
	/* 16-bit code */
I
Ingo Molnar 已提交
107
	[GDT_ENTRY_APMBIOS_BASE+1]	= { { { 0x0000ffff, 0x00009a00 } } },
108
	/* data */
I
Ingo Molnar 已提交
109
	[GDT_ENTRY_APMBIOS_BASE+2]	= { { { 0x0000ffff, 0x00409200 } } },
110

111
	[GDT_ENTRY_ESPFIX_SS]		= { { { 0x0000ffff, 0x00cf9200 } } },
I
Ingo Molnar 已提交
112
	[GDT_ENTRY_PERCPU]		= { { { 0x0000ffff, 0x00cf9200 } } },
113
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
114
#endif
B
Brian Gerst 已提交
115
} };
116
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
117

118 119 120 121 122 123 124
static int __init x86_xsave_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
	return 1;
}
__setup("noxsave", x86_xsave_setup);

125
#ifdef CONFIG_X86_32
126 127
static int cachesize_override __cpuinitdata = -1;
static int disable_x86_serial_nr __cpuinitdata = 1;
L
Linus Torvalds 已提交
128

129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_fxsr_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_FXSR);
	setup_clear_cpu_cap(X86_FEATURE_XMM);
	return 1;
}
__setup("nofxsr", x86_fxsr_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

156 157 158 159 160 161 162
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
163 164 165 166 167 168 169 170 171 172 173
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

174 175
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
176 177 178 179 180 181 182 183 184 185 186 187

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
static int __cpuinit have_cpuid_p(void)
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

static void __cpuinit squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
I
Ingo Molnar 已提交
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

	printk(KERN_NOTICE "CPU serial number disabled.\n");
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
204 205 206 207 208 209 210 211
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
212
#else
213 214 215 216
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
217 218 219 220 221
/* Probe for the CPUID instruction */
static inline int have_cpuid_p(void)
{
	return 1;
}
222 223 224
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
225
#endif
226

227 228 229 230 231 232 233 234 235
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
236

237 238 239 240 241 242 243 244 245 246 247
static const struct cpuid_dependent_feature __cpuinitconst
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
{
	const struct cpuid_dependent_feature *df;
248

249
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
250 251 252

		if (!cpu_has(c, df->feature))
			continue;
253 254 255 256 257 258 259
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
260
		if (!((s32)df->level < 0 ?
261
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
262 263 264 265 266 267 268 269 270 271
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

		printk(KERN_WARNING
		       "CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
				x86_cap_flags[df->feature], df->level);
272
	}
273
}
274

275 276 277
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
278 279
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
280 281 282
 */

/* Look up CPU names by table lookup. */
283
static const char *__cpuinit table_lookup_model(struct cpuinfo_x86 *c)
284
{
285
	const struct cpu_model_info *info;
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

	info = this_cpu->c_models;

	while (info && info->family) {
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
	return NULL;		/* Not found */
}

303 304
__u32 cpu_caps_cleared[NCAPINTS] __cpuinitdata;
__u32 cpu_caps_set[NCAPINTS] __cpuinitdata;
305

306 307 308 309 310 311 312 313
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
	loadsegment(gs, 0);
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
314
	load_stack_canary_segment();
315 316
}

I
Ingo Molnar 已提交
317 318 319 320
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
321
void switch_to_new_gdt(int cpu)
322 323 324
{
	struct desc_ptr gdt_descr;

325
	gdt_descr.address = (long)get_cpu_gdt_table(cpu);
326 327
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
328
	/* Reload the per-cpu base */
329 330

	load_percpu_segment(cpu);
331 332
}

333
static const struct cpu_dev *__cpuinitdata cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
334

335
static void __cpuinit default_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
336
{
337 338 339
#ifdef CONFIG_X86_64
	display_cacheinfo(c);
#else
L
Linus Torvalds 已提交
340 341 342 343 344 345 346 347 348
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
349
#endif
L
Linus Torvalds 已提交
350 351
}

352
static const struct cpu_dev __cpuinitconst default_cpu = {
L
Linus Torvalds 已提交
353
	.c_init	= default_init,
354
	.c_vendor = "Unknown",
Y
Yinghai Lu 已提交
355
	.c_x86_vendor = X86_VENDOR_UNKNOWN,
L
Linus Torvalds 已提交
356 357
};

358
static void __cpuinit get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
359 360 361 362
{
	unsigned int *v;
	char *p, *q;

363
	if (c->extended_cpuid_level < 0x80000004)
364
		return;
L
Linus Torvalds 已提交
365

I
Ingo Molnar 已提交
366
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
367 368 369 370 371
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

I
Ingo Molnar 已提交
372 373 374 375
	/*
	 * Intel chips right-justify this string for some dumb reason;
	 * undo that brain damage:
	 */
L
Linus Torvalds 已提交
376
	p = q = &c->x86_model_id[0];
377
	while (*p == ' ')
378
		p++;
379
	if (p != q) {
380 381 382 383
		while (*p)
			*q++ = *p++;
		while (q <= &c->x86_model_id[48])
			*q++ = '\0';	/* Zero-pad the rest */
L
Linus Torvalds 已提交
384 385 386
	}
}

387
void __cpuinit display_cacheinfo(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
388
{
389
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
390

391
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
392 393

	if (n >= 0x80000005) {
394
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
395
		printk(KERN_INFO "CPU: L1 I Cache: %dK (%d bytes/line), D cache %dK (%d bytes/line)\n",
396 397
				edx>>24, edx&0xFF, ecx>>24, ecx&0xFF);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
398 399 400 401
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
402 403 404 405 406
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

407
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
408
	l2size = ecx >> 16;
409

410 411 412
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
413 414
	/* do processor-specific cache resizing */
	if (this_cpu->c_size_cache)
415
		l2size = this_cpu->c_size_cache(c, l2size);
L
Linus Torvalds 已提交
416 417 418 419 420

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

421
	if (l2size == 0)
L
Linus Torvalds 已提交
422
		return;		/* Again, no L2 cache is possible */
423
#endif
L
Linus Torvalds 已提交
424 425 426 427

	c->x86_cache_size = l2size;

	printk(KERN_INFO "CPU: L2 Cache: %dK (%d bytes/line)\n",
428
			l2size, ecx & 0xFF);
L
Linus Torvalds 已提交
429 430
}

431
void __cpuinit detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
432
{
433
#ifdef CONFIG_X86_HT
434 435
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
L
Linus Torvalds 已提交
436

437
	if (!cpu_has(c, X86_FEATURE_HT))
438
		return;
L
Linus Torvalds 已提交
439

440 441
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
442

443 444
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
445

446
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
447

448 449 450 451
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
		printk(KERN_INFO  "CPU: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
452 453
		goto out;
	}
454

I
Ingo Molnar 已提交
455 456
	if (smp_num_siblings <= 1)
		goto out;
457

I
Ingo Molnar 已提交
458 459 460 461 462 463
	if (smp_num_siblings > nr_cpu_ids) {
		pr_warning("CPU: Unsupported number of siblings %d",
			   smp_num_siblings);
		smp_num_siblings = 1;
		return;
	}
464

I
Ingo Molnar 已提交
465 466
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
467

I
Ingo Molnar 已提交
468
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
469

I
Ingo Molnar 已提交
470
	index_msb = get_count_order(smp_num_siblings);
471

I
Ingo Molnar 已提交
472
	core_bits = get_count_order(c->x86_max_cores);
473

I
Ingo Molnar 已提交
474 475
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
476

477 478 479 480 481 482
out:
	if ((c->x86_max_cores * smp_num_siblings) > 1) {
		printk(KERN_INFO  "CPU: Physical Processor ID: %d\n",
		       c->phys_proc_id);
		printk(KERN_INFO  "CPU: Processor Core ID: %d\n",
		       c->cpu_core_id);
483 484
	}
#endif
485
}
L
Linus Torvalds 已提交
486

487
static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
488 489
{
	char *v = c->x86_vendor_id;
I
Ingo Molnar 已提交
490
	int i;
L
Linus Torvalds 已提交
491 492

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
493 494 495 496 497 498
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
499

Y
Yinghai Lu 已提交
500 501 502
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
503 504
		}
	}
Y
Yinghai Lu 已提交
505

506 507 508
	printk_once(KERN_ERR
			"CPU: vendor_id '%s' unknown, using generic init.\n" \
			"CPU: Your system may be unstable.\n", v);
Y
Yinghai Lu 已提交
509

510 511
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
512 513
}

514
void __cpuinit cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
515 516
{
	/* Get vendor name */
517 518 519 520
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
521 522

	c->x86 = 4;
523
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
524 525
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
526

L
Linus Torvalds 已提交
527
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
528 529 530
		c->x86 = (tfms >> 8) & 0xf;
		c->x86_model = (tfms >> 4) & 0xf;
		c->x86_mask = tfms & 0xf;
I
Ingo Molnar 已提交
531

532
		if (c->x86 == 0xf)
L
Linus Torvalds 已提交
533
			c->x86 += (tfms >> 20) & 0xff;
534
		if (c->x86 >= 0x6)
535
			c->x86_model += ((tfms >> 16) & 0xf) << 4;
I
Ingo Molnar 已提交
536

H
Huang, Ying 已提交
537 538
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
539
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
540
		}
L
Linus Torvalds 已提交
541 542
	}
}
543 544

static void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c)
545 546
{
	u32 tfms, xlvl;
547
	u32 ebx;
548

549 550 551
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
		u32 capability, excap;
I
Ingo Molnar 已提交
552

553 554 555 556
		cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
		c->x86_capability[0] = capability;
		c->x86_capability[4] = excap;
	}
557

558 559 560
	/* AMD-defined flags: level 0x80000001 */
	xlvl = cpuid_eax(0x80000000);
	c->extended_cpuid_level = xlvl;
I
Ingo Molnar 已提交
561

562 563 564 565
	if ((xlvl & 0xffff0000) == 0x80000000) {
		if (xlvl >= 0x80000001) {
			c->x86_capability[1] = cpuid_edx(0x80000001);
			c->x86_capability[6] = cpuid_ecx(0x80000001);
566 567 568
		}
	}

569 570 571 572 573
	if (c->extended_cpuid_level >= 0x80000008) {
		u32 eax = cpuid_eax(0x80000008);

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
574
	}
575 576 577
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
578
#endif
579 580 581

	if (c->extended_cpuid_level >= 0x80000007)
		c->x86_power = cpuid_edx(0x80000007);
582 583

}
L
Linus Torvalds 已提交
584

Y
Yinghai Lu 已提交
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
static void __cpuinit identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

611 612 613 614 615 616 617 618 619
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
620
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
621
{
622 623
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
624 625
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
626
#else
H
Huang, Ying 已提交
627
	c->x86_clflush_size = 32;
628 629
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
630
#endif
631
	c->x86_cache_alignment = c->x86_clflush_size;
632

633
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
634
	c->extended_cpuid_level = 0;
635

Y
Yinghai Lu 已提交
636 637 638 639
	if (!have_cpuid_p())
		identify_cpu_without_cpuid(c);

	/* cyrix could have cpuid enabled via c_identify()*/
640 641 642 643 644
	if (!have_cpuid_p())
		return;

	cpu_detect(c);

645
	get_cpu_vendor(c);
646

647
	get_cpu_cap(c);
648

Y
Yinghai Lu 已提交
649 650
	if (this_cpu->c_early_init)
		this_cpu->c_early_init(c);
651

I
Ingo Molnar 已提交
652
#ifdef CONFIG_SMP
653
	c->cpu_index = boot_cpu_id;
I
Ingo Molnar 已提交
654
#endif
655
	filter_cpuid_features(c, false);
656 657
}

658 659
void __init early_cpu_init(void)
{
660
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
661 662
	int count = 0;

663
	printk(KERN_INFO "KERNEL supported cpus:\n");
Y
Yinghai Lu 已提交
664
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
665
		const struct cpu_dev *cpudev = *cdev;
Y
Yinghai Lu 已提交
666
		unsigned int j;
667

Y
Yinghai Lu 已提交
668 669 670 671 672 673 674 675
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

		for (j = 0; j < 2; j++) {
			if (!cpudev->c_ident[j])
				continue;
676
			printk(KERN_INFO "  %s %s\n", cpudev->c_vendor,
Y
Yinghai Lu 已提交
677 678 679
				cpudev->c_ident[j]);
		}
	}
680 681

	early_identify_cpu(&boot_cpu_data);
682
}
683

684 685
/*
 * The NOPL instruction is supposed to exist on all CPUs with
686
 * family >= 6; unfortunately, that's not true in practice because
687
 * of early VIA chips and (more importantly) broken virtualizers that
688 689 690
 * are not easy to detect.  In the latter case it doesn't even *fail*
 * reliably, so probing for it doesn't even work.  Disable it completely
 * unless we can find a reliable way to detect all the broken cases.
691 692 693 694
 */
static void __cpuinit detect_nopl(struct cpuinfo_x86 *c)
{
	clear_cpu_cap(c, X86_FEATURE_NOPL);
695 696
}

697
static void __cpuinit generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
698
{
Y
Yinghai Lu 已提交
699
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
700

701
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
702
		identify_cpu_without_cpuid(c);
703

Y
Yinghai Lu 已提交
704
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
705
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
706
		return;
L
Linus Torvalds 已提交
707

708
	cpu_detect(c);
L
Linus Torvalds 已提交
709

710
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
711

712
	get_cpu_cap(c);
L
Linus Torvalds 已提交
713

714 715
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
716 717
#ifdef CONFIG_X86_32
# ifdef CONFIG_X86_HT
718
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
719
# else
720
		c->apicid = c->initial_apicid;
721 722
# endif
#endif
L
Linus Torvalds 已提交
723

724 725
#ifdef CONFIG_X86_HT
		c->phys_proc_id = c->initial_apicid;
726
#endif
727
	}
L
Linus Torvalds 已提交
728

729
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
730

731 732
	init_scattered_cpuid_features(c);
	detect_nopl(c);
L
Linus Torvalds 已提交
733 734 735 736 737
}

/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
Y
Yinghai Lu 已提交
738
static void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
739 740 741 742 743 744 745 746 747
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
748
	c->x86_max_cores = 1;
749
	c->x86_coreid_bits = 0;
750
#ifdef CONFIG_X86_64
751
	c->x86_clflush_size = 64;
752 753
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
754 755
#else
	c->cpuid_level = -1;	/* CPUID not detected */
756
	c->x86_clflush_size = 32;
757 758
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
759 760
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
761 762 763 764
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

765
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
766 767
		this_cpu->c_identify(c);

768 769 770 771 772 773
	/* Clear/Set all flags overriden by options, after probe */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

774
#ifdef CONFIG_X86_64
775
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
776 777
#endif

L
Linus Torvalds 已提交
778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

	/*
I
Ingo Molnar 已提交
795 796
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
797 798
	 */

799 800 801
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
802
	/* If the model name is still unset, do table lookup. */
803
	if (!c->x86_model_id[0]) {
804
		const char *p;
L
Linus Torvalds 已提交
805
		p = table_lookup_model(c);
806
		if (p)
L
Linus Torvalds 已提交
807 808 809 810
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
811
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
812 813
	}

814 815 816 817
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

818
	init_hypervisor(c);
819 820 821 822 823 824 825 826 827 828

	/*
	 * Clear/Set all flags overriden by options, need do it
	 * before following smp all cpus cap AND.
	 */
	for (i = 0; i < NCAPINTS; i++) {
		c->x86_capability[i] &= ~cpu_caps_cleared[i];
		c->x86_capability[i] |= cpu_caps_set[i];
	}

L
Linus Torvalds 已提交
829 830 831 832 833 834
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
835
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
836
		/* AND the already accumulated flags with these */
837
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
838 839 840
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
	}

841
#ifdef CONFIG_X86_MCE
L
Linus Torvalds 已提交
842 843
	/* Init Machine Check Exception if available. */
	mcheck_init(c);
844
#endif
845 846

	select_idle_routine(c);
847 848 849 850

#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
	numa_add_cpu(smp_processor_id());
#endif
851
}
S
Shaohua Li 已提交
852

853 854 855 856 857 858 859 860 861 862
#ifdef CONFIG_X86_64
static void vgetcpu_set_mode(void)
{
	if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
		vgetcpu_mode = VGETCPU_RDTSCP;
	else
		vgetcpu_mode = VGETCPU_LSL;
}
#endif

863 864 865
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
866
	init_c1e_mask();
867
#ifdef CONFIG_X86_32
868
	sysenter_setup();
L
Li Shaohua 已提交
869
	enable_sep_cpu();
870 871
#else
	vgetcpu_set_mode();
872
#endif
I
Ingo Molnar 已提交
873
	init_hw_perf_counters();
874
}
S
Shaohua Li 已提交
875

876 877 878 879
void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
880
#ifdef CONFIG_X86_32
881
	enable_sep_cpu();
882
#endif
883
	mtrr_ap_init();
L
Linus Torvalds 已提交
884 885
}

886
struct msr_range {
I
Ingo Molnar 已提交
887 888
	unsigned	min;
	unsigned	max;
889
};
L
Linus Torvalds 已提交
890

891
static const struct msr_range msr_range_array[] __cpuinitconst = {
892 893 894 895 896
	{ 0x00000000, 0x00000418},
	{ 0xc0000000, 0xc000040b},
	{ 0xc0010000, 0xc0010142},
	{ 0xc0011000, 0xc001103b},
};
L
Linus Torvalds 已提交
897

898 899
static void __cpuinit print_cpu_msr(void)
{
I
Ingo Molnar 已提交
900
	unsigned index_min, index_max;
901 902 903 904 905 906 907
	unsigned index;
	u64 val;
	int i;

	for (i = 0; i < ARRAY_SIZE(msr_range_array); i++) {
		index_min = msr_range_array[i].min;
		index_max = msr_range_array[i].max;
I
Ingo Molnar 已提交
908

909 910 911 912
		for (index = index_min; index < index_max; index++) {
			if (rdmsrl_amd_safe(index, &val))
				continue;
			printk(KERN_INFO " MSR%08x: %016llx\n", index, val);
L
Linus Torvalds 已提交
913
		}
914 915
	}
}
916

917
static int show_msr __cpuinitdata;
I
Ingo Molnar 已提交
918

919 920 921
static __init int setup_show_msr(char *arg)
{
	int num;
922

923
	get_option(&arg, &num);
924

925 926 927
	if (num > 0)
		show_msr = num;
	return 1;
L
Linus Torvalds 已提交
928
}
929
__setup("show_msr=", setup_show_msr);
L
Linus Torvalds 已提交
930

A
Andi Kleen 已提交
931 932 933 934 935 936 937
static __init int setup_noclflush(char *arg)
{
	setup_clear_cpu_cap(X86_FEATURE_CLFLSH);
	return 1;
}
__setup("noclflush", setup_noclflush);

938
void __cpuinit print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
939
{
940
	const char *vendor = NULL;
L
Linus Torvalds 已提交
941

I
Ingo Molnar 已提交
942
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
943
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
944 945 946 947
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
948

949
	if (vendor && !strstr(c->x86_model_id, vendor))
950
		printk(KERN_CONT "%s ", vendor);
L
Linus Torvalds 已提交
951

952 953
	if (c->x86_model_id[0])
		printk(KERN_CONT "%s", c->x86_model_id);
L
Linus Torvalds 已提交
954
	else
955
		printk(KERN_CONT "%d86", c->x86);
L
Linus Torvalds 已提交
956

957
	if (c->x86_mask || c->cpuid_level >= 0)
958
		printk(KERN_CONT " stepping %02x\n", c->x86_mask);
L
Linus Torvalds 已提交
959
	else
960
		printk(KERN_CONT "\n");
961 962 963 964 965 966 967 968

#ifdef CONFIG_SMP
	if (c->cpu_index < show_msr)
		print_cpu_msr();
#else
	if (show_msr)
		print_cpu_msr();
#endif
L
Linus Torvalds 已提交
969 970
}

971 972 973
static __init int setup_disablecpuid(char *arg)
{
	int bit;
I
Ingo Molnar 已提交
974

975 976 977 978
	if (get_option(&arg, &bit) && bit < NCAPINTS*32)
		setup_clear_cpu_cap(bit);
	else
		return 0;
I
Ingo Molnar 已提交
979

980 981 982 983
	return 1;
}
__setup("clearcpuid=", setup_disablecpuid);

984 985 986
#ifdef CONFIG_X86_64
struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };

987 988
DEFINE_PER_CPU_FIRST(union irq_stack_union,
		     irq_stack_union) __aligned(PAGE_SIZE);
I
Ingo Molnar 已提交
989

990
DEFINE_PER_CPU(char *, irq_stack_ptr) =
991
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
992

993 994 995 996
DEFINE_PER_CPU(unsigned long, kernel_stack) =
	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(kernel_stack);

997
DEFINE_PER_CPU(unsigned int, irq_count) = -1;
998

I
Ingo Molnar 已提交
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

1010
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
1011
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
1012 1013 1014

/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
1015
{
1016 1017 1018 1019 1020 1021 1022 1023
	/*
	 * LSTAR and STAR live in a bit strange symbiosis.
	 * They both write to the same internal register. STAR allows to
	 * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip.
	 */
	wrmsrl(MSR_STAR,  ((u64)__USER32_CS)<<48  | ((u64)__KERNEL_CS)<<32);
	wrmsrl(MSR_LSTAR, system_call);
	wrmsrl(MSR_CSTAR, ignore_sysret);
1024

1025 1026 1027
#ifdef CONFIG_IA32_EMULATION
	syscall32_cpu_init();
#endif
1028

1029 1030 1031
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL);
L
Linus Torvalds 已提交
1032
}
1033

1034 1035 1036 1037 1038 1039 1040 1041
unsigned long kernel_eflags;

/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

I
Ingo Molnar 已提交
1042
#else	/* CONFIG_X86_64 */
1043

1044 1045 1046
#ifdef CONFIG_CC_STACKPROTECTOR
DEFINE_PER_CPU(unsigned long, stack_canary);
#endif
1047

1048
/* Make sure %fs and %gs are initialized properly in idle threads */
A
Adrian Bunk 已提交
1049
struct pt_regs * __cpuinit idle_regs(struct pt_regs *regs)
1050 1051
{
	memset(regs, 0, sizeof(struct pt_regs));
1052
	regs->fs = __KERNEL_PERCPU;
1053
	regs->gs = __KERNEL_STACK_CANARY;
I
Ingo Molnar 已提交
1054

1055 1056
	return regs;
}
I
Ingo Molnar 已提交
1057
#endif	/* CONFIG_X86_64 */
1058

1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1074

1075 1076 1077 1078 1079
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1080
 * A lot of state is already set up in PDA init for 64 bit
1081
 */
1082
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1083

1084 1085
void __cpuinit cpu_init(void)
{
I
Ingo Molnar 已提交
1086
	struct orig_ist *orig_ist;
1087
	struct task_struct *me;
I
Ingo Molnar 已提交
1088 1089 1090
	struct tss_struct *t;
	unsigned long v;
	int cpu;
1091 1092
	int i;

I
Ingo Molnar 已提交
1093 1094 1095 1096
	cpu = stack_smp_processor_id();
	t = &per_cpu(init_tss, cpu);
	orig_ist = &per_cpu(orig_ist, cpu);

1097 1098 1099 1100 1101
#ifdef CONFIG_NUMA
	if (cpu != 0 && percpu_read(node_number) == 0 &&
	    cpu_to_node(cpu) != NUMA_NO_NODE)
		percpu_write(node_number, cpu_to_node(cpu));
#endif
1102 1103 1104

	me = current;

1105
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask))
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
		panic("CPU#%d already initialized!\n", cpu);

	printk(KERN_INFO "Initializing CPU#%d\n", cpu);

	clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1117
	switch_to_new_gdt(cpu);
1118 1119
	loadsegment(fs, 0);

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	load_idt((const struct desc_ptr *)&idt_descr);

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

	check_efer();
Y
Yinghai Lu 已提交
1130
	if (cpu != 0)
1131 1132 1133 1134 1135 1136
		enable_x2apic();

	/*
	 * set up and load the per-CPU TSS
	 */
	if (!orig_ist->ist[0]) {
1137
		char *estacks = per_cpu(exception_stacks, cpu);
I
Ingo Molnar 已提交
1138

1139
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1140
			estacks += exception_stack_sizes[v];
1141 1142 1143 1144 1145 1146
			orig_ist->ist[v] = t->x86_tss.ist[v] =
					(unsigned long)estacks;
		}
	}

	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
I
Ingo Molnar 已提交
1147

1148 1149 1150 1151 1152 1153 1154 1155 1156
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

	atomic_inc(&init_mm.mm_count);
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1157
	BUG_ON(me->mm);
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
	enter_lazy_tlb(&init_mm, me);

	load_sp0(t, &current->thread);
	set_tss_desc(cpu, t);
	load_TR_desc();
	load_LDT(&init_mm.context);

#ifdef CONFIG_KGDB
	/*
	 * If the kgdb is connected no debug regs should be altered.  This
	 * is only applicable when KGDB and a KGDB I/O module are built
	 * into the kernel and you are using early debugging with
	 * kgdbwait. KGDB will control the kernel HW breakpoint registers.
	 */
	if (kgdb_connected && arch_kgdb_ops.correct_hw_break)
		arch_kgdb_ops.correct_hw_break();
1174
	else
1175
#endif
1176
		clear_all_debug_regs();
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187

	fpu_init();

	raw_local_save_flags(kernel_eflags);

	if (is_uv_system())
		uv_cpu_init();
}

#else

1188
void __cpuinit cpu_init(void)
1189
{
1190 1191
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1192
	struct tss_struct *t = &per_cpu(init_tss, cpu);
1193
	struct thread_struct *thread = &curr->thread;
1194

1195
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) {
1196
		printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
1197 1198
		for (;;)
			local_irq_enable();
1199 1200 1201 1202 1203 1204 1205
	}

	printk(KERN_INFO "Initializing CPU#%d\n", cpu);

	if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
		clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

1206
	load_idt(&idt_descr);
1207
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1208 1209 1210 1211 1212

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
	atomic_inc(&init_mm.mm_count);
1213
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1214
	BUG_ON(curr->mm);
1215
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1216

1217
	load_sp0(t, thread);
1218
	set_tss_desc(cpu, t);
L
Linus Torvalds 已提交
1219 1220 1221
	load_TR_desc();
	load_LDT(&init_mm.context);

1222 1223
	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);

1224
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1225 1226
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1227
#endif
L
Linus Torvalds 已提交
1228

1229
	clear_all_debug_regs();
L
Linus Torvalds 已提交
1230 1231 1232 1233

	/*
	 * Force FPU initialization:
	 */
1234 1235 1236 1237
	if (cpu_has_xsave)
		current_thread_info()->status = TS_XSAVE;
	else
		current_thread_info()->status = 0;
L
Linus Torvalds 已提交
1238 1239
	clear_used_math();
	mxcsr_feature_mask_init();
1240 1241 1242 1243

	/*
	 * Boot processor to setup the FP and extended state context info.
	 */
1244
	if (smp_processor_id() == boot_cpu_id)
1245 1246 1247
		init_thread_xstate();

	xsave_init();
L
Linus Torvalds 已提交
1248
}
1249
#endif