common.c 28.9 KB
Newer Older
1
#include <linux/bootmem.h>
2
#include <linux/linkage.h>
3
#include <linux/bitops.h>
4
#include <linux/kernel.h>
5
#include <linux/module.h>
6 7
#include <linux/percpu.h>
#include <linux/string.h>
L
Linus Torvalds 已提交
8
#include <linux/delay.h>
9 10 11
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/kgdb.h>
L
Linus Torvalds 已提交
12
#include <linux/smp.h>
13 14 15
#include <linux/io.h>

#include <asm/stackprotector.h>
L
Linus Torvalds 已提交
16
#include <asm/mmu_context.h>
17 18 19
#include <asm/hypervisor.h>
#include <asm/processor.h>
#include <asm/sections.h>
I
Ingo Molnar 已提交
20
#include <asm/topology.h>
21 22 23 24 25 26 27 28
#include <asm/cpumask.h>
#include <asm/pgtable.h>
#include <asm/atomic.h>
#include <asm/proto.h>
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/i387.h>
29
#include <asm/mtrr.h>
30 31 32
#include <asm/numa.h>
#include <asm/asm.h>
#include <asm/cpu.h>
33
#include <asm/mce.h>
34
#include <asm/msr.h>
35
#include <asm/pat.h>
I
Ingo Molnar 已提交
36
#include <asm/smp.h>
37 38

#ifdef CONFIG_X86_LOCAL_APIC
T
Tejun Heo 已提交
39
#include <asm/uv/uv.h>
L
Linus Torvalds 已提交
40 41 42 43
#endif

#include "cpu.h"

44 45
/* all of these masks are initialized in setup_cpu_local_masks() */
cpumask_var_t cpu_initialized_mask;
46 47
cpumask_var_t cpu_callout_mask;
cpumask_var_t cpu_callin_mask;
48 49 50 51

/* representing cpus for which sibling maps can be computed */
cpumask_var_t cpu_sibling_setup_mask;

B
Brian Gerst 已提交
52
/* correctly size the local cpu masks */
53
void __init setup_cpu_local_masks(void)
B
Brian Gerst 已提交
54 55 56 57 58 59 60
{
	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
	alloc_bootmem_cpumask_var(&cpu_callin_mask);
	alloc_bootmem_cpumask_var(&cpu_callout_mask);
	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
}

61
static const struct cpu_dev *this_cpu __cpuinitdata;
62

B
Brian Gerst 已提交
63
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
Y
Yinghai Lu 已提交
64
#ifdef CONFIG_X86_64
B
Brian Gerst 已提交
65 66 67 68 69
	/*
	 * We need valid kernel segments for data and code in long mode too
	 * IRET will check the segment types  kkeil 2000/10/28
	 * Also sysret mandates a special GDT layout
	 *
70
	 * TLS descriptors are currently at a different place compared to i386.
B
Brian Gerst 已提交
71 72
	 * Hopefully nobody expects them at a fixed place (Wine?)
	 */
I
Ingo Molnar 已提交
73 74 75 76 77 78
	[GDT_ENTRY_KERNEL32_CS]		= { { { 0x0000ffff, 0x00cf9b00 } } },
	[GDT_ENTRY_KERNEL_CS]		= { { { 0x0000ffff, 0x00af9b00 } } },
	[GDT_ENTRY_KERNEL_DS]		= { { { 0x0000ffff, 0x00cf9300 } } },
	[GDT_ENTRY_DEFAULT_USER32_CS]	= { { { 0x0000ffff, 0x00cffb00 } } },
	[GDT_ENTRY_DEFAULT_USER_DS]	= { { { 0x0000ffff, 0x00cff300 } } },
	[GDT_ENTRY_DEFAULT_USER_CS]	= { { { 0x0000ffff, 0x00affb00 } } },
Y
Yinghai Lu 已提交
79
#else
I
Ingo Molnar 已提交
80 81 82 83
	[GDT_ENTRY_KERNEL_CS]		= { { { 0x0000ffff, 0x00cf9a00 } } },
	[GDT_ENTRY_KERNEL_DS]		= { { { 0x0000ffff, 0x00cf9200 } } },
	[GDT_ENTRY_DEFAULT_USER_CS]	= { { { 0x0000ffff, 0x00cffa00 } } },
	[GDT_ENTRY_DEFAULT_USER_DS]	= { { { 0x0000ffff, 0x00cff200 } } },
84 85 86 87 88
	/*
	 * Segments used for calling PnP BIOS have byte granularity.
	 * They code segments and data segments have fixed 64k limits,
	 * the transfer segment sizes are set at run time.
	 */
89
	/* 32-bit code */
I
Ingo Molnar 已提交
90
	[GDT_ENTRY_PNPBIOS_CS32]	= { { { 0x0000ffff, 0x00409a00 } } },
91
	/* 16-bit code */
I
Ingo Molnar 已提交
92
	[GDT_ENTRY_PNPBIOS_CS16]	= { { { 0x0000ffff, 0x00009a00 } } },
93
	/* 16-bit data */
I
Ingo Molnar 已提交
94
	[GDT_ENTRY_PNPBIOS_DS]		= { { { 0x0000ffff, 0x00009200 } } },
95
	/* 16-bit data */
I
Ingo Molnar 已提交
96
	[GDT_ENTRY_PNPBIOS_TS1]		= { { { 0x00000000, 0x00009200 } } },
97
	/* 16-bit data */
I
Ingo Molnar 已提交
98
	[GDT_ENTRY_PNPBIOS_TS2]		= { { { 0x00000000, 0x00009200 } } },
99 100 101 102
	/*
	 * The APM segments have byte granularity and their bases
	 * are set at run time.  All have 64k limits.
	 */
103
	/* 32-bit code */
I
Ingo Molnar 已提交
104
	[GDT_ENTRY_APMBIOS_BASE]	= { { { 0x0000ffff, 0x00409a00 } } },
105
	/* 16-bit code */
I
Ingo Molnar 已提交
106
	[GDT_ENTRY_APMBIOS_BASE+1]	= { { { 0x0000ffff, 0x00009a00 } } },
107
	/* data */
I
Ingo Molnar 已提交
108
	[GDT_ENTRY_APMBIOS_BASE+2]	= { { { 0x0000ffff, 0x00409200 } } },
109

I
Ingo Molnar 已提交
110 111
	[GDT_ENTRY_ESPFIX_SS]		= { { { 0x00000000, 0x00c09200 } } },
	[GDT_ENTRY_PERCPU]		= { { { 0x0000ffff, 0x00cf9200 } } },
112
	GDT_STACK_CANARY_INIT
Y
Yinghai Lu 已提交
113
#endif
B
Brian Gerst 已提交
114
} };
115
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
116

117
#ifdef CONFIG_X86_32
118 119
static int cachesize_override __cpuinitdata = -1;
static int disable_x86_serial_nr __cpuinitdata = 1;
L
Linus Torvalds 已提交
120

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
static int __init cachesize_setup(char *str)
{
	get_option(&str, &cachesize_override);
	return 1;
}
__setup("cachesize=", cachesize_setup);

static int __init x86_fxsr_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_FXSR);
	setup_clear_cpu_cap(X86_FEATURE_XMM);
	return 1;
}
__setup("nofxsr", x86_fxsr_setup);

static int __init x86_sep_setup(char *s)
{
	setup_clear_cpu_cap(X86_FEATURE_SEP);
	return 1;
}
__setup("nosep", x86_sep_setup);

/* Standard macro to see if a specific flag is changeable */
static inline int flag_is_changeable_p(u32 flag)
{
	u32 f1, f2;

148 149 150 151 152 153 154
	/*
	 * Cyrix and IDT cpus allow disabling of CPUID
	 * so the code below may return different results
	 * when it is executed before and after enabling
	 * the CPUID. Add "volatile" to not allow gcc to
	 * optimize the subsequent calls to this function.
	 */
I
Ingo Molnar 已提交
155 156 157 158 159 160 161 162 163 164 165
	asm volatile ("pushfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "movl %0, %1	\n\t"
		      "xorl %2, %0	\n\t"
		      "pushl %0		\n\t"
		      "popfl		\n\t"
		      "pushfl		\n\t"
		      "popl %0		\n\t"
		      "popfl		\n\t"

166 167
		      : "=&r" (f1), "=&r" (f2)
		      : "ir" (flag));
168 169 170 171 172 173 174 175 176 177 178 179

	return ((f1^f2) & flag) != 0;
}

/* Probe for the CPUID instruction */
static int __cpuinit have_cpuid_p(void)
{
	return flag_is_changeable_p(X86_EFLAGS_ID);
}

static void __cpuinit squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
I
Ingo Molnar 已提交
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
	unsigned long lo, hi;

	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
		return;

	/* Disable processor serial number: */

	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
	lo |= 0x200000;
	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);

	printk(KERN_NOTICE "CPU serial number disabled.\n");
	clear_cpu_cap(c, X86_FEATURE_PN);

	/* Disabling the serial number may affect the cpuid level */
	c->cpuid_level = cpuid_eax(0);
196 197 198 199 200 201 202 203
}

static int __init x86_serial_nr_setup(char *s)
{
	disable_x86_serial_nr = 0;
	return 1;
}
__setup("serialnumber", x86_serial_nr_setup);
204
#else
205 206 207 208
static inline int flag_is_changeable_p(u32 flag)
{
	return 1;
}
209 210 211 212 213
/* Probe for the CPUID instruction */
static inline int have_cpuid_p(void)
{
	return 1;
}
214 215 216
static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
{
}
217
#endif
218

219 220 221 222 223 224 225 226 227
/*
 * Some CPU features depend on higher CPUID levels, which may not always
 * be available due to CPUID level capping or broken virtualization
 * software.  Add those features to this table to auto-disable them.
 */
struct cpuid_dependent_feature {
	u32 feature;
	u32 level;
};
I
Ingo Molnar 已提交
228

229 230 231 232 233 234 235 236 237 238 239
static const struct cpuid_dependent_feature __cpuinitconst
cpuid_dependent_features[] = {
	{ X86_FEATURE_MWAIT,		0x00000005 },
	{ X86_FEATURE_DCA,		0x00000009 },
	{ X86_FEATURE_XSAVE,		0x0000000d },
	{ 0, 0 }
};

static void __cpuinit filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
{
	const struct cpuid_dependent_feature *df;
240

241
	for (df = cpuid_dependent_features; df->feature; df++) {
I
Ingo Molnar 已提交
242 243 244

		if (!cpu_has(c, df->feature))
			continue;
245 246 247 248 249 250 251
		/*
		 * Note: cpuid_level is set to -1 if unavailable, but
		 * extended_extended_level is set to 0 if unavailable
		 * and the legitimate extended levels are all negative
		 * when signed; hence the weird messing around with
		 * signs here...
		 */
I
Ingo Molnar 已提交
252
		if (!((s32)df->level < 0 ?
253
		     (u32)df->level > (u32)c->extended_cpuid_level :
I
Ingo Molnar 已提交
254 255 256 257 258 259 260 261 262 263
		     (s32)df->level > (s32)c->cpuid_level))
			continue;

		clear_cpu_cap(c, df->feature);
		if (!warn)
			continue;

		printk(KERN_WARNING
		       "CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
				x86_cap_flags[df->feature], df->level);
264
	}
265
}
266

267 268 269
/*
 * Naming convention should be: <Name> [(<Codename>)]
 * This table only is used unless init_<vendor>() below doesn't set it;
I
Ingo Molnar 已提交
270 271
 * in particular, if CPUID levels 0x80000002..4 are supported, this
 * isn't used
272 273 274
 */

/* Look up CPU names by table lookup. */
275
static const char *__cpuinit table_lookup_model(struct cpuinfo_x86 *c)
276
{
277
	const struct cpu_model_info *info;
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294

	if (c->x86_model >= 16)
		return NULL;	/* Range check */

	if (!this_cpu)
		return NULL;

	info = this_cpu->c_models;

	while (info && info->family) {
		if (info->family == c->x86)
			return info->model_names[c->x86_model];
		info++;
	}
	return NULL;		/* Not found */
}

295 296
__u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;

297 298 299 300 301 302 303 304
void load_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	loadsegment(fs, __KERNEL_PERCPU);
#else
	loadsegment(gs, 0);
	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
#endif
305
	load_stack_canary_segment();
306 307
}

I
Ingo Molnar 已提交
308 309 310 311
/*
 * Current gdt points %fs at the "master" per-cpu area: after this,
 * it's on the real one.
 */
312
void switch_to_new_gdt(int cpu)
313 314 315
{
	struct desc_ptr gdt_descr;

316
	gdt_descr.address = (long)get_cpu_gdt_table(cpu);
317 318
	gdt_descr.size = GDT_SIZE - 1;
	load_gdt(&gdt_descr);
319
	/* Reload the per-cpu base */
320 321

	load_percpu_segment(cpu);
322 323
}

324
static const struct cpu_dev *__cpuinitdata cpu_devs[X86_VENDOR_NUM] = {};
L
Linus Torvalds 已提交
325

326
static void __cpuinit default_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
327
{
328 329 330
#ifdef CONFIG_X86_64
	display_cacheinfo(c);
#else
L
Linus Torvalds 已提交
331 332 333 334 335 336 337 338 339
	/* Not much we can do here... */
	/* Check if at least it has cpuid */
	if (c->cpuid_level == -1) {
		/* No cpuid. It must be an ancient CPU */
		if (c->x86 == 4)
			strcpy(c->x86_model_id, "486");
		else if (c->x86 == 3)
			strcpy(c->x86_model_id, "386");
	}
340
#endif
L
Linus Torvalds 已提交
341 342
}

343
static const struct cpu_dev __cpuinitconst default_cpu = {
L
Linus Torvalds 已提交
344
	.c_init	= default_init,
345
	.c_vendor = "Unknown",
Y
Yinghai Lu 已提交
346
	.c_x86_vendor = X86_VENDOR_UNKNOWN,
L
Linus Torvalds 已提交
347 348
};

349
static void __cpuinit get_model_name(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
350 351 352 353
{
	unsigned int *v;
	char *p, *q;

354
	if (c->extended_cpuid_level < 0x80000004)
355
		return;
L
Linus Torvalds 已提交
356

I
Ingo Molnar 已提交
357
	v = (unsigned int *)c->x86_model_id;
L
Linus Torvalds 已提交
358 359 360 361 362
	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
	c->x86_model_id[48] = 0;

I
Ingo Molnar 已提交
363 364 365 366
	/*
	 * Intel chips right-justify this string for some dumb reason;
	 * undo that brain damage:
	 */
L
Linus Torvalds 已提交
367
	p = q = &c->x86_model_id[0];
368
	while (*p == ' ')
369
		p++;
370
	if (p != q) {
371 372 373 374
		while (*p)
			*q++ = *p++;
		while (q <= &c->x86_model_id[48])
			*q++ = '\0';	/* Zero-pad the rest */
L
Linus Torvalds 已提交
375 376 377
	}
}

378
void __cpuinit display_cacheinfo(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
379
{
380
	unsigned int n, dummy, ebx, ecx, edx, l2size;
L
Linus Torvalds 已提交
381

382
	n = c->extended_cpuid_level;
L
Linus Torvalds 已提交
383 384

	if (n >= 0x80000005) {
385
		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
386
		printk(KERN_INFO "CPU: L1 I Cache: %dK (%d bytes/line), D cache %dK (%d bytes/line)\n",
387 388
				edx>>24, edx&0xFF, ecx>>24, ecx&0xFF);
		c->x86_cache_size = (ecx>>24) + (edx>>24);
389 390 391 392
#ifdef CONFIG_X86_64
		/* On K8 L1 TLB is inclusive, so don't count it */
		c->x86_tlbsize = 0;
#endif
L
Linus Torvalds 已提交
393 394 395 396 397
	}

	if (n < 0x80000006)	/* Some chips just has a large L1. */
		return;

398
	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
399
	l2size = ecx >> 16;
400

401 402 403
#ifdef CONFIG_X86_64
	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
#else
L
Linus Torvalds 已提交
404 405
	/* do processor-specific cache resizing */
	if (this_cpu->c_size_cache)
406
		l2size = this_cpu->c_size_cache(c, l2size);
L
Linus Torvalds 已提交
407 408 409 410 411

	/* Allow user to override all this if necessary. */
	if (cachesize_override != -1)
		l2size = cachesize_override;

412
	if (l2size == 0)
L
Linus Torvalds 已提交
413
		return;		/* Again, no L2 cache is possible */
414
#endif
L
Linus Torvalds 已提交
415 416 417 418

	c->x86_cache_size = l2size;

	printk(KERN_INFO "CPU: L2 Cache: %dK (%d bytes/line)\n",
419
			l2size, ecx & 0xFF);
L
Linus Torvalds 已提交
420 421
}

422
void __cpuinit detect_ht(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
423
{
424
#ifdef CONFIG_X86_HT
425 426
	u32 eax, ebx, ecx, edx;
	int index_msb, core_bits;
L
Linus Torvalds 已提交
427

428
	if (!cpu_has(c, X86_FEATURE_HT))
429
		return;
L
Linus Torvalds 已提交
430

431 432
	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
		goto out;
L
Linus Torvalds 已提交
433

434 435
	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
		return;
L
Linus Torvalds 已提交
436

437
	cpuid(1, &eax, &ebx, &ecx, &edx);
L
Linus Torvalds 已提交
438

439 440 441 442
	smp_num_siblings = (ebx & 0xff0000) >> 16;

	if (smp_num_siblings == 1) {
		printk(KERN_INFO  "CPU: Hyper-Threading is disabled\n");
I
Ingo Molnar 已提交
443 444
		goto out;
	}
445

I
Ingo Molnar 已提交
446 447
	if (smp_num_siblings <= 1)
		goto out;
448

I
Ingo Molnar 已提交
449 450 451 452 453 454
	if (smp_num_siblings > nr_cpu_ids) {
		pr_warning("CPU: Unsupported number of siblings %d",
			   smp_num_siblings);
		smp_num_siblings = 1;
		return;
	}
455

I
Ingo Molnar 已提交
456 457
	index_msb = get_count_order(smp_num_siblings);
	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
458

I
Ingo Molnar 已提交
459
	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
460

I
Ingo Molnar 已提交
461
	index_msb = get_count_order(smp_num_siblings);
462

I
Ingo Molnar 已提交
463
	core_bits = get_count_order(c->x86_max_cores);
464

I
Ingo Molnar 已提交
465 466
	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
				       ((1 << core_bits) - 1);
L
Linus Torvalds 已提交
467

468 469 470 471 472 473
out:
	if ((c->x86_max_cores * smp_num_siblings) > 1) {
		printk(KERN_INFO  "CPU: Physical Processor ID: %d\n",
		       c->phys_proc_id);
		printk(KERN_INFO  "CPU: Processor Core ID: %d\n",
		       c->cpu_core_id);
474 475
	}
#endif
476
}
L
Linus Torvalds 已提交
477

478
static void __cpuinit get_cpu_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
479 480
{
	char *v = c->x86_vendor_id;
481
	static int printed;
I
Ingo Molnar 已提交
482
	int i;
L
Linus Torvalds 已提交
483 484

	for (i = 0; i < X86_VENDOR_NUM; i++) {
Y
Yinghai Lu 已提交
485 486 487 488 489 490
		if (!cpu_devs[i])
			break;

		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
		    (cpu_devs[i]->c_ident[1] &&
		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
I
Ingo Molnar 已提交
491

Y
Yinghai Lu 已提交
492 493 494
			this_cpu = cpu_devs[i];
			c->x86_vendor = this_cpu->c_x86_vendor;
			return;
L
Linus Torvalds 已提交
495 496
		}
	}
Y
Yinghai Lu 已提交
497

498 499
	if (!printed) {
		printed++;
I
Ingo Molnar 已提交
500 501 502
		printk(KERN_ERR
		    "CPU: vendor_id '%s' unknown, using generic init.\n", v);

503 504
		printk(KERN_ERR "CPU: Your system may be unstable.\n");
	}
Y
Yinghai Lu 已提交
505

506 507
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	this_cpu = &default_cpu;
L
Linus Torvalds 已提交
508 509
}

510
void __cpuinit cpu_detect(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
511 512
{
	/* Get vendor name */
513 514 515 516
	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
	      (unsigned int *)&c->x86_vendor_id[0],
	      (unsigned int *)&c->x86_vendor_id[8],
	      (unsigned int *)&c->x86_vendor_id[4]);
L
Linus Torvalds 已提交
517 518

	c->x86 = 4;
519
	/* Intel-defined flags: level 0x00000001 */
L
Linus Torvalds 已提交
520 521
	if (c->cpuid_level >= 0x00000001) {
		u32 junk, tfms, cap0, misc;
I
Ingo Molnar 已提交
522

L
Linus Torvalds 已提交
523
		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
524 525 526
		c->x86 = (tfms >> 8) & 0xf;
		c->x86_model = (tfms >> 4) & 0xf;
		c->x86_mask = tfms & 0xf;
I
Ingo Molnar 已提交
527

528
		if (c->x86 == 0xf)
L
Linus Torvalds 已提交
529
			c->x86 += (tfms >> 20) & 0xff;
530
		if (c->x86 >= 0x6)
531
			c->x86_model += ((tfms >> 16) & 0xf) << 4;
I
Ingo Molnar 已提交
532

H
Huang, Ying 已提交
533 534
		if (cap0 & (1<<19)) {
			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
535
			c->x86_cache_alignment = c->x86_clflush_size;
H
Huang, Ying 已提交
536
		}
L
Linus Torvalds 已提交
537 538
	}
}
539 540

static void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c)
541 542
{
	u32 tfms, xlvl;
543
	u32 ebx;
544

545 546 547
	/* Intel-defined flags: level 0x00000001 */
	if (c->cpuid_level >= 0x00000001) {
		u32 capability, excap;
I
Ingo Molnar 已提交
548

549 550 551 552
		cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
		c->x86_capability[0] = capability;
		c->x86_capability[4] = excap;
	}
553

554 555 556
	/* AMD-defined flags: level 0x80000001 */
	xlvl = cpuid_eax(0x80000000);
	c->extended_cpuid_level = xlvl;
I
Ingo Molnar 已提交
557

558 559 560 561
	if ((xlvl & 0xffff0000) == 0x80000000) {
		if (xlvl >= 0x80000001) {
			c->x86_capability[1] = cpuid_edx(0x80000001);
			c->x86_capability[6] = cpuid_ecx(0x80000001);
562 563 564
		}
	}

565 566 567 568 569
	if (c->extended_cpuid_level >= 0x80000008) {
		u32 eax = cpuid_eax(0x80000008);

		c->x86_virt_bits = (eax >> 8) & 0xff;
		c->x86_phys_bits = eax & 0xff;
570
	}
571 572 573
#ifdef CONFIG_X86_32
	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
		c->x86_phys_bits = 36;
574
#endif
575 576 577

	if (c->extended_cpuid_level >= 0x80000007)
		c->x86_power = cpuid_edx(0x80000007);
578 579

}
L
Linus Torvalds 已提交
580

Y
Yinghai Lu 已提交
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
static void __cpuinit identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_32
	int i;

	/*
	 * First of all, decide if this is a 486 or higher
	 * It's a 486 if we can modify the AC flag
	 */
	if (flag_is_changeable_p(X86_EFLAGS_AC))
		c->x86 = 4;
	else
		c->x86 = 3;

	for (i = 0; i < X86_VENDOR_NUM; i++)
		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
			c->x86_vendor_id[0] = 0;
			cpu_devs[i]->c_identify(c);
			if (c->x86_vendor_id[0]) {
				get_cpu_vendor(c);
				break;
			}
		}
#endif
}

607 608 609 610 611 612 613 614 615
/*
 * Do minimum CPU detection early.
 * Fields really needed: vendor, cpuid_level, family, model, mask,
 * cache alignment.
 * The others are not touched to avoid unwanted side effects.
 *
 * WARNING: this function is only called on the BP.  Don't add code here
 * that is supposed to run on all CPUs.
 */
616
static void __init early_identify_cpu(struct cpuinfo_x86 *c)
617
{
618 619
#ifdef CONFIG_X86_64
	c->x86_clflush_size = 64;
620 621
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
622
#else
H
Huang, Ying 已提交
623
	c->x86_clflush_size = 32;
624 625
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
626
#endif
627
	c->x86_cache_alignment = c->x86_clflush_size;
628

629
	memset(&c->x86_capability, 0, sizeof c->x86_capability);
630
	c->extended_cpuid_level = 0;
631

Y
Yinghai Lu 已提交
632 633 634 635
	if (!have_cpuid_p())
		identify_cpu_without_cpuid(c);

	/* cyrix could have cpuid enabled via c_identify()*/
636 637 638 639 640
	if (!have_cpuid_p())
		return;

	cpu_detect(c);

641
	get_cpu_vendor(c);
642

643
	get_cpu_cap(c);
644

Y
Yinghai Lu 已提交
645 646
	if (this_cpu->c_early_init)
		this_cpu->c_early_init(c);
647

I
Ingo Molnar 已提交
648
#ifdef CONFIG_SMP
649
	c->cpu_index = boot_cpu_id;
I
Ingo Molnar 已提交
650
#endif
651
	filter_cpuid_features(c, false);
652 653
}

654 655
void __init early_cpu_init(void)
{
656
	const struct cpu_dev *const *cdev;
Y
Yinghai Lu 已提交
657 658
	int count = 0;

659
	printk(KERN_INFO "KERNEL supported cpus:\n");
Y
Yinghai Lu 已提交
660
	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
661
		const struct cpu_dev *cpudev = *cdev;
Y
Yinghai Lu 已提交
662
		unsigned int j;
663

Y
Yinghai Lu 已提交
664 665 666 667 668 669 670 671
		if (count >= X86_VENDOR_NUM)
			break;
		cpu_devs[count] = cpudev;
		count++;

		for (j = 0; j < 2; j++) {
			if (!cpudev->c_ident[j])
				continue;
672
			printk(KERN_INFO "  %s %s\n", cpudev->c_vendor,
Y
Yinghai Lu 已提交
673 674 675
				cpudev->c_ident[j]);
		}
	}
676 677

	early_identify_cpu(&boot_cpu_data);
678
}
679

680 681
/*
 * The NOPL instruction is supposed to exist on all CPUs with
682
 * family >= 6; unfortunately, that's not true in practice because
683
 * of early VIA chips and (more importantly) broken virtualizers that
684 685 686
 * are not easy to detect.  In the latter case it doesn't even *fail*
 * reliably, so probing for it doesn't even work.  Disable it completely
 * unless we can find a reliable way to detect all the broken cases.
687 688 689 690
 */
static void __cpuinit detect_nopl(struct cpuinfo_x86 *c)
{
	clear_cpu_cap(c, X86_FEATURE_NOPL);
691 692
}

693
static void __cpuinit generic_identify(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
694
{
Y
Yinghai Lu 已提交
695
	c->extended_cpuid_level = 0;
L
Linus Torvalds 已提交
696

697
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
698
		identify_cpu_without_cpuid(c);
699

Y
Yinghai Lu 已提交
700
	/* cyrix could have cpuid enabled via c_identify()*/
I
Ingo Molnar 已提交
701
	if (!have_cpuid_p())
Y
Yinghai Lu 已提交
702
		return;
L
Linus Torvalds 已提交
703

704
	cpu_detect(c);
L
Linus Torvalds 已提交
705

706
	get_cpu_vendor(c);
L
Linus Torvalds 已提交
707

708
	get_cpu_cap(c);
L
Linus Torvalds 已提交
709

710 711
	if (c->cpuid_level >= 0x00000001) {
		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
712 713
#ifdef CONFIG_X86_32
# ifdef CONFIG_X86_HT
714
		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
715
# else
716
		c->apicid = c->initial_apicid;
717 718
# endif
#endif
L
Linus Torvalds 已提交
719

720 721
#ifdef CONFIG_X86_HT
		c->phys_proc_id = c->initial_apicid;
722
#endif
723
	}
L
Linus Torvalds 已提交
724

725
	get_model_name(c); /* Default name */
L
Linus Torvalds 已提交
726

727 728
	init_scattered_cpuid_features(c);
	detect_nopl(c);
L
Linus Torvalds 已提交
729 730 731 732 733
}

/*
 * This does the hard work of actually picking apart the CPU stuff...
 */
Y
Yinghai Lu 已提交
734
static void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
735 736 737 738 739 740 741 742 743
{
	int i;

	c->loops_per_jiffy = loops_per_jiffy;
	c->x86_cache_size = -1;
	c->x86_vendor = X86_VENDOR_UNKNOWN;
	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
	c->x86_vendor_id[0] = '\0'; /* Unset */
	c->x86_model_id[0] = '\0';  /* Unset */
744
	c->x86_max_cores = 1;
745
	c->x86_coreid_bits = 0;
746
#ifdef CONFIG_X86_64
747
	c->x86_clflush_size = 64;
748 749
	c->x86_phys_bits = 36;
	c->x86_virt_bits = 48;
750 751
#else
	c->cpuid_level = -1;	/* CPUID not detected */
752
	c->x86_clflush_size = 32;
753 754
	c->x86_phys_bits = 32;
	c->x86_virt_bits = 32;
755 756
#endif
	c->x86_cache_alignment = c->x86_clflush_size;
L
Linus Torvalds 已提交
757 758 759 760
	memset(&c->x86_capability, 0, sizeof c->x86_capability);

	generic_identify(c);

761
	if (this_cpu->c_identify)
L
Linus Torvalds 已提交
762 763
		this_cpu->c_identify(c);

764
#ifdef CONFIG_X86_64
765
	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
766 767
#endif

L
Linus Torvalds 已提交
768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
	/*
	 * Vendor-specific initialization.  In this section we
	 * canonicalize the feature flags, meaning if there are
	 * features a certain CPU supports which CPUID doesn't
	 * tell us, CPUID claiming incorrect flags, or other bugs,
	 * we handle them here.
	 *
	 * At the end of this section, c->x86_capability better
	 * indicate the features this CPU genuinely supports!
	 */
	if (this_cpu->c_init)
		this_cpu->c_init(c);

	/* Disable the PN if appropriate */
	squash_the_stupid_serial_number(c);

	/*
I
Ingo Molnar 已提交
785 786
	 * The vendor-specific functions might have changed features.
	 * Now we do "generic changes."
L
Linus Torvalds 已提交
787 788
	 */

789 790 791
	/* Filter out anything that depends on CPUID levels we don't have */
	filter_cpuid_features(c, true);

L
Linus Torvalds 已提交
792
	/* If the model name is still unset, do table lookup. */
793
	if (!c->x86_model_id[0]) {
794
		const char *p;
L
Linus Torvalds 已提交
795
		p = table_lookup_model(c);
796
		if (p)
L
Linus Torvalds 已提交
797 798 799 800
			strcpy(c->x86_model_id, p);
		else
			/* Last resort... */
			sprintf(c->x86_model_id, "%02x/%02x",
801
				c->x86, c->x86_model);
L
Linus Torvalds 已提交
802 803
	}

804 805 806 807
#ifdef CONFIG_X86_64
	detect_ht(c);
#endif

808
	init_hypervisor(c);
L
Linus Torvalds 已提交
809 810 811 812 813 814
	/*
	 * On SMP, boot_cpu_data holds the common feature set between
	 * all CPUs; so make sure that we indicate which features are
	 * common between the CPUs.  The first time this routine gets
	 * executed, c == &boot_cpu_data.
	 */
815
	if (c != &boot_cpu_data) {
L
Linus Torvalds 已提交
816
		/* AND the already accumulated flags with these */
817
		for (i = 0; i < NCAPINTS; i++)
L
Linus Torvalds 已提交
818 819 820
			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
	}

821 822
	/* Clear all flags overriden by options */
	for (i = 0; i < NCAPINTS; i++)
823
		c->x86_capability[i] &= ~cleared_cpu_caps[i];
824

825
#ifdef CONFIG_X86_MCE
L
Linus Torvalds 已提交
826 827
	/* Init Machine Check Exception if available. */
	mcheck_init(c);
828
#endif
829 830

	select_idle_routine(c);
831 832 833 834

#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
	numa_add_cpu(smp_processor_id());
#endif
835
}
S
Shaohua Li 已提交
836

837 838 839 840 841 842 843 844 845 846
#ifdef CONFIG_X86_64
static void vgetcpu_set_mode(void)
{
	if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP))
		vgetcpu_mode = VGETCPU_RDTSCP;
	else
		vgetcpu_mode = VGETCPU_LSL;
}
#endif

847 848 849
void __init identify_boot_cpu(void)
{
	identify_cpu(&boot_cpu_data);
850
	init_c1e_mask();
851
#ifdef CONFIG_X86_32
852
	sysenter_setup();
L
Li Shaohua 已提交
853
	enable_sep_cpu();
854 855
#else
	vgetcpu_set_mode();
856
#endif
857
}
S
Shaohua Li 已提交
858

859 860 861 862
void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
{
	BUG_ON(c == &boot_cpu_data);
	identify_cpu(c);
863
#ifdef CONFIG_X86_32
864
	enable_sep_cpu();
865
#endif
866
	mtrr_ap_init();
L
Linus Torvalds 已提交
867 868
}

869
struct msr_range {
I
Ingo Molnar 已提交
870 871
	unsigned	min;
	unsigned	max;
872
};
L
Linus Torvalds 已提交
873

874
static const struct msr_range msr_range_array[] __cpuinitconst = {
875 876 877 878 879
	{ 0x00000000, 0x00000418},
	{ 0xc0000000, 0xc000040b},
	{ 0xc0010000, 0xc0010142},
	{ 0xc0011000, 0xc001103b},
};
L
Linus Torvalds 已提交
880

881 882
static void __cpuinit print_cpu_msr(void)
{
I
Ingo Molnar 已提交
883
	unsigned index_min, index_max;
884 885 886 887 888 889 890
	unsigned index;
	u64 val;
	int i;

	for (i = 0; i < ARRAY_SIZE(msr_range_array); i++) {
		index_min = msr_range_array[i].min;
		index_max = msr_range_array[i].max;
I
Ingo Molnar 已提交
891

892 893 894 895
		for (index = index_min; index < index_max; index++) {
			if (rdmsrl_amd_safe(index, &val))
				continue;
			printk(KERN_INFO " MSR%08x: %016llx\n", index, val);
L
Linus Torvalds 已提交
896
		}
897 898
	}
}
899

900
static int show_msr __cpuinitdata;
I
Ingo Molnar 已提交
901

902 903 904
static __init int setup_show_msr(char *arg)
{
	int num;
905

906
	get_option(&arg, &num);
907

908 909 910
	if (num > 0)
		show_msr = num;
	return 1;
L
Linus Torvalds 已提交
911
}
912
__setup("show_msr=", setup_show_msr);
L
Linus Torvalds 已提交
913

A
Andi Kleen 已提交
914 915 916 917 918 919 920
static __init int setup_noclflush(char *arg)
{
	setup_clear_cpu_cap(X86_FEATURE_CLFLSH);
	return 1;
}
__setup("noclflush", setup_noclflush);

921
void __cpuinit print_cpu_info(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
922
{
923
	const char *vendor = NULL;
L
Linus Torvalds 已提交
924

I
Ingo Molnar 已提交
925
	if (c->x86_vendor < X86_VENDOR_NUM) {
L
Linus Torvalds 已提交
926
		vendor = this_cpu->c_vendor;
I
Ingo Molnar 已提交
927 928 929 930
	} else {
		if (c->cpuid_level >= 0)
			vendor = c->x86_vendor_id;
	}
L
Linus Torvalds 已提交
931

932
	if (vendor && !strstr(c->x86_model_id, vendor))
933
		printk(KERN_CONT "%s ", vendor);
L
Linus Torvalds 已提交
934

935 936
	if (c->x86_model_id[0])
		printk(KERN_CONT "%s", c->x86_model_id);
L
Linus Torvalds 已提交
937
	else
938
		printk(KERN_CONT "%d86", c->x86);
L
Linus Torvalds 已提交
939

940
	if (c->x86_mask || c->cpuid_level >= 0)
941
		printk(KERN_CONT " stepping %02x\n", c->x86_mask);
L
Linus Torvalds 已提交
942
	else
943
		printk(KERN_CONT "\n");
944 945 946 947 948 949 950 951

#ifdef CONFIG_SMP
	if (c->cpu_index < show_msr)
		print_cpu_msr();
#else
	if (show_msr)
		print_cpu_msr();
#endif
L
Linus Torvalds 已提交
952 953
}

954 955 956
static __init int setup_disablecpuid(char *arg)
{
	int bit;
I
Ingo Molnar 已提交
957

958 959 960 961
	if (get_option(&arg, &bit) && bit < NCAPINTS*32)
		setup_clear_cpu_cap(bit);
	else
		return 0;
I
Ingo Molnar 已提交
962

963 964 965 966
	return 1;
}
__setup("clearcpuid=", setup_disablecpuid);

967 968 969
#ifdef CONFIG_X86_64
struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };

970 971
DEFINE_PER_CPU_FIRST(union irq_stack_union,
		     irq_stack_union) __aligned(PAGE_SIZE);
I
Ingo Molnar 已提交
972

973
DEFINE_PER_CPU(char *, irq_stack_ptr) =
974
	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
975

976 977 978
DEFINE_PER_CPU(unsigned long, kernel_stack) =
	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(kernel_stack);
979

980
DEFINE_PER_CPU(unsigned int, irq_count) = -1;
981

I
Ingo Molnar 已提交
982 983 984 985 986 987 988 989 990 991 992
/*
 * Special IST stacks which the CPU switches to when it calls
 * an IST-marked descriptor entry. Up to 7 stacks (hardware
 * limit), all of them are 4K, except the debug stack which
 * is 8K.
 */
static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
	  [0 ... N_EXCEPTION_STACKS - 1]	= EXCEPTION_STKSZ,
	  [DEBUG_STACK - 1]			= DEBUG_STKSZ
};

993 994 995
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ])
	__aligned(PAGE_SIZE);
996 997 998

/* May not be marked __init: used by software suspend */
void syscall_init(void)
L
Linus Torvalds 已提交
999
{
1000 1001 1002 1003 1004 1005 1006 1007
	/*
	 * LSTAR and STAR live in a bit strange symbiosis.
	 * They both write to the same internal register. STAR allows to
	 * set CS/DS but only a 32bit target. LSTAR sets the 64bit rip.
	 */
	wrmsrl(MSR_STAR,  ((u64)__USER32_CS)<<48  | ((u64)__KERNEL_CS)<<32);
	wrmsrl(MSR_LSTAR, system_call);
	wrmsrl(MSR_CSTAR, ignore_sysret);
1008

1009 1010 1011
#ifdef CONFIG_IA32_EMULATION
	syscall32_cpu_init();
#endif
1012

1013 1014 1015
	/* Flags to clear on syscall */
	wrmsrl(MSR_SYSCALL_MASK,
	       X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL);
L
Linus Torvalds 已提交
1016
}
1017

1018 1019 1020 1021 1022 1023 1024 1025
unsigned long kernel_eflags;

/*
 * Copies of the original ist values from the tss are only accessed during
 * debugging, no special alignment required.
 */
DEFINE_PER_CPU(struct orig_ist, orig_ist);

I
Ingo Molnar 已提交
1026
#else	/* CONFIG_X86_64 */
1027

1028 1029 1030
#ifdef CONFIG_CC_STACKPROTECTOR
DEFINE_PER_CPU(unsigned long, stack_canary);
#endif
1031

1032
/* Make sure %fs and %gs are initialized properly in idle threads */
A
Adrian Bunk 已提交
1033
struct pt_regs * __cpuinit idle_regs(struct pt_regs *regs)
1034 1035
{
	memset(regs, 0, sizeof(struct pt_regs));
1036
	regs->fs = __KERNEL_PERCPU;
1037
	regs->gs = __KERNEL_STACK_CANARY;
I
Ingo Molnar 已提交
1038

1039 1040
	return regs;
}
I
Ingo Molnar 已提交
1041
#endif	/* CONFIG_X86_64 */
1042

1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
/*
 * Clear all 6 debug registers:
 */
static void clear_all_debug_regs(void)
{
	int i;

	for (i = 0; i < 8; i++) {
		/* Ignore db4, db5 */
		if ((i == 4) || (i == 5))
			continue;

		set_debugreg(0, i);
	}
}
1058

1059 1060 1061 1062 1063
/*
 * cpu_init() initializes state that is per-CPU. Some data is already
 * initialized (naturally) in the bootstrap process, such as the GDT
 * and IDT. We reload them nevertheless, this function acts as a
 * 'CPU state barrier', nothing should get across.
1064
 * A lot of state is already set up in PDA init for 64 bit
1065
 */
1066
#ifdef CONFIG_X86_64
I
Ingo Molnar 已提交
1067

1068 1069
void __cpuinit cpu_init(void)
{
I
Ingo Molnar 已提交
1070
	struct orig_ist *orig_ist;
1071
	struct task_struct *me;
I
Ingo Molnar 已提交
1072 1073 1074
	struct tss_struct *t;
	unsigned long v;
	int cpu;
1075 1076
	int i;

I
Ingo Molnar 已提交
1077 1078 1079 1080
	cpu = stack_smp_processor_id();
	t = &per_cpu(init_tss, cpu);
	orig_ist = &per_cpu(orig_ist, cpu);

1081 1082 1083 1084 1085
#ifdef CONFIG_NUMA
	if (cpu != 0 && percpu_read(node_number) == 0 &&
	    cpu_to_node(cpu) != NUMA_NO_NODE)
		percpu_write(node_number, cpu_to_node(cpu));
#endif
1086 1087 1088

	me = current;

1089
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask))
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
		panic("CPU#%d already initialized!\n", cpu);

	printk(KERN_INFO "Initializing CPU#%d\n", cpu);

	clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

	/*
	 * Initialize the per-CPU GDT with the boot GDT,
	 * and set up the GDT descriptor:
	 */

1101
	switch_to_new_gdt(cpu);
1102 1103
	loadsegment(fs, 0);

1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
	load_idt((const struct desc_ptr *)&idt_descr);

	memset(me->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
	syscall_init();

	wrmsrl(MSR_FS_BASE, 0);
	wrmsrl(MSR_KERNEL_GS_BASE, 0);
	barrier();

	check_efer();
Y
Yinghai Lu 已提交
1114
	if (cpu != 0)
1115 1116 1117 1118 1119 1120
		enable_x2apic();

	/*
	 * set up and load the per-CPU TSS
	 */
	if (!orig_ist->ist[0]) {
1121
		char *estacks = per_cpu(exception_stacks, cpu);
I
Ingo Molnar 已提交
1122

1123
		for (v = 0; v < N_EXCEPTION_STACKS; v++) {
I
Ingo Molnar 已提交
1124
			estacks += exception_stack_sizes[v];
1125 1126 1127 1128 1129 1130
			orig_ist->ist[v] = t->x86_tss.ist[v] =
					(unsigned long)estacks;
		}
	}

	t->x86_tss.io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
I
Ingo Molnar 已提交
1131

1132 1133 1134 1135 1136 1137 1138 1139 1140
	/*
	 * <= is required because the CPU will access up to
	 * 8 bits beyond the end of the IO permission bitmap.
	 */
	for (i = 0; i <= IO_BITMAP_LONGS; i++)
		t->io_bitmap[i] = ~0UL;

	atomic_inc(&init_mm.mm_count);
	me->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1141
	BUG_ON(me->mm);
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
	enter_lazy_tlb(&init_mm, me);

	load_sp0(t, &current->thread);
	set_tss_desc(cpu, t);
	load_TR_desc();
	load_LDT(&init_mm.context);

#ifdef CONFIG_KGDB
	/*
	 * If the kgdb is connected no debug regs should be altered.  This
	 * is only applicable when KGDB and a KGDB I/O module are built
	 * into the kernel and you are using early debugging with
	 * kgdbwait. KGDB will control the kernel HW breakpoint registers.
	 */
	if (kgdb_connected && arch_kgdb_ops.correct_hw_break)
		arch_kgdb_ops.correct_hw_break();
1158
	else
1159
#endif
1160
		clear_all_debug_regs();
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171

	fpu_init();

	raw_local_save_flags(kernel_eflags);

	if (is_uv_system())
		uv_cpu_init();
}

#else

1172
void __cpuinit cpu_init(void)
1173
{
1174 1175
	int cpu = smp_processor_id();
	struct task_struct *curr = current;
1176
	struct tss_struct *t = &per_cpu(init_tss, cpu);
1177
	struct thread_struct *thread = &curr->thread;
1178

1179
	if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) {
1180
		printk(KERN_WARNING "CPU#%d already initialized!\n", cpu);
1181 1182
		for (;;)
			local_irq_enable();
1183 1184 1185 1186 1187 1188 1189
	}

	printk(KERN_INFO "Initializing CPU#%d\n", cpu);

	if (cpu_has_vme || cpu_has_tsc || cpu_has_de)
		clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);

1190
	load_idt(&idt_descr);
1191
	switch_to_new_gdt(cpu);
L
Linus Torvalds 已提交
1192 1193 1194 1195 1196

	/*
	 * Set up and load the per-CPU TSS and LDT
	 */
	atomic_inc(&init_mm.mm_count);
1197
	curr->active_mm = &init_mm;
S
Stoyan Gaydarov 已提交
1198
	BUG_ON(curr->mm);
1199
	enter_lazy_tlb(&init_mm, curr);
L
Linus Torvalds 已提交
1200

1201
	load_sp0(t, thread);
1202
	set_tss_desc(cpu, t);
L
Linus Torvalds 已提交
1203 1204 1205
	load_TR_desc();
	load_LDT(&init_mm.context);

1206
#ifdef CONFIG_DOUBLEFAULT
L
Linus Torvalds 已提交
1207 1208
	/* Set up doublefault TSS pointer in the GDT */
	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1209
#endif
L
Linus Torvalds 已提交
1210

1211
	clear_all_debug_regs();
L
Linus Torvalds 已提交
1212 1213 1214 1215

	/*
	 * Force FPU initialization:
	 */
1216 1217 1218 1219
	if (cpu_has_xsave)
		current_thread_info()->status = TS_XSAVE;
	else
		current_thread_info()->status = 0;
L
Linus Torvalds 已提交
1220 1221
	clear_used_math();
	mxcsr_feature_mask_init();
1222 1223 1224 1225

	/*
	 * Boot processor to setup the FP and extended state context info.
	 */
1226
	if (smp_processor_id() == boot_cpu_id)
1227 1228 1229
		init_thread_xstate();

	xsave_init();
L
Linus Torvalds 已提交
1230
}
1231
#endif