smpboot.c 41.5 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/export.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/realmode.h>
62 63
#include <asm/cpu.h>
#include <asm/numa.h>
64 65 66
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
67
#include <asm/mwait.h>
I
Ingo Molnar 已提交
68
#include <asm/apic.h>
69
#include <asm/io_apic.h>
70
#include <asm/fpu/internal.h>
71
#include <asm/setup.h>
T
Tejun Heo 已提交
72
#include <asm/uv/uv.h>
73
#include <linux/mc146818rtc.h>
74
#include <asm/i8259.h>
75
#include <asm/realmode.h>
76
#include <asm/misc.h>
77

78 79 80 81 82
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
83
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
84 85

/* representing HT siblings of each logical CPU */
86
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
87 88 89
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
90
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
91 92
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

93
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
94

95
/* Per CPU bogomips and other parameters */
96
DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
97
EXPORT_PER_CPU_SYMBOL(cpu_info);
98

99 100 101 102 103 104
/* Logical package management. We might want to allocate that dynamically */
static int *physical_to_logical_pkg __read_mostly;
static unsigned long *physical_package_map __read_mostly;;
static unsigned int max_physical_pkg_id __read_mostly;
unsigned int __max_logical_packages __read_mostly;
EXPORT_SYMBOL(__max_logical_packages);
105
static unsigned int logical_packages __read_mostly;
106

107 108 109
/* Maximum number of SMT threads on any online core */
int __max_smt_threads __read_mostly;

110 111 112 113 114 115 116 117 118 119 120
/* Flag to indicate if a complete sched domain rebuild is required */
bool x86_topology_update;

int arch_update_cpu_topology(void)
{
	int retval = x86_topology_update;

	x86_topology_update = false;
	return retval;
}

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
{
	unsigned long flags;

	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0xa, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);
	local_flush_tlb();
	pr_debug("1.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) =
							start_eip >> 4;
	pr_debug("2.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) =
							start_eip & 0xf;
	pr_debug("3.\n");
}

static inline void smpboot_restore_warm_reset_vector(void)
{
	unsigned long flags;

	/*
	 * Install writable page 0 entry to set BIOS data area.
	 */
	local_flush_tlb();

	/*
	 * Paranoid:  Set warm reset code and vector here back
	 * to default values.
	 */
	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);

	*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
}

158
/*
159 160
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
161
 */
162
static void smp_callin(void)
163 164 165 166 167
{
	int cpuid, phys_id;

	/*
	 * If waken up by an INIT in an 82489DX configuration
168 169 170
	 * cpu_callout_mask guarantees we don't get here before
	 * an INIT_deassert IPI reaches our local APIC, so it is
	 * now safe to touch our local APIC.
171
	 */
172
	cpuid = smp_processor_id();
173 174 175 176

	/*
	 * (This works even if the APIC is not enabled.)
	 */
177
	phys_id = read_apic_id();
178 179 180 181 182 183 184

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */
185
	apic_ap_setup();
186

187 188 189 190 191 192
	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

193 194
	/*
	 * Get our bogomips.
195 196 197
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
198 199
	 */
	calibrate_delay();
200
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
201
	pr_debug("Stack at about %p\n", &cpuid);
202

203 204 205 206 207 208 209
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

210 211
	notify_cpu_starting(cpuid);

212 213 214
	/*
	 * Allow the master to continue.
	 */
215
	cpumask_set_cpu(cpuid, cpu_callin_mask);
216 217
}

218 219
static int cpu0_logical_apicid;
static int enable_start_cpu0;
220 221 222
/*
 * Activate a secondary processor.
 */
223
static void notrace start_secondary(void *unused)
224 225 226 227 228 229
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
230
	cpu_init();
231
	x86_cpuinit.early_percpu_clock_init();
232 233
	preempt_disable();
	smp_callin();
234

235 236
	enable_start_cpu0 = 0;

237
#ifdef CONFIG_X86_32
238
	/* switch away from the initial page table */
239 240 241 242
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

243 244 245 246 247 248 249 250
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	/*
251 252 253 254
	 * Lock vector_lock and initialize the vectors on this cpu
	 * before setting the cpu online. We must set it online with
	 * vector_lock held to prevent a concurrent setup/teardown
	 * from seeing a half valid vector space.
255
	 */
256
	lock_vector_lock();
257
	setup_vector_irq(smp_processor_id());
258
	set_cpu_online(smp_processor_id(), true);
259
	unlock_vector_lock();
260
	cpu_set_state_online(smp_processor_id());
261
	x86_platform.nmi_init();
262

263 264 265
	/* enable local interrupts */
	local_irq_enable();

266 267
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
268

269
	x86_cpuinit.setup_percpu_clockev();
270 271

	wmb();
272
	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
273 274
}

275 276 277 278 279 280
/**
 * topology_update_package_map - Update the physical to logical package map
 * @pkg:	The physical package id as retrieved via CPUID
 * @cpu:	The cpu for which this is updated
 */
int topology_update_package_map(unsigned int pkg, unsigned int cpu)
281
{
282
	unsigned int new;
283 284 285 286 287 288 289 290 291 292 293 294

	/* Called from early boot ? */
	if (!physical_package_map)
		return 0;

	if (pkg >= max_physical_pkg_id)
		return -EINVAL;

	/* Set the logical package id */
	if (test_and_set_bit(pkg, physical_package_map))
		goto found;

295 296 297
	if (logical_packages >= __max_logical_packages) {
		pr_warn("Package %u of CPU %u exceeds BIOS package data %u.\n",
			logical_packages, cpu, __max_logical_packages);
298 299
		return -ENOSPC;
	}
300 301

	new = logical_packages++;
302 303 304 305
	if (new != pkg) {
		pr_info("CPU %u Converting physical %u to logical package %u\n",
			cpu, pkg, new);
	}
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
	physical_to_logical_pkg[pkg] = new;

found:
	cpu_data(cpu).logical_proc_id = physical_to_logical_pkg[pkg];
	return 0;
}

/**
 * topology_phys_to_logical_pkg - Map a physical package id to a logical
 *
 * Returns logical package id or -1 if not found
 */
int topology_phys_to_logical_pkg(unsigned int phys_pkg)
{
	if (phys_pkg >= max_physical_pkg_id)
		return -1;
	return physical_to_logical_pkg[phys_pkg];
}
EXPORT_SYMBOL(topology_phys_to_logical_pkg);

326
static void __init smp_init_package_map(struct cpuinfo_x86 *c, unsigned int cpu)
327
{
328
	unsigned int ncpus;
329 330 331 332 333
	size_t size;

	/*
	 * Today neither Intel nor AMD support heterogenous systems. That
	 * might change in the future....
334 335 336 337 338 339 340 341 342 343 344 345 346
	 *
	 * While ideally we'd want '* smp_num_siblings' in the below @ncpus
	 * computation, this won't actually work since some Intel BIOSes
	 * report inconsistent HT data when they disable HT.
	 *
	 * In particular, they reduce the APIC-IDs to only include the cores,
	 * but leave the CPUID topology to say there are (2) siblings.
	 * This means we don't know how many threads there will be until
	 * after the APIC enumeration.
	 *
	 * By not including this we'll sometimes over-estimate the number of
	 * logical packages by the amount of !present siblings, but this is
	 * still better than MAX_LOCAL_APIC.
347 348 349 350 351
	 *
	 * We use total_cpus not nr_cpu_ids because nr_cpu_ids can be limited
	 * on the command line leading to a similar issue as the HT disable
	 * problem because the hyperthreads are usually enumerated after the
	 * primary cores.
352
	 */
353
	ncpus = boot_cpu_data.x86_max_cores;
354 355 356 357 358
	if (!ncpus) {
		pr_warn("x86_max_cores == zero !?!?");
		ncpus = 1;
	}

359
	__max_logical_packages = DIV_ROUND_UP(total_cpus, ncpus);
360
	logical_packages = 0;
361 362 363 364 365 366 367 368 369 370 371 372

	/*
	 * Possibly larger than what we need as the number of apic ids per
	 * package can be smaller than the actual used apic ids.
	 */
	max_physical_pkg_id = DIV_ROUND_UP(MAX_LOCAL_APIC, ncpus);
	size = max_physical_pkg_id * sizeof(unsigned int);
	physical_to_logical_pkg = kmalloc(size, GFP_KERNEL);
	memset(physical_to_logical_pkg, 0xff, size);
	size = BITS_TO_LONGS(max_physical_pkg_id) * sizeof(unsigned long);
	physical_package_map = kzalloc(size, GFP_KERNEL);

373
	pr_info("Max logical packages: %u\n", __max_logical_packages);
374 375

	topology_update_package_map(c->phys_proc_id, cpu);
376 377
}

378 379 380 381 382 383 384
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
385
	smp_init_package_map(c, id);
386 387
}

388 389 390 391
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
392
void smp_store_cpu_info(int id)
393 394 395
{
	struct cpuinfo_x86 *c = &cpu_data(id);

396
	*c = boot_cpu_data;
397
	c->cpu_index = id;
398 399 400 401 402
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
403 404
}

405 406 407 408 409 410 411 412
static bool
topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
}

413
static bool
414
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
415
{
416 417
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

418
	return !WARN_ONCE(!topology_same_node(c, o),
419 420 421 422 423
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

424
#define link_mask(mfunc, c1, c2)					\
425
do {									\
426 427
	cpumask_set_cpu((c1), mfunc(c2));				\
	cpumask_set_cpu((c2), mfunc(c1));				\
428 429
} while (0)

430
static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
431
{
432
	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
433 434 435 436
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
437
		    c->cpu_core_id == o->cpu_core_id)
438 439 440 441 442 443 444 445 446 447
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

448
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
449 450 451 452 453 454 455 456
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
457 458
}

459 460 461 462 463 464
/*
 * Unlike the other levels, we do not enforce keeping a
 * multicore group inside a NUMA node.  If this happens, we will
 * discard the MC level of the topology later.
 */
static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
465
{
466 467
	if (c->phys_proc_id == o->phys_proc_id)
		return true;
468 469
	return false;
}
470

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC)
static inline int x86_sched_itmt_flags(void)
{
	return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0;
}

#ifdef CONFIG_SCHED_MC
static int x86_core_flags(void)
{
	return cpu_core_flags() | x86_sched_itmt_flags();
}
#endif
#ifdef CONFIG_SCHED_SMT
static int x86_smt_flags(void)
{
	return cpu_smt_flags() | x86_sched_itmt_flags();
}
#endif
#endif

491
static struct sched_domain_topology_level x86_numa_in_package_topology[] = {
492
#ifdef CONFIG_SCHED_SMT
493
	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
494 495
#endif
#ifdef CONFIG_SCHED_MC
496
	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
497 498 499
#endif
	{ NULL, },
};
500 501 502

static struct sched_domain_topology_level x86_topology[] = {
#ifdef CONFIG_SCHED_SMT
503
	{ cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) },
504 505
#endif
#ifdef CONFIG_SCHED_MC
506
	{ cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) },
507 508 509 510 511
#endif
	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
	{ NULL, },
};

512
/*
513 514
 * Set if a package/die has multiple NUMA nodes inside.
 * AMD Magny-Cours and Intel Cluster-on-Die have this.
515
 */
516
static bool x86_has_numa_in_package;
517

518
void set_cpu_sibling_map(int cpu)
519
{
520
	bool has_smt = smp_num_siblings > 1;
521
	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
522
	struct cpuinfo_x86 *c = &cpu_data(cpu);
523
	struct cpuinfo_x86 *o;
524
	int i, threads;
525

526
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
527

528
	if (!has_mp) {
529
		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
530
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
531
		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
532 533 534 535
		c->booted_cores = 1;
		return;
	}

536
	for_each_cpu(i, cpu_sibling_setup_mask) {
537 538 539
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
540
			link_mask(topology_sibling_cpumask, cpu, i);
541

542
		if ((i == cpu) || (has_mp && match_llc(c, o)))
543
			link_mask(cpu_llc_shared_mask, cpu, i);
544

545 546 547 548
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
549
	 * topology_sibling_cpumask links to be set-up.
550 551 552 553
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

554
		if ((i == cpu) || (has_mp && match_die(c, o))) {
555
			link_mask(topology_core_cpumask, cpu, i);
556

557 558 559
			/*
			 *  Does this new cpu bringup a new core?
			 */
560 561
			if (cpumask_weight(
			    topology_sibling_cpumask(cpu)) == 1) {
562 563 564 565
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
566 567
				if (cpumask_first(
				    topology_sibling_cpumask(i)) == i)
568 569 570 571 572 573 574 575 576 577
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
578
		if (match_die(c, o) && !topology_same_node(c, o))
579
			x86_has_numa_in_package = true;
580
	}
581 582 583 584

	threads = cpumask_weight(topology_sibling_cpumask(cpu));
	if (threads > __max_smt_threads)
		__max_smt_threads = threads;
585 586
}

587
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
588
const struct cpumask *cpu_coregroup_mask(int cpu)
589
{
590
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
591 592
}

I
Ingo Molnar 已提交
593
static void impress_friends(void)
594 595 596 597 598 599
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
600
	pr_debug("Before bogomips\n");
601
	for_each_possible_cpu(cpu)
602
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
603
			bogosum += cpu_data(cpu).loops_per_jiffy;
604
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
605
		num_online_cpus(),
606 607 608
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

609
	pr_debug("Before bogocount - setting activated=1\n");
610 611
}

612
void __inquire_remote_apic(int apicid)
613 614
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
615
	const char * const names[] = { "ID", "VERSION", "SPIV" };
616 617 618
	int timeout;
	u32 status;

619
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
620 621

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
622
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
623 624 625 626 627 628

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
629
			pr_cont("a previous APIC delivery may have failed\n");
630

631
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
632 633 634 635 636 637 638 639 640 641

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
642
			pr_cont("%08x\n", status);
643 644
			break;
		default:
645
			pr_cont("failed\n");
646 647 648 649
		}
	}
}

650 651 652 653 654 655 656 657
/*
 * The Multiprocessor Specification 1.4 (1997) example code suggests
 * that there should be a 10ms delay between the BSP asserting INIT
 * and de-asserting INIT, when starting a remote processor.
 * But that slows boot and resume on modern processors, which include
 * many cores and don't require that delay.
 *
 * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
658
 * Modern processor families are quirked to remove the delay entirely.
659 660 661
 */
#define UDELAY_10MS_DEFAULT 10000

662
static unsigned int init_udelay = UINT_MAX;
663 664 665 666 667 668 669 670 671

static int __init cpu_init_udelay(char *str)
{
	get_option(&str, &init_udelay);

	return 0;
}
early_param("cpu_init_udelay", cpu_init_udelay);

672 673 674
static void __init smp_quirk_init_udelay(void)
{
	/* if cmdline changed it from default, leave it alone */
675
	if (init_udelay != UINT_MAX)
676 677 678 679
		return;

	/* if modern processor, use no delay */
	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
680
	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) {
681
		init_udelay = 0;
682 683
		return;
	}
684 685
	/* else, use legacy delay */
	init_udelay = UDELAY_10MS_DEFAULT;
686 687
}

688 689 690 691 692
/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
693
int
694
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
695 696 697 698 699 700 701
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
702
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
703

704
	pr_debug("Waiting for send to finish...\n");
705 706 707 708 709 710
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
711
	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
712 713 714 715 716
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
717
	pr_debug("NMI sent\n");
718 719

	if (send_status)
720
		pr_err("APIC never delivered???\n");
721
	if (accept_status)
722
		pr_err("APIC delivery error (%lx)\n", accept_status);
723 724 725 726

	return (send_status | accept_status);
}

727
static int
728
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
729
{
730
	unsigned long send_status = 0, accept_status = 0;
731 732
	int maxlvt, num_starts, j;

733 734
	maxlvt = lapic_get_maxlvt();

735 736 737
	/*
	 * Be paranoid about clearing APIC errors.
	 */
738
	if (APIC_INTEGRATED(boot_cpu_apic_version)) {
739 740
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
741 742 743
		apic_read(APIC_ESR);
	}

744
	pr_debug("Asserting INIT\n");
745 746 747 748 749 750 751

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
752 753
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
754

755
	pr_debug("Waiting for send to finish...\n");
756 757
	send_status = safe_apic_wait_icr_idle();

758
	udelay(init_udelay);
759

760
	pr_debug("Deasserting INIT\n");
761 762 763

	/* Target chip */
	/* Send IPI */
764
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
765

766
	pr_debug("Waiting for send to finish...\n");
767 768 769 770 771 772 773 774 775 776
	send_status = safe_apic_wait_icr_idle();

	mb();

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
777
	if (APIC_INTEGRATED(boot_cpu_apic_version))
778 779 780 781 782 783 784
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Run STARTUP IPI loop.
	 */
785
	pr_debug("#startup loops: %d\n", num_starts);
786 787

	for (j = 1; j <= num_starts; j++) {
788
		pr_debug("Sending STARTUP #%d\n", j);
789 790
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
791
		apic_read(APIC_ESR);
792
		pr_debug("After apic_write\n");
793 794 795 796 797 798 799 800

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
801 802
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
803 804 805 806

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
L
Len Brown 已提交
807 808 809
		if (init_udelay == 0)
			udelay(10);
		else
810
			udelay(300);
811

812
		pr_debug("Startup point 1\n");
813

814
		pr_debug("Waiting for send to finish...\n");
815 816 817 818 819
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
L
Len Brown 已提交
820 821 822
		if (init_udelay == 0)
			udelay(10);
		else
823
			udelay(200);
824

825
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
826 827 828 829 830
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
831
	pr_debug("After Startup\n");
832 833

	if (send_status)
834
		pr_err("APIC never delivered???\n");
835
	if (accept_status)
836
		pr_err("APIC delivery error (%lx)\n", accept_status);
837 838 839 840

	return (send_status | accept_status);
}

841
/* reduce the number of lines printed when booting a large cpu count system */
842
static void announce_cpu(int cpu, int apicid)
843 844
{
	static int current_node = -1;
845
	int node = early_cpu_to_node(cpu);
846
	static int width, node_width;
847 848 849

	if (!width)
		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
850

851 852 853 854 855 856
	if (!node_width)
		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */

	if (cpu == 1)
		printk(KERN_INFO "x86: Booting SMP configuration:\n");

857 858 859
	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
860
				pr_cont("\n");
861
			current_node = node;
862 863 864

			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
			       node_width - num_digits(node), " ", node);
865
		}
866 867 868 869 870 871 872

		/* Add padding for the BSP */
		if (cpu == 1)
			pr_cont("%*s", width + 1, " ");

		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);

873 874 875 876 877
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
901
static int
902 903 904 905 906 907
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

908 909
	preempt_disable();

910 911 912
	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
913 914 915 916
	if (cpu) {
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
		goto out;
	}
917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}
935 936 937

out:
	preempt_enable();
938 939 940 941

	return boot_error;
}

942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958
void common_cpu_up(unsigned int cpu, struct task_struct *idle)
{
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();

	per_cpu(current_task, cpu) = idle;

#ifdef CONFIG_X86_32
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
	per_cpu(cpu_current_top_of_stack, cpu) =
		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
#else
	initial_gs = per_cpu_offset(cpu);
#endif
}

959 960 961
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
962 963
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
964
 */
965
static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
966
{
967
	volatile u32 *trampoline_status =
968
		(volatile u32 *) __va(real_mode_header->trampoline_status);
969
	/* start_ip had better be page-aligned! */
970
	unsigned long start_ip = real_mode_header->trampoline_start;
971

972
	unsigned long boot_error = 0;
973
	int cpu0_nmi_registered = 0;
974
	unsigned long timeout;
975

976
	idle->thread.sp = (unsigned long)task_pt_regs(idle);
977
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
978
	initial_code = (unsigned long)start_secondary;
979
	initial_stack  = idle->thread.sp;
980

981 982 983 984 985 986 987
	/*
	 * Enable the espfix hack for this CPU
	*/
#ifdef CONFIG_X86_ESPFIX64
	init_espfix_ap(cpu);
#endif

988 989
	/* So we see what's up */
	announce_cpu(cpu, apicid);
990 991 992 993 994 995

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

J
Jack Steiner 已提交
996
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
997

998
		pr_debug("Setting warm reset code and vector.\n");
999

J
Jack Steiner 已提交
1000 1001 1002
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
1003
		*/
1004
		if (APIC_INTEGRATED(boot_cpu_apic_version)) {
1005 1006 1007
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
1008
	}
1009

1010 1011 1012 1013 1014 1015 1016 1017 1018
	/*
	 * AP might wait on cpu_callout_mask in cpu_init() with
	 * cpu_initialized_mask set if previous attempt to online
	 * it timed-out. Clear cpu_initialized_mask so that after
	 * INIT/SIPI it could start with a clean state.
	 */
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
	smp_mb();

1019
	/*
1020 1021 1022 1023
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
1024
	 */
1025 1026 1027
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
1028 1029
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
1030 1031 1032

	if (!boot_error) {
		/*
1033
		 * Wait 10s total for first sign of life from AP
1034
		 */
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
		boot_error = -1;
		timeout = jiffies + 10*HZ;
		while (time_before(jiffies, timeout)) {
			if (cpumask_test_cpu(cpu, cpu_initialized_mask)) {
				/*
				 * Tell AP to proceed with initialization
				 */
				cpumask_set_cpu(cpu, cpu_callout_mask);
				boot_error = 0;
				break;
			}
			schedule();
		}
	}
1049

1050
	if (!boot_error) {
1051
		/*
1052
		 * Wait till AP completes initial initialization
1053
		 */
1054
		while (!cpumask_test_cpu(cpu, cpu_callin_mask)) {
1055 1056 1057 1058 1059 1060 1061
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
1062 1063 1064 1065
		}
	}

	/* mark "stuck" area as not stuck */
1066
	*trampoline_status = 0;
1067

1068 1069 1070 1071 1072 1073
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
1074 1075 1076 1077 1078 1079 1080
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

1081 1082 1083
	return boot_error;
}

1084
int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
1085
{
1086
	int apicid = apic->cpu_present_to_apicid(cpu);
1087 1088 1089 1090 1091
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

1092
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
1093

1094
	if (apicid == BAD_APICID ||
1095
	    !physid_isset(apicid, phys_cpu_present_map) ||
1096
	    !apic->apic_id_valid(apicid)) {
1097
		pr_err("%s: bad cpu %d\n", __func__, cpu);
1098 1099 1100 1101 1102 1103
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
1104
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
1105
		pr_debug("do_boot_cpu %d Already started\n", cpu);
1106 1107 1108 1109 1110 1111 1112 1113 1114
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

1115 1116 1117 1118
	/* x86 CPUs take themselves offline, so delayed offline is OK. */
	err = cpu_check_up_prepare(cpu);
	if (err && err != -EBUSY)
		return err;
1119

1120
	/* the FPU context is blank, nobody can own it */
1121
	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
1122

1123 1124
	common_cpu_up(cpu, tidle);

1125
	err = do_boot_cpu(apicid, cpu, tidle);
1126
	if (err) {
1127
		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
1128
		return -EIO;
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

1139
	while (!cpu_online(cpu)) {
1140 1141 1142 1143 1144 1145 1146
		cpu_relax();
		touch_nmi_watchdog();
	}

	return 0;
}

1147 1148 1149 1150 1151 1152 1153 1154
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

1155 1156 1157 1158 1159 1160 1161
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
1162 1163
	pr_info("SMP disabled\n");

1164 1165
	disable_ioapic_support();

1166 1167
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
1168

1169
	if (smp_found_config)
1170
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1171
	else
1172
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1173 1174
	cpumask_set_cpu(0, topology_sibling_cpumask(0));
	cpumask_set_cpu(0, topology_core_cpumask(0));
1175 1176
}

1177 1178 1179 1180 1181 1182 1183
enum {
	SMP_OK,
	SMP_NO_CONFIG,
	SMP_NO_APIC,
	SMP_FORCE_UP,
};

1184 1185 1186 1187 1188
/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
1189
	preempt_disable();
1190

1191
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1192 1193 1194 1195
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

1196 1197
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
1198 1199 1200 1201

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
1202
				set_cpu_present(cpu, false);
1203 1204 1205 1206 1207 1208
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
1209
				set_cpu_possible(cpu, false);
1210 1211 1212 1213 1214 1215 1216
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

1217
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1218
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
1219 1220
			hard_smp_processor_id());

1221 1222 1223 1224 1225 1226 1227 1228
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1229
		preempt_enable();
1230
		pr_notice("SMP motherboard not detected\n");
1231
		return SMP_NO_CONFIG;
1232 1233 1234 1235 1236 1237
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1238
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1239 1240
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1241 1242
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1243
	preempt_enable();
1244 1245 1246 1247

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
1248
	if (APIC_INTEGRATED(boot_cpu_apic_version) &&
1249
	    !boot_cpu_has(X86_FEATURE_APIC)) {
1250 1251 1252
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1253
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1254
		}
1255
		return SMP_NO_APIC;
1256 1257 1258 1259 1260 1261
	}

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1262
		pr_info("SMP mode deactivated\n");
1263
		return SMP_FORCE_UP;
1264 1265
	}

1266
	return SMP_OK;
1267 1268 1269 1270 1271 1272 1273
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1274
	for_each_possible_cpu(i) {
1275 1276
		c = &cpu_data(i);
		/* mark all to hotplug */
1277
		c->cpu_index = nr_cpu_ids;
1278 1279 1280 1281 1282 1283 1284 1285 1286
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1287 1288
	unsigned int i;

1289
	smp_cpu_index_default();
1290

1291 1292 1293
	/*
	 * Setup boot CPU information
	 */
1294
	smp_store_boot_cpu_info(); /* Final full version of the data */
1295 1296
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1297

1298
	for_each_possible_cpu(i) {
1299 1300
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1301
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1302
	}
1303 1304 1305 1306 1307 1308 1309 1310 1311 1312

	/*
	 * Set 'default' x86 topology, this matches default_topology() in that
	 * it has NUMA nodes as a topology level. See also
	 * native_smp_cpus_done().
	 *
	 * Must be done before set_cpus_sibling_map() is ran.
	 */
	set_sched_topology(x86_topology);

1313 1314
	set_cpu_sibling_map(0);

1315 1316
	switch (smp_sanity_check(max_cpus)) {
	case SMP_NO_CONFIG:
1317
		disable_smp();
1318 1319 1320 1321 1322 1323 1324 1325
		if (APIC_init_uniprocessor())
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
		return;
	case SMP_NO_APIC:
		disable_smp();
		return;
	case SMP_FORCE_UP:
		disable_smp();
1326
		apic_bsp_setup(false);
1327
		return;
1328 1329
	case SMP_OK:
		break;
1330 1331
	}

1332
	if (read_apic_id() != boot_cpu_physical_apicid) {
1333
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1334
		     read_apic_id(), boot_cpu_physical_apicid);
1335 1336 1337
		/* Or can we switch back to PIC here? */
	}

1338
	default_setup_apic_routing();
1339
	cpu0_logical_apicid = apic_bsp_setup(false);
1340

1341
	pr_info("CPU0: ");
1342
	print_cpu_info(&cpu_data(0));
1343 1344 1345

	if (is_uv_system())
		uv_system_init();
1346 1347

	set_mtrr_aps_delayed_init();
1348 1349

	smp_quirk_init_udelay();
1350
}
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1362 1363 1364 1365 1366 1367
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1368
	switch_to_new_gdt(me);
1369 1370
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1371
	cpu_set_state_online(me);
1372 1373
}

1374 1375
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1376
	pr_debug("Boot done\n");
1377

1378 1379 1380
	if (x86_has_numa_in_package)
		set_sched_topology(x86_numa_in_package_topology);

D
Don Zickus 已提交
1381
	nmi_selftest();
1382 1383
	impress_friends();
	setup_ioapic_dest();
1384
	mtrr_aps_init();
1385 1386
}

1387 1388 1389 1390 1391 1392 1393 1394 1395
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1396
/*
1397
 * cpu_possible_mask should be static, it cannot change as cpu's
1398 1399 1400
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1401
 * cpu_present_mask on the other hand can change dynamically.
1402 1403 1404 1405 1406 1407
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1408
 * - The user can overwrite it with possible_cpus=NUM
1409 1410 1411 1412 1413 1414
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1415
	int i, possible;
1416

1417 1418
	/* No boot processor was found in mptable or ACPI MADT */
	if (!num_processors) {
1419 1420 1421
		if (boot_cpu_has(X86_FEATURE_APIC)) {
			int apicid = boot_cpu_physical_apicid;
			int cpu = hard_smp_processor_id();
1422

1423
			pr_warn("Boot CPU (id %d) not listed by BIOS\n", cpu);
1424

1425 1426 1427 1428 1429
			/* Make sure boot cpu is enumerated */
			if (apic->cpu_present_to_apicid(0) == BAD_APICID &&
			    apic->apic_id_valid(apicid))
				generic_processor_info(apicid, boot_cpu_apic_version);
		}
1430 1431 1432 1433

		if (!num_processors)
			num_processors = 1;
	}
1434

1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1446 1447
		possible = setup_possible_cpus;

1448 1449
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1450 1451
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1452
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1453 1454
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1455
	}
1456

1457 1458 1459 1460
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1461
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1462 1463 1464 1465
			possible, setup_max_cpus);
		possible = i;
	}

1466
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1467 1468 1469
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1470
		set_cpu_possible(i, true);
1471 1472
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1473 1474

	nr_cpu_ids = possible;
1475
}
1476

1477 1478
#ifdef CONFIG_HOTPLUG_CPU

1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
/* Recompute SMT state for all CPUs on offline */
static void recompute_smt_state(void)
{
	int max_threads, cpu;

	max_threads = 0;
	for_each_online_cpu (cpu) {
		int threads = cpumask_weight(topology_sibling_cpumask(cpu));

		if (threads > max_threads)
			max_threads = threads;
	}
	__max_smt_threads = max_threads;
}

1494 1495 1496 1497 1498
static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1499 1500
	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1501 1502 1503
		/*/
		 * last thread sibling in this cpu core going down
		 */
1504
		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1505 1506 1507
			cpu_data(sibling).booted_cores--;
	}

1508 1509
	for_each_cpu(sibling, topology_sibling_cpumask(cpu))
		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1510 1511 1512
	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
	cpumask_clear(cpu_llc_shared_mask(cpu));
1513 1514
	cpumask_clear(topology_sibling_cpumask(cpu));
	cpumask_clear(topology_core_cpumask(cpu));
1515 1516
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1517
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1518
	recompute_smt_state();
1519 1520
}

1521
static void remove_cpu_from_maps(int cpu)
1522
{
1523 1524 1525
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1526
	/* was set by cpu_init() */
1527
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1528
	numa_remove_cpu(cpu);
1529 1530
}

1531
void cpu_disable_common(void)
1532 1533 1534 1535 1536 1537
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1538
	lock_vector_lock();
1539
	remove_cpu_from_maps(cpu);
1540
	unlock_vector_lock();
1541
	fixup_irqs();
1542 1543 1544 1545
}

int native_cpu_disable(void)
{
1546 1547 1548 1549 1550 1551
	int ret;

	ret = check_irq_vectors_for_cpu_disable();
	if (ret)
		return ret;

1552 1553
	clear_local_APIC();
	cpu_disable_common();
1554

1555 1556 1557
	return 0;
}

1558
int common_cpu_die(unsigned int cpu)
1559
{
1560
	int ret = 0;
1561

1562
	/* We don't do anything here: idle task is faking death itself. */
1563

1564
	/* They ack this in play_dead() by setting CPU_DEAD */
1565
	if (cpu_wait_death(cpu, 5)) {
1566 1567 1568 1569
		if (system_state == SYSTEM_RUNNING)
			pr_info("CPU %u is now offline\n", cpu);
	} else {
		pr_err("CPU %u didn't die...\n", cpu);
1570
		ret = -1;
1571
	}
1572 1573 1574 1575 1576 1577 1578

	return ret;
}

void native_cpu_die(unsigned int cpu)
{
	common_cpu_die(cpu);
1579
}
1580 1581 1582 1583 1584 1585 1586

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();

	/* Ack it */
1587
	(void)cpu_report_death();
1588 1589 1590 1591 1592 1593 1594

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1595 1596 1597 1598 1599 1600 1601 1602
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1603 1604 1605 1606 1607 1608 1609 1610 1611
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
1612
	void *mwait_ptr;
1613
	int i;
1614

1615
	if (!this_cpu_has(X86_FEATURE_MWAIT))
1616
		return;
1617
	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1618
		return;
1619
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1644 1645 1646 1647 1648 1649 1650
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1651 1652
	wbinvd();

1653
	while (1) {
1654 1655 1656 1657 1658 1659 1660
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
1661
		mb();
1662
		clflush(mwait_ptr);
1663
		mb();
1664
		__monitor(mwait_ptr, 0, 0);
1665 1666
		mb();
		__mwait(eax, 0);
1667 1668 1669 1670 1671
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1672 1673 1674
	}
}

1675
void hlt_play_dead(void)
1676
{
1677
	if (__this_cpu_read(cpu_info.x86) >= 4)
1678 1679
		wbinvd();

1680 1681
	while (1) {
		native_halt();
1682 1683 1684 1685 1686
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1687 1688 1689
	}
}

1690 1691 1692
void native_play_dead(void)
{
	play_dead_common();
1693
	tboot_shutdown(TB_SHUTDOWN_WFS);
1694 1695

	mwait_play_dead();	/* Only returns on failure */
1696 1697
	if (cpuidle_play_dead())
		hlt_play_dead();
1698 1699
}

1700
#else /* ... !CONFIG_HOTPLUG_CPU */
1701
int native_cpu_disable(void)
1702 1703 1704 1705
{
	return -ENOSYS;
}

1706
void native_cpu_die(unsigned int cpu)
1707 1708 1709 1710
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1711 1712 1713 1714 1715 1716

void native_play_dead(void)
{
	BUG();
}

1717
#endif