smpboot.c 40.7 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/module.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/idle.h>
62
#include <asm/realmode.h>
63 64
#include <asm/cpu.h>
#include <asm/numa.h>
65 66 67
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
68
#include <asm/mwait.h>
I
Ingo Molnar 已提交
69
#include <asm/apic.h>
70
#include <asm/io_apic.h>
71
#include <asm/fpu/internal.h>
72
#include <asm/setup.h>
T
Tejun Heo 已提交
73
#include <asm/uv/uv.h>
74
#include <linux/mc146818rtc.h>
75
#include <asm/i8259.h>
76
#include <asm/realmode.h>
77
#include <asm/misc.h>
78

79 80 81 82 83
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
84
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
85 86

/* representing HT siblings of each logical CPU */
87
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
88 89 90
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
91
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
92 93
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

94
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
95

96
/* Per CPU bogomips and other parameters */
97
DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
98
EXPORT_PER_CPU_SYMBOL(cpu_info);
99

100 101 102 103 104 105 106 107
/* Logical package management. We might want to allocate that dynamically */
static int *physical_to_logical_pkg __read_mostly;
static unsigned long *physical_package_map __read_mostly;;
static unsigned long *logical_package_map  __read_mostly;
static unsigned int max_physical_pkg_id __read_mostly;
unsigned int __max_logical_packages __read_mostly;
EXPORT_SYMBOL(__max_logical_packages);

108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
{
	unsigned long flags;

	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0xa, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);
	local_flush_tlb();
	pr_debug("1.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) =
							start_eip >> 4;
	pr_debug("2.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) =
							start_eip & 0xf;
	pr_debug("3.\n");
}

static inline void smpboot_restore_warm_reset_vector(void)
{
	unsigned long flags;

	/*
	 * Install writable page 0 entry to set BIOS data area.
	 */
	local_flush_tlb();

	/*
	 * Paranoid:  Set warm reset code and vector here back
	 * to default values.
	 */
	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);

	*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
}

145
/*
146 147
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
148
 */
149
static void smp_callin(void)
150 151 152 153 154
{
	int cpuid, phys_id;

	/*
	 * If waken up by an INIT in an 82489DX configuration
155 156 157
	 * cpu_callout_mask guarantees we don't get here before
	 * an INIT_deassert IPI reaches our local APIC, so it is
	 * now safe to touch our local APIC.
158
	 */
159
	cpuid = smp_processor_id();
160 161 162 163

	/*
	 * (This works even if the APIC is not enabled.)
	 */
164
	phys_id = read_apic_id();
165 166 167 168 169 170 171

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */
172
	apic_ap_setup();
173

174 175 176 177 178 179
	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

180 181
	/*
	 * Get our bogomips.
182 183 184
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
185 186
	 */
	calibrate_delay();
187
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
188
	pr_debug("Stack at about %p\n", &cpuid);
189

190 191 192 193 194 195 196
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

197 198
	notify_cpu_starting(cpuid);

199 200 201
	/*
	 * Allow the master to continue.
	 */
202
	cpumask_set_cpu(cpuid, cpu_callin_mask);
203 204
}

205 206
static int cpu0_logical_apicid;
static int enable_start_cpu0;
207 208 209
/*
 * Activate a secondary processor.
 */
210
static void notrace start_secondary(void *unused)
211 212 213 214 215 216
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
217
	cpu_init();
218
	x86_cpuinit.early_percpu_clock_init();
219 220
	preempt_disable();
	smp_callin();
221

222 223
	enable_start_cpu0 = 0;

224
#ifdef CONFIG_X86_32
225
	/* switch away from the initial page table */
226 227 228 229
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

230 231 232 233 234 235 236 237
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	/*
238 239 240 241
	 * Lock vector_lock and initialize the vectors on this cpu
	 * before setting the cpu online. We must set it online with
	 * vector_lock held to prevent a concurrent setup/teardown
	 * from seeing a half valid vector space.
242
	 */
243
	lock_vector_lock();
244
	setup_vector_irq(smp_processor_id());
245
	set_cpu_online(smp_processor_id(), true);
246
	unlock_vector_lock();
247
	cpu_set_state_online(smp_processor_id());
248
	x86_platform.nmi_init();
249

250 251 252
	/* enable local interrupts */
	local_irq_enable();

253 254
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
255

256
	x86_cpuinit.setup_percpu_clockev();
257 258

	wmb();
259
	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
260 261
}

262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
int topology_update_package_map(unsigned int apicid, unsigned int cpu)
{
	unsigned int new, pkg = apicid >> boot_cpu_data.x86_coreid_bits;

	/* Called from early boot ? */
	if (!physical_package_map)
		return 0;

	if (pkg >= max_physical_pkg_id)
		return -EINVAL;

	/* Set the logical package id */
	if (test_and_set_bit(pkg, physical_package_map))
		goto found;

	new = find_first_zero_bit(logical_package_map, __max_logical_packages);
	if (new >= __max_logical_packages) {
		physical_to_logical_pkg[pkg] = -1;
		pr_warn("APIC(%x) Package %u exceeds logical package map\n",
			apicid, pkg);
		return -ENOSPC;
	}
	set_bit(new, logical_package_map);
	pr_info("APIC(%x) Converting physical %u to logical package %u\n",
		apicid, pkg, new);
	physical_to_logical_pkg[pkg] = new;

found:
	cpu_data(cpu).logical_proc_id = physical_to_logical_pkg[pkg];
	return 0;
}

/**
 * topology_phys_to_logical_pkg - Map a physical package id to a logical
 *
 * Returns logical package id or -1 if not found
 */
int topology_phys_to_logical_pkg(unsigned int phys_pkg)
{
	if (phys_pkg >= max_physical_pkg_id)
		return -1;
	return physical_to_logical_pkg[phys_pkg];
}
EXPORT_SYMBOL(topology_phys_to_logical_pkg);

static void __init smp_init_package_map(void)
{
	unsigned int ncpus, cpu;
	size_t size;

	/*
	 * Today neither Intel nor AMD support heterogenous systems. That
	 * might change in the future....
315 316 317 318 319 320 321 322 323 324 325 326 327
	 *
	 * While ideally we'd want '* smp_num_siblings' in the below @ncpus
	 * computation, this won't actually work since some Intel BIOSes
	 * report inconsistent HT data when they disable HT.
	 *
	 * In particular, they reduce the APIC-IDs to only include the cores,
	 * but leave the CPUID topology to say there are (2) siblings.
	 * This means we don't know how many threads there will be until
	 * after the APIC enumeration.
	 *
	 * By not including this we'll sometimes over-estimate the number of
	 * logical packages by the amount of !present siblings, but this is
	 * still better than MAX_LOCAL_APIC.
328 329 330 331 332
	 *
	 * We use total_cpus not nr_cpu_ids because nr_cpu_ids can be limited
	 * on the command line leading to a similar issue as the HT disable
	 * problem because the hyperthreads are usually enumerated after the
	 * primary cores.
333
	 */
334
	ncpus = boot_cpu_data.x86_max_cores;
335 336 337 338 339
	if (!ncpus) {
		pr_warn("x86_max_cores == zero !?!?");
		ncpus = 1;
	}

340
	__max_logical_packages = DIV_ROUND_UP(total_cpus, ncpus);
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370

	/*
	 * Possibly larger than what we need as the number of apic ids per
	 * package can be smaller than the actual used apic ids.
	 */
	max_physical_pkg_id = DIV_ROUND_UP(MAX_LOCAL_APIC, ncpus);
	size = max_physical_pkg_id * sizeof(unsigned int);
	physical_to_logical_pkg = kmalloc(size, GFP_KERNEL);
	memset(physical_to_logical_pkg, 0xff, size);
	size = BITS_TO_LONGS(max_physical_pkg_id) * sizeof(unsigned long);
	physical_package_map = kzalloc(size, GFP_KERNEL);
	size = BITS_TO_LONGS(__max_logical_packages) * sizeof(unsigned long);
	logical_package_map = kzalloc(size, GFP_KERNEL);

	pr_info("Max logical packages: %u\n", __max_logical_packages);

	for_each_present_cpu(cpu) {
		unsigned int apicid = apic->cpu_present_to_apicid(cpu);

		if (apicid == BAD_APICID || !apic->apic_id_valid(apicid))
			continue;
		if (!topology_update_package_map(apicid, cpu))
			continue;
		pr_warn("CPU %u APICId %x disabled\n", cpu, apicid);
		per_cpu(x86_bios_cpu_apicid, cpu) = BAD_APICID;
		set_cpu_possible(cpu, false);
		set_cpu_present(cpu, false);
	}
}

371 372 373 374 375 376 377
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
378
	smp_init_package_map();
379 380
}

381 382 383 384
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
385
void smp_store_cpu_info(int id)
386 387 388
{
	struct cpuinfo_x86 *c = &cpu_data(id);

389
	*c = boot_cpu_data;
390
	c->cpu_index = id;
391 392 393 394 395
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
396 397
}

398 399 400 401 402 403 404 405
static bool
topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
}

406
static bool
407
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
408
{
409 410
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

411
	return !WARN_ONCE(!topology_same_node(c, o),
412 413 414 415 416
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

417
#define link_mask(mfunc, c1, c2)					\
418
do {									\
419 420
	cpumask_set_cpu((c1), mfunc(c2));				\
	cpumask_set_cpu((c2), mfunc(c1));				\
421 422
} while (0)

423
static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
424
{
425
	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
426 427 428 429
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
430
		    c->cpu_core_id == o->cpu_core_id)
431 432 433 434 435 436 437 438 439 440
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

441
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
442 443 444 445 446 447 448 449
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
450 451
}

452 453 454 455 456 457
/*
 * Unlike the other levels, we do not enforce keeping a
 * multicore group inside a NUMA node.  If this happens, we will
 * discard the MC level of the topology later.
 */
static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
458
{
459 460
	if (c->phys_proc_id == o->phys_proc_id)
		return true;
461 462
	return false;
}
463

464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
static struct sched_domain_topology_level numa_inside_package_topology[] = {
#ifdef CONFIG_SCHED_SMT
	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
#endif
#ifdef CONFIG_SCHED_MC
	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
#endif
	{ NULL, },
};
/*
 * set_sched_topology() sets the topology internal to a CPU.  The
 * NUMA topologies are layered on top of it to build the full
 * system topology.
 *
 * If NUMA nodes are observed to occur within a CPU package, this
 * function should be called.  It forces the sched domain code to
 * only use the SMT level for the CPU portion of the topology.
 * This essentially falls back to relying on NUMA information
 * from the SRAT table to describe the entire system topology
 * (except for hyperthreads).
 */
static void primarily_use_numa_for_topology(void)
{
	set_sched_topology(numa_inside_package_topology);
}

490
void set_cpu_sibling_map(int cpu)
491
{
492
	bool has_smt = smp_num_siblings > 1;
493
	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
494
	struct cpuinfo_x86 *c = &cpu_data(cpu);
495 496
	struct cpuinfo_x86 *o;
	int i;
497

498
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
499

500
	if (!has_mp) {
501
		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
502
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
503
		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
504 505 506 507
		c->booted_cores = 1;
		return;
	}

508
	for_each_cpu(i, cpu_sibling_setup_mask) {
509 510 511
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
512
			link_mask(topology_sibling_cpumask, cpu, i);
513

514
		if ((i == cpu) || (has_mp && match_llc(c, o)))
515
			link_mask(cpu_llc_shared_mask, cpu, i);
516

517 518 519 520
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
521
	 * topology_sibling_cpumask links to be set-up.
522 523 524 525
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

526
		if ((i == cpu) || (has_mp && match_die(c, o))) {
527
			link_mask(topology_core_cpumask, cpu, i);
528

529 530 531
			/*
			 *  Does this new cpu bringup a new core?
			 */
532 533
			if (cpumask_weight(
			    topology_sibling_cpumask(cpu)) == 1) {
534 535 536 537
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
538 539
				if (cpumask_first(
				    topology_sibling_cpumask(i)) == i)
540 541 542 543 544 545 546 547 548 549
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
550
		if (match_die(c, o) && !topology_same_node(c, o))
551
			primarily_use_numa_for_topology();
552 553 554
	}
}

555
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
556
const struct cpumask *cpu_coregroup_mask(int cpu)
557
{
558
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
559 560
}

I
Ingo Molnar 已提交
561
static void impress_friends(void)
562 563 564 565 566 567
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
568
	pr_debug("Before bogomips\n");
569
	for_each_possible_cpu(cpu)
570
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
571
			bogosum += cpu_data(cpu).loops_per_jiffy;
572
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
573
		num_online_cpus(),
574 575 576
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

577
	pr_debug("Before bogocount - setting activated=1\n");
578 579
}

580
void __inquire_remote_apic(int apicid)
581 582
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
583
	const char * const names[] = { "ID", "VERSION", "SPIV" };
584 585 586
	int timeout;
	u32 status;

587
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
588 589

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
590
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
591 592 593 594 595 596

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
597
			pr_cont("a previous APIC delivery may have failed\n");
598

599
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
600 601 602 603 604 605 606 607 608 609

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
610
			pr_cont("%08x\n", status);
611 612
			break;
		default:
613
			pr_cont("failed\n");
614 615 616 617
		}
	}
}

618 619 620 621 622 623 624 625
/*
 * The Multiprocessor Specification 1.4 (1997) example code suggests
 * that there should be a 10ms delay between the BSP asserting INIT
 * and de-asserting INIT, when starting a remote processor.
 * But that slows boot and resume on modern processors, which include
 * many cores and don't require that delay.
 *
 * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
626
 * Modern processor families are quirked to remove the delay entirely.
627 628 629
 */
#define UDELAY_10MS_DEFAULT 10000

630
static unsigned int init_udelay = UINT_MAX;
631 632 633 634 635 636 637 638 639

static int __init cpu_init_udelay(char *str)
{
	get_option(&str, &init_udelay);

	return 0;
}
early_param("cpu_init_udelay", cpu_init_udelay);

640 641 642
static void __init smp_quirk_init_udelay(void)
{
	/* if cmdline changed it from default, leave it alone */
643
	if (init_udelay != UINT_MAX)
644 645 646 647
		return;

	/* if modern processor, use no delay */
	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
648
	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) {
649
		init_udelay = 0;
650 651
		return;
	}
652 653
	/* else, use legacy delay */
	init_udelay = UDELAY_10MS_DEFAULT;
654 655
}

656 657 658 659 660
/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
661
int
662
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
663 664 665 666 667 668 669
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
670
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
671

672
	pr_debug("Waiting for send to finish...\n");
673 674 675 676 677 678
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
679
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
680 681 682 683 684
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
685
	pr_debug("NMI sent\n");
686 687

	if (send_status)
688
		pr_err("APIC never delivered???\n");
689
	if (accept_status)
690
		pr_err("APIC delivery error (%lx)\n", accept_status);
691 692 693 694

	return (send_status | accept_status);
}

695
static int
696
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
697
{
698
	unsigned long send_status = 0, accept_status = 0;
699 700
	int maxlvt, num_starts, j;

701 702
	maxlvt = lapic_get_maxlvt();

703 704 705 706
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
707 708
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
709 710 711
		apic_read(APIC_ESR);
	}

712
	pr_debug("Asserting INIT\n");
713 714 715 716 717 718 719

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
720 721
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
722

723
	pr_debug("Waiting for send to finish...\n");
724 725
	send_status = safe_apic_wait_icr_idle();

726
	udelay(init_udelay);
727

728
	pr_debug("Deasserting INIT\n");
729 730 731

	/* Target chip */
	/* Send IPI */
732
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
733

734
	pr_debug("Waiting for send to finish...\n");
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
	send_status = safe_apic_wait_icr_idle();

	mb();

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Run STARTUP IPI loop.
	 */
753
	pr_debug("#startup loops: %d\n", num_starts);
754 755

	for (j = 1; j <= num_starts; j++) {
756
		pr_debug("Sending STARTUP #%d\n", j);
757 758
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
759
		apic_read(APIC_ESR);
760
		pr_debug("After apic_write\n");
761 762 763 764 765 766 767 768

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
769 770
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
771 772 773 774

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
L
Len Brown 已提交
775 776 777
		if (init_udelay == 0)
			udelay(10);
		else
778
			udelay(300);
779

780
		pr_debug("Startup point 1\n");
781

782
		pr_debug("Waiting for send to finish...\n");
783 784 785 786 787
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
L
Len Brown 已提交
788 789 790
		if (init_udelay == 0)
			udelay(10);
		else
791
			udelay(200);
792

793
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
794 795 796 797 798
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
799
	pr_debug("After Startup\n");
800 801

	if (send_status)
802
		pr_err("APIC never delivered???\n");
803
	if (accept_status)
804
		pr_err("APIC delivery error (%lx)\n", accept_status);
805 806 807 808

	return (send_status | accept_status);
}

809 810 811 812 813 814 815 816
void smp_announce(void)
{
	int num_nodes = num_online_nodes();

	printk(KERN_INFO "x86: Booted up %d node%s, %d CPUs\n",
	       num_nodes, (num_nodes > 1 ? "s" : ""), num_online_cpus());
}

817
/* reduce the number of lines printed when booting a large cpu count system */
818
static void announce_cpu(int cpu, int apicid)
819 820
{
	static int current_node = -1;
821
	int node = early_cpu_to_node(cpu);
822
	static int width, node_width;
823 824 825

	if (!width)
		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
826

827 828 829 830 831 832
	if (!node_width)
		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */

	if (cpu == 1)
		printk(KERN_INFO "x86: Booting SMP configuration:\n");

833 834 835
	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
836
				pr_cont("\n");
837
			current_node = node;
838 839 840

			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
			       node_width - num_digits(node), " ", node);
841
		}
842 843 844 845 846 847 848

		/* Add padding for the BSP */
		if (cpu == 1)
			pr_cont("%*s", width + 1, " ");

		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);

849 850 851 852 853
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
877
static int
878 879 880 881 882 883
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

884 885
	preempt_disable();

886 887 888
	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
889 890 891 892
	if (cpu) {
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
		goto out;
	}
893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}
911 912 913

out:
	preempt_enable();
914 915 916 917

	return boot_error;
}

918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
void common_cpu_up(unsigned int cpu, struct task_struct *idle)
{
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();

	per_cpu(current_task, cpu) = idle;

#ifdef CONFIG_X86_32
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
	per_cpu(cpu_current_top_of_stack, cpu) =
		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
#else
	clear_tsk_thread_flag(idle, TIF_FORK);
	initial_gs = per_cpu_offset(cpu);
#endif
}

936 937 938
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
939 940
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
941
 */
942
static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
943
{
944
	volatile u32 *trampoline_status =
945
		(volatile u32 *) __va(real_mode_header->trampoline_status);
946
	/* start_ip had better be page-aligned! */
947
	unsigned long start_ip = real_mode_header->trampoline_start;
948

949
	unsigned long boot_error = 0;
950
	int cpu0_nmi_registered = 0;
951
	unsigned long timeout;
952

953 954
	idle->thread.sp = (unsigned long) (((struct pt_regs *)
			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
955

956
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
957
	initial_code = (unsigned long)start_secondary;
958
	stack_start  = idle->thread.sp;
959

960 961 962 963 964 965 966
	/*
	 * Enable the espfix hack for this CPU
	*/
#ifdef CONFIG_X86_ESPFIX64
	init_espfix_ap(cpu);
#endif

967 968
	/* So we see what's up */
	announce_cpu(cpu, apicid);
969 970 971 972 973 974

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

J
Jack Steiner 已提交
975
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
976

977
		pr_debug("Setting warm reset code and vector.\n");
978

J
Jack Steiner 已提交
979 980 981
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
982 983 984 985 986
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
987
	}
988

989 990 991 992 993 994 995 996 997
	/*
	 * AP might wait on cpu_callout_mask in cpu_init() with
	 * cpu_initialized_mask set if previous attempt to online
	 * it timed-out. Clear cpu_initialized_mask so that after
	 * INIT/SIPI it could start with a clean state.
	 */
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
	smp_mb();

998
	/*
999 1000 1001 1002
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
1003
	 */
1004 1005 1006
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
1007 1008
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
1009 1010 1011

	if (!boot_error) {
		/*
1012
		 * Wait 10s total for first sign of life from AP
1013
		 */
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
		boot_error = -1;
		timeout = jiffies + 10*HZ;
		while (time_before(jiffies, timeout)) {
			if (cpumask_test_cpu(cpu, cpu_initialized_mask)) {
				/*
				 * Tell AP to proceed with initialization
				 */
				cpumask_set_cpu(cpu, cpu_callout_mask);
				boot_error = 0;
				break;
			}
			schedule();
		}
	}
1028

1029
	if (!boot_error) {
1030
		/*
1031
		 * Wait till AP completes initial initialization
1032
		 */
1033
		while (!cpumask_test_cpu(cpu, cpu_callin_mask)) {
1034 1035 1036 1037 1038 1039 1040
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
1041 1042 1043 1044
		}
	}

	/* mark "stuck" area as not stuck */
1045
	*trampoline_status = 0;
1046

1047 1048 1049 1050 1051 1052
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
1053 1054 1055 1056 1057 1058 1059
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

1060 1061 1062
	return boot_error;
}

1063
int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
1064
{
1065
	int apicid = apic->cpu_present_to_apicid(cpu);
1066 1067 1068 1069 1070
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

1071
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
1072

1073
	if (apicid == BAD_APICID ||
1074
	    !physid_isset(apicid, phys_cpu_present_map) ||
1075
	    !apic->apic_id_valid(apicid)) {
1076
		pr_err("%s: bad cpu %d\n", __func__, cpu);
1077 1078 1079 1080 1081 1082
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
1083
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
1084
		pr_debug("do_boot_cpu %d Already started\n", cpu);
1085 1086 1087 1088 1089 1090 1091 1092 1093
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

1094 1095 1096 1097
	/* x86 CPUs take themselves offline, so delayed offline is OK. */
	err = cpu_check_up_prepare(cpu);
	if (err && err != -EBUSY)
		return err;
1098

1099 1100 1101
	/* the FPU context is blank, nobody can own it */
	__cpu_disable_lazy_restore(cpu);

1102 1103
	common_cpu_up(cpu, tidle);

1104 1105 1106 1107 1108 1109 1110
	/*
	 * We have to walk the irq descriptors to setup the vector
	 * space for the cpu which comes online.  Prevent irq
	 * alloc/free across the bringup.
	 */
	irq_lock_sparse();

1111
	err = do_boot_cpu(apicid, cpu, tidle);
1112

1113
	if (err) {
1114
		irq_unlock_sparse();
1115
		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
1116
		return -EIO;
1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

1127
	while (!cpu_online(cpu)) {
1128 1129 1130 1131
		cpu_relax();
		touch_nmi_watchdog();
	}

1132 1133
	irq_unlock_sparse();

1134 1135 1136
	return 0;
}

1137 1138 1139 1140 1141 1142 1143 1144
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

1145 1146 1147 1148 1149 1150 1151
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
1152 1153
	pr_info("SMP disabled\n");

1154 1155
	disable_ioapic_support();

1156 1157
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
1158

1159
	if (smp_found_config)
1160
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1161
	else
1162
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1163 1164
	cpumask_set_cpu(0, topology_sibling_cpumask(0));
	cpumask_set_cpu(0, topology_core_cpumask(0));
1165 1166
}

1167 1168 1169 1170 1171 1172 1173
enum {
	SMP_OK,
	SMP_NO_CONFIG,
	SMP_NO_APIC,
	SMP_FORCE_UP,
};

1174 1175 1176 1177 1178
/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
1179
	preempt_disable();
1180

1181
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1182 1183 1184 1185
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

1186 1187
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
1188 1189 1190 1191

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
1192
				set_cpu_present(cpu, false);
1193 1194 1195 1196 1197 1198
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
1199
				set_cpu_possible(cpu, false);
1200 1201 1202 1203 1204 1205 1206
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

1207
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1208
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
1209 1210
			hard_smp_processor_id());

1211 1212 1213 1214 1215 1216 1217 1218
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1219
		preempt_enable();
1220
		pr_notice("SMP motherboard not detected\n");
1221
		return SMP_NO_CONFIG;
1222 1223 1224 1225 1226 1227
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1228
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1229 1230
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1231 1232
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1233
	preempt_enable();
1234 1235 1236 1237 1238

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
1239
	    !boot_cpu_has(X86_FEATURE_APIC)) {
1240 1241 1242
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1243
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1244
		}
1245
		return SMP_NO_APIC;
1246 1247 1248 1249 1250 1251
	}

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1252
		pr_info("SMP mode deactivated\n");
1253
		return SMP_FORCE_UP;
1254 1255
	}

1256
	return SMP_OK;
1257 1258 1259 1260 1261 1262 1263
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1264
	for_each_possible_cpu(i) {
1265 1266
		c = &cpu_data(i);
		/* mark all to hotplug */
1267
		c->cpu_index = nr_cpu_ids;
1268 1269 1270 1271 1272 1273 1274 1275 1276
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1277 1278
	unsigned int i;

1279
	smp_cpu_index_default();
1280

1281 1282 1283
	/*
	 * Setup boot CPU information
	 */
1284
	smp_store_boot_cpu_info(); /* Final full version of the data */
1285 1286
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1287

1288
	current_thread_info()->cpu = 0;  /* needed? */
1289
	for_each_possible_cpu(i) {
1290 1291
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1292
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1293
	}
1294 1295
	set_cpu_sibling_map(0);

1296 1297
	switch (smp_sanity_check(max_cpus)) {
	case SMP_NO_CONFIG:
1298
		disable_smp();
1299 1300 1301 1302 1303 1304 1305 1306
		if (APIC_init_uniprocessor())
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
		return;
	case SMP_NO_APIC:
		disable_smp();
		return;
	case SMP_FORCE_UP:
		disable_smp();
1307
		apic_bsp_setup(false);
1308
		return;
1309 1310
	case SMP_OK:
		break;
1311 1312
	}

1313 1314
	default_setup_apic_routing();

1315
	if (read_apic_id() != boot_cpu_physical_apicid) {
1316
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1317
		     read_apic_id(), boot_cpu_physical_apicid);
1318 1319 1320
		/* Or can we switch back to PIC here? */
	}

1321
	cpu0_logical_apicid = apic_bsp_setup(false);
1322

1323
	pr_info("CPU%d: ", 0);
1324
	print_cpu_info(&cpu_data(0));
1325 1326 1327

	if (is_uv_system())
		uv_system_init();
1328 1329

	set_mtrr_aps_delayed_init();
1330 1331

	smp_quirk_init_udelay();
1332
}
1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1344 1345 1346 1347 1348 1349
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1350
	switch_to_new_gdt(me);
1351 1352
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1353
	cpu_set_state_online(me);
1354 1355
}

1356 1357
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1358
	pr_debug("Boot done\n");
1359

D
Don Zickus 已提交
1360
	nmi_selftest();
1361 1362
	impress_friends();
	setup_ioapic_dest();
1363
	mtrr_aps_init();
1364 1365
}

1366 1367 1368 1369 1370 1371 1372 1373 1374
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1375
/*
1376
 * cpu_possible_mask should be static, it cannot change as cpu's
1377 1378 1379
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1380
 * cpu_present_mask on the other hand can change dynamically.
1381 1382 1383 1384 1385 1386
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1387
 * - The user can overwrite it with possible_cpus=NUM
1388 1389 1390 1391 1392 1393
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1394
	int i, possible;
1395

1396 1397 1398 1399
	/* no processor from mptable or madt */
	if (!num_processors)
		num_processors = 1;

1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1411 1412
		possible = setup_possible_cpus;

1413 1414
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1415 1416
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1417
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1418 1419
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1420
	}
1421

1422 1423 1424 1425
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1426
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1427 1428 1429 1430
			possible, setup_max_cpus);
		possible = i;
	}

1431
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1432 1433 1434
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1435
		set_cpu_possible(i, true);
1436 1437
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1438 1439

	nr_cpu_ids = possible;
1440
}
1441

1442 1443 1444 1445 1446 1447 1448
#ifdef CONFIG_HOTPLUG_CPU

static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1449 1450
	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1451 1452 1453
		/*/
		 * last thread sibling in this cpu core going down
		 */
1454
		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1455 1456 1457
			cpu_data(sibling).booted_cores--;
	}

1458 1459
	for_each_cpu(sibling, topology_sibling_cpumask(cpu))
		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1460 1461 1462
	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
	cpumask_clear(cpu_llc_shared_mask(cpu));
1463 1464
	cpumask_clear(topology_sibling_cpumask(cpu));
	cpumask_clear(topology_core_cpumask(cpu));
1465 1466
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1467
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1468 1469
}

1470
static void remove_cpu_from_maps(int cpu)
1471
{
1472 1473 1474
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1475
	/* was set by cpu_init() */
1476
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1477
	numa_remove_cpu(cpu);
1478 1479
}

1480
void cpu_disable_common(void)
1481 1482 1483 1484 1485 1486
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1487
	lock_vector_lock();
1488
	remove_cpu_from_maps(cpu);
1489
	unlock_vector_lock();
1490
	fixup_irqs();
1491 1492 1493 1494
}

int native_cpu_disable(void)
{
1495 1496 1497 1498 1499 1500
	int ret;

	ret = check_irq_vectors_for_cpu_disable();
	if (ret)
		return ret;

1501 1502
	clear_local_APIC();
	cpu_disable_common();
1503

1504 1505 1506
	return 0;
}

1507
int common_cpu_die(unsigned int cpu)
1508
{
1509
	int ret = 0;
1510

1511
	/* We don't do anything here: idle task is faking death itself. */
1512

1513
	/* They ack this in play_dead() by setting CPU_DEAD */
1514
	if (cpu_wait_death(cpu, 5)) {
1515 1516 1517 1518
		if (system_state == SYSTEM_RUNNING)
			pr_info("CPU %u is now offline\n", cpu);
	} else {
		pr_err("CPU %u didn't die...\n", cpu);
1519
		ret = -1;
1520
	}
1521 1522 1523 1524 1525 1526 1527

	return ret;
}

void native_cpu_die(unsigned int cpu)
{
	common_cpu_die(cpu);
1528
}
1529 1530 1531 1532 1533

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();
1534
	amd_e400_remove_cpu(raw_smp_processor_id());
1535 1536

	/* Ack it */
1537
	(void)cpu_report_death();
1538 1539 1540 1541 1542 1543 1544

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1545 1546 1547 1548 1549 1550 1551 1552
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1553 1554 1555 1556 1557 1558 1559 1560 1561
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
1562
	void *mwait_ptr;
1563
	int i;
1564

1565
	if (!this_cpu_has(X86_FEATURE_MWAIT))
1566
		return;
1567
	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1568
		return;
1569
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1594 1595 1596 1597 1598 1599 1600
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1601 1602
	wbinvd();

1603
	while (1) {
1604 1605 1606 1607 1608 1609 1610
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
1611
		mb();
1612
		clflush(mwait_ptr);
1613
		mb();
1614
		__monitor(mwait_ptr, 0, 0);
1615 1616
		mb();
		__mwait(eax, 0);
1617 1618 1619 1620 1621
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1622 1623 1624 1625 1626
	}
}

static inline void hlt_play_dead(void)
{
1627
	if (__this_cpu_read(cpu_info.x86) >= 4)
1628 1629
		wbinvd();

1630 1631
	while (1) {
		native_halt();
1632 1633 1634 1635 1636
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1637 1638 1639
	}
}

1640 1641 1642
void native_play_dead(void)
{
	play_dead_common();
1643
	tboot_shutdown(TB_SHUTDOWN_WFS);
1644 1645

	mwait_play_dead();	/* Only returns on failure */
1646 1647
	if (cpuidle_play_dead())
		hlt_play_dead();
1648 1649
}

1650
#else /* ... !CONFIG_HOTPLUG_CPU */
1651
int native_cpu_disable(void)
1652 1653 1654 1655
{
	return -ENOSYS;
}

1656
void native_cpu_die(unsigned int cpu)
1657 1658 1659 1660
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1661 1662 1663 1664 1665 1666

void native_play_dead(void)
{
	BUG();
}

1667
#endif