smpboot.c 35.3 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/module.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/idle.h>
62
#include <asm/realmode.h>
63 64
#include <asm/cpu.h>
#include <asm/numa.h>
65 66 67
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
68
#include <asm/mwait.h>
I
Ingo Molnar 已提交
69
#include <asm/apic.h>
70
#include <asm/io_apic.h>
71 72
#include <asm/i387.h>
#include <asm/fpu-internal.h>
73
#include <asm/setup.h>
T
Tejun Heo 已提交
74
#include <asm/uv/uv.h>
75
#include <linux/mc146818rtc.h>
76
#include <asm/smpboot_hooks.h>
77
#include <asm/i8259.h>
78
#include <asm/realmode.h>
79
#include <asm/misc.h>
80

81 82 83
/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

84 85 86 87 88
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
89
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
90 91

/* representing HT siblings of each logical CPU */
92
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
93 94 95
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
96
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
97 98
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

99
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
100

101 102 103
/* Per CPU bogomips and other parameters */
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
104

105
atomic_t init_deasserted;
106 107

/*
108 109
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
110
 */
111
static void smp_callin(void)
112 113 114 115 116 117 118 119 120
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
121 122
	 *
	 * Since CPU0 is not wakened up by INIT, it doesn't wait for the IPI.
123
	 */
124
	cpuid = smp_processor_id();
125 126 127
	if (apic->wait_for_init_deassert && cpuid)
		while (!atomic_read(&init_deasserted))
			cpu_relax();
128 129 130 131

	/*
	 * (This works even if the APIC is not enabled.)
	 */
132
	phys_id = read_apic_id();
133
	if (cpumask_test_cpu(cpuid, cpu_callin_mask)) {
134 135 136
		panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
					phys_id, cpuid);
	}
137
	pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
155
		if (cpumask_test_cpu(cpuid, cpu_callout_mask))
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
			break;
		cpu_relax();
	}

	if (!time_before(jiffies, timeout)) {
		panic("%s: CPU%d started up but did not get a callout!\n",
		      __func__, cpuid);
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

172
	pr_debug("CALLIN, before setup_local_APIC()\n");
173 174
	if (apic->smp_callin_clear_local_apic)
		apic->smp_callin_clear_local_apic();
175 176 177
	setup_local_APIC();
	end_local_APIC_setup();

178 179 180
	/*
	 * Need to setup vector mappings before we enable interrupts.
	 */
181
	setup_vector_irq(smp_processor_id());
182 183 184 185 186 187 188

	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

189 190
	/*
	 * Get our bogomips.
191 192 193
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
194 195
	 */
	calibrate_delay();
196
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
197
	pr_debug("Stack at about %p\n", &cpuid);
198

199 200 201 202 203 204 205
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

206 207
	notify_cpu_starting(cpuid);

208 209 210
	/*
	 * Allow the master to continue.
	 */
211
	cpumask_set_cpu(cpuid, cpu_callin_mask);
212 213
}

214 215
static int cpu0_logical_apicid;
static int enable_start_cpu0;
216 217 218
/*
 * Activate a secondary processor.
 */
219
static void notrace start_secondary(void *unused)
220 221 222 223 224 225
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
226
	cpu_init();
227
	x86_cpuinit.early_percpu_clock_init();
228 229
	preempt_disable();
	smp_callin();
230

231 232
	enable_start_cpu0 = 0;

233
#ifdef CONFIG_X86_32
234
	/* switch away from the initial page table */
235 236 237 238
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

239 240 241 242 243 244 245 246
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	/*
247 248 249
	 * We need to hold vector_lock so there the set of online cpus
	 * does not change while we are assigning vectors to cpus.  Holding
	 * this lock ensures we don't half assign or remove an irq from a cpu.
250
	 */
251
	lock_vector_lock();
252
	set_cpu_online(smp_processor_id(), true);
253
	unlock_vector_lock();
254
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
255
	x86_platform.nmi_init();
256

257 258 259
	/* enable local interrupts */
	local_irq_enable();

260 261
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
262

263
	x86_cpuinit.setup_percpu_clockev();
264 265

	wmb();
T
Thomas Gleixner 已提交
266
	cpu_startup_entry(CPUHP_ONLINE);
267 268
}

269 270 271 272 273 274 275 276 277
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
}

278 279 280 281
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
282
void smp_store_cpu_info(int id)
283 284 285
{
	struct cpuinfo_x86 *c = &cpu_data(id);

286
	*c = boot_cpu_data;
287
	c->cpu_index = id;
288 289 290 291 292
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
293 294
}

295
static bool
296
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
297
{
298 299 300 301 302 303 304 305 306 307 308 309 310 311
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return !WARN_ONCE(cpu_to_node(cpu1) != cpu_to_node(cpu2),
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

#define link_mask(_m, c1, c2)						\
do {									\
	cpumask_set_cpu((c1), cpu_##_m##_mask(c2));			\
	cpumask_set_cpu((c2), cpu_##_m##_mask(c1));			\
} while (0)

312
static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
313
{
A
Andreas Herrmann 已提交
314
	if (cpu_has_topoext) {
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
		    c->compute_unit_id == o->compute_unit_id)
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

330
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
331 332 333 334 335 336 337 338
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
339 340
}

341
static bool match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
342
{
343 344 345
	if (c->phys_proc_id == o->phys_proc_id) {
		if (cpu_has(c, X86_FEATURE_AMD_DCM))
			return true;
346

347 348
		return topology_sane(c, o, "mc");
	}
349 350
	return false;
}
351

352
void set_cpu_sibling_map(int cpu)
353
{
354
	bool has_smt = smp_num_siblings > 1;
355
	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
356
	struct cpuinfo_x86 *c = &cpu_data(cpu);
357 358
	struct cpuinfo_x86 *o;
	int i;
359

360
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
361

362
	if (!has_mp) {
363
		cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
364 365
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
		cpumask_set_cpu(cpu, cpu_core_mask(cpu));
366 367 368 369
		c->booted_cores = 1;
		return;
	}

370
	for_each_cpu(i, cpu_sibling_setup_mask) {
371 372 373 374 375
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
			link_mask(sibling, cpu, i);

376
		if ((i == cpu) || (has_mp && match_llc(c, o)))
377 378
			link_mask(llc_shared, cpu, i);

379 380 381 382 383 384 385 386 387
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
	 * cpu_sibling_mask links to be set-up.
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

388
		if ((i == cpu) || (has_mp && match_mc(c, o))) {
389 390
			link_mask(core, cpu, i);

391 392 393
			/*
			 *  Does this new cpu bringup a new core?
			 */
394
			if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) {
395 396 397 398
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
399
				if (cpumask_first(cpu_sibling_mask(i)) == i)
400 401 402 403 404 405 406 407 408 409 410 411 412
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
	}
}

413
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
414
const struct cpumask *cpu_coregroup_mask(int cpu)
415
{
416
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
417 418
}

I
Ingo Molnar 已提交
419
static void impress_friends(void)
420 421 422 423 424 425
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
426
	pr_debug("Before bogomips\n");
427
	for_each_possible_cpu(cpu)
428
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
429
			bogosum += cpu_data(cpu).loops_per_jiffy;
430
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
431
		num_online_cpus(),
432 433 434
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

435
	pr_debug("Before bogocount - setting activated=1\n");
436 437
}

438
void __inquire_remote_apic(int apicid)
439 440
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
441
	const char * const names[] = { "ID", "VERSION", "SPIV" };
442 443 444
	int timeout;
	u32 status;

445
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
446 447

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
448
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
449 450 451 452 453 454

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
455
			pr_cont("a previous APIC delivery may have failed\n");
456

457
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
458 459 460 461 462 463 464 465 466 467

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
468
			pr_cont("%08x\n", status);
469 470
			break;
		default:
471
			pr_cont("failed\n");
472 473 474 475 476 477 478 479 480
		}
	}
}

/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
481
int
482
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
483 484 485 486 487 488 489
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
490
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
491

492
	pr_debug("Waiting for send to finish...\n");
493 494 495 496 497 498
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
499
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
500 501 502 503 504
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
505
	pr_debug("NMI sent\n");
506 507

	if (send_status)
508
		pr_err("APIC never delivered???\n");
509
	if (accept_status)
510
		pr_err("APIC delivery error (%lx)\n", accept_status);
511 512 513 514

	return (send_status | accept_status);
}

515
static int
516
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
517 518 519 520
{
	unsigned long send_status, accept_status = 0;
	int maxlvt, num_starts, j;

521 522
	maxlvt = lapic_get_maxlvt();

523 524 525 526
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
527 528
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
529 530 531
		apic_read(APIC_ESR);
	}

532
	pr_debug("Asserting INIT\n");
533 534 535 536 537 538 539

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
540 541
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
542

543
	pr_debug("Waiting for send to finish...\n");
544 545 546 547
	send_status = safe_apic_wait_icr_idle();

	mdelay(10);

548
	pr_debug("Deasserting INIT\n");
549 550 551

	/* Target chip */
	/* Send IPI */
552
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
553

554
	pr_debug("Waiting for send to finish...\n");
555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
	send_status = safe_apic_wait_icr_idle();

	mb();
	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Paravirt / VMI wants a startup IPI hook here to set up the
	 * target processor state.
	 */
	startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
576
			 stack_start);
577 578 579 580

	/*
	 * Run STARTUP IPI loop.
	 */
581
	pr_debug("#startup loops: %d\n", num_starts);
582 583

	for (j = 1; j <= num_starts; j++) {
584
		pr_debug("Sending STARTUP #%d\n", j);
585 586
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
587
		apic_read(APIC_ESR);
588
		pr_debug("After apic_write\n");
589 590 591 592 593 594 595 596

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
597 598
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
599 600 601 602 603 604

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

605
		pr_debug("Startup point 1\n");
606

607
		pr_debug("Waiting for send to finish...\n");
608 609 610 611 612 613
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
614
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
615 616 617 618 619
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
620
	pr_debug("After Startup\n");
621 622

	if (send_status)
623
		pr_err("APIC never delivered???\n");
624
	if (accept_status)
625
		pr_err("APIC delivery error (%lx)\n", accept_status);
626 627 628 629

	return (send_status | accept_status);
}

630 631 632 633 634 635 636 637
void smp_announce(void)
{
	int num_nodes = num_online_nodes();

	printk(KERN_INFO "x86: Booted up %d node%s, %d CPUs\n",
	       num_nodes, (num_nodes > 1 ? "s" : ""), num_online_cpus());
}

638
/* reduce the number of lines printed when booting a large cpu count system */
639
static void announce_cpu(int cpu, int apicid)
640 641
{
	static int current_node = -1;
642
	int node = early_cpu_to_node(cpu);
643
	static int width, node_width;
644 645 646

	if (!width)
		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
647

648 649 650 651 652 653
	if (!node_width)
		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */

	if (cpu == 1)
		printk(KERN_INFO "x86: Booting SMP configuration:\n");

654 655 656
	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
657
				pr_cont("\n");
658
			current_node = node;
659 660 661

			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
			       node_width - num_digits(node), " ", node);
662
		}
663 664 665 666 667 668 669

		/* Add padding for the BSP */
		if (cpu == 1)
			pr_cont("%*s", width + 1, " ");

		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);

670 671 672 673 674
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
698
static int
699 700 701 702 703 704
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

705 706
	preempt_disable();

707 708 709
	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
710 711 712 713
	if (cpu) {
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
		goto out;
	}
714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}
732 733 734

out:
	preempt_enable();
735 736 737 738

	return boot_error;
}

739 740 741
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
742 743
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
744
 */
745
static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
746
{
747
	volatile u32 *trampoline_status =
748
		(volatile u32 *) __va(real_mode_header->trampoline_status);
749
	/* start_ip had better be page-aligned! */
750
	unsigned long start_ip = real_mode_header->trampoline_start;
751

752
	unsigned long boot_error = 0;
753
	int timeout;
754
	int cpu0_nmi_registered = 0;
755

756 757
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();
758

759 760 761
	idle->thread.sp = (unsigned long) (((struct pt_regs *)
			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
	per_cpu(current_task, cpu) = idle;
762

763
#ifdef CONFIG_X86_32
764 765 766
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
#else
767
	clear_tsk_thread_flag(idle, TIF_FORK);
768
	initial_gs = per_cpu_offset(cpu);
769
#endif
770
	per_cpu(kernel_stack, cpu) =
771
		(unsigned long)task_stack_page(idle) -
772
		KERNEL_STACK_OFFSET + THREAD_SIZE;
773
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
774
	initial_code = (unsigned long)start_secondary;
775
	stack_start  = idle->thread.sp;
776

777 778
	/* So we see what's up */
	announce_cpu(cpu, apicid);
779 780 781 782 783 784 785 786

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

J
Jack Steiner 已提交
787
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
788

789
		pr_debug("Setting warm reset code and vector.\n");
790

J
Jack Steiner 已提交
791 792 793
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
794 795 796 797 798
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
799
	}
800 801

	/*
802 803 804 805
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
806
	 */
807 808 809
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
810 811
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
812 813 814 815 816

	if (!boot_error) {
		/*
		 * allow APs to start initializing.
		 */
817
		pr_debug("Before Callout %d\n", cpu);
818
		cpumask_set_cpu(cpu, cpu_callout_mask);
819
		pr_debug("After Callout %d\n", cpu);
820 821 822 823 824

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
825
			if (cpumask_test_cpu(cpu, cpu_callin_mask))
826 827
				break;	/* It has booted */
			udelay(100);
828 829 830 831 832 833 834
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
835 836
		}

837 838
		if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
			print_cpu_msr(&cpu_data(cpu));
839
			pr_debug("CPU%d: has booted.\n", cpu);
840
		} else {
841
			boot_error = 1;
842
			if (*trampoline_status == 0xA5A5A5A5)
843
				/* trampoline started but...? */
844
				pr_err("CPU%d: Stuck ??\n", cpu);
845 846
			else
				/* trampoline code not run */
847
				pr_err("CPU%d: Not responding\n", cpu);
848 849
			if (apic->inquire_remote_apic)
				apic->inquire_remote_apic(apicid);
850 851
		}
	}
852

853 854
	if (boot_error) {
		/* Try to put things back the way they were before ... */
855
		numa_remove_cpu(cpu); /* was set by numa_add_cpu */
856 857 858 859 860 861

		/* was set by do_boot_cpu() */
		cpumask_clear_cpu(cpu, cpu_callout_mask);

		/* was set by cpu_init() */
		cpumask_clear_cpu(cpu, cpu_initialized_mask);
862 863 864
	}

	/* mark "stuck" area as not stuck */
865
	*trampoline_status = 0;
866

867 868 869 870 871 872
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
873 874 875 876 877 878 879
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

880 881 882
	return boot_error;
}

883
int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
884
{
885
	int apicid = apic->cpu_present_to_apicid(cpu);
886 887 888 889 890
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

891
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
892

893
	if (apicid == BAD_APICID ||
894
	    !physid_isset(apicid, phys_cpu_present_map) ||
895
	    !apic->apic_id_valid(apicid)) {
896
		pr_err("%s: bad cpu %d\n", __func__, cpu);
897 898 899 900 901 902
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
903
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
904
		pr_debug("do_boot_cpu %d Already started\n", cpu);
905 906 907 908 909 910 911 912 913 914 915
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;

916 917 918
	/* the FPU context is blank, nobody can own it */
	__cpu_disable_lazy_restore(cpu);

919
	err = do_boot_cpu(apicid, cpu, tidle);
920
	if (err) {
921
		pr_debug("do_boot_cpu failed %d\n", err);
922
		return -EIO;
923 924 925 926 927 928 929 930 931 932
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

933
	while (!cpu_online(cpu)) {
934 935 936 937 938 939 940
		cpu_relax();
		touch_nmi_watchdog();
	}

	return 0;
}

941 942 943 944 945 946 947 948
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

949 950 951 952 953 954 955
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
956 957
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
958
	smpboot_clear_io_apic_irqs();
959

960
	if (smp_found_config)
961
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
962
	else
963
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
964 965
	cpumask_set_cpu(0, cpu_sibling_mask(0));
	cpumask_set_cpu(0, cpu_core_mask(0));
966 967 968 969 970 971 972
}

/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
973
	preempt_disable();
974

975
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
976 977 978 979
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

980 981
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
982 983 984 985

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
986
				set_cpu_present(cpu, false);
987 988 989 990 991 992
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
993
				set_cpu_possible(cpu, false);
994 995 996 997 998 999 1000
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

1001
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1002
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
1003 1004
			hard_smp_processor_id());

1005 1006 1007 1008 1009 1010 1011 1012
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1013
		preempt_enable();
1014
		pr_notice("SMP motherboard not detected\n");
1015 1016
		disable_smp();
		if (APIC_init_uniprocessor())
1017
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
1018 1019 1020 1021 1022 1023 1024
		return -1;
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1025
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1026 1027
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1028 1029
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1030
	preempt_enable();
1031 1032 1033 1034 1035 1036

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
	    !cpu_has_apic) {
1037 1038 1039
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1040
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1041
		}
1042
		smpboot_clear_io_apic();
1043
		disable_ioapic_support();
1044 1045 1046 1047 1048 1049 1050 1051 1052
		return -1;
	}

	verify_local_APIC();

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1053
		pr_info("SMP mode deactivated\n");
1054
		smpboot_clear_io_apic();
1055

1056 1057
		connect_bsp_APIC();
		setup_local_APIC();
1058
		bsp_end_local_APIC_setup();
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069
		return -1;
	}

	return 0;
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1070
	for_each_possible_cpu(i) {
1071 1072
		c = &cpu_data(i);
		/* mark all to hotplug */
1073
		c->cpu_index = nr_cpu_ids;
1074 1075 1076 1077 1078 1079 1080 1081 1082
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1083 1084
	unsigned int i;

1085
	preempt_disable();
1086
	smp_cpu_index_default();
1087

1088 1089 1090
	/*
	 * Setup boot CPU information
	 */
1091
	smp_store_boot_cpu_info(); /* Final full version of the data */
1092 1093
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1094

1095
	current_thread_info()->cpu = 0;  /* needed? */
1096
	for_each_possible_cpu(i) {
1097 1098
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1099
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1100
	}
1101 1102
	set_cpu_sibling_map(0);

1103

1104
	if (smp_sanity_check(max_cpus) < 0) {
1105
		pr_info("SMP disabled\n");
1106
		disable_smp();
1107
		goto out;
1108 1109
	}

1110 1111
	default_setup_apic_routing();

J
Jack Steiner 已提交
1112
	preempt_disable();
1113
	if (read_apic_id() != boot_cpu_physical_apicid) {
1114
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1115
		     read_apic_id(), boot_cpu_physical_apicid);
1116 1117
		/* Or can we switch back to PIC here? */
	}
J
Jack Steiner 已提交
1118
	preempt_enable();
1119 1120

	connect_bsp_APIC();
1121

1122 1123 1124 1125 1126
	/*
	 * Switch from PIC to APIC mode.
	 */
	setup_local_APIC();

1127 1128 1129 1130 1131
	if (x2apic_mode)
		cpu0_logical_apicid = apic_read(APIC_LDR);
	else
		cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));

1132 1133 1134 1135 1136
	/*
	 * Enable IO APIC before setting up error vector
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		enable_IO_APIC();
1137

1138
	bsp_end_local_APIC_setup();
1139

1140 1141
	if (apic->setup_portio_remap)
		apic->setup_portio_remap();
1142 1143 1144 1145 1146 1147

	smpboot_setup_io_apic();
	/*
	 * Set up local APIC timer on boot CPU.
	 */

1148
	pr_info("CPU%d: ", 0);
1149
	print_cpu_info(&cpu_data(0));
1150
	x86_init.timers.setup_percpu_clockev();
1151 1152 1153

	if (is_uv_system())
		uv_system_init();
1154 1155

	set_mtrr_aps_delayed_init();
1156 1157
out:
	preempt_enable();
1158
}
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1170 1171 1172 1173 1174 1175
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1176
	switch_to_new_gdt(me);
1177 1178
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1179 1180 1181
	per_cpu(cpu_state, me) = CPU_ONLINE;
}

1182 1183
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1184
	pr_debug("Boot done\n");
1185

D
Don Zickus 已提交
1186
	nmi_selftest();
1187 1188 1189 1190
	impress_friends();
#ifdef CONFIG_X86_IO_APIC
	setup_ioapic_dest();
#endif
1191
	mtrr_aps_init();
1192 1193
}

1194 1195 1196 1197 1198 1199 1200 1201 1202
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1203
/*
1204
 * cpu_possible_mask should be static, it cannot change as cpu's
1205 1206 1207
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1208
 * cpu_present_mask on the other hand can change dynamically.
1209 1210 1211 1212 1213 1214
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1215
 * - The user can overwrite it with possible_cpus=NUM
1216 1217 1218 1219 1220 1221
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1222
	int i, possible;
1223

1224 1225 1226 1227
	/* no processor from mptable or madt */
	if (!num_processors)
		num_processors = 1;

1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1239 1240
		possible = setup_possible_cpus;

1241 1242
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1243 1244
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1245
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1246 1247
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1248
	}
1249

1250 1251 1252 1253
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1254
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1255 1256 1257 1258
			possible, setup_max_cpus);
		possible = i;
	}

1259
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1260 1261 1262
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1263
		set_cpu_possible(i, true);
1264 1265
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1266 1267

	nr_cpu_ids = possible;
1268
}
1269

1270 1271 1272 1273 1274 1275 1276
#ifdef CONFIG_HOTPLUG_CPU

static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1277 1278
	for_each_cpu(sibling, cpu_core_mask(cpu)) {
		cpumask_clear_cpu(cpu, cpu_core_mask(sibling));
1279 1280 1281
		/*/
		 * last thread sibling in this cpu core going down
		 */
1282
		if (cpumask_weight(cpu_sibling_mask(cpu)) == 1)
1283 1284 1285
			cpu_data(sibling).booted_cores--;
	}

1286 1287 1288 1289
	for_each_cpu(sibling, cpu_sibling_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling));
	cpumask_clear(cpu_sibling_mask(cpu));
	cpumask_clear(cpu_core_mask(cpu));
1290 1291
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1292
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1293 1294
}

1295 1296
static void __ref remove_cpu_from_maps(int cpu)
{
1297 1298 1299
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1300
	/* was set by cpu_init() */
1301
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1302
	numa_remove_cpu(cpu);
1303 1304
}

1305
void cpu_disable_common(void)
1306 1307 1308 1309 1310 1311
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1312
	lock_vector_lock();
1313
	remove_cpu_from_maps(cpu);
1314
	unlock_vector_lock();
1315
	fixup_irqs();
1316 1317 1318 1319
}

int native_cpu_disable(void)
{
1320 1321 1322 1323 1324 1325
	int ret;

	ret = check_irq_vectors_for_cpu_disable();
	if (ret)
		return ret;

1326 1327 1328
	clear_local_APIC();

	cpu_disable_common();
1329 1330 1331
	return 0;
}

1332
void native_cpu_die(unsigned int cpu)
1333 1334 1335 1336 1337 1338 1339
{
	/* We don't do anything here: idle task is faking death itself. */
	unsigned int i;

	for (i = 0; i < 10; i++) {
		/* They ack this in play_dead by setting CPU_DEAD */
		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1340 1341
			if (system_state == SYSTEM_RUNNING)
				pr_info("CPU %u is now offline\n", cpu);
1342 1343 1344 1345
			return;
		}
		msleep(100);
	}
1346
	pr_err("CPU %u didn't die...\n", cpu);
1347
}
1348 1349 1350 1351 1352

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();
1353
	amd_e400_remove_cpu(raw_smp_processor_id());
1354 1355 1356

	mb();
	/* Ack it */
T
Tejun Heo 已提交
1357
	__this_cpu_write(cpu_state, CPU_DEAD);
1358 1359 1360 1361 1362 1363 1364

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1365 1366 1367 1368 1369 1370 1371 1372
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1373 1374 1375 1376 1377 1378 1379 1380 1381
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
1382
	void *mwait_ptr;
1383
	int i;
1384

1385
	if (!this_cpu_has(X86_FEATURE_MWAIT))
1386
		return;
1387
	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1388
		return;
1389
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1414 1415 1416 1417 1418 1419 1420
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1421 1422
	wbinvd();

1423
	while (1) {
1424 1425 1426 1427 1428 1429 1430
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
1431
		mb();
1432
		clflush(mwait_ptr);
1433
		mb();
1434
		__monitor(mwait_ptr, 0, 0);
1435 1436
		mb();
		__mwait(eax, 0);
1437 1438 1439 1440 1441
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1442 1443 1444 1445 1446
	}
}

static inline void hlt_play_dead(void)
{
1447
	if (__this_cpu_read(cpu_info.x86) >= 4)
1448 1449
		wbinvd();

1450 1451
	while (1) {
		native_halt();
1452 1453 1454 1455 1456
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1457 1458 1459
	}
}

1460 1461 1462
void native_play_dead(void)
{
	play_dead_common();
1463
	tboot_shutdown(TB_SHUTDOWN_WFS);
1464 1465

	mwait_play_dead();	/* Only returns on failure */
1466 1467
	if (cpuidle_play_dead())
		hlt_play_dead();
1468 1469
}

1470
#else /* ... !CONFIG_HOTPLUG_CPU */
1471
int native_cpu_disable(void)
1472 1473 1474 1475
{
	return -ENOSYS;
}

1476
void native_cpu_die(unsigned int cpu)
1477 1478 1479 1480
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1481 1482 1483 1484 1485 1486

void native_play_dead(void)
{
	BUG();
}

1487
#endif