smpboot.c 35.3 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/module.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/idle.h>
62
#include <asm/realmode.h>
63 64
#include <asm/cpu.h>
#include <asm/numa.h>
65 66 67
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
68
#include <asm/mwait.h>
I
Ingo Molnar 已提交
69
#include <asm/apic.h>
70
#include <asm/io_apic.h>
71 72
#include <asm/i387.h>
#include <asm/fpu-internal.h>
73
#include <asm/setup.h>
T
Tejun Heo 已提交
74
#include <asm/uv/uv.h>
75
#include <linux/mc146818rtc.h>
76
#include <asm/smpboot_hooks.h>
77
#include <asm/i8259.h>
78
#include <asm/realmode.h>
79
#include <asm/misc.h>
80

81 82 83
/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

84 85 86 87 88
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
89
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
90 91

/* representing HT siblings of each logical CPU */
92
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
93 94 95
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
96
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
97 98
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

99
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
100

101 102 103
/* Per CPU bogomips and other parameters */
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
104

105
atomic_t init_deasserted;
106 107

/*
108 109
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
110
 */
111
static void smp_callin(void)
112 113 114 115 116 117 118 119 120
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
121 122
	 *
	 * Since CPU0 is not wakened up by INIT, it doesn't wait for the IPI.
123
	 */
124
	cpuid = smp_processor_id();
125 126 127
	if (apic->wait_for_init_deassert && cpuid)
		while (!atomic_read(&init_deasserted))
			cpu_relax();
128 129 130 131

	/*
	 * (This works even if the APIC is not enabled.)
	 */
132
	phys_id = read_apic_id();
133
	if (cpumask_test_cpu(cpuid, cpu_callin_mask)) {
134 135 136
		panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
					phys_id, cpuid);
	}
137
	pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
155
		if (cpumask_test_cpu(cpuid, cpu_callout_mask))
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
			break;
		cpu_relax();
	}

	if (!time_before(jiffies, timeout)) {
		panic("%s: CPU%d started up but did not get a callout!\n",
		      __func__, cpuid);
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

172
	pr_debug("CALLIN, before setup_local_APIC()\n");
173 174
	if (apic->smp_callin_clear_local_apic)
		apic->smp_callin_clear_local_apic();
175 176 177
	setup_local_APIC();
	end_local_APIC_setup();

178 179 180
	/*
	 * Need to setup vector mappings before we enable interrupts.
	 */
181
	setup_vector_irq(smp_processor_id());
182 183 184 185 186 187 188

	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

189 190
	/*
	 * Get our bogomips.
191 192 193
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
194 195
	 */
	calibrate_delay();
196
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
197
	pr_debug("Stack at about %p\n", &cpuid);
198

199 200 201 202 203 204 205
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

206 207
	notify_cpu_starting(cpuid);

208 209 210
	/*
	 * Allow the master to continue.
	 */
211
	cpumask_set_cpu(cpuid, cpu_callin_mask);
212 213
}

214 215
static int cpu0_logical_apicid;
static int enable_start_cpu0;
216 217 218
/*
 * Activate a secondary processor.
 */
219
static void notrace start_secondary(void *unused)
220 221 222 223 224 225
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
226
	cpu_init();
227
	x86_cpuinit.early_percpu_clock_init();
228 229
	preempt_disable();
	smp_callin();
230

231 232
	enable_start_cpu0 = 0;

233
#ifdef CONFIG_X86_32
234
	/* switch away from the initial page table */
235 236 237 238
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

239 240 241 242 243 244 245 246
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	/*
247 248 249
	 * We need to hold vector_lock so there the set of online cpus
	 * does not change while we are assigning vectors to cpus.  Holding
	 * this lock ensures we don't half assign or remove an irq from a cpu.
250
	 */
251
	lock_vector_lock();
252
	set_cpu_online(smp_processor_id(), true);
253
	unlock_vector_lock();
254
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
255
	x86_platform.nmi_init();
256

257 258 259
	/* enable local interrupts */
	local_irq_enable();

260 261
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
262

263
	x86_cpuinit.setup_percpu_clockev();
264 265

	wmb();
T
Thomas Gleixner 已提交
266
	cpu_startup_entry(CPUHP_ONLINE);
267 268
}

269 270 271 272 273 274 275 276 277
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
}

278 279 280 281
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
282
void smp_store_cpu_info(int id)
283 284 285
{
	struct cpuinfo_x86 *c = &cpu_data(id);

286
	*c = boot_cpu_data;
287
	c->cpu_index = id;
288 289 290 291 292
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
293 294
}

295
static bool
296
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
297
{
298 299 300 301 302 303 304 305 306 307 308 309 310 311
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return !WARN_ONCE(cpu_to_node(cpu1) != cpu_to_node(cpu2),
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

#define link_mask(_m, c1, c2)						\
do {									\
	cpumask_set_cpu((c1), cpu_##_m##_mask(c2));			\
	cpumask_set_cpu((c2), cpu_##_m##_mask(c1));			\
} while (0)

312
static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
313
{
A
Andreas Herrmann 已提交
314
	if (cpu_has_topoext) {
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
		    c->compute_unit_id == o->compute_unit_id)
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

330
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
331 332 333 334 335 336 337 338
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
339 340
}

341
static bool match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
342
{
343 344 345
	if (c->phys_proc_id == o->phys_proc_id) {
		if (cpu_has(c, X86_FEATURE_AMD_DCM))
			return true;
346

347 348
		return topology_sane(c, o, "mc");
	}
349 350
	return false;
}
351

352
void set_cpu_sibling_map(int cpu)
353
{
354
	bool has_smt = smp_num_siblings > 1;
355
	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
356
	struct cpuinfo_x86 *c = &cpu_data(cpu);
357 358
	struct cpuinfo_x86 *o;
	int i;
359

360
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
361

362
	if (!has_mp) {
363
		cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
364 365
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
		cpumask_set_cpu(cpu, cpu_core_mask(cpu));
366 367 368 369
		c->booted_cores = 1;
		return;
	}

370
	for_each_cpu(i, cpu_sibling_setup_mask) {
371 372 373 374 375
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
			link_mask(sibling, cpu, i);

376
		if ((i == cpu) || (has_mp && match_llc(c, o)))
377 378
			link_mask(llc_shared, cpu, i);

379 380 381 382 383 384 385 386 387
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
	 * cpu_sibling_mask links to be set-up.
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

388
		if ((i == cpu) || (has_mp && match_mc(c, o))) {
389 390
			link_mask(core, cpu, i);

391 392 393
			/*
			 *  Does this new cpu bringup a new core?
			 */
394
			if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) {
395 396 397 398
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
399
				if (cpumask_first(cpu_sibling_mask(i)) == i)
400 401 402 403 404 405 406 407 408 409 410 411 412
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
	}
}

413
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
414
const struct cpumask *cpu_coregroup_mask(int cpu)
415
{
416
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
417 418
}

I
Ingo Molnar 已提交
419
static void impress_friends(void)
420 421 422 423 424 425
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
426
	pr_debug("Before bogomips\n");
427
	for_each_possible_cpu(cpu)
428
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
429
			bogosum += cpu_data(cpu).loops_per_jiffy;
430
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
431
		num_online_cpus(),
432 433 434
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

435
	pr_debug("Before bogocount - setting activated=1\n");
436 437
}

438
void __inquire_remote_apic(int apicid)
439 440
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
441
	const char * const names[] = { "ID", "VERSION", "SPIV" };
442 443 444
	int timeout;
	u32 status;

445
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
446 447

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
448
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
449 450 451 452 453 454

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
455
			pr_cont("a previous APIC delivery may have failed\n");
456

457
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
458 459 460 461 462 463 464 465 466 467

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
468
			pr_cont("%08x\n", status);
469 470
			break;
		default:
471
			pr_cont("failed\n");
472 473 474 475 476 477 478 479 480
		}
	}
}

/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
481
int
482
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
483 484 485 486 487 488 489
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
490
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
491

492
	pr_debug("Waiting for send to finish...\n");
493 494 495 496 497 498
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
499
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
500 501 502 503 504
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
505
	pr_debug("NMI sent\n");
506 507

	if (send_status)
508
		pr_err("APIC never delivered???\n");
509
	if (accept_status)
510
		pr_err("APIC delivery error (%lx)\n", accept_status);
511 512 513 514

	return (send_status | accept_status);
}

515
static int
516
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
517 518 519 520
{
	unsigned long send_status, accept_status = 0;
	int maxlvt, num_starts, j;

521 522
	maxlvt = lapic_get_maxlvt();

523 524 525 526
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
527 528
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
529 530 531
		apic_read(APIC_ESR);
	}

532
	pr_debug("Asserting INIT\n");
533 534 535 536 537 538 539

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
540 541
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
542

543
	pr_debug("Waiting for send to finish...\n");
544 545 546 547
	send_status = safe_apic_wait_icr_idle();

	mdelay(10);

548
	pr_debug("Deasserting INIT\n");
549 550 551

	/* Target chip */
	/* Send IPI */
552
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
553

554
	pr_debug("Waiting for send to finish...\n");
555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
	send_status = safe_apic_wait_icr_idle();

	mb();
	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Paravirt / VMI wants a startup IPI hook here to set up the
	 * target processor state.
	 */
	startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
576
			 stack_start);
577 578 579 580

	/*
	 * Run STARTUP IPI loop.
	 */
581
	pr_debug("#startup loops: %d\n", num_starts);
582 583

	for (j = 1; j <= num_starts; j++) {
584
		pr_debug("Sending STARTUP #%d\n", j);
585 586
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
587
		apic_read(APIC_ESR);
588
		pr_debug("After apic_write\n");
589 590 591 592 593 594 595 596

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
597 598
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
599 600 601 602 603 604

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

605
		pr_debug("Startup point 1\n");
606

607
		pr_debug("Waiting for send to finish...\n");
608 609 610 611 612 613
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
614
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
615 616 617 618 619
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
620
	pr_debug("After Startup\n");
621 622

	if (send_status)
623
		pr_err("APIC never delivered???\n");
624
	if (accept_status)
625
		pr_err("APIC delivery error (%lx)\n", accept_status);
626 627 628 629

	return (send_status | accept_status);
}

630 631 632 633 634 635 636 637
void smp_announce(void)
{
	int num_nodes = num_online_nodes();

	printk(KERN_INFO "x86: Booted up %d node%s, %d CPUs\n",
	       num_nodes, (num_nodes > 1 ? "s" : ""), num_online_cpus());
}

638
/* reduce the number of lines printed when booting a large cpu count system */
639
static void announce_cpu(int cpu, int apicid)
640 641
{
	static int current_node = -1;
642
	int node = early_cpu_to_node(cpu);
643
	static int width, node_width;
644 645 646

	if (!width)
		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
647

648 649 650 651 652 653
	if (!node_width)
		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */

	if (cpu == 1)
		printk(KERN_INFO "x86: Booting SMP configuration:\n");

654 655 656
	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
657
				pr_cont("\n");
658
			current_node = node;
659 660 661

			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
			       node_width - num_digits(node), " ", node);
662
		}
663 664 665 666 667 668 669

		/* Add padding for the BSP */
		if (cpu == 1)
			pr_cont("%*s", width + 1, " ");

		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);

670 671 672 673 674
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
698
static int
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
	if (cpu)
		return wakeup_secondary_cpu_via_init(apicid, start_ip);

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}

	return boot_error;
}

732 733 734
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
735 736
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
737
 */
738
static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
739
{
740
	volatile u32 *trampoline_status =
741
		(volatile u32 *) __va(real_mode_header->trampoline_status);
742
	/* start_ip had better be page-aligned! */
743
	unsigned long start_ip = real_mode_header->trampoline_start;
744

745
	unsigned long boot_error = 0;
746
	int timeout;
747
	int cpu0_nmi_registered = 0;
748

749 750
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();
751

752 753 754
	idle->thread.sp = (unsigned long) (((struct pt_regs *)
			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
	per_cpu(current_task, cpu) = idle;
755

756
#ifdef CONFIG_X86_32
757 758 759
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
#else
760
	clear_tsk_thread_flag(idle, TIF_FORK);
761
	initial_gs = per_cpu_offset(cpu);
762
	per_cpu(kernel_stack, cpu) =
763
		(unsigned long)task_stack_page(idle) -
764
		KERNEL_STACK_OFFSET + THREAD_SIZE;
765
#endif
766
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
767
	initial_code = (unsigned long)start_secondary;
768
	stack_start  = idle->thread.sp;
769

770 771
	/* So we see what's up */
	announce_cpu(cpu, apicid);
772 773 774 775 776 777 778 779

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

J
Jack Steiner 已提交
780
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
781

782
		pr_debug("Setting warm reset code and vector.\n");
783

J
Jack Steiner 已提交
784 785 786
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
787 788 789 790 791
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
792
	}
793 794

	/*
795 796 797 798
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
799
	 */
800 801 802
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
803 804
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
805 806 807 808 809

	if (!boot_error) {
		/*
		 * allow APs to start initializing.
		 */
810
		pr_debug("Before Callout %d\n", cpu);
811
		cpumask_set_cpu(cpu, cpu_callout_mask);
812
		pr_debug("After Callout %d\n", cpu);
813 814 815 816 817

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
818
			if (cpumask_test_cpu(cpu, cpu_callin_mask))
819 820
				break;	/* It has booted */
			udelay(100);
821 822 823 824 825 826 827
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
828 829
		}

830 831
		if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
			print_cpu_msr(&cpu_data(cpu));
832
			pr_debug("CPU%d: has booted.\n", cpu);
833
		} else {
834
			boot_error = 1;
835
			if (*trampoline_status == 0xA5A5A5A5)
836
				/* trampoline started but...? */
837
				pr_err("CPU%d: Stuck ??\n", cpu);
838 839
			else
				/* trampoline code not run */
840
				pr_err("CPU%d: Not responding\n", cpu);
841 842
			if (apic->inquire_remote_apic)
				apic->inquire_remote_apic(apicid);
843 844
		}
	}
845

846 847
	if (boot_error) {
		/* Try to put things back the way they were before ... */
848
		numa_remove_cpu(cpu); /* was set by numa_add_cpu */
849 850 851 852 853 854 855 856

		/* was set by do_boot_cpu() */
		cpumask_clear_cpu(cpu, cpu_callout_mask);

		/* was set by cpu_init() */
		cpumask_clear_cpu(cpu, cpu_initialized_mask);

		set_cpu_present(cpu, false);
857 858 859 860
		per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
	}

	/* mark "stuck" area as not stuck */
861
	*trampoline_status = 0;
862

863 864 865 866 867 868
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
869 870 871 872 873 874 875
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

876 877 878
	return boot_error;
}

879
int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
880
{
881
	int apicid = apic->cpu_present_to_apicid(cpu);
882 883 884 885 886
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

887
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
888

889
	if (apicid == BAD_APICID ||
890
	    !physid_isset(apicid, phys_cpu_present_map) ||
891
	    !apic->apic_id_valid(apicid)) {
892
		pr_err("%s: bad cpu %d\n", __func__, cpu);
893 894 895 896 897 898
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
899
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
900
		pr_debug("do_boot_cpu %d Already started\n", cpu);
901 902 903 904 905 906 907 908 909 910 911
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;

912 913 914
	/* the FPU context is blank, nobody can own it */
	__cpu_disable_lazy_restore(cpu);

915
	err = do_boot_cpu(apicid, cpu, tidle);
916
	if (err) {
917
		pr_debug("do_boot_cpu failed %d\n", err);
918
		return -EIO;
919 920 921 922 923 924 925 926 927 928
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

929
	while (!cpu_online(cpu)) {
930 931 932 933 934 935 936
		cpu_relax();
		touch_nmi_watchdog();
	}

	return 0;
}

937 938 939 940 941 942 943 944
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

945 946 947 948 949 950 951
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
952 953
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
954
	smpboot_clear_io_apic_irqs();
955

956
	if (smp_found_config)
957
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
958
	else
959
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
960 961
	cpumask_set_cpu(0, cpu_sibling_mask(0));
	cpumask_set_cpu(0, cpu_core_mask(0));
962 963 964 965 966 967 968
}

/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
969
	preempt_disable();
970

971
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
972 973 974 975
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

976 977
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
978 979 980 981

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
982
				set_cpu_present(cpu, false);
983 984 985 986 987 988
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
989
				set_cpu_possible(cpu, false);
990 991 992 993 994 995 996
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

997
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
998
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
999 1000
			hard_smp_processor_id());

1001 1002 1003 1004 1005 1006 1007 1008
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1009
		preempt_enable();
1010
		pr_notice("SMP motherboard not detected\n");
1011 1012
		disable_smp();
		if (APIC_init_uniprocessor())
1013
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
1014 1015 1016 1017 1018 1019 1020
		return -1;
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1021
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1022 1023
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1024 1025
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1026
	preempt_enable();
1027 1028 1029 1030 1031 1032

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
	    !cpu_has_apic) {
1033 1034 1035
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1036
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1037
		}
1038
		smpboot_clear_io_apic();
1039
		disable_ioapic_support();
1040 1041 1042 1043 1044 1045 1046 1047 1048
		return -1;
	}

	verify_local_APIC();

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1049
		pr_info("SMP mode deactivated\n");
1050
		smpboot_clear_io_apic();
1051

1052 1053
		connect_bsp_APIC();
		setup_local_APIC();
1054
		bsp_end_local_APIC_setup();
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
		return -1;
	}

	return 0;
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1066
	for_each_possible_cpu(i) {
1067 1068
		c = &cpu_data(i);
		/* mark all to hotplug */
1069
		c->cpu_index = nr_cpu_ids;
1070 1071 1072 1073 1074 1075 1076 1077 1078
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1079 1080
	unsigned int i;

1081
	preempt_disable();
1082
	smp_cpu_index_default();
1083

1084 1085 1086
	/*
	 * Setup boot CPU information
	 */
1087
	smp_store_boot_cpu_info(); /* Final full version of the data */
1088 1089
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1090

1091
	current_thread_info()->cpu = 0;  /* needed? */
1092
	for_each_possible_cpu(i) {
1093 1094
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1095
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1096
	}
1097 1098
	set_cpu_sibling_map(0);

1099

1100
	if (smp_sanity_check(max_cpus) < 0) {
1101
		pr_info("SMP disabled\n");
1102
		disable_smp();
1103
		goto out;
1104 1105
	}

1106 1107
	default_setup_apic_routing();

J
Jack Steiner 已提交
1108
	preempt_disable();
1109
	if (read_apic_id() != boot_cpu_physical_apicid) {
1110
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1111
		     read_apic_id(), boot_cpu_physical_apicid);
1112 1113
		/* Or can we switch back to PIC here? */
	}
J
Jack Steiner 已提交
1114
	preempt_enable();
1115 1116

	connect_bsp_APIC();
1117

1118 1119 1120 1121 1122
	/*
	 * Switch from PIC to APIC mode.
	 */
	setup_local_APIC();

1123 1124 1125 1126 1127
	if (x2apic_mode)
		cpu0_logical_apicid = apic_read(APIC_LDR);
	else
		cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));

1128 1129 1130 1131 1132
	/*
	 * Enable IO APIC before setting up error vector
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		enable_IO_APIC();
1133

1134
	bsp_end_local_APIC_setup();
1135

1136 1137
	if (apic->setup_portio_remap)
		apic->setup_portio_remap();
1138 1139 1140 1141 1142 1143

	smpboot_setup_io_apic();
	/*
	 * Set up local APIC timer on boot CPU.
	 */

1144
	pr_info("CPU%d: ", 0);
1145
	print_cpu_info(&cpu_data(0));
1146
	x86_init.timers.setup_percpu_clockev();
1147 1148 1149

	if (is_uv_system())
		uv_system_init();
1150 1151

	set_mtrr_aps_delayed_init();
1152 1153
out:
	preempt_enable();
1154
}
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1166 1167 1168 1169 1170 1171
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1172
	switch_to_new_gdt(me);
1173 1174
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1175 1176 1177
	per_cpu(cpu_state, me) = CPU_ONLINE;
}

1178 1179
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1180
	pr_debug("Boot done\n");
1181

D
Don Zickus 已提交
1182
	nmi_selftest();
1183 1184 1185 1186
	impress_friends();
#ifdef CONFIG_X86_IO_APIC
	setup_ioapic_dest();
#endif
1187
	mtrr_aps_init();
1188 1189
}

1190 1191 1192 1193 1194 1195 1196 1197 1198
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1199
/*
1200
 * cpu_possible_mask should be static, it cannot change as cpu's
1201 1202 1203
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1204
 * cpu_present_mask on the other hand can change dynamically.
1205 1206 1207 1208 1209 1210
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1211
 * - The user can overwrite it with possible_cpus=NUM
1212 1213 1214 1215 1216 1217
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1218
	int i, possible;
1219

1220 1221 1222 1223
	/* no processor from mptable or madt */
	if (!num_processors)
		num_processors = 1;

1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1235 1236
		possible = setup_possible_cpus;

1237 1238
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1239 1240
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1241
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1242 1243
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1244
	}
1245

1246 1247 1248 1249
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1250
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1251 1252 1253 1254
			possible, setup_max_cpus);
		possible = i;
	}

1255
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1256 1257 1258
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1259
		set_cpu_possible(i, true);
1260 1261
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1262 1263

	nr_cpu_ids = possible;
1264
}
1265

1266 1267 1268 1269 1270 1271 1272
#ifdef CONFIG_HOTPLUG_CPU

static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1273 1274
	for_each_cpu(sibling, cpu_core_mask(cpu)) {
		cpumask_clear_cpu(cpu, cpu_core_mask(sibling));
1275 1276 1277
		/*/
		 * last thread sibling in this cpu core going down
		 */
1278
		if (cpumask_weight(cpu_sibling_mask(cpu)) == 1)
1279 1280 1281
			cpu_data(sibling).booted_cores--;
	}

1282 1283 1284 1285
	for_each_cpu(sibling, cpu_sibling_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling));
	cpumask_clear(cpu_sibling_mask(cpu));
	cpumask_clear(cpu_core_mask(cpu));
1286 1287
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1288
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1289 1290
}

1291 1292
static void __ref remove_cpu_from_maps(int cpu)
{
1293 1294 1295
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1296
	/* was set by cpu_init() */
1297
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1298
	numa_remove_cpu(cpu);
1299 1300
}

1301
void cpu_disable_common(void)
1302 1303 1304 1305 1306 1307
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1308
	lock_vector_lock();
1309
	remove_cpu_from_maps(cpu);
1310
	unlock_vector_lock();
1311
	fixup_irqs();
1312 1313 1314 1315
}

int native_cpu_disable(void)
{
1316 1317 1318 1319 1320 1321
	int ret;

	ret = check_irq_vectors_for_cpu_disable();
	if (ret)
		return ret;

1322 1323 1324
	clear_local_APIC();

	cpu_disable_common();
1325 1326 1327
	return 0;
}

1328
void native_cpu_die(unsigned int cpu)
1329 1330 1331 1332 1333 1334 1335
{
	/* We don't do anything here: idle task is faking death itself. */
	unsigned int i;

	for (i = 0; i < 10; i++) {
		/* They ack this in play_dead by setting CPU_DEAD */
		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1336 1337
			if (system_state == SYSTEM_RUNNING)
				pr_info("CPU %u is now offline\n", cpu);
1338 1339 1340 1341
			return;
		}
		msleep(100);
	}
1342
	pr_err("CPU %u didn't die...\n", cpu);
1343
}
1344 1345 1346 1347 1348

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();
1349
	amd_e400_remove_cpu(raw_smp_processor_id());
1350 1351 1352

	mb();
	/* Ack it */
T
Tejun Heo 已提交
1353
	__this_cpu_write(cpu_state, CPU_DEAD);
1354 1355 1356 1357 1358 1359 1360

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1361 1362 1363 1364 1365 1366 1367 1368
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1369 1370 1371 1372 1373 1374 1375 1376 1377
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
1378
	void *mwait_ptr;
1379
	int i;
1380

1381
	if (!this_cpu_has(X86_FEATURE_MWAIT))
1382
		return;
1383
	if (!this_cpu_has(X86_FEATURE_CLFLSH))
1384
		return;
1385
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1410 1411 1412 1413 1414 1415 1416
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1417 1418
	wbinvd();

1419
	while (1) {
1420 1421 1422 1423 1424 1425 1426
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
1427
		mb();
1428
		clflush(mwait_ptr);
1429
		mb();
1430
		__monitor(mwait_ptr, 0, 0);
1431 1432
		mb();
		__mwait(eax, 0);
1433 1434 1435 1436 1437
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1438 1439 1440 1441 1442
	}
}

static inline void hlt_play_dead(void)
{
1443
	if (__this_cpu_read(cpu_info.x86) >= 4)
1444 1445
		wbinvd();

1446 1447
	while (1) {
		native_halt();
1448 1449 1450 1451 1452
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1453 1454 1455
	}
}

1456 1457 1458
void native_play_dead(void)
{
	play_dead_common();
1459
	tboot_shutdown(TB_SHUTDOWN_WFS);
1460 1461

	mwait_play_dead();	/* Only returns on failure */
1462 1463
	if (cpuidle_play_dead())
		hlt_play_dead();
1464 1465
}

1466
#else /* ... !CONFIG_HOTPLUG_CPU */
1467
int native_cpu_disable(void)
1468 1469 1470 1471
{
	return -ENOSYS;
}

1472
void native_cpu_die(unsigned int cpu)
1473 1474 1475 1476
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1477 1478 1479 1480 1481 1482

void native_play_dead(void)
{
	BUG();
}

1483
#endif