smpboot.c 35.3 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/module.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/idle.h>
62
#include <asm/realmode.h>
63 64
#include <asm/cpu.h>
#include <asm/numa.h>
65 66 67
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
68
#include <asm/mwait.h>
I
Ingo Molnar 已提交
69
#include <asm/apic.h>
70
#include <asm/io_apic.h>
71 72
#include <asm/i387.h>
#include <asm/fpu-internal.h>
73
#include <asm/setup.h>
T
Tejun Heo 已提交
74
#include <asm/uv/uv.h>
75
#include <linux/mc146818rtc.h>
76

77
#include <asm/smpboot_hooks.h>
78
#include <asm/i8259.h>
79

80 81
#include <asm/realmode.h>

82 83 84
/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

85
#ifdef CONFIG_HOTPLUG_CPU
86 87 88 89 90 91
/*
 * We need this for trampoline_base protection from concurrent accesses when
 * off- and onlining cores wildly.
 */
static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex);

92
void cpu_hotplug_driver_lock(void)
93
{
94
	mutex_lock(&x86_cpu_hotplug_driver_mutex);
95 96
}

97
void cpu_hotplug_driver_unlock(void)
98
{
99
	mutex_unlock(&x86_cpu_hotplug_driver_mutex);
100 101 102 103
}

ssize_t arch_cpu_probe(const char *buf, size_t count) { return -1; }
ssize_t arch_cpu_release(const char *buf, size_t count) { return -1; }
104
#endif
105

106 107 108 109 110
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
111
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
112 113

/* representing HT siblings of each logical CPU */
114
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
115 116 117
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
118
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
119 120
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

121
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
122

123 124 125
/* Per CPU bogomips and other parameters */
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
126

127
atomic_t init_deasserted;
128 129

/*
130 131
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
132
 */
I
Ingo Molnar 已提交
133
static void __cpuinit smp_callin(void)
134 135 136 137 138 139 140 141 142
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
143 144
	 *
	 * Since CPU0 is not wakened up by INIT, it doesn't wait for the IPI.
145
	 */
146 147
	cpuid = smp_processor_id();
	if (apic->wait_for_init_deassert && cpuid != 0)
148
		apic->wait_for_init_deassert(&init_deasserted);
149 150 151 152

	/*
	 * (This works even if the APIC is not enabled.)
	 */
153
	phys_id = read_apic_id();
154
	if (cpumask_test_cpu(cpuid, cpu_callin_mask)) {
155 156 157
		panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
					phys_id, cpuid);
	}
158
	pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
176
		if (cpumask_test_cpu(cpuid, cpu_callout_mask))
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
			break;
		cpu_relax();
	}

	if (!time_before(jiffies, timeout)) {
		panic("%s: CPU%d started up but did not get a callout!\n",
		      __func__, cpuid);
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

193
	pr_debug("CALLIN, before setup_local_APIC()\n");
194 195
	if (apic->smp_callin_clear_local_apic)
		apic->smp_callin_clear_local_apic();
196 197 198
	setup_local_APIC();
	end_local_APIC_setup();

199 200 201
	/*
	 * Need to setup vector mappings before we enable interrupts.
	 */
202
	setup_vector_irq(smp_processor_id());
203 204 205 206 207 208 209

	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

210 211
	/*
	 * Get our bogomips.
212 213 214
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
215 216
	 */
	calibrate_delay();
217
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
218
	pr_debug("Stack at about %p\n", &cpuid);
219

220 221 222 223 224 225 226
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

227 228
	notify_cpu_starting(cpuid);

229 230 231
	/*
	 * Allow the master to continue.
	 */
232
	cpumask_set_cpu(cpuid, cpu_callin_mask);
233 234
}

235 236
static int cpu0_logical_apicid;
static int enable_start_cpu0;
237 238 239
/*
 * Activate a secondary processor.
 */
240
notrace static void __cpuinit start_secondary(void *unused)
241 242 243 244 245 246
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
247
	cpu_init();
248
	x86_cpuinit.early_percpu_clock_init();
249 250
	preempt_disable();
	smp_callin();
251

252 253
	enable_start_cpu0 = 0;

254
#ifdef CONFIG_X86_32
255
	/* switch away from the initial page table */
256 257 258 259
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

260 261 262 263 264 265 266 267
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	/*
268 269 270
	 * We need to hold vector_lock so there the set of online cpus
	 * does not change while we are assigning vectors to cpus.  Holding
	 * this lock ensures we don't half assign or remove an irq from a cpu.
271
	 */
272
	lock_vector_lock();
273
	set_cpu_online(smp_processor_id(), true);
274
	unlock_vector_lock();
275
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
276
	x86_platform.nmi_init();
277

278 279 280
	/* enable local interrupts */
	local_irq_enable();

281 282
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
283

284
	x86_cpuinit.setup_percpu_clockev();
285 286 287 288 289

	wmb();
	cpu_idle();
}

290 291 292 293 294 295 296 297 298
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
}

299 300 301 302 303 304 305 306
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
void __cpuinit smp_store_cpu_info(int id)
{
	struct cpuinfo_x86 *c = &cpu_data(id);

307
	*c = boot_cpu_data;
308
	c->cpu_index = id;
309 310 311 312 313
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
314 315
}

316 317
static bool __cpuinit
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
318
{
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return !WARN_ONCE(cpu_to_node(cpu1) != cpu_to_node(cpu2),
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

#define link_mask(_m, c1, c2)						\
do {									\
	cpumask_set_cpu((c1), cpu_##_m##_mask(c2));			\
	cpumask_set_cpu((c2), cpu_##_m##_mask(c1));			\
} while (0)

static bool __cpuinit match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
A
Andreas Herrmann 已提交
335
	if (cpu_has_topoext) {
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
		    c->compute_unit_id == o->compute_unit_id)
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

static bool __cpuinit match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
360 361
}

362 363
static bool __cpuinit match_mc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
364 365 366
	if (c->phys_proc_id == o->phys_proc_id) {
		if (cpu_has(c, X86_FEATURE_AMD_DCM))
			return true;
367

368 369
		return topology_sane(c, o, "mc");
	}
370 371
	return false;
}
372

373 374
void __cpuinit set_cpu_sibling_map(int cpu)
{
375 376
	bool has_mc = boot_cpu_data.x86_max_cores > 1;
	bool has_smt = smp_num_siblings > 1;
377
	struct cpuinfo_x86 *c = &cpu_data(cpu);
378 379
	struct cpuinfo_x86 *o;
	int i;
380

381
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
382

383
	if (!has_smt && !has_mc) {
384
		cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
385 386
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
		cpumask_set_cpu(cpu, cpu_core_mask(cpu));
387 388 389 390
		c->booted_cores = 1;
		return;
	}

391
	for_each_cpu(i, cpu_sibling_setup_mask) {
392 393 394 395 396 397 398 399
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
			link_mask(sibling, cpu, i);

		if ((i == cpu) || (has_mc && match_llc(c, o)))
			link_mask(llc_shared, cpu, i);

400 401 402 403 404 405 406 407 408
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
	 * cpu_sibling_mask links to be set-up.
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

409 410 411
		if ((i == cpu) || (has_mc && match_mc(c, o))) {
			link_mask(core, cpu, i);

412 413 414
			/*
			 *  Does this new cpu bringup a new core?
			 */
415
			if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) {
416 417 418 419
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
420
				if (cpumask_first(cpu_sibling_mask(i)) == i)
421 422 423 424 425 426 427 428 429 430 431 432 433
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
	}
}

434
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
435
const struct cpumask *cpu_coregroup_mask(int cpu)
436
{
437
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
438 439
}

I
Ingo Molnar 已提交
440
static void impress_friends(void)
441 442 443 444 445 446
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
447
	pr_debug("Before bogomips\n");
448
	for_each_possible_cpu(cpu)
449
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
450
			bogosum += cpu_data(cpu).loops_per_jiffy;
451
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
452
		num_online_cpus(),
453 454 455
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

456
	pr_debug("Before bogocount - setting activated=1\n");
457 458
}

459
void __inquire_remote_apic(int apicid)
460 461
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
462
	const char * const names[] = { "ID", "VERSION", "SPIV" };
463 464 465
	int timeout;
	u32 status;

466
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
467 468

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
469
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
470 471 472 473 474 475

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
476
			pr_cont("a previous APIC delivery may have failed\n");
477

478
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
479 480 481 482 483 484 485 486 487 488

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
489
			pr_cont("%08x\n", status);
490 491
			break;
		default:
492
			pr_cont("failed\n");
493 494 495 496 497 498 499 500 501
		}
	}
}

/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
502
int __cpuinit
503
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
504 505 506 507 508 509 510
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
511
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
512

513
	pr_debug("Waiting for send to finish...\n");
514 515 516 517 518 519
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
520
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
521 522 523 524 525
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
526
	pr_debug("NMI sent\n");
527 528

	if (send_status)
529
		pr_err("APIC never delivered???\n");
530
	if (accept_status)
531
		pr_err("APIC delivery error (%lx)\n", accept_status);
532 533 534 535

	return (send_status | accept_status);
}

536
static int __cpuinit
537
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
538 539 540 541
{
	unsigned long send_status, accept_status = 0;
	int maxlvt, num_starts, j;

542 543
	maxlvt = lapic_get_maxlvt();

544 545 546 547
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
548 549
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
550 551 552
		apic_read(APIC_ESR);
	}

553
	pr_debug("Asserting INIT\n");
554 555 556 557 558 559 560

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
561 562
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
563

564
	pr_debug("Waiting for send to finish...\n");
565 566 567 568
	send_status = safe_apic_wait_icr_idle();

	mdelay(10);

569
	pr_debug("Deasserting INIT\n");
570 571 572

	/* Target chip */
	/* Send IPI */
573
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
574

575
	pr_debug("Waiting for send to finish...\n");
576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
	send_status = safe_apic_wait_icr_idle();

	mb();
	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Paravirt / VMI wants a startup IPI hook here to set up the
	 * target processor state.
	 */
	startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
597
			 stack_start);
598 599 600 601

	/*
	 * Run STARTUP IPI loop.
	 */
602
	pr_debug("#startup loops: %d\n", num_starts);
603 604

	for (j = 1; j <= num_starts; j++) {
605
		pr_debug("Sending STARTUP #%d\n", j);
606 607
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
608
		apic_read(APIC_ESR);
609
		pr_debug("After apic_write\n");
610 611 612 613 614 615 616 617

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
618 619
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
620 621 622 623 624 625

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

626
		pr_debug("Startup point 1\n");
627

628
		pr_debug("Waiting for send to finish...\n");
629 630 631 632 633 634
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
635
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
636 637 638 639 640
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
641
	pr_debug("After Startup\n");
642 643

	if (send_status)
644
		pr_err("APIC never delivered???\n");
645
	if (accept_status)
646
		pr_err("APIC delivery error (%lx)\n", accept_status);
647 648 649 650

	return (send_status | accept_status);
}

651 652 653 654
/* reduce the number of lines printed when booting a large cpu count system */
static void __cpuinit announce_cpu(int cpu, int apicid)
{
	static int current_node = -1;
655
	int node = early_cpu_to_node(cpu);
656 657 658 659

	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
660
				pr_cont(" OK\n");
661 662 663
			current_node = node;
			pr_info("Booting Node %3d, Processors ", node);
		}
664
		pr_cont(" #%d%s", cpu, cpu == (nr_cpu_ids - 1) ? " OK\n" : "");
665 666 667 668 669 670
		return;
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
static int __cpuinit
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
	if (cpu)
		return wakeup_secondary_cpu_via_init(apicid, start_ip);

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}

	return boot_error;
}

728 729 730
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
731 732
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
733
 */
734
static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
735
{
736
	volatile u32 *trampoline_status =
737
		(volatile u32 *) __va(real_mode_header->trampoline_status);
738
	/* start_ip had better be page-aligned! */
739
	unsigned long start_ip = real_mode_header->trampoline_start;
740

741
	unsigned long boot_error = 0;
742
	int timeout;
743
	int cpu0_nmi_registered = 0;
744

745 746
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();
747

748 749 750
	idle->thread.sp = (unsigned long) (((struct pt_regs *)
			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
	per_cpu(current_task, cpu) = idle;
751

752
#ifdef CONFIG_X86_32
753 754 755
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
#else
756
	clear_tsk_thread_flag(idle, TIF_FORK);
757
	initial_gs = per_cpu_offset(cpu);
758
	per_cpu(kernel_stack, cpu) =
759
		(unsigned long)task_stack_page(idle) -
760
		KERNEL_STACK_OFFSET + THREAD_SIZE;
761
#endif
762
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
763
	initial_code = (unsigned long)start_secondary;
764
	stack_start  = idle->thread.sp;
765

766 767
	/* So we see what's up */
	announce_cpu(cpu, apicid);
768 769 770 771 772 773 774 775

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

J
Jack Steiner 已提交
776
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
777

778
		pr_debug("Setting warm reset code and vector.\n");
779

J
Jack Steiner 已提交
780 781 782
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
783 784 785 786 787
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
788
	}
789 790

	/*
791 792 793 794
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
795
	 */
796 797 798
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
799 800
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
801 802 803 804 805

	if (!boot_error) {
		/*
		 * allow APs to start initializing.
		 */
806
		pr_debug("Before Callout %d\n", cpu);
807
		cpumask_set_cpu(cpu, cpu_callout_mask);
808
		pr_debug("After Callout %d\n", cpu);
809 810 811 812 813

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
814
			if (cpumask_test_cpu(cpu, cpu_callin_mask))
815 816
				break;	/* It has booted */
			udelay(100);
817 818 819 820 821 822 823
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
824 825
		}

826 827
		if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
			print_cpu_msr(&cpu_data(cpu));
828
			pr_debug("CPU%d: has booted.\n", cpu);
829
		} else {
830
			boot_error = 1;
831
			if (*trampoline_status == 0xA5A5A5A5)
832
				/* trampoline started but...? */
833
				pr_err("CPU%d: Stuck ??\n", cpu);
834 835
			else
				/* trampoline code not run */
836
				pr_err("CPU%d: Not responding\n", cpu);
837 838
			if (apic->inquire_remote_apic)
				apic->inquire_remote_apic(apicid);
839 840
		}
	}
841

842 843
	if (boot_error) {
		/* Try to put things back the way they were before ... */
844
		numa_remove_cpu(cpu); /* was set by numa_add_cpu */
845 846 847 848 849 850 851 852

		/* was set by do_boot_cpu() */
		cpumask_clear_cpu(cpu, cpu_callout_mask);

		/* was set by cpu_init() */
		cpumask_clear_cpu(cpu, cpu_initialized_mask);

		set_cpu_present(cpu, false);
853 854 855 856
		per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
	}

	/* mark "stuck" area as not stuck */
857
	*trampoline_status = 0;
858

859 860 861 862 863 864
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
865 866 867 868 869 870 871
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

872 873 874
	return boot_error;
}

875
int __cpuinit native_cpu_up(unsigned int cpu, struct task_struct *tidle)
876
{
877
	int apicid = apic->cpu_present_to_apicid(cpu);
878 879 880 881 882
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

883
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
884

885
	if (apicid == BAD_APICID ||
886
	    !physid_isset(apicid, phys_cpu_present_map) ||
887
	    !apic->apic_id_valid(apicid)) {
888
		pr_err("%s: bad cpu %d\n", __func__, cpu);
889 890 891 892 893 894
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
895
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
896
		pr_debug("do_boot_cpu %d Already started\n", cpu);
897 898 899 900 901 902 903 904 905 906 907
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;

908 909 910
	/* the FPU context is blank, nobody can own it */
	__cpu_disable_lazy_restore(cpu);

911
	err = do_boot_cpu(apicid, cpu, tidle);
912
	if (err) {
913
		pr_debug("do_boot_cpu failed %d\n", err);
914
		return -EIO;
915 916 917 918 919 920 921 922 923 924
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

925
	while (!cpu_online(cpu)) {
926 927 928 929 930 931 932
		cpu_relax();
		touch_nmi_watchdog();
	}

	return 0;
}

933 934 935 936 937 938 939 940
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

941 942 943 944 945 946 947
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
948 949
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
950
	smpboot_clear_io_apic_irqs();
951

952
	if (smp_found_config)
953
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
954
	else
955
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
956 957
	cpumask_set_cpu(0, cpu_sibling_mask(0));
	cpumask_set_cpu(0, cpu_core_mask(0));
958 959 960 961 962 963 964
}

/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
965
	preempt_disable();
966

967
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
968 969 970 971
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

972 973
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
974 975 976 977

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
978
				set_cpu_present(cpu, false);
979 980 981 982 983 984
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
985
				set_cpu_possible(cpu, false);
986 987 988 989 990 991 992
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

993
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
994
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
995 996
			hard_smp_processor_id());

997 998 999 1000 1001 1002 1003 1004
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1005
		preempt_enable();
1006
		pr_notice("SMP motherboard not detected\n");
1007 1008
		disable_smp();
		if (APIC_init_uniprocessor())
1009
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
1010 1011 1012 1013 1014 1015 1016
		return -1;
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1017
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1018 1019
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1020 1021
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1022
	preempt_enable();
1023 1024 1025 1026 1027 1028

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
	    !cpu_has_apic) {
1029 1030 1031
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1032
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1033
		}
1034
		smpboot_clear_io_apic();
1035
		disable_ioapic_support();
1036 1037 1038 1039 1040 1041 1042 1043 1044
		return -1;
	}

	verify_local_APIC();

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1045
		pr_info("SMP mode deactivated\n");
1046
		smpboot_clear_io_apic();
1047

1048 1049
		connect_bsp_APIC();
		setup_local_APIC();
1050
		bsp_end_local_APIC_setup();
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
		return -1;
	}

	return 0;
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1062
	for_each_possible_cpu(i) {
1063 1064
		c = &cpu_data(i);
		/* mark all to hotplug */
1065
		c->cpu_index = nr_cpu_ids;
1066 1067 1068 1069 1070 1071 1072 1073 1074
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1075 1076
	unsigned int i;

1077
	preempt_disable();
1078
	smp_cpu_index_default();
1079

1080 1081 1082
	/*
	 * Setup boot CPU information
	 */
1083
	smp_store_boot_cpu_info(); /* Final full version of the data */
1084 1085
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1086

1087
	current_thread_info()->cpu = 0;  /* needed? */
1088
	for_each_possible_cpu(i) {
1089 1090
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1091
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1092
	}
1093 1094
	set_cpu_sibling_map(0);

1095

1096
	if (smp_sanity_check(max_cpus) < 0) {
1097
		pr_info("SMP disabled\n");
1098
		disable_smp();
1099
		goto out;
1100 1101
	}

1102 1103
	default_setup_apic_routing();

J
Jack Steiner 已提交
1104
	preempt_disable();
1105
	if (read_apic_id() != boot_cpu_physical_apicid) {
1106
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1107
		     read_apic_id(), boot_cpu_physical_apicid);
1108 1109
		/* Or can we switch back to PIC here? */
	}
J
Jack Steiner 已提交
1110
	preempt_enable();
1111 1112

	connect_bsp_APIC();
1113

1114 1115 1116 1117 1118
	/*
	 * Switch from PIC to APIC mode.
	 */
	setup_local_APIC();

1119 1120 1121 1122 1123
	if (x2apic_mode)
		cpu0_logical_apicid = apic_read(APIC_LDR);
	else
		cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));

1124 1125 1126 1127 1128
	/*
	 * Enable IO APIC before setting up error vector
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		enable_IO_APIC();
1129

1130
	bsp_end_local_APIC_setup();
1131

1132 1133
	if (apic->setup_portio_remap)
		apic->setup_portio_remap();
1134 1135 1136 1137 1138 1139

	smpboot_setup_io_apic();
	/*
	 * Set up local APIC timer on boot CPU.
	 */

1140
	pr_info("CPU%d: ", 0);
1141
	print_cpu_info(&cpu_data(0));
1142
	x86_init.timers.setup_percpu_clockev();
1143 1144 1145

	if (is_uv_system())
		uv_system_init();
1146 1147

	set_mtrr_aps_delayed_init();
1148 1149
out:
	preempt_enable();
1150
}
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1162 1163 1164 1165 1166 1167
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1168
	switch_to_new_gdt(me);
1169 1170
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1171 1172 1173
	per_cpu(cpu_state, me) = CPU_ONLINE;
}

1174 1175
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1176
	pr_debug("Boot done\n");
1177

D
Don Zickus 已提交
1178
	nmi_selftest();
1179 1180 1181 1182
	impress_friends();
#ifdef CONFIG_X86_IO_APIC
	setup_ioapic_dest();
#endif
1183
	mtrr_aps_init();
1184 1185
}

1186 1187 1188 1189 1190 1191 1192 1193 1194
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1195
/*
1196
 * cpu_possible_mask should be static, it cannot change as cpu's
1197 1198 1199
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1200
 * cpu_present_mask on the other hand can change dynamically.
1201 1202 1203 1204 1205 1206
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1207
 * - The user can overwrite it with possible_cpus=NUM
1208 1209 1210 1211 1212 1213
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1214
	int i, possible;
1215

1216 1217 1218 1219
	/* no processor from mptable or madt */
	if (!num_processors)
		num_processors = 1;

1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1231 1232
		possible = setup_possible_cpus;

1233 1234
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1235 1236
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1237
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1238 1239
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1240
	}
1241

1242 1243 1244 1245
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1246
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1247 1248 1249 1250
			possible, setup_max_cpus);
		possible = i;
	}

1251
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1252 1253 1254
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1255
		set_cpu_possible(i, true);
1256 1257
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1258 1259

	nr_cpu_ids = possible;
1260
}
1261

1262 1263 1264 1265 1266 1267 1268
#ifdef CONFIG_HOTPLUG_CPU

static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1269 1270
	for_each_cpu(sibling, cpu_core_mask(cpu)) {
		cpumask_clear_cpu(cpu, cpu_core_mask(sibling));
1271 1272 1273
		/*/
		 * last thread sibling in this cpu core going down
		 */
1274
		if (cpumask_weight(cpu_sibling_mask(cpu)) == 1)
1275 1276 1277
			cpu_data(sibling).booted_cores--;
	}

1278 1279 1280 1281
	for_each_cpu(sibling, cpu_sibling_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling));
	cpumask_clear(cpu_sibling_mask(cpu));
	cpumask_clear(cpu_core_mask(cpu));
1282 1283
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1284
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1285 1286
}

1287 1288
static void __ref remove_cpu_from_maps(int cpu)
{
1289 1290 1291
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1292
	/* was set by cpu_init() */
1293
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1294
	numa_remove_cpu(cpu);
1295 1296
}

1297
void cpu_disable_common(void)
1298 1299 1300 1301 1302 1303
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1304
	lock_vector_lock();
1305
	remove_cpu_from_maps(cpu);
1306
	unlock_vector_lock();
1307
	fixup_irqs();
1308 1309 1310 1311 1312 1313 1314
}

int native_cpu_disable(void)
{
	clear_local_APIC();

	cpu_disable_common();
1315 1316 1317
	return 0;
}

1318
void native_cpu_die(unsigned int cpu)
1319 1320 1321 1322 1323 1324 1325
{
	/* We don't do anything here: idle task is faking death itself. */
	unsigned int i;

	for (i = 0; i < 10; i++) {
		/* They ack this in play_dead by setting CPU_DEAD */
		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1326 1327
			if (system_state == SYSTEM_RUNNING)
				pr_info("CPU %u is now offline\n", cpu);
1328 1329 1330 1331
			return;
		}
		msleep(100);
	}
1332
	pr_err("CPU %u didn't die...\n", cpu);
1333
}
1334 1335 1336 1337 1338

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();
1339
	amd_e400_remove_cpu(raw_smp_processor_id());
1340 1341 1342

	mb();
	/* Ack it */
T
Tejun Heo 已提交
1343
	__this_cpu_write(cpu_state, CPU_DEAD);
1344 1345 1346 1347 1348 1349 1350

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1351 1352 1353 1354 1355 1356 1357 1358
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
	int i;
1369
	void *mwait_ptr;
1370
	struct cpuinfo_x86 *c = __this_cpu_ptr(&cpu_info);
1371

1372
	if (!(this_cpu_has(X86_FEATURE_MWAIT) && mwait_usable(c)))
1373
		return;
1374
	if (!this_cpu_has(X86_FEATURE_CLFLSH))
1375
		return;
1376
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1401 1402 1403 1404 1405 1406 1407
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1408 1409
	wbinvd();

1410
	while (1) {
1411 1412 1413 1414 1415 1416 1417 1418 1419
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
		clflush(mwait_ptr);
		__monitor(mwait_ptr, 0, 0);
1420 1421
		mb();
		__mwait(eax, 0);
1422 1423 1424 1425 1426
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1427 1428 1429 1430 1431
	}
}

static inline void hlt_play_dead(void)
{
1432
	if (__this_cpu_read(cpu_info.x86) >= 4)
1433 1434
		wbinvd();

1435 1436
	while (1) {
		native_halt();
1437 1438 1439 1440 1441
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1442 1443 1444
	}
}

1445 1446 1447
void native_play_dead(void)
{
	play_dead_common();
1448
	tboot_shutdown(TB_SHUTDOWN_WFS);
1449 1450

	mwait_play_dead();	/* Only returns on failure */
1451 1452
	if (cpuidle_play_dead())
		hlt_play_dead();
1453 1454
}

1455
#else /* ... !CONFIG_HOTPLUG_CPU */
1456
int native_cpu_disable(void)
1457 1458 1459 1460
{
	return -ENOSYS;
}

1461
void native_cpu_die(unsigned int cpu)
1462 1463 1464 1465
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1466 1467 1468 1469 1470 1471

void native_play_dead(void)
{
	BUG();
}

1472
#endif