smpboot.c 37.0 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/module.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/idle.h>
62
#include <asm/realmode.h>
63 64
#include <asm/cpu.h>
#include <asm/numa.h>
65 66 67
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
68
#include <asm/mwait.h>
I
Ingo Molnar 已提交
69
#include <asm/apic.h>
70
#include <asm/io_apic.h>
71
#include <asm/fpu/internal.h>
72
#include <asm/setup.h>
T
Tejun Heo 已提交
73
#include <asm/uv/uv.h>
74
#include <linux/mc146818rtc.h>
75
#include <asm/i8259.h>
76
#include <asm/realmode.h>
77
#include <asm/misc.h>
78

79 80 81 82 83
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
84
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
85 86

/* representing HT siblings of each logical CPU */
87
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
88 89 90
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
91
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
92 93
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

94
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
95

96
/* Per CPU bogomips and other parameters */
97
DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
98
EXPORT_PER_CPU_SYMBOL(cpu_info);
99

100
atomic_t init_deasserted;
101

102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
{
	unsigned long flags;

	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0xa, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);
	local_flush_tlb();
	pr_debug("1.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) =
							start_eip >> 4;
	pr_debug("2.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) =
							start_eip & 0xf;
	pr_debug("3.\n");
}

static inline void smpboot_restore_warm_reset_vector(void)
{
	unsigned long flags;

	/*
	 * Install writable page 0 entry to set BIOS data area.
	 */
	local_flush_tlb();

	/*
	 * Paranoid:  Set warm reset code and vector here back
	 * to default values.
	 */
	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);

	*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
}

139
/*
140 141
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
142
 */
143
static void smp_callin(void)
144 145 146 147 148 149 150 151
{
	int cpuid, phys_id;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
152 153
	 *
	 * Since CPU0 is not wakened up by INIT, it doesn't wait for the IPI.
154
	 */
155
	cpuid = smp_processor_id();
156 157 158
	if (apic->wait_for_init_deassert && cpuid)
		while (!atomic_read(&init_deasserted))
			cpu_relax();
159 160 161 162

	/*
	 * (This works even if the APIC is not enabled.)
	 */
163
	phys_id = read_apic_id();
164 165 166 167 168 169 170

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */
171
	apic_ap_setup();
172

173 174 175 176 177 178
	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

179 180
	/*
	 * Get our bogomips.
181 182 183
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
184 185
	 */
	calibrate_delay();
186
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
187
	pr_debug("Stack at about %p\n", &cpuid);
188

189 190 191 192 193 194 195
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

196 197
	notify_cpu_starting(cpuid);

198 199 200
	/*
	 * Allow the master to continue.
	 */
201
	cpumask_set_cpu(cpuid, cpu_callin_mask);
202 203
}

204 205
static int cpu0_logical_apicid;
static int enable_start_cpu0;
206 207 208
/*
 * Activate a secondary processor.
 */
209
static void notrace start_secondary(void *unused)
210 211 212 213 214 215
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
216
	cpu_init();
217
	x86_cpuinit.early_percpu_clock_init();
218 219
	preempt_disable();
	smp_callin();
220

221 222
	enable_start_cpu0 = 0;

223
#ifdef CONFIG_X86_32
224
	/* switch away from the initial page table */
225 226 227 228
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

229 230 231 232 233 234 235 236
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	/*
237 238 239 240
	 * Lock vector_lock and initialize the vectors on this cpu
	 * before setting the cpu online. We must set it online with
	 * vector_lock held to prevent a concurrent setup/teardown
	 * from seeing a half valid vector space.
241
	 */
242
	lock_vector_lock();
243
	setup_vector_irq(smp_processor_id());
244
	set_cpu_online(smp_processor_id(), true);
245
	unlock_vector_lock();
246
	cpu_set_state_online(smp_processor_id());
247
	x86_platform.nmi_init();
248

249 250 251
	/* enable local interrupts */
	local_irq_enable();

252 253
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
254

255
	x86_cpuinit.setup_percpu_clockev();
256 257

	wmb();
T
Thomas Gleixner 已提交
258
	cpu_startup_entry(CPUHP_ONLINE);
259 260
}

261 262 263 264 265 266 267 268 269
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
}

270 271 272 273
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
274
void smp_store_cpu_info(int id)
275 276 277
{
	struct cpuinfo_x86 *c = &cpu_data(id);

278
	*c = boot_cpu_data;
279
	c->cpu_index = id;
280 281 282 283 284
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
285 286
}

287 288 289 290 291 292 293 294
static bool
topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
}

295
static bool
296
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
297
{
298 299
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

300
	return !WARN_ONCE(!topology_same_node(c, o),
301 302 303 304 305
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

306
#define link_mask(mfunc, c1, c2)					\
307
do {									\
308 309
	cpumask_set_cpu((c1), mfunc(c2));				\
	cpumask_set_cpu((c2), mfunc(c1));				\
310 311
} while (0)

312
static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
313
{
A
Andreas Herrmann 已提交
314
	if (cpu_has_topoext) {
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
		    c->compute_unit_id == o->compute_unit_id)
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

330
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
331 332 333 334 335 336 337 338
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
339 340
}

341 342 343 344 345 346
/*
 * Unlike the other levels, we do not enforce keeping a
 * multicore group inside a NUMA node.  If this happens, we will
 * discard the MC level of the topology later.
 */
static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
347
{
348 349
	if (c->phys_proc_id == o->phys_proc_id)
		return true;
350 351
	return false;
}
352

353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
static struct sched_domain_topology_level numa_inside_package_topology[] = {
#ifdef CONFIG_SCHED_SMT
	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
#endif
#ifdef CONFIG_SCHED_MC
	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
#endif
	{ NULL, },
};
/*
 * set_sched_topology() sets the topology internal to a CPU.  The
 * NUMA topologies are layered on top of it to build the full
 * system topology.
 *
 * If NUMA nodes are observed to occur within a CPU package, this
 * function should be called.  It forces the sched domain code to
 * only use the SMT level for the CPU portion of the topology.
 * This essentially falls back to relying on NUMA information
 * from the SRAT table to describe the entire system topology
 * (except for hyperthreads).
 */
static void primarily_use_numa_for_topology(void)
{
	set_sched_topology(numa_inside_package_topology);
}

379
void set_cpu_sibling_map(int cpu)
380
{
381
	bool has_smt = smp_num_siblings > 1;
382
	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
383
	struct cpuinfo_x86 *c = &cpu_data(cpu);
384 385
	struct cpuinfo_x86 *o;
	int i;
386

387
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
388

389
	if (!has_mp) {
390
		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
391
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
392
		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
393 394 395 396
		c->booted_cores = 1;
		return;
	}

397
	for_each_cpu(i, cpu_sibling_setup_mask) {
398 399 400
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
401
			link_mask(topology_sibling_cpumask, cpu, i);
402

403
		if ((i == cpu) || (has_mp && match_llc(c, o)))
404
			link_mask(cpu_llc_shared_mask, cpu, i);
405

406 407 408 409
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
410
	 * topology_sibling_cpumask links to be set-up.
411 412 413 414
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

415
		if ((i == cpu) || (has_mp && match_die(c, o))) {
416
			link_mask(topology_core_cpumask, cpu, i);
417

418 419 420
			/*
			 *  Does this new cpu bringup a new core?
			 */
421 422
			if (cpumask_weight(
			    topology_sibling_cpumask(cpu)) == 1) {
423 424 425 426
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
427 428
				if (cpumask_first(
				    topology_sibling_cpumask(i)) == i)
429 430 431 432 433 434 435 436 437 438
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
439
		if (match_die(c, o) && !topology_same_node(c, o))
440
			primarily_use_numa_for_topology();
441 442 443
	}
}

444
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
445
const struct cpumask *cpu_coregroup_mask(int cpu)
446
{
447
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
448 449
}

I
Ingo Molnar 已提交
450
static void impress_friends(void)
451 452 453 454 455 456
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
457
	pr_debug("Before bogomips\n");
458
	for_each_possible_cpu(cpu)
459
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
460
			bogosum += cpu_data(cpu).loops_per_jiffy;
461
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
462
		num_online_cpus(),
463 464 465
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

466
	pr_debug("Before bogocount - setting activated=1\n");
467 468
}

469
void __inquire_remote_apic(int apicid)
470 471
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
472
	const char * const names[] = { "ID", "VERSION", "SPIV" };
473 474 475
	int timeout;
	u32 status;

476
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
477 478

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
479
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
480 481 482 483 484 485

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
486
			pr_cont("a previous APIC delivery may have failed\n");
487

488
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
489 490 491 492 493 494 495 496 497 498

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
499
			pr_cont("%08x\n", status);
500 501
			break;
		default:
502
			pr_cont("failed\n");
503 504 505 506
		}
	}
}

507 508 509 510 511 512 513 514
/*
 * The Multiprocessor Specification 1.4 (1997) example code suggests
 * that there should be a 10ms delay between the BSP asserting INIT
 * and de-asserting INIT, when starting a remote processor.
 * But that slows boot and resume on modern processors, which include
 * many cores and don't require that delay.
 *
 * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
515
 * Modern processor families are quirked to remove the delay entirely.
516 517 518 519 520 521 522 523 524 525 526 527 528
 */
#define UDELAY_10MS_DEFAULT 10000

static unsigned int init_udelay = UDELAY_10MS_DEFAULT;

static int __init cpu_init_udelay(char *str)
{
	get_option(&str, &init_udelay);

	return 0;
}
early_param("cpu_init_udelay", cpu_init_udelay);

529 530 531 532 533 534 535 536 537 538 539 540
static void __init smp_quirk_init_udelay(void)
{
	/* if cmdline changed it from default, leave it alone */
	if (init_udelay != UDELAY_10MS_DEFAULT)
		return;

	/* if modern processor, use no delay */
	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF)))
		init_udelay = 0;
}

541 542 543 544 545
/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
546
int
547
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
548 549 550 551 552 553 554
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
555
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
556

557
	pr_debug("Waiting for send to finish...\n");
558 559 560 561 562 563
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
564
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
565 566 567 568 569
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
570
	pr_debug("NMI sent\n");
571 572

	if (send_status)
573
		pr_err("APIC never delivered???\n");
574
	if (accept_status)
575
		pr_err("APIC delivery error (%lx)\n", accept_status);
576 577 578 579

	return (send_status | accept_status);
}

580
static int
581
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
582
{
583
	unsigned long send_status = 0, accept_status = 0;
584 585
	int maxlvt, num_starts, j;

586 587
	maxlvt = lapic_get_maxlvt();

588 589 590 591
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
592 593
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
594 595 596
		apic_read(APIC_ESR);
	}

597
	pr_debug("Asserting INIT\n");
598 599 600 601 602 603 604

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
605 606
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
607

608
	pr_debug("Waiting for send to finish...\n");
609 610
	send_status = safe_apic_wait_icr_idle();

611
	udelay(init_udelay);
612

613
	pr_debug("Deasserting INIT\n");
614 615 616

	/* Target chip */
	/* Send IPI */
617
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
618

619
	pr_debug("Waiting for send to finish...\n");
620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
	send_status = safe_apic_wait_icr_idle();

	mb();
	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Paravirt / VMI wants a startup IPI hook here to set up the
	 * target processor state.
	 */
	startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
641
			 stack_start);
642 643 644 645

	/*
	 * Run STARTUP IPI loop.
	 */
646
	pr_debug("#startup loops: %d\n", num_starts);
647 648

	for (j = 1; j <= num_starts; j++) {
649
		pr_debug("Sending STARTUP #%d\n", j);
650 651
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
652
		apic_read(APIC_ESR);
653
		pr_debug("After apic_write\n");
654 655 656 657 658 659 660 661

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
662 663
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
664 665 666 667 668 669

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

670
		pr_debug("Startup point 1\n");
671

672
		pr_debug("Waiting for send to finish...\n");
673 674 675 676 677 678 679
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);

680
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
681 682 683 684 685
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
686
	pr_debug("After Startup\n");
687 688

	if (send_status)
689
		pr_err("APIC never delivered???\n");
690
	if (accept_status)
691
		pr_err("APIC delivery error (%lx)\n", accept_status);
692 693 694 695

	return (send_status | accept_status);
}

696 697 698 699 700 701 702 703
void smp_announce(void)
{
	int num_nodes = num_online_nodes();

	printk(KERN_INFO "x86: Booted up %d node%s, %d CPUs\n",
	       num_nodes, (num_nodes > 1 ? "s" : ""), num_online_cpus());
}

704
/* reduce the number of lines printed when booting a large cpu count system */
705
static void announce_cpu(int cpu, int apicid)
706 707
{
	static int current_node = -1;
708
	int node = early_cpu_to_node(cpu);
709
	static int width, node_width;
710 711 712

	if (!width)
		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
713

714 715 716 717 718 719
	if (!node_width)
		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */

	if (cpu == 1)
		printk(KERN_INFO "x86: Booting SMP configuration:\n");

720 721 722
	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
723
				pr_cont("\n");
724
			current_node = node;
725 726 727

			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
			       node_width - num_digits(node), " ", node);
728
		}
729 730 731 732 733 734 735

		/* Add padding for the BSP */
		if (cpu == 1)
			pr_cont("%*s", width + 1, " ");

		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);

736 737 738 739 740
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
764
static int
765 766 767 768 769 770
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

771 772
	preempt_disable();

773 774 775
	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
776 777 778 779
	if (cpu) {
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
		goto out;
	}
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}
798 799 800

out:
	preempt_enable();
801 802 803 804

	return boot_error;
}

805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
void common_cpu_up(unsigned int cpu, struct task_struct *idle)
{
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();

	per_cpu(current_task, cpu) = idle;

#ifdef CONFIG_X86_32
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
	per_cpu(cpu_current_top_of_stack, cpu) =
		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
#else
	clear_tsk_thread_flag(idle, TIF_FORK);
	initial_gs = per_cpu_offset(cpu);
#endif
}

823 824 825
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
826 827
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
828
 */
829
static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
830
{
831
	volatile u32 *trampoline_status =
832
		(volatile u32 *) __va(real_mode_header->trampoline_status);
833
	/* start_ip had better be page-aligned! */
834
	unsigned long start_ip = real_mode_header->trampoline_start;
835

836
	unsigned long boot_error = 0;
837
	int cpu0_nmi_registered = 0;
838
	unsigned long timeout;
839

840 841
	idle->thread.sp = (unsigned long) (((struct pt_regs *)
			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
842

843
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
844
	initial_code = (unsigned long)start_secondary;
845
	stack_start  = idle->thread.sp;
846

847 848 849 850 851 852 853
	/*
	 * Enable the espfix hack for this CPU
	*/
#ifdef CONFIG_X86_ESPFIX64
	init_espfix_ap(cpu);
#endif

854 855
	/* So we see what's up */
	announce_cpu(cpu, apicid);
856 857 858 859 860 861 862 863

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

J
Jack Steiner 已提交
864
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
865

866
		pr_debug("Setting warm reset code and vector.\n");
867

J
Jack Steiner 已提交
868 869 870
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
871 872 873 874 875
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
876
	}
877

878 879 880 881 882 883 884 885 886
	/*
	 * AP might wait on cpu_callout_mask in cpu_init() with
	 * cpu_initialized_mask set if previous attempt to online
	 * it timed-out. Clear cpu_initialized_mask so that after
	 * INIT/SIPI it could start with a clean state.
	 */
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
	smp_mb();

887
	/*
888 889 890 891
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
892
	 */
893 894 895
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
896 897
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
898 899 900

	if (!boot_error) {
		/*
901
		 * Wait 10s total for a response from AP
902
		 */
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917
		boot_error = -1;
		timeout = jiffies + 10*HZ;
		while (time_before(jiffies, timeout)) {
			if (cpumask_test_cpu(cpu, cpu_initialized_mask)) {
				/*
				 * Tell AP to proceed with initialization
				 */
				cpumask_set_cpu(cpu, cpu_callout_mask);
				boot_error = 0;
				break;
			}
			udelay(100);
			schedule();
		}
	}
918

919
	if (!boot_error) {
920
		/*
921
		 * Wait till AP completes initial initialization
922
		 */
923
		while (!cpumask_test_cpu(cpu, cpu_callin_mask)) {
924 925 926 927 928 929
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
930
			udelay(100);
931
			schedule();
932 933 934 935
		}
	}

	/* mark "stuck" area as not stuck */
936
	*trampoline_status = 0;
937

938 939 940 941 942 943
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
944 945 946 947 948 949 950
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

951 952 953
	return boot_error;
}

954
int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
955
{
956
	int apicid = apic->cpu_present_to_apicid(cpu);
957 958 959 960 961
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

962
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
963

964
	if (apicid == BAD_APICID ||
965
	    !physid_isset(apicid, phys_cpu_present_map) ||
966
	    !apic->apic_id_valid(apicid)) {
967
		pr_err("%s: bad cpu %d\n", __func__, cpu);
968 969 970 971 972 973
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
974
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
975
		pr_debug("do_boot_cpu %d Already started\n", cpu);
976 977 978 979 980 981 982 983 984
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

985 986 987 988
	/* x86 CPUs take themselves offline, so delayed offline is OK. */
	err = cpu_check_up_prepare(cpu);
	if (err && err != -EBUSY)
		return err;
989

990 991 992
	/* the FPU context is blank, nobody can own it */
	__cpu_disable_lazy_restore(cpu);

993 994
	common_cpu_up(cpu, tidle);

995
	err = do_boot_cpu(apicid, cpu, tidle);
996
	if (err) {
997
		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
998
		return -EIO;
999 1000 1001 1002 1003 1004 1005 1006 1007 1008
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

1009
	while (!cpu_online(cpu)) {
1010 1011 1012 1013 1014 1015 1016
		cpu_relax();
		touch_nmi_watchdog();
	}

	return 0;
}

1017 1018 1019 1020 1021 1022 1023 1024
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

1025 1026 1027 1028 1029 1030 1031
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
1032 1033
	pr_info("SMP disabled\n");

1034 1035
	disable_ioapic_support();

1036 1037
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
1038

1039
	if (smp_found_config)
1040
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1041
	else
1042
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1043 1044
	cpumask_set_cpu(0, topology_sibling_cpumask(0));
	cpumask_set_cpu(0, topology_core_cpumask(0));
1045 1046
}

1047 1048 1049 1050 1051 1052 1053
enum {
	SMP_OK,
	SMP_NO_CONFIG,
	SMP_NO_APIC,
	SMP_FORCE_UP,
};

1054 1055 1056 1057 1058
/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
1059
	preempt_disable();
1060

1061
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1062 1063 1064 1065
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

1066 1067
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
1068 1069 1070 1071

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
1072
				set_cpu_present(cpu, false);
1073 1074 1075 1076 1077 1078
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
1079
				set_cpu_possible(cpu, false);
1080 1081 1082 1083 1084 1085 1086
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

1087
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1088
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
1089 1090
			hard_smp_processor_id());

1091 1092 1093 1094 1095 1096 1097 1098
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1099
		preempt_enable();
1100
		pr_notice("SMP motherboard not detected\n");
1101
		return SMP_NO_CONFIG;
1102 1103 1104 1105 1106 1107
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1108
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1109 1110
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1111 1112
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1113
	preempt_enable();
1114 1115 1116 1117 1118 1119

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
	    !cpu_has_apic) {
1120 1121 1122
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1123
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1124
		}
1125
		return SMP_NO_APIC;
1126 1127 1128 1129 1130 1131
	}

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1132
		pr_info("SMP mode deactivated\n");
1133
		return SMP_FORCE_UP;
1134 1135
	}

1136
	return SMP_OK;
1137 1138 1139 1140 1141 1142 1143
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1144
	for_each_possible_cpu(i) {
1145 1146
		c = &cpu_data(i);
		/* mark all to hotplug */
1147
		c->cpu_index = nr_cpu_ids;
1148 1149 1150 1151 1152 1153 1154 1155 1156
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1157 1158
	unsigned int i;

1159
	smp_cpu_index_default();
1160

1161 1162 1163
	/*
	 * Setup boot CPU information
	 */
1164
	smp_store_boot_cpu_info(); /* Final full version of the data */
1165 1166
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1167

1168
	current_thread_info()->cpu = 0;  /* needed? */
1169
	for_each_possible_cpu(i) {
1170 1171
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1172
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1173
	}
1174 1175
	set_cpu_sibling_map(0);

1176 1177
	switch (smp_sanity_check(max_cpus)) {
	case SMP_NO_CONFIG:
1178
		disable_smp();
1179 1180 1181 1182 1183 1184 1185 1186
		if (APIC_init_uniprocessor())
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
		return;
	case SMP_NO_APIC:
		disable_smp();
		return;
	case SMP_FORCE_UP:
		disable_smp();
1187
		apic_bsp_setup(false);
1188
		return;
1189 1190
	case SMP_OK:
		break;
1191 1192
	}

1193 1194
	default_setup_apic_routing();

1195
	if (read_apic_id() != boot_cpu_physical_apicid) {
1196
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1197
		     read_apic_id(), boot_cpu_physical_apicid);
1198 1199 1200
		/* Or can we switch back to PIC here? */
	}

1201
	cpu0_logical_apicid = apic_bsp_setup(false);
1202

1203
	pr_info("CPU%d: ", 0);
1204
	print_cpu_info(&cpu_data(0));
1205 1206 1207

	if (is_uv_system())
		uv_system_init();
1208 1209

	set_mtrr_aps_delayed_init();
1210 1211

	smp_quirk_init_udelay();
1212
}
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1224 1225 1226 1227 1228 1229
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1230
	switch_to_new_gdt(me);
1231 1232
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1233
	cpu_set_state_online(me);
1234 1235
}

1236 1237
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1238
	pr_debug("Boot done\n");
1239

D
Don Zickus 已提交
1240
	nmi_selftest();
1241 1242
	impress_friends();
	setup_ioapic_dest();
1243
	mtrr_aps_init();
1244 1245
}

1246 1247 1248 1249 1250 1251 1252 1253 1254
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1255
/*
1256
 * cpu_possible_mask should be static, it cannot change as cpu's
1257 1258 1259
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1260
 * cpu_present_mask on the other hand can change dynamically.
1261 1262 1263 1264 1265 1266
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1267
 * - The user can overwrite it with possible_cpus=NUM
1268 1269 1270 1271 1272 1273
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1274
	int i, possible;
1275

1276 1277 1278 1279
	/* no processor from mptable or madt */
	if (!num_processors)
		num_processors = 1;

1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1291 1292
		possible = setup_possible_cpus;

1293 1294
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1295 1296
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1297
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1298 1299
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1300
	}
1301

1302 1303 1304 1305
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1306
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1307 1308 1309 1310
			possible, setup_max_cpus);
		possible = i;
	}

1311
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1312 1313 1314
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1315
		set_cpu_possible(i, true);
1316 1317
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1318 1319

	nr_cpu_ids = possible;
1320
}
1321

1322 1323 1324 1325 1326 1327 1328
#ifdef CONFIG_HOTPLUG_CPU

static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1329 1330
	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1331 1332 1333
		/*/
		 * last thread sibling in this cpu core going down
		 */
1334
		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1335 1336 1337
			cpu_data(sibling).booted_cores--;
	}

1338 1339
	for_each_cpu(sibling, topology_sibling_cpumask(cpu))
		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1340 1341 1342
	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
	cpumask_clear(cpu_llc_shared_mask(cpu));
1343 1344
	cpumask_clear(topology_sibling_cpumask(cpu));
	cpumask_clear(topology_core_cpumask(cpu));
1345 1346
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1347
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1348 1349
}

1350 1351
static void __ref remove_cpu_from_maps(int cpu)
{
1352 1353 1354
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1355
	/* was set by cpu_init() */
1356
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1357
	numa_remove_cpu(cpu);
1358 1359
}

1360
void cpu_disable_common(void)
1361 1362 1363 1364 1365 1366
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1367
	lock_vector_lock();
1368
	remove_cpu_from_maps(cpu);
1369
	unlock_vector_lock();
1370
	fixup_irqs();
1371 1372 1373 1374
}

int native_cpu_disable(void)
{
1375 1376 1377 1378 1379 1380
	int ret;

	ret = check_irq_vectors_for_cpu_disable();
	if (ret)
		return ret;

1381 1382
	clear_local_APIC();
	cpu_disable_common();
1383

1384 1385 1386
	return 0;
}

1387
int common_cpu_die(unsigned int cpu)
1388
{
1389
	int ret = 0;
1390

1391
	/* We don't do anything here: idle task is faking death itself. */
1392

1393
	/* They ack this in play_dead() by setting CPU_DEAD */
1394
	if (cpu_wait_death(cpu, 5)) {
1395 1396 1397 1398
		if (system_state == SYSTEM_RUNNING)
			pr_info("CPU %u is now offline\n", cpu);
	} else {
		pr_err("CPU %u didn't die...\n", cpu);
1399
		ret = -1;
1400
	}
1401 1402 1403 1404 1405 1406 1407

	return ret;
}

void native_cpu_die(unsigned int cpu)
{
	common_cpu_die(cpu);
1408
}
1409 1410 1411 1412 1413

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();
1414
	amd_e400_remove_cpu(raw_smp_processor_id());
1415 1416

	/* Ack it */
1417
	(void)cpu_report_death();
1418 1419 1420 1421 1422 1423 1424

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1425 1426 1427 1428 1429 1430 1431 1432
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1433 1434 1435 1436 1437 1438 1439 1440 1441
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
1442
	void *mwait_ptr;
1443
	int i;
1444

1445
	if (!this_cpu_has(X86_FEATURE_MWAIT))
1446
		return;
1447
	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1448
		return;
1449
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1474 1475 1476 1477 1478 1479 1480
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1481 1482
	wbinvd();

1483
	while (1) {
1484 1485 1486 1487 1488 1489 1490
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
1491
		mb();
1492
		clflush(mwait_ptr);
1493
		mb();
1494
		__monitor(mwait_ptr, 0, 0);
1495 1496
		mb();
		__mwait(eax, 0);
1497 1498 1499 1500 1501
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1502 1503 1504 1505 1506
	}
}

static inline void hlt_play_dead(void)
{
1507
	if (__this_cpu_read(cpu_info.x86) >= 4)
1508 1509
		wbinvd();

1510 1511
	while (1) {
		native_halt();
1512 1513 1514 1515 1516
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1517 1518 1519
	}
}

1520 1521 1522
void native_play_dead(void)
{
	play_dead_common();
1523
	tboot_shutdown(TB_SHUTDOWN_WFS);
1524 1525

	mwait_play_dead();	/* Only returns on failure */
1526 1527
	if (cpuidle_play_dead())
		hlt_play_dead();
1528 1529
}

1530
#else /* ... !CONFIG_HOTPLUG_CPU */
1531
int native_cpu_disable(void)
1532 1533 1534 1535
{
	return -ENOSYS;
}

1536
void native_cpu_die(unsigned int cpu)
1537 1538 1539 1540
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1541 1542 1543 1544 1545 1546

void native_play_dead(void)
{
	BUG();
}

1547
#endif