smpboot.c 36.4 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/module.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/idle.h>
62
#include <asm/realmode.h>
63 64
#include <asm/cpu.h>
#include <asm/numa.h>
65 66 67
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
68
#include <asm/mwait.h>
I
Ingo Molnar 已提交
69
#include <asm/apic.h>
70
#include <asm/io_apic.h>
71 72
#include <asm/i387.h>
#include <asm/fpu-internal.h>
73
#include <asm/setup.h>
T
Tejun Heo 已提交
74
#include <asm/uv/uv.h>
75
#include <linux/mc146818rtc.h>
76
#include <asm/smpboot_hooks.h>
77
#include <asm/i8259.h>
78
#include <asm/realmode.h>
79
#include <asm/misc.h>
80

81 82 83
/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

84 85 86 87 88
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
89
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
90 91

/* representing HT siblings of each logical CPU */
92
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
93 94 95
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
96
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
97 98
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

99
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
100

101 102 103
/* Per CPU bogomips and other parameters */
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
104

105
atomic_t init_deasserted;
106 107

/*
108 109
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
110
 */
111
static void smp_callin(void)
112 113
{
	int cpuid, phys_id;
114
	unsigned long timeout;
115 116 117 118 119 120

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
121 122
	 *
	 * Since CPU0 is not wakened up by INIT, it doesn't wait for the IPI.
123
	 */
124
	cpuid = smp_processor_id();
125 126 127
	if (apic->wait_for_init_deassert && cpuid)
		while (!atomic_read(&init_deasserted))
			cpu_relax();
128 129 130 131

	/*
	 * (This works even if the APIC is not enabled.)
	 */
132
	phys_id = read_apic_id();
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
	if (cpumask_test_cpu(cpuid, cpu_callin_mask)) {
		panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
					phys_id, cpuid);
	}
	pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
		if (cpumask_test_cpu(cpuid, cpu_callout_mask))
			break;
		cpu_relax();
	}

	if (!time_before(jiffies, timeout)) {
		panic("%s: CPU%d started up but did not get a callout!\n",
		      __func__, cpuid);
	}
164 165 166 167 168 169 170 171 172 173

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */
	setup_local_APIC();
	end_local_APIC_setup();

174 175 176
	/*
	 * Need to setup vector mappings before we enable interrupts.
	 */
177
	setup_vector_irq(smp_processor_id());
178 179 180 181 182 183 184

	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

185 186
	/*
	 * Get our bogomips.
187 188 189
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
190 191
	 */
	calibrate_delay();
192
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
193
	pr_debug("Stack at about %p\n", &cpuid);
194

195 196 197 198 199 200 201
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

202 203
	notify_cpu_starting(cpuid);

204 205 206
	/*
	 * Allow the master to continue.
	 */
207
	cpumask_set_cpu(cpuid, cpu_callin_mask);
208 209
}

210 211
static int cpu0_logical_apicid;
static int enable_start_cpu0;
212 213 214
/*
 * Activate a secondary processor.
 */
215
static void notrace start_secondary(void *unused)
216 217 218 219 220 221
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
222
	cpu_init();
223
	x86_cpuinit.early_percpu_clock_init();
224 225
	preempt_disable();
	smp_callin();
226

227 228
	enable_start_cpu0 = 0;

229
#ifdef CONFIG_X86_32
230
	/* switch away from the initial page table */
231 232 233 234
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

235 236 237 238 239 240 241
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

242 243 244
	/*
	 * Enable the espfix hack for this CPU
	 */
245
#ifdef CONFIG_X86_ESPFIX64
246 247 248
	init_espfix_ap();
#endif

249
	/*
250 251 252
	 * We need to hold vector_lock so there the set of online cpus
	 * does not change while we are assigning vectors to cpus.  Holding
	 * this lock ensures we don't half assign or remove an irq from a cpu.
253
	 */
254
	lock_vector_lock();
255
	set_cpu_online(smp_processor_id(), true);
256
	unlock_vector_lock();
257
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
258
	x86_platform.nmi_init();
259

260 261 262
	/* enable local interrupts */
	local_irq_enable();

263 264
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
265

266
	x86_cpuinit.setup_percpu_clockev();
267 268

	wmb();
T
Thomas Gleixner 已提交
269
	cpu_startup_entry(CPUHP_ONLINE);
270 271
}

272 273 274 275 276 277 278 279 280
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
}

281 282 283 284
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
285
void smp_store_cpu_info(int id)
286 287 288
{
	struct cpuinfo_x86 *c = &cpu_data(id);

289
	*c = boot_cpu_data;
290
	c->cpu_index = id;
291 292 293 294 295
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
296 297
}

298 299 300 301 302 303 304 305
static bool
topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
}

306
static bool
307
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
308
{
309 310
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

311
	return !WARN_ONCE(!topology_same_node(c, o),
312 313 314 315 316 317 318 319 320 321 322
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

#define link_mask(_m, c1, c2)						\
do {									\
	cpumask_set_cpu((c1), cpu_##_m##_mask(c2));			\
	cpumask_set_cpu((c2), cpu_##_m##_mask(c1));			\
} while (0)

323
static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
324
{
A
Andreas Herrmann 已提交
325
	if (cpu_has_topoext) {
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
		    c->compute_unit_id == o->compute_unit_id)
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

341
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
342 343 344 345 346 347 348 349
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
350 351
}

352 353 354 355 356 357
/*
 * Unlike the other levels, we do not enforce keeping a
 * multicore group inside a NUMA node.  If this happens, we will
 * discard the MC level of the topology later.
 */
static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
358
{
359 360
	if (c->phys_proc_id == o->phys_proc_id)
		return true;
361 362
	return false;
}
363

364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
static struct sched_domain_topology_level numa_inside_package_topology[] = {
#ifdef CONFIG_SCHED_SMT
	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
#endif
#ifdef CONFIG_SCHED_MC
	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
#endif
	{ NULL, },
};
/*
 * set_sched_topology() sets the topology internal to a CPU.  The
 * NUMA topologies are layered on top of it to build the full
 * system topology.
 *
 * If NUMA nodes are observed to occur within a CPU package, this
 * function should be called.  It forces the sched domain code to
 * only use the SMT level for the CPU portion of the topology.
 * This essentially falls back to relying on NUMA information
 * from the SRAT table to describe the entire system topology
 * (except for hyperthreads).
 */
static void primarily_use_numa_for_topology(void)
{
	set_sched_topology(numa_inside_package_topology);
}

390
void set_cpu_sibling_map(int cpu)
391
{
392
	bool has_smt = smp_num_siblings > 1;
393
	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
394
	struct cpuinfo_x86 *c = &cpu_data(cpu);
395 396
	struct cpuinfo_x86 *o;
	int i;
397

398
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
399

400
	if (!has_mp) {
401
		cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
402 403
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
		cpumask_set_cpu(cpu, cpu_core_mask(cpu));
404 405 406 407
		c->booted_cores = 1;
		return;
	}

408
	for_each_cpu(i, cpu_sibling_setup_mask) {
409 410 411 412 413
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
			link_mask(sibling, cpu, i);

414
		if ((i == cpu) || (has_mp && match_llc(c, o)))
415 416
			link_mask(llc_shared, cpu, i);

417 418 419 420 421 422 423 424 425
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
	 * cpu_sibling_mask links to be set-up.
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

426
		if ((i == cpu) || (has_mp && match_die(c, o))) {
427 428
			link_mask(core, cpu, i);

429 430 431
			/*
			 *  Does this new cpu bringup a new core?
			 */
432
			if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) {
433 434 435 436
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
437
				if (cpumask_first(cpu_sibling_mask(i)) == i)
438 439 440 441 442 443 444 445 446 447
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
448 449
		if (match_die(c, o) == !topology_same_node(c, o))
			primarily_use_numa_for_topology();
450 451 452
	}
}

453
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
454
const struct cpumask *cpu_coregroup_mask(int cpu)
455
{
456
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
457 458
}

I
Ingo Molnar 已提交
459
static void impress_friends(void)
460 461 462 463 464 465
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
466
	pr_debug("Before bogomips\n");
467
	for_each_possible_cpu(cpu)
468
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
469
			bogosum += cpu_data(cpu).loops_per_jiffy;
470
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
471
		num_online_cpus(),
472 473 474
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

475
	pr_debug("Before bogocount - setting activated=1\n");
476 477
}

478
void __inquire_remote_apic(int apicid)
479 480
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
481
	const char * const names[] = { "ID", "VERSION", "SPIV" };
482 483 484
	int timeout;
	u32 status;

485
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
486 487

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
488
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
489 490 491 492 493 494

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
495
			pr_cont("a previous APIC delivery may have failed\n");
496

497
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
498 499 500 501 502 503 504 505 506 507

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
508
			pr_cont("%08x\n", status);
509 510
			break;
		default:
511
			pr_cont("failed\n");
512 513 514 515 516 517 518 519 520
		}
	}
}

/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
521
int
522
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
523 524 525 526 527 528 529
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
530
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
531

532
	pr_debug("Waiting for send to finish...\n");
533 534 535 536 537 538
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
539
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
540 541 542 543 544
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
545
	pr_debug("NMI sent\n");
546 547

	if (send_status)
548
		pr_err("APIC never delivered???\n");
549
	if (accept_status)
550
		pr_err("APIC delivery error (%lx)\n", accept_status);
551 552 553 554

	return (send_status | accept_status);
}

555
static int
556
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
557 558 559 560
{
	unsigned long send_status, accept_status = 0;
	int maxlvt, num_starts, j;

561 562
	maxlvt = lapic_get_maxlvt();

563 564 565 566
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
567 568
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
569 570 571
		apic_read(APIC_ESR);
	}

572
	pr_debug("Asserting INIT\n");
573 574 575 576 577 578 579

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
580 581
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
582

583
	pr_debug("Waiting for send to finish...\n");
584 585 586 587
	send_status = safe_apic_wait_icr_idle();

	mdelay(10);

588
	pr_debug("Deasserting INIT\n");
589 590 591

	/* Target chip */
	/* Send IPI */
592
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
593

594
	pr_debug("Waiting for send to finish...\n");
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
	send_status = safe_apic_wait_icr_idle();

	mb();
	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Paravirt / VMI wants a startup IPI hook here to set up the
	 * target processor state.
	 */
	startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
616
			 stack_start);
617 618 619 620

	/*
	 * Run STARTUP IPI loop.
	 */
621
	pr_debug("#startup loops: %d\n", num_starts);
622 623

	for (j = 1; j <= num_starts; j++) {
624
		pr_debug("Sending STARTUP #%d\n", j);
625 626
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
627
		apic_read(APIC_ESR);
628
		pr_debug("After apic_write\n");
629 630 631 632 633 634 635 636

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
637 638
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
639 640 641 642 643 644

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

645
		pr_debug("Startup point 1\n");
646

647
		pr_debug("Waiting for send to finish...\n");
648 649 650 651 652 653
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
654
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
655 656 657 658 659
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
660
	pr_debug("After Startup\n");
661 662

	if (send_status)
663
		pr_err("APIC never delivered???\n");
664
	if (accept_status)
665
		pr_err("APIC delivery error (%lx)\n", accept_status);
666 667 668 669

	return (send_status | accept_status);
}

670 671 672 673 674 675 676 677
void smp_announce(void)
{
	int num_nodes = num_online_nodes();

	printk(KERN_INFO "x86: Booted up %d node%s, %d CPUs\n",
	       num_nodes, (num_nodes > 1 ? "s" : ""), num_online_cpus());
}

678
/* reduce the number of lines printed when booting a large cpu count system */
679
static void announce_cpu(int cpu, int apicid)
680 681
{
	static int current_node = -1;
682
	int node = early_cpu_to_node(cpu);
683
	static int width, node_width;
684 685 686

	if (!width)
		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
687

688 689 690 691 692 693
	if (!node_width)
		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */

	if (cpu == 1)
		printk(KERN_INFO "x86: Booting SMP configuration:\n");

694 695 696
	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
697
				pr_cont("\n");
698
			current_node = node;
699 700 701

			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
			       node_width - num_digits(node), " ", node);
702
		}
703 704 705 706 707 708 709

		/* Add padding for the BSP */
		if (cpu == 1)
			pr_cont("%*s", width + 1, " ");

		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);

710 711 712 713 714
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
738
static int
739 740 741 742 743 744
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

745 746
	preempt_disable();

747 748 749
	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
750 751 752 753
	if (cpu) {
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
		goto out;
	}
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}
772 773 774

out:
	preempt_enable();
775 776 777 778

	return boot_error;
}

779 780 781
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
782 783
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
784
 */
785
static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
786
{
787
	volatile u32 *trampoline_status =
788
		(volatile u32 *) __va(real_mode_header->trampoline_status);
789
	/* start_ip had better be page-aligned! */
790
	unsigned long start_ip = real_mode_header->trampoline_start;
791

792
	unsigned long boot_error = 0;
793
	int timeout;
794
	int cpu0_nmi_registered = 0;
795

796 797
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();
798

799 800 801
	idle->thread.sp = (unsigned long) (((struct pt_regs *)
			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
	per_cpu(current_task, cpu) = idle;
802

803
#ifdef CONFIG_X86_32
804 805 806
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
#else
807
	clear_tsk_thread_flag(idle, TIF_FORK);
808
	initial_gs = per_cpu_offset(cpu);
809
#endif
810
	per_cpu(kernel_stack, cpu) =
811
		(unsigned long)task_stack_page(idle) -
812
		KERNEL_STACK_OFFSET + THREAD_SIZE;
813
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
814
	initial_code = (unsigned long)start_secondary;
815
	stack_start  = idle->thread.sp;
816

817 818
	/* So we see what's up */
	announce_cpu(cpu, apicid);
819 820 821 822 823 824 825 826

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

J
Jack Steiner 已提交
827
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
828

829
		pr_debug("Setting warm reset code and vector.\n");
830

J
Jack Steiner 已提交
831 832 833
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
834 835 836 837 838
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
839
	}
840 841

	/*
842 843 844 845
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
846
	 */
847 848 849
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
850 851
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
852 853 854

	if (!boot_error) {
		/*
855
		 * allow APs to start initializing.
856
		 */
857 858 859
		pr_debug("Before Callout %d\n", cpu);
		cpumask_set_cpu(cpu, cpu_callout_mask);
		pr_debug("After Callout %d\n", cpu);
860 861

		/*
862
		 * Wait 5s total for a response
863
		 */
864 865 866 867
		for (timeout = 0; timeout < 50000; timeout++) {
			if (cpumask_test_cpu(cpu, cpu_callin_mask))
				break;	/* It has booted */
			udelay(100);
868 869 870 871 872 873 874
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
875
		}
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901

		if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
			print_cpu_msr(&cpu_data(cpu));
			pr_debug("CPU%d: has booted.\n", cpu);
		} else {
			boot_error = 1;
			if (*trampoline_status == 0xA5A5A5A5)
				/* trampoline started but...? */
				pr_err("CPU%d: Stuck ??\n", cpu);
			else
				/* trampoline code not run */
				pr_err("CPU%d: Not responding\n", cpu);
			if (apic->inquire_remote_apic)
				apic->inquire_remote_apic(apicid);
		}
	}

	if (boot_error) {
		/* Try to put things back the way they were before ... */
		numa_remove_cpu(cpu); /* was set by numa_add_cpu */

		/* was set by do_boot_cpu() */
		cpumask_clear_cpu(cpu, cpu_callout_mask);

		/* was set by cpu_init() */
		cpumask_clear_cpu(cpu, cpu_initialized_mask);
902 903 904
	}

	/* mark "stuck" area as not stuck */
905
	*trampoline_status = 0;
906

907 908 909 910 911 912
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
913 914 915 916 917 918 919
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

920 921 922
	return boot_error;
}

923
int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
924
{
925
	int apicid = apic->cpu_present_to_apicid(cpu);
926 927 928 929 930
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

931
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
932

933
	if (apicid == BAD_APICID ||
934
	    !physid_isset(apicid, phys_cpu_present_map) ||
935
	    !apic->apic_id_valid(apicid)) {
936
		pr_err("%s: bad cpu %d\n", __func__, cpu);
937 938 939 940 941 942
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
943
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
944
		pr_debug("do_boot_cpu %d Already started\n", cpu);
945 946 947 948 949 950 951 952 953 954 955
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;

956 957 958
	/* the FPU context is blank, nobody can own it */
	__cpu_disable_lazy_restore(cpu);

959
	err = do_boot_cpu(apicid, cpu, tidle);
960
	if (err) {
961
		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
962
		return -EIO;
963 964 965 966 967 968 969 970 971 972
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

973
	while (!cpu_online(cpu)) {
974 975 976 977 978 979 980
		cpu_relax();
		touch_nmi_watchdog();
	}

	return 0;
}

981 982 983 984 985 986 987 988
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

989 990 991 992 993 994 995
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
996 997
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
998
	smpboot_clear_io_apic_irqs();
999

1000
	if (smp_found_config)
1001
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1002
	else
1003
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1004 1005
	cpumask_set_cpu(0, cpu_sibling_mask(0));
	cpumask_set_cpu(0, cpu_core_mask(0));
1006 1007 1008 1009 1010 1011 1012
}

/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
1013
	preempt_disable();
1014

1015
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1016 1017 1018 1019
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

1020 1021
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
1022 1023 1024 1025

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
1026
				set_cpu_present(cpu, false);
1027 1028 1029 1030 1031 1032
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
1033
				set_cpu_possible(cpu, false);
1034 1035 1036 1037 1038 1039 1040
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

1041
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1042
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
1043 1044
			hard_smp_processor_id());

1045 1046 1047 1048 1049 1050 1051 1052
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1053
		preempt_enable();
1054
		pr_notice("SMP motherboard not detected\n");
1055 1056
		disable_smp();
		if (APIC_init_uniprocessor())
1057
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
1058 1059 1060 1061 1062 1063 1064
		return -1;
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1065
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1066 1067
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1068 1069
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1070
	preempt_enable();
1071 1072 1073 1074 1075 1076

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
	    !cpu_has_apic) {
1077 1078 1079
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1080
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1081
		}
1082
		smpboot_clear_io_apic();
1083
		disable_ioapic_support();
1084 1085 1086 1087 1088 1089 1090 1091 1092
		return -1;
	}

	verify_local_APIC();

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1093
		pr_info("SMP mode deactivated\n");
1094
		smpboot_clear_io_apic();
1095

1096 1097
		connect_bsp_APIC();
		setup_local_APIC();
1098
		bsp_end_local_APIC_setup();
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
		return -1;
	}

	return 0;
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1110
	for_each_possible_cpu(i) {
1111 1112
		c = &cpu_data(i);
		/* mark all to hotplug */
1113
		c->cpu_index = nr_cpu_ids;
1114 1115 1116 1117 1118 1119 1120 1121 1122
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1123 1124
	unsigned int i;

1125
	preempt_disable();
1126
	smp_cpu_index_default();
1127

1128 1129 1130
	/*
	 * Setup boot CPU information
	 */
1131
	smp_store_boot_cpu_info(); /* Final full version of the data */
1132 1133
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1134

1135
	current_thread_info()->cpu = 0;  /* needed? */
1136
	for_each_possible_cpu(i) {
1137 1138
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1139
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1140
	}
1141 1142
	set_cpu_sibling_map(0);

1143

1144
	if (smp_sanity_check(max_cpus) < 0) {
1145
		pr_info("SMP disabled\n");
1146
		disable_smp();
1147
		goto out;
1148 1149
	}

1150 1151
	default_setup_apic_routing();

J
Jack Steiner 已提交
1152
	preempt_disable();
1153
	if (read_apic_id() != boot_cpu_physical_apicid) {
1154
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1155
		     read_apic_id(), boot_cpu_physical_apicid);
1156 1157
		/* Or can we switch back to PIC here? */
	}
J
Jack Steiner 已提交
1158
	preempt_enable();
1159 1160

	connect_bsp_APIC();
1161

1162 1163 1164 1165 1166
	/*
	 * Switch from PIC to APIC mode.
	 */
	setup_local_APIC();

1167 1168 1169 1170 1171
	if (x2apic_mode)
		cpu0_logical_apicid = apic_read(APIC_LDR);
	else
		cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));

1172 1173 1174 1175 1176
	/*
	 * Enable IO APIC before setting up error vector
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		enable_IO_APIC();
1177

1178
	bsp_end_local_APIC_setup();
1179 1180 1181 1182 1183
	smpboot_setup_io_apic();
	/*
	 * Set up local APIC timer on boot CPU.
	 */

1184
	pr_info("CPU%d: ", 0);
1185
	print_cpu_info(&cpu_data(0));
1186
	x86_init.timers.setup_percpu_clockev();
1187 1188 1189

	if (is_uv_system())
		uv_system_init();
1190 1191

	set_mtrr_aps_delayed_init();
1192 1193
out:
	preempt_enable();
1194
}
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1206 1207 1208 1209 1210 1211
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1212
	switch_to_new_gdt(me);
1213 1214
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1215 1216 1217
	per_cpu(cpu_state, me) = CPU_ONLINE;
}

1218 1219
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1220
	pr_debug("Boot done\n");
1221

D
Don Zickus 已提交
1222
	nmi_selftest();
1223 1224 1225 1226
	impress_friends();
#ifdef CONFIG_X86_IO_APIC
	setup_ioapic_dest();
#endif
1227
	mtrr_aps_init();
1228 1229
}

1230 1231 1232 1233 1234 1235 1236 1237 1238
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1239
/*
1240
 * cpu_possible_mask should be static, it cannot change as cpu's
1241 1242 1243
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1244
 * cpu_present_mask on the other hand can change dynamically.
1245 1246 1247 1248 1249 1250
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1251
 * - The user can overwrite it with possible_cpus=NUM
1252 1253 1254 1255 1256 1257
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1258
	int i, possible;
1259

1260 1261 1262 1263
	/* no processor from mptable or madt */
	if (!num_processors)
		num_processors = 1;

1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1275 1276
		possible = setup_possible_cpus;

1277 1278
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1279 1280
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1281
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1282 1283
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1284
	}
1285

1286 1287 1288 1289
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1290
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1291 1292 1293 1294
			possible, setup_max_cpus);
		possible = i;
	}

1295
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1296 1297 1298
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1299
		set_cpu_possible(i, true);
1300 1301
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1302 1303

	nr_cpu_ids = possible;
1304
}
1305

1306 1307 1308 1309 1310 1311 1312
#ifdef CONFIG_HOTPLUG_CPU

static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1313 1314
	for_each_cpu(sibling, cpu_core_mask(cpu)) {
		cpumask_clear_cpu(cpu, cpu_core_mask(sibling));
1315 1316 1317
		/*/
		 * last thread sibling in this cpu core going down
		 */
1318
		if (cpumask_weight(cpu_sibling_mask(cpu)) == 1)
1319 1320 1321
			cpu_data(sibling).booted_cores--;
	}

1322 1323 1324 1325
	for_each_cpu(sibling, cpu_sibling_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling));
	cpumask_clear(cpu_sibling_mask(cpu));
	cpumask_clear(cpu_core_mask(cpu));
1326 1327
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1328
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1329 1330
}

1331 1332
static void __ref remove_cpu_from_maps(int cpu)
{
1333 1334 1335
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1336
	/* was set by cpu_init() */
1337
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1338
	numa_remove_cpu(cpu);
1339 1340
}

1341
void cpu_disable_common(void)
1342 1343 1344 1345 1346 1347
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1348
	lock_vector_lock();
1349
	remove_cpu_from_maps(cpu);
1350
	unlock_vector_lock();
1351
	fixup_irqs();
1352 1353 1354 1355
}

int native_cpu_disable(void)
{
1356 1357 1358 1359 1360 1361
	int ret;

	ret = check_irq_vectors_for_cpu_disable();
	if (ret)
		return ret;

1362 1363 1364
	clear_local_APIC();

	cpu_disable_common();
1365 1366 1367
	return 0;
}

1368
void native_cpu_die(unsigned int cpu)
1369 1370 1371 1372 1373 1374 1375
{
	/* We don't do anything here: idle task is faking death itself. */
	unsigned int i;

	for (i = 0; i < 10; i++) {
		/* They ack this in play_dead by setting CPU_DEAD */
		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1376 1377
			if (system_state == SYSTEM_RUNNING)
				pr_info("CPU %u is now offline\n", cpu);
1378 1379 1380 1381
			return;
		}
		msleep(100);
	}
1382
	pr_err("CPU %u didn't die...\n", cpu);
1383
}
1384 1385 1386 1387 1388

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();
1389
	amd_e400_remove_cpu(raw_smp_processor_id());
1390 1391 1392

	mb();
	/* Ack it */
T
Tejun Heo 已提交
1393
	__this_cpu_write(cpu_state, CPU_DEAD);
1394 1395 1396 1397 1398 1399 1400

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1401 1402 1403 1404 1405 1406 1407 1408
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1409 1410 1411 1412 1413 1414 1415 1416 1417
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
1418
	void *mwait_ptr;
1419
	int i;
1420

1421
	if (!this_cpu_has(X86_FEATURE_MWAIT))
1422
		return;
1423
	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1424
		return;
1425
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1450 1451 1452 1453 1454 1455 1456
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1457 1458
	wbinvd();

1459
	while (1) {
1460 1461 1462 1463 1464 1465 1466
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
1467
		mb();
1468
		clflush(mwait_ptr);
1469
		mb();
1470
		__monitor(mwait_ptr, 0, 0);
1471 1472
		mb();
		__mwait(eax, 0);
1473 1474 1475 1476 1477
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1478 1479 1480 1481 1482
	}
}

static inline void hlt_play_dead(void)
{
1483
	if (__this_cpu_read(cpu_info.x86) >= 4)
1484 1485
		wbinvd();

1486 1487
	while (1) {
		native_halt();
1488 1489 1490 1491 1492
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1493 1494 1495
	}
}

1496 1497 1498
void native_play_dead(void)
{
	play_dead_common();
1499
	tboot_shutdown(TB_SHUTDOWN_WFS);
1500 1501

	mwait_play_dead();	/* Only returns on failure */
1502 1503
	if (cpuidle_play_dead())
		hlt_play_dead();
1504 1505
}

1506
#else /* ... !CONFIG_HOTPLUG_CPU */
1507
int native_cpu_disable(void)
1508 1509 1510 1511
{
	return -ENOSYS;
}

1512
void native_cpu_die(unsigned int cpu)
1513 1514 1515 1516
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1517 1518 1519 1520 1521 1522

void native_play_dead(void)
{
	BUG();
}

1523
#endif