smpboot.c 41.2 KB
Newer Older
1
 /*
2 3
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
I
Ingo Molnar 已提交
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

44 45
#include <linux/init.h>
#include <linux/smp.h>
46
#include <linux/export.h>
47
#include <linux/sched.h>
48
#include <linux/percpu.h>
G
Glauber Costa 已提交
49
#include <linux/bootmem.h>
50 51
#include <linux/err.h>
#include <linux/nmi.h>
52
#include <linux/tboot.h>
53
#include <linux/stackprotector.h>
54
#include <linux/gfp.h>
55
#include <linux/cpuidle.h>
56

57
#include <asm/acpi.h>
58
#include <asm/desc.h>
59 60
#include <asm/nmi.h>
#include <asm/irq.h>
61
#include <asm/idle.h>
62
#include <asm/realmode.h>
63 64
#include <asm/cpu.h>
#include <asm/numa.h>
65 66 67
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
68
#include <asm/mwait.h>
I
Ingo Molnar 已提交
69
#include <asm/apic.h>
70
#include <asm/io_apic.h>
71
#include <asm/fpu/internal.h>
72
#include <asm/setup.h>
T
Tejun Heo 已提交
73
#include <asm/uv/uv.h>
74
#include <linux/mc146818rtc.h>
75
#include <asm/i8259.h>
76
#include <asm/realmode.h>
77
#include <asm/misc.h>
78

79 80 81 82 83
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
84
DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
85 86

/* representing HT siblings of each logical CPU */
87
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
88 89 90
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
91
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
92 93
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

94
DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
95

96
/* Per CPU bogomips and other parameters */
97
DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
98
EXPORT_PER_CPU_SYMBOL(cpu_info);
99

100 101 102 103 104 105 106 107
/* Logical package management. We might want to allocate that dynamically */
static int *physical_to_logical_pkg __read_mostly;
static unsigned long *physical_package_map __read_mostly;;
static unsigned long *logical_package_map  __read_mostly;
static unsigned int max_physical_pkg_id __read_mostly;
unsigned int __max_logical_packages __read_mostly;
EXPORT_SYMBOL(__max_logical_packages);

108 109 110
/* Maximum number of SMT threads on any online core */
int __max_smt_threads __read_mostly;

111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
{
	unsigned long flags;

	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0xa, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);
	local_flush_tlb();
	pr_debug("1.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) =
							start_eip >> 4;
	pr_debug("2.\n");
	*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) =
							start_eip & 0xf;
	pr_debug("3.\n");
}

static inline void smpboot_restore_warm_reset_vector(void)
{
	unsigned long flags;

	/*
	 * Install writable page 0 entry to set BIOS data area.
	 */
	local_flush_tlb();

	/*
	 * Paranoid:  Set warm reset code and vector here back
	 * to default values.
	 */
	spin_lock_irqsave(&rtc_lock, flags);
	CMOS_WRITE(0, 0xf);
	spin_unlock_irqrestore(&rtc_lock, flags);

	*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;
}

148
/*
149 150
 * Report back to the Boot Processor during boot time or to the caller processor
 * during CPU online.
151
 */
152
static void smp_callin(void)
153 154 155 156 157
{
	int cpuid, phys_id;

	/*
	 * If waken up by an INIT in an 82489DX configuration
158 159 160
	 * cpu_callout_mask guarantees we don't get here before
	 * an INIT_deassert IPI reaches our local APIC, so it is
	 * now safe to touch our local APIC.
161
	 */
162
	cpuid = smp_processor_id();
163 164 165 166

	/*
	 * (This works even if the APIC is not enabled.)
	 */
167
	phys_id = read_apic_id();
168 169 170 171 172 173 174

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */
175
	apic_ap_setup();
176

177 178 179 180 181 182
	/*
	 * Save our processor parameters. Note: this information
	 * is needed for clock calibration.
	 */
	smp_store_cpu_info(cpuid);

183 184
	/*
	 * Get our bogomips.
185 186 187
	 * Update loops_per_jiffy in cpu_data. Previous call to
	 * smp_store_cpu_info() stored a value that is close but not as
	 * accurate as the value just calculated.
188 189
	 */
	calibrate_delay();
190
	cpu_data(cpuid).loops_per_jiffy = loops_per_jiffy;
191
	pr_debug("Stack at about %p\n", &cpuid);
192

193 194 195 196 197 198 199
	/*
	 * This must be done before setting cpu_online_mask
	 * or calling notify_cpu_starting.
	 */
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

200 201
	notify_cpu_starting(cpuid);

202 203 204
	/*
	 * Allow the master to continue.
	 */
205
	cpumask_set_cpu(cpuid, cpu_callin_mask);
206 207
}

208 209
static int cpu0_logical_apicid;
static int enable_start_cpu0;
210 211 212
/*
 * Activate a secondary processor.
 */
213
static void notrace start_secondary(void *unused)
214 215 216 217 218 219
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
220
	cpu_init();
221
	x86_cpuinit.early_percpu_clock_init();
222 223
	preempt_disable();
	smp_callin();
224

225 226
	enable_start_cpu0 = 0;

227
#ifdef CONFIG_X86_32
228
	/* switch away from the initial page table */
229 230 231 232
	load_cr3(swapper_pg_dir);
	__flush_tlb_all();
#endif

233 234 235 236 237 238 239 240
	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	/*
241 242 243 244
	 * Lock vector_lock and initialize the vectors on this cpu
	 * before setting the cpu online. We must set it online with
	 * vector_lock held to prevent a concurrent setup/teardown
	 * from seeing a half valid vector space.
245
	 */
246
	lock_vector_lock();
247
	setup_vector_irq(smp_processor_id());
248
	set_cpu_online(smp_processor_id(), true);
249
	unlock_vector_lock();
250
	cpu_set_state_online(smp_processor_id());
251
	x86_platform.nmi_init();
252

253 254 255
	/* enable local interrupts */
	local_irq_enable();

256 257
	/* to prevent fake stack check failure in clock setup */
	boot_init_stack_canary();
258

259
	x86_cpuinit.setup_percpu_clockev();
260 261

	wmb();
262
	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
263 264
}

265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
int topology_update_package_map(unsigned int apicid, unsigned int cpu)
{
	unsigned int new, pkg = apicid >> boot_cpu_data.x86_coreid_bits;

	/* Called from early boot ? */
	if (!physical_package_map)
		return 0;

	if (pkg >= max_physical_pkg_id)
		return -EINVAL;

	/* Set the logical package id */
	if (test_and_set_bit(pkg, physical_package_map))
		goto found;

	new = find_first_zero_bit(logical_package_map, __max_logical_packages);
	if (new >= __max_logical_packages) {
		physical_to_logical_pkg[pkg] = -1;
		pr_warn("APIC(%x) Package %u exceeds logical package map\n",
			apicid, pkg);
		return -ENOSPC;
	}
	set_bit(new, logical_package_map);
	pr_info("APIC(%x) Converting physical %u to logical package %u\n",
		apicid, pkg, new);
	physical_to_logical_pkg[pkg] = new;

found:
	cpu_data(cpu).logical_proc_id = physical_to_logical_pkg[pkg];
	return 0;
}

/**
 * topology_phys_to_logical_pkg - Map a physical package id to a logical
 *
 * Returns logical package id or -1 if not found
 */
int topology_phys_to_logical_pkg(unsigned int phys_pkg)
{
	if (phys_pkg >= max_physical_pkg_id)
		return -1;
	return physical_to_logical_pkg[phys_pkg];
}
EXPORT_SYMBOL(topology_phys_to_logical_pkg);

static void __init smp_init_package_map(void)
{
	unsigned int ncpus, cpu;
	size_t size;

	/*
	 * Today neither Intel nor AMD support heterogenous systems. That
	 * might change in the future....
318 319 320 321 322 323 324 325 326 327 328 329 330
	 *
	 * While ideally we'd want '* smp_num_siblings' in the below @ncpus
	 * computation, this won't actually work since some Intel BIOSes
	 * report inconsistent HT data when they disable HT.
	 *
	 * In particular, they reduce the APIC-IDs to only include the cores,
	 * but leave the CPUID topology to say there are (2) siblings.
	 * This means we don't know how many threads there will be until
	 * after the APIC enumeration.
	 *
	 * By not including this we'll sometimes over-estimate the number of
	 * logical packages by the amount of !present siblings, but this is
	 * still better than MAX_LOCAL_APIC.
331 332 333 334 335
	 *
	 * We use total_cpus not nr_cpu_ids because nr_cpu_ids can be limited
	 * on the command line leading to a similar issue as the HT disable
	 * problem because the hyperthreads are usually enumerated after the
	 * primary cores.
336
	 */
337
	ncpus = boot_cpu_data.x86_max_cores;
338 339 340 341 342
	if (!ncpus) {
		pr_warn("x86_max_cores == zero !?!?");
		ncpus = 1;
	}

343
	__max_logical_packages = DIV_ROUND_UP(total_cpus, ncpus);
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373

	/*
	 * Possibly larger than what we need as the number of apic ids per
	 * package can be smaller than the actual used apic ids.
	 */
	max_physical_pkg_id = DIV_ROUND_UP(MAX_LOCAL_APIC, ncpus);
	size = max_physical_pkg_id * sizeof(unsigned int);
	physical_to_logical_pkg = kmalloc(size, GFP_KERNEL);
	memset(physical_to_logical_pkg, 0xff, size);
	size = BITS_TO_LONGS(max_physical_pkg_id) * sizeof(unsigned long);
	physical_package_map = kzalloc(size, GFP_KERNEL);
	size = BITS_TO_LONGS(__max_logical_packages) * sizeof(unsigned long);
	logical_package_map = kzalloc(size, GFP_KERNEL);

	pr_info("Max logical packages: %u\n", __max_logical_packages);

	for_each_present_cpu(cpu) {
		unsigned int apicid = apic->cpu_present_to_apicid(cpu);

		if (apicid == BAD_APICID || !apic->apic_id_valid(apicid))
			continue;
		if (!topology_update_package_map(apicid, cpu))
			continue;
		pr_warn("CPU %u APICId %x disabled\n", cpu, apicid);
		per_cpu(x86_bios_cpu_apicid, cpu) = BAD_APICID;
		set_cpu_possible(cpu, false);
		set_cpu_present(cpu, false);
	}
}

374 375 376 377 378 379 380
void __init smp_store_boot_cpu_info(void)
{
	int id = 0; /* CPU 0 */
	struct cpuinfo_x86 *c = &cpu_data(id);

	*c = boot_cpu_data;
	c->cpu_index = id;
381
	smp_init_package_map();
382 383
}

384 385 386 387
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */
388
void smp_store_cpu_info(int id)
389 390 391
{
	struct cpuinfo_x86 *c = &cpu_data(id);

392
	*c = boot_cpu_data;
393
	c->cpu_index = id;
394 395 396 397 398
	/*
	 * During boot time, CPU0 has this setup already. Save the info when
	 * bringing up AP or offlined CPU0.
	 */
	identify_secondary_cpu(c);
399 400
}

401 402 403 404 405 406 407 408
static bool
topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));
}

409
static bool
410
topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name)
411
{
412 413
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

414
	return !WARN_ONCE(!topology_same_node(c, o),
415 416 417 418 419
		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "
		"[node: %d != %d]. Ignoring dependency.\n",
		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));
}

420
#define link_mask(mfunc, c1, c2)					\
421
do {									\
422 423
	cpumask_set_cpu((c1), mfunc(c2));				\
	cpumask_set_cpu((c2), mfunc(c1));				\
424 425
} while (0)

426
static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
427
{
428
	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
429 430 431 432
		int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

		if (c->phys_proc_id == o->phys_proc_id &&
		    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2) &&
433
		    c->cpu_core_id == o->cpu_core_id)
434 435 436 437 438 439 440 441 442 443
			return topology_sane(c, o, "smt");

	} else if (c->phys_proc_id == o->phys_proc_id &&
		   c->cpu_core_id == o->cpu_core_id) {
		return topology_sane(c, o, "smt");
	}

	return false;
}

444
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
445 446 447 448 449 450 451 452
{
	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;

	if (per_cpu(cpu_llc_id, cpu1) != BAD_APICID &&
	    per_cpu(cpu_llc_id, cpu1) == per_cpu(cpu_llc_id, cpu2))
		return topology_sane(c, o, "llc");

	return false;
453 454
}

455 456 457 458 459 460
/*
 * Unlike the other levels, we do not enforce keeping a
 * multicore group inside a NUMA node.  If this happens, we will
 * discard the MC level of the topology later.
 */
static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
461
{
462 463
	if (c->phys_proc_id == o->phys_proc_id)
		return true;
464 465
	return false;
}
466

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
static struct sched_domain_topology_level numa_inside_package_topology[] = {
#ifdef CONFIG_SCHED_SMT
	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
#endif
#ifdef CONFIG_SCHED_MC
	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
#endif
	{ NULL, },
};
/*
 * set_sched_topology() sets the topology internal to a CPU.  The
 * NUMA topologies are layered on top of it to build the full
 * system topology.
 *
 * If NUMA nodes are observed to occur within a CPU package, this
 * function should be called.  It forces the sched domain code to
 * only use the SMT level for the CPU portion of the topology.
 * This essentially falls back to relying on NUMA information
 * from the SRAT table to describe the entire system topology
 * (except for hyperthreads).
 */
static void primarily_use_numa_for_topology(void)
{
	set_sched_topology(numa_inside_package_topology);
}

493
void set_cpu_sibling_map(int cpu)
494
{
495
	bool has_smt = smp_num_siblings > 1;
496
	bool has_mp = has_smt || boot_cpu_data.x86_max_cores > 1;
497
	struct cpuinfo_x86 *c = &cpu_data(cpu);
498
	struct cpuinfo_x86 *o;
499
	int i, threads;
500

501
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
502

503
	if (!has_mp) {
504
		cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu));
505
		cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
506
		cpumask_set_cpu(cpu, topology_core_cpumask(cpu));
507 508 509 510
		c->booted_cores = 1;
		return;
	}

511
	for_each_cpu(i, cpu_sibling_setup_mask) {
512 513 514
		o = &cpu_data(i);

		if ((i == cpu) || (has_smt && match_smt(c, o)))
515
			link_mask(topology_sibling_cpumask, cpu, i);
516

517
		if ((i == cpu) || (has_mp && match_llc(c, o)))
518
			link_mask(cpu_llc_shared_mask, cpu, i);
519

520 521 522 523
	}

	/*
	 * This needs a separate iteration over the cpus because we rely on all
524
	 * topology_sibling_cpumask links to be set-up.
525 526 527 528
	 */
	for_each_cpu(i, cpu_sibling_setup_mask) {
		o = &cpu_data(i);

529
		if ((i == cpu) || (has_mp && match_die(c, o))) {
530
			link_mask(topology_core_cpumask, cpu, i);
531

532 533 534
			/*
			 *  Does this new cpu bringup a new core?
			 */
535 536
			if (cpumask_weight(
			    topology_sibling_cpumask(cpu)) == 1) {
537 538 539 540
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
541 542
				if (cpumask_first(
				    topology_sibling_cpumask(i)) == i)
543 544 545 546 547 548 549 550 551 552
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
553
		if (match_die(c, o) && !topology_same_node(c, o))
554
			primarily_use_numa_for_topology();
555
	}
556 557 558 559

	threads = cpumask_weight(topology_sibling_cpumask(cpu));
	if (threads > __max_smt_threads)
		__max_smt_threads = threads;
560 561
}

562
/* maps the cpu to the sched domain representing multi-core */
R
Rusty Russell 已提交
563
const struct cpumask *cpu_coregroup_mask(int cpu)
564
{
565
	return cpu_llc_shared_mask(cpu);
R
Rusty Russell 已提交
566 567
}

I
Ingo Molnar 已提交
568
static void impress_friends(void)
569 570 571 572 573 574
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
575
	pr_debug("Before bogomips\n");
576
	for_each_possible_cpu(cpu)
577
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
578
			bogosum += cpu_data(cpu).loops_per_jiffy;
579
	pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n",
580
		num_online_cpus(),
581 582 583
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

584
	pr_debug("Before bogocount - setting activated=1\n");
585 586
}

587
void __inquire_remote_apic(int apicid)
588 589
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
590
	const char * const names[] = { "ID", "VERSION", "SPIV" };
591 592 593
	int timeout;
	u32 status;

594
	pr_info("Inquiring remote APIC 0x%x...\n", apicid);
595 596

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
597
		pr_info("... APIC 0x%x %s: ", apicid, names[i]);
598 599 600 601 602 603

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
604
			pr_cont("a previous APIC delivery may have failed\n");
605

606
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
607 608 609 610 611 612 613 614 615 616

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
617
			pr_cont("%08x\n", status);
618 619
			break;
		default:
620
			pr_cont("failed\n");
621 622 623 624
		}
	}
}

625 626 627 628 629 630 631 632
/*
 * The Multiprocessor Specification 1.4 (1997) example code suggests
 * that there should be a 10ms delay between the BSP asserting INIT
 * and de-asserting INIT, when starting a remote processor.
 * But that slows boot and resume on modern processors, which include
 * many cores and don't require that delay.
 *
 * Cmdline "init_cpu_udelay=" is available to over-ride this delay.
633
 * Modern processor families are quirked to remove the delay entirely.
634 635 636
 */
#define UDELAY_10MS_DEFAULT 10000

637
static unsigned int init_udelay = UINT_MAX;
638 639 640 641 642 643 644 645 646

static int __init cpu_init_udelay(char *str)
{
	get_option(&str, &init_udelay);

	return 0;
}
early_param("cpu_init_udelay", cpu_init_udelay);

647 648 649
static void __init smp_quirk_init_udelay(void)
{
	/* if cmdline changed it from default, leave it alone */
650
	if (init_udelay != UINT_MAX)
651 652 653 654
		return;

	/* if modern processor, use no delay */
	if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) ||
655
	    ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) {
656
		init_udelay = 0;
657 658
		return;
	}
659 660
	/* else, use legacy delay */
	init_udelay = UDELAY_10MS_DEFAULT;
661 662
}

663 664 665 666 667
/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
668
int
669
wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip)
670 671 672 673 674 675 676
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
677
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, apicid);
678

679
	pr_debug("Waiting for send to finish...\n");
680 681 682 683 684 685
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
686
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
687 688 689 690 691
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
692
	pr_debug("NMI sent\n");
693 694

	if (send_status)
695
		pr_err("APIC never delivered???\n");
696
	if (accept_status)
697
		pr_err("APIC delivery error (%lx)\n", accept_status);
698 699 700 701

	return (send_status | accept_status);
}

702
static int
703
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
704
{
705
	unsigned long send_status = 0, accept_status = 0;
706 707
	int maxlvt, num_starts, j;

708 709
	maxlvt = lapic_get_maxlvt();

710 711 712 713
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
714 715
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
716 717 718
		apic_read(APIC_ESR);
	}

719
	pr_debug("Asserting INIT\n");
720 721 722 723 724 725 726

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
727 728
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
729

730
	pr_debug("Waiting for send to finish...\n");
731 732
	send_status = safe_apic_wait_icr_idle();

733
	udelay(init_udelay);
734

735
	pr_debug("Deasserting INIT\n");
736 737 738

	/* Target chip */
	/* Send IPI */
739
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
740

741
	pr_debug("Waiting for send to finish...\n");
742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
	send_status = safe_apic_wait_icr_idle();

	mb();

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Run STARTUP IPI loop.
	 */
760
	pr_debug("#startup loops: %d\n", num_starts);
761 762

	for (j = 1; j <= num_starts; j++) {
763
		pr_debug("Sending STARTUP #%d\n", j);
764 765
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
766
		apic_read(APIC_ESR);
767
		pr_debug("After apic_write\n");
768 769 770 771 772 773 774 775

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
776 777
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
778 779 780 781

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
L
Len Brown 已提交
782 783 784
		if (init_udelay == 0)
			udelay(10);
		else
785
			udelay(300);
786

787
		pr_debug("Startup point 1\n");
788

789
		pr_debug("Waiting for send to finish...\n");
790 791 792 793 794
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
L
Len Brown 已提交
795 796 797
		if (init_udelay == 0)
			udelay(10);
		else
798
			udelay(200);
799

800
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
801 802 803 804 805
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
806
	pr_debug("After Startup\n");
807 808

	if (send_status)
809
		pr_err("APIC never delivered???\n");
810
	if (accept_status)
811
		pr_err("APIC delivery error (%lx)\n", accept_status);
812 813 814 815

	return (send_status | accept_status);
}

816 817 818 819 820 821 822 823
void smp_announce(void)
{
	int num_nodes = num_online_nodes();

	printk(KERN_INFO "x86: Booted up %d node%s, %d CPUs\n",
	       num_nodes, (num_nodes > 1 ? "s" : ""), num_online_cpus());
}

824
/* reduce the number of lines printed when booting a large cpu count system */
825
static void announce_cpu(int cpu, int apicid)
826 827
{
	static int current_node = -1;
828
	int node = early_cpu_to_node(cpu);
829
	static int width, node_width;
830 831 832

	if (!width)
		width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */
833

834 835 836 837 838 839
	if (!node_width)
		node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */

	if (cpu == 1)
		printk(KERN_INFO "x86: Booting SMP configuration:\n");

840 841 842
	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
843
				pr_cont("\n");
844
			current_node = node;
845 846 847

			printk(KERN_INFO ".... node %*s#%d, CPUs:  ",
			       node_width - num_digits(node), " ", node);
848
		}
849 850 851 852 853 854 855

		/* Add padding for the BSP */
		if (cpu == 1)
			pr_cont("%*s", width + 1, " ");

		pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu);

856 857 858 859 860
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
static int wakeup_cpu0_nmi(unsigned int cmd, struct pt_regs *regs)
{
	int cpu;

	cpu = smp_processor_id();
	if (cpu == 0 && !cpu_online(cpu) && enable_start_cpu0)
		return NMI_HANDLED;

	return NMI_DONE;
}

/*
 * Wake up AP by INIT, INIT, STARTUP sequence.
 *
 * Instead of waiting for STARTUP after INITs, BSP will execute the BIOS
 * boot-strap code which is not a desired behavior for waking up BSP. To
 * void the boot-strap code, wake up CPU0 by NMI instead.
 *
 * This works to wake up soft offlined CPU0 only. If CPU0 is hard offlined
 * (i.e. physically hot removed and then hot added), NMI won't wake it up.
 * We'll change this code in the future to wake up hard offlined CPU0 if
 * real platform and request are available.
 */
884
static int
885 886 887 888 889 890
wakeup_cpu_via_init_nmi(int cpu, unsigned long start_ip, int apicid,
	       int *cpu0_nmi_registered)
{
	int id;
	int boot_error;

891 892
	preempt_disable();

893 894 895
	/*
	 * Wake up AP by INIT, INIT, STARTUP sequence.
	 */
896 897 898 899
	if (cpu) {
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
		goto out;
	}
900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917

	/*
	 * Wake up BSP by nmi.
	 *
	 * Register a NMI handler to help wake up CPU0.
	 */
	boot_error = register_nmi_handler(NMI_LOCAL,
					  wakeup_cpu0_nmi, 0, "wake_cpu0");

	if (!boot_error) {
		enable_start_cpu0 = 1;
		*cpu0_nmi_registered = 1;
		if (apic->dest_logical == APIC_DEST_LOGICAL)
			id = cpu0_logical_apicid;
		else
			id = apicid;
		boot_error = wakeup_secondary_cpu_via_nmi(id, start_ip);
	}
918 919 920

out:
	preempt_enable();
921 922 923 924

	return boot_error;
}

925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
void common_cpu_up(unsigned int cpu, struct task_struct *idle)
{
	/* Just in case we booted with a single CPU. */
	alternatives_enable_smp();

	per_cpu(current_task, cpu) = idle;

#ifdef CONFIG_X86_32
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
	per_cpu(cpu_current_top_of_stack, cpu) =
		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
#else
	clear_tsk_thread_flag(idle, TIF_FORK);
	initial_gs = per_cpu_offset(cpu);
#endif
}

943 944 945
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
946 947
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
948
 */
949
static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
950
{
951
	volatile u32 *trampoline_status =
952
		(volatile u32 *) __va(real_mode_header->trampoline_status);
953
	/* start_ip had better be page-aligned! */
954
	unsigned long start_ip = real_mode_header->trampoline_start;
955

956
	unsigned long boot_error = 0;
957
	int cpu0_nmi_registered = 0;
958
	unsigned long timeout;
959

960 961
	idle->thread.sp = (unsigned long) (((struct pt_regs *)
			  (THREAD_SIZE +  task_stack_page(idle))) - 1);
962

963
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
964
	initial_code = (unsigned long)start_secondary;
965
	stack_start  = idle->thread.sp;
966

967 968 969 970 971 972 973
	/*
	 * Enable the espfix hack for this CPU
	*/
#ifdef CONFIG_X86_ESPFIX64
	init_espfix_ap(cpu);
#endif

974 975
	/* So we see what's up */
	announce_cpu(cpu, apicid);
976 977 978 979 980 981

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

J
Jack Steiner 已提交
982
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
983

984
		pr_debug("Setting warm reset code and vector.\n");
985

J
Jack Steiner 已提交
986 987 988
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
989 990 991 992 993
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
J
Jack Steiner 已提交
994
	}
995

996 997 998 999 1000 1001 1002 1003 1004
	/*
	 * AP might wait on cpu_callout_mask in cpu_init() with
	 * cpu_initialized_mask set if previous attempt to online
	 * it timed-out. Clear cpu_initialized_mask so that after
	 * INIT/SIPI it could start with a clean state.
	 */
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
	smp_mb();

1005
	/*
1006 1007 1008 1009
	 * Wake up a CPU in difference cases:
	 * - Use the method in the APIC driver if it's defined
	 * Otherwise,
	 * - Use an INIT boot APIC message for APs or NMI for BSP.
1010
	 */
1011 1012 1013
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
1014 1015
		boot_error = wakeup_cpu_via_init_nmi(cpu, start_ip, apicid,
						     &cpu0_nmi_registered);
1016 1017 1018

	if (!boot_error) {
		/*
1019
		 * Wait 10s total for first sign of life from AP
1020
		 */
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
		boot_error = -1;
		timeout = jiffies + 10*HZ;
		while (time_before(jiffies, timeout)) {
			if (cpumask_test_cpu(cpu, cpu_initialized_mask)) {
				/*
				 * Tell AP to proceed with initialization
				 */
				cpumask_set_cpu(cpu, cpu_callout_mask);
				boot_error = 0;
				break;
			}
			schedule();
		}
	}
1035

1036
	if (!boot_error) {
1037
		/*
1038
		 * Wait till AP completes initial initialization
1039
		 */
1040
		while (!cpumask_test_cpu(cpu, cpu_callin_mask)) {
1041 1042 1043 1044 1045 1046 1047
			/*
			 * Allow other tasks to run while we wait for the
			 * AP to come online. This also gives a chance
			 * for the MTRR work(triggered by the AP coming online)
			 * to be completed in the stop machine context.
			 */
			schedule();
1048 1049 1050 1051
		}
	}

	/* mark "stuck" area as not stuck */
1052
	*trampoline_status = 0;
1053

1054 1055 1056 1057 1058 1059
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
1060 1061 1062 1063 1064 1065 1066
	/*
	 * Clean up the nmi handler. Do this after the callin and callout sync
	 * to avoid impact of possible long unregister time.
	 */
	if (cpu0_nmi_registered)
		unregister_nmi_handler(NMI_LOCAL, "wake_cpu0");

1067 1068 1069
	return boot_error;
}

1070
int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
1071
{
1072
	int apicid = apic->cpu_present_to_apicid(cpu);
1073 1074 1075 1076 1077
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

1078
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
1079

1080
	if (apicid == BAD_APICID ||
1081
	    !physid_isset(apicid, phys_cpu_present_map) ||
1082
	    !apic->apic_id_valid(apicid)) {
1083
		pr_err("%s: bad cpu %d\n", __func__, cpu);
1084 1085 1086 1087 1088 1089
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
1090
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
1091
		pr_debug("do_boot_cpu %d Already started\n", cpu);
1092 1093 1094 1095 1096 1097 1098 1099 1100
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

1101 1102 1103 1104
	/* x86 CPUs take themselves offline, so delayed offline is OK. */
	err = cpu_check_up_prepare(cpu);
	if (err && err != -EBUSY)
		return err;
1105

1106 1107 1108
	/* the FPU context is blank, nobody can own it */
	__cpu_disable_lazy_restore(cpu);

1109 1110
	common_cpu_up(cpu, tidle);

1111 1112 1113 1114 1115 1116 1117
	/*
	 * We have to walk the irq descriptors to setup the vector
	 * space for the cpu which comes online.  Prevent irq
	 * alloc/free across the bringup.
	 */
	irq_lock_sparse();

1118
	err = do_boot_cpu(apicid, cpu, tidle);
1119

1120
	if (err) {
1121
		irq_unlock_sparse();
1122
		pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu);
1123
		return -EIO;
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

1134
	while (!cpu_online(cpu)) {
1135 1136 1137 1138
		cpu_relax();
		touch_nmi_watchdog();
	}

1139 1140
	irq_unlock_sparse();

1141 1142 1143
	return 0;
}

1144 1145 1146 1147 1148 1149 1150 1151
/**
 * arch_disable_smp_support() - disables SMP support for x86 at runtime
 */
void arch_disable_smp_support(void)
{
	disable_ioapic_support();
}

1152 1153 1154 1155 1156 1157 1158
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
1159 1160
	pr_info("SMP disabled\n");

1161 1162
	disable_ioapic_support();

1163 1164
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
1165

1166
	if (smp_found_config)
1167
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1168
	else
1169
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
1170 1171
	cpumask_set_cpu(0, topology_sibling_cpumask(0));
	cpumask_set_cpu(0, topology_core_cpumask(0));
1172 1173
}

1174 1175 1176 1177 1178 1179 1180
enum {
	SMP_OK,
	SMP_NO_CONFIG,
	SMP_NO_APIC,
	SMP_FORCE_UP,
};

1181 1182 1183 1184 1185
/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
J
Jack Steiner 已提交
1186
	preempt_disable();
1187

1188
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
1189 1190 1191 1192
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

1193 1194
		pr_warn("More than 8 CPUs detected - skipping them\n"
			"Use CONFIG_X86_BIGSMP\n");
1195 1196 1197 1198

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
1199
				set_cpu_present(cpu, false);
1200 1201 1202 1203 1204 1205
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
1206
				set_cpu_possible(cpu, false);
1207 1208 1209 1210 1211 1212 1213
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

1214
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
1215
		pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
M
Michael Tokarev 已提交
1216 1217
			hard_smp_processor_id());

1218 1219 1220 1221 1222 1223 1224 1225
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
J
Jack Steiner 已提交
1226
		preempt_enable();
1227
		pr_notice("SMP motherboard not detected\n");
1228
		return SMP_NO_CONFIG;
1229 1230 1231 1232 1233 1234
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1235
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1236 1237
		pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
			  boot_cpu_physical_apicid);
1238 1239
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
J
Jack Steiner 已提交
1240
	preempt_enable();
1241 1242 1243 1244 1245

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
1246
	    !boot_cpu_has(X86_FEATURE_APIC)) {
1247 1248 1249
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
1250
			pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
1251
		}
1252
		return SMP_NO_APIC;
1253 1254 1255 1256 1257 1258
	}

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1259
		pr_info("SMP mode deactivated\n");
1260
		return SMP_FORCE_UP;
1261 1262
	}

1263
	return SMP_OK;
1264 1265 1266 1267 1268 1269 1270
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1271
	for_each_possible_cpu(i) {
1272 1273
		c = &cpu_data(i);
		/* mark all to hotplug */
1274
		c->cpu_index = nr_cpu_ids;
1275 1276 1277 1278 1279 1280 1281 1282 1283
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1284 1285
	unsigned int i;

1286
	smp_cpu_index_default();
1287

1288 1289 1290
	/*
	 * Setup boot CPU information
	 */
1291
	smp_store_boot_cpu_info(); /* Final full version of the data */
1292 1293
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
	mb();
1294

1295
	for_each_possible_cpu(i) {
1296 1297
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1298
		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1299
	}
1300 1301
	set_cpu_sibling_map(0);

1302 1303
	switch (smp_sanity_check(max_cpus)) {
	case SMP_NO_CONFIG:
1304
		disable_smp();
1305 1306 1307 1308 1309 1310 1311 1312
		if (APIC_init_uniprocessor())
			pr_notice("Local APIC not detected. Using dummy APIC emulation.\n");
		return;
	case SMP_NO_APIC:
		disable_smp();
		return;
	case SMP_FORCE_UP:
		disable_smp();
1313
		apic_bsp_setup(false);
1314
		return;
1315 1316
	case SMP_OK:
		break;
1317 1318
	}

1319
	if (read_apic_id() != boot_cpu_physical_apicid) {
1320
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1321
		     read_apic_id(), boot_cpu_physical_apicid);
1322 1323 1324
		/* Or can we switch back to PIC here? */
	}

1325
	default_setup_apic_routing();
1326
	cpu0_logical_apicid = apic_bsp_setup(false);
1327

1328
	pr_info("CPU%d: ", 0);
1329
	print_cpu_info(&cpu_data(0));
1330 1331 1332

	if (is_uv_system())
		uv_system_init();
1333 1334

	set_mtrr_aps_delayed_init();
1335 1336

	smp_quirk_init_udelay();
1337
}
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348

void arch_enable_nonboot_cpus_begin(void)
{
	set_mtrr_aps_delayed_init();
}

void arch_enable_nonboot_cpus_end(void)
{
	mtrr_aps_init();
}

1349 1350 1351 1352 1353 1354
/*
 * Early setup to make printk work.
 */
void __init native_smp_prepare_boot_cpu(void)
{
	int me = smp_processor_id();
1355
	switch_to_new_gdt(me);
1356 1357
	/* already set me in cpu_online_mask in boot_cpu_init() */
	cpumask_set_cpu(me, cpu_callout_mask);
1358
	cpu_set_state_online(me);
1359 1360
}

1361 1362
void __init native_smp_cpus_done(unsigned int max_cpus)
{
1363
	pr_debug("Boot done\n");
1364

D
Don Zickus 已提交
1365
	nmi_selftest();
1366 1367
	impress_friends();
	setup_ioapic_dest();
1368
	mtrr_aps_init();
1369 1370
}

1371 1372 1373 1374 1375 1376 1377 1378 1379
static int __initdata setup_possible_cpus = -1;
static int __init _setup_possible_cpus(char *str)
{
	get_option(&str, &setup_possible_cpus);
	return 0;
}
early_param("possible_cpus", _setup_possible_cpus);


1380
/*
1381
 * cpu_possible_mask should be static, it cannot change as cpu's
1382 1383 1384
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
1385
 * cpu_present_mask on the other hand can change dynamically.
1386 1387 1388 1389 1390 1391
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
1392
 * - The user can overwrite it with possible_cpus=NUM
1393 1394 1395 1396 1397 1398
 * - Otherwise don't reserve additional CPUs.
 * We do this because additional CPUs waste a lot of memory.
 * -AK
 */
__init void prefill_possible_map(void)
{
T
Thomas Gleixner 已提交
1399
	int i, possible;
1400

1401 1402 1403 1404
	/* no processor from mptable or madt */
	if (!num_processors)
		num_processors = 1;

1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
	i = setup_max_cpus ?: 1;
	if (setup_possible_cpus == -1) {
		possible = num_processors;
#ifdef CONFIG_HOTPLUG_CPU
		if (setup_max_cpus)
			possible += disabled_cpus;
#else
		if (possible > i)
			possible = i;
#endif
	} else
1416 1417
		possible = setup_possible_cpus;

1418 1419
	total_cpus = max_t(int, possible, num_processors + disabled_cpus);

1420 1421
	/* nr_cpu_ids could be reduced via nr_cpus= */
	if (possible > nr_cpu_ids) {
1422
		pr_warn("%d Processors exceeds NR_CPUS limit of %d\n",
1423 1424
			possible, nr_cpu_ids);
		possible = nr_cpu_ids;
1425
	}
1426

1427 1428 1429 1430
#ifdef CONFIG_HOTPLUG_CPU
	if (!setup_max_cpus)
#endif
	if (possible > i) {
1431
		pr_warn("%d Processors exceeds max_cpus limit of %u\n",
1432 1433 1434 1435
			possible, setup_max_cpus);
		possible = i;
	}

1436
	pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
1437 1438 1439
		possible, max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
1440
		set_cpu_possible(i, true);
1441 1442
	for (; i < NR_CPUS; i++)
		set_cpu_possible(i, false);
1443 1444

	nr_cpu_ids = possible;
1445
}
1446

1447 1448
#ifdef CONFIG_HOTPLUG_CPU

1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463
/* Recompute SMT state for all CPUs on offline */
static void recompute_smt_state(void)
{
	int max_threads, cpu;

	max_threads = 0;
	for_each_online_cpu (cpu) {
		int threads = cpumask_weight(topology_sibling_cpumask(cpu));

		if (threads > max_threads)
			max_threads = threads;
	}
	__max_smt_threads = max_threads;
}

1464 1465 1466 1467 1468
static void remove_siblinginfo(int cpu)
{
	int sibling;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

1469 1470
	for_each_cpu(sibling, topology_core_cpumask(cpu)) {
		cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
1471 1472 1473
		/*/
		 * last thread sibling in this cpu core going down
		 */
1474
		if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1)
1475 1476 1477
			cpu_data(sibling).booted_cores--;
	}

1478 1479
	for_each_cpu(sibling, topology_sibling_cpumask(cpu))
		cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
1480 1481 1482
	for_each_cpu(sibling, cpu_llc_shared_mask(cpu))
		cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling));
	cpumask_clear(cpu_llc_shared_mask(cpu));
1483 1484
	cpumask_clear(topology_sibling_cpumask(cpu));
	cpumask_clear(topology_core_cpumask(cpu));
1485 1486
	c->phys_proc_id = 0;
	c->cpu_core_id = 0;
1487
	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1488
	recompute_smt_state();
1489 1490
}

1491
static void remove_cpu_from_maps(int cpu)
1492
{
1493 1494 1495
	set_cpu_online(cpu, false);
	cpumask_clear_cpu(cpu, cpu_callout_mask);
	cpumask_clear_cpu(cpu, cpu_callin_mask);
1496
	/* was set by cpu_init() */
1497
	cpumask_clear_cpu(cpu, cpu_initialized_mask);
1498
	numa_remove_cpu(cpu);
1499 1500
}

1501
void cpu_disable_common(void)
1502 1503 1504 1505 1506 1507
{
	int cpu = smp_processor_id();

	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
1508
	lock_vector_lock();
1509
	remove_cpu_from_maps(cpu);
1510
	unlock_vector_lock();
1511
	fixup_irqs();
1512 1513 1514 1515
}

int native_cpu_disable(void)
{
1516 1517 1518 1519 1520 1521
	int ret;

	ret = check_irq_vectors_for_cpu_disable();
	if (ret)
		return ret;

1522 1523
	clear_local_APIC();
	cpu_disable_common();
1524

1525 1526 1527
	return 0;
}

1528
int common_cpu_die(unsigned int cpu)
1529
{
1530
	int ret = 0;
1531

1532
	/* We don't do anything here: idle task is faking death itself. */
1533

1534
	/* They ack this in play_dead() by setting CPU_DEAD */
1535
	if (cpu_wait_death(cpu, 5)) {
1536 1537 1538 1539
		if (system_state == SYSTEM_RUNNING)
			pr_info("CPU %u is now offline\n", cpu);
	} else {
		pr_err("CPU %u didn't die...\n", cpu);
1540
		ret = -1;
1541
	}
1542 1543 1544 1545 1546 1547 1548

	return ret;
}

void native_cpu_die(unsigned int cpu)
{
	common_cpu_die(cpu);
1549
}
1550 1551 1552 1553 1554

void play_dead_common(void)
{
	idle_task_exit();
	reset_lazy_tlbstate();
1555
	amd_e400_remove_cpu(raw_smp_processor_id());
1556 1557

	/* Ack it */
1558
	(void)cpu_report_death();
1559 1560 1561 1562 1563 1564 1565

	/*
	 * With physical CPU hotplug, we should halt the cpu
	 */
	local_irq_disable();
}

1566 1567 1568 1569 1570 1571 1572 1573
static bool wakeup_cpu0(void)
{
	if (smp_processor_id() == 0 && enable_start_cpu0)
		return true;

	return false;
}

1574 1575 1576 1577 1578 1579 1580 1581 1582
/*
 * We need to flush the caches before going to sleep, lest we have
 * dirty data in our caches when we come back up.
 */
static inline void mwait_play_dead(void)
{
	unsigned int eax, ebx, ecx, edx;
	unsigned int highest_cstate = 0;
	unsigned int highest_subcstate = 0;
1583
	void *mwait_ptr;
1584
	int i;
1585

1586
	if (!this_cpu_has(X86_FEATURE_MWAIT))
1587
		return;
1588
	if (!this_cpu_has(X86_FEATURE_CLFLUSH))
1589
		return;
1590
	if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
		return;

	eax = CPUID_MWAIT_LEAF;
	ecx = 0;
	native_cpuid(&eax, &ebx, &ecx, &edx);

	/*
	 * eax will be 0 if EDX enumeration is not valid.
	 * Initialized below to cstate, sub_cstate value when EDX is valid.
	 */
	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
		eax = 0;
	} else {
		edx >>= MWAIT_SUBSTATE_SIZE;
		for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
			if (edx & MWAIT_SUBSTATE_MASK) {
				highest_cstate = i;
				highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
			}
		}
		eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
			(highest_subcstate - 1);
	}

1615 1616 1617 1618 1619 1620 1621
	/*
	 * This should be a memory location in a cache line which is
	 * unlikely to be touched by other processors.  The actual
	 * content is immaterial as it is not actually modified in any way.
	 */
	mwait_ptr = &current_thread_info()->flags;

1622 1623
	wbinvd();

1624
	while (1) {
1625 1626 1627 1628 1629 1630 1631
		/*
		 * The CLFLUSH is a workaround for erratum AAI65 for
		 * the Xeon 7400 series.  It's not clear it is actually
		 * needed, but it should be harmless in either case.
		 * The WBINVD is insufficient due to the spurious-wakeup
		 * case where we return around the loop.
		 */
1632
		mb();
1633
		clflush(mwait_ptr);
1634
		mb();
1635
		__monitor(mwait_ptr, 0, 0);
1636 1637
		mb();
		__mwait(eax, 0);
1638 1639 1640 1641 1642
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1643 1644 1645
	}
}

1646
void hlt_play_dead(void)
1647
{
1648
	if (__this_cpu_read(cpu_info.x86) >= 4)
1649 1650
		wbinvd();

1651 1652
	while (1) {
		native_halt();
1653 1654 1655 1656 1657
		/*
		 * If NMI wants to wake up CPU0, start CPU0.
		 */
		if (wakeup_cpu0())
			start_cpu0();
1658 1659 1660
	}
}

1661 1662 1663
void native_play_dead(void)
{
	play_dead_common();
1664
	tboot_shutdown(TB_SHUTDOWN_WFS);
1665 1666

	mwait_play_dead();	/* Only returns on failure */
1667 1668
	if (cpuidle_play_dead())
		hlt_play_dead();
1669 1670
}

1671
#else /* ... !CONFIG_HOTPLUG_CPU */
1672
int native_cpu_disable(void)
1673 1674 1675 1676
{
	return -ENOSYS;
}

1677
void native_cpu_die(unsigned int cpu)
1678 1679 1680 1681
{
	/* We said "no" in __cpu_disable */
	BUG();
}
1682 1683 1684 1685 1686 1687

void native_play_dead(void)
{
	BUG();
}

1688
#endif