smpboot.c 30.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 *	x86 SMP booting functions
 *
 *	(c) 1995 Alan Cox, Building #3 <alan@redhat.com>
 *	(c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
15
 *	This code is released under the GNU General Public License version 2
L
Linus Torvalds 已提交
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIP report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
33 34 35 36
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *					Various cleanups.
 *					Probably mostly hotplug CPU ready now.
A
Ashok Raj 已提交
37
 *	Ashok Raj			: CPU hotplug support
L
Linus Torvalds 已提交
38 39
 */

40

L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
#include <linux/config.h>
#include <linux/init.h>

#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/smp_lock.h>
#include <linux/bootmem.h>
#include <linux/thread_info.h>
#include <linux/module.h>

#include <linux/delay.h>
#include <linux/mc146818rtc.h>
#include <asm/mtrr.h>
#include <asm/pgalloc.h>
#include <asm/desc.h>
#include <asm/kdebug.h>
#include <asm/tlbflush.h>
#include <asm/proto.h>
59
#include <asm/nmi.h>
60 61
#include <asm/irq.h>
#include <asm/hw_irq.h>
L
Linus Torvalds 已提交
62 63 64 65

/* Number of siblings per CPU package */
int smp_num_siblings = 1;
/* Package ID of each logical CPU */
66
u8 phys_proc_id[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
67
/* core ID of each logical CPU */
68
u8 cpu_core_id[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
L
Linus Torvalds 已提交
69 70

/* Bitmask of currently online CPUs */
71
cpumask_t cpu_online_map __read_mostly;
L
Linus Torvalds 已提交
72

73 74 75 76 77 78
EXPORT_SYMBOL(cpu_online_map);

/*
 * Private maps to synchronize booting between AP and BP.
 * Probably not needed anymore, but it makes for easier debugging. -AK
 */
L
Linus Torvalds 已提交
79 80
cpumask_t cpu_callin_map;
cpumask_t cpu_callout_map;
81 82 83

cpumask_t cpu_possible_map;
EXPORT_SYMBOL(cpu_possible_map);
L
Linus Torvalds 已提交
84 85 86 87

/* Per CPU bogomips and other parameters */
struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;

88 89 90
/* Set when the idlers are all forked */
int smp_threads_ready;

91
/* representing HT siblings of each logical CPU */
92
cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
93 94

/* representing HT and core siblings of each logical CPU */
95
cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
96
EXPORT_SYMBOL(cpu_core_map);
L
Linus Torvalds 已提交
97 98 99 100 101

/*
 * Trampoline 80x86 program as an array.
 */

102 103
extern unsigned char trampoline_data[];
extern unsigned char trampoline_end[];
L
Linus Torvalds 已提交
104

A
Ashok Raj 已提交
105 106 107 108 109 110 111 112 113 114 115 116 117
/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

/*
 * Store all idle threads, this can be reused instead of creating
 * a new thread. Also avoids complicated thread destroy functionality
 * for idle threads.
 */
struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;

#define get_idle_for_cpu(x)     (idle_thread_array[(x)])
#define set_idle_for_cpu(x,p)   (idle_thread_array[(x)] = (p))

L
Linus Torvalds 已提交
118 119 120 121 122 123
/*
 * Currently trivial. Write the real->protected mode
 * bootstrap into the page concerned. The caller
 * has made sure it's suitably aligned.
 */

124
static unsigned long __cpuinit setup_trampoline(void)
L
Linus Torvalds 已提交
125 126 127 128 129 130 131 132 133 134 135
{
	void *tramp = __va(SMP_TRAMPOLINE_BASE); 
	memcpy(tramp, trampoline_data, trampoline_end - trampoline_data);
	return virt_to_phys(tramp);
}

/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */

136
static void __cpuinit smp_store_cpu_info(int id)
L
Linus Torvalds 已提交
137 138 139 140 141
{
	struct cpuinfo_x86 *c = cpu_data + id;

	*c = boot_cpu_data;
	identify_cpu(c);
142
	print_cpu_info(c);
L
Linus Torvalds 已提交
143 144 145
}

/*
146 147 148 149 150
 * New Funky TSC sync algorithm borrowed from IA64.
 * Main advantage is that it doesn't reset the TSCs fully and
 * in general looks more robust and it works better than my earlier
 * attempts. I believe it was written by David Mosberger. Some minor
 * adjustments for x86-64 by me -AK
L
Linus Torvalds 已提交
151
 *
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
 * Original comment reproduced below.
 *
 * Synchronize TSC of the current (slave) CPU with the TSC of the
 * MASTER CPU (normally the time-keeper CPU).  We use a closed loop to
 * eliminate the possibility of unaccounted-for errors (such as
 * getting a machine check in the middle of a calibration step).  The
 * basic idea is for the slave to ask the master what itc value it has
 * and to read its own itc before and after the master responds.  Each
 * iteration gives us three timestamps:
 *
 *	slave		master
 *
 *	t0 ---\
 *             ---\
 *		   --->
 *			tm
 *		   /---
 *	       /---
 *	t1 <---
 *
 *
 * The goal is to adjust the slave's TSC such that tm falls exactly
 * half-way between t0 and t1.  If we achieve this, the clocks are
 * synchronized provided the interconnect between the slave and the
 * master is symmetric.  Even if the interconnect were asymmetric, we
 * would still know that the synchronization error is smaller than the
 * roundtrip latency (t0 - t1).
 *
 * When the interconnect is quiet and symmetric, this lets us
 * synchronize the TSC to within one or two cycles.  However, we can
 * only *guarantee* that the synchronization is accurate to within a
 * round-trip time, which is typically in the range of several hundred
 * cycles (e.g., ~500 cycles).  In practice, this means that the TSCs
 * are usually almost perfectly synchronized, but we shouldn't assume
 * that the accuracy is much better than half a micro second or so.
 *
 * [there are other errors like the latency of RDTSC and of the
 * WRMSR. These can also account to hundreds of cycles. So it's
 * probably worse. It claims 153 cycles error on a dual Opteron,
 * but I suspect the numbers are actually somewhat worse -AK]
L
Linus Torvalds 已提交
192 193
 */

194 195 196 197 198 199 200 201 202
#define MASTER	0
#define SLAVE	(SMP_CACHE_BYTES/8)

/* Intentionally don't use cpu_relax() while TSC synchronization
   because we don't want to go into funky power save modi or cause
   hypervisors to schedule us away.  Going to sleep would likely affect
   latency and low latency is the primary objective here. -AK */
#define no_cpu_relax() barrier()

203
static __cpuinitdata DEFINE_SPINLOCK(tsc_sync_lock);
204 205 206 207
static volatile __cpuinitdata unsigned long go[SLAVE + 1];
static int notscsync __cpuinitdata;

#undef DEBUG_TSC_SYNC
L
Linus Torvalds 已提交
208

209 210
#define NUM_ROUNDS	64	/* magic value */
#define NUM_ITERS	5	/* likewise */
L
Linus Torvalds 已提交
211

212 213
/* Callback on boot CPU */
static __cpuinit void sync_master(void *arg)
L
Linus Torvalds 已提交
214
{
215 216 217 218 219 220 221 222 223 224 225 226 227 228
	unsigned long flags, i;

	go[MASTER] = 0;

	local_irq_save(flags);
	{
		for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
			while (!go[MASTER])
				no_cpu_relax();
			go[MASTER] = 0;
			rdtscll(go[SLAVE]);
		}
	}
	local_irq_restore(flags);
229
}
L
Linus Torvalds 已提交
230

231
/*
232 233 234
 * Return the number of cycles by which our tsc differs from the tsc
 * on the master (time-keeper) CPU.  A positive number indicates our
 * tsc is ahead of the master, negative that it is behind.
235
 */
236 237
static inline long
get_delta(long *rt, long *master)
238
{
239 240 241
	unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
	unsigned long tcenter, t0, t1, tm;
	int i;
242

243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
	for (i = 0; i < NUM_ITERS; ++i) {
		rdtscll(t0);
		go[MASTER] = 1;
		while (!(tm = go[SLAVE]))
			no_cpu_relax();
		go[SLAVE] = 0;
		rdtscll(t1);

		if (t1 - t0 < best_t1 - best_t0)
			best_t0 = t0, best_t1 = t1, best_tm = tm;
	}

	*rt = best_t1 - best_t0;
	*master = best_tm - best_t0;

	/* average best_t0 and best_t1 without overflow: */
	tcenter = (best_t0/2 + best_t1/2);
	if (best_t0 % 2 + best_t1 % 2 == 2)
		++tcenter;
	return tcenter - best_tm;
L
Linus Torvalds 已提交
263 264
}

E
Eric W. Biederman 已提交
265
static __cpuinit void sync_tsc(unsigned int master)
L
Linus Torvalds 已提交
266
{
267 268 269
	int i, done = 0;
	long delta, adj, adjust_latency = 0;
	unsigned long flags, rt, master_time_stamp, bound;
270
#ifdef DEBUG_TSC_SYNC
271 272 273 274 275 276 277 278
	static struct syncdebug {
		long rt;	/* roundtrip time */
		long master;	/* master's timestamp */
		long diff;	/* difference between midpoint and master's timestamp */
		long lat;	/* estimate of tsc adjustment latency */
	} t[NUM_ROUNDS] __cpuinitdata;
#endif

E
Eric W. Biederman 已提交
279 280 281
	printk(KERN_INFO "CPU %d: Syncing TSC to CPU %u.\n",
		smp_processor_id(), master);

282 283
	go[MASTER] = 1;

E
Eric W. Biederman 已提交
284 285 286 287 288 289
	/* It is dangerous to broadcast IPI as cpus are coming up,
	 * as they may not be ready to accept them.  So since
	 * we only need to send the ipi to the boot cpu direct
	 * the message, and avoid the race.
	 */
	smp_call_function_single(master, sync_master, NULL, 1, 0);
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313

	while (go[MASTER])	/* wait for master to be ready */
		no_cpu_relax();

	spin_lock_irqsave(&tsc_sync_lock, flags);
	{
		for (i = 0; i < NUM_ROUNDS; ++i) {
			delta = get_delta(&rt, &master_time_stamp);
			if (delta == 0) {
				done = 1;	/* let's lock on to this... */
				bound = rt;
			}

			if (!done) {
				unsigned long t;
				if (i > 0) {
					adjust_latency += -delta;
					adj = -delta + adjust_latency/4;
				} else
					adj = -delta;

				rdtscll(t);
				wrmsrl(MSR_IA32_TSC, t + adj);
			}
314
#ifdef DEBUG_TSC_SYNC
315 316 317 318 319 320 321 322 323
			t[i].rt = rt;
			t[i].master = master_time_stamp;
			t[i].diff = delta;
			t[i].lat = adjust_latency/4;
#endif
		}
	}
	spin_unlock_irqrestore(&tsc_sync_lock, flags);

324
#ifdef DEBUG_TSC_SYNC
325 326 327 328 329 330 331 332
	for (i = 0; i < NUM_ROUNDS; ++i)
		printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
		       t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif

	printk(KERN_INFO
	       "CPU %d: synchronized TSC with CPU %u (last diff %ld cycles, "
	       "maxerr %lu cycles)\n",
E
Eric W. Biederman 已提交
333
	       smp_processor_id(), master, delta, rt);
334
}
L
Linus Torvalds 已提交
335

336
static void __cpuinit tsc_sync_wait(void)
337
{
338
	if (notscsync || !cpu_has_tsc)
339
		return;
340
	sync_tsc(0);
341
}
L
Linus Torvalds 已提交
342

343
static __init int notscsync_setup(char *s)
344
{
345 346
	notscsync = 1;
	return 0;
L
Linus Torvalds 已提交
347
}
348
__setup("notscsync", notscsync_setup);
L
Linus Torvalds 已提交
349

350
static atomic_t init_deasserted __cpuinitdata;
L
Linus Torvalds 已提交
351

352 353 354 355 356
/*
 * Report back to the Boot Processor.
 * Running on AP.
 */
void __cpuinit smp_callin(void)
L
Linus Torvalds 已提交
357 358 359 360 361 362 363 364 365 366
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
	 */
367 368
	while (!atomic_read(&init_deasserted))
		cpu_relax();
L
Linus Torvalds 已提交
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398

	/*
	 * (This works even if the APIC is not enabled.)
	 */
	phys_id = GET_APIC_ID(apic_read(APIC_ID));
	cpuid = smp_processor_id();
	if (cpu_isset(cpuid, cpu_callin_map)) {
		panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
					phys_id, cpuid);
	}
	Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
		if (cpu_isset(cpuid, cpu_callout_map))
			break;
399
		cpu_relax();
L
Linus Torvalds 已提交
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
	}

	if (!time_before(jiffies, timeout)) {
		panic("smp_callin: CPU%d started up but did not get a callout!\n",
			cpuid);
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

	Dprintk("CALLIN, before setup_local_APIC().\n");
	setup_local_APIC();

	/*
	 * Get our bogomips.
419 420 421
 	 *
 	 * Need to enable IRQs because it can take longer and then
	 * the NMI watchdog might kill us.
L
Linus Torvalds 已提交
422
	 */
423
	local_irq_enable();
L
Linus Torvalds 已提交
424
	calibrate_delay();
425
	local_irq_disable();
L
Linus Torvalds 已提交
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
	Dprintk("Stack at about %p\n",&cpuid);

	disable_APIC_timer();

	/*
	 * Save our processor parameters
	 */
 	smp_store_cpu_info(cpuid);

	/*
	 * Allow the master to continue.
	 */
	cpu_set(cpuid, cpu_callin_map);
}

441 442 443
/* representing cpus for which sibling maps can be computed */
static cpumask_t cpu_sibling_setup_map;

444 445 446
static inline void set_cpu_sibling_map(int cpu)
{
	int i;
447 448 449
	struct cpuinfo_x86 *c = cpu_data;

	cpu_set(cpu, cpu_sibling_setup_map);
450 451

	if (smp_num_siblings > 1) {
452 453 454
		for_each_cpu_mask(i, cpu_sibling_setup_map) {
			if (phys_proc_id[cpu] == phys_proc_id[i] &&
			    cpu_core_id[cpu] == cpu_core_id[i]) {
455 456
				cpu_set(i, cpu_sibling_map[cpu]);
				cpu_set(cpu, cpu_sibling_map[i]);
457 458
				cpu_set(i, cpu_core_map[cpu]);
				cpu_set(cpu, cpu_core_map[i]);
459 460 461 462 463 464
			}
		}
	} else {
		cpu_set(cpu, cpu_sibling_map[cpu]);
	}

465
	if (current_cpu_data.x86_max_cores == 1) {
466
		cpu_core_map[cpu] = cpu_sibling_map[cpu];
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
		c[cpu].booted_cores = 1;
		return;
	}

	for_each_cpu_mask(i, cpu_sibling_setup_map) {
		if (phys_proc_id[cpu] == phys_proc_id[i]) {
			cpu_set(i, cpu_core_map[cpu]);
			cpu_set(cpu, cpu_core_map[i]);
			/*
			 *  Does this new cpu bringup a new core?
			 */
			if (cpus_weight(cpu_sibling_map[cpu]) == 1) {
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
				if (first_cpu(cpu_sibling_map[i]) == i)
					c[cpu].booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					c[i].booted_cores++;
			} else if (i != cpu && !c[cpu].booted_cores)
				c[cpu].booted_cores = c[i].booted_cores;
		}
494 495 496
	}
}

L
Linus Torvalds 已提交
497
/*
498
 * Setup code on secondary processor (after comming out of the trampoline)
L
Linus Torvalds 已提交
499
 */
500
void __cpuinit start_secondary(void)
L
Linus Torvalds 已提交
501 502 503 504 505 506 507
{
	/*
	 * Dont put anything before smp_callin(), SMP
	 * booting is too fragile that we want to limit the
	 * things done here to the most necessary things.
	 */
	cpu_init();
508
	preempt_disable();
L
Linus Torvalds 已提交
509 510 511 512 513 514 515 516
	smp_callin();

	/* otherwise gcc will move up the smp_processor_id before the cpu_init */
	barrier();

	Dprintk("cpu %d: setting up apic clock\n", smp_processor_id()); 	
	setup_secondary_APIC_clock();

517
	Dprintk("cpu %d: enabling apic timer\n", smp_processor_id());
L
Linus Torvalds 已提交
518 519 520 521 522 523 524

	if (nmi_watchdog == NMI_IO_APIC) {
		disable_8259A_irq(0);
		enable_NMI_through_LVT0(NULL);
		enable_8259A_irq(0);
	}

525
	enable_APIC_timer();
L
Linus Torvalds 已提交
526

527 528 529 530 531 532
	/*
	 * The sibling maps must be set before turing the online map on for
	 * this cpu
	 */
	set_cpu_sibling_map(smp_processor_id());

533 534 535 536 537 538 539 540
	/* 
  	 * Wait for TSC sync to not schedule things before.
	 * We still process interrupts, which could see an inconsistent
	 * time in that window unfortunately. 
	 * Do this here because TSC sync has global unprotected state.
 	 */
	tsc_sync_wait();

541 542 543 544 545 546 547 548 549 550
	/*
	 * We need to hold call_lock, so there is no inconsistency
	 * between the time smp_call_function() determines number of
	 * IPI receipients, and the time when the determination is made
	 * for which cpus receive the IPI in genapic_flat.c. Holding this
	 * lock helps us to not include this cpu in a currently in progress
	 * smp_call_function().
	 */
	lock_ipi_call_lock();

L
Linus Torvalds 已提交
551
	/*
552
	 * Allow the master to continue.
L
Linus Torvalds 已提交
553 554
	 */
	cpu_set(smp_processor_id(), cpu_online_map);
555 556 557
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
	unlock_ipi_call_lock();

L
Linus Torvalds 已提交
558 559 560
	cpu_idle();
}

561
extern volatile unsigned long init_rsp;
L
Linus Torvalds 已提交
562 563
extern void (*initial_code)(void);

564
#ifdef APIC_DEBUG
565
static void inquire_remote_apic(int apicid)
L
Linus Torvalds 已提交
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
	char *names[] = { "ID", "VERSION", "SPIV" };
	int timeout, status;

	printk(KERN_INFO "Inquiring remote APIC #%d...\n", apicid);

	for (i = 0; i < sizeof(regs) / sizeof(*regs); i++) {
		printk("... APIC #%d %s: ", apicid, names[i]);

		/*
		 * Wait for idle.
		 */
		apic_wait_icr_idle();

581 582
		apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
		apic_write(APIC_ICR, APIC_DM_REMRD | regs[i]);
L
Linus Torvalds 已提交
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
			printk("%08x\n", status);
			break;
		default:
			printk("failed\n");
		}
	}
}
#endif

602 603 604 605
/*
 * Kick the secondary to wake up.
 */
static int __cpuinit wakeup_secondary_via_INIT(int phys_apicid, unsigned int start_rip)
L
Linus Torvalds 已提交
606 607 608 609 610 611 612 613 614
{
	unsigned long send_status = 0, accept_status = 0;
	int maxlvt, timeout, num_starts, j;

	Dprintk("Asserting INIT.\n");

	/*
	 * Turn INIT on target chip
	 */
615
	apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
L
Linus Torvalds 已提交
616 617 618 619

	/*
	 * Send IPI
	 */
620
	apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
L
Linus Torvalds 已提交
621 622 623 624 625 626 627 628 629 630 631 632 633 634 635
				| APIC_DM_INIT);

	Dprintk("Waiting for send to finish...\n");
	timeout = 0;
	do {
		Dprintk("+");
		udelay(100);
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
	} while (send_status && (timeout++ < 1000));

	mdelay(10);

	Dprintk("Deasserting INIT.\n");

	/* Target chip */
636
	apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
L
Linus Torvalds 已提交
637 638

	/* Send IPI */
639
	apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
L
Linus Torvalds 已提交
640 641 642 643 644 645 646 647 648 649 650

	Dprintk("Waiting for send to finish...\n");
	timeout = 0;
	do {
		Dprintk("+");
		udelay(100);
		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
	} while (send_status && (timeout++ < 1000));

	atomic_set(&init_deasserted, 1);

651
	num_starts = 2;
L
Linus Torvalds 已提交
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671

	/*
	 * Run STARTUP IPI loop.
	 */
	Dprintk("#startup loops: %d.\n", num_starts);

	maxlvt = get_maxlvt();

	for (j = 1; j <= num_starts; j++) {
		Dprintk("Sending STARTUP #%d.\n",j);
		apic_read_around(APIC_SPIV);
		apic_write(APIC_ESR, 0);
		apic_read(APIC_ESR);
		Dprintk("After apic_write.\n");

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
672
		apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
L
Linus Torvalds 已提交
673 674 675

		/* Boot on the stack */
		/* Kick the second */
676
		apic_write(APIC_ICR, APIC_DM_STARTUP | (start_rip >> 12));
L
Linus Torvalds 已提交
677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

		Dprintk("Startup point 1.\n");

		Dprintk("Waiting for send to finish...\n");
		timeout = 0;
		do {
			Dprintk("+");
			udelay(100);
			send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
		} while (send_status && (timeout++ < 1000));

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
		/*
		 * Due to the Pentium erratum 3AP.
		 */
		if (maxlvt > 3) {
			apic_read_around(APIC_SPIV);
			apic_write(APIC_ESR, 0);
		}
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
	Dprintk("After Startup.\n");

	if (send_status)
		printk(KERN_ERR "APIC never delivered???\n");
	if (accept_status)
		printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);

	return (send_status | accept_status);
}

A
Ashok Raj 已提交
718 719 720 721 722 723 724 725 726 727 728 729 730 731
struct create_idle {
	struct task_struct *idle;
	struct completion done;
	int cpu;
};

void do_fork_idle(void *_c_idle)
{
	struct create_idle *c_idle = _c_idle;

	c_idle->idle = fork_idle(c_idle->cpu);
	complete(&c_idle->done);
}

732 733 734 735
/*
 * Boot one CPU.
 */
static int __cpuinit do_boot_cpu(int cpu, int apicid)
L
Linus Torvalds 已提交
736 737
{
	unsigned long boot_error;
738
	int timeout;
L
Linus Torvalds 已提交
739
	unsigned long start_rip;
A
Ashok Raj 已提交
740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
	struct create_idle c_idle = {
		.cpu = cpu,
		.done = COMPLETION_INITIALIZER(c_idle.done),
	};
	DECLARE_WORK(work, do_fork_idle, &c_idle);

	c_idle.idle = get_idle_for_cpu(cpu);

	if (c_idle.idle) {
		c_idle.idle->thread.rsp = (unsigned long) (((struct pt_regs *)
			(THREAD_SIZE + (unsigned long) c_idle.idle->thread_info)) - 1);
		init_idle(c_idle.idle, cpu);
		goto do_rest;
	}

L
Linus Torvalds 已提交
755
	/*
A
Ashok Raj 已提交
756 757 758 759 760 761 762 763
	 * During cold boot process, keventd thread is not spun up yet.
	 * When we do cpu hot-add, we create idle threads on the fly, we should
	 * not acquire any attributes from the calling context. Hence the clean
	 * way to create kernel_threads() is to do that from keventd().
	 * We do the current_is_keventd() due to the fact that ACPI notifier
	 * was also queuing to keventd() and when the caller is already running
	 * in context of keventd(), we would end up with locking up the keventd
	 * thread.
L
Linus Torvalds 已提交
764
	 */
A
Ashok Raj 已提交
765 766 767 768 769 770 771 772
	if (!keventd_up() || current_is_keventd())
		work.func(work.data);
	else {
		schedule_work(&work);
		wait_for_completion(&c_idle.done);
	}

	if (IS_ERR(c_idle.idle)) {
773
		printk("failed fork for CPU %d\n", cpu);
A
Ashok Raj 已提交
774
		return PTR_ERR(c_idle.idle);
775
	}
L
Linus Torvalds 已提交
776

A
Ashok Raj 已提交
777 778 779 780 781
	set_idle_for_cpu(cpu, c_idle.idle);

do_rest:

	cpu_pda[cpu].pcurrent = c_idle.idle;
L
Linus Torvalds 已提交
782 783 784

	start_rip = setup_trampoline();

A
Ashok Raj 已提交
785
	init_rsp = c_idle.idle->thread.rsp;
L
Linus Torvalds 已提交
786 787
	per_cpu(init_tss,cpu).rsp0 = init_rsp;
	initial_code = start_secondary;
A
Ashok Raj 已提交
788
	clear_ti_thread_flag(c_idle.idle->thread_info, TIF_FORK);
L
Linus Torvalds 已提交
789

790 791 792
	printk(KERN_INFO "Booting processor %d/%d APIC 0x%x\n", cpu,
		cpus_weight(cpu_present_map),
		apicid);
L
Linus Torvalds 已提交
793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

	Dprintk("Setting warm reset code and vector.\n");

	CMOS_WRITE(0xa, 0xf);
	local_flush_tlb();
	Dprintk("1.\n");
	*((volatile unsigned short *) phys_to_virt(0x469)) = start_rip >> 4;
	Dprintk("2.\n");
	*((volatile unsigned short *) phys_to_virt(0x467)) = start_rip & 0xf;
	Dprintk("3.\n");

	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[apicid])) {
		apic_read_around(APIC_SPIV);
		apic_write(APIC_ESR, 0);
		apic_read(APIC_ESR);
	}

	/*
	 * Status is now clean
	 */
	boot_error = 0;

	/*
	 * Starting actual IPI sequence...
	 */
828
	boot_error = wakeup_secondary_via_INIT(apicid, start_rip);
L
Linus Torvalds 已提交
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858

	if (!boot_error) {
		/*
		 * allow APs to start initializing.
		 */
		Dprintk("Before Callout %d.\n", cpu);
		cpu_set(cpu, cpu_callout_map);
		Dprintk("After Callout %d.\n", cpu);

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
			if (cpu_isset(cpu, cpu_callin_map))
				break;	/* It has booted */
			udelay(100);
		}

		if (cpu_isset(cpu, cpu_callin_map)) {
			/* number CPUs logically, starting from 1 (BSP is 0) */
			Dprintk("CPU has booted.\n");
		} else {
			boot_error = 1;
			if (*((volatile unsigned char *)phys_to_virt(SMP_TRAMPOLINE_BASE))
					== 0xA5)
				/* trampoline started but...? */
				printk("Stuck ??\n");
			else
				/* trampoline code not run */
				printk("Not responding.\n");
859
#ifdef APIC_DEBUG
L
Linus Torvalds 已提交
860 861 862 863 864 865 866
			inquire_remote_apic(apicid);
#endif
		}
	}
	if (boot_error) {
		cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
		clear_bit(cpu, &cpu_initialized); /* was set by cpu_init() */
867 868
		cpu_clear(cpu, cpu_present_map);
		cpu_clear(cpu, cpu_possible_map);
L
Linus Torvalds 已提交
869 870
		x86_cpu_to_apicid[cpu] = BAD_APICID;
		x86_cpu_to_log_apicid[cpu] = BAD_APICID;
871
		return -EIO;
L
Linus Torvalds 已提交
872
	}
873 874

	return 0;
L
Linus Torvalds 已提交
875 876
}

877 878 879
cycles_t cacheflush_time;
unsigned long cache_decay_ticks;

L
Linus Torvalds 已提交
880
/*
881
 * Cleanup possible dangling ends...
L
Linus Torvalds 已提交
882
 */
883
static __cpuinit void smp_cleanup_boot(void)
L
Linus Torvalds 已提交
884
{
885 886 887 888 889
	/*
	 * Paranoid:  Set warm reset code and vector here back
	 * to default values.
	 */
	CMOS_WRITE(0, 0xf);
L
Linus Torvalds 已提交
890

891 892 893 894 895 896 897 898 899 900 901
	/*
	 * Reset trampoline flag
	 */
	*((volatile int *) phys_to_virt(0x467)) = 0;
}

/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
902
static __init void disable_smp(void)
903 904 905 906 907 908 909 910 911 912 913
{
	cpu_present_map = cpumask_of_cpu(0);
	cpu_possible_map = cpumask_of_cpu(0);
	if (smp_found_config)
		phys_cpu_present_map = physid_mask_of_physid(boot_cpu_id);
	else
		phys_cpu_present_map = physid_mask_of_physid(0);
	cpu_set(0, cpu_sibling_map[0]);
	cpu_set(0, cpu_core_map[0]);
}

914
#ifdef CONFIG_HOTPLUG_CPU
915 916 917

int additional_cpus __initdata = -1;

918 919 920 921 922 923 924 925 926
/*
 * cpu_possible_map should be static, it cannot change as cpu's
 * are onlined, or offlined. The reason is per-cpu data-structures
 * are allocated by some modules at init time, and dont expect to
 * do this dynamically on cpu arrival/departure.
 * cpu_present_map on the other hand can change dynamically.
 * In case when cpu_hotplug is not compiled, then we resort to current
 * behaviour, which is cpu_possible == cpu_present.
 * - Ashok Raj
927 928 929 930 931 932 933
 *
 * Three ways to find out the number of additional hotplug CPUs:
 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
 * - otherwise use half of the available CPUs or 2, whatever is more.
 * - The user can overwrite it with additional_cpus=NUM
 * We do this because additional CPUs waste a lot of memory.
 * -AK
934
 */
935
__init void prefill_possible_map(void)
936 937
{
	int i;
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
	int possible;

 	if (additional_cpus == -1) {
 		if (disabled_cpus > 0) {
 			additional_cpus = disabled_cpus;
 		} else {
 			additional_cpus = num_processors / 2;
 			if (additional_cpus == 0)
 				additional_cpus = 2;
 		}
 	}
	possible = num_processors + additional_cpus;
	if (possible > NR_CPUS) 
		possible = NR_CPUS;

	printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
		possible,
	        max_t(int, possible - num_processors, 0));

	for (i = 0; i < possible; i++)
958 959 960 961
		cpu_set(i, cpu_possible_map);
}
#endif

962 963 964
/*
 * Various sanity checks.
 */
965
static int __init smp_sanity_check(unsigned max_cpus)
966
{
L
Linus Torvalds 已提交
967 968 969 970 971 972 973 974 975 976 977 978
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
		printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
		       hard_smp_processor_id());
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config) {
		printk(KERN_NOTICE "SMP motherboard not detected.\n");
979
		disable_smp();
L
Linus Torvalds 已提交
980 981 982
		if (APIC_init_uniprocessor())
			printk(KERN_NOTICE "Local APIC not detected."
					   " Using dummy APIC emulation.\n");
983
		return -1;
L
Linus Torvalds 已提交
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
	if (!physid_isset(boot_cpu_id, phys_cpu_present_map)) {
		printk(KERN_NOTICE "weird, boot CPU (#%d) not listed by the BIOS.\n",
								 boot_cpu_id);
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_id]) && !cpu_has_apic) {
		printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
			boot_cpu_id);
		printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
1003 1004
		nr_ioapics = 0;
		return -1;
L
Linus Torvalds 已提交
1005 1006 1007 1008 1009 1010 1011
	}

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
		printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
1012 1013
		nr_ioapics = 0;
		return -1;
L
Linus Torvalds 已提交
1014 1015
	}

1016 1017
	return 0;
}
L
Linus Torvalds 已提交
1018

1019 1020 1021 1022
/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
1023
void __init smp_prepare_cpus(unsigned int max_cpus)
1024 1025 1026 1027
{
	nmi_watchdog_default();
	current_cpu_data = boot_cpu_data;
	current_thread_info()->cpu = 0;  /* needed? */
1028
	set_cpu_sibling_map(0);
L
Linus Torvalds 已提交
1029

1030 1031 1032 1033
	if (smp_sanity_check(max_cpus) < 0) {
		printk(KERN_INFO "SMP disabled\n");
		disable_smp();
		return;
L
Linus Torvalds 已提交
1034 1035
	}

1036

L
Linus Torvalds 已提交
1037
	/*
1038
	 * Switch from PIC to APIC mode.
L
Linus Torvalds 已提交
1039
	 */
1040 1041
	connect_bsp_APIC();
	setup_local_APIC();
L
Linus Torvalds 已提交
1042

1043 1044 1045 1046
	if (GET_APIC_ID(apic_read(APIC_ID)) != boot_cpu_id) {
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
		      GET_APIC_ID(apic_read(APIC_ID)), boot_cpu_id);
		/* Or can we switch back to PIC here? */
L
Linus Torvalds 已提交
1047 1048 1049
	}

	/*
1050
	 * Now start the IO-APICs
L
Linus Torvalds 已提交
1051 1052 1053 1054 1055 1056 1057
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		setup_IO_APIC();
	else
		nr_ioapics = 0;

	/*
1058
	 * Set up local APIC timer on boot CPU.
L
Linus Torvalds 已提交
1059 1060
	 */

1061
	setup_boot_APIC_clock();
L
Linus Torvalds 已提交
1062 1063
}

1064 1065 1066 1067
/*
 * Early setup to make printk work.
 */
void __init smp_prepare_boot_cpu(void)
L
Linus Torvalds 已提交
1068
{
1069 1070 1071
	int me = smp_processor_id();
	cpu_set(me, cpu_online_map);
	cpu_set(me, cpu_callout_map);
1072
	per_cpu(cpu_state, me) = CPU_ONLINE;
L
Linus Torvalds 已提交
1073 1074
}

1075 1076 1077 1078
/*
 * Entry point to boot a CPU.
 */
int __cpuinit __cpu_up(unsigned int cpu)
L
Linus Torvalds 已提交
1079
{
1080 1081
	int err;
	int apicid = cpu_present_to_apicid(cpu);
L
Linus Torvalds 已提交
1082

1083
	WARN_ON(irqs_disabled());
L
Linus Torvalds 已提交
1084

1085 1086 1087 1088 1089 1090 1091 1092
	Dprintk("++++++++++++++++++++=_---CPU UP  %u\n", cpu);

	if (apicid == BAD_APICID || apicid == boot_cpu_id ||
	    !physid_isset(apicid, phys_cpu_present_map)) {
		printk("__cpu_up: bad cpu %d\n", cpu);
		return -EINVAL;
	}

A
Ashok Raj 已提交
1093 1094 1095 1096 1097 1098 1099 1100
	/*
	 * Already booted CPU?
	 */
 	if (cpu_isset(cpu, cpu_callin_map)) {
		Dprintk("do_boot_cpu %d Already started\n", cpu);
 		return -ENOSYS;
	}

1101
	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
1102 1103 1104 1105 1106
	/* Boot it! */
	err = do_boot_cpu(cpu, apicid);
	if (err < 0) {
		Dprintk("do_boot_cpu failed %d\n", err);
		return err;
L
Linus Torvalds 已提交
1107
	}
1108

L
Linus Torvalds 已提交
1109 1110 1111 1112
	/* Unleash the CPU! */
	Dprintk("waiting for cpu %d\n", cpu);

	while (!cpu_isset(cpu, cpu_online_map))
1113
		cpu_relax();
A
Ashok Raj 已提交
1114 1115 1116
	err = 0;

	return err;
L
Linus Torvalds 已提交
1117 1118
}

1119 1120 1121
/*
 * Finish the SMP boot.
 */
1122
void __init smp_cpus_done(unsigned int max_cpus)
L
Linus Torvalds 已提交
1123
{
1124 1125
	smp_cleanup_boot();

L
Linus Torvalds 已提交
1126 1127 1128 1129
#ifdef CONFIG_X86_IO_APIC
	setup_ioapic_dest();
#endif

1130
	time_init_gtod();
1131 1132

	check_nmi_watchdog();
1133
}
A
Ashok Raj 已提交
1134 1135 1136

#ifdef CONFIG_HOTPLUG_CPU

1137
static void remove_siblinginfo(int cpu)
A
Ashok Raj 已提交
1138 1139
{
	int sibling;
1140
	struct cpuinfo_x86 *c = cpu_data;
A
Ashok Raj 已提交
1141

1142 1143 1144 1145 1146 1147 1148 1149 1150
	for_each_cpu_mask(sibling, cpu_core_map[cpu]) {
		cpu_clear(cpu, cpu_core_map[sibling]);
		/*
		 * last thread sibling in this cpu core going down
		 */
		if (cpus_weight(cpu_sibling_map[cpu]) == 1)
			c[sibling].booted_cores--;
	}
			
A
Ashok Raj 已提交
1151 1152 1153 1154 1155 1156
	for_each_cpu_mask(sibling, cpu_sibling_map[cpu])
		cpu_clear(cpu, cpu_sibling_map[sibling]);
	cpus_clear(cpu_sibling_map[cpu]);
	cpus_clear(cpu_core_map[cpu]);
	phys_proc_id[cpu] = BAD_APICID;
	cpu_core_id[cpu] = BAD_APICID;
1157
	cpu_clear(cpu, cpu_sibling_setup_map);
A
Ashok Raj 已提交
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
}

void remove_cpu_from_maps(void)
{
	int cpu = smp_processor_id();

	cpu_clear(cpu, cpu_callout_map);
	cpu_clear(cpu, cpu_callin_map);
	clear_bit(cpu, &cpu_initialized); /* was set by cpu_init() */
}

int __cpu_disable(void)
{
	int cpu = smp_processor_id();

	/*
	 * Perhaps use cpufreq to drop frequency, but that could go
	 * into generic code.
 	 *
	 * We won't take down the boot processor on i386 due to some
	 * interrupts only being able to be serviced by the BSP.
	 * Especially so if we're not using an IOAPIC	-zwane
	 */
	if (cpu == 0)
		return -EBUSY;

1184
	clear_local_APIC();
A
Ashok Raj 已提交
1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211

	/*
	 * HACK:
	 * Allow any queued timer interrupts to get serviced
	 * This is only a temporary solution until we cleanup
	 * fixup_irqs as we do for IA64.
	 */
	local_irq_enable();
	mdelay(1);

	local_irq_disable();
	remove_siblinginfo(cpu);

	/* It's now safe to remove this processor from the online map */
	cpu_clear(cpu, cpu_online_map);
	remove_cpu_from_maps();
	fixup_irqs(cpu_online_map);
	return 0;
}

void __cpu_die(unsigned int cpu)
{
	/* We don't do anything here: idle task is faking death itself. */
	unsigned int i;

	for (i = 0; i < 10; i++) {
		/* They ack this in play_dead by setting CPU_DEAD */
1212 1213
		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
			printk ("CPU %d is now offline\n", cpu);
A
Ashok Raj 已提交
1214
			return;
1215
		}
1216
		msleep(100);
A
Ashok Raj 已提交
1217 1218 1219 1220
	}
 	printk(KERN_ERR "CPU %u didn't die...\n", cpu);
}

1221 1222 1223 1224 1225 1226
static __init int setup_additional_cpus(char *s)
{
	return get_option(&s, &additional_cpus);
}
__setup("additional_cpus=", setup_additional_cpus);

A
Ashok Raj 已提交
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
#else /* ... !CONFIG_HOTPLUG_CPU */

int __cpu_disable(void)
{
	return -ENOSYS;
}

void __cpu_die(unsigned int cpu)
{
	/* We said "no" in __cpu_disable */
	BUG();
}
#endif /* CONFIG_HOTPLUG_CPU */