voyager_smp.c 51.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/* -*- mode: c; c-basic-offset: 8 -*- */

/* Copyright (C) 1999,2001
 *
 * Author: J.E.J.Bottomley@HansenPartnership.com
 *
 * linux/arch/i386/kernel/voyager_smp.c
 *
 * This file provides all the same external entries as smp.c but uses
 * the voyager hal to provide the functionality
 */
12
#include <linux/module.h>
L
Linus Torvalds 已提交
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/delay.h>
#include <linux/mc146818rtc.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/bootmem.h>
#include <linux/completion.h>
#include <asm/desc.h>
#include <asm/voyager.h>
#include <asm/vic.h>
#include <asm/mtrr.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm/arch_hooks.h>

/* TLB state -- visible externally, indexed physically */
32
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = { &init_mm, 0 };
L
Linus Torvalds 已提交
33 34

/* CPU IRQ affinity -- set to all ones initially */
I
Ingo Molnar 已提交
35 36
static unsigned long cpu_irq_affinity[NR_CPUS] __cacheline_aligned =
	{[0 ... NR_CPUS-1]  = ~0UL };
L
Linus Torvalds 已提交
37 38 39

/* per CPU data structure (for /proc/cpuinfo et al), visible externally
 * indexed physically */
40
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
41
EXPORT_PER_CPU_SYMBOL(cpu_info);
L
Linus Torvalds 已提交
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73

/* physical ID of the CPU used to boot the system */
unsigned char boot_cpu_id;

/* The memory line addresses for the Quad CPIs */
struct voyager_qic_cpi *voyager_quad_cpi_addr[NR_CPUS] __cacheline_aligned;

/* The masks for the Extended VIC processors, filled in by cat_init */
__u32 voyager_extended_vic_processors = 0;

/* Masks for the extended Quad processors which cannot be VIC booted */
__u32 voyager_allowed_boot_processors = 0;

/* The mask for the Quad Processors (both extended and non-extended) */
__u32 voyager_quad_processors = 0;

/* Total count of live CPUs, used in process.c to display
 * the CPU information and in irq.c for the per CPU irq
 * activity count.  Finally exported by i386_ksyms.c */
static int voyager_extended_cpus = 1;

/* Have we found an SMP box - used by time.c to do the profiling
   interrupt for timeslicing; do not set to 1 until the per CPU timer
   interrupt is active */
int smp_found_config = 0;

/* Used for the invalidate map that's also checked in the spinlock */
static volatile unsigned long smp_invalidate_needed;

/* Bitmask of currently online CPUs - used by setup.c for
   /proc/cpuinfo, visible externally but still physical */
cpumask_t cpu_online_map = CPU_MASK_NONE;
74
EXPORT_SYMBOL(cpu_online_map);
L
Linus Torvalds 已提交
75 76 77 78 79 80 81 82 83 84 85 86

/* Bitmask of CPUs present in the system - exported by i386_syms.c, used
 * by scheduler but indexed physically */
cpumask_t phys_cpu_present_map = CPU_MASK_NONE;

/* The internal functions */
static void send_CPI(__u32 cpuset, __u8 cpi);
static void ack_CPI(__u8 cpi);
static int ack_QIC_CPI(__u8 cpi);
static void ack_special_QIC_CPI(__u8 cpi);
static void ack_VIC_CPI(__u8 cpi);
static void send_CPI_allbutself(__u8 cpi);
J
James Bottomley 已提交
87 88
static void mask_vic_irq(unsigned int irq);
static void unmask_vic_irq(unsigned int irq);
L
Linus Torvalds 已提交
89 90 91 92 93 94 95 96 97 98 99 100
static unsigned int startup_vic_irq(unsigned int irq);
static void enable_local_vic_irq(unsigned int irq);
static void disable_local_vic_irq(unsigned int irq);
static void before_handle_vic_irq(unsigned int irq);
static void after_handle_vic_irq(unsigned int irq);
static void set_vic_irq_affinity(unsigned int irq, cpumask_t mask);
static void ack_vic_irq(unsigned int irq);
static void vic_enable_cpi(void);
static void do_boot_cpu(__u8 cpuid);
static void do_quad_bootstrap(void);

int hard_smp_processor_id(void);
101
int safe_smp_processor_id(void);
L
Linus Torvalds 已提交
102 103

/* Inline functions */
I
Ingo Molnar 已提交
104
static inline void send_one_QIC_CPI(__u8 cpu, __u8 cpi)
L
Linus Torvalds 已提交
105 106
{
	voyager_quad_cpi_addr[cpu]->qic_cpi[cpi].cpi =
I
Ingo Molnar 已提交
107
	    (smp_processor_id() << 16) + cpi;
L
Linus Torvalds 已提交
108 109
}

I
Ingo Molnar 已提交
110
static inline void send_QIC_CPI(__u32 cpuset, __u8 cpi)
L
Linus Torvalds 已提交
111 112 113 114
{
	int cpu;

	for_each_online_cpu(cpu) {
I
Ingo Molnar 已提交
115
		if (cpuset & (1 << cpu)) {
L
Linus Torvalds 已提交
116
#ifdef VOYAGER_DEBUG
I
Ingo Molnar 已提交
117 118 119 120
			if (!cpu_isset(cpu, cpu_online_map))
				VDEBUG(("CPU%d sending cpi %d to CPU%d not in "
					"cpu_online_map\n",
					hard_smp_processor_id(), cpi, cpu));
L
Linus Torvalds 已提交
121 122 123 124 125 126
#endif
			send_one_QIC_CPI(cpu, cpi - QIC_CPI_OFFSET);
		}
	}
}

I
Ingo Molnar 已提交
127
static inline void wrapper_smp_local_timer_interrupt(void)
128 129
{
	irq_enter();
130
	smp_local_timer_interrupt();
131 132 133
	irq_exit();
}

I
Ingo Molnar 已提交
134
static inline void send_one_CPI(__u8 cpu, __u8 cpi)
L
Linus Torvalds 已提交
135
{
I
Ingo Molnar 已提交
136
	if (voyager_quad_processors & (1 << cpu))
L
Linus Torvalds 已提交
137 138
		send_one_QIC_CPI(cpu, cpi - QIC_CPI_OFFSET);
	else
I
Ingo Molnar 已提交
139
		send_CPI(1 << cpu, cpi);
L
Linus Torvalds 已提交
140 141
}

I
Ingo Molnar 已提交
142
static inline void send_CPI_allbutself(__u8 cpi)
L
Linus Torvalds 已提交
143 144 145 146 147 148
{
	__u8 cpu = smp_processor_id();
	__u32 mask = cpus_addr(cpu_online_map)[0] & ~(1 << cpu);
	send_CPI(mask, cpi);
}

I
Ingo Molnar 已提交
149
static inline int is_cpu_quad(void)
L
Linus Torvalds 已提交
150 151 152 153 154
{
	__u8 cpumask = inb(VIC_PROC_WHO_AM_I);
	return ((cpumask & QUAD_IDENTIFIER) == QUAD_IDENTIFIER);
}

I
Ingo Molnar 已提交
155
static inline int is_cpu_extended(void)
L
Linus Torvalds 已提交
156 157 158
{
	__u8 cpu = hard_smp_processor_id();

I
Ingo Molnar 已提交
159
	return (voyager_extended_vic_processors & (1 << cpu));
L
Linus Torvalds 已提交
160 161
}

I
Ingo Molnar 已提交
162
static inline int is_cpu_vic_boot(void)
L
Linus Torvalds 已提交
163 164 165
{
	__u8 cpu = hard_smp_processor_id();

I
Ingo Molnar 已提交
166 167
	return (voyager_extended_vic_processors
		& voyager_allowed_boot_processors & (1 << cpu));
L
Linus Torvalds 已提交
168 169
}

I
Ingo Molnar 已提交
170
static inline void ack_CPI(__u8 cpi)
L
Linus Torvalds 已提交
171
{
I
Ingo Molnar 已提交
172
	switch (cpi) {
L
Linus Torvalds 已提交
173
	case VIC_CPU_BOOT_CPI:
I
Ingo Molnar 已提交
174
		if (is_cpu_quad() && !is_cpu_vic_boot())
L
Linus Torvalds 已提交
175 176 177 178 179
			ack_QIC_CPI(cpi);
		else
			ack_VIC_CPI(cpi);
		break;
	case VIC_SYS_INT:
I
Ingo Molnar 已提交
180
	case VIC_CMN_INT:
L
Linus Torvalds 已提交
181 182
		/* These are slightly strange.  Even on the Quad card,
		 * They are vectored as VIC CPIs */
I
Ingo Molnar 已提交
183
		if (is_cpu_quad())
L
Linus Torvalds 已提交
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
			ack_special_QIC_CPI(cpi);
		else
			ack_VIC_CPI(cpi);
		break;
	default:
		printk("VOYAGER ERROR: CPI%d is in common CPI code\n", cpi);
		break;
	}
}

/* local variables */

/* The VIC IRQ descriptors -- these look almost identical to the
 * 8259 IRQs except that masks and things must be kept per processor
 */
J
James Bottomley 已提交
199
static struct irq_chip vic_chip = {
I
Ingo Molnar 已提交
200 201 202 203 204
	.name = "VIC",
	.startup = startup_vic_irq,
	.mask = mask_vic_irq,
	.unmask = unmask_vic_irq,
	.set_affinity = set_vic_irq_affinity,
L
Linus Torvalds 已提交
205 206 207 208 209 210 211 212
};

/* used to count up as CPUs are brought on line (starts at 0) */
static int cpucount = 0;

/* steal a page from the bottom of memory for the trampoline and
 * squirrel its address away here.  This will be in kernel virtual
 * space */
213
unsigned char *trampoline_base;
L
Linus Torvalds 已提交
214 215 216 217

/* The per cpu profile stuff - used in smp_local_timer_interrupt */
static DEFINE_PER_CPU(int, prof_multiplier) = 1;
static DEFINE_PER_CPU(int, prof_old_multiplier) = 1;
I
Ingo Molnar 已提交
218
static DEFINE_PER_CPU(int, prof_counter) = 1;
L
Linus Torvalds 已提交
219 220 221 222 223 224 225 226 227 228 229

/* the map used to check if a CPU has booted */
static __u32 cpu_booted_map;

/* the synchronize flag used to hold all secondary CPUs spinning in
 * a tight loop until the boot sequence is ready for them */
static cpumask_t smp_commenced_mask = CPU_MASK_NONE;

/* This is for the new dynamic CPU boot code */
cpumask_t cpu_callin_map = CPU_MASK_NONE;
cpumask_t cpu_callout_map = CPU_MASK_NONE;
230
cpumask_t cpu_possible_map = CPU_MASK_NONE;
231
EXPORT_SYMBOL(cpu_possible_map);
L
Linus Torvalds 已提交
232 233 234 235 236 237 238 239

/* The per processor IRQ masks (these are usually kept in sync) */
static __u16 vic_irq_mask[NR_CPUS] __cacheline_aligned;

/* the list of IRQs to be enabled by the VIC_ENABLE_IRQ_CPI */
static __u16 vic_irq_enable_mask[NR_CPUS] __cacheline_aligned = { 0 };

/* Lock for enable/disable of VIC interrupts */
I
Ingo Molnar 已提交
240
static __cacheline_aligned DEFINE_SPINLOCK(vic_irq_lock);
L
Linus Torvalds 已提交
241

I
Ingo Molnar 已提交
242
/* The boot processor is correctly set up in PC mode when it
L
Linus Torvalds 已提交
243 244 245 246 247 248 249 250 251 252 253 254 255
 * comes up, but the secondaries need their master/slave 8259
 * pairs initializing correctly */

/* Interrupt counters (per cpu) and total - used to try to
 * even up the interrupt handling routines */
static long vic_intr_total = 0;
static long vic_intr_count[NR_CPUS] __cacheline_aligned = { 0 };
static unsigned long vic_tick[NR_CPUS] __cacheline_aligned = { 0 };

/* Since we can only use CPI0, we fake all the other CPIs */
static unsigned long vic_cpi_mailbox[NR_CPUS] __cacheline_aligned;

/* debugging routine to read the isr of the cpu's pic */
I
Ingo Molnar 已提交
256
static inline __u16 vic_read_isr(void)
L
Linus Torvalds 已提交
257 258 259 260 261 262 263 264 265 266 267
{
	__u16 isr;

	outb(0x0b, 0xa0);
	isr = inb(0xa0) << 8;
	outb(0x0b, 0x20);
	isr |= inb(0x20);

	return isr;
}

I
Ingo Molnar 已提交
268
static __init void qic_setup(void)
L
Linus Torvalds 已提交
269
{
I
Ingo Molnar 已提交
270
	if (!is_cpu_quad()) {
L
Linus Torvalds 已提交
271 272 273 274 275
		/* not a quad, no setup */
		return;
	}
	outb(QIC_DEFAULT_MASK0, QIC_MASK_REGISTER0);
	outb(QIC_CPI_ENABLE, QIC_MASK_REGISTER1);
I
Ingo Molnar 已提交
276 277

	if (is_cpu_extended()) {
L
Linus Torvalds 已提交
278 279 280 281 282 283 284 285 286
		/* the QIC duplicate of the VIC base register */
		outb(VIC_DEFAULT_CPI_BASE, QIC_VIC_CPI_BASE_REGISTER);
		outb(QIC_DEFAULT_CPI_BASE, QIC_CPI_BASE_REGISTER);

		/* FIXME: should set up the QIC timer and memory parity
		 * error vectors here */
	}
}

I
Ingo Molnar 已提交
287
static __init void vic_setup_pic(void)
L
Linus Torvalds 已提交
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
{
	outb(1, VIC_REDIRECT_REGISTER_1);
	/* clear the claim registers for dynamic routing */
	outb(0, VIC_CLAIM_REGISTER_0);
	outb(0, VIC_CLAIM_REGISTER_1);

	outb(0, VIC_PRIORITY_REGISTER);
	/* Set the Primary and Secondary Microchannel vector
	 * bases to be the same as the ordinary interrupts
	 *
	 * FIXME: This would be more efficient using separate
	 * vectors. */
	outb(FIRST_EXTERNAL_VECTOR, VIC_PRIMARY_MC_BASE);
	outb(FIRST_EXTERNAL_VECTOR, VIC_SECONDARY_MC_BASE);
	/* Now initiallise the master PIC belonging to this CPU by
	 * sending the four ICWs */

	/* ICW1: level triggered, ICW4 needed */
	outb(0x19, 0x20);

	/* ICW2: vector base */
	outb(FIRST_EXTERNAL_VECTOR, 0x21);

	/* ICW3: slave at line 2 */
	outb(0x04, 0x21);

	/* ICW4: 8086 mode */
	outb(0x01, 0x21);

	/* now the same for the slave PIC */

	/* ICW1: level trigger, ICW4 needed */
	outb(0x19, 0xA0);

	/* ICW2: slave vector base */
	outb(FIRST_EXTERNAL_VECTOR + 8, 0xA1);
I
Ingo Molnar 已提交
324

L
Linus Torvalds 已提交
325 326 327 328 329 330 331
	/* ICW3: slave ID */
	outb(0x02, 0xA1);

	/* ICW4: 8086 mode */
	outb(0x01, 0xA1);
}

I
Ingo Molnar 已提交
332
static void do_quad_bootstrap(void)
L
Linus Torvalds 已提交
333
{
I
Ingo Molnar 已提交
334
	if (is_cpu_quad() && is_cpu_vic_boot()) {
L
Linus Torvalds 已提交
335 336 337 338 339 340
		int i;
		unsigned long flags;
		__u8 cpuid = hard_smp_processor_id();

		local_irq_save(flags);

I
Ingo Molnar 已提交
341
		for (i = 0; i < 4; i++) {
L
Linus Torvalds 已提交
342
			/* FIXME: this would be >>3 &0x7 on the 32 way */
I
Ingo Molnar 已提交
343
			if (((cpuid >> 2) & 0x03) == i)
L
Linus Torvalds 已提交
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
				/* don't lower our own mask! */
				continue;

			/* masquerade as local Quad CPU */
			outb(QIC_CPUID_ENABLE | i, QIC_PROCESSOR_ID);
			/* enable the startup CPI */
			outb(QIC_BOOT_CPI_MASK, QIC_MASK_REGISTER1);
			/* restore cpu id */
			outb(0, QIC_PROCESSOR_ID);
		}
		local_irq_restore(flags);
	}
}

/* Set up all the basic stuff: read the SMP config and make all the
 * SMP information reflect only the boot cpu.  All others will be
 * brought on-line later. */
I
Ingo Molnar 已提交
361
void __init find_smp_config(void)
L
Linus Torvalds 已提交
362 363 364 365 366 367 368 369
{
	int i;

	boot_cpu_id = hard_smp_processor_id();

	printk("VOYAGER SMP: Boot cpu is %d\n", boot_cpu_id);

	/* initialize the CPU structures (moved from smp_boot_cpus) */
I
Ingo Molnar 已提交
370
	for (i = 0; i < NR_CPUS; i++) {
L
Linus Torvalds 已提交
371 372 373 374 375
		cpu_irq_affinity[i] = ~0;
	}
	cpu_online_map = cpumask_of_cpu(boot_cpu_id);

	/* The boot CPU must be extended */
I
Ingo Molnar 已提交
376
	voyager_extended_vic_processors = 1 << boot_cpu_id;
S
Simon Arlott 已提交
377
	/* initially, all of the first 8 CPUs can boot */
L
Linus Torvalds 已提交
378 379 380 381
	voyager_allowed_boot_processors = 0xff;
	/* set up everything for just this CPU, we can alter
	 * this as we start the other CPUs later */
	/* now get the CPU disposition from the extended CMOS */
I
Ingo Molnar 已提交
382 383 384 385 386 387 388 389 390 391
	cpus_addr(phys_cpu_present_map)[0] =
	    voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK);
	cpus_addr(phys_cpu_present_map)[0] |=
	    voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK + 1) << 8;
	cpus_addr(phys_cpu_present_map)[0] |=
	    voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK +
				       2) << 16;
	cpus_addr(phys_cpu_present_map)[0] |=
	    voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK +
				       3) << 24;
392
	cpu_possible_map = phys_cpu_present_map;
I
Ingo Molnar 已提交
393 394
	printk("VOYAGER SMP: phys_cpu_present_map = 0x%lx\n",
	       cpus_addr(phys_cpu_present_map)[0]);
L
Linus Torvalds 已提交
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
	/* Here we set up the VIC to enable SMP */
	/* enable the CPIs by writing the base vector to their register */
	outb(VIC_DEFAULT_CPI_BASE, VIC_CPI_BASE_REGISTER);
	outb(1, VIC_REDIRECT_REGISTER_1);
	/* set the claim registers for static routing --- Boot CPU gets
	 * all interrupts untill all other CPUs started */
	outb(0xff, VIC_CLAIM_REGISTER_0);
	outb(0xff, VIC_CLAIM_REGISTER_1);
	/* Set the Primary and Secondary Microchannel vector
	 * bases to be the same as the ordinary interrupts
	 *
	 * FIXME: This would be more efficient using separate
	 * vectors. */
	outb(FIRST_EXTERNAL_VECTOR, VIC_PRIMARY_MC_BASE);
	outb(FIRST_EXTERNAL_VECTOR, VIC_SECONDARY_MC_BASE);

	/* Finally tell the firmware that we're driving */
	outb(inb(VOYAGER_SUS_IN_CONTROL_PORT) | VOYAGER_IN_CONTROL_FLAG,
	     VOYAGER_SUS_IN_CONTROL_PORT);

	current_thread_info()->cpu = boot_cpu_id;
J
Jeremy Fitzhardinge 已提交
416
	x86_write_percpu(cpu_number, boot_cpu_id);
L
Linus Torvalds 已提交
417 418 419 420 421
}

/*
 *	The bootstrap kernel entry code has set these up. Save them
 *	for a given CPU, id is physical */
I
Ingo Molnar 已提交
422
void __init smp_store_cpu_info(int id)
L
Linus Torvalds 已提交
423
{
424
	struct cpuinfo_x86 *c = &cpu_data(id);
L
Linus Torvalds 已提交
425 426 427

	*c = boot_cpu_data;

J
Jeremy Fitzhardinge 已提交
428
	identify_secondary_cpu(c);
L
Linus Torvalds 已提交
429 430 431
}

/* set up the trampoline and return the physical address of the code */
I
Ingo Molnar 已提交
432
static __u32 __init setup_trampoline(void)
L
Linus Torvalds 已提交
433 434
{
	/* these two are global symbols in trampoline.S */
J
Jan Beulich 已提交
435 436
	extern const __u8 trampoline_end[];
	extern const __u8 trampoline_data[];
L
Linus Torvalds 已提交
437

438
	memcpy(trampoline_base, trampoline_data,
L
Linus Torvalds 已提交
439
	       trampoline_end - trampoline_data);
440
	return virt_to_phys(trampoline_base);
L
Linus Torvalds 已提交
441 442 443
}

/* Routine initially called when a non-boot CPU is brought online */
I
Ingo Molnar 已提交
444
static void __init start_secondary(void *unused)
L
Linus Torvalds 已提交
445 446 447
{
	__u8 cpuid = hard_smp_processor_id();

J
Jeremy Fitzhardinge 已提交
448
	cpu_init();
L
Linus Torvalds 已提交
449 450 451 452 453

	/* OK, we're in the routine */
	ack_CPI(VIC_CPU_BOOT_CPI);

	/* setup the 8259 master slave pair belonging to this CPU ---
I
Ingo Molnar 已提交
454 455
	 * we won't actually receive any until the boot CPU
	 * relinquishes it's static routing mask */
L
Linus Torvalds 已提交
456 457 458 459
	vic_setup_pic();

	qic_setup();

I
Ingo Molnar 已提交
460
	if (is_cpu_quad() && !is_cpu_vic_boot()) {
L
Linus Torvalds 已提交
461 462 463
		/* clear the boot CPI */
		__u8 dummy;

I
Ingo Molnar 已提交
464 465
		dummy =
		    voyager_quad_cpi_addr[cpuid]->qic_cpi[VIC_CPU_BOOT_CPI].cpi;
L
Linus Torvalds 已提交
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
		printk("read dummy %d\n", dummy);
	}

	/* lower the mask to receive CPIs */
	vic_enable_cpi();

	VDEBUG(("VOYAGER SMP: CPU%d, stack at about %p\n", cpuid, &cpuid));

	/* enable interrupts */
	local_irq_enable();

	/* get our bogomips */
	calibrate_delay();

	/* save our processor parameters */
	smp_store_cpu_info(cpuid);

	/* if we're a quad, we may need to bootstrap other CPUs */
	do_quad_bootstrap();

	/* FIXME: this is rather a poor hack to prevent the CPU
	 * activating softirqs while it's supposed to be waiting for
	 * permission to proceed.  Without this, the new per CPU stuff
	 * in the softirqs will fail */
	local_irq_disable();
	cpu_set(cpuid, cpu_callin_map);

	/* signal that we're done */
	cpu_booted_map = 1;

	while (!cpu_isset(cpuid, smp_commenced_mask))
		rep_nop();
	local_irq_enable();

	local_flush_tlb();

	cpu_set(cpuid, cpu_online_map);
	wmb();
	cpu_idle();
}

/* Routine to kick start the given CPU and wait for it to report ready
 * (or timeout in startup).  When this routine returns, the requested
 * CPU is either fully running and configured or known to be dead.
 *
 * We call this routine sequentially 1 CPU at a time, so no need for
 * locking */

I
Ingo Molnar 已提交
514
static void __init do_boot_cpu(__u8 cpu)
L
Linus Torvalds 已提交
515 516 517 518
{
	struct task_struct *idle;
	int timeout;
	unsigned long flags;
I
Ingo Molnar 已提交
519 520 521
	int quad_boot = (1 << cpu) & voyager_quad_processors
	    & ~(voyager_extended_vic_processors
		& voyager_allowed_boot_processors);
L
Linus Torvalds 已提交
522 523 524 525 526

	/* This is an area in head.S which was used to set up the
	 * initial kernel stack.  We need to alter this to give the
	 * booting CPU a new stack (taken from its idle process) */
	extern struct {
527
		__u8 *sp;
L
Linus Torvalds 已提交
528 529 530 531
		unsigned short ss;
	} stack_start;
	/* This is the format of the CPI IDT gate (in real mode) which
	 * we're hijacking to boot the CPU */
I
Ingo Molnar 已提交
532
	union IDTFormat {
L
Linus Torvalds 已提交
533
		struct seg {
I
Ingo Molnar 已提交
534 535
			__u16 Offset;
			__u16 Segment;
L
Linus Torvalds 已提交
536 537 538 539 540 541 542 543 544 545 546 547 548 549 550
		} idt;
		__u32 val;
	} hijack_source;

	__u32 *hijack_vector;
	__u32 start_phys_address = setup_trampoline();

	/* There's a clever trick to this: The linux trampoline is
	 * compiled to begin at absolute location zero, so make the
	 * address zero but have the data segment selector compensate
	 * for the actual address */
	hijack_source.idt.Offset = start_phys_address & 0x000F;
	hijack_source.idt.Segment = (start_phys_address >> 4) & 0xFFFF;

	cpucount++;
J
James Bottomley 已提交
551 552
	alternatives_smp_switch(1);

L
Linus Torvalds 已提交
553
	idle = fork_idle(cpu);
I
Ingo Molnar 已提交
554
	if (IS_ERR(idle))
L
Linus Torvalds 已提交
555
		panic("failed fork for CPU%d", cpu);
556
	idle->thread.ip = (unsigned long)start_secondary;
L
Linus Torvalds 已提交
557
	/* init_tasks (in sched.c) is indexed logically */
558
	stack_start.sp = (void *)idle->thread.sp;
L
Linus Torvalds 已提交
559

J
Jeremy Fitzhardinge 已提交
560
	init_gdt(cpu);
I
Ingo Molnar 已提交
561
	per_cpu(current_task, cpu) = idle;
J
Jeremy Fitzhardinge 已提交
562
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
L
Linus Torvalds 已提交
563 564 565
	irq_ctx_init(cpu);

	/* Note: Don't modify initial ss override */
I
Ingo Molnar 已提交
566
	VDEBUG(("VOYAGER SMP: Booting CPU%d at 0x%lx[%x:%x], stack %p\n", cpu,
L
Linus Torvalds 已提交
567
		(unsigned long)hijack_source.val, hijack_source.idt.Segment,
568
		hijack_source.idt.Offset, stack_start.sp));
569 570 571 572 573

	/* init lowmem identity mapping */
	clone_pgd_range(swapper_pg_dir, swapper_pg_dir + USER_PGD_PTRS,
			min_t(unsigned long, KERNEL_PGD_PTRS, USER_PGD_PTRS));
	flush_tlb_all();
L
Linus Torvalds 已提交
574

I
Ingo Molnar 已提交
575
	if (quad_boot) {
L
Linus Torvalds 已提交
576
		printk("CPU %d: non extended Quad boot\n", cpu);
I
Ingo Molnar 已提交
577 578 579
		hijack_vector =
		    (__u32 *)
		    phys_to_virt((VIC_CPU_BOOT_CPI + QIC_DEFAULT_CPI_BASE) * 4);
L
Linus Torvalds 已提交
580 581 582
		*hijack_vector = hijack_source.val;
	} else {
		printk("CPU%d: extended VIC boot\n", cpu);
I
Ingo Molnar 已提交
583 584 585
		hijack_vector =
		    (__u32 *)
		    phys_to_virt((VIC_CPU_BOOT_CPI + VIC_DEFAULT_CPI_BASE) * 4);
L
Linus Torvalds 已提交
586 587
		*hijack_vector = hijack_source.val;
		/* VIC errata, may also receive interrupt at this address */
I
Ingo Molnar 已提交
588 589 590 591
		hijack_vector =
		    (__u32 *)
		    phys_to_virt((VIC_CPU_BOOT_ERRATA_CPI +
				  VIC_DEFAULT_CPI_BASE) * 4);
L
Linus Torvalds 已提交
592 593 594 595 596 597 598
		*hijack_vector = hijack_source.val;
	}
	/* All non-boot CPUs start with interrupts fully masked.  Need
	 * to lower the mask of the CPI we're about to send.  We do
	 * this in the VIC by masquerading as the processor we're
	 * about to boot and lowering its interrupt mask */
	local_irq_save(flags);
I
Ingo Molnar 已提交
599
	if (quad_boot) {
L
Linus Torvalds 已提交
600 601 602 603
		send_one_QIC_CPI(cpu, VIC_CPU_BOOT_CPI);
	} else {
		outb(VIC_CPU_MASQUERADE_ENABLE | cpu, VIC_PROCESSOR_ID);
		/* here we're altering registers belonging to `cpu' */
I
Ingo Molnar 已提交
604

L
Linus Torvalds 已提交
605 606 607 608 609 610
		outb(VIC_BOOT_INTERRUPT_MASK, 0x21);
		/* now go back to our original identity */
		outb(boot_cpu_id, VIC_PROCESSOR_ID);

		/* and boot the CPU */

I
Ingo Molnar 已提交
611
		send_CPI((1 << cpu), VIC_CPU_BOOT_CPI);
L
Linus Torvalds 已提交
612 613 614 615 616
	}
	cpu_booted_map = 0;
	local_irq_restore(flags);

	/* now wait for it to become ready (or timeout) */
I
Ingo Molnar 已提交
617 618
	for (timeout = 0; timeout < 50000; timeout++) {
		if (cpu_booted_map)
L
Linus Torvalds 已提交
619 620 621 622
			break;
		udelay(100);
	}
	/* reset the page table */
623
	zap_low_mappings();
I
Ingo Molnar 已提交
624

L
Linus Torvalds 已提交
625 626 627
	if (cpu_booted_map) {
		VDEBUG(("CPU%d: Booted successfully, back in CPU %d\n",
			cpu, smp_processor_id()));
I
Ingo Molnar 已提交
628

L
Linus Torvalds 已提交
629
		printk("CPU%d: ", cpu);
630
		print_cpu_info(&cpu_data(cpu));
L
Linus Torvalds 已提交
631 632
		wmb();
		cpu_set(cpu, cpu_callout_map);
633
		cpu_set(cpu, cpu_present_map);
I
Ingo Molnar 已提交
634
	} else {
L
Linus Torvalds 已提交
635
		printk("CPU%d FAILED TO BOOT: ", cpu);
I
Ingo Molnar 已提交
636 637 638
		if (*
		    ((volatile unsigned char *)phys_to_virt(start_phys_address))
		    == 0xA5)
L
Linus Torvalds 已提交
639 640 641
			printk("Stuck.\n");
		else
			printk("Not responding.\n");
I
Ingo Molnar 已提交
642

L
Linus Torvalds 已提交
643 644 645 646
		cpucount--;
	}
}

I
Ingo Molnar 已提交
647
void __init smp_boot_cpus(void)
L
Linus Torvalds 已提交
648 649 650 651 652 653
{
	int i;

	/* CAT BUS initialisation must be done after the memory */
	/* FIXME: The L4 has a catbus too, it just needs to be
	 * accessed in a totally different way */
I
Ingo Molnar 已提交
654
	if (voyager_level == 5) {
L
Linus Torvalds 已提交
655 656 657 658
		voyager_cat_init();

		/* now that the cat has probed the Voyager System Bus, sanity
		 * check the cpu map */
I
Ingo Molnar 已提交
659 660 661
		if (((voyager_quad_processors | voyager_extended_vic_processors)
		     & cpus_addr(phys_cpu_present_map)[0]) !=
		    cpus_addr(phys_cpu_present_map)[0]) {
L
Linus Torvalds 已提交
662
			/* should panic */
I
Ingo Molnar 已提交
663 664
			printk("\n\n***WARNING*** "
			       "Sanity check of CPU present map FAILED\n");
L
Linus Torvalds 已提交
665
		}
I
Ingo Molnar 已提交
666 667 668
	} else if (voyager_level == 4)
		voyager_extended_vic_processors =
		    cpus_addr(phys_cpu_present_map)[0];
L
Linus Torvalds 已提交
669 670 671 672 673 674 675 676

	/* this sets up the idle task to run on the current cpu */
	voyager_extended_cpus = 1;
	/* Remove the global_irq_holder setting, it triggers a BUG() on
	 * schedule at the moment */
	//global_irq_holder = boot_cpu_id;

	/* FIXME: Need to do something about this but currently only works
I
Ingo Molnar 已提交
677 678
	 * on CPUs with a tsc which none of mine have.
	 smp_tune_scheduling();
L
Linus Torvalds 已提交
679 680 681
	 */
	smp_store_cpu_info(boot_cpu_id);
	printk("CPU%d: ", boot_cpu_id);
682
	print_cpu_info(&cpu_data(boot_cpu_id));
L
Linus Torvalds 已提交
683

I
Ingo Molnar 已提交
684
	if (is_cpu_quad()) {
L
Linus Torvalds 已提交
685 686 687 688 689 690 691 692 693 694 695
		/* booting on a Quad CPU */
		printk("VOYAGER SMP: Boot CPU is Quad\n");
		qic_setup();
		do_quad_bootstrap();
	}

	/* enable our own CPIs */
	vic_enable_cpi();

	cpu_set(boot_cpu_id, cpu_online_map);
	cpu_set(boot_cpu_id, cpu_callout_map);
I
Ingo Molnar 已提交
696 697

	/* loop over all the extended VIC CPUs and boot them.  The
L
Linus Torvalds 已提交
698
	 * Quad CPUs must be bootstrapped by their extended VIC cpu */
I
Ingo Molnar 已提交
699 700
	for (i = 0; i < NR_CPUS; i++) {
		if (i == boot_cpu_id || !cpu_isset(i, phys_cpu_present_map))
L
Linus Torvalds 已提交
701 702 703 704 705 706 707 708 709 710 711 712
			continue;
		do_boot_cpu(i);
		/* This udelay seems to be needed for the Quad boots
		 * don't remove unless you know what you're doing */
		udelay(1000);
	}
	/* we could compute the total bogomips here, but why bother?,
	 * Code added from smpboot.c */
	{
		unsigned long bogosum = 0;
		for (i = 0; i < NR_CPUS; i++)
			if (cpu_isset(i, cpu_online_map))
713
				bogosum += cpu_data(i).loops_per_jiffy;
I
Ingo Molnar 已提交
714 715 716 717
		printk(KERN_INFO "Total of %d processors activated "
		       "(%lu.%02lu BogoMIPS).\n",
		       cpucount + 1, bogosum / (500000 / HZ),
		       (bogosum / (5000 / HZ)) % 100);
L
Linus Torvalds 已提交
718 719
	}
	voyager_extended_cpus = hweight32(voyager_extended_vic_processors);
I
Ingo Molnar 已提交
720 721 722
	printk("VOYAGER: Extended (interrupt handling CPUs): "
	       "%d, non-extended: %d\n", voyager_extended_cpus,
	       num_booting_cpus() - voyager_extended_cpus);
L
Linus Torvalds 已提交
723 724 725 726
	/* that's it, switch to symmetric mode */
	outb(0, VIC_PRIORITY_REGISTER);
	outb(0, VIC_CLAIM_REGISTER_0);
	outb(0, VIC_CLAIM_REGISTER_1);
I
Ingo Molnar 已提交
727

L
Linus Torvalds 已提交
728 729 730 731 732
	VDEBUG(("VOYAGER SMP: Booted with %d CPUs\n", num_booting_cpus()));
}

/* Reload the secondary CPUs task structure (this function does not
 * return ) */
I
Ingo Molnar 已提交
733
void __init initialize_secondary(void)
L
Linus Torvalds 已提交
734 735 736 737 738 739 740 741 742 743 744
{
#if 0
	// AC kernels only
	set_current(hard_get_current());
#endif

	/*
	 * We don't actually need to load the full TSS,
	 * basically just the stack pointer and the eip.
	 */

I
Ingo Molnar 已提交
745
	asm volatile ("movl %0,%%esp\n\t"
746 747
		      "jmp *%1"::"r" (current->thread.sp),
		      "r"(current->thread.ip));
L
Linus Torvalds 已提交
748 749 750 751 752 753 754 755
}

/* handle a Voyager SYS_INT -- If we don't, the base board will
 * panic the system.
 *
 * System interrupts occur because some problem was detected on the
 * various busses.  To find out what you have to probe all the
 * hardware via the CAT bus.  FIXME: At the moment we do nothing. */
756
void smp_vic_sys_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
757 758
{
	ack_CPI(VIC_SYS_INT);
I
Ingo Molnar 已提交
759
	printk("Voyager SYSTEM INTERRUPT\n");
L
Linus Torvalds 已提交
760 761 762 763 764
}

/* Handle a voyager CMN_INT; These interrupts occur either because of
 * a system status change or because a single bit memory error
 * occurred.  FIXME: At the moment, ignore all this. */
765
void smp_vic_cmn_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
766 767 768 769 770 771
{
	static __u8 in_cmn_int = 0;
	static DEFINE_SPINLOCK(cmn_int_lock);

	/* common ints are broadcast, so make sure we only do this once */
	_raw_spin_lock(&cmn_int_lock);
I
Ingo Molnar 已提交
772
	if (in_cmn_int)
L
Linus Torvalds 已提交
773 774 775 776 777 778 779
		goto unlock_end;

	in_cmn_int++;
	_raw_spin_unlock(&cmn_int_lock);

	VDEBUG(("Voyager COMMON INTERRUPT\n"));

I
Ingo Molnar 已提交
780
	if (voyager_level == 5)
L
Linus Torvalds 已提交
781 782 783 784
		voyager_cat_do_common_interrupt();

	_raw_spin_lock(&cmn_int_lock);
	in_cmn_int = 0;
I
Ingo Molnar 已提交
785
      unlock_end:
L
Linus Torvalds 已提交
786 787 788 789 790 791 792
	_raw_spin_unlock(&cmn_int_lock);
	ack_CPI(VIC_CMN_INT);
}

/*
 * Reschedule call back. Nothing to do, all the work is done
 * automatically when we return from the interrupt.  */
I
Ingo Molnar 已提交
793
static void smp_reschedule_interrupt(void)
L
Linus Torvalds 已提交
794 795 796 797
{
	/* do nothing */
}

I
Ingo Molnar 已提交
798
static struct mm_struct *flush_mm;
L
Linus Torvalds 已提交
799 800 801 802
static unsigned long flush_va;
static DEFINE_SPINLOCK(tlbstate_lock);

/*
I
Ingo Molnar 已提交
803
 * We cannot call mmdrop() because we are in interrupt context,
L
Linus Torvalds 已提交
804 805 806 807 808
 * instead update mm->cpu_vm_mask.
 *
 * We need to reload %cr3 since the page tables may be going
 * away from under us..
 */
809
static inline void voyager_leave_mm(unsigned long cpu)
L
Linus Torvalds 已提交
810 811 812 813 814 815 816 817 818 819
{
	if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
		BUG();
	cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask);
	load_cr3(swapper_pg_dir);
}

/*
 * Invalidate call-back
 */
I
Ingo Molnar 已提交
820
static void smp_invalidate_interrupt(void)
L
Linus Torvalds 已提交
821 822 823 824 825 826 827
{
	__u8 cpu = smp_processor_id();

	if (!test_bit(cpu, &smp_invalidate_needed))
		return;
	/* This will flood messages.  Don't uncomment unless you see
	 * Problems with cross cpu invalidation
I
Ingo Molnar 已提交
828 829 830
	 VDEBUG(("VOYAGER SMP: CPU%d received INVALIDATE_CPI\n",
	 smp_processor_id()));
	 */
L
Linus Torvalds 已提交
831 832 833

	if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
		if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
834
			if (flush_va == TLB_FLUSH_ALL)
L
Linus Torvalds 已提交
835 836 837 838
				local_flush_tlb();
			else
				__flush_tlb_one(flush_va);
		} else
839
			voyager_leave_mm(cpu);
L
Linus Torvalds 已提交
840 841 842 843 844 845 846 847 848 849
	}
	smp_mb__before_clear_bit();
	clear_bit(cpu, &smp_invalidate_needed);
	smp_mb__after_clear_bit();
}

/* All the new flush operations for 2.4 */

/* This routine is called with a physical cpu mask */
static void
I
Ingo Molnar 已提交
850 851
voyager_flush_tlb_others(unsigned long cpumask, struct mm_struct *mm,
			 unsigned long va)
L
Linus Torvalds 已提交
852 853 854 855 856 857 858 859 860 861 862 863 864
{
	int stuck = 50000;

	if (!cpumask)
		BUG();
	if ((cpumask & cpus_addr(cpu_online_map)[0]) != cpumask)
		BUG();
	if (cpumask & (1 << smp_processor_id()))
		BUG();
	if (!mm)
		BUG();

	spin_lock(&tlbstate_lock);
I
Ingo Molnar 已提交
865

L
Linus Torvalds 已提交
866 867 868 869 870 871 872 873 874 875 876
	flush_mm = mm;
	flush_va = va;
	atomic_set_mask(cpumask, &smp_invalidate_needed);
	/*
	 * We have to send the CPI only to
	 * CPUs affected.
	 */
	send_CPI(cpumask, VIC_INVALIDATE_CPI);

	while (smp_invalidate_needed) {
		mb();
I
Ingo Molnar 已提交
877 878 879
		if (--stuck == 0) {
			printk("***WARNING*** Stuck doing invalidate CPI "
			       "(CPU%d)\n", smp_processor_id());
L
Linus Torvalds 已提交
880 881 882 883 884
			break;
		}
	}

	/* Uncomment only to debug invalidation problems
I
Ingo Molnar 已提交
885 886
	   VDEBUG(("VOYAGER SMP: Completed invalidate CPI (CPU%d)\n", cpu));
	 */
L
Linus Torvalds 已提交
887 888 889 890 891 892

	flush_mm = NULL;
	flush_va = 0;
	spin_unlock(&tlbstate_lock);
}

I
Ingo Molnar 已提交
893
void flush_tlb_current_task(void)
L
Linus Torvalds 已提交
894 895 896 897 898 899 900 901 902
{
	struct mm_struct *mm = current->mm;
	unsigned long cpu_mask;

	preempt_disable();

	cpu_mask = cpus_addr(mm->cpu_vm_mask)[0] & ~(1 << smp_processor_id());
	local_flush_tlb();
	if (cpu_mask)
903
		voyager_flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
L
Linus Torvalds 已提交
904 905 906 907

	preempt_enable();
}

I
Ingo Molnar 已提交
908
void flush_tlb_mm(struct mm_struct *mm)
L
Linus Torvalds 已提交
909 910 911 912 913 914 915 916 917 918 919
{
	unsigned long cpu_mask;

	preempt_disable();

	cpu_mask = cpus_addr(mm->cpu_vm_mask)[0] & ~(1 << smp_processor_id());

	if (current->active_mm == mm) {
		if (current->mm)
			local_flush_tlb();
		else
920
			voyager_leave_mm(smp_processor_id());
L
Linus Torvalds 已提交
921 922
	}
	if (cpu_mask)
923
		voyager_flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
L
Linus Torvalds 已提交
924 925 926 927

	preempt_enable();
}

I
Ingo Molnar 已提交
928
void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
L
Linus Torvalds 已提交
929 930 931 932 933 934 935 936
{
	struct mm_struct *mm = vma->vm_mm;
	unsigned long cpu_mask;

	preempt_disable();

	cpu_mask = cpus_addr(mm->cpu_vm_mask)[0] & ~(1 << smp_processor_id());
	if (current->active_mm == mm) {
I
Ingo Molnar 已提交
937
		if (current->mm)
L
Linus Torvalds 已提交
938
			__flush_tlb_one(va);
I
Ingo Molnar 已提交
939
		else
940
			voyager_leave_mm(smp_processor_id());
L
Linus Torvalds 已提交
941 942 943
	}

	if (cpu_mask)
J
Jeremy Fitzhardinge 已提交
944
		voyager_flush_tlb_others(cpu_mask, mm, va);
L
Linus Torvalds 已提交
945 946 947

	preempt_enable();
}
I
Ingo Molnar 已提交
948

949
EXPORT_SYMBOL(flush_tlb_page);
L
Linus Torvalds 已提交
950 951

/* enable the requested IRQs */
I
Ingo Molnar 已提交
952
static void smp_enable_irq_interrupt(void)
L
Linus Torvalds 已提交
953 954 955 956 957
{
	__u8 irq;
	__u8 cpu = get_cpu();

	VDEBUG(("VOYAGER SMP: CPU%d enabling irq mask 0x%x\n", cpu,
I
Ingo Molnar 已提交
958
		vic_irq_enable_mask[cpu]));
L
Linus Torvalds 已提交
959 960

	spin_lock(&vic_irq_lock);
I
Ingo Molnar 已提交
961 962
	for (irq = 0; irq < 16; irq++) {
		if (vic_irq_enable_mask[cpu] & (1 << irq))
L
Linus Torvalds 已提交
963 964 965 966 967 968 969
			enable_local_vic_irq(irq);
	}
	vic_irq_enable_mask[cpu] = 0;
	spin_unlock(&vic_irq_lock);

	put_cpu_no_resched();
}
I
Ingo Molnar 已提交
970

L
Linus Torvalds 已提交
971 972 973
/*
 *	CPU halt call-back
 */
I
Ingo Molnar 已提交
974
static void smp_stop_cpu_function(void *dummy)
L
Linus Torvalds 已提交
975 976 977 978
{
	VDEBUG(("VOYAGER SMP: CPU%d is STOPPING\n", smp_processor_id()));
	cpu_clear(smp_processor_id(), cpu_online_map);
	local_irq_disable();
I
Ingo Molnar 已提交
979
	for (;;)
Z
Zachary Amsden 已提交
980
		halt();
L
Linus Torvalds 已提交
981 982 983 984 985 986 987 988 989 990 991 992
}

static DEFINE_SPINLOCK(call_lock);

struct call_data_struct {
	void (*func) (void *info);
	void *info;
	volatile unsigned long started;
	volatile unsigned long finished;
	int wait;
};

I
Ingo Molnar 已提交
993
static struct call_data_struct *call_data;
L
Linus Torvalds 已提交
994 995 996

/* execute a thread on a new CPU.  The function to be called must be
 * previously set up.  This is used to schedule a function for
S
Simon Arlott 已提交
997
 * execution on all CPUs - set up the function then broadcast a
L
Linus Torvalds 已提交
998
 * function_interrupt CPI to come here on each CPU */
I
Ingo Molnar 已提交
999
static void smp_call_function_interrupt(void)
L
Linus Torvalds 已提交
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
{
	void (*func) (void *info) = call_data->func;
	void *info = call_data->info;
	/* must take copy of wait because call_data may be replaced
	 * unless the function is waiting for us to finish */
	int wait = call_data->wait;
	__u8 cpu = smp_processor_id();

	/*
	 * Notify initiating CPU that I've grabbed the data and am
	 * about to execute the function
	 */
	mb();
I
Ingo Molnar 已提交
1013
	if (!test_and_clear_bit(cpu, &call_data->started)) {
L
Linus Torvalds 已提交
1014
		/* If the bit wasn't set, this could be a replay */
I
Ingo Molnar 已提交
1015 1016
		printk(KERN_WARNING "VOYAGER SMP: CPU %d received call funtion"
		       " with no call pending\n", cpu);
L
Linus Torvalds 已提交
1017 1018 1019 1020 1021 1022
		return;
	}
	/*
	 * At this point the info structure may be out of scope unless wait==1
	 */
	irq_enter();
I
Ingo Molnar 已提交
1023
	(*func) (info);
1024
	__get_cpu_var(irq_stat).irq_call_count++;
L
Linus Torvalds 已提交
1025 1026 1027 1028 1029 1030 1031
	irq_exit();
	if (wait) {
		mb();
		clear_bit(cpu, &call_data->finished);
	}
}

1032
static int
I
Ingo Molnar 已提交
1033 1034
voyager_smp_call_function_mask(cpumask_t cpumask,
			       void (*func) (void *info), void *info, int wait)
L
Linus Torvalds 已提交
1035 1036
{
	struct call_data_struct data;
J
Jeremy Fitzhardinge 已提交
1037
	u32 mask = cpus_addr(cpumask)[0];
L
Linus Torvalds 已提交
1038

I
Ingo Molnar 已提交
1039
	mask &= ~(1 << smp_processor_id());
L
Linus Torvalds 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057

	if (!mask)
		return 0;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	data.func = func;
	data.info = info;
	data.started = mask;
	data.wait = wait;
	if (wait)
		data.finished = mask;

	spin_lock(&call_lock);
	call_data = &data;
	wmb();
	/* Send a message to all other CPUs and wait for them to respond */
1058
	send_CPI(mask, VIC_CALL_FUNCTION_CPI);
L
Linus Torvalds 已提交
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071

	/* Wait for response */
	while (data.started)
		barrier();

	if (wait)
		while (data.finished)
			barrier();

	spin_unlock(&call_lock);

	return 0;
}
1072

L
Linus Torvalds 已提交
1073 1074 1075 1076 1077 1078
/* Sorry about the name.  In an APIC based system, the APICs
 * themselves are programmed to send a timer interrupt.  This is used
 * by linux to reschedule the processor.  Voyager doesn't have this,
 * so we use the system clock to interrupt one processor, which in
 * turn, broadcasts a timer CPI to all the others --- we receive that
 * CPI here.  We don't use this actually for counting so losing
I
Ingo Molnar 已提交
1079
 * ticks doesn't matter
L
Linus Torvalds 已提交
1080
 *
S
Simon Arlott 已提交
1081
 * FIXME: For those CPUs which actually have a local APIC, we could
L
Linus Torvalds 已提交
1082 1083 1084 1085 1086
 * try to use it to trigger this interrupt instead of having to
 * broadcast the timer tick.  Unfortunately, all my pentium DYADs have
 * no local APIC, so I can't do this
 *
 * This function is currently a placeholder and is unused in the code */
1087
void smp_apic_timer_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1088
{
1089 1090 1091
	struct pt_regs *old_regs = set_irq_regs(regs);
	wrapper_smp_local_timer_interrupt();
	set_irq_regs(old_regs);
L
Linus Torvalds 已提交
1092 1093 1094
}

/* All of the QUAD interrupt GATES */
1095
void smp_qic_timer_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1096
{
1097
	struct pt_regs *old_regs = set_irq_regs(regs);
1098 1099
	ack_QIC_CPI(QIC_TIMER_CPI);
	wrapper_smp_local_timer_interrupt();
1100
	set_irq_regs(old_regs);
L
Linus Torvalds 已提交
1101 1102
}

1103
void smp_qic_invalidate_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1104 1105 1106 1107 1108
{
	ack_QIC_CPI(QIC_INVALIDATE_CPI);
	smp_invalidate_interrupt();
}

1109
void smp_qic_reschedule_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1110 1111 1112 1113 1114
{
	ack_QIC_CPI(QIC_RESCHEDULE_CPI);
	smp_reschedule_interrupt();
}

1115
void smp_qic_enable_irq_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1116 1117 1118 1119 1120
{
	ack_QIC_CPI(QIC_ENABLE_IRQ_CPI);
	smp_enable_irq_interrupt();
}

1121
void smp_qic_call_function_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1122 1123 1124 1125 1126
{
	ack_QIC_CPI(QIC_CALL_FUNCTION_CPI);
	smp_call_function_interrupt();
}

1127
void smp_vic_cpi_interrupt(struct pt_regs *regs)
L
Linus Torvalds 已提交
1128
{
1129
	struct pt_regs *old_regs = set_irq_regs(regs);
L
Linus Torvalds 已提交
1130 1131
	__u8 cpu = smp_processor_id();

I
Ingo Molnar 已提交
1132
	if (is_cpu_quad())
L
Linus Torvalds 已提交
1133 1134 1135 1136
		ack_QIC_CPI(VIC_CPI_LEVEL0);
	else
		ack_VIC_CPI(VIC_CPI_LEVEL0);

I
Ingo Molnar 已提交
1137
	if (test_and_clear_bit(VIC_TIMER_CPI, &vic_cpi_mailbox[cpu]))
1138
		wrapper_smp_local_timer_interrupt();
I
Ingo Molnar 已提交
1139
	if (test_and_clear_bit(VIC_INVALIDATE_CPI, &vic_cpi_mailbox[cpu]))
L
Linus Torvalds 已提交
1140
		smp_invalidate_interrupt();
I
Ingo Molnar 已提交
1141
	if (test_and_clear_bit(VIC_RESCHEDULE_CPI, &vic_cpi_mailbox[cpu]))
L
Linus Torvalds 已提交
1142
		smp_reschedule_interrupt();
I
Ingo Molnar 已提交
1143
	if (test_and_clear_bit(VIC_ENABLE_IRQ_CPI, &vic_cpi_mailbox[cpu]))
L
Linus Torvalds 已提交
1144
		smp_enable_irq_interrupt();
I
Ingo Molnar 已提交
1145
	if (test_and_clear_bit(VIC_CALL_FUNCTION_CPI, &vic_cpi_mailbox[cpu]))
L
Linus Torvalds 已提交
1146
		smp_call_function_interrupt();
1147
	set_irq_regs(old_regs);
L
Linus Torvalds 已提交
1148 1149
}

I
Ingo Molnar 已提交
1150
static void do_flush_tlb_all(void *info)
L
Linus Torvalds 已提交
1151 1152 1153 1154 1155
{
	unsigned long cpu = smp_processor_id();

	__flush_tlb_all();
	if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY)
1156
		voyager_leave_mm(cpu);
L
Linus Torvalds 已提交
1157 1158 1159
}

/* flush the TLB of every active CPU in the system */
I
Ingo Molnar 已提交
1160
void flush_tlb_all(void)
L
Linus Torvalds 已提交
1161 1162 1163 1164 1165 1166
{
	on_each_cpu(do_flush_tlb_all, 0, 1, 1);
}

/* used to set up the trampoline for other CPUs when the memory manager
 * is sorted out */
I
Ingo Molnar 已提交
1167
void __init smp_alloc_memory(void)
L
Linus Torvalds 已提交
1168
{
1169
	trampoline_base = alloc_bootmem_low_pages(PAGE_SIZE);
I
Ingo Molnar 已提交
1170
	if (__pa(trampoline_base) >= 0x93000)
L
Linus Torvalds 已提交
1171 1172 1173 1174
		BUG();
}

/* send a reschedule CPI to one CPU by physical CPU number*/
I
Ingo Molnar 已提交
1175
static void voyager_smp_send_reschedule(int cpu)
L
Linus Torvalds 已提交
1176 1177 1178 1179
{
	send_one_CPI(cpu, VIC_RESCHEDULE_CPI);
}

I
Ingo Molnar 已提交
1180
int hard_smp_processor_id(void)
L
Linus Torvalds 已提交
1181 1182 1183
{
	__u8 i;
	__u8 cpumask = inb(VIC_PROC_WHO_AM_I);
I
Ingo Molnar 已提交
1184
	if ((cpumask & QUAD_IDENTIFIER) == QUAD_IDENTIFIER)
L
Linus Torvalds 已提交
1185 1186
		return cpumask & 0x1F;

I
Ingo Molnar 已提交
1187 1188
	for (i = 0; i < 8; i++) {
		if (cpumask & (1 << i))
L
Linus Torvalds 已提交
1189 1190 1191 1192 1193 1194
			return i;
	}
	printk("** WARNING ** Illegal cpuid returned by VIC: %d", cpumask);
	return 0;
}

I
Ingo Molnar 已提交
1195
int safe_smp_processor_id(void)
1196 1197 1198 1199
{
	return hard_smp_processor_id();
}

L
Linus Torvalds 已提交
1200
/* broadcast a halt to all other CPUs */
I
Ingo Molnar 已提交
1201
static void voyager_smp_send_stop(void)
L
Linus Torvalds 已提交
1202 1203 1204 1205 1206 1207
{
	smp_call_function(smp_stop_cpu_function, NULL, 1, 1);
}

/* this function is triggered in time.c when a clock tick fires
 * we need to re-broadcast the tick to all CPUs */
I
Ingo Molnar 已提交
1208
void smp_vic_timer_interrupt(void)
L
Linus Torvalds 已提交
1209 1210
{
	send_CPI_allbutself(VIC_TIMER_CPI);
1211
	smp_local_timer_interrupt();
L
Linus Torvalds 已提交
1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
}

/* local (per CPU) timer interrupt.  It does both profiling and
 * process statistics/rescheduling.
 *
 * We do profiling in every local tick, statistics/rescheduling
 * happen only every 'profiling multiplier' ticks. The default
 * multiplier is 1 and it can be changed by writing the new multiplier
 * value into /proc/profile.
 */
I
Ingo Molnar 已提交
1222
void smp_local_timer_interrupt(void)
L
Linus Torvalds 已提交
1223 1224 1225 1226
{
	int cpu = smp_processor_id();
	long weight;

1227
	profile_tick(CPU_PROFILING);
L
Linus Torvalds 已提交
1228 1229 1230 1231 1232 1233 1234 1235 1236
	if (--per_cpu(prof_counter, cpu) <= 0) {
		/*
		 * The multiplier may have changed since the last time we got
		 * to this point as a result of the user writing to
		 * /proc/profile. In this case we need to adjust the APIC
		 * timer accordingly.
		 *
		 * Interrupts are already masked off at this point.
		 */
I
Ingo Molnar 已提交
1237
		per_cpu(prof_counter, cpu) = per_cpu(prof_multiplier, cpu);
L
Linus Torvalds 已提交
1238
		if (per_cpu(prof_counter, cpu) !=
I
Ingo Molnar 已提交
1239
		    per_cpu(prof_old_multiplier, cpu)) {
L
Linus Torvalds 已提交
1240 1241
			/* FIXME: need to update the vic timer tick here */
			per_cpu(prof_old_multiplier, cpu) =
I
Ingo Molnar 已提交
1242
			    per_cpu(prof_counter, cpu);
L
Linus Torvalds 已提交
1243 1244
		}

1245
		update_process_times(user_mode_vm(get_irq_regs()));
L
Linus Torvalds 已提交
1246 1247
	}

I
Ingo Molnar 已提交
1248
	if (((1 << cpu) & voyager_extended_vic_processors) == 0)
L
Linus Torvalds 已提交
1249 1250 1251 1252 1253 1254
		/* only extended VIC processors participate in
		 * interrupt distribution */
		return;

	/*
	 * We take the 'long' return path, and there every subsystem
S
Simon Arlott 已提交
1255
	 * grabs the appropriate locks (kernel lock/ irq lock).
L
Linus Torvalds 已提交
1256 1257 1258 1259 1260 1261 1262 1263
	 *
	 * we might want to decouple profiling from the 'long path',
	 * and do the profiling totally in assembly.
	 *
	 * Currently this isn't too much of an issue (performance wise),
	 * we can take more than 100K local irqs per second on a 100 MHz P5.
	 */

I
Ingo Molnar 已提交
1264
	if ((++vic_tick[cpu] & 0x7) != 0)
L
Linus Torvalds 已提交
1265 1266 1267 1268
		return;
	/* get here every 16 ticks (about every 1/6 of a second) */

	/* Change our priority to give someone else a chance at getting
I
Ingo Molnar 已提交
1269
	 * the IRQ. The algorithm goes like this:
L
Linus Torvalds 已提交
1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
	 *
	 * In the VIC, the dynamically routed interrupt is always
	 * handled by the lowest priority eligible (i.e. receiving
	 * interrupts) CPU.  If >1 eligible CPUs are equal lowest, the
	 * lowest processor number gets it.
	 *
	 * The priority of a CPU is controlled by a special per-CPU
	 * VIC priority register which is 3 bits wide 0 being lowest
	 * and 7 highest priority..
	 *
	 * Therefore we subtract the average number of interrupts from
	 * the number we've fielded.  If this number is negative, we
	 * lower the activity count and if it is positive, we raise
	 * it.
	 *
	 * I'm afraid this still leads to odd looking interrupt counts:
	 * the totals are all roughly equal, but the individual ones
	 * look rather skewed.
	 *
	 * FIXME: This algorithm is total crap when mixed with SMP
	 * affinity code since we now try to even up the interrupt
	 * counts when an affinity binding is keeping them on a
	 * particular CPU*/
I
Ingo Molnar 已提交
1293
	weight = (vic_intr_count[cpu] * voyager_extended_cpus
L
Linus Torvalds 已提交
1294 1295
		  - vic_intr_total) >> 4;
	weight += 4;
I
Ingo Molnar 已提交
1296
	if (weight > 7)
L
Linus Torvalds 已提交
1297
		weight = 7;
I
Ingo Molnar 已提交
1298
	if (weight < 0)
L
Linus Torvalds 已提交
1299
		weight = 0;
I
Ingo Molnar 已提交
1300 1301

	outb((__u8) weight, VIC_PRIORITY_REGISTER);
L
Linus Torvalds 已提交
1302 1303

#ifdef VOYAGER_DEBUG
I
Ingo Molnar 已提交
1304
	if ((vic_tick[cpu] & 0xFFF) == 0) {
L
Linus Torvalds 已提交
1305 1306 1307 1308 1309 1310 1311 1312
		/* print this message roughly every 25 secs */
		printk("VOYAGER SMP: vic_tick[%d] = %lu, weight = %ld\n",
		       cpu, vic_tick[cpu], weight);
	}
#endif
}

/* setup the profiling timer */
I
Ingo Molnar 已提交
1313
int setup_profiling_timer(unsigned int multiplier)
L
Linus Torvalds 已提交
1314 1315 1316
{
	int i;

I
Ingo Molnar 已提交
1317
	if ((!multiplier))
L
Linus Torvalds 已提交
1318 1319
		return -EINVAL;

I
Ingo Molnar 已提交
1320
	/*
L
Linus Torvalds 已提交
1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
	 * Set the new multiplier for each CPU. CPUs don't start using the
	 * new values until the next timer interrupt in which they do process
	 * accounting.
	 */
	for (i = 0; i < NR_CPUS; ++i)
		per_cpu(prof_multiplier, i) = multiplier;

	return 0;
}

J
James Bottomley 已提交
1331 1332 1333
/* This is a bit of a mess, but forced on us by the genirq changes
 * there's no genirq handler that really does what voyager wants
 * so hack it up with the simple IRQ handler */
1334
static void handle_vic_irq(unsigned int irq, struct irq_desc *desc)
J
James Bottomley 已提交
1335 1336 1337 1338 1339 1340
{
	before_handle_vic_irq(irq);
	handle_simple_irq(irq, desc);
	after_handle_vic_irq(irq);
}

L
Linus Torvalds 已提交
1341 1342 1343 1344 1345 1346 1347 1348 1349
/*  The CPIs are handled in the per cpu 8259s, so they must be
 *  enabled to be received: FIX: enabling the CPIs in the early
 *  boot sequence interferes with bug checking; enable them later
 *  on in smp_init */
#define VIC_SET_GATE(cpi, vector) \
	set_intr_gate((cpi) + VIC_DEFAULT_CPI_BASE, (vector))
#define QIC_SET_GATE(cpi, vector) \
	set_intr_gate((cpi) + QIC_DEFAULT_CPI_BASE, (vector))

I
Ingo Molnar 已提交
1350
void __init smp_intr_init(void)
L
Linus Torvalds 已提交
1351 1352 1353 1354
{
	int i;

	/* initialize the per cpu irq mask to all disabled */
I
Ingo Molnar 已提交
1355
	for (i = 0; i < NR_CPUS; i++)
L
Linus Torvalds 已提交
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
		vic_irq_mask[i] = 0xFFFF;

	VIC_SET_GATE(VIC_CPI_LEVEL0, vic_cpi_interrupt);

	VIC_SET_GATE(VIC_SYS_INT, vic_sys_interrupt);
	VIC_SET_GATE(VIC_CMN_INT, vic_cmn_interrupt);

	QIC_SET_GATE(QIC_TIMER_CPI, qic_timer_interrupt);
	QIC_SET_GATE(QIC_INVALIDATE_CPI, qic_invalidate_interrupt);
	QIC_SET_GATE(QIC_RESCHEDULE_CPI, qic_reschedule_interrupt);
	QIC_SET_GATE(QIC_ENABLE_IRQ_CPI, qic_enable_irq_interrupt);
	QIC_SET_GATE(QIC_CALL_FUNCTION_CPI, qic_call_function_interrupt);

I
Ingo Molnar 已提交
1369
	/* now put the VIC descriptor into the first 48 IRQs
L
Linus Torvalds 已提交
1370 1371 1372
	 *
	 * This is for later: first 16 correspond to PC IRQs; next 16
	 * are Primary MC IRQs and final 16 are Secondary MC IRQs */
I
Ingo Molnar 已提交
1373
	for (i = 0; i < 48; i++)
J
James Bottomley 已提交
1374
		set_irq_chip_and_handler(i, &vic_chip, handle_vic_irq);
L
Linus Torvalds 已提交
1375 1376 1377 1378
}

/* send a CPI at level cpi to a set of cpus in cpuset (set 1 bit per
 * processor to receive CPI */
I
Ingo Molnar 已提交
1379
static void send_CPI(__u32 cpuset, __u8 cpi)
L
Linus Torvalds 已提交
1380 1381 1382 1383
{
	int cpu;
	__u32 quad_cpuset = (cpuset & voyager_quad_processors);

I
Ingo Molnar 已提交
1384 1385
	if (cpi < VIC_START_FAKE_CPI) {
		/* fake CPI are only used for booting, so send to the
L
Linus Torvalds 已提交
1386
		 * extended quads as well---Quads must be VIC booted */
I
Ingo Molnar 已提交
1387
		outb((__u8) (cpuset), VIC_CPI_Registers[cpi]);
L
Linus Torvalds 已提交
1388 1389
		return;
	}
I
Ingo Molnar 已提交
1390
	if (quad_cpuset)
L
Linus Torvalds 已提交
1391 1392 1393
		send_QIC_CPI(quad_cpuset, cpi);
	cpuset &= ~quad_cpuset;
	cpuset &= 0xff;		/* only first 8 CPUs vaild for VIC CPI */
I
Ingo Molnar 已提交
1394
	if (cpuset == 0)
L
Linus Torvalds 已提交
1395 1396
		return;
	for_each_online_cpu(cpu) {
I
Ingo Molnar 已提交
1397
		if (cpuset & (1 << cpu))
L
Linus Torvalds 已提交
1398 1399
			set_bit(cpi, &vic_cpi_mailbox[cpu]);
	}
I
Ingo Molnar 已提交
1400 1401
	if (cpuset)
		outb((__u8) cpuset, VIC_CPI_Registers[VIC_CPI_LEVEL0]);
L
Linus Torvalds 已提交
1402 1403 1404 1405 1406 1407 1408 1409
}

/* Acknowledge receipt of CPI in the QIC, clear in QIC hardware and
 * set the cache line to shared by reading it.
 *
 * DON'T make this inline otherwise the cache line read will be
 * optimised away
 * */
I
Ingo Molnar 已提交
1410 1411
static int ack_QIC_CPI(__u8 cpi)
{
L
Linus Torvalds 已提交
1412 1413 1414 1415
	__u8 cpu = hard_smp_processor_id();

	cpi &= 7;

I
Ingo Molnar 已提交
1416
	outb(1 << cpi, QIC_INTERRUPT_CLEAR1);
L
Linus Torvalds 已提交
1417 1418 1419
	return voyager_quad_cpi_addr[cpu]->qic_cpi[cpi].cpi;
}

I
Ingo Molnar 已提交
1420
static void ack_special_QIC_CPI(__u8 cpi)
L
Linus Torvalds 已提交
1421
{
I
Ingo Molnar 已提交
1422
	switch (cpi) {
L
Linus Torvalds 已提交
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434
	case VIC_CMN_INT:
		outb(QIC_CMN_INT, QIC_INTERRUPT_CLEAR0);
		break;
	case VIC_SYS_INT:
		outb(QIC_SYS_INT, QIC_INTERRUPT_CLEAR0);
		break;
	}
	/* also clear at the VIC, just in case (nop for non-extended proc) */
	ack_VIC_CPI(cpi);
}

/* Acknowledge receipt of CPI in the VIC (essentially an EOI) */
I
Ingo Molnar 已提交
1435
static void ack_VIC_CPI(__u8 cpi)
L
Linus Torvalds 已提交
1436 1437 1438 1439 1440 1441 1442 1443
{
#ifdef VOYAGER_DEBUG
	unsigned long flags;
	__u16 isr;
	__u8 cpu = smp_processor_id();

	local_irq_save(flags);
	isr = vic_read_isr();
I
Ingo Molnar 已提交
1444
	if ((isr & (1 << (cpi & 7))) == 0) {
L
Linus Torvalds 已提交
1445 1446 1447 1448 1449 1450
		printk("VOYAGER SMP: CPU%d lost CPI%d\n", cpu, cpi);
	}
#endif
	/* send specific EOI; the two system interrupts have
	 * bit 4 set for a separate vector but behave as the
	 * corresponding 3 bit intr */
I
Ingo Molnar 已提交
1451
	outb_p(0x60 | (cpi & 7), 0x20);
L
Linus Torvalds 已提交
1452 1453

#ifdef VOYAGER_DEBUG
I
Ingo Molnar 已提交
1454
	if ((vic_read_isr() & (1 << (cpi & 7))) != 0) {
L
Linus Torvalds 已提交
1455 1456 1457 1458 1459 1460 1461
		printk("VOYAGER SMP: CPU%d still asserting CPI%d\n", cpu, cpi);
	}
	local_irq_restore(flags);
#endif
}

/* cribbed with thanks from irq.c */
I
Ingo Molnar 已提交
1462
#define __byte(x,y)	(((unsigned char *)&(y))[x])
L
Linus Torvalds 已提交
1463 1464 1465
#define cached_21(cpu)	(__byte(0,vic_irq_mask[cpu]))
#define cached_A1(cpu)	(__byte(1,vic_irq_mask[cpu]))

I
Ingo Molnar 已提交
1466
static unsigned int startup_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1467
{
J
James Bottomley 已提交
1468
	unmask_vic_irq(irq);
L
Linus Torvalds 已提交
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493

	return 0;
}

/* The enable and disable routines.  This is where we run into
 * conflicting architectural philosophy.  Fundamentally, the voyager
 * architecture does not expect to have to disable interrupts globally
 * (the IRQ controllers belong to each CPU).  The processor masquerade
 * which is used to start the system shouldn't be used in a running OS
 * since it will cause great confusion if two separate CPUs drive to
 * the same IRQ controller (I know, I've tried it).
 *
 * The solution is a variant on the NCR lazy SPL design:
 *
 * 1) To disable an interrupt, do nothing (other than set the
 *    IRQ_DISABLED flag).  This dares the interrupt actually to arrive.
 *
 * 2) If the interrupt dares to come in, raise the local mask against
 *    it (this will result in all the CPU masks being raised
 *    eventually).
 *
 * 3) To enable the interrupt, lower the mask on the local CPU and
 *    broadcast an Interrupt enable CPI which causes all other CPUs to
 *    adjust their masks accordingly.  */

I
Ingo Molnar 已提交
1494
static void unmask_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1495 1496 1497 1498
{
	/* linux doesn't to processor-irq affinity, so enable on
	 * all CPUs we know about */
	int cpu = smp_processor_id(), real_cpu;
I
Ingo Molnar 已提交
1499
	__u16 mask = (1 << irq);
L
Linus Torvalds 已提交
1500 1501 1502
	__u32 processorList = 0;
	unsigned long flags;

J
James Bottomley 已提交
1503
	VDEBUG(("VOYAGER: unmask_vic_irq(%d) CPU%d affinity 0x%lx\n",
L
Linus Torvalds 已提交
1504 1505 1506
		irq, cpu, cpu_irq_affinity[cpu]));
	spin_lock_irqsave(&vic_irq_lock, flags);
	for_each_online_cpu(real_cpu) {
I
Ingo Molnar 已提交
1507
		if (!(voyager_extended_vic_processors & (1 << real_cpu)))
L
Linus Torvalds 已提交
1508
			continue;
I
Ingo Molnar 已提交
1509
		if (!(cpu_irq_affinity[real_cpu] & mask)) {
L
Linus Torvalds 已提交
1510 1511 1512
			/* irq has no affinity for this CPU, ignore */
			continue;
		}
I
Ingo Molnar 已提交
1513
		if (real_cpu == cpu) {
L
Linus Torvalds 已提交
1514
			enable_local_vic_irq(irq);
I
Ingo Molnar 已提交
1515
		} else if (vic_irq_mask[real_cpu] & mask) {
L
Linus Torvalds 已提交
1516
			vic_irq_enable_mask[real_cpu] |= mask;
I
Ingo Molnar 已提交
1517
			processorList |= (1 << real_cpu);
L
Linus Torvalds 已提交
1518 1519 1520
		}
	}
	spin_unlock_irqrestore(&vic_irq_lock, flags);
I
Ingo Molnar 已提交
1521
	if (processorList)
L
Linus Torvalds 已提交
1522 1523 1524
		send_CPI(processorList, VIC_ENABLE_IRQ_CPI);
}

I
Ingo Molnar 已提交
1525
static void mask_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1526 1527 1528 1529
{
	/* lazy disable, do nothing */
}

I
Ingo Molnar 已提交
1530
static void enable_local_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1531 1532 1533 1534 1535 1536
{
	__u8 cpu = smp_processor_id();
	__u16 mask = ~(1 << irq);
	__u16 old_mask = vic_irq_mask[cpu];

	vic_irq_mask[cpu] &= mask;
I
Ingo Molnar 已提交
1537
	if (vic_irq_mask[cpu] == old_mask)
L
Linus Torvalds 已提交
1538 1539 1540 1541 1542 1543
		return;

	VDEBUG(("VOYAGER DEBUG: Enabling irq %d in hardware on CPU %d\n",
		irq, cpu));

	if (irq & 8) {
I
Ingo Molnar 已提交
1544
		outb_p(cached_A1(cpu), 0xA1);
L
Linus Torvalds 已提交
1545
		(void)inb_p(0xA1);
I
Ingo Molnar 已提交
1546 1547
	} else {
		outb_p(cached_21(cpu), 0x21);
L
Linus Torvalds 已提交
1548 1549 1550 1551
		(void)inb_p(0x21);
	}
}

I
Ingo Molnar 已提交
1552
static void disable_local_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1553 1554 1555 1556 1557
{
	__u8 cpu = smp_processor_id();
	__u16 mask = (1 << irq);
	__u16 old_mask = vic_irq_mask[cpu];

I
Ingo Molnar 已提交
1558
	if (irq == 7)
L
Linus Torvalds 已提交
1559 1560 1561
		return;

	vic_irq_mask[cpu] |= mask;
I
Ingo Molnar 已提交
1562
	if (old_mask == vic_irq_mask[cpu])
L
Linus Torvalds 已提交
1563 1564 1565 1566 1567 1568
		return;

	VDEBUG(("VOYAGER DEBUG: Disabling irq %d in hardware on CPU %d\n",
		irq, cpu));

	if (irq & 8) {
I
Ingo Molnar 已提交
1569
		outb_p(cached_A1(cpu), 0xA1);
L
Linus Torvalds 已提交
1570
		(void)inb_p(0xA1);
I
Ingo Molnar 已提交
1571 1572
	} else {
		outb_p(cached_21(cpu), 0x21);
L
Linus Torvalds 已提交
1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
		(void)inb_p(0x21);
	}
}

/* The VIC is level triggered, so the ack can only be issued after the
 * interrupt completes.  However, we do Voyager lazy interrupt
 * handling here: It is an extremely expensive operation to mask an
 * interrupt in the vic, so we merely set a flag (IRQ_DISABLED).  If
 * this interrupt actually comes in, then we mask and ack here to push
 * the interrupt off to another CPU */
I
Ingo Molnar 已提交
1583
static void before_handle_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1584 1585 1586 1587 1588 1589 1590 1591
{
	irq_desc_t *desc = irq_desc + irq;
	__u8 cpu = smp_processor_id();

	_raw_spin_lock(&vic_irq_lock);
	vic_intr_total++;
	vic_intr_count[cpu]++;

I
Ingo Molnar 已提交
1592
	if (!(cpu_irq_affinity[cpu] & (1 << irq))) {
L
Linus Torvalds 已提交
1593 1594
		/* The irq is not in our affinity mask, push it off
		 * onto another CPU */
I
Ingo Molnar 已提交
1595 1596
		VDEBUG(("VOYAGER DEBUG: affinity triggered disable of irq %d "
			"on cpu %d\n", irq, cpu));
L
Linus Torvalds 已提交
1597 1598 1599 1600
		disable_local_vic_irq(irq);
		/* set IRQ_INPROGRESS to prevent the handler in irq.c from
		 * actually calling the interrupt routine */
		desc->status |= IRQ_REPLAY | IRQ_INPROGRESS;
I
Ingo Molnar 已提交
1601
	} else if (desc->status & IRQ_DISABLED) {
L
Linus Torvalds 已提交
1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617
		/* Damn, the interrupt actually arrived, do the lazy
		 * disable thing. The interrupt routine in irq.c will
		 * not handle a IRQ_DISABLED interrupt, so nothing more
		 * need be done here */
		VDEBUG(("VOYAGER DEBUG: lazy disable of irq %d on CPU %d\n",
			irq, cpu));
		disable_local_vic_irq(irq);
		desc->status |= IRQ_REPLAY;
	} else {
		desc->status &= ~IRQ_REPLAY;
	}

	_raw_spin_unlock(&vic_irq_lock);
}

/* Finish the VIC interrupt: basically mask */
I
Ingo Molnar 已提交
1618
static void after_handle_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
{
	irq_desc_t *desc = irq_desc + irq;

	_raw_spin_lock(&vic_irq_lock);
	{
		unsigned int status = desc->status & ~IRQ_INPROGRESS;
#ifdef VOYAGER_DEBUG
		__u16 isr;
#endif

		desc->status = status;
		if ((status & IRQ_DISABLED))
			disable_local_vic_irq(irq);
#ifdef VOYAGER_DEBUG
		/* DEBUG: before we ack, check what's in progress */
		isr = vic_read_isr();
I
Ingo Molnar 已提交
1635
		if ((isr & (1 << irq) && !(status & IRQ_REPLAY)) == 0) {
L
Linus Torvalds 已提交
1636 1637 1638
			int i;
			__u8 cpu = smp_processor_id();
			__u8 real_cpu;
I
Ingo Molnar 已提交
1639
			int mask;	/* Um... initialize me??? --RR */
L
Linus Torvalds 已提交
1640 1641 1642

			printk("VOYAGER SMP: CPU%d lost interrupt %d\n",
			       cpu, irq);
1643
			for_each_possible_cpu(real_cpu, mask) {
L
Linus Torvalds 已提交
1644 1645 1646 1647

				outb(VIC_CPU_MASQUERADE_ENABLE | real_cpu,
				     VIC_PROCESSOR_ID);
				isr = vic_read_isr();
I
Ingo Molnar 已提交
1648 1649 1650 1651
				if (isr & (1 << irq)) {
					printk
					    ("VOYAGER SMP: CPU%d ack irq %d\n",
					     real_cpu, irq);
L
Linus Torvalds 已提交
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
					ack_vic_irq(irq);
				}
				outb(cpu, VIC_PROCESSOR_ID);
			}
		}
#endif /* VOYAGER_DEBUG */
		/* as soon as we ack, the interrupt is eligible for
		 * receipt by another CPU so everything must be in
		 * order here  */
		ack_vic_irq(irq);
I
Ingo Molnar 已提交
1662
		if (status & IRQ_REPLAY) {
L
Linus Torvalds 已提交
1663 1664 1665 1666 1667 1668 1669 1670
			/* replay is set if we disable the interrupt
			 * in the before_handle_vic_irq() routine, so
			 * clear the in progress bit here to allow the
			 * next CPU to handle this correctly */
			desc->status &= ~(IRQ_REPLAY | IRQ_INPROGRESS);
		}
#ifdef VOYAGER_DEBUG
		isr = vic_read_isr();
I
Ingo Molnar 已提交
1671 1672 1673
		if ((isr & (1 << irq)) != 0)
			printk("VOYAGER SMP: after_handle_vic_irq() after "
			       "ack irq=%d, isr=0x%x\n", irq, isr);
L
Linus Torvalds 已提交
1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697
#endif /* VOYAGER_DEBUG */
	}
	_raw_spin_unlock(&vic_irq_lock);

	/* All code after this point is out of the main path - the IRQ
	 * may be intercepted by another CPU if reasserted */
}

/* Linux processor - interrupt affinity manipulations.
 *
 * For each processor, we maintain a 32 bit irq affinity mask.
 * Initially it is set to all 1's so every processor accepts every
 * interrupt.  In this call, we change the processor's affinity mask:
 *
 * Change from enable to disable:
 *
 * If the interrupt ever comes in to the processor, we will disable it
 * and ack it to push it off to another CPU, so just accept the mask here.
 *
 * Change from disable to enable:
 *
 * change the mask and then do an interrupt enable CPI to re-enable on
 * the selected processors */

I
Ingo Molnar 已提交
1698
void set_vic_irq_affinity(unsigned int irq, cpumask_t mask)
L
Linus Torvalds 已提交
1699 1700 1701 1702 1703 1704 1705
{
	/* Only extended processors handle interrupts */
	unsigned long real_mask;
	unsigned long irq_mask = 1 << irq;
	int cpu;

	real_mask = cpus_addr(mask)[0] & voyager_extended_vic_processors;
I
Ingo Molnar 已提交
1706 1707

	if (cpus_addr(mask)[0] == 0)
S
Simon Arlott 已提交
1708
		/* can't have no CPUs to accept the interrupt -- extremely
L
Linus Torvalds 已提交
1709 1710 1711
		 * bad things will happen */
		return;

I
Ingo Molnar 已提交
1712
	if (irq == 0)
L
Linus Torvalds 已提交
1713 1714 1715 1716 1717 1718 1719 1720
		/* can't change the affinity of the timer IRQ.  This
		 * is due to the constraint in the voyager
		 * architecture that the CPI also comes in on and IRQ
		 * line and we have chosen IRQ0 for this.  If you
		 * raise the mask on this interrupt, the processor
		 * will no-longer be able to accept VIC CPIs */
		return;

I
Ingo Molnar 已提交
1721
	if (irq >= 32)
L
Linus Torvalds 已提交
1722 1723 1724 1725 1726 1727 1728
		/* You can only have 32 interrupts in a voyager system
		 * (and 32 only if you have a secondary microchannel
		 * bus) */
		return;

	for_each_online_cpu(cpu) {
		unsigned long cpu_mask = 1 << cpu;
I
Ingo Molnar 已提交
1729 1730

		if (cpu_mask & real_mask) {
L
Linus Torvalds 已提交
1731 1732 1733 1734 1735 1736 1737 1738 1739
			/* enable the interrupt for this cpu */
			cpu_irq_affinity[cpu] |= irq_mask;
		} else {
			/* disable the interrupt for this cpu */
			cpu_irq_affinity[cpu] &= ~irq_mask;
		}
	}
	/* this is magic, we now have the correct affinity maps, so
	 * enable the interrupt.  This will send an enable CPI to
S
Simon Arlott 已提交
1740
	 * those CPUs who need to enable it in their local masks,
L
Linus Torvalds 已提交
1741 1742 1743 1744 1745
	 * causing them to correct for the new affinity . If the
	 * interrupt is currently globally disabled, it will simply be
	 * disabled again as it comes in (voyager lazy disable).  If
	 * the affinity map is tightened to disable the interrupt on a
	 * cpu, it will be pushed off when it comes in */
J
James Bottomley 已提交
1746
	unmask_vic_irq(irq);
L
Linus Torvalds 已提交
1747 1748
}

I
Ingo Molnar 已提交
1749
static void ack_vic_irq(unsigned int irq)
L
Linus Torvalds 已提交
1750 1751
{
	if (irq & 8) {
I
Ingo Molnar 已提交
1752 1753
		outb(0x62, 0x20);	/* Specific EOI to cascade */
		outb(0x60 | (irq & 7), 0xA0);
L
Linus Torvalds 已提交
1754
	} else {
I
Ingo Molnar 已提交
1755
		outb(0x60 | (irq & 7), 0x20);
L
Linus Torvalds 已提交
1756 1757 1758 1759 1760 1761
	}
}

/* enable the CPIs.  In the VIC, the CPIs are delivered by the 8259
 * but are not vectored by it.  This means that the 8259 mask must be
 * lowered to receive them */
I
Ingo Molnar 已提交
1762
static __init void vic_enable_cpi(void)
L
Linus Torvalds 已提交
1763 1764
{
	__u8 cpu = smp_processor_id();
I
Ingo Molnar 已提交
1765

L
Linus Torvalds 已提交
1766 1767 1768 1769 1770 1771 1772 1773
	/* just take a copy of the current mask (nop for boot cpu) */
	vic_irq_mask[cpu] = vic_irq_mask[boot_cpu_id];

	enable_local_vic_irq(VIC_CPI_LEVEL0);
	enable_local_vic_irq(VIC_CPI_LEVEL1);
	/* for sys int and cmn int */
	enable_local_vic_irq(7);

I
Ingo Molnar 已提交
1774
	if (is_cpu_quad()) {
L
Linus Torvalds 已提交
1775 1776 1777 1778 1779 1780 1781 1782 1783 1784
		outb(QIC_DEFAULT_MASK0, QIC_MASK_REGISTER0);
		outb(QIC_CPI_ENABLE, QIC_MASK_REGISTER1);
		VDEBUG(("VOYAGER SMP: QIC ENABLE CPI: CPU%d: MASK 0x%x\n",
			cpu, QIC_CPI_ENABLE));
	}

	VDEBUG(("VOYAGER SMP: ENABLE CPI: CPU%d: MASK 0x%x\n",
		cpu, vic_irq_mask[cpu]));
}

I
Ingo Molnar 已提交
1785
void voyager_smp_dump()
L
Linus Torvalds 已提交
1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
{
	int old_cpu = smp_processor_id(), cpu;

	/* dump the interrupt masks of each processor */
	for_each_online_cpu(cpu) {
		__u16 imr, isr, irr;
		unsigned long flags;

		local_irq_save(flags);
		outb(VIC_CPU_MASQUERADE_ENABLE | cpu, VIC_PROCESSOR_ID);
		imr = (inb(0xa1) << 8) | inb(0x21);
		outb(0x0a, 0xa0);
		irr = inb(0xa0) << 8;
		outb(0x0a, 0x20);
		irr |= inb(0x20);
		outb(0x0b, 0xa0);
		isr = inb(0xa0) << 8;
		outb(0x0b, 0x20);
		isr |= inb(0x20);
		outb(old_cpu, VIC_PROCESSOR_ID);
		local_irq_restore(flags);
		printk("\tCPU%d: mask=0x%x, IMR=0x%x, IRR=0x%x, ISR=0x%x\n",
		       cpu, vic_irq_mask[cpu], imr, irr, isr);
#if 0
		/* These lines are put in to try to unstick an un ack'd irq */
I
Ingo Molnar 已提交
1811
		if (isr != 0) {
L
Linus Torvalds 已提交
1812
			int irq;
I
Ingo Molnar 已提交
1813 1814
			for (irq = 0; irq < 16; irq++) {
				if (isr & (1 << irq)) {
L
Linus Torvalds 已提交
1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829
					printk("\tCPU%d: ack irq %d\n",
					       cpu, irq);
					local_irq_save(flags);
					outb(VIC_CPU_MASQUERADE_ENABLE | cpu,
					     VIC_PROCESSOR_ID);
					ack_vic_irq(irq);
					outb(old_cpu, VIC_PROCESSOR_ID);
					local_irq_restore(flags);
				}
			}
		}
#endif
	}
}

I
Ingo Molnar 已提交
1830
void smp_voyager_power_off(void *dummy)
L
Linus Torvalds 已提交
1831
{
I
Ingo Molnar 已提交
1832
	if (smp_processor_id() == boot_cpu_id)
L
Linus Torvalds 已提交
1833 1834 1835 1836 1837
		voyager_power_off();
	else
		smp_stop_cpu_function(NULL);
}

I
Ingo Molnar 已提交
1838
static void __init voyager_smp_prepare_cpus(unsigned int max_cpus)
L
Linus Torvalds 已提交
1839 1840 1841 1842 1843
{
	/* FIXME: ignore max_cpus for now */
	smp_boot_cpus();
}

R
Randy Dunlap 已提交
1844
static void __cpuinit voyager_smp_prepare_boot_cpu(void)
L
Linus Torvalds 已提交
1845
{
J
Jeremy Fitzhardinge 已提交
1846 1847 1848
	init_gdt(smp_processor_id());
	switch_to_new_gdt();

L
Linus Torvalds 已提交
1849 1850
	cpu_set(smp_processor_id(), cpu_online_map);
	cpu_set(smp_processor_id(), cpu_callout_map);
1851
	cpu_set(smp_processor_id(), cpu_possible_map);
1852
	cpu_set(smp_processor_id(), cpu_present_map);
L
Linus Torvalds 已提交
1853 1854
}

I
Ingo Molnar 已提交
1855
static int __cpuinit voyager_cpu_up(unsigned int cpu)
L
Linus Torvalds 已提交
1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
{
	/* This only works at boot for x86.  See "rewrite" above. */
	if (cpu_isset(cpu, smp_commenced_mask))
		return -ENOSYS;

	/* In case one didn't come up */
	if (!cpu_isset(cpu, cpu_callin_map))
		return -EIO;
	/* Unleash the CPU! */
	cpu_set(cpu, smp_commenced_mask);
	while (!cpu_isset(cpu, cpu_online_map))
		mb();
	return 0;
}

I
Ingo Molnar 已提交
1871
static void __init voyager_smp_cpus_done(unsigned int max_cpus)
L
Linus Torvalds 已提交
1872 1873 1874
{
	zap_low_mappings();
}
1875

I
Ingo Molnar 已提交
1876
void __init smp_setup_processor_id(void)
1877 1878
{
	current_thread_info()->cpu = hard_smp_processor_id();
J
Jeremy Fitzhardinge 已提交
1879
	x86_write_percpu(cpu_number, hard_smp_processor_id());
1880
}
J
Jeremy Fitzhardinge 已提交
1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891

struct smp_ops smp_ops = {
	.smp_prepare_boot_cpu = voyager_smp_prepare_boot_cpu,
	.smp_prepare_cpus = voyager_smp_prepare_cpus,
	.cpu_up = voyager_cpu_up,
	.smp_cpus_done = voyager_smp_cpus_done,

	.smp_send_stop = voyager_smp_send_stop,
	.smp_send_reschedule = voyager_smp_send_reschedule,
	.smp_call_function_mask = voyager_smp_call_function_mask,
};