smp.c 9.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 *
 * Copyright (C) 2000, 2001 Kanoj Sarcar
 * Copyright (C) 2000, 2001 Ralf Baechle
 * Copyright (C) 2000, 2001 Silicon Graphics, Inc.
 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation
 */
#include <linux/cache.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/threads.h>
#include <linux/module.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <linux/sched.h>
#include <linux/cpumask.h>
R
Rojhalat Ibrahim 已提交
32
#include <linux/cpu.h>
A
Alexey Dobriyan 已提交
33
#include <linux/err.h>
L
Linus Torvalds 已提交
34 35 36 37

#include <asm/atomic.h>
#include <asm/cpu.h>
#include <asm/processor.h>
38
#include <asm/r4k-timer.h>
L
Linus Torvalds 已提交
39 40
#include <asm/system.h>
#include <asm/mmu_context.h>
41
#include <asm/time.h>
L
Linus Torvalds 已提交
42

43 44 45 46
#ifdef CONFIG_MIPS_MT_SMTC
#include <asm/mipsmtregs.h>
#endif /* CONFIG_MIPS_MT_SMTC */

L
Linus Torvalds 已提交
47 48 49 50 51 52 53 54 55
cpumask_t phys_cpu_present_map;		/* Bitmask of available CPUs */
volatile cpumask_t cpu_callin_map;	/* Bitmask of started secondaries */
cpumask_t cpu_online_map;		/* Bitmask of currently online CPUs */
int __cpu_number_map[NR_CPUS];		/* Map physical to logical */
int __cpu_logical_map[NR_CPUS];		/* Map logical to physical */

EXPORT_SYMBOL(phys_cpu_present_map);
EXPORT_SYMBOL(cpu_online_map);

56
extern void cpu_idle(void);
L
Linus Torvalds 已提交
57

58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
/* Number of TCs (or siblings in Intel speak) per CPU core */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* representing the TCs (or siblings in Intel speak) of each logical CPU */
cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(cpu_sibling_map);

/* representing cpus for which sibling maps can be computed */
static cpumask_t cpu_sibling_setup_map;

static inline void set_cpu_sibling_map(int cpu)
{
	int i;

	cpu_set(cpu, cpu_sibling_setup_map);

	if (smp_num_siblings > 1) {
		for_each_cpu_mask(i, cpu_sibling_setup_map) {
			if (cpu_data[cpu].core == cpu_data[i].core) {
				cpu_set(i, cpu_sibling_map[cpu]);
				cpu_set(cpu, cpu_sibling_map[i]);
			}
		}
	} else
		cpu_set(cpu, cpu_sibling_map[cpu]);
}

86 87 88 89
struct plat_smp_ops *mp_ops;

__cpuinit void register_smp_ops(struct plat_smp_ops *ops)
{
90 91
	if (mp_ops)
		printk(KERN_WARNING "Overriding previously set SMP ops\n");
92 93 94 95

	mp_ops = ops;
}

L
Linus Torvalds 已提交
96 97 98 99
/*
 * First C code run on the secondary CPUs after being started up by
 * the master.
 */
100
asmlinkage __cpuinit void start_secondary(void)
L
Linus Torvalds 已提交
101
{
102
	unsigned int cpu;
L
Linus Torvalds 已提交
103

104 105 106 107
#ifdef CONFIG_MIPS_MT_SMTC
	/* Only do cpu_probe for first TC of CPU */
	if ((read_c0_tcbind() & TCBIND_CURTC) == 0)
#endif /* CONFIG_MIPS_MT_SMTC */
L
Linus Torvalds 已提交
108 109 110
	cpu_probe();
	cpu_report();
	per_cpu_trap_init();
111
	mips_clockevent_init();
112
	mp_ops->init_secondary();
L
Linus Torvalds 已提交
113 114 115 116 117 118 119

	/*
	 * XXX parity protection should be folded in here when it's converted
	 * to an option instead of something based on .cputype
	 */

	calibrate_delay();
120 121
	preempt_disable();
	cpu = smp_processor_id();
L
Linus Torvalds 已提交
122 123
	cpu_data[cpu].udelay_val = loops_per_jiffy;

124
	mp_ops->smp_finish();
125
	set_cpu_sibling_map(cpu);
L
Linus Torvalds 已提交
126 127 128

	cpu_set(cpu, cpu_callin_map);

129 130
	synchronise_count_slave();

L
Linus Torvalds 已提交
131 132 133
	cpu_idle();
}

134
void arch_send_call_function_ipi(cpumask_t mask)
L
Linus Torvalds 已提交
135
{
136
	mp_ops->send_ipi_mask(mask, SMP_CALL_FUNCTION);
L
Linus Torvalds 已提交
137 138
}

139 140 141 142
/*
 * We reuse the same vector for the single IPI
 */
void arch_send_call_function_single_ipi(int cpu)
143
{
144
	mp_ops->send_ipi_mask(cpumask_of_cpu(cpu), SMP_CALL_FUNCTION);
145
}
146

147 148 149
/*
 * Call into both interrupt handlers, as we share the IPI for them
 */
L
Linus Torvalds 已提交
150 151 152
void smp_call_function_interrupt(void)
{
	irq_enter();
153 154
	generic_smp_call_function_single_interrupt();
	generic_smp_call_function_interrupt();
L
Linus Torvalds 已提交
155
	irq_exit();
156 157
}

L
Linus Torvalds 已提交
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
static void stop_this_cpu(void *dummy)
{
	/*
	 * Remove this CPU:
	 */
	cpu_clear(smp_processor_id(), cpu_online_map);
	local_irq_enable();	/* May need to service _machine_restart IPI */
	for (;;);		/* Wait if available. */
}

void smp_send_stop(void)
{
	smp_call_function(stop_this_cpu, NULL, 1, 0);
}

void __init smp_cpus_done(unsigned int max_cpus)
{
175
	mp_ops->cpus_done();
176
	synchronise_count_master();
L
Linus Torvalds 已提交
177 178 179 180 181 182 183
}

/* called from main before smp_init() */
void __init smp_prepare_cpus(unsigned int max_cpus)
{
	init_new_context(current, &init_mm);
	current_thread_info()->cpu = 0;
184
	mp_ops->prepare_cpus(max_cpus);
185
	set_cpu_sibling_map(0);
186 187 188
#ifndef CONFIG_HOTPLUG_CPU
	cpu_present_map = cpu_possible_map;
#endif
L
Linus Torvalds 已提交
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
}

/* preload SMP state for boot cpu */
void __devinit smp_prepare_boot_cpu(void)
{
	/*
	 * This assumes that bootup is always handled by the processor
	 * with the logic and physical number 0.
	 */
	__cpu_number_map[0] = 0;
	__cpu_logical_map[0] = 0;
	cpu_set(0, phys_cpu_present_map);
	cpu_set(0, cpu_online_map);
	cpu_set(0, cpu_callin_map);
}

/*
206 207 208
 * Called once for each "cpu_possible(cpu)".  Needs to spin up the cpu
 * and keep control until "cpu_online(cpu)" is set.  Note: cpu is
 * physical, not logical.
L
Linus Torvalds 已提交
209
 */
210
int __cpuinit __cpu_up(unsigned int cpu)
L
Linus Torvalds 已提交
211 212 213 214
{
	struct task_struct *idle;

	/*
215
	 * Processor goes to start_secondary(), sets online flag
L
Linus Torvalds 已提交
216 217 218 219 220
	 * The following code is purely to make sure
	 * Linux can schedule processes on this slave.
	 */
	idle = fork_idle(cpu);
	if (IS_ERR(idle))
221
		panic(KERN_ERR "Fork failed for CPU %d", cpu);
L
Linus Torvalds 已提交
222

223
	mp_ops->boot_secondary(cpu, idle);
L
Linus Torvalds 已提交
224

225 226 227
	/*
	 * Trust is futile.  We should really have timeouts ...
	 */
L
Linus Torvalds 已提交
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
	while (!cpu_isset(cpu, cpu_callin_map))
		udelay(100);

	cpu_set(cpu, cpu_online_map);

	return 0;
}

/* Not really SMP stuff ... */
int setup_profiling_timer(unsigned int multiplier)
{
	return 0;
}

static void flush_tlb_all_ipi(void *info)
{
	local_flush_tlb_all();
}

void flush_tlb_all(void)
{
249
	on_each_cpu(flush_tlb_all_ipi, NULL, 1, 1);
L
Linus Torvalds 已提交
250 251 252 253 254 255 256
}

static void flush_tlb_mm_ipi(void *mm)
{
	local_flush_tlb_mm((struct mm_struct *)mm);
}

257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
/*
 * Special Variant of smp_call_function for use by TLB functions:
 *
 *  o No return value
 *  o collapses to normal function call on UP kernels
 *  o collapses to normal function call on systems with a single shared
 *    primary cache.
 *  o CONFIG_MIPS_MT_SMTC currently implies there is only one physical core.
 */
static inline void smp_on_other_tlbs(void (*func) (void *info), void *info)
{
#ifndef CONFIG_MIPS_MT_SMTC
	smp_call_function(func, info, 1, 1);
#endif
}

static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
{
	preempt_disable();

	smp_on_other_tlbs(func, info);
	func(info);

	preempt_enable();
}

L
Linus Torvalds 已提交
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
/*
 * The following tlb flush calls are invoked when old translations are
 * being torn down, or pte attributes are changing. For single threaded
 * address spaces, a new context is obtained on the current cpu, and tlb
 * context on other cpus are invalidated to force a new context allocation
 * at switch_mm time, should the mm ever be used on other cpus. For
 * multithreaded address spaces, intercpu interrupts have to be sent.
 * Another case where intercpu interrupts are required is when the target
 * mm might be active on another cpu (eg debuggers doing the flushes on
 * behalf of debugees, kswapd stealing pages from another process etc).
 * Kanoj 07/00.
 */

void flush_tlb_mm(struct mm_struct *mm)
{
	preempt_disable();

	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
R
Ralf Baechle 已提交
301
		smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
L
Linus Torvalds 已提交
302
	} else {
303 304 305 306
		cpumask_t mask = cpu_online_map;
		unsigned int cpu;

		cpu_clear(smp_processor_id(), mask);
R
Ralf Baechle 已提交
307
		for_each_cpu_mask(cpu, mask)
308 309
			if (cpu_context(cpu, mm))
				cpu_context(cpu, mm) = 0;
L
Linus Torvalds 已提交
310 311 312 313 314 315 316 317 318 319 320 321 322 323
	}
	local_flush_tlb_mm(mm);

	preempt_enable();
}

struct flush_tlb_data {
	struct vm_area_struct *vma;
	unsigned long addr1;
	unsigned long addr2;
};

static void flush_tlb_range_ipi(void *info)
{
R
Ralf Baechle 已提交
324
	struct flush_tlb_data *fd = info;
L
Linus Torvalds 已提交
325 326 327 328 329 330 331 332 333 334

	local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
}

void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
	struct mm_struct *mm = vma->vm_mm;

	preempt_disable();
	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
335 336 337 338 339
		struct flush_tlb_data fd = {
			.vma = vma,
			.addr1 = start,
			.addr2 = end,
		};
L
Linus Torvalds 已提交
340

R
Ralf Baechle 已提交
341
		smp_on_other_tlbs(flush_tlb_range_ipi, &fd);
L
Linus Torvalds 已提交
342
	} else {
343 344 345 346
		cpumask_t mask = cpu_online_map;
		unsigned int cpu;

		cpu_clear(smp_processor_id(), mask);
R
Ralf Baechle 已提交
347
		for_each_cpu_mask(cpu, mask)
348 349
			if (cpu_context(cpu, mm))
				cpu_context(cpu, mm) = 0;
L
Linus Torvalds 已提交
350 351 352 353 354 355 356
	}
	local_flush_tlb_range(vma, start, end);
	preempt_enable();
}

static void flush_tlb_kernel_range_ipi(void *info)
{
R
Ralf Baechle 已提交
357
	struct flush_tlb_data *fd = info;
L
Linus Torvalds 已提交
358 359 360 361 362 363

	local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
}

void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
364 365 366 367
	struct flush_tlb_data fd = {
		.addr1 = start,
		.addr2 = end,
	};
L
Linus Torvalds 已提交
368

R
Ralf Baechle 已提交
369
	on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1, 1);
L
Linus Torvalds 已提交
370 371 372 373
}

static void flush_tlb_page_ipi(void *info)
{
R
Ralf Baechle 已提交
374
	struct flush_tlb_data *fd = info;
L
Linus Torvalds 已提交
375 376 377 378 379 380 381 382

	local_flush_tlb_page(fd->vma, fd->addr1);
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
	preempt_disable();
	if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
383 384 385 386
		struct flush_tlb_data fd = {
			.vma = vma,
			.addr1 = page,
		};
L
Linus Torvalds 已提交
387

R
Ralf Baechle 已提交
388
		smp_on_other_tlbs(flush_tlb_page_ipi, &fd);
L
Linus Torvalds 已提交
389
	} else {
390 391 392 393
		cpumask_t mask = cpu_online_map;
		unsigned int cpu;

		cpu_clear(smp_processor_id(), mask);
R
Ralf Baechle 已提交
394
		for_each_cpu_mask(cpu, mask)
395 396
			if (cpu_context(cpu, vma->vm_mm))
				cpu_context(cpu, vma->vm_mm) = 0;
L
Linus Torvalds 已提交
397 398 399 400 401 402 403 404 405 406 407 408 409 410
	}
	local_flush_tlb_page(vma, page);
	preempt_enable();
}

static void flush_tlb_one_ipi(void *info)
{
	unsigned long vaddr = (unsigned long) info;

	local_flush_tlb_one(vaddr);
}

void flush_tlb_one(unsigned long vaddr)
{
411
	smp_on_each_tlb(flush_tlb_one_ipi, (void *) vaddr);
L
Linus Torvalds 已提交
412 413 414 415
}

EXPORT_SYMBOL(flush_tlb_page);
EXPORT_SYMBOL(flush_tlb_one);