smp.c 17.7 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * SMP initialisation and IPI support
 * Based on arch/arm/kernel/smp.c
 *
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/acpi.h>
C
Catalin Marinas 已提交
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/seq_file.h>
#include <linux/irq.h>
#include <linux/percpu.h>
#include <linux/clockchips.h>
#include <linux/completion.h>
#include <linux/of.h>
39
#include <linux/irq_work.h>
C
Catalin Marinas 已提交
40

41
#include <asm/alternative.h>
C
Catalin Marinas 已提交
42 43
#include <asm/atomic.h>
#include <asm/cacheflush.h>
44
#include <asm/cpu.h>
C
Catalin Marinas 已提交
45
#include <asm/cputype.h>
M
Mark Rutland 已提交
46
#include <asm/cpu_ops.h>
C
Catalin Marinas 已提交
47 48 49 50
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/processor.h>
51
#include <asm/smp_plat.h>
C
Catalin Marinas 已提交
52 53 54
#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/ptrace.h>
55
#include <asm/virt.h>
C
Catalin Marinas 已提交
56

N
Nicolas Pitre 已提交
57 58 59
#define CREATE_TRACE_POINTS
#include <trace/events/ipi.h>

C
Catalin Marinas 已提交
60 61 62 63 64 65 66 67 68 69 70
/*
 * as from 2.5, kernels no longer have an init_tasks structure
 * so we need some other way of telling a new secondary core
 * where to place its SVC stack
 */
struct secondary_data secondary_data;

enum ipi_msg_type {
	IPI_RESCHEDULE,
	IPI_CALL_FUNC,
	IPI_CPU_STOP,
71
	IPI_TIMER,
72
	IPI_IRQ_WORK,
C
Catalin Marinas 已提交
73 74 75 76 77 78
};

/*
 * Boot a secondary CPU, and assign it the specified idle task.
 * This also gives us the initial stack to use for this CPU.
 */
79
static int boot_secondary(unsigned int cpu, struct task_struct *idle)
C
Catalin Marinas 已提交
80
{
81 82
	if (cpu_ops[cpu]->cpu_boot)
		return cpu_ops[cpu]->cpu_boot(cpu);
C
Catalin Marinas 已提交
83

84
	return -EOPNOTSUPP;
C
Catalin Marinas 已提交
85 86 87 88
}

static DECLARE_COMPLETION(cpu_running);

89
int __cpu_up(unsigned int cpu, struct task_struct *idle)
C
Catalin Marinas 已提交
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
{
	int ret;

	/*
	 * We need to tell the secondary core where to find its stack and the
	 * page tables.
	 */
	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
	__flush_dcache_area(&secondary_data, sizeof(secondary_data));

	/*
	 * Now bring the CPU into our world.
	 */
	ret = boot_secondary(cpu, idle);
	if (ret == 0) {
		/*
		 * CPU was successfully started, wait for it to come online or
		 * time out.
		 */
		wait_for_completion_timeout(&cpu_running,
					    msecs_to_jiffies(1000));

		if (!cpu_online(cpu)) {
			pr_crit("CPU%u: failed to come online\n", cpu);
			ret = -EIO;
		}
	} else {
		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
	}

	secondary_data.stack = NULL;

	return ret;
}

125 126 127 128 129
static void smp_store_cpu_info(unsigned int cpuid)
{
	store_cpu_topology(cpuid);
}

C
Catalin Marinas 已提交
130 131 132 133
/*
 * This is the secondary CPU boot entry.  We're using this CPUs
 * idle thread stack, but a set of temporary page tables.
 */
134
asmlinkage void secondary_start_kernel(void)
C
Catalin Marinas 已提交
135 136 137 138 139 140 141 142 143 144 145
{
	struct mm_struct *mm = &init_mm;
	unsigned int cpu = smp_processor_id();

	/*
	 * All kernel threads share the same mm context; grab a
	 * reference and switch to it.
	 */
	atomic_inc(&mm->mm_count);
	current->active_mm = mm;

146 147
	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));

C
Catalin Marinas 已提交
148 149 150 151 152
	/*
	 * TTBR0 is only used for the identity mapping at this stage. Make it
	 * point to zero page to avoid speculatively fetching new entries.
	 */
	cpu_set_reserved_ttbr0();
153
	local_flush_tlb_all();
154
	cpu_set_default_tcr_t0sz();
C
Catalin Marinas 已提交
155 156 157 158

	preempt_disable();
	trace_hardirqs_off();

159 160 161 162 163 164 165
	/*
	 * If the system has established the capabilities, make sure
	 * this CPU ticks all of those. If it doesn't, the CPU will
	 * fail to come online.
	 */
	verify_local_cpu_capabilities();

166 167
	if (cpu_ops[cpu]->cpu_postboot)
		cpu_ops[cpu]->cpu_postboot();
C
Catalin Marinas 已提交
168

169 170 171 172 173
	/*
	 * Log the CPU info before it is marked online and might get read.
	 */
	cpuinfo_store_cpu();

174 175 176 177 178
	/*
	 * Enable GIC and timers.
	 */
	notify_cpu_starting(cpu);

179 180
	smp_store_cpu_info(cpu);

C
Catalin Marinas 已提交
181 182 183 184 185
	/*
	 * OK, now it's safe to let the boot CPU continue.  Wait for
	 * the CPU migration code to notice that the CPU is online
	 * before we continue.
	 */
186 187
	pr_info("CPU%u: Booted secondary processor [%08x]\n",
					 cpu, read_cpuid_id());
C
Catalin Marinas 已提交
188
	set_cpu_online(cpu, true);
189
	complete(&cpu_running);
C
Catalin Marinas 已提交
190

191
	local_dbg_enable();
192
	local_irq_enable();
193
	local_async_enable();
194

C
Catalin Marinas 已提交
195 196 197
	/*
	 * OK, it's off to the idle thread for us
	 */
T
Thomas Gleixner 已提交
198
	cpu_startup_entry(CPUHP_ONLINE);
C
Catalin Marinas 已提交
199 200
}

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
#ifdef CONFIG_HOTPLUG_CPU
static int op_cpu_disable(unsigned int cpu)
{
	/*
	 * If we don't have a cpu_die method, abort before we reach the point
	 * of no return. CPU0 may not have an cpu_ops, so test for it.
	 */
	if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_die)
		return -EOPNOTSUPP;

	/*
	 * We may need to abort a hot unplug for some other mechanism-specific
	 * reason.
	 */
	if (cpu_ops[cpu]->cpu_disable)
		return cpu_ops[cpu]->cpu_disable(cpu);

	return 0;
}

/*
 * __cpu_disable runs on the processor to be shutdown.
 */
int __cpu_disable(void)
{
	unsigned int cpu = smp_processor_id();
	int ret;

	ret = op_cpu_disable(cpu);
	if (ret)
		return ret;

	/*
	 * Take this CPU offline.  Once we clear this, we can't return,
	 * and we must not schedule until we're ready to give up the cpu.
	 */
	set_cpu_online(cpu, false);

	/*
	 * OK - migrate IRQs away from this CPU
	 */
242 243
	irq_migrate_all_off_this_cpu();

244 245 246
	return 0;
}

247 248 249 250 251 252 253 254
static int op_cpu_kill(unsigned int cpu)
{
	/*
	 * If we have no means of synchronising with the dying CPU, then assume
	 * that it is really dead. We can only wait for an arbitrary length of
	 * time and hope that it's dead, so let's skip the wait and just hope.
	 */
	if (!cpu_ops[cpu]->cpu_kill)
255
		return 0;
256 257 258 259

	return cpu_ops[cpu]->cpu_kill(cpu);
}

260 261 262 263 264 265
/*
 * called on the thread which is asking for a CPU to be shutdown -
 * waits until shutdown has completed, or it is timed out.
 */
void __cpu_die(unsigned int cpu)
{
266 267
	int err;

268
	if (!cpu_wait_death(cpu, 5)) {
269 270 271 272
		pr_crit("CPU%u: cpu didn't die\n", cpu);
		return;
	}
	pr_notice("CPU%u: shutdown\n", cpu);
273 274 275 276 277 278 279

	/*
	 * Now that the dying CPU is beyond the point of no return w.r.t.
	 * in-kernel synchronisation, try to get the firwmare to help us to
	 * verify that it has really left the kernel before we consider
	 * clobbering anything it might still be using.
	 */
280 281 282 283
	err = op_cpu_kill(cpu);
	if (err)
		pr_warn("CPU%d may not have shut down cleanly: %d\n",
			cpu, err);
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
}

/*
 * Called from the idle thread for the CPU which has been shutdown.
 *
 * Note that we disable IRQs here, but do not re-enable them
 * before returning to the caller. This is also the behaviour
 * of the other hotplug-cpu capable cores, so presumably coming
 * out of idle fixes this.
 */
void cpu_die(void)
{
	unsigned int cpu = smp_processor_id();

	idle_task_exit();

	local_irq_disable();

	/* Tell __cpu_die() that this CPU is now safe to dispose of */
303
	(void)cpu_report_death();
304 305 306 307 308 309 310 311 312 313 314 315

	/*
	 * Actually shutdown the CPU. This must never fail. The specific hotplug
	 * mechanism must perform all required cache maintenance to ensure that
	 * no dirty lines are lost in the process of shutting down the CPU.
	 */
	cpu_ops[cpu]->cpu_die(cpu);

	BUG();
}
#endif

316 317 318 319 320 321 322 323 324 325 326
static void __init hyp_mode_check(void)
{
	if (is_hyp_mode_available())
		pr_info("CPU: All CPU(s) started at EL2\n");
	else if (is_hyp_mode_mismatched())
		WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
			   "CPU: CPUs started in inconsistent modes");
	else
		pr_info("CPU: All CPU(s) started at EL1\n");
}

C
Catalin Marinas 已提交
327 328
void __init smp_cpus_done(unsigned int max_cpus)
{
329
	pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
330
	setup_cpu_features();
331 332
	hyp_mode_check();
	apply_alternatives_all();
C
Catalin Marinas 已提交
333 334 335 336
}

void __init smp_prepare_boot_cpu(void)
{
337
	cpuinfo_store_boot_cpu();
338
	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
C
Catalin Marinas 已提交
339 340
}

341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
static u64 __init of_get_cpu_mpidr(struct device_node *dn)
{
	const __be32 *cell;
	u64 hwid;

	/*
	 * A cpu node with missing "reg" property is
	 * considered invalid to build a cpu_logical_map
	 * entry.
	 */
	cell = of_get_property(dn, "reg", NULL);
	if (!cell) {
		pr_err("%s: missing reg property\n", dn->full_name);
		return INVALID_HWID;
	}

	hwid = of_read_number(cell, of_n_addr_cells(dn));
	/*
	 * Non affinity bits must be set to 0 in the DT
	 */
	if (hwid & ~MPIDR_HWID_BITMASK) {
		pr_err("%s: invalid reg property\n", dn->full_name);
		return INVALID_HWID;
	}
	return hwid;
}

/*
 * Duplicate MPIDRs are a recipe for disaster. Scan all initialized
 * entries and check for duplicates. If any is found just ignore the
 * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid
 * matching valid MPIDR values.
 */
static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid)
{
	unsigned int i;

	for (i = 1; (i < cpu) && (i < NR_CPUS); i++)
		if (cpu_logical_map(i) == hwid)
			return true;
	return false;
}

384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
/*
 * Initialize cpu operations for a logical cpu and
 * set it in the possible mask on success
 */
static int __init smp_cpu_setup(int cpu)
{
	if (cpu_read_ops(cpu))
		return -ENODEV;

	if (cpu_ops[cpu]->cpu_init(cpu))
		return -ENODEV;

	set_cpu_possible(cpu, true);

	return 0;
}

401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
static bool bootcpu_valid __initdata;
static unsigned int cpu_count = 1;

#ifdef CONFIG_ACPI
/*
 * acpi_map_gic_cpu_interface - parse processor MADT entry
 *
 * Carry out sanity checks on MADT processor entry and initialize
 * cpu_logical_map on success
 */
static void __init
acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
{
	u64 hwid = processor->arm_mpidr;

416 417
	if (!(processor->flags & ACPI_MADT_ENABLED)) {
		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
418 419 420
		return;
	}

421 422
	if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) {
		pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid);
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
		return;
	}

	if (is_mpidr_duplicate(cpu_count, hwid)) {
		pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);
		return;
	}

	/* Check if GICC structure of boot CPU is available in the MADT */
	if (cpu_logical_map(0) == hwid) {
		if (bootcpu_valid) {
			pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n",
			       hwid);
			return;
		}
		bootcpu_valid = true;
		return;
	}

	if (cpu_count >= NR_CPUS)
		return;

	/* map the logical cpu id to cpu MPIDR */
	cpu_logical_map(cpu_count) = hwid;

	cpu_count++;
}

static int __init
acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header,
			     const unsigned long end)
{
	struct acpi_madt_generic_interrupt *processor;

	processor = (struct acpi_madt_generic_interrupt *)header;
458
	if (BAD_MADT_GICC_ENTRY(processor, end))
459 460 461 462 463 464 465 466 467 468 469 470
		return -EINVAL;

	acpi_table_print_madt_entry(header);

	acpi_map_gic_cpu_interface(processor);

	return 0;
}
#else
#define acpi_table_parse_madt(...)	do { } while (0)
#endif

C
Catalin Marinas 已提交
471
/*
472 473 474
 * Enumerate the possible CPU set from the device tree and build the
 * cpu logical map array containing MPIDR values related to logical
 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
C
Catalin Marinas 已提交
475
 */
476
static void __init of_parse_and_init_cpus(void)
C
Catalin Marinas 已提交
477 478 479 480
{
	struct device_node *dn = NULL;

	while ((dn = of_find_node_by_type(dn, "cpu"))) {
481
		u64 hwid = of_get_cpu_mpidr(dn);
482

483
		if (hwid == INVALID_HWID)
484 485
			goto next;

486 487 488
		if (is_mpidr_duplicate(cpu_count, hwid)) {
			pr_err("%s: duplicate cpu reg properties in the DT\n",
				dn->full_name);
489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
			goto next;
		}

		/*
		 * The numbering scheme requires that the boot CPU
		 * must be assigned logical id 0. Record it so that
		 * the logical map built from DT is validated and can
		 * be used.
		 */
		if (hwid == cpu_logical_map(0)) {
			if (bootcpu_valid) {
				pr_err("%s: duplicate boot cpu reg property in DT\n",
					dn->full_name);
				goto next;
			}

			bootcpu_valid = true;

			/*
			 * cpu_logical_map has already been
			 * initialized and the boot cpu doesn't need
			 * the enable-method so continue without
			 * incrementing cpu.
			 */
			continue;
		}

516
		if (cpu_count >= NR_CPUS)
C
Catalin Marinas 已提交
517 518
			goto next;

519
		pr_debug("cpu logical map 0x%llx\n", hwid);
520
		cpu_logical_map(cpu_count) = hwid;
C
Catalin Marinas 已提交
521
next:
522
		cpu_count++;
C
Catalin Marinas 已提交
523
	}
524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
}

/*
 * Enumerate the possible CPU set from the device tree or ACPI and build the
 * cpu logical map array containing MPIDR values related to logical
 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
 */
void __init smp_init_cpus(void)
{
	int i;

	if (acpi_disabled)
		of_parse_and_init_cpus();
	else
		/*
		 * do a walk of MADT to determine how many CPUs
		 * we have including disabled CPUs, and get information
		 * we need for SMP init
		 */
		acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
				      acpi_parse_gic_cpu_interface, 0);
C
Catalin Marinas 已提交
545

546 547 548
	if (cpu_count > NR_CPUS)
		pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
			cpu_count, NR_CPUS);
549 550

	if (!bootcpu_valid) {
551
		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
552 553 554 555
		return;
	}

	/*
556 557 558 559 560
	 * We need to set the cpu_logical_map entries before enabling
	 * the cpus so that cpu processor description entries (DT cpu nodes
	 * and ACPI MADT entries) can be retrieved by matching the cpu hwid
	 * with entries in cpu_logical_map while initializing the cpus.
	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
561
	 */
562 563 564 565 566 567
	for (i = 1; i < NR_CPUS; i++) {
		if (cpu_logical_map(i) != INVALID_HWID) {
			if (smp_cpu_setup(i))
				cpu_logical_map(i) = INVALID_HWID;
		}
	}
C
Catalin Marinas 已提交
568 569 570 571
}

void __init smp_prepare_cpus(unsigned int max_cpus)
{
M
Mark Rutland 已提交
572 573
	int err;
	unsigned int cpu, ncores = num_possible_cpus();
C
Catalin Marinas 已提交
574

575 576 577 578
	init_cpu_topology();

	smp_store_cpu_info(smp_processor_id());

C
Catalin Marinas 已提交
579 580 581 582 583 584
	/*
	 * are we trying to boot more cores than exist?
	 */
	if (max_cpus > ncores)
		max_cpus = ncores;

585 586 587 588
	/* Don't bother if we're effectively UP */
	if (max_cpus <= 1)
		return;

C
Catalin Marinas 已提交
589 590 591 592
	/*
	 * Initialise the present map (which describes the set of CPUs
	 * actually populated at the present time) and release the
	 * secondaries from the bootloader.
593 594
	 *
	 * Make sure we online at most (max_cpus - 1) additional CPUs.
C
Catalin Marinas 已提交
595
	 */
596
	max_cpus--;
C
Catalin Marinas 已提交
597 598 599 600
	for_each_possible_cpu(cpu) {
		if (max_cpus == 0)
			break;

601 602 603
		if (cpu == smp_processor_id())
			continue;

M
Mark Rutland 已提交
604
		if (!cpu_ops[cpu])
C
Catalin Marinas 已提交
605 606
			continue;

M
Mark Rutland 已提交
607
		err = cpu_ops[cpu]->cpu_prepare(cpu);
608 609
		if (err)
			continue;
C
Catalin Marinas 已提交
610 611 612 613 614 615

		set_cpu_present(cpu, true);
		max_cpus--;
	}
}

616
void (*__smp_cross_call)(const struct cpumask *, unsigned int);
C
Catalin Marinas 已提交
617 618 619

void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
{
N
Nicolas Pitre 已提交
620
	__smp_cross_call = fn;
C
Catalin Marinas 已提交
621 622
}

N
Nicolas Pitre 已提交
623 624
static const char *ipi_types[NR_IPI] __tracepoint_string = {
#define S(x,s)	[x] = s
C
Catalin Marinas 已提交
625 626 627
	S(IPI_RESCHEDULE, "Rescheduling interrupts"),
	S(IPI_CALL_FUNC, "Function call interrupts"),
	S(IPI_CPU_STOP, "CPU stop interrupts"),
628
	S(IPI_TIMER, "Timer broadcast interrupts"),
629
	S(IPI_IRQ_WORK, "IRQ work interrupts"),
C
Catalin Marinas 已提交
630 631
};

N
Nicolas Pitre 已提交
632 633 634 635 636 637
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
{
	trace_ipi_raise(target, ipi_types[ipinr]);
	__smp_cross_call(target, ipinr);
}

C
Catalin Marinas 已提交
638 639 640 641 642
void show_ipi_list(struct seq_file *p, int prec)
{
	unsigned int cpu, i;

	for (i = 0; i < NR_IPI; i++) {
N
Nicolas Pitre 已提交
643
		seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
C
Catalin Marinas 已提交
644
			   prec >= 4 ? " " : "");
645
		for_each_online_cpu(cpu)
C
Catalin Marinas 已提交
646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
			seq_printf(p, "%10u ",
				   __get_irq_stat(cpu, ipi_irqs[i]));
		seq_printf(p, "      %s\n", ipi_types[i]);
	}
}

u64 smp_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = 0;
	int i;

	for (i = 0; i < NR_IPI; i++)
		sum += __get_irq_stat(cpu, ipi_irqs[i]);

	return sum;
}

N
Nicolas Pitre 已提交
663 664 665 666 667 668 669
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_CALL_FUNC);
}

void arch_send_call_function_single_ipi(int cpu)
{
670
	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
N
Nicolas Pitre 已提交
671 672 673 674 675 676 677 678 679 680
}

#ifdef CONFIG_IRQ_WORK
void arch_irq_work_raise(void)
{
	if (__smp_cross_call)
		smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
}
#endif

C
Catalin Marinas 已提交
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
static DEFINE_RAW_SPINLOCK(stop_lock);

/*
 * ipi_cpu_stop - handle IPI from smp_send_stop()
 */
static void ipi_cpu_stop(unsigned int cpu)
{
	if (system_state == SYSTEM_BOOTING ||
	    system_state == SYSTEM_RUNNING) {
		raw_spin_lock(&stop_lock);
		pr_crit("CPU%u: stopping\n", cpu);
		dump_stack();
		raw_spin_unlock(&stop_lock);
	}

	set_cpu_online(cpu, false);

	local_irq_disable();

	while (1)
		cpu_relax();
}

/*
 * Main handler for inter-processor interrupts
 */
void handle_IPI(int ipinr, struct pt_regs *regs)
{
	unsigned int cpu = smp_processor_id();
	struct pt_regs *old_regs = set_irq_regs(regs);

N
Nicolas Pitre 已提交
712
	if ((unsigned)ipinr < NR_IPI) {
713
		trace_ipi_entry_rcuidle(ipi_types[ipinr]);
N
Nicolas Pitre 已提交
714 715
		__inc_irq_stat(cpu, ipi_irqs[ipinr]);
	}
C
Catalin Marinas 已提交
716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733

	switch (ipinr) {
	case IPI_RESCHEDULE:
		scheduler_ipi();
		break;

	case IPI_CALL_FUNC:
		irq_enter();
		generic_smp_call_function_interrupt();
		irq_exit();
		break;

	case IPI_CPU_STOP:
		irq_enter();
		ipi_cpu_stop(cpu);
		irq_exit();
		break;

734 735 736 737 738 739 740 741
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
	case IPI_TIMER:
		irq_enter();
		tick_receive_broadcast();
		irq_exit();
		break;
#endif

742 743 744 745 746 747 748 749
#ifdef CONFIG_IRQ_WORK
	case IPI_IRQ_WORK:
		irq_enter();
		irq_work_run();
		irq_exit();
		break;
#endif

C
Catalin Marinas 已提交
750 751 752 753
	default:
		pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
		break;
	}
N
Nicolas Pitre 已提交
754 755

	if ((unsigned)ipinr < NR_IPI)
756
		trace_ipi_exit_rcuidle(ipi_types[ipinr]);
C
Catalin Marinas 已提交
757 758 759 760 761 762 763 764
	set_irq_regs(old_regs);
}

void smp_send_reschedule(int cpu)
{
	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
}

765 766 767 768 769 770 771
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
void tick_broadcast(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_TIMER);
}
#endif

C
Catalin Marinas 已提交
772 773 774 775 776 777 778 779
void smp_send_stop(void)
{
	unsigned long timeout;

	if (num_online_cpus() > 1) {
		cpumask_t mask;

		cpumask_copy(&mask, cpu_online_mask);
780
		cpumask_clear_cpu(smp_processor_id(), &mask);
C
Catalin Marinas 已提交
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800

		smp_cross_call(&mask, IPI_CPU_STOP);
	}

	/* Wait up to one second for other CPUs to stop */
	timeout = USEC_PER_SEC;
	while (num_online_cpus() > 1 && timeout--)
		udelay(1);

	if (num_online_cpus() > 1)
		pr_warning("SMP: failed to stop secondary CPUs\n");
}

/*
 * not supported here
 */
int setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}