smp.c 18.9 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * SMP initialisation and IPI support
 * Based on arch/arm/kernel/smp.c
 *
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/acpi.h>
C
Catalin Marinas 已提交
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/seq_file.h>
#include <linux/irq.h>
#include <linux/percpu.h>
#include <linux/clockchips.h>
#include <linux/completion.h>
#include <linux/of.h>
39
#include <linux/irq_work.h>
C
Catalin Marinas 已提交
40

41
#include <asm/alternative.h>
C
Catalin Marinas 已提交
42 43
#include <asm/atomic.h>
#include <asm/cacheflush.h>
44
#include <asm/cpu.h>
C
Catalin Marinas 已提交
45
#include <asm/cputype.h>
M
Mark Rutland 已提交
46
#include <asm/cpu_ops.h>
C
Catalin Marinas 已提交
47 48 49 50
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/processor.h>
51
#include <asm/smp_plat.h>
C
Catalin Marinas 已提交
52 53 54
#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/ptrace.h>
55
#include <asm/virt.h>
C
Catalin Marinas 已提交
56

N
Nicolas Pitre 已提交
57 58 59
#define CREATE_TRACE_POINTS
#include <trace/events/ipi.h>

C
Catalin Marinas 已提交
60 61 62 63 64 65 66 67 68 69 70
/*
 * as from 2.5, kernels no longer have an init_tasks structure
 * so we need some other way of telling a new secondary core
 * where to place its SVC stack
 */
struct secondary_data secondary_data;

enum ipi_msg_type {
	IPI_RESCHEDULE,
	IPI_CALL_FUNC,
	IPI_CPU_STOP,
71
	IPI_TIMER,
72
	IPI_IRQ_WORK,
73
	IPI_WAKEUP
C
Catalin Marinas 已提交
74 75 76 77 78 79
};

/*
 * Boot a secondary CPU, and assign it the specified idle task.
 * This also gives us the initial stack to use for this CPU.
 */
80
static int boot_secondary(unsigned int cpu, struct task_struct *idle)
C
Catalin Marinas 已提交
81
{
82 83
	if (cpu_ops[cpu]->cpu_boot)
		return cpu_ops[cpu]->cpu_boot(cpu);
C
Catalin Marinas 已提交
84

85
	return -EOPNOTSUPP;
C
Catalin Marinas 已提交
86 87 88 89
}

static DECLARE_COMPLETION(cpu_running);

90
int __cpu_up(unsigned int cpu, struct task_struct *idle)
C
Catalin Marinas 已提交
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
{
	int ret;

	/*
	 * We need to tell the secondary core where to find its stack and the
	 * page tables.
	 */
	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
	__flush_dcache_area(&secondary_data, sizeof(secondary_data));

	/*
	 * Now bring the CPU into our world.
	 */
	ret = boot_secondary(cpu, idle);
	if (ret == 0) {
		/*
		 * CPU was successfully started, wait for it to come online or
		 * time out.
		 */
		wait_for_completion_timeout(&cpu_running,
					    msecs_to_jiffies(1000));

		if (!cpu_online(cpu)) {
			pr_crit("CPU%u: failed to come online\n", cpu);
			ret = -EIO;
		}
	} else {
		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
	}

	secondary_data.stack = NULL;

	return ret;
}

126 127 128 129 130
static void smp_store_cpu_info(unsigned int cpuid)
{
	store_cpu_topology(cpuid);
}

C
Catalin Marinas 已提交
131 132 133 134
/*
 * This is the secondary CPU boot entry.  We're using this CPUs
 * idle thread stack, but a set of temporary page tables.
 */
135
asmlinkage void secondary_start_kernel(void)
C
Catalin Marinas 已提交
136 137 138 139 140 141 142 143 144 145 146
{
	struct mm_struct *mm = &init_mm;
	unsigned int cpu = smp_processor_id();

	/*
	 * All kernel threads share the same mm context; grab a
	 * reference and switch to it.
	 */
	atomic_inc(&mm->mm_count);
	current->active_mm = mm;

147 148
	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));

C
Catalin Marinas 已提交
149 150 151 152
	/*
	 * TTBR0 is only used for the identity mapping at this stage. Make it
	 * point to zero page to avoid speculatively fetching new entries.
	 */
M
Mark Rutland 已提交
153
	cpu_uninstall_idmap();
C
Catalin Marinas 已提交
154 155 156 157

	preempt_disable();
	trace_hardirqs_off();

158 159 160 161 162 163 164
	/*
	 * If the system has established the capabilities, make sure
	 * this CPU ticks all of those. If it doesn't, the CPU will
	 * fail to come online.
	 */
	verify_local_cpu_capabilities();

165 166
	if (cpu_ops[cpu]->cpu_postboot)
		cpu_ops[cpu]->cpu_postboot();
C
Catalin Marinas 已提交
167

168 169 170 171 172
	/*
	 * Log the CPU info before it is marked online and might get read.
	 */
	cpuinfo_store_cpu();

173 174 175 176 177
	/*
	 * Enable GIC and timers.
	 */
	notify_cpu_starting(cpu);

178 179
	smp_store_cpu_info(cpu);

C
Catalin Marinas 已提交
180 181 182 183 184
	/*
	 * OK, now it's safe to let the boot CPU continue.  Wait for
	 * the CPU migration code to notice that the CPU is online
	 * before we continue.
	 */
185 186
	pr_info("CPU%u: Booted secondary processor [%08x]\n",
					 cpu, read_cpuid_id());
C
Catalin Marinas 已提交
187
	set_cpu_online(cpu, true);
188
	complete(&cpu_running);
C
Catalin Marinas 已提交
189

190
	local_dbg_enable();
191
	local_irq_enable();
192
	local_async_enable();
193

C
Catalin Marinas 已提交
194 195 196
	/*
	 * OK, it's off to the idle thread for us
	 */
T
Thomas Gleixner 已提交
197
	cpu_startup_entry(CPUHP_ONLINE);
C
Catalin Marinas 已提交
198 199
}

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
#ifdef CONFIG_HOTPLUG_CPU
static int op_cpu_disable(unsigned int cpu)
{
	/*
	 * If we don't have a cpu_die method, abort before we reach the point
	 * of no return. CPU0 may not have an cpu_ops, so test for it.
	 */
	if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_die)
		return -EOPNOTSUPP;

	/*
	 * We may need to abort a hot unplug for some other mechanism-specific
	 * reason.
	 */
	if (cpu_ops[cpu]->cpu_disable)
		return cpu_ops[cpu]->cpu_disable(cpu);

	return 0;
}

/*
 * __cpu_disable runs on the processor to be shutdown.
 */
int __cpu_disable(void)
{
	unsigned int cpu = smp_processor_id();
	int ret;

	ret = op_cpu_disable(cpu);
	if (ret)
		return ret;

	/*
	 * Take this CPU offline.  Once we clear this, we can't return,
	 * and we must not schedule until we're ready to give up the cpu.
	 */
	set_cpu_online(cpu, false);

	/*
	 * OK - migrate IRQs away from this CPU
	 */
241 242
	irq_migrate_all_off_this_cpu();

243 244 245
	return 0;
}

246 247 248 249 250 251 252 253
static int op_cpu_kill(unsigned int cpu)
{
	/*
	 * If we have no means of synchronising with the dying CPU, then assume
	 * that it is really dead. We can only wait for an arbitrary length of
	 * time and hope that it's dead, so let's skip the wait and just hope.
	 */
	if (!cpu_ops[cpu]->cpu_kill)
254
		return 0;
255 256 257 258

	return cpu_ops[cpu]->cpu_kill(cpu);
}

259 260 261 262 263 264
/*
 * called on the thread which is asking for a CPU to be shutdown -
 * waits until shutdown has completed, or it is timed out.
 */
void __cpu_die(unsigned int cpu)
{
265 266
	int err;

267
	if (!cpu_wait_death(cpu, 5)) {
268 269 270 271
		pr_crit("CPU%u: cpu didn't die\n", cpu);
		return;
	}
	pr_notice("CPU%u: shutdown\n", cpu);
272 273 274 275 276 277 278

	/*
	 * Now that the dying CPU is beyond the point of no return w.r.t.
	 * in-kernel synchronisation, try to get the firwmare to help us to
	 * verify that it has really left the kernel before we consider
	 * clobbering anything it might still be using.
	 */
279 280 281 282
	err = op_cpu_kill(cpu);
	if (err)
		pr_warn("CPU%d may not have shut down cleanly: %d\n",
			cpu, err);
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
}

/*
 * Called from the idle thread for the CPU which has been shutdown.
 *
 * Note that we disable IRQs here, but do not re-enable them
 * before returning to the caller. This is also the behaviour
 * of the other hotplug-cpu capable cores, so presumably coming
 * out of idle fixes this.
 */
void cpu_die(void)
{
	unsigned int cpu = smp_processor_id();

	idle_task_exit();

	local_irq_disable();

	/* Tell __cpu_die() that this CPU is now safe to dispose of */
302
	(void)cpu_report_death();
303 304 305 306 307 308 309 310 311 312 313 314

	/*
	 * Actually shutdown the CPU. This must never fail. The specific hotplug
	 * mechanism must perform all required cache maintenance to ensure that
	 * no dirty lines are lost in the process of shutting down the CPU.
	 */
	cpu_ops[cpu]->cpu_die(cpu);

	BUG();
}
#endif

315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
/*
 * Kill the calling secondary CPU, early in bringup before it is turned
 * online.
 */
void cpu_die_early(void)
{
	int cpu = smp_processor_id();

	pr_crit("CPU%d: will not boot\n", cpu);

	/* Mark this CPU absent */
	set_cpu_present(cpu, 0);

#ifdef CONFIG_HOTPLUG_CPU
	/* Check if we can park ourselves */
	if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die)
		cpu_ops[cpu]->cpu_die(cpu);
#endif

	cpu_park_loop();
}

337 338 339 340 341 342 343 344 345 346 347
static void __init hyp_mode_check(void)
{
	if (is_hyp_mode_available())
		pr_info("CPU: All CPU(s) started at EL2\n");
	else if (is_hyp_mode_mismatched())
		WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
			   "CPU: CPUs started in inconsistent modes");
	else
		pr_info("CPU: All CPU(s) started at EL1\n");
}

C
Catalin Marinas 已提交
348 349
void __init smp_cpus_done(unsigned int max_cpus)
{
350
	pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
351
	setup_cpu_features();
352 353
	hyp_mode_check();
	apply_alternatives_all();
C
Catalin Marinas 已提交
354 355 356 357
}

void __init smp_prepare_boot_cpu(void)
{
358
	cpuinfo_store_boot_cpu();
359
	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
C
Catalin Marinas 已提交
360 361
}

362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
static u64 __init of_get_cpu_mpidr(struct device_node *dn)
{
	const __be32 *cell;
	u64 hwid;

	/*
	 * A cpu node with missing "reg" property is
	 * considered invalid to build a cpu_logical_map
	 * entry.
	 */
	cell = of_get_property(dn, "reg", NULL);
	if (!cell) {
		pr_err("%s: missing reg property\n", dn->full_name);
		return INVALID_HWID;
	}

	hwid = of_read_number(cell, of_n_addr_cells(dn));
	/*
	 * Non affinity bits must be set to 0 in the DT
	 */
	if (hwid & ~MPIDR_HWID_BITMASK) {
		pr_err("%s: invalid reg property\n", dn->full_name);
		return INVALID_HWID;
	}
	return hwid;
}

/*
 * Duplicate MPIDRs are a recipe for disaster. Scan all initialized
 * entries and check for duplicates. If any is found just ignore the
 * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid
 * matching valid MPIDR values.
 */
static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid)
{
	unsigned int i;

	for (i = 1; (i < cpu) && (i < NR_CPUS); i++)
		if (cpu_logical_map(i) == hwid)
			return true;
	return false;
}

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
/*
 * Initialize cpu operations for a logical cpu and
 * set it in the possible mask on success
 */
static int __init smp_cpu_setup(int cpu)
{
	if (cpu_read_ops(cpu))
		return -ENODEV;

	if (cpu_ops[cpu]->cpu_init(cpu))
		return -ENODEV;

	set_cpu_possible(cpu, true);

	return 0;
}

422 423 424 425 426 427 428 429 430 431 432 433 434 435 436
static bool bootcpu_valid __initdata;
static unsigned int cpu_count = 1;

#ifdef CONFIG_ACPI
/*
 * acpi_map_gic_cpu_interface - parse processor MADT entry
 *
 * Carry out sanity checks on MADT processor entry and initialize
 * cpu_logical_map on success
 */
static void __init
acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
{
	u64 hwid = processor->arm_mpidr;

437 438
	if (!(processor->flags & ACPI_MADT_ENABLED)) {
		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
439 440 441
		return;
	}

442 443
	if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) {
		pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid);
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
		return;
	}

	if (is_mpidr_duplicate(cpu_count, hwid)) {
		pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);
		return;
	}

	/* Check if GICC structure of boot CPU is available in the MADT */
	if (cpu_logical_map(0) == hwid) {
		if (bootcpu_valid) {
			pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n",
			       hwid);
			return;
		}
		bootcpu_valid = true;
		return;
	}

	if (cpu_count >= NR_CPUS)
		return;

	/* map the logical cpu id to cpu MPIDR */
	cpu_logical_map(cpu_count) = hwid;

469 470 471 472 473 474 475 476 477 478 479
	/*
	 * Set-up the ACPI parking protocol cpu entries
	 * while initializing the cpu_logical_map to
	 * avoid parsing MADT entries multiple times for
	 * nothing (ie a valid cpu_logical_map entry should
	 * contain a valid parking protocol data set to
	 * initialize the cpu if the parking protocol is
	 * the only available enable method).
	 */
	acpi_set_mailbox_entry(cpu_count, processor);

480 481 482 483 484 485 486 487 488 489
	cpu_count++;
}

static int __init
acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header,
			     const unsigned long end)
{
	struct acpi_madt_generic_interrupt *processor;

	processor = (struct acpi_madt_generic_interrupt *)header;
490
	if (BAD_MADT_GICC_ENTRY(processor, end))
491 492 493 494 495 496 497 498 499 500 501 502
		return -EINVAL;

	acpi_table_print_madt_entry(header);

	acpi_map_gic_cpu_interface(processor);

	return 0;
}
#else
#define acpi_table_parse_madt(...)	do { } while (0)
#endif

C
Catalin Marinas 已提交
503
/*
504 505 506
 * Enumerate the possible CPU set from the device tree and build the
 * cpu logical map array containing MPIDR values related to logical
 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
C
Catalin Marinas 已提交
507
 */
508
static void __init of_parse_and_init_cpus(void)
C
Catalin Marinas 已提交
509 510 511 512
{
	struct device_node *dn = NULL;

	while ((dn = of_find_node_by_type(dn, "cpu"))) {
513
		u64 hwid = of_get_cpu_mpidr(dn);
514

515
		if (hwid == INVALID_HWID)
516 517
			goto next;

518 519 520
		if (is_mpidr_duplicate(cpu_count, hwid)) {
			pr_err("%s: duplicate cpu reg properties in the DT\n",
				dn->full_name);
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
			goto next;
		}

		/*
		 * The numbering scheme requires that the boot CPU
		 * must be assigned logical id 0. Record it so that
		 * the logical map built from DT is validated and can
		 * be used.
		 */
		if (hwid == cpu_logical_map(0)) {
			if (bootcpu_valid) {
				pr_err("%s: duplicate boot cpu reg property in DT\n",
					dn->full_name);
				goto next;
			}

			bootcpu_valid = true;

			/*
			 * cpu_logical_map has already been
			 * initialized and the boot cpu doesn't need
			 * the enable-method so continue without
			 * incrementing cpu.
			 */
			continue;
		}

548
		if (cpu_count >= NR_CPUS)
C
Catalin Marinas 已提交
549 550
			goto next;

551
		pr_debug("cpu logical map 0x%llx\n", hwid);
552
		cpu_logical_map(cpu_count) = hwid;
C
Catalin Marinas 已提交
553
next:
554
		cpu_count++;
C
Catalin Marinas 已提交
555
	}
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
}

/*
 * Enumerate the possible CPU set from the device tree or ACPI and build the
 * cpu logical map array containing MPIDR values related to logical
 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
 */
void __init smp_init_cpus(void)
{
	int i;

	if (acpi_disabled)
		of_parse_and_init_cpus();
	else
		/*
		 * do a walk of MADT to determine how many CPUs
		 * we have including disabled CPUs, and get information
		 * we need for SMP init
		 */
		acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
				      acpi_parse_gic_cpu_interface, 0);
C
Catalin Marinas 已提交
577

578 579 580
	if (cpu_count > NR_CPUS)
		pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
			cpu_count, NR_CPUS);
581 582

	if (!bootcpu_valid) {
583
		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
584 585 586 587
		return;
	}

	/*
588 589 590 591 592
	 * We need to set the cpu_logical_map entries before enabling
	 * the cpus so that cpu processor description entries (DT cpu nodes
	 * and ACPI MADT entries) can be retrieved by matching the cpu hwid
	 * with entries in cpu_logical_map while initializing the cpus.
	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
593
	 */
594 595 596 597 598 599
	for (i = 1; i < NR_CPUS; i++) {
		if (cpu_logical_map(i) != INVALID_HWID) {
			if (smp_cpu_setup(i))
				cpu_logical_map(i) = INVALID_HWID;
		}
	}
C
Catalin Marinas 已提交
600 601 602 603
}

void __init smp_prepare_cpus(unsigned int max_cpus)
{
M
Mark Rutland 已提交
604 605
	int err;
	unsigned int cpu, ncores = num_possible_cpus();
C
Catalin Marinas 已提交
606

607 608 609 610
	init_cpu_topology();

	smp_store_cpu_info(smp_processor_id());

C
Catalin Marinas 已提交
611 612 613 614 615 616
	/*
	 * are we trying to boot more cores than exist?
	 */
	if (max_cpus > ncores)
		max_cpus = ncores;

617 618 619 620
	/* Don't bother if we're effectively UP */
	if (max_cpus <= 1)
		return;

C
Catalin Marinas 已提交
621 622 623 624
	/*
	 * Initialise the present map (which describes the set of CPUs
	 * actually populated at the present time) and release the
	 * secondaries from the bootloader.
625 626
	 *
	 * Make sure we online at most (max_cpus - 1) additional CPUs.
C
Catalin Marinas 已提交
627
	 */
628
	max_cpus--;
C
Catalin Marinas 已提交
629 630 631 632
	for_each_possible_cpu(cpu) {
		if (max_cpus == 0)
			break;

633 634 635
		if (cpu == smp_processor_id())
			continue;

M
Mark Rutland 已提交
636
		if (!cpu_ops[cpu])
C
Catalin Marinas 已提交
637 638
			continue;

M
Mark Rutland 已提交
639
		err = cpu_ops[cpu]->cpu_prepare(cpu);
640 641
		if (err)
			continue;
C
Catalin Marinas 已提交
642 643 644 645 646 647

		set_cpu_present(cpu, true);
		max_cpus--;
	}
}

648
void (*__smp_cross_call)(const struct cpumask *, unsigned int);
C
Catalin Marinas 已提交
649 650 651

void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
{
N
Nicolas Pitre 已提交
652
	__smp_cross_call = fn;
C
Catalin Marinas 已提交
653 654
}

N
Nicolas Pitre 已提交
655 656
static const char *ipi_types[NR_IPI] __tracepoint_string = {
#define S(x,s)	[x] = s
C
Catalin Marinas 已提交
657 658 659
	S(IPI_RESCHEDULE, "Rescheduling interrupts"),
	S(IPI_CALL_FUNC, "Function call interrupts"),
	S(IPI_CPU_STOP, "CPU stop interrupts"),
660
	S(IPI_TIMER, "Timer broadcast interrupts"),
661
	S(IPI_IRQ_WORK, "IRQ work interrupts"),
662
	S(IPI_WAKEUP, "CPU wake-up interrupts"),
C
Catalin Marinas 已提交
663 664
};

N
Nicolas Pitre 已提交
665 666 667 668 669 670
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
{
	trace_ipi_raise(target, ipi_types[ipinr]);
	__smp_cross_call(target, ipinr);
}

C
Catalin Marinas 已提交
671 672 673 674 675
void show_ipi_list(struct seq_file *p, int prec)
{
	unsigned int cpu, i;

	for (i = 0; i < NR_IPI; i++) {
N
Nicolas Pitre 已提交
676
		seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
C
Catalin Marinas 已提交
677
			   prec >= 4 ? " " : "");
678
		for_each_online_cpu(cpu)
C
Catalin Marinas 已提交
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
			seq_printf(p, "%10u ",
				   __get_irq_stat(cpu, ipi_irqs[i]));
		seq_printf(p, "      %s\n", ipi_types[i]);
	}
}

u64 smp_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = 0;
	int i;

	for (i = 0; i < NR_IPI; i++)
		sum += __get_irq_stat(cpu, ipi_irqs[i]);

	return sum;
}

N
Nicolas Pitre 已提交
696 697 698 699 700 701 702
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_CALL_FUNC);
}

void arch_send_call_function_single_ipi(int cpu)
{
703
	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
N
Nicolas Pitre 已提交
704 705
}

706 707 708 709 710 711 712
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_WAKEUP);
}
#endif

N
Nicolas Pitre 已提交
713 714 715 716 717 718 719 720
#ifdef CONFIG_IRQ_WORK
void arch_irq_work_raise(void)
{
	if (__smp_cross_call)
		smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
}
#endif

C
Catalin Marinas 已提交
721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
static DEFINE_RAW_SPINLOCK(stop_lock);

/*
 * ipi_cpu_stop - handle IPI from smp_send_stop()
 */
static void ipi_cpu_stop(unsigned int cpu)
{
	if (system_state == SYSTEM_BOOTING ||
	    system_state == SYSTEM_RUNNING) {
		raw_spin_lock(&stop_lock);
		pr_crit("CPU%u: stopping\n", cpu);
		dump_stack();
		raw_spin_unlock(&stop_lock);
	}

	set_cpu_online(cpu, false);

	local_irq_disable();

	while (1)
		cpu_relax();
}

/*
 * Main handler for inter-processor interrupts
 */
void handle_IPI(int ipinr, struct pt_regs *regs)
{
	unsigned int cpu = smp_processor_id();
	struct pt_regs *old_regs = set_irq_regs(regs);

N
Nicolas Pitre 已提交
752
	if ((unsigned)ipinr < NR_IPI) {
753
		trace_ipi_entry_rcuidle(ipi_types[ipinr]);
N
Nicolas Pitre 已提交
754 755
		__inc_irq_stat(cpu, ipi_irqs[ipinr]);
	}
C
Catalin Marinas 已提交
756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773

	switch (ipinr) {
	case IPI_RESCHEDULE:
		scheduler_ipi();
		break;

	case IPI_CALL_FUNC:
		irq_enter();
		generic_smp_call_function_interrupt();
		irq_exit();
		break;

	case IPI_CPU_STOP:
		irq_enter();
		ipi_cpu_stop(cpu);
		irq_exit();
		break;

774 775 776 777 778 779 780 781
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
	case IPI_TIMER:
		irq_enter();
		tick_receive_broadcast();
		irq_exit();
		break;
#endif

782 783 784 785 786 787 788 789
#ifdef CONFIG_IRQ_WORK
	case IPI_IRQ_WORK:
		irq_enter();
		irq_work_run();
		irq_exit();
		break;
#endif

790 791 792 793 794 795 796 797
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
	case IPI_WAKEUP:
		WARN_ONCE(!acpi_parking_protocol_valid(cpu),
			  "CPU%u: Wake-up IPI outside the ACPI parking protocol\n",
			  cpu);
		break;
#endif

C
Catalin Marinas 已提交
798 799 800 801
	default:
		pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
		break;
	}
N
Nicolas Pitre 已提交
802 803

	if ((unsigned)ipinr < NR_IPI)
804
		trace_ipi_exit_rcuidle(ipi_types[ipinr]);
C
Catalin Marinas 已提交
805 806 807 808 809 810 811 812
	set_irq_regs(old_regs);
}

void smp_send_reschedule(int cpu)
{
	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
}

813 814 815 816 817 818 819
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
void tick_broadcast(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_TIMER);
}
#endif

C
Catalin Marinas 已提交
820 821 822 823 824 825 826 827
void smp_send_stop(void)
{
	unsigned long timeout;

	if (num_online_cpus() > 1) {
		cpumask_t mask;

		cpumask_copy(&mask, cpu_online_mask);
828
		cpumask_clear_cpu(smp_processor_id(), &mask);
C
Catalin Marinas 已提交
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848

		smp_cross_call(&mask, IPI_CPU_STOP);
	}

	/* Wait up to one second for other CPUs to stop */
	timeout = USEC_PER_SEC;
	while (num_online_cpus() > 1 && timeout--)
		udelay(1);

	if (num_online_cpus() > 1)
		pr_warning("SMP: failed to stop secondary CPUs\n");
}

/*
 * not supported here
 */
int setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}