smp.c 20.8 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * SMP initialisation and IPI support
 * Based on arch/arm/kernel/smp.c
 *
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/acpi.h>
C
Catalin Marinas 已提交
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/seq_file.h>
#include <linux/irq.h>
#include <linux/percpu.h>
#include <linux/clockchips.h>
#include <linux/completion.h>
#include <linux/of.h>
39
#include <linux/irq_work.h>
C
Catalin Marinas 已提交
40

41
#include <asm/alternative.h>
C
Catalin Marinas 已提交
42 43
#include <asm/atomic.h>
#include <asm/cacheflush.h>
44
#include <asm/cpu.h>
C
Catalin Marinas 已提交
45
#include <asm/cputype.h>
M
Mark Rutland 已提交
46
#include <asm/cpu_ops.h>
C
Catalin Marinas 已提交
47 48 49 50
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/processor.h>
51
#include <asm/smp_plat.h>
C
Catalin Marinas 已提交
52 53 54
#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/ptrace.h>
55
#include <asm/virt.h>
C
Catalin Marinas 已提交
56

N
Nicolas Pitre 已提交
57 58 59
#define CREATE_TRACE_POINTS
#include <trace/events/ipi.h>

C
Catalin Marinas 已提交
60 61 62 63 64 65
/*
 * as from 2.5, kernels no longer have an init_tasks structure
 * so we need some other way of telling a new secondary core
 * where to place its SVC stack
 */
struct secondary_data secondary_data;
66 67
/* Number of CPUs which aren't online, but looping in kernel text. */
int cpus_stuck_in_kernel;
C
Catalin Marinas 已提交
68 69 70 71 72

enum ipi_msg_type {
	IPI_RESCHEDULE,
	IPI_CALL_FUNC,
	IPI_CPU_STOP,
73
	IPI_TIMER,
74
	IPI_IRQ_WORK,
75
	IPI_WAKEUP
C
Catalin Marinas 已提交
76 77
};

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
#ifdef CONFIG_ARM64_VHE

/* Whether the boot CPU is running in HYP mode or not*/
static bool boot_cpu_hyp_mode;

static inline void save_boot_cpu_run_el(void)
{
	boot_cpu_hyp_mode = is_kernel_in_hyp_mode();
}

static inline bool is_boot_cpu_in_hyp_mode(void)
{
	return boot_cpu_hyp_mode;
}

/*
 * Verify that a secondary CPU is running the kernel at the same
 * EL as that of the boot CPU.
 */
void verify_cpu_run_el(void)
{
	bool in_el2 = is_kernel_in_hyp_mode();
	bool boot_cpu_el2 = is_boot_cpu_in_hyp_mode();

	if (in_el2 ^ boot_cpu_el2) {
		pr_crit("CPU%d: mismatched Exception Level(EL%d) with boot CPU(EL%d)\n",
					smp_processor_id(),
					in_el2 ? 2 : 1,
					boot_cpu_el2 ? 2 : 1);
		cpu_panic_kernel();
	}
}

#else
static inline void save_boot_cpu_run_el(void) {}
#endif

115 116 117 118 119 120 121 122 123 124
#ifdef CONFIG_HOTPLUG_CPU
static int op_cpu_kill(unsigned int cpu);
#else
static inline int op_cpu_kill(unsigned int cpu)
{
	return -ENOSYS;
}
#endif


C
Catalin Marinas 已提交
125 126 127 128
/*
 * Boot a secondary CPU, and assign it the specified idle task.
 * This also gives us the initial stack to use for this CPU.
 */
129
static int boot_secondary(unsigned int cpu, struct task_struct *idle)
C
Catalin Marinas 已提交
130
{
131 132
	if (cpu_ops[cpu]->cpu_boot)
		return cpu_ops[cpu]->cpu_boot(cpu);
C
Catalin Marinas 已提交
133

134
	return -EOPNOTSUPP;
C
Catalin Marinas 已提交
135 136 137 138
}

static DECLARE_COMPLETION(cpu_running);

139
int __cpu_up(unsigned int cpu, struct task_struct *idle)
C
Catalin Marinas 已提交
140 141
{
	int ret;
142
	long status;
C
Catalin Marinas 已提交
143 144 145 146 147 148

	/*
	 * We need to tell the secondary core where to find its stack and the
	 * page tables.
	 */
	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
149
	update_cpu_boot_status(CPU_MMU_OFF);
C
Catalin Marinas 已提交
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
	__flush_dcache_area(&secondary_data, sizeof(secondary_data));

	/*
	 * Now bring the CPU into our world.
	 */
	ret = boot_secondary(cpu, idle);
	if (ret == 0) {
		/*
		 * CPU was successfully started, wait for it to come online or
		 * time out.
		 */
		wait_for_completion_timeout(&cpu_running,
					    msecs_to_jiffies(1000));

		if (!cpu_online(cpu)) {
			pr_crit("CPU%u: failed to come online\n", cpu);
			ret = -EIO;
		}
	} else {
		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
	}

	secondary_data.stack = NULL;
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
	status = READ_ONCE(secondary_data.status);
	if (ret && status) {

		if (status == CPU_MMU_OFF)
			status = READ_ONCE(__early_cpu_boot_status);

		switch (status) {
		default:
			pr_err("CPU%u: failed in unknown state : 0x%lx\n",
					cpu, status);
			break;
		case CPU_KILL_ME:
			if (!op_cpu_kill(cpu)) {
				pr_crit("CPU%u: died during early boot\n", cpu);
				break;
			}
			/* Fall through */
			pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
		case CPU_STUCK_IN_KERNEL:
			pr_crit("CPU%u: is stuck in kernel\n", cpu);
			cpus_stuck_in_kernel++;
			break;
		case CPU_PANIC_KERNEL:
			panic("CPU%u detected unsupported configuration\n", cpu);
		}
	}
C
Catalin Marinas 已提交
199 200 201 202

	return ret;
}

203 204 205 206 207
static void smp_store_cpu_info(unsigned int cpuid)
{
	store_cpu_topology(cpuid);
}

C
Catalin Marinas 已提交
208 209 210 211
/*
 * This is the secondary CPU boot entry.  We're using this CPUs
 * idle thread stack, but a set of temporary page tables.
 */
212
asmlinkage void secondary_start_kernel(void)
C
Catalin Marinas 已提交
213 214 215 216 217 218 219 220 221 222 223
{
	struct mm_struct *mm = &init_mm;
	unsigned int cpu = smp_processor_id();

	/*
	 * All kernel threads share the same mm context; grab a
	 * reference and switch to it.
	 */
	atomic_inc(&mm->mm_count);
	current->active_mm = mm;

224 225
	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));

C
Catalin Marinas 已提交
226 227 228 229
	/*
	 * TTBR0 is only used for the identity mapping at this stage. Make it
	 * point to zero page to avoid speculatively fetching new entries.
	 */
M
Mark Rutland 已提交
230
	cpu_uninstall_idmap();
C
Catalin Marinas 已提交
231 232 233 234

	preempt_disable();
	trace_hardirqs_off();

235 236 237 238 239 240 241
	/*
	 * If the system has established the capabilities, make sure
	 * this CPU ticks all of those. If it doesn't, the CPU will
	 * fail to come online.
	 */
	verify_local_cpu_capabilities();

242 243
	if (cpu_ops[cpu]->cpu_postboot)
		cpu_ops[cpu]->cpu_postboot();
C
Catalin Marinas 已提交
244

245 246 247 248 249
	/*
	 * Log the CPU info before it is marked online and might get read.
	 */
	cpuinfo_store_cpu();

250 251 252 253 254
	/*
	 * Enable GIC and timers.
	 */
	notify_cpu_starting(cpu);

255 256
	smp_store_cpu_info(cpu);

C
Catalin Marinas 已提交
257 258 259 260 261
	/*
	 * OK, now it's safe to let the boot CPU continue.  Wait for
	 * the CPU migration code to notice that the CPU is online
	 * before we continue.
	 */
262 263
	pr_info("CPU%u: Booted secondary processor [%08x]\n",
					 cpu, read_cpuid_id());
264 265 266
	update_cpu_boot_status(CPU_BOOT_SUCCESS);
	/* Make sure the status update is visible before we complete */
	smp_wmb();
C
Catalin Marinas 已提交
267
	set_cpu_online(cpu, true);
268
	complete(&cpu_running);
C
Catalin Marinas 已提交
269

270
	local_dbg_enable();
271
	local_irq_enable();
272
	local_async_enable();
273

C
Catalin Marinas 已提交
274 275 276
	/*
	 * OK, it's off to the idle thread for us
	 */
277
	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
C
Catalin Marinas 已提交
278 279
}

280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
#ifdef CONFIG_HOTPLUG_CPU
static int op_cpu_disable(unsigned int cpu)
{
	/*
	 * If we don't have a cpu_die method, abort before we reach the point
	 * of no return. CPU0 may not have an cpu_ops, so test for it.
	 */
	if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_die)
		return -EOPNOTSUPP;

	/*
	 * We may need to abort a hot unplug for some other mechanism-specific
	 * reason.
	 */
	if (cpu_ops[cpu]->cpu_disable)
		return cpu_ops[cpu]->cpu_disable(cpu);

	return 0;
}

/*
 * __cpu_disable runs on the processor to be shutdown.
 */
int __cpu_disable(void)
{
	unsigned int cpu = smp_processor_id();
	int ret;

	ret = op_cpu_disable(cpu);
	if (ret)
		return ret;

	/*
	 * Take this CPU offline.  Once we clear this, we can't return,
	 * and we must not schedule until we're ready to give up the cpu.
	 */
	set_cpu_online(cpu, false);

	/*
	 * OK - migrate IRQs away from this CPU
	 */
321 322
	irq_migrate_all_off_this_cpu();

323 324 325
	return 0;
}

326 327 328 329 330 331 332 333
static int op_cpu_kill(unsigned int cpu)
{
	/*
	 * If we have no means of synchronising with the dying CPU, then assume
	 * that it is really dead. We can only wait for an arbitrary length of
	 * time and hope that it's dead, so let's skip the wait and just hope.
	 */
	if (!cpu_ops[cpu]->cpu_kill)
334
		return 0;
335 336 337 338

	return cpu_ops[cpu]->cpu_kill(cpu);
}

339 340 341 342 343 344
/*
 * called on the thread which is asking for a CPU to be shutdown -
 * waits until shutdown has completed, or it is timed out.
 */
void __cpu_die(unsigned int cpu)
{
345 346
	int err;

347
	if (!cpu_wait_death(cpu, 5)) {
348 349 350 351
		pr_crit("CPU%u: cpu didn't die\n", cpu);
		return;
	}
	pr_notice("CPU%u: shutdown\n", cpu);
352 353 354 355 356 357 358

	/*
	 * Now that the dying CPU is beyond the point of no return w.r.t.
	 * in-kernel synchronisation, try to get the firwmare to help us to
	 * verify that it has really left the kernel before we consider
	 * clobbering anything it might still be using.
	 */
359 360 361 362
	err = op_cpu_kill(cpu);
	if (err)
		pr_warn("CPU%d may not have shut down cleanly: %d\n",
			cpu, err);
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
}

/*
 * Called from the idle thread for the CPU which has been shutdown.
 *
 * Note that we disable IRQs here, but do not re-enable them
 * before returning to the caller. This is also the behaviour
 * of the other hotplug-cpu capable cores, so presumably coming
 * out of idle fixes this.
 */
void cpu_die(void)
{
	unsigned int cpu = smp_processor_id();

	idle_task_exit();

	local_irq_disable();

	/* Tell __cpu_die() that this CPU is now safe to dispose of */
382
	(void)cpu_report_death();
383 384 385 386 387 388 389 390 391 392 393 394

	/*
	 * Actually shutdown the CPU. This must never fail. The specific hotplug
	 * mechanism must perform all required cache maintenance to ensure that
	 * no dirty lines are lost in the process of shutting down the CPU.
	 */
	cpu_ops[cpu]->cpu_die(cpu);

	BUG();
}
#endif

395 396 397 398 399 400 401 402 403 404 405 406 407 408
/*
 * Kill the calling secondary CPU, early in bringup before it is turned
 * online.
 */
void cpu_die_early(void)
{
	int cpu = smp_processor_id();

	pr_crit("CPU%d: will not boot\n", cpu);

	/* Mark this CPU absent */
	set_cpu_present(cpu, 0);

#ifdef CONFIG_HOTPLUG_CPU
409
	update_cpu_boot_status(CPU_KILL_ME);
410 411 412 413
	/* Check if we can park ourselves */
	if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die)
		cpu_ops[cpu]->cpu_die(cpu);
#endif
414
	update_cpu_boot_status(CPU_STUCK_IN_KERNEL);
415 416 417 418

	cpu_park_loop();
}

419 420 421 422 423 424 425 426 427 428 429
static void __init hyp_mode_check(void)
{
	if (is_hyp_mode_available())
		pr_info("CPU: All CPU(s) started at EL2\n");
	else if (is_hyp_mode_mismatched())
		WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
			   "CPU: CPUs started in inconsistent modes");
	else
		pr_info("CPU: All CPU(s) started at EL1\n");
}

C
Catalin Marinas 已提交
430 431
void __init smp_cpus_done(unsigned int max_cpus)
{
432
	pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
433
	setup_cpu_features();
434 435
	hyp_mode_check();
	apply_alternatives_all();
C
Catalin Marinas 已提交
436 437 438 439
}

void __init smp_prepare_boot_cpu(void)
{
440
	cpuinfo_store_boot_cpu();
441
	save_boot_cpu_run_el();
442
	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
C
Catalin Marinas 已提交
443 444
}

445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
static u64 __init of_get_cpu_mpidr(struct device_node *dn)
{
	const __be32 *cell;
	u64 hwid;

	/*
	 * A cpu node with missing "reg" property is
	 * considered invalid to build a cpu_logical_map
	 * entry.
	 */
	cell = of_get_property(dn, "reg", NULL);
	if (!cell) {
		pr_err("%s: missing reg property\n", dn->full_name);
		return INVALID_HWID;
	}

	hwid = of_read_number(cell, of_n_addr_cells(dn));
	/*
	 * Non affinity bits must be set to 0 in the DT
	 */
	if (hwid & ~MPIDR_HWID_BITMASK) {
		pr_err("%s: invalid reg property\n", dn->full_name);
		return INVALID_HWID;
	}
	return hwid;
}

/*
 * Duplicate MPIDRs are a recipe for disaster. Scan all initialized
 * entries and check for duplicates. If any is found just ignore the
 * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid
 * matching valid MPIDR values.
 */
static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid)
{
	unsigned int i;

	for (i = 1; (i < cpu) && (i < NR_CPUS); i++)
		if (cpu_logical_map(i) == hwid)
			return true;
	return false;
}

488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
/*
 * Initialize cpu operations for a logical cpu and
 * set it in the possible mask on success
 */
static int __init smp_cpu_setup(int cpu)
{
	if (cpu_read_ops(cpu))
		return -ENODEV;

	if (cpu_ops[cpu]->cpu_init(cpu))
		return -ENODEV;

	set_cpu_possible(cpu, true);

	return 0;
}

505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
static bool bootcpu_valid __initdata;
static unsigned int cpu_count = 1;

#ifdef CONFIG_ACPI
/*
 * acpi_map_gic_cpu_interface - parse processor MADT entry
 *
 * Carry out sanity checks on MADT processor entry and initialize
 * cpu_logical_map on success
 */
static void __init
acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
{
	u64 hwid = processor->arm_mpidr;

520 521
	if (!(processor->flags & ACPI_MADT_ENABLED)) {
		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
522 523 524
		return;
	}

525 526
	if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) {
		pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid);
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
		return;
	}

	if (is_mpidr_duplicate(cpu_count, hwid)) {
		pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);
		return;
	}

	/* Check if GICC structure of boot CPU is available in the MADT */
	if (cpu_logical_map(0) == hwid) {
		if (bootcpu_valid) {
			pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n",
			       hwid);
			return;
		}
		bootcpu_valid = true;
		return;
	}

	if (cpu_count >= NR_CPUS)
		return;

	/* map the logical cpu id to cpu MPIDR */
	cpu_logical_map(cpu_count) = hwid;

552 553 554 555 556 557 558 559 560 561 562
	/*
	 * Set-up the ACPI parking protocol cpu entries
	 * while initializing the cpu_logical_map to
	 * avoid parsing MADT entries multiple times for
	 * nothing (ie a valid cpu_logical_map entry should
	 * contain a valid parking protocol data set to
	 * initialize the cpu if the parking protocol is
	 * the only available enable method).
	 */
	acpi_set_mailbox_entry(cpu_count, processor);

563 564 565 566 567 568 569 570 571 572
	cpu_count++;
}

static int __init
acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header,
			     const unsigned long end)
{
	struct acpi_madt_generic_interrupt *processor;

	processor = (struct acpi_madt_generic_interrupt *)header;
573
	if (BAD_MADT_GICC_ENTRY(processor, end))
574 575 576 577 578 579 580 581 582 583 584 585
		return -EINVAL;

	acpi_table_print_madt_entry(header);

	acpi_map_gic_cpu_interface(processor);

	return 0;
}
#else
#define acpi_table_parse_madt(...)	do { } while (0)
#endif

C
Catalin Marinas 已提交
586
/*
587 588 589
 * Enumerate the possible CPU set from the device tree and build the
 * cpu logical map array containing MPIDR values related to logical
 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
C
Catalin Marinas 已提交
590
 */
591
static void __init of_parse_and_init_cpus(void)
C
Catalin Marinas 已提交
592 593 594 595
{
	struct device_node *dn = NULL;

	while ((dn = of_find_node_by_type(dn, "cpu"))) {
596
		u64 hwid = of_get_cpu_mpidr(dn);
597

598
		if (hwid == INVALID_HWID)
599 600
			goto next;

601 602 603
		if (is_mpidr_duplicate(cpu_count, hwid)) {
			pr_err("%s: duplicate cpu reg properties in the DT\n",
				dn->full_name);
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630
			goto next;
		}

		/*
		 * The numbering scheme requires that the boot CPU
		 * must be assigned logical id 0. Record it so that
		 * the logical map built from DT is validated and can
		 * be used.
		 */
		if (hwid == cpu_logical_map(0)) {
			if (bootcpu_valid) {
				pr_err("%s: duplicate boot cpu reg property in DT\n",
					dn->full_name);
				goto next;
			}

			bootcpu_valid = true;

			/*
			 * cpu_logical_map has already been
			 * initialized and the boot cpu doesn't need
			 * the enable-method so continue without
			 * incrementing cpu.
			 */
			continue;
		}

631
		if (cpu_count >= NR_CPUS)
C
Catalin Marinas 已提交
632 633
			goto next;

634
		pr_debug("cpu logical map 0x%llx\n", hwid);
635
		cpu_logical_map(cpu_count) = hwid;
C
Catalin Marinas 已提交
636
next:
637
		cpu_count++;
C
Catalin Marinas 已提交
638
	}
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
}

/*
 * Enumerate the possible CPU set from the device tree or ACPI and build the
 * cpu logical map array containing MPIDR values related to logical
 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
 */
void __init smp_init_cpus(void)
{
	int i;

	if (acpi_disabled)
		of_parse_and_init_cpus();
	else
		/*
		 * do a walk of MADT to determine how many CPUs
		 * we have including disabled CPUs, and get information
		 * we need for SMP init
		 */
		acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
				      acpi_parse_gic_cpu_interface, 0);
C
Catalin Marinas 已提交
660

661 662 663
	if (cpu_count > NR_CPUS)
		pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n",
			cpu_count, NR_CPUS);
664 665

	if (!bootcpu_valid) {
666
		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
667 668 669 670
		return;
	}

	/*
671 672 673 674 675
	 * We need to set the cpu_logical_map entries before enabling
	 * the cpus so that cpu processor description entries (DT cpu nodes
	 * and ACPI MADT entries) can be retrieved by matching the cpu hwid
	 * with entries in cpu_logical_map while initializing the cpus.
	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
676
	 */
677 678 679 680 681 682
	for (i = 1; i < NR_CPUS; i++) {
		if (cpu_logical_map(i) != INVALID_HWID) {
			if (smp_cpu_setup(i))
				cpu_logical_map(i) = INVALID_HWID;
		}
	}
C
Catalin Marinas 已提交
683 684 685 686
}

void __init smp_prepare_cpus(unsigned int max_cpus)
{
M
Mark Rutland 已提交
687 688
	int err;
	unsigned int cpu, ncores = num_possible_cpus();
C
Catalin Marinas 已提交
689

690 691 692 693
	init_cpu_topology();

	smp_store_cpu_info(smp_processor_id());

C
Catalin Marinas 已提交
694 695 696 697 698 699
	/*
	 * are we trying to boot more cores than exist?
	 */
	if (max_cpus > ncores)
		max_cpus = ncores;

700 701 702 703
	/* Don't bother if we're effectively UP */
	if (max_cpus <= 1)
		return;

C
Catalin Marinas 已提交
704 705 706 707
	/*
	 * Initialise the present map (which describes the set of CPUs
	 * actually populated at the present time) and release the
	 * secondaries from the bootloader.
708 709
	 *
	 * Make sure we online at most (max_cpus - 1) additional CPUs.
C
Catalin Marinas 已提交
710
	 */
711
	max_cpus--;
C
Catalin Marinas 已提交
712 713 714 715
	for_each_possible_cpu(cpu) {
		if (max_cpus == 0)
			break;

716 717 718
		if (cpu == smp_processor_id())
			continue;

M
Mark Rutland 已提交
719
		if (!cpu_ops[cpu])
C
Catalin Marinas 已提交
720 721
			continue;

M
Mark Rutland 已提交
722
		err = cpu_ops[cpu]->cpu_prepare(cpu);
723 724
		if (err)
			continue;
C
Catalin Marinas 已提交
725 726 727 728 729 730

		set_cpu_present(cpu, true);
		max_cpus--;
	}
}

731
void (*__smp_cross_call)(const struct cpumask *, unsigned int);
C
Catalin Marinas 已提交
732 733 734

void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
{
N
Nicolas Pitre 已提交
735
	__smp_cross_call = fn;
C
Catalin Marinas 已提交
736 737
}

N
Nicolas Pitre 已提交
738 739
static const char *ipi_types[NR_IPI] __tracepoint_string = {
#define S(x,s)	[x] = s
C
Catalin Marinas 已提交
740 741 742
	S(IPI_RESCHEDULE, "Rescheduling interrupts"),
	S(IPI_CALL_FUNC, "Function call interrupts"),
	S(IPI_CPU_STOP, "CPU stop interrupts"),
743
	S(IPI_TIMER, "Timer broadcast interrupts"),
744
	S(IPI_IRQ_WORK, "IRQ work interrupts"),
745
	S(IPI_WAKEUP, "CPU wake-up interrupts"),
C
Catalin Marinas 已提交
746 747
};

N
Nicolas Pitre 已提交
748 749 750 751 752 753
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
{
	trace_ipi_raise(target, ipi_types[ipinr]);
	__smp_cross_call(target, ipinr);
}

C
Catalin Marinas 已提交
754 755 756 757 758
void show_ipi_list(struct seq_file *p, int prec)
{
	unsigned int cpu, i;

	for (i = 0; i < NR_IPI; i++) {
N
Nicolas Pitre 已提交
759
		seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
C
Catalin Marinas 已提交
760
			   prec >= 4 ? " " : "");
761
		for_each_online_cpu(cpu)
C
Catalin Marinas 已提交
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
			seq_printf(p, "%10u ",
				   __get_irq_stat(cpu, ipi_irqs[i]));
		seq_printf(p, "      %s\n", ipi_types[i]);
	}
}

u64 smp_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = 0;
	int i;

	for (i = 0; i < NR_IPI; i++)
		sum += __get_irq_stat(cpu, ipi_irqs[i]);

	return sum;
}

N
Nicolas Pitre 已提交
779 780 781 782 783 784 785
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_CALL_FUNC);
}

void arch_send_call_function_single_ipi(int cpu)
{
786
	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
N
Nicolas Pitre 已提交
787 788
}

789 790 791 792 793 794 795
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_WAKEUP);
}
#endif

N
Nicolas Pitre 已提交
796 797 798 799 800 801 802 803
#ifdef CONFIG_IRQ_WORK
void arch_irq_work_raise(void)
{
	if (__smp_cross_call)
		smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
}
#endif

C
Catalin Marinas 已提交
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
static DEFINE_RAW_SPINLOCK(stop_lock);

/*
 * ipi_cpu_stop - handle IPI from smp_send_stop()
 */
static void ipi_cpu_stop(unsigned int cpu)
{
	if (system_state == SYSTEM_BOOTING ||
	    system_state == SYSTEM_RUNNING) {
		raw_spin_lock(&stop_lock);
		pr_crit("CPU%u: stopping\n", cpu);
		dump_stack();
		raw_spin_unlock(&stop_lock);
	}

	set_cpu_online(cpu, false);

	local_irq_disable();

	while (1)
		cpu_relax();
}

/*
 * Main handler for inter-processor interrupts
 */
void handle_IPI(int ipinr, struct pt_regs *regs)
{
	unsigned int cpu = smp_processor_id();
	struct pt_regs *old_regs = set_irq_regs(regs);

N
Nicolas Pitre 已提交
835
	if ((unsigned)ipinr < NR_IPI) {
836
		trace_ipi_entry_rcuidle(ipi_types[ipinr]);
N
Nicolas Pitre 已提交
837 838
		__inc_irq_stat(cpu, ipi_irqs[ipinr]);
	}
C
Catalin Marinas 已提交
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856

	switch (ipinr) {
	case IPI_RESCHEDULE:
		scheduler_ipi();
		break;

	case IPI_CALL_FUNC:
		irq_enter();
		generic_smp_call_function_interrupt();
		irq_exit();
		break;

	case IPI_CPU_STOP:
		irq_enter();
		ipi_cpu_stop(cpu);
		irq_exit();
		break;

857 858 859 860 861 862 863 864
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
	case IPI_TIMER:
		irq_enter();
		tick_receive_broadcast();
		irq_exit();
		break;
#endif

865 866 867 868 869 870 871 872
#ifdef CONFIG_IRQ_WORK
	case IPI_IRQ_WORK:
		irq_enter();
		irq_work_run();
		irq_exit();
		break;
#endif

873 874 875 876 877 878 879 880
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
	case IPI_WAKEUP:
		WARN_ONCE(!acpi_parking_protocol_valid(cpu),
			  "CPU%u: Wake-up IPI outside the ACPI parking protocol\n",
			  cpu);
		break;
#endif

C
Catalin Marinas 已提交
881 882 883 884
	default:
		pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
		break;
	}
N
Nicolas Pitre 已提交
885 886

	if ((unsigned)ipinr < NR_IPI)
887
		trace_ipi_exit_rcuidle(ipi_types[ipinr]);
C
Catalin Marinas 已提交
888 889 890 891 892 893 894 895
	set_irq_regs(old_regs);
}

void smp_send_reschedule(int cpu)
{
	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
}

896 897 898 899 900 901 902
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
void tick_broadcast(const struct cpumask *mask)
{
	smp_cross_call(mask, IPI_TIMER);
}
#endif

C
Catalin Marinas 已提交
903 904 905 906 907 908 909 910
void smp_send_stop(void)
{
	unsigned long timeout;

	if (num_online_cpus() > 1) {
		cpumask_t mask;

		cpumask_copy(&mask, cpu_online_mask);
911
		cpumask_clear_cpu(smp_processor_id(), &mask);
C
Catalin Marinas 已提交
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931

		smp_cross_call(&mask, IPI_CPU_STOP);
	}

	/* Wait up to one second for other CPUs to stop */
	timeout = USEC_PER_SEC;
	while (num_online_cpus() > 1 && timeout--)
		udelay(1);

	if (num_online_cpus() > 1)
		pr_warning("SMP: failed to stop secondary CPUs\n");
}

/*
 * not supported here
 */
int setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}