diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index a7c2337b0c7d66ff7f7709ec684150b10243d3ab..e6b62c7d6f0e9d71da0093a6d51696d4a26189a7 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -32,7 +32,7 @@ void init_IRQ(void); #ifdef CONFIG_SMP #include -extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, +extern bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace #endif diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 87f8d0e5e314a87acbae464484c5d565f2874018..fd9f9f419a6afca6309dca7813e027aea6ceb882 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -850,7 +850,8 @@ static void raise_nmi(cpumask_t *mask) __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask); } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) { nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_nmi); + return true; } diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h index fac08e18bcd512eec9bbafe7fa79a38ccea1d48c..9e926b4f7d47614ee4fd53cba48c04fbc09d421b 100644 --- a/arch/arm64/include/asm/irq.h +++ b/arch/arm64/include/asm/irq.h @@ -6,6 +6,12 @@ #include +#ifdef CONFIG_SMP +extern bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, + bool exclude_self); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace +#endif + struct pt_regs; int set_handle_irq(void (*handle_irq)(struct pt_regs *)); diff --git a/arch/arm64/include/asm/nmi.h b/arch/arm64/include/asm/nmi.h new file mode 100644 index 0000000000000000000000000000000000000000..4cd14b6af88b9658c7552a73618fb5a99f87d93e --- /dev/null +++ b/arch/arm64/include/asm/nmi.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_NMI_H +#define __ASM_NMI_H + +#ifndef __ASSEMBLER__ + +#include + +extern bool arm64_supports_nmi(void); +extern void arm64_send_nmi(cpumask_t *mask); + +void set_smp_dynamic_ipi(int ipi); +void dynamic_ipi_setup(int cpu); +void dynamic_ipi_teardown(int cpu); + +#endif /* !__ASSEMBLER__ */ +#endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 7c2bb4e724767bc3e51e0e335be51e616f5809f7..3dd4f65d93ca3039c41dde6963057f4efcccfb19 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -34,7 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idreg-override.o idle.o \ - patching.o + patching.o ipi_nmi.o obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ sys_compat.o diff --git a/arch/arm64/kernel/ipi_nmi.c b/arch/arm64/kernel/ipi_nmi.c new file mode 100644 index 0000000000000000000000000000000000000000..3b105852fc176c757ca001ba1208b143f2b091fa --- /dev/null +++ b/arch/arm64/kernel/ipi_nmi.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NMI support for IPIs + * + * Copyright (C) 2020 Linaro Limited + * Author: Sumit Garg + */ + +#include +#include +#include +#include +#include + +#include + +static struct irq_desc *ipi_nmi_desc __read_mostly; +static int ipi_nmi_id __read_mostly; + +bool arm64_supports_nmi(void) +{ + if (ipi_nmi_desc) + return true; + + return false; +} + +void arm64_send_nmi(cpumask_t *mask) +{ + if (WARN_ON_ONCE(!ipi_nmi_desc)) + return; + + __ipi_send_mask(ipi_nmi_desc, mask); +} + +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +{ + if (!ipi_nmi_desc) + return false; + + nmi_trigger_cpumask_backtrace(mask, exclude_self, arm64_send_nmi); + + return true; +} + +static irqreturn_t ipi_nmi_handler(int irq, void *data) +{ + irqreturn_t ret = IRQ_NONE; + + if (nmi_cpu_backtrace(get_irq_regs())) + ret = IRQ_HANDLED; + +#ifdef CONFIG_KGDB + if (!kgdb_nmicallback(smp_processor_id(), get_irq_regs())) + ret = IRQ_HANDLED; +#endif + + return ret; +} + +void dynamic_ipi_setup(int cpu) +{ + if (!ipi_nmi_desc) + return; + + if (!prepare_percpu_nmi(ipi_nmi_id)) + enable_percpu_nmi(ipi_nmi_id, IRQ_TYPE_NONE); +} + +void dynamic_ipi_teardown(int cpu) +{ + if (!ipi_nmi_desc) + return; + + disable_percpu_nmi(ipi_nmi_id); + teardown_percpu_nmi(ipi_nmi_id); +} + +void __init set_smp_dynamic_ipi(int ipi) +{ + if (!request_percpu_nmi(ipi, ipi_nmi_handler, "IPI", &cpu_number)) { + ipi_nmi_desc = irq_to_desc(ipi); + ipi_nmi_id = ipi; + } +} diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c index 4e1f983df3d1c22df953e9249312683aa3e0d04e..98d8c1027f06c10b6b6c42d87b7d67812c8900d8 100644 --- a/arch/arm64/kernel/kgdb.c +++ b/arch/arm64/kernel/kgdb.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -356,3 +357,20 @@ int kgdb_arch_remove_breakpoint(struct kgdb_bkpt *bpt) return aarch64_insn_write((void *)bpt->bpt_addr, *(u32 *)bpt->saved_instr); } + +void kgdb_roundup_cpus(void) +{ + struct cpumask mask; + + if (!arm64_supports_nmi()) { + kgdb_smp_call_nmi_hook(); + return; + } + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(raw_smp_processor_id(), &mask); + if (cpumask_empty(&mask)) + return; + + arm64_send_nmi(&mask); +} diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index d00d4cbb31b1671c46379cb09674561483ef31b3..0449a8a1627d3a8a74d56966e2d7f1b719245fb9 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -939,6 +940,8 @@ static void ipi_setup(int cpu) for (i = 0; i < nr_ipi; i++) enable_percpu_irq(ipi_irq_base + i, 0); + + dynamic_ipi_setup(cpu); } #ifdef CONFIG_HOTPLUG_CPU @@ -951,6 +954,8 @@ static void ipi_teardown(int cpu) for (i = 0; i < nr_ipi; i++) disable_percpu_irq(ipi_irq_base + i); + + dynamic_ipi_teardown(cpu); } #endif @@ -972,6 +977,9 @@ void __init set_smp_ipi_range(int ipi_base, int n) irq_set_status_flags(ipi_base + i, IRQ_HIDDEN); } + if (n > nr_ipi) + set_smp_dynamic_ipi(ipi_base + nr_ipi); + ipi_irq_base = ipi_base; /* Setup the boot CPU immediately */ diff --git a/arch/mips/include/asm/irq.h b/arch/mips/include/asm/irq.h index 44f9824c1d8c3b855fcf4b5c6a814476d3e9452a..daf16173486a78215c00bbfebe53240f130f368e 100644 --- a/arch/mips/include/asm/irq.h +++ b/arch/mips/include/asm/irq.h @@ -77,7 +77,7 @@ extern int cp0_fdc_irq; extern int get_c0_fdc_int(void); -void arch_trigger_cpumask_backtrace(const struct cpumask *mask, +bool arch_trigger_cpumask_backtrace(const struct cpumask *mask, bool exclude_self); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index a3225912c862da7bbe3bd315950f5073006f99e9..197a0e2ae12f04398c58f06aec8cfbd1fc50bddf 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -750,9 +750,10 @@ static void raise_backtrace(cpumask_t *mask) } } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) { nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace); + return true; } int mips_get_process_fp_mode(struct task_struct *task) diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h index c3c7adef74de0605a795d278121712dbf51f47c3..135f65adcf63012bc567e97294923cf4e98cbd19 100644 --- a/arch/powerpc/include/asm/nmi.h +++ b/arch/powerpc/include/asm/nmi.h @@ -12,7 +12,7 @@ static inline void watchdog_nmi_set_timeout_pct(u64 pct) {} #endif #ifdef CONFIG_NMI_IPI -extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, +extern bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace #endif diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c index 5de8597eaab8dc428828abb0c6fba244e62fe23b..0fee4bded7ba111281af77b0bbe44dad8bd4dfa0 100644 --- a/arch/powerpc/kernel/stacktrace.c +++ b/arch/powerpc/kernel/stacktrace.c @@ -221,8 +221,9 @@ static void raise_backtrace_ipi(cpumask_t *mask) } } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) { nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace_ipi); + return true; } #endif /* defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_NMI_IPI) */ diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index 154df2cf19f42d8fdf1ea2bc49a56a31fdf09398..00a0051a9da0ab1a0a2f791ef696f2fe2767c265 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -87,7 +87,7 @@ static inline unsigned long get_softint(void) return retval; } -void arch_trigger_cpumask_backtrace(const struct cpumask *mask, +bool arch_trigger_cpumask_backtrace(const struct cpumask *mask, bool exclude_self); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index b51d8fb0ecdc2d693b3e30fb0b908a074df362e3..93972780d6e704903c45a51ea1e1f5b2ec06a478 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -236,7 +236,7 @@ static void __global_reg_poll(struct global_reg_snapshot *gp) } } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) { struct thread_info *tp = current_thread_info(); struct pt_regs *regs = get_irq_regs(); @@ -291,6 +291,8 @@ void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags); + + return true; } #ifdef CONFIG_MAGIC_SYSRQ diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index 768aa234cbb4a6c2fb3c6362156825e11ee046d1..f731638cc38edab549fe7e948e84df456db26579 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -43,7 +43,7 @@ extern void init_ISA_irqs(void); extern void __init init_IRQ(void); #ifdef CONFIG_X86_LOCAL_APIC -void arch_trigger_cpumask_backtrace(const struct cpumask *mask, +bool arch_trigger_cpumask_backtrace(const struct cpumask *mask, bool exclude_self); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace diff --git a/arch/x86/kernel/apic/hw_nmi.c b/arch/x86/kernel/apic/hw_nmi.c index 34a992e275ef42b94e6f0fb057ff160617cd972a..e7dcd28bc824ff3bb51117af18d71ee91d91904f 100644 --- a/arch/x86/kernel/apic/hw_nmi.c +++ b/arch/x86/kernel/apic/hw_nmi.c @@ -34,10 +34,11 @@ static void nmi_raise_cpu_backtrace(cpumask_t *mask) apic->send_IPI_mask(mask, NMI_VECTOR); } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) { nmi_trigger_cpumask_backtrace(mask, exclude_self, nmi_raise_cpu_backtrace); + return true; } static int nmi_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index a605aa79435a4dca64dd1eb012f0f1fe7f394ff3..43beff9c5a4112912b49eaaea9d289c404e6f9df 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -524,6 +524,7 @@ static u32 gic_get_ppi_index(struct irq_data *d) static int gic_irq_nmi_setup(struct irq_data *d) { struct irq_desc *desc = irq_to_desc(d->irq); + u32 idx; if (!gic_supports_nmi()) return -EINVAL; @@ -541,16 +542,22 @@ static int gic_irq_nmi_setup(struct irq_data *d) return -EINVAL; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_ppi_index(d); + switch (get_intid_range(d)) { + case SGI_RANGE: + break; + case PPI_RANGE: + case EPPI_RANGE: + idx = gic_get_ppi_index(d); /* Setting up PPI as NMI, only switch handler for first NMI */ if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { refcount_set(&ppi_nmi_refs[idx], 1); desc->handle_irq = handle_percpu_devid_fasteoi_nmi; } - } else { + break; + default: desc->handle_irq = handle_fasteoi_nmi; + break; } gic_irq_set_prio(d, GICD_INT_NMI_PRI); @@ -561,6 +568,7 @@ static int gic_irq_nmi_setup(struct irq_data *d) static void gic_irq_nmi_teardown(struct irq_data *d) { struct irq_desc *desc = irq_to_desc(d->irq); + u32 idx; if (WARN_ON(!gic_supports_nmi())) return; @@ -578,14 +586,20 @@ static void gic_irq_nmi_teardown(struct irq_data *d) return; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_ppi_index(d); + switch (get_intid_range(d)) { + case SGI_RANGE: + break; + case PPI_RANGE: + case EPPI_RANGE: + idx = gic_get_ppi_index(d); /* Tearing down NMI, only switch handler for last NMI */ if (refcount_dec_and_test(&ppi_nmi_refs[idx])) desc->handle_irq = handle_percpu_devid_irq; - } else { + break; + default: desc->handle_irq = handle_fasteoi_irq; + break; } gic_irq_set_prio(d, GICD_INT_DEF_PRI); @@ -1976,6 +1990,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_dist_init(); gic_cpu_init(); + gic_enable_nmi_support(); gic_smp_init(); gic_cpu_pm_init(); @@ -1988,8 +2003,6 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gicv2m_init(handle, gic_data.domain); } - gic_enable_nmi_support(); - return 0; out_free: diff --git a/include/linux/kgdb.h b/include/linux/kgdb.h index 258cdde8d356b9b8bf1acd900588ae4ce50bc680..87713bd390f354f045a05220c6b20b789709d3db 100644 --- a/include/linux/kgdb.h +++ b/include/linux/kgdb.h @@ -199,6 +199,18 @@ kgdb_arch_handle_qxfer_pkt(char *remcom_in_buffer, extern void kgdb_call_nmi_hook(void *ignored); +/** + * kgdb_smp_call_nmi_hook - Provide default fallback mechanism to + * round-up CPUs + * + * If you're using the default implementation of kgdb_roundup_cpus() + * this function will be called. And if an arch detects at runtime to + * not support NMI based roundup then it can fallback to default + * mechanism using this API. + */ + +extern void kgdb_smp_call_nmi_hook(void); + /** * kgdb_roundup_cpus - Get other CPUs into a holding pattern * diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 048c0b9aa623dd05484c00e05c316fadf761eeac..7d8a77cd1e036dbb6f2ad2dc8fa8a099f9cc6a04 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -145,26 +145,22 @@ static inline void touch_nmi_watchdog(void) #ifdef arch_trigger_cpumask_backtrace static inline bool trigger_all_cpu_backtrace(void) { - arch_trigger_cpumask_backtrace(cpu_online_mask, false); - return true; + return arch_trigger_cpumask_backtrace(cpu_online_mask, false); } static inline bool trigger_allbutself_cpu_backtrace(void) { - arch_trigger_cpumask_backtrace(cpu_online_mask, true); - return true; + return arch_trigger_cpumask_backtrace(cpu_online_mask, true); } static inline bool trigger_cpumask_backtrace(struct cpumask *mask) { - arch_trigger_cpumask_backtrace(mask, false); - return true; + return arch_trigger_cpumask_backtrace(mask, false); } static inline bool trigger_single_cpu_backtrace(int cpu) { - arch_trigger_cpumask_backtrace(cpumask_of(cpu), false); - return true; + return arch_trigger_cpumask_backtrace(cpumask_of(cpu), false); } /* generic implementation */ diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index d5e9ccde3ab8e97f51e93b8fb57ba0e2a101ecfc..14d40a7d6a4b6f4afa8860427a0ec670a3281521 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -238,7 +238,7 @@ NOKPROBE_SYMBOL(kgdb_call_nmi_hook); static DEFINE_PER_CPU(call_single_data_t, kgdb_roundup_csd) = CSD_INIT(kgdb_call_nmi_hook, NULL); -void __weak kgdb_roundup_cpus(void) +void kgdb_smp_call_nmi_hook(void) { call_single_data_t *csd; int this_cpu = raw_smp_processor_id(); @@ -269,6 +269,12 @@ void __weak kgdb_roundup_cpus(void) kgdb_info[cpu].rounding_up = false; } } +NOKPROBE_SYMBOL(kgdb_smp_call_nmi_hook); + +void __weak kgdb_roundup_cpus(void) +{ + kgdb_smp_call_nmi_hook(); +} NOKPROBE_SYMBOL(kgdb_roundup_cpus); #endif