diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 2f6dbaa12d43188669bfa34eb6ca5c4cc340799a..7c2c552ac7ed52fb2e7e5028b9ffebc6caac702a 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -560,6 +560,25 @@ config ARCH_ENABLE_THP_MIGRATION def_bool y depends on TRANSPARENT_HUGEPAGE +config PARAVIRT + bool "Enable paravirtualization code" + help + This changes the kernel so it can modify itself when it is run + under a hypervisor, potentially improving performance significantly + over full virtualization. + +config PARAVIRT_TIME_ACCOUNTING + bool "Paravirtual steal time accounting" + select PARAVIRT + default n + help + Select this option to enable fine granularity task steal time + accounting. Time spent executing other tasks in parallel with + the current vCPU is discounted from the vCPU capacity. To account for + that, there can be a small performance impact. + + If in doubt, say N here. + config ARCH_MEMORY_PROBE def_bool y depends on MEMORY_HOTPLUG diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index 43ec73c9cc2d6ddcbcf7d87efbde0807d2ce6f9a..26179147db7a469c32c34828130029c0f89f3d9d 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -35,6 +35,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_USERFAULTFD=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set +CONFIG_PARAVIRT_TIME_ACCOUNTING=y CONFIG_CPU_HAS_LSX=y CONFIG_CPU_HAS_LASX=y CONFIG_NUMA=y diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h index bd9bb0bf0f70397520df6c8c0c7e1789b62e1d66..9ec29edcdb1784d00c7f078b365c93d934cbe1f6 100644 --- a/arch/loongarch/include/asm/kvm_para.h +++ b/arch/loongarch/include/asm/kvm_para.h @@ -2,6 +2,10 @@ #ifndef _ASM_LOONGARCH_KVM_PARA_H #define _ASM_LOONGARCH_KVM_PARA_H +#include + +#define KVM_HYPERCALL ".word 0x002b8000" + /* * Hypcall code field */ @@ -25,6 +29,148 @@ #define KVM_RET_SUC 1 #define KVM_RET_NOT_SUPPORTED -1 +/* + * Hypercalls interface for KVM. + * + * a0: function identifier + * a1-a6: args + * Return value will be placed in v0. + * Up to 6 arguments are passed in a1, a2, a3, a4, a5, a6. + */ +static inline long kvm_hypercall0(u64 fid) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall1(u64 fid, unsigned long arg0) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun), "r" (a1) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall2(u64 fid, + unsigned long arg0, unsigned long arg1) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun), "r" (a1), "r" (a2) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall3(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "r" (fun), "r" (a1), "r" (a2), "r" (a3) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall4(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2, + unsigned long arg3) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + register unsigned long a4 asm("a4") = arg3; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall5(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + register unsigned long a4 asm("a4") = arg3; + register unsigned long a5 asm("a5") = arg4; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4), "r" (a5) + : "memory" + ); + + return ret; +} + +static inline long kvm_hypercall6(u64 fid, + unsigned long arg0, unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4, unsigned long arg5) +{ + register long ret asm("v0"); + register unsigned long fun asm("a0") = fid; + register unsigned long a1 asm("a1") = arg0; + register unsigned long a2 asm("a2") = arg1; + register unsigned long a3 asm("a3") = arg2; + register unsigned long a4 asm("a4") = arg3; + register unsigned long a5 asm("a5") = arg4; + register unsigned long a6 asm("a6") = arg5; + + __asm__ __volatile__( + KVM_HYPERCALL + : "=r" (ret) + : "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6) + : "memory" + ); + + return ret; +} + static inline bool kvm_check_and_clear_guest_paused(void) { return false; diff --git a/arch/loongarch/include/asm/loongson.h b/arch/loongarch/include/asm/loongson.h index e4108f674c4e8ebccfb07a2e880e0d94278b2d76..a093df3184e8e6a9f815d06531cd5872957b9eaf 100644 --- a/arch/loongarch/include/asm/loongson.h +++ b/arch/loongarch/include/asm/loongson.h @@ -14,8 +14,6 @@ #include #include -extern const struct plat_smp_ops loongson3_smp_ops; - #define LOONGSON_REG(x) \ (*(volatile u32 *)((char *)TO_UNCACHE(LOONGSON_REG_BASE) + (x))) diff --git a/arch/loongarch/include/asm/paravirt.h b/arch/loongarch/include/asm/paravirt.h new file mode 100644 index 0000000000000000000000000000000000000000..5a447f06dbdc963203ee58e77cbb6f6c782a450a --- /dev/null +++ b/arch/loongarch/include/asm/paravirt.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_PARAVIRT_H +#define _ASM_LOONGARCH_PARAVIRT_H +#include + +#ifdef CONFIG_PARAVIRT +static inline bool kvm_para_available(void) +{ + return true; +} +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +struct pv_time_ops { + unsigned long long (*steal_clock)(int cpu); +}; +struct kvm_steal_time { + __u64 steal; + __u32 version; + __u32 flags; + __u32 pad[12]; +}; +extern struct pv_time_ops pv_time_ops; + +static inline u64 paravirt_steal_clock(int cpu) +{ + return pv_time_ops.steal_clock(cpu); +} + +static inline bool pv_feature_support(int feature) +{ + return kvm_hypercall1(KVM_HC_FUNC_FEATURE, feature) == KVM_RET_SUC; +} +static inline void pv_notify_host(int feature, unsigned long data) +{ + kvm_hypercall2(KVM_HC_FUNC_NOTIFY, feature, data); +} + +int __init pv_time_init(void); +int __init pv_ipi_init(void); +#else +static inline bool kvm_para_available(void) +{ + return false; +} + +static inline int pv_time_init(void) +{ + return 0; +} + +static inline int pv_ipi_init(void) +{ + return 0; +} +#endif +#endif /* _ASM_LOONGARCH_PARAVIRT_H */ diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h index 92cfe170313542c97778e46d1d403e55f7c0dd5d..aad5784121416225997b8970d7dd5e683c201787 100644 --- a/arch/loongarch/include/asm/smp.h +++ b/arch/loongarch/include/asm/smp.h @@ -12,6 +12,11 @@ #include #include +struct smp_ops { + void (*send_call_func_ipi)(const struct cpumask *mask, unsigned int action); + void (*send_call_func_single_ipi)(int cpu, unsigned int action); +}; + extern int smp_num_siblings; extern int num_processors; extern int disabled_cpus; @@ -78,15 +83,8 @@ extern void calculate_cpu_foreign_map(void); */ extern void show_ipi_list(struct seq_file *p, int prec); -static inline void arch_send_call_function_single_ipi(int cpu) -{ - loongson3_send_ipi_single(cpu, SMP_CALL_FUNCTION); -} - -static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask) -{ - loongson3_send_ipi_mask(mask, SMP_CALL_FUNCTION); -} +void arch_send_call_function_single_ipi(int cpu); +void arch_send_call_function_ipi_mask(const struct cpumask *mask); #ifdef CONFIG_HOTPLUG_CPU static inline int __cpu_disable(void) diff --git a/arch/loongarch/include/uapi/asm/kvm_para.h b/arch/loongarch/include/uapi/asm/kvm_para.h new file mode 100644 index 0000000000000000000000000000000000000000..331b6c7c5202ab7f666d78a84fce9ef552f3c1d2 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/kvm_para.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_LOONGARCH_KVM_PARA_H +#define _UAPI_ASM_LOONGARCH_KVM_PARA_H + +/* Device Control API on vcpu fd */ +#define KVM_LARCH_VCPU_PVTIME_CTRL 2 +#define KVM_LARCH_VCPU_PVTIME_IPA 0 + +#endif /* _UAPI_ASM_LOONGARCH_KVM_PARA_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index c39f35231035fce5079317609f25003d281e2988..3afde226091f25b4831cabe4c3140c516a60f6a9 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -36,4 +36,6 @@ obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o +obj-$(CONFIG_PARAVIRT) += paravirt.o + CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c new file mode 100644 index 0000000000000000000000000000000000000000..1d0f51d8fcc68a2289364a196f07f03aa5a1fea6 --- /dev/null +++ b/arch/loongarch/kernel/paravirt.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Loongson Technology Co., Ltd. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * LoongArch paravirtualization support. + * + * Author: zhangyanyue + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct static_key paravirt_steal_enabled; +struct static_key paravirt_steal_rq_enabled; + +static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); +static int has_steal_clock; +static unsigned int pv_feature_disabled; + +static u64 dummy_steal_clock(int cpu) +{ + return 0; +} + +static u64 pv_steal_clock(int cpu) +{ + u64 steal; + struct kvm_steal_time *src; + int version; + + src = &per_cpu(steal_time, cpu); + do { + + version = src->version; + /* Make sure that the version is read before the steal. */ + virt_rmb(); + steal = src->steal; + /* Make sure that the steal is read before the next version. */ + virt_rmb(); + + } while ((version & 1) || (version != src->version)); + return steal; +} + +struct pv_time_ops pv_time_ops = { + .steal_clock = dummy_steal_clock, +}; +EXPORT_SYMBOL_GPL(pv_time_ops); + +phys_addr_t slow_virt_to_phys(const void *vaddr) +{ + unsigned long addr = (unsigned long) vaddr; + unsigned long page_mask = 0; + pgd_t *pgd = pgd_offset_k(addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + phys_addr_t phys_addr; + + /* See arch/loongarch/kernel/numa.c: setup_per_cpu_areas() */ + if (nr_node_ids < 8) + return virt_to_phys((void *)vaddr); + + if (pgd_none(*pgd)) + goto out; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) + goto out; + + pud = pud_offset(p4d, addr); + if (pud_none(*pud) || pud_bad(*pud)) + goto out; + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + goto out; + +#ifdef KVM_HUGE_TLB_SUPPORT + if (pmd_huge(*pmd)) { + pte = (pte_t *)pmd; + page_mask = PMD_MASK; + goto out; + } +#endif + + pte = pte_offset_kernel(pmd, addr); + page_mask = PAGE_MASK; + +out: + phys_addr = (pte_pfn(*pte) << PAGE_SHIFT) | (addr & ~page_mask); + return phys_addr; +} + +static void pv_register_steal_time(void) +{ + int cpu = smp_processor_id(); + struct kvm_steal_time *st; + + if (!has_steal_clock) + return; + + st = &per_cpu(steal_time, cpu); + pv_notify_host(KVM_FEATURE_STEAL_TIME, slow_virt_to_phys(st)); + + pr_info("pv stealtime: cpu %d, st:0x%llx phys:0x%llx\n", + cpu, (unsigned long long)st, (unsigned long long) slow_virt_to_phys(st)); +} + +#ifdef CONFIG_SMP +static void pv_disable_steal_time(void) +{ + if (has_steal_clock) + pv_notify_host(KVM_FEATURE_STEAL_TIME, 0); +} + +static int pv_cpu_online(unsigned int cpu) +{ + unsigned long flags; + + local_irq_save(flags); + pv_register_steal_time(); + local_irq_restore(flags); + return 0; +} + +static int pv_cpu_down_prepare(unsigned int cpu) +{ + unsigned long flags; + + local_irq_save(flags); + pv_disable_steal_time(); + local_irq_restore(flags); + return 0; +} +#endif +static void pv_cpu_reboot(void *unused) +{ + pv_disable_steal_time(); +} +static int pv_reboot_notify(struct notifier_block *nb, unsigned long code, + void *unused) +{ + on_each_cpu(pv_cpu_reboot, NULL, 1); + return NOTIFY_DONE; +} +static struct notifier_block pv_reboot_nb = { + .notifier_call = pv_reboot_notify, +}; +int __init pv_time_init(void) +{ + if (!cpu_has_hypervisor) + return 0; + if (!kvm_para_available()) + return 0; + + if (!(pv_feature_disabled & (1 << KVM_FEATURE_STEAL_TIME)) && + pv_feature_support(KVM_FEATURE_STEAL_TIME)) { + + register_reboot_notifier(&pv_reboot_nb); + + has_steal_clock = 1; + pv_time_ops.steal_clock = pv_steal_clock; + pv_register_steal_time(); + static_key_slow_inc(¶virt_steal_enabled); + static_key_slow_inc(¶virt_steal_rq_enabled); + +#ifdef CONFIG_SMP + if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "loongarch/pv:online", + pv_cpu_online, pv_cpu_down_prepare) < 0) + pr_err("failed to install cpu hotplug callbacks\n"); +#endif + } + return 0; +} + +static void pv_send_ipi(const struct cpumask *mask, unsigned int action) +{ + unsigned long flags; + unsigned int cpu, i, min = 0, max = 0; + u64 ipi_bitmap = 0; + long ret; + + if (cpumask_empty(mask)) + return; + + local_irq_save(flags); + + for_each_cpu(i, mask) { + cpu = cpu_logical_map(i); + if (!ipi_bitmap) { + min = max = cpu; + } else if (cpu < min && (max - cpu) < BITS_PER_LONG) { + ipi_bitmap <<= min - cpu; + min = cpu; + } else if (cpu > min && cpu < min + BITS_PER_LONG) { + max = cpu < max ? max : cpu; + } else { + ret = kvm_hypercall3(KVM_HC_FUNC_IPI, ipi_bitmap, min, action); + WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret); + min = max = cpu; + ipi_bitmap = 0; + } + __set_bit(cpu - min, (unsigned long *)&ipi_bitmap); + } + + if (ipi_bitmap) { + ret = kvm_hypercall3(KVM_HC_FUNC_IPI, ipi_bitmap, min, action); + WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret); + } + + local_irq_restore(flags); +} + +extern struct smp_ops smp_ops; +int __init pv_ipi_init(void) +{ + if (!cpu_has_hypervisor) + return 0; + if (!IS_ENABLED(CONFIG_SMP)) + return 0; + if (!kvm_para_available()) + return 0; + + if (!(pv_feature_disabled & (1 << KVM_FEATURE_MULTI_IPI)) && + pv_feature_support(KVM_FEATURE_MULTI_IPI)) { + smp_ops.send_call_func_ipi = pv_send_ipi; + } + return 0; +} + +static int __init set_pv_ipi(char *str) +{ + pv_feature_disabled |= (1 << KVM_FEATURE_MULTI_IPI); + return 0; +} +early_param("no_pvipi", set_pv_ipi); + +static int __init set_pv_time(char *str) +{ + pv_feature_disabled |= (1 << KVM_FEATURE_STEAL_TIME); + return 0; +} +early_param("no_pvtime", set_pv_time); diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 45cb89308fcf2da24fa1d6c9fd30da6be0253449..fd39844347b59c654d6b1bd8f52dafe3d9c8e1e8 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,7 @@ #include #include #include +#include #include "legacy_boot.h" #define SMBIOS_BIOSSIZE_OFFSET 0x09 @@ -334,6 +336,8 @@ void __init platform_init(void) pr_info("The BIOS Version: %s\n", b_info.bios_version); efi_runtime_init(); + + pv_ipi_init(); } static void __init check_kernel_sections_mem(void) diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 4d2cb1e2d6611373ede87672b20329fe8847d217..f5a94559d44129fe908c4603ec4ace93ecdf87dc 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "legacy_boot.h" int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ @@ -151,6 +152,22 @@ void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) ipi_write_action(cpu_logical_map(i), (u32)action); } +struct smp_ops smp_ops = { + .send_call_func_single_ipi = loongson3_send_ipi_single, + .send_call_func_ipi = loongson3_send_ipi_mask, +}; +EXPORT_SYMBOL(smp_ops); + +void arch_send_call_function_single_ipi(int cpu) +{ + smp_ops.send_call_func_single_ipi(cpu, SMP_CALL_FUNCTION); +} + +void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + smp_ops.send_call_func_ipi(mask, SMP_CALL_FUNCTION); +} + /* * This function sends a 'reschedule' IPI to another CPU. * it goes straight through and wastes no time serializing diff --git a/arch/loongarch/kernel/time.c b/arch/loongarch/kernel/time.c index 8d331a5fae5a03801a05062204c50850f03334b6..18fa38705da7a2acee3a3ee630ebcbc6461ef047 100644 --- a/arch/loongarch/kernel/time.c +++ b/arch/loongarch/kernel/time.c @@ -16,6 +16,7 @@ #include #include #include +#include u64 cpu_clock_freq; EXPORT_SYMBOL(cpu_clock_freq); @@ -228,4 +229,6 @@ void __init time_init(void) constant_clockevent_init(); constant_clocksource_init(); + + pv_time_init(); } diff --git a/arch/loongarch/kvm/loongarch.c b/arch/loongarch/kvm/loongarch.c index 03da5959aaebebfd9023ffd4a7ab3c7dfa58f30c..9e0dc6a4e6e2945ef08a929bda1272cbd96a55aa 100644 --- a/arch/loongarch/kvm/loongarch.c +++ b/arch/loongarch/kvm/loongarch.c @@ -28,6 +28,7 @@ #include "kvmcpu.h" #include #include +#include #include "intc/ls3a_ipi.h" #include "intc/ls7a_irq.h"