LoongArch: LSVZ: Support PV IPI to reduce VM exit

LoongArch inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I76XVK

--------------------------------

PV IPI with less VM exit can improve performance than
iocsr emulation when sending ipi interrupt to multi cpus.

This patch supports:
1. sending PV IPI by hypercall
2. recording and updating steal time for VM

Change-Id: Ia3ce493b83c067cc9e7c66b3094e4d87ff630f1d
Signed-off-by: Nzhoumin <zhoumin@loongson.cn>

arch/loongarch/Kconfig

@@ -561,6 +561,25 @@ config ARCH_ENABLE_THP_MIGRATION
 	def_bool y
 	depends on TRANSPARENT_HUGEPAGE
 
+config PARAVIRT
+	bool "Enable paravirtualization code"
+	help
+	  This changes the kernel so it can modify itself when it is run
+	  under a hypervisor, potentially improving performance significantly
+	  over full virtualization.
+
+config PARAVIRT_TIME_ACCOUNTING
+	bool "Paravirtual steal time accounting"
+	select PARAVIRT
+	default n
+	help
+	  Select this option to enable fine granularity task steal time
+	  accounting. Time spent executing other tasks in parallel with
+	  the current vCPU is discounted from the vCPU capacity. To account for
+	  that, there can be a small performance impact.
+
+	  If in doubt, say N here.
+
 config ARCH_MEMORY_PROBE
 	def_bool y
 	depends on MEMORY_HOTPLUG

arch/loongarch/configs/loongson3_defconfig

@@ -35,6 +35,7 @@ CONFIG_BPF_SYSCALL=y
 CONFIG_USERFAULTFD=y
 CONFIG_PERF_EVENTS=y
 # CONFIG_COMPAT_BRK is not set
+CONFIG_PARAVIRT_TIME_ACCOUNTING=y
 CONFIG_CPU_HAS_LSX=y
 CONFIG_CPU_HAS_LASX=y
 CONFIG_NUMA=y

arch/loongarch/include/asm/kvm_para.h

@@ -2,6 +2,10 @@
 #ifndef _ASM_LOONGARCH_KVM_PARA_H
 #define _ASM_LOONGARCH_KVM_PARA_H
 
+#include <uapi/asm/kvm_para.h>
+
+#define KVM_HYPERCALL ".word 0x002b8000"
+
 /*
  * Hypcall code field
  */
@@ -25,6 +29,148 @@
 #define KVM_RET_SUC	1
 #define KVM_RET_NOT_SUPPORTED	-1
 
+/*
+ * Hypercalls interface for KVM.
+ *
+ * a0: function identifier
+ * a1-a6: args
+ * Return value will be placed in v0.
+ * Up to 6 arguments are passed in a1, a2, a3, a4, a5, a6.
+ */
+static inline long kvm_hypercall0(u64 fid)
+{
+	register long ret asm("v0");
+	register unsigned long fun asm("a0") = fid;
+
+	__asm__ __volatile__(
+		KVM_HYPERCALL
+		: "=r" (ret)
+		: "r" (fun)
+		: "memory"
+	);
+
+	return ret;
+}
+
+static inline long kvm_hypercall1(u64 fid, unsigned long arg0)
+{
+	register long ret asm("v0");
+	register unsigned long fun asm("a0") = fid;
+	register unsigned long a1 asm("a1") = arg0;
+
+	__asm__ __volatile__(
+		KVM_HYPERCALL
+		: "=r" (ret)
+		: "r" (fun), "r" (a1)
+		: "memory"
+	);
+
+	return ret;
+}
+
+static inline long kvm_hypercall2(u64 fid,
+					unsigned long arg0, unsigned long arg1)
+{
+	register long ret asm("v0");
+	register unsigned long fun asm("a0") = fid;
+	register unsigned long a1 asm("a1") = arg0;
+	register unsigned long a2 asm("a2") = arg1;
+
+	__asm__ __volatile__(
+		KVM_HYPERCALL
+		: "=r" (ret)
+		: "r" (fun), "r" (a1), "r" (a2)
+		: "memory"
+	);
+
+	return ret;
+}
+
+static inline long kvm_hypercall3(u64 fid,
+	unsigned long arg0, unsigned long arg1, unsigned long arg2)
+{
+	register long ret asm("v0");
+	register unsigned long fun asm("a0") = fid;
+	register unsigned long a1 asm("a1") = arg0;
+	register unsigned long a2 asm("a2") = arg1;
+	register unsigned long a3 asm("a3") = arg2;
+
+	__asm__ __volatile__(
+		KVM_HYPERCALL
+		: "=r" (ret)
+		: "r" (fun), "r" (a1), "r" (a2), "r" (a3)
+		: "memory"
+	);
+
+	return ret;
+}
+
+static inline long kvm_hypercall4(u64 fid,
+	unsigned long arg0, unsigned long arg1, unsigned long arg2,
+	unsigned long arg3)
+{
+	register long ret asm("v0");
+	register unsigned long fun asm("a0") = fid;
+	register unsigned long a1 asm("a1") = arg0;
+	register unsigned long a2 asm("a2") = arg1;
+	register unsigned long a3 asm("a3") = arg2;
+	register unsigned long a4 asm("a4") = arg3;
+
+	__asm__ __volatile__(
+		KVM_HYPERCALL
+		: "=r" (ret)
+		: "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4)
+		: "memory"
+	);
+
+	return ret;
+}
+
+static inline long kvm_hypercall5(u64 fid,
+	unsigned long arg0, unsigned long arg1, unsigned long arg2,
+	unsigned long arg3, unsigned long arg4)
+{
+	register long ret asm("v0");
+	register unsigned long fun asm("a0") = fid;
+	register unsigned long a1 asm("a1") = arg0;
+	register unsigned long a2 asm("a2") = arg1;
+	register unsigned long a3 asm("a3") = arg2;
+	register unsigned long a4 asm("a4") = arg3;
+	register unsigned long a5 asm("a5") = arg4;
+
+	__asm__ __volatile__(
+		KVM_HYPERCALL
+		: "=r" (ret)
+		: "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4), "r" (a5)
+		: "memory"
+	);
+
+	return ret;
+}
+
+static inline long kvm_hypercall6(u64 fid,
+	unsigned long arg0, unsigned long arg1, unsigned long arg2,
+	unsigned long arg3, unsigned long arg4, unsigned long arg5)
+{
+	register long ret asm("v0");
+	register unsigned long fun asm("a0") = fid;
+	register unsigned long a1 asm("a1") = arg0;
+	register unsigned long a2 asm("a2") = arg1;
+	register unsigned long a3 asm("a3") = arg2;
+	register unsigned long a4 asm("a4") = arg3;
+	register unsigned long a5 asm("a5") = arg4;
+	register unsigned long a6 asm("a6") = arg5;
+
+	__asm__ __volatile__(
+		KVM_HYPERCALL
+		: "=r" (ret)
+		: "i"(fun), "r" (a1), "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6)
+		: "memory"
+	);
+
+	return ret;
+}
+
 static inline bool kvm_check_and_clear_guest_paused(void)
 {
 	return false;

arch/loongarch/include/asm/loongson.h

@@ -14,8 +14,6 @@
 #include <asm/addrspace.h>
 #include <asm/bootinfo.h>
 
-extern const struct plat_smp_ops loongson3_smp_ops;
-
 #define LOONGSON_REG(x) \
 	(*(volatile u32 *)((char *)TO_UNCACHE(LOONGSON_REG_BASE) + (x)))
 

arch/loongarch/include/asm/paravirt.h

+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_LOONGARCH_PARAVIRT_H
+#define _ASM_LOONGARCH_PARAVIRT_H
+#include <asm/kvm_para.h>
+
+#ifdef CONFIG_PARAVIRT
+static inline bool kvm_para_available(void)
+{
+	return true;
+}
+struct static_key;
+extern struct static_key paravirt_steal_enabled;
+extern struct static_key paravirt_steal_rq_enabled;
+
+struct pv_time_ops {
+	unsigned long long (*steal_clock)(int cpu);
+};
+struct kvm_steal_time {
+	__u64 steal;
+	__u32 version;
+	__u32 flags;
+	__u32 pad[12];
+};
+extern struct pv_time_ops pv_time_ops;
+
+static inline u64 paravirt_steal_clock(int cpu)
+{
+	return pv_time_ops.steal_clock(cpu);
+}
+
+static inline bool pv_feature_support(int feature)
+{
+	return kvm_hypercall1(KVM_HC_FUNC_FEATURE, feature) == KVM_RET_SUC;
+}
+static inline void pv_notify_host(int feature, unsigned long data)
+{
+	kvm_hypercall2(KVM_HC_FUNC_NOTIFY, feature, data);
+}
+
+int __init pv_time_init(void);
+int __init pv_ipi_init(void);
+#else
+static inline bool kvm_para_available(void)
+{
+	return false;
+}
+
+static inline int pv_time_init(void)
+{
+	return 0;
+}
+
+static inline int pv_ipi_init(void)
+{
+	return 0;
+}
+#endif
+#endif /* _ASM_LOONGARCH_PARAVIRT_H */

arch/loongarch/include/asm/smp.h

@@ -12,6 +12,11 @@
 #include <linux/threads.h>
 #include <linux/cpumask.h>
 
+struct smp_ops {
+	void (*send_call_func_ipi)(const struct cpumask *mask, unsigned int action);
+	void (*send_call_func_single_ipi)(int cpu, unsigned int action);
+};
+
 extern int smp_num_siblings;
 extern int num_processors;
 extern int disabled_cpus;
@@ -78,15 +83,8 @@ extern void calculate_cpu_foreign_map(void);
  */
 extern void show_ipi_list(struct seq_file *p, int prec);
 
-static inline void arch_send_call_function_single_ipi(int cpu)
-{
-	loongson3_send_ipi_single(cpu, SMP_CALL_FUNCTION);
-}
-
-static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
-{
-	loongson3_send_ipi_mask(mask, SMP_CALL_FUNCTION);
-}
+void arch_send_call_function_single_ipi(int cpu);
+void arch_send_call_function_ipi_mask(const struct cpumask *mask);
 
 #ifdef CONFIG_HOTPLUG_CPU
 static inline int __cpu_disable(void)

arch/loongarch/include/uapi/asm/kvm_para.h

+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _UAPI_ASM_LOONGARCH_KVM_PARA_H
+#define _UAPI_ASM_LOONGARCH_KVM_PARA_H
+
+/* Device Control API on vcpu fd */
+#define KVM_LARCH_VCPU_PVTIME_CTRL  2
+#define KVM_LARCH_VCPU_PVTIME_IPA   0
+
+#endif /* _UAPI_ASM_LOONGARCH_KVM_PARA_H */

arch/loongarch/kernel/Makefile

@@ -36,4 +36,6 @@ obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o
 
 obj-$(CONFIG_PERF_EVENTS)	+= perf_event.o perf_regs.o
 
+obj-$(CONFIG_PARAVIRT)		+= paravirt.o
+
 CPPFLAGS_vmlinux.lds		:= $(KBUILD_CFLAGS)

arch/loongarch/kernel/paravirt.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 Loongson Technology Co., Ltd.
+ *
+ * This program is free software; you can redistribute	it and/or modify it
+ * under  the terms of	the GNU General	 Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ *
+ * LoongArch paravirtualization support.
+ *
+ * Author: zhangyanyue <zhangyanyue@loongson.cn>
+ */
+#include <linux/export.h>
+#include <linux/types.h>
+#include <linux/cpu.h>
+#include <linux/hugetlb.h>
+#include <linux/kvm_para.h>
+#include <linux/init.h>
+#include <linux/reboot.h>
+#include <asm/paravirt.h>
+#include <asm/cpu.h>
+#include <asm/io.h>
+#include <asm/pgtable.h>
+
+struct static_key paravirt_steal_enabled;
+struct static_key paravirt_steal_rq_enabled;
+
+static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64);
+static int has_steal_clock;
+static unsigned int pv_feature_disabled;
+
+static u64 dummy_steal_clock(int cpu)
+{
+	return 0;
+}
+
+static u64 pv_steal_clock(int cpu)
+{
+	u64 steal;
+	struct kvm_steal_time *src;
+	int version;
+
+	src = &per_cpu(steal_time, cpu);
+	do {
+
+		version = src->version;
+		/* Make sure that the version is read before the steal. */
+		virt_rmb();
+		steal = src->steal;
+		/* Make sure that the steal is read before the next version. */
+		virt_rmb();
+
+	} while ((version & 1) || (version != src->version));
+	return steal;
+}
+
+struct pv_time_ops pv_time_ops = {
+	.steal_clock = dummy_steal_clock,
+};
+EXPORT_SYMBOL_GPL(pv_time_ops);
+
+phys_addr_t slow_virt_to_phys(const void *vaddr)
+{
+	unsigned long addr = (unsigned long) vaddr;
+	unsigned long page_mask = 0;
+	pgd_t *pgd = pgd_offset_k(addr);
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+	pte_t *pte;
+	phys_addr_t phys_addr;
+
+	/* See arch/loongarch/kernel/numa.c: setup_per_cpu_areas() */
+	if (nr_node_ids < 8)
+		return virt_to_phys((void *)vaddr);
+
+	if (pgd_none(*pgd))
+		goto out;
+
+	p4d = p4d_offset(pgd, addr);
+	if (p4d_none(*p4d))
+		goto out;
+
+	pud = pud_offset(p4d, addr);
+	if (pud_none(*pud) || pud_bad(*pud))
+		goto out;
+
+	pmd = pmd_offset(pud, addr);
+	if (pmd_none(*pmd) || pmd_bad(*pmd))
+		goto out;
+
+#ifdef KVM_HUGE_TLB_SUPPORT
+	if (pmd_huge(*pmd)) {
+		pte = (pte_t *)pmd;
+		page_mask = PMD_MASK;
+		goto out;
+	}
+#endif
+
+	pte = pte_offset_kernel(pmd, addr);
+	page_mask = PAGE_MASK;
+
+out:
+	phys_addr = (pte_pfn(*pte) << PAGE_SHIFT) | (addr & ~page_mask);
+	return phys_addr;
+}
+
+static void pv_register_steal_time(void)
+{
+	int cpu = smp_processor_id();
+	struct kvm_steal_time *st;
+
+	if (!has_steal_clock)
+		return;
+
+	st = &per_cpu(steal_time, cpu);
+	pv_notify_host(KVM_FEATURE_STEAL_TIME, slow_virt_to_phys(st));
+
+	pr_info("pv stealtime: cpu %d, st:0x%llx phys:0x%llx\n",
+			cpu, (unsigned long long)st, (unsigned long long) slow_virt_to_phys(st));
+}
+
+#ifdef CONFIG_SMP
+static void pv_disable_steal_time(void)
+{
+	if (has_steal_clock)
+		pv_notify_host(KVM_FEATURE_STEAL_TIME, 0);
+}
+
+static int pv_cpu_online(unsigned int cpu)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	pv_register_steal_time();
+	local_irq_restore(flags);
+	return 0;
+}
+
+static int pv_cpu_down_prepare(unsigned int cpu)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	pv_disable_steal_time();
+	local_irq_restore(flags);
+	return 0;
+}
+#endif
+static void pv_cpu_reboot(void *unused)
+{
+	pv_disable_steal_time();
+}
+static int pv_reboot_notify(struct notifier_block *nb, unsigned long code,
+		void *unused)
+{
+	on_each_cpu(pv_cpu_reboot, NULL, 1);
+	return NOTIFY_DONE;
+}
+static struct notifier_block pv_reboot_nb = {
+	.notifier_call	= pv_reboot_notify,
+};
+int __init pv_time_init(void)
+{
+	if (!cpu_has_hypervisor)
+		return 0;
+	if (!kvm_para_available())
+		return 0;
+
+	if (!(pv_feature_disabled & (1 << KVM_FEATURE_STEAL_TIME)) &&
+			pv_feature_support(KVM_FEATURE_STEAL_TIME)) {
+
+		register_reboot_notifier(&pv_reboot_nb);
+
+		has_steal_clock = 1;
+		pv_time_ops.steal_clock = pv_steal_clock;
+		pv_register_steal_time();
+		static_key_slow_inc(&paravirt_steal_enabled);
+		static_key_slow_inc(&paravirt_steal_rq_enabled);
+
+#ifdef CONFIG_SMP
+		if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "loongarch/pv:online",
+					pv_cpu_online, pv_cpu_down_prepare) < 0)
+			pr_err("failed to install cpu hotplug callbacks\n");
+#endif
+	}
+	return 0;
+}
+
+static void pv_send_ipi(const struct cpumask *mask, unsigned int action)
+{
+	unsigned long flags;
+	unsigned int cpu, i, min = 0, max = 0;
+	u64 ipi_bitmap = 0;
+	long ret;
+
+	if (cpumask_empty(mask))
+		return;
+
+	local_irq_save(flags);
+
+	for_each_cpu(i, mask) {
+		cpu = cpu_logical_map(i);
+		if (!ipi_bitmap) {
+			min = max = cpu;
+		} else if (cpu < min && (max - cpu) < BITS_PER_LONG) {
+			ipi_bitmap <<= min - cpu;
+			min = cpu;
+		} else if (cpu > min && cpu < min + BITS_PER_LONG) {
+			max = cpu < max ? max : cpu;
+		} else {
+			ret = kvm_hypercall3(KVM_HC_FUNC_IPI, ipi_bitmap, min, action);
+			WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret);
+			min = max = cpu;
+			ipi_bitmap = 0;
+		}
+		__set_bit(cpu - min, (unsigned long *)&ipi_bitmap);
+	}
+
+	if (ipi_bitmap) {
+		ret = kvm_hypercall3(KVM_HC_FUNC_IPI, ipi_bitmap, min, action);
+		WARN_ONCE(ret < 0, "KVM: failed to send PV IPI: %ld", ret);
+	}
+
+	local_irq_restore(flags);
+}
+
+extern struct smp_ops smp_ops;
+int __init pv_ipi_init(void)
+{
+	if (!cpu_has_hypervisor)
+		return 0;
+	if (!IS_ENABLED(CONFIG_SMP))
+		return 0;
+	if (!kvm_para_available())
+		return 0;
+
+	if (!(pv_feature_disabled & (1 << KVM_FEATURE_MULTI_IPI)) &&
+			pv_feature_support(KVM_FEATURE_MULTI_IPI)) {
+		smp_ops.send_call_func_ipi = pv_send_ipi;
+	}
+	return 0;
+}
+
+static int __init set_pv_ipi(char *str)
+{
+	pv_feature_disabled |= (1 << KVM_FEATURE_MULTI_IPI);
+	return 0;
+}
+early_param("no_pvipi", set_pv_ipi);
+
+static int __init set_pv_time(char *str)
+{
+	pv_feature_disabled |= (1 << KVM_FEATURE_STEAL_TIME);
+	return 0;
+}
+early_param("no_pvtime", set_pv_time);

arch/loongarch/kernel/setup.c

@@ -29,6 +29,7 @@
 #include <linux/device.h>
 #include <linux/dma-map-ops.h>
 #include <linux/swiotlb.h>
+#include <asm/smp.h>
 
 #include <asm/addrspace.h>
 #include <asm/alternative.h>
@@ -44,6 +45,7 @@
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/time.h>
+#include <asm/paravirt.h>
 #include "legacy_boot.h"
 
 #define SMBIOS_BIOSSIZE_OFFSET		0x09
@@ -334,6 +336,8 @@ void __init platform_init(void)
 	pr_info("The BIOS Version: %s\n", b_info.bios_version);
 
 	efi_runtime_init();
+
+	pv_ipi_init();
 }
 
 static void __init check_kernel_sections_mem(void)

arch/loongarch/kernel/smp.c

@@ -30,6 +30,7 @@
 #include <asm/processor.h>
 #include <asm/setup.h>
 #include <asm/time.h>
+#include <asm/paravirt.h>
 #include "legacy_boot.h"
 
 int __cpu_number_map[NR_CPUS];   /* Map physical to logical */
@@ -151,6 +152,22 @@ void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action)
 		ipi_write_action(cpu_logical_map(i), (u32)action);
 }
 
+struct smp_ops smp_ops = {
+	.send_call_func_single_ipi = loongson3_send_ipi_single,
+	.send_call_func_ipi = loongson3_send_ipi_mask,
+};
+EXPORT_SYMBOL(smp_ops);
+
+void arch_send_call_function_single_ipi(int cpu)
+{
+	smp_ops.send_call_func_single_ipi(cpu, SMP_CALL_FUNCTION);
+}
+
+void arch_send_call_function_ipi_mask(const struct cpumask *mask)
+{
+	smp_ops.send_call_func_ipi(mask, SMP_CALL_FUNCTION);
+}
+
 /*
  * This function sends a 'reschedule' IPI to another CPU.
  * it goes straight through and wastes no time serializing

arch/loongarch/kernel/time.c

@@ -16,6 +16,7 @@
 #include <asm/cpu-features.h>
 #include <asm/loongarch.h>
 #include <asm/time.h>
+#include <asm/paravirt.h>
 
 u64 cpu_clock_freq;
 EXPORT_SYMBOL(cpu_clock_freq);
@@ -228,4 +229,6 @@ void __init time_init(void)
 
 	constant_clockevent_init();
 	constant_clocksource_init();
+
+	pv_time_init();
 }

arch/loongarch/kvm/loongarch.c

@@ -28,6 +28,7 @@
 #include "kvmcpu.h"
 #include <asm/setup.h>
 #include <asm/time.h>
+#include <asm/paravirt.h>
 
 #include "intc/ls3a_ipi.h"
 #include "intc/ls7a_irq.h"