!467 LoongArch: add kvm support

Merge Pull Request from: @Hongchen_Zhang 
 
Added simulation of csr devices, interrupt controller devices, mmu, 
timers, etc. in kvm under loongarch structure.
kvm-based acceleration for qemu applications on 5000 servers or 5000 
desktops with loongarch cpu

Characteristic Details: 
1) cpu 
  - Support up to 64 vcpu, support vcpu hot-plug
2) memory
  - Back-end memory supports normal huge pages and transparent huge pages
  - Support balloon 
  - Support memory hot-swap
3) Peripherals
  - Support standard interfaces PCI, SATA, SCSI, USB, virtio devices
  - Support multiple NICs, multiple disks
  - Supports hot-plug of devices
4) boot
  - Support UEFI bios boot
  - Support boot order
  - Direct kernel boot support
  - Support tpm
5) Migration
  - Support for virtual machine snapshots
  - Support for virtual machine saving and recovery
  - Support for shared storage migration
  - Support for incremental migration
  - Supports full copy migration of storage

Test passed with below step:
1、Install virt-manager
2、Install libvirt
3、Install qemu
4、Download loongarch's qcow2 file
5、Create a virtual machine on virtu-manager using the qcow2 file
6、Start the virtual machine and find that the virtual machine can run normally 
 
Link:https://gitee.com/openeuler/kernel/pulls/467 

Reviewed-by: Guo Dongtai <guodongtai@kylinos.cn> 
Reviewed-by: Kevin Zhu <zhukeqian1@huawei.com> 
Signed-off-by: Zheng Zengkai <zhengzengkai@huawei.com> 
Acked-by: Zheng Zengkai <zhengzengkai@huawei.com> 

arch/loongarch/Kbuild

@@ -2,5 +2,9 @@ obj-y += kernel/
 obj-y += mm/
 obj-y += vdso/
 
+ifdef CONFIG_KVM
+obj-y += kvm/
+endif
+
 # for cleaning
 subdir- += boot

arch/loongarch/Kconfig

@@ -106,6 +106,7 @@ config LOONGARCH
 	select HAVE_SETUP_PER_CPU_AREA if NUMA
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_TIF_NOHZ
+	select HAVE_KVM
 	select HAVE_VIRT_CPU_ACCOUNTING_GEN if !SMP
 	select IRQ_FORCED_THREADING
 	select IRQ_LOONGARCH_CPU
@@ -539,3 +540,4 @@ source "drivers/cpufreq/Kconfig"
 endmenu
 
 source "drivers/firmware/Kconfig"
+source "arch/loongarch/kvm/Kconfig"

arch/loongarch/configs/loongson3_defconfig

@@ -49,6 +49,11 @@ CONFIG_CPU_FREQ_GOV_POWERSAVE=y
 CONFIG_LOONGSON3_ACPI_CPUFREQ=y
 CONFIG_EFI_CAPSULE_LOADER=m
 CONFIG_EFI_TEST=m
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=m
+CONFIG_VHOST_NET=m
+CONFIG_VHOST_SCSI=m
+CONFIG_VHOST_VSOCK=m
 CONFIG_MODULES=y
 CONFIG_MODULE_FORCE_LOAD=y
 CONFIG_MODULE_UNLOAD=y

arch/loongarch/include/asm/Kbuild

@@ -26,4 +26,3 @@ generic-y += poll.h
 generic-y += param.h
 generic-y += posix_types.h
 generic-y += resource.h
-generic-y += kvm_para.h

arch/loongarch/include/asm/inst.h

@@ -42,17 +42,18 @@ enum reg1i21_op {
 };
 
 enum reg2i12_op {
-	addiw_op	= 0x0a,
-	addid_op	= 0x0b,
-	lu52id_op	= 0x0c,
-	ldb_op		= 0xa0,
-	ldh_op		= 0xa1,
-	ldw_op		= 0xa2,
-	ldd_op		= 0xa3,
-	stb_op		= 0xa4,
-	sth_op		= 0xa5,
-	stw_op		= 0xa6,
-	std_op		= 0xa7,
+	slti_op = 0x8, sltui_op, addiw_op, addid_op,
+	lu52id_op, cache_op = 0x18, xvldreplb_op = 0xca,
+	ldb_op = 0xa0, ldh_op, ldw_op, ldd_op, stb_op, sth_op,
+	stw_op, std_op, ldbu_op, ldhu_op, ldwu_op, preld_op,
+	flds_op, fsts_op, fldd_op, fstd_op, vld_op, vst_op, xvld_op,
+	xvst_op, ldlw_op = 0xb8, ldrw_op, ldld_op, ldrd_op, stlw_op,
+	strw_op, stld_op, strd_op, vldreplb_op = 0xc2,
+};
+
+enum reg2i14_op {
+	llw_op = 0x20, scw_op, lld_op, scd_op, ldptrw_op, stptrw_op,
+	ldptrd_op, stptrd_op,
 };
 
 enum reg2i16_op {
@@ -65,6 +66,49 @@ enum reg2i16_op {
 	bgeu_op		= 0x1b,
 };
 
+enum reg3_op {
+	asrtled_op = 0x2, asrtgtd_op,
+	addw_op = 0x20, addd_op, subw_op, subd_op,
+	slt_op, sltu_op, maskeqz_op, masknez_op,
+	nor_op, and_op, or_op, xor_op, orn_op,
+	andn_op, sllw_op, srlw_op, sraw_op, slld_op,
+	srld_op, srad_op, rotrb_op, rotrh_op,
+	rotrw_op, rotrd_op, mulw_op, mulhw_op,
+	mulhwu_op, muld_op, mulhd_op, mulhdu_op,
+	mulwdw_op, mulwdwu_op, divw_op, modw_op,
+	divwu_op, modwu_op, divd_op, modd_op,
+	divdu_op, moddu_op, crcwbw_op,
+	crcwhw_op, crcwww_op, crcwdw_op, crccwbw_op,
+	crccwhw_op, crccwww_op, crccwdw_op, addu12iw_op,
+	addu12id_op,
+	adcb_op = 0x60, adch_op, adcw_op, adcd_op,
+	sbcb_op, sbch_op, sbcw_op, sbcd_op,
+	rcrb_op, rcrh_op, rcrw_op, rcrd_op,
+	ldxb_op = 0x7000, ldxh_op = 0x7008, ldxw_op = 0x7010, ldxd_op = 0x7018,
+	stxb_op = 0x7020, stxh_op = 0x7028, stxw_op = 0x7030, stxd_op = 0x7038,
+	ldxbu_op = 0x7040, ldxhu_op = 0x7048, ldxwu_op = 0x7050,
+	preldx_op = 0x7058, fldxs_op = 0x7060, fldxd_op = 0x7068,
+	fstxs_op = 0x7070, fstxd_op = 0x7078, vldx_op = 0x7080,
+	vstx_op = 0x7088, xvldx_op = 0x7090, xvstx_op = 0x7098,
+	amswapw_op = 0x70c0, amswapd_op, amaddw_op, amaddd_op, amandw_op,
+	amandd_op, amorw_op, amord_op, amxorw_op, amxord_op, ammaxw_op,
+	ammaxd_op, amminw_op, ammind_op, ammaxwu_op, ammaxdu_op,
+	amminwu_op, ammindu_op, amswap_dbw_op, amswap_dbd_op, amadd_dbw_op,
+	amadd_dbd_op, amand_dbw_op, amand_dbd_op, amor_dbw_op, amor_dbd_op,
+	amxor_dbw_op, amxor_dbd_op, ammax_dbw_op, ammax_dbd_op, ammin_dbw_op,
+	ammin_dbd_op, ammax_dbwu_op, ammax_dbdu_op, ammin_dbwu_op,
+	ammin_dbdu_op, fldgts_op = 0x70e8, fldgtd_op,
+	fldles_op, fldled_op, fstgts_op, fstgtd_op, fstles_op, fstled_op,
+	ldgtb_op, ldgth_op, ldgtw_op, ldgtd_op, ldleb_op, ldleh_op, ldlew_op,
+	ldled_op, stgtb_op, stgth_op, stgtw_op, stgtd_op, stleb_op, stleh_op,
+	stlew_op, stled_op,
+};
+
+enum reg2_op {
+	iocsrrdb_op = 0x19200, iocsrrdh_op, iocsrrdw_op, iocsrrdd_op,
+	iocsrwrb_op, iocsrwrh_op, iocsrwrw_op, iocsrwrd_op,
+};
+
 struct reg0i26_format {
 	unsigned int immediate_h : 10;
 	unsigned int immediate_l : 16;
@@ -84,6 +128,12 @@ struct reg1i21_format {
 	unsigned int opcode : 6;
 };
 
+struct reg2_format {
+	unsigned int rd : 5;
+	unsigned int rj : 5;
+	unsigned int opcode : 22;
+};
+
 struct reg2i12_format {
 	unsigned int rd : 5;
 	unsigned int rj : 5;
@@ -91,6 +141,18 @@ struct reg2i12_format {
 	unsigned int opcode : 10;
 };
 
+struct reg2i14_format {
+	unsigned int rd : 5;
+	unsigned int rj : 5;
+	unsigned int simmediate : 14;
+	unsigned int opcode : 8;
+};
+
+struct reg0i15_format {
+	unsigned int simmediate	: 15;
+	unsigned int opcode	: 17;
+};
+
 struct reg2i16_format {
 	unsigned int rd : 5;
 	unsigned int rj : 5;
@@ -98,13 +160,32 @@ struct reg2i16_format {
 	unsigned int opcode : 6;
 };
 
+struct reg3_format {
+	unsigned int rd : 5;
+	unsigned int rj : 5;
+	unsigned int rk : 5;
+	unsigned int opcode : 17;
+};
+
+struct reg2csr_format {
+	unsigned int rd : 5;
+	unsigned int rj : 5;
+	unsigned int csr : 14;
+	unsigned int opcode : 8;
+};
+
 union loongarch_instruction {
 	unsigned int word;
 	struct reg0i26_format reg0i26_format;
 	struct reg1i20_format reg1i20_format;
 	struct reg1i21_format reg1i21_format;
+	struct reg3_format reg3_format;
+	struct reg2_format reg2_format;
 	struct reg2i12_format reg2i12_format;
+	struct reg2i14_format reg2i14_format;
 	struct reg2i16_format reg2i16_format;
+	struct reg2csr_format	reg2csr_format;
+	struct reg0i15_format	reg0i15_format;
 };
 
 #define LOONGARCH_INSN_SIZE	sizeof(union loongarch_instruction)

arch/loongarch/include/asm/kvm_host.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef __LOONGARCH_KVM_HOST_H__
+#define __LOONGARCH_KVM_HOST_H__
+
+#include <linux/cpumask.h>
+#include <linux/mutex.h>
+#include <linux/hrtimer.h>
+#include <linux/interrupt.h>
+#include <linux/types.h>
+#include <linux/kvm.h>
+#include <linux/kvm_types.h>
+#include <linux/threads.h>
+#include <linux/spinlock.h>
+#include <asm/inst.h>
+
+/* Loongarch KVM register ids */
+#define LOONGARCH_CSR_32(_R, _S)					\
+	(KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U32 | (8 * (_R) + (_S)))
+
+#define LOONGARCH_CSR_64(_R, _S)					\
+	(KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U64 | (8 * (_R) + (_S)))
+
+#define KVM_IOC_CSRID(id)	LOONGARCH_CSR_64(id, 0)
+#define KVM_GET_IOC_CSRIDX(id)	((id & KVM_CSR_IDX_MASK) >> 3)
+
+#define LOONGSON_VIRT_REG_BASE	0x1f000000
+#define KVM_MAX_VCPUS		256
+#define KVM_USER_MEM_SLOTS	256
+/* memory slots that does not exposed to userspace */
+#define KVM_PRIVATE_MEM_SLOTS	0
+
+#define KVM_HALT_POLL_NS_DEFAULT 500000
+
+#define KVM_REQ_RECORD_STEAL	KVM_ARCH_REQ(1)
+#define KVM_INVALID_ADDR		0xdeadbeef
+#define KVM_HVA_ERR_BAD			(-1UL)
+#define KVM_HVA_ERR_RO_BAD		(-2UL)
+static inline bool kvm_is_error_hva(unsigned long addr)
+{
+	return IS_ERR_VALUE(addr);
+}
+
+struct kvm_vm_stat {
+	ulong remote_tlb_flush;
+	u64 vm_ioctl_irq_line;
+	u64 ls7a_ioapic_update;
+	u64 ls7a_ioapic_set_irq;
+	u64 ioapic_reg_write;
+	u64 ioapic_reg_read;
+	u64 set_ls7a_ioapic;
+	u64 get_ls7a_ioapic;
+	u64 set_ls3a_ext_irq;
+	u64 get_ls3a_ext_irq;
+	u64 trigger_ls3a_ext_irq;
+	u64 pip_read_exits;
+	u64 pip_write_exits;
+	u64 ls7a_msi_irq;
+};
+struct kvm_vcpu_stat {
+	u64 excep_exits[EXCCODE_INT_START];
+	u64 idle_exits;
+	u64 signal_exits;
+	u64 int_exits;
+	u64 rdcsr_cpu_feature_exits;
+	u64 rdcsr_misc_func_exits;
+	u64 rdcsr_ipi_access_exits;
+	u64 cpucfg_exits;
+	u64 huge_dec_exits;
+	u64 huge_thp_exits;
+	u64 huge_adjust_exits;
+	u64 huge_set_exits;
+	u64 huge_merge_exits;
+	u64 halt_successful_poll;
+	u64 halt_attempted_poll;
+	u64 halt_poll_success_ns;
+	u64 halt_poll_fail_ns;
+	u64 halt_poll_invalid;
+	u64 halt_wakeup;
+};
+
+#define KVM_MEMSLOT_DISABLE_THP		(1UL << 17)
+struct kvm_arch_memory_slot {
+	unsigned int flags;
+};
+
+enum {
+	IOCSR_FEATURES,
+	IOCSR_VENDOR,
+	IOCSR_CPUNAME,
+	IOCSR_NODECNT,
+	IOCSR_MISC_FUNC,
+	IOCSR_MAX
+};
+
+struct kvm_context {
+	unsigned long gid_mask;
+	unsigned long gid_ver_mask;
+	unsigned long gid_fisrt_ver;
+	unsigned long vpid_cache;
+	struct kvm_vcpu *last_vcpu;
+};
+
+struct kvm_arch {
+	/* Guest physical mm */
+	struct mm_struct gpa_mm;
+	/* Mask of CPUs needing GPA ASID flush */
+	cpumask_t asid_flush_mask;
+
+	unsigned char online_vcpus;
+	unsigned char is_migrate;
+	s64 stablecounter_gftoffset;
+	u32 cpucfg_lasx;
+	struct ls7a_kvm_ioapic *v_ioapic;
+	struct ls3a_kvm_ipi *v_gipi;
+	struct ls3a_kvm_routerirq *v_routerirq;
+	struct ls3a_kvm_extirq *v_extirq;
+	spinlock_t iocsr_lock;
+	struct kvm_iocsr_entry iocsr[IOCSR_MAX];
+	struct kvm_cpucfg cpucfgs;
+	struct kvm_context __percpu *vmcs;
+};
+
+
+#define LOONGARCH_CSRS	0x100
+#define CSR_UCWIN_BASE	0x100
+#define CSR_UCWIN_SIZE	0x10
+#define CSR_DMWIN_BASE	0x180
+#define CSR_DMWIN_SIZE	0x4
+#define CSR_PERF_BASE	0x200
+#define CSR_PERF_SIZE	0x8
+#define CSR_DEBUG_BASE	0x500
+#define CSR_DEBUG_SIZE	0x3
+#define CSR_ALL_SIZE	0x800
+
+struct loongarch_csrs {
+	unsigned long csrs[CSR_ALL_SIZE];
+};
+
+/* Resume Flags */
+#define RESUME_FLAG_DR		(1<<0)	/* Reload guest nonvolatile state? */
+#define RESUME_FLAG_HOST	(1<<1)	/* Resume host? */
+
+#define RESUME_GUEST		0
+#define RESUME_GUEST_DR		RESUME_FLAG_DR
+#define RESUME_HOST		RESUME_FLAG_HOST
+
+enum emulation_result {
+	EMULATE_DONE,		/* no further processing */
+	EMULATE_DO_MMIO,	/* kvm_run filled with MMIO request */
+	EMULATE_FAIL,		/* can't emulate this instruction */
+	EMULATE_WAIT,		/* WAIT instruction */
+	EMULATE_PRIV_FAIL,
+	EMULATE_EXCEPT,		/* A guest exception has been generated */
+	EMULATE_PV_HYPERCALL,	/* HYPCALL instruction */
+	EMULATE_DEBUG,		/* Emulate guest kernel debug */
+	EMULATE_DO_IOCSR,	/* handle IOCSR request */
+};
+
+#define KVM_LARCH_FPU		(0x1 << 0)
+#define KVM_LARCH_LSX		(0x1 << 1)
+#define KVM_LARCH_LASX		(0x1 << 2)
+#define KVM_LARCH_DATA_HWBP	(0x1 << 3)
+#define KVM_LARCH_INST_HWBP	(0x1 << 4)
+#define KVM_LARCH_HWBP		(KVM_LARCH_DATA_HWBP | KVM_LARCH_INST_HWBP)
+#define KVM_LARCH_RESET		(0x1 << 5)
+#define KVM_LARCH_PERF		(0x1 << 6)
+
+struct kvm_vcpu_arch {
+	unsigned long guest_eentry;
+	unsigned long host_eentry;
+	int (*vcpu_run)(struct kvm_run *run, struct kvm_vcpu *vcpu);
+	int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu);
+
+	/* Host registers preserved across guest mode execution */
+	unsigned long host_stack;
+	unsigned long host_gp;
+	unsigned long host_pgd;
+	unsigned long host_pgdhi;
+	unsigned long host_entryhi;
+
+	/* Host CSR registers used when handling exits from guest */
+	unsigned long badv;
+	unsigned long host_estat;
+	unsigned long badi;
+	unsigned long host_ecfg;
+	unsigned long host_percpu;
+
+	u32 is_hypcall;
+	/* GPRS */
+	unsigned long gprs[32];
+	unsigned long pc;
+
+	/* FPU State */
+	struct loongarch_fpu fpu FPU_ALIGN;
+	/* Which auxiliary state is loaded (KVM_LOONGARCH_AUX_*) */
+	unsigned int aux_inuse;
+
+	/* CSR State */
+	struct loongarch_csrs *csr;
+
+	/* GPR used as IO source/target */
+	u32 io_gpr;
+
+	struct hrtimer swtimer;
+	/* Count timer control KVM register */
+	u32 count_ctl;
+
+	/* Bitmask of exceptions that are pending */
+	unsigned long irq_pending;
+	/* Bitmask of pending exceptions to be cleared */
+	unsigned long irq_clear;
+
+	/* Cache some mmu pages needed inside spinlock regions */
+	struct kvm_mmu_memory_cache mmu_page_cache;
+
+	/* vcpu's vpid is different on each host cpu in an smp system */
+	u64 vpid[NR_CPUS];
+
+	/* Period of stable timer tick in ns */
+	u64 timer_period;
+	/* Frequency of stable timer in Hz */
+	u64 timer_mhz;
+	/* Stable bias from the raw time */
+	u64 timer_bias;
+	/* Dynamic nanosecond bias (multiple of timer_period) to avoid overflow */
+	s64 timer_dyn_bias;
+	/* Save ktime */
+	ktime_t stable_ktime_saved;
+
+	u64 core_ext_ioisr[4];
+
+	/* Last CPU the VCPU state was loaded on */
+	int last_sched_cpu;
+	/* Last CPU the VCPU actually executed guest code on */
+	int last_exec_cpu;
+
+	u8 fpu_enabled;
+	u8 lsx_enabled;
+	/* paravirt steal time */
+	struct {
+		u64 guest_addr;
+		u64 last_steal;
+		struct gfn_to_pfn_cache cache;
+	} st;
+	struct kvm_guest_debug_arch guest_debug;
+	/* save host pmu csr */
+	u64 perf_ctrl[4];
+	u64 perf_cntr[4];
+
+};
+
+static inline unsigned long readl_sw_gcsr(struct loongarch_csrs *csr, int reg)
+{
+	return csr->csrs[reg];
+}
+
+static inline void writel_sw_gcsr(struct loongarch_csrs *csr, int reg, \
+		unsigned long val)
+{
+	csr->csrs[reg] = val;
+}
+
+/* Helpers */
+static inline bool _kvm_guest_has_fpu(struct kvm_vcpu_arch *arch)
+{
+	return cpu_has_fpu && arch->fpu_enabled;
+}
+
+
+static inline bool _kvm_guest_has_lsx(struct kvm_vcpu_arch *arch)
+{
+	return cpu_has_lsx && arch->lsx_enabled;
+}
+
+bool _kvm_guest_has_lasx(struct kvm_vcpu *vcpu);
+void _kvm_init_fault(void);
+
+/* Debug: dump vcpu state */
+int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
+
+/* MMU handling */
+int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write);
+void kvm_flush_tlb_all(void);
+void _kvm_destroy_mm(struct kvm *kvm);
+pgd_t *kvm_pgd_alloc(void);
+void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
+
+enum _kvm_fault_result {
+	KVM_LOONGARCH_MAPPED = 0,
+	KVM_LOONGARCH_GVA,
+	KVM_LOONGARCH_GPA,
+	KVM_LOONGARCH_TLB,
+	KVM_LOONGARCH_TLBINV,
+	KVM_LOONGARCH_TLBMOD,
+};
+
+#define KVM_ARCH_WANT_MMU_NOTIFIER
+int kvm_unmap_hva_range(struct kvm *kvm,
+			unsigned long start, unsigned long end, bool blockable);
+int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
+
+static inline void update_pc(struct kvm_vcpu_arch *arch)
+{
+	arch->pc += 4;
+}
+
+/**
+ * kvm_is_ifetch_fault() - Find whether a TLBL exception is due to ifetch fault.
+ * @vcpu:	Virtual CPU.
+ *
+ * Returns:	Whether the TLBL exception was likely due to an instruction
+ *		fetch fault rather than a data load fault.
+ */
+static inline bool kvm_is_ifetch_fault(struct kvm_vcpu_arch *arch)
+{
+	if (arch->pc == arch->badv)
+		return true;
+
+	return false;
+}
+
+/* Misc */
+static inline void kvm_arch_hardware_unsetup(void) {}
+static inline void kvm_arch_sync_events(struct kvm *kvm) {}
+static inline void kvm_arch_free_memslot(struct kvm *kvm,
+					struct kvm_memory_slot *slot) {}
+static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {}
+static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
+static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
+
+extern int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu);
+extern void kvm_exception_entry(void);
+#endif /* __LOONGARCH_KVM_HOST_H__ */

arch/loongarch/include/asm/kvm_para.h

+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_LOONGARCH_KVM_PARA_H
+#define _ASM_LOONGARCH_KVM_PARA_H
+
+/*
+ * Hypcall code field
+ */
+#define KVM_HC_CODE_SERIVCE     0x0
+#define KVM_HC_CODE_SWDBG       0x5
+/*
+ *	function id
+ *	0x00000 ~ 0xfffff      Standard Hypervisor Calls
+ */
+#define KVM_HC_FUNC_FEATURE	0x0
+#define KVM_HC_FUNC_NOTIFY	0x1
+#define KVM_HC_FUNC_IPI		0x2
+/*
+ * LoongArch support PV feature list
+ */
+#define KVM_FEATURE_STEAL_TIME	0
+#define KVM_FEATURE_MULTI_IPI	1
+/*
+ * LoongArch hypcall return code
+ */
+#define KVM_RET_SUC	1
+#define KVM_RET_NOT_SUPPORTED	-1
+
+static inline bool kvm_check_and_clear_guest_paused(void)
+{
+	return false;
+}
+
+static inline unsigned int kvm_arch_para_features(void)
+{
+	return 0;
+}
+
+static inline unsigned int kvm_arch_para_hints(void)
+{
+	return 0;
+}
+
+#endif /* _ASM_LOONGARCH_KVM_PARA_H */

arch/loongarch/include/asm/kvm_types.h

+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_LOONGARCH64_KVM_TYPES_H
+#define _ASM_LOONGARCH64_KVM_TYPES_H
+
+#define KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 4
+
+#endif /* _ASM_LOONGARCH64_KVM_TYPES_H */
+

arch/loongarch/include/asm/smp.h

@@ -78,16 +78,6 @@ extern void calculate_cpu_foreign_map(void);
  */
 extern void show_ipi_list(struct seq_file *p, int prec);
 
-/*
- * This function sends a 'reschedule' IPI to another CPU.
- * it goes straight through and wastes no time serializing
- * anything. Worst case is that we lose a reschedule ...
- */
-static inline void smp_send_reschedule(int cpu)
-{
-	loongson3_send_ipi_single(cpu, SMP_RESCHEDULE);
-}
-
 static inline void arch_send_call_function_single_ipi(int cpu)
 {
 	loongson3_send_ipi_single(cpu, SMP_CALL_FUNCTION);

arch/loongarch/include/uapi/asm/kvm.h

+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2020  Loongson Technologies, Inc.  All rights reserved.
+ * Authors: Sanjay Lal <sanjayl@kymasys.com>
+ * Authors: Xing Li <lixing@loongson.cn>
+ */
+
+#ifndef __LINUX_KVM_LOONGARCH_H
+#define __LINUX_KVM_LOONGARCH_H
+
+#include <linux/types.h>
+#ifndef __KERNEL__
+#include <stdint.h>
+#endif
+
+#define __KVM_HAVE_GUEST_DEBUG
+#define KVM_GUESTDBG_USE_SW_BP 0x00010000
+#define KVM_GUESTDBG_USE_HW_BP 0x00020000
+#define KVM_DATA_HW_BREAKPOINT_NUM 8
+#define KVM_INST_HW_BREAKPOINT_NUM 8
+
+/*
+ * KVM Loongarch specific structures and definitions.
+ *
+ * Some parts derived from the x86 version of this file.
+ */
+
+#define __KVM_HAVE_READONLY_MEM
+
+#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
+
+/*
+ * for KVM_GET_REGS and KVM_SET_REGS
+ */
+struct kvm_regs {
+	/* out (KVM_GET_REGS) / in (KVM_SET_REGS) */
+	__u64 gpr[32];
+	__u64 pc;
+};
+
+/*
+ * for KVM_GET_CPUCFG
+ */
+struct kvm_cpucfg {
+	/* out (KVM_GET_CPUCFG) */
+	__u32 cpucfg[64];
+};
+
+/*
+ * for KVM_GET_FPU and KVM_SET_FPU
+ */
+struct kvm_fpu {
+	__u32 fcsr;
+	__u32 none;
+	__u64 fcc;    /* 8x8 */
+	struct kvm_fpureg {
+		__u64 val64[4];	//support max 256 bits
+	} fpr[32];
+};
+
+/*
+ * For LOONGARCH, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access various
+ * registers.  The id field is broken down as follows:
+ *
+ *  bits[63..52] - As per linux/kvm.h
+ *  bits[51..32] - Must be zero.
+ *  bits[31..16] - Register set.
+ *
+ * Register set = 0: GP registers from kvm_regs (see definitions below).
+ *
+ * Register set = 1: CSR registers.
+ *
+ * Register set = 2: KVM specific registers (see definitions below).
+ *
+ * Register set = 3: FPU / SIMD registers (see definitions below).
+ *
+ * Other sets registers may be added in the future.  Each set would
+ * have its own identifier in bits[31..16].
+ */
+
+#define KVM_REG_LOONGARCH_GP		(KVM_REG_LOONGARCH | 0x00000ULL)
+#define KVM_REG_LOONGARCH_CSR		(KVM_REG_LOONGARCH | 0x10000ULL)
+#define KVM_REG_LOONGARCH_KVM		(KVM_REG_LOONGARCH | 0x20000ULL)
+#define KVM_REG_LOONGARCH_FPU		(KVM_REG_LOONGARCH | 0x30000ULL)
+#define KVM_REG_LOONGARCH_MASK		(KVM_REG_LOONGARCH | 0x30000ULL)
+#define KVM_CSR_IDX_MASK		(0x10000 - 1)
+
+/*
+ * KVM_REG_LOONGARCH_KVM - KVM specific control registers.
+ */
+
+#define KVM_REG_LOONGARCH_COUNTER	(KVM_REG_LOONGARCH_KVM | KVM_REG_SIZE_U64 | 3)
+#define KVM_REG_LOONGARCH_VCPU_RESET	(KVM_REG_LOONGARCH_KVM | KVM_REG_SIZE_U64 | 4)
+
+#define __KVM_HAVE_IRQ_LINE
+
+struct kvm_debug_exit_arch {
+	__u64 era;
+	__u32 fwps;
+	__u32 mwps;
+	__u32 exception;
+};
+
+/* for KVM_SET_GUEST_DEBUG */
+struct hw_breakpoint {
+    __u64 addr;
+    __u64 mask;
+    __u32 asid;
+    __u32 ctrl;
+};
+
+struct kvm_guest_debug_arch {
+	struct hw_breakpoint data_breakpoint[KVM_DATA_HW_BREAKPOINT_NUM];
+	struct hw_breakpoint inst_breakpoint[KVM_INST_HW_BREAKPOINT_NUM];
+	int inst_bp_nums, data_bp_nums;
+};
+
+/* definition of registers in kvm_run */
+struct kvm_sync_regs {
+};
+
+/* dummy definition */
+struct kvm_sregs {
+};
+
+struct kvm_iocsr_entry {
+	__u32 addr;
+	__u32 pad;
+	__u64 data;
+};
+
+struct kvm_csr_entry {
+	__u32 index;
+	__u32 reserved;
+	__u64 data;
+};
+
+/* for KVM_GET_MSRS and KVM_SET_MSRS */
+struct kvm_msrs {
+	__u32 ncsrs; /* number of msrs in entries */
+	__u32 pad;
+
+	struct kvm_csr_entry entries[0];
+};
+
+struct kvm_loongarch_interrupt {
+	/* in */
+	__u32 cpu;
+	__u32 irq;
+};
+
+#define KVM_IRQCHIP_LS7A_IOAPIC	0x0
+#define KVM_IRQCHIP_LS3A_GIPI	0x1
+#define KVM_IRQCHIP_LS3A_HT_IRQ	0x2
+#define KVM_IRQCHIP_LS3A_ROUTE	0x3
+#define KVM_IRQCHIP_LS3A_EXTIRQ	0x4
+#define KVM_IRQCHIP_LS3A_IPMASK	0x5
+#define KVM_NR_IRQCHIPS          1
+#define KVM_IRQCHIP_NUM_PINS    64
+
+#define KVM_MAX_CORES			256
+#define KVM_EXTIOI_IRQS			(256)
+#define KVM_EXTIOI_IRQS_BITMAP_SIZE	(KVM_EXTIOI_IRQS / 8)
+/* map to ipnum per 32 irqs */
+#define KVM_EXTIOI_IRQS_IPMAP_SIZE	(KVM_EXTIOI_IRQS / 32)
+#define KVM_EXTIOI_IRQS_PER_GROUP	32
+#define KVM_EXTIOI_IRQS_COREMAP_SIZE	(KVM_EXTIOI_IRQS)
+#define KVM_EXTIOI_IRQS_NODETYPE_SIZE	16
+
+struct ls7a_ioapic_state {
+	/* 0x000 interrupt id register */
+	__u64 int_id;
+	/* 0x020 interrupt mask register */
+	__u64 int_mask;
+	/* 0x040 1=msi */
+	__u64 htmsi_en;
+	/* 0x060 edge=1 level  =0 */
+	__u64 intedge;
+	/* 0x080 for clean edge int,set 1 clean,set 0 is noused */
+	__u64 intclr;
+	/* 0x0c0 */
+	__u64 auto_crtl0;
+	/* 0x0e0 */
+	__u64 auto_crtl1;
+	/* 0x100 - 0x140 */
+	__u8 route_entry[64];
+	/* 0x200 - 0x240 */
+	__u8 htmsi_vector[64];
+	/* 0x300 */
+	__u64 intisr_chip0;
+	/* 0x320 */
+	__u64 intisr_chip1;
+	/* edge detection */
+	__u64 last_intirr;
+	/* 0x380 interrupt request register */
+	__u64 intirr;
+	/* 0x3a0 interrupt service register */
+	__u64 intisr;
+	/* 0x3e0 interrupt level polarity selection register,
+	 * 0 for high level tirgger
+	 */
+	__u64 int_polarity;
+};
+
+struct loongarch_gipi_single {
+	__u32 status;
+	__u32 en;
+	__u32 set;
+	__u32 clear;
+	__u64 buf[4];
+};
+
+struct loongarch_gipiState {
+	struct loongarch_gipi_single core[KVM_MAX_CORES];
+};
+
+struct kvm_loongarch_ls3a_extirq_state {
+	union ext_en_r {
+		uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE];
+	} ext_en_r;
+	union bounce_r {
+		uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE];
+	} bounce_r;
+	union ext_isr_r {
+		uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE];
+	} ext_isr_r;
+	union ext_core_isr_r {
+		uint64_t reg_u64[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE];
+	} ext_core_isr_r;
+	union ip_map_r {
+		uint64_t reg_u64;
+		uint32_t reg_u32[KVM_EXTIOI_IRQS_IPMAP_SIZE / 4];
+		uint8_t reg_u8[KVM_EXTIOI_IRQS_IPMAP_SIZE];
+	} ip_map_r;
+	union core_map_r {
+		uint64_t reg_u64[KVM_EXTIOI_IRQS_COREMAP_SIZE / 8];
+		uint32_t reg_u32[KVM_EXTIOI_IRQS_COREMAP_SIZE / 4];
+		uint8_t reg_u8[KVM_EXTIOI_IRQS_COREMAP_SIZE];
+	} core_map_r;
+	union node_type_r {
+		uint64_t reg_u64[KVM_EXTIOI_IRQS_NODETYPE_SIZE / 4];
+		uint32_t reg_u32[KVM_EXTIOI_IRQS_NODETYPE_SIZE / 2];
+		uint16_t reg_u16[KVM_EXTIOI_IRQS_NODETYPE_SIZE];
+		uint8_t reg_u8[KVM_EXTIOI_IRQS_NODETYPE_SIZE * 2];
+	} node_type_r;
+};
+
+struct loongarch_kvm_irqchip {
+	__u16 chip_id;
+	__u16 len;
+	__u16 vcpu_id;
+	__u16 reserved;
+	char data[0];
+};
+
+#endif /* __LINUX_KVM_LOONGARCH_H */

arch/loongarch/kernel/asm-offsets.c

@@ -9,6 +9,7 @@
 #include <linux/mm.h>
 #include <linux/kbuild.h>
 #include <linux/suspend.h>
+#include <linux/kvm_host.h>
 #include <asm/cpu-info.h>
 #include <asm/ptrace.h>
 #include <asm/processor.h>
@@ -269,3 +270,35 @@ void output_pbe_defines(void)
 }
 #endif
 
+void output_kvm_defines(void)
+{
+	COMMENT(" KVM/LOONGISA Specific offsets. ");
+
+	OFFSET(VCPU_FCSR0, kvm_vcpu_arch, fpu.fcsr);
+	OFFSET(VCPU_FCC, kvm_vcpu_arch, fpu.fcc);
+	BLANK();
+
+	OFFSET(KVM_VCPU_ARCH, kvm_vcpu, arch);
+	OFFSET(KVM_VCPU_KVM, kvm_vcpu, kvm);
+	OFFSET(KVM_VCPU_RUN, kvm_vcpu, run);
+	BLANK();
+
+	OFFSET(KVM_ARCH_HSTACK, kvm_vcpu_arch, host_stack);
+	OFFSET(KVM_ARCH_HGP, kvm_vcpu_arch, host_gp);
+	OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit);
+	OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd);
+	OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry);
+	OFFSET(KVM_ARCH_GPC, kvm_vcpu_arch, pc);
+	OFFSET(KVM_ARCH_GGPR, kvm_vcpu_arch, gprs);
+	OFFSET(KVM_ARCH_HESTAT, kvm_vcpu_arch, host_estat);
+	OFFSET(KVM_ARCH_HBADV, kvm_vcpu_arch, badv);
+	OFFSET(KVM_ARCH_HBADI, kvm_vcpu_arch, badi);
+	OFFSET(KVM_ARCH_ISHYPCALL, kvm_vcpu_arch, is_hypcall);
+	OFFSET(KVM_ARCH_HECFG, kvm_vcpu_arch, host_ecfg);
+	OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry);
+	OFFSET(KVM_ARCH_HPERCPU, kvm_vcpu_arch, host_percpu);
+
+	OFFSET(KVM_GPGD, kvm, arch.gpa_mm.pgd);
+	BLANK();
+}
+

arch/loongarch/kernel/smp.c

@@ -151,6 +151,17 @@ void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action)
 		ipi_write_action(cpu_logical_map(i), (u32)action);
 }
 
+/*
+ * This function sends a 'reschedule' IPI to another CPU.
+ * it goes straight through and wastes no time serializing
+ * anything. Worst case is that we lose a reschedule ...
+ */
+void smp_send_reschedule(int cpu)
+{
+	loongson3_send_ipi_single(cpu, SMP_RESCHEDULE);
+}
+EXPORT_SYMBOL_GPL(smp_send_reschedule);
+
 irqreturn_t loongson3_ipi_interrupt(int irq, void *dev)
 {
 	unsigned int action;

arch/loongarch/kvm/Kconfig

+# SPDX-License-Identifier: GPL-2.0
+#
+# KVM configuration
+#
+source "virt/kvm/Kconfig"
+
+menuconfig VIRTUALIZATION
+	bool "Virtualization"
+	help
+	  Say Y here to get to see options for using your Linux host to run
+	  other operating systems inside virtual machines (guests).
+	  This option alone does not add any kernel code.
+
+	  If you say N, all options in this submenu will be skipped and disabled.
+
+if VIRTUALIZATION
+
+config KVM
+	tristate "Kernel-based Virtual Machine (KVM) support"
+	depends on HAVE_KVM
+	select EXPORT_UASM
+	select PREEMPT_NOTIFIERS
+	select ANON_INODES
+	select KVM_GENERIC_DIRTYLOG_READ_PROTECT
+	select HAVE_KVM_VCPU_ASYNC_IOCTL
+	select KVM_MMIO
+	select MMU_NOTIFIER
+	select HAVE_KVM_IRQCHIP
+	select HAVE_KVM_IRQFD
+	select HAVE_KVM_IRQ_ROUTING
+	select HAVE_KVM_EVENTFD
+	select HAVE_KVM_MSI
+	select SRCU
+	select KVM_VFIO
+	help
+	  Support for hosting Guest kernels.
+
+choice
+	prompt "Virtualization mode"
+	depends on KVM
+	default KVM_LOONGARCH_LVZ
+
+config KVM_LOONGARCH_LVZ
+	bool "LOONGARCH Virtualization (VZ) ASE"
+	help
+	  Use the LOONGARCH Virtualization (VZ) ASE to virtualize guests. This
+	  supports running unmodified guest kernels, but requires hardware
+	  support.
+
+endchoice
+
+source "drivers/vhost/Kconfig"
+
+endif # VIRTUALIZATION

arch/loongarch/kvm/Makefile

+# SPDX-License-Identifier: GPL-2.0
+# Makefile for KVM support for LoongArch
+#
+
+OBJECT_FILES_NON_STANDARD_entry.o	:= y
+
+common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o \
+				irqchip.o eventfd.o)
+
+KVM := ../../../virt/kvm
+common-objs-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o
+
+EXTRA_CFLAGS += -Ivirt/kvm -Iarch/loongarch/kvm
+
+kvm-objs := $(common-objs-y) loongarch.o emulate.o interrupt.o
+kvm-objs += hypcall.o
+kvm-objs += mmu.o
+kvm-objs += kvm_compat.o
+
+kvm-objs		+= exit.o intc/ls7a_irq.o intc/ls3a_ipi.o intc/irqchip-debug.o\
+				timer.o intc/ls3a_ext_irq.o irqfd.o csr.o
+obj-$(CONFIG_KVM)	+= kvm.o
+obj-y			+= entry.o fpu.o

arch/loongarch/kvm/csr.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/kvm_host.h>
+#include <asm/inst.h>
+#include <asm/numa.h>
+#include "kvmcpu.h"
+#include "intc/ls3a_ipi.h"
+#include "intc/ls3a_ext_irq.h"
+#include "kvm_compat.h"
+#include "kvmcsr.h"
+#include "irq.h"
+
+#define CASE_READ_SW_GCSR(csr, regid, csrid)                    \
+	do {                                                    \
+		if (regid == csrid) {                           \
+			return kvm_read_sw_gcsr(csr, csrid);    \
+		}                                               \
+	} while (0)
+
+unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	unsigned long val = 0;
+
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRCTL);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO1);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO2);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRERA);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRSAVE);
+	/* read sw csr when not config pmu to guest */
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL0);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL1);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL2);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL3);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR0);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR1);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR2);
+	CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR3);
+
+	val = 0;
+	if (csrid < 4096)
+		val = kvm_read_sw_gcsr(csr, csrid);
+	else
+		pr_warn_once("Unsupport csrread 0x%x with pc %lx\n",
+			csrid, vcpu->arch.pc);
+	return val;
+}
+
+#define CASE_WRITE_SW_GCSR(csr, regid, csrid, val)              \
+	do {                                                    \
+		if (regid == csrid) {                           \
+			kvm_write_sw_gcsr(csr, csrid, val);     \
+			return ;                                \
+		}                                               \
+	} while (0)
+
+void _kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid,
+	unsigned long val)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRCTL, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO1, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO2, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRERA, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRSAVE, val);
+
+	/* give pmu register to guest when config perfctrl */
+	CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL0, val);
+	CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL1, val);
+	CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL2, val);
+	CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL3, val);
+	/* write sw pmu csr if not config ctrl */
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR0, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR1, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR2, val);
+	CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR3, val);
+
+
+	if (csrid < 4096)
+		kvm_write_sw_gcsr(csr, csrid, val);
+	else
+		pr_warn_once("Unsupport csrwrite 0x%x with pc %lx\n",
+				csrid, vcpu->arch.pc);
+}
+
+#define CASE_CHANGE_SW_GCSR(csr, regid, csrid, mask, val)               \
+	do {                                                            \
+		if (regid == csrid) {                                   \
+			kvm_change_sw_gcsr(csr, csrid, mask, val);      \
+			return ;                                        \
+		}                                                       \
+	} while (0)
+
+void _kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid,
+	unsigned long csr_mask, unsigned long val)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+
+	CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_IMPCTL1, csr_mask, val);
+	CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRCTL, csr_mask, val);
+	CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO1, csr_mask, val);
+	CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO2, csr_mask, val);
+	CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY, csr_mask, val);
+	CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRERA, csr_mask, val);
+	CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRSAVE, csr_mask, val);
+
+	if (csrid < 4096) {
+		unsigned long orig;
+
+		orig = kvm_read_sw_gcsr(csr, csrid);
+		orig &= ~csr_mask;
+		orig |= val & csr_mask;
+		kvm_write_sw_gcsr(csr, csrid, orig);
+	}
+	pr_warn_once("Unsupport csrxchg 0x%x with pc %lx\n",
+				csrid, vcpu->arch.pc);
+}
+
+int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+
+	GET_HW_GCSR(id, KVM_CSR_CRMD, v);
+	GET_HW_GCSR(id, KVM_CSR_PRMD, v);
+	GET_HW_GCSR(id, KVM_CSR_EUEN, v);
+	GET_HW_GCSR(id, KVM_CSR_MISC, v);
+	GET_HW_GCSR(id, KVM_CSR_ECFG, v);
+	GET_HW_GCSR(id, KVM_CSR_ESTAT, v);
+	GET_HW_GCSR(id, KVM_CSR_ERA, v);
+	GET_HW_GCSR(id, KVM_CSR_BADV, v);
+	GET_HW_GCSR(id, KVM_CSR_BADI, v);
+	GET_HW_GCSR(id, KVM_CSR_EENTRY, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBIDX, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBEHI, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBELO0, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBELO1, v);
+	GET_HW_GCSR(id, KVM_CSR_ASID, v);
+	GET_HW_GCSR(id, KVM_CSR_PGDL, v);
+	GET_HW_GCSR(id, KVM_CSR_PGDH, v);
+	GET_HW_GCSR(id, KVM_CSR_PWCTL0, v);
+	GET_HW_GCSR(id, KVM_CSR_PWCTL1, v);
+	GET_HW_GCSR(id, KVM_CSR_STLBPGSIZE, v);
+	GET_HW_GCSR(id, KVM_CSR_RVACFG, v);
+	GET_HW_GCSR(id, KVM_CSR_CPUID, v);
+	GET_HW_GCSR(id, KVM_CSR_PRCFG1, v);
+	GET_HW_GCSR(id, KVM_CSR_PRCFG2, v);
+	GET_HW_GCSR(id, KVM_CSR_PRCFG3, v);
+	GET_HW_GCSR(id, KVM_CSR_KS0, v);
+	GET_HW_GCSR(id, KVM_CSR_KS1, v);
+	GET_HW_GCSR(id, KVM_CSR_KS2, v);
+	GET_HW_GCSR(id, KVM_CSR_KS3, v);
+	GET_HW_GCSR(id, KVM_CSR_KS4, v);
+	GET_HW_GCSR(id, KVM_CSR_KS5, v);
+	GET_HW_GCSR(id, KVM_CSR_KS6, v);
+	GET_HW_GCSR(id, KVM_CSR_KS7, v);
+	GET_HW_GCSR(id, KVM_CSR_TMID, v);
+	GET_HW_GCSR(id, KVM_CSR_TCFG, v);
+	GET_HW_GCSR(id, KVM_CSR_TVAL, v);
+	GET_HW_GCSR(id, KVM_CSR_CNTC, v);
+	GET_HW_GCSR(id, KVM_CSR_LLBCTL, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBRENTRY, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBRBADV, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBRERA, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBRSAVE, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBRELO0, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBRELO1, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBREHI, v);
+	GET_HW_GCSR(id, KVM_CSR_TLBRPRMD, v);
+	GET_HW_GCSR(id, KVM_CSR_DMWIN0, v);
+	GET_HW_GCSR(id, KVM_CSR_DMWIN1, v);
+	GET_HW_GCSR(id, KVM_CSR_DMWIN2, v);
+	GET_HW_GCSR(id, KVM_CSR_DMWIN3, v);
+	GET_HW_GCSR(id, KVM_CSR_MWPS, v);
+	GET_HW_GCSR(id, KVM_CSR_FWPS, v);
+
+	GET_SW_GCSR(csr, id, KVM_CSR_IMPCTL1, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_IMPCTL2, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_ERRCTL, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_ERRINFO1, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_ERRINFO2, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_MERRENTRY, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_MERRERA, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_ERRSAVE, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_CTAG, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_DEBUG, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_DERA, v);
+	GET_SW_GCSR(csr, id, KVM_CSR_DESAVE, v);
+
+	GET_SW_GCSR(csr, id, KVM_CSR_TINTCLR, v);
+
+	if (force && (id < CSR_ALL_SIZE)) {
+		*v = kvm_read_sw_gcsr(csr, id);
+		return 0;
+	}
+
+	return -1;
+}
+
+int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	int ret;
+
+	SET_HW_GCSR(csr, id, KVM_CSR_CRMD, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_PRMD, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_EUEN, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_MISC, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_ECFG, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_ERA, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_BADV, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_BADI, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_EENTRY, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBIDX, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBEHI, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBELO0, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBELO1, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_ASID, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_PGDL, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_PGDH, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_PWCTL0, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_PWCTL1, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_STLBPGSIZE, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_RVACFG, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_CPUID, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS0, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS1, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS2, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS3, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS4, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS5, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS6, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_KS7, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TMID, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TCFG, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TVAL, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_CNTC, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_LLBCTL, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBRENTRY, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBRBADV, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBRERA, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBRSAVE, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBRELO0, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBRELO1, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBREHI, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_TLBRPRMD, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_DMWIN0, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_DMWIN1, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_DMWIN2, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_DMWIN3, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_MWPS, v);
+	SET_HW_GCSR(csr, id, KVM_CSR_FWPS, v);
+
+	SET_SW_GCSR(csr, id, KVM_CSR_IMPCTL1, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_IMPCTL2, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_ERRCTL, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_ERRINFO1, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_ERRINFO2, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_MERRENTRY, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_MERRERA, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_ERRSAVE, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_CTAG, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_DEBUG, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_DERA, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_DESAVE, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_PRCFG1, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_PRCFG2, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_PRCFG3, v);
+
+	SET_SW_GCSR(csr, id, KVM_CSR_PGD, v);
+	SET_SW_GCSR(csr, id, KVM_CSR_TINTCLR, v);
+
+	ret = -1;
+	switch (id) {
+	case KVM_CSR_ESTAT:
+		kvm_write_gcsr_estat(*v);
+		/* estat IP0~IP7 inject through guestexcept */
+		kvm_write_csr_gintc(((*v) >> 2)  & 0xff);
+		ret = 0;
+		break;
+	default:
+		if (force && (id < CSR_ALL_SIZE)) {
+			kvm_set_sw_gcsr(csr, id, *v);
+			ret = 0;
+		}
+		break;
+	}
+
+	return ret;
+}
+
+struct kvm_iocsr {
+	u32 start, end;
+	int (*get) (struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, u64 *res);
+	int (*set) (struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, u64 val);
+};
+
+static struct kvm_iocsr_entry *_kvm_find_iocsr(struct kvm *kvm, u32 addr)
+{
+	int i = 0;
+
+	for (i = 0; i < IOCSR_MAX; i++) {
+		if (addr == kvm->arch.iocsr[i].addr)
+			return &kvm->arch.iocsr[i];
+	}
+
+	return NULL;
+}
+
+static int kvm_iocsr_common_get(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		u32 addr, u64 *res)
+{
+	int r = EMULATE_FAIL;
+	struct kvm_iocsr_entry *entry;
+
+	spin_lock(&vcpu->kvm->arch.iocsr_lock);
+	entry = _kvm_find_iocsr(vcpu->kvm, addr);
+	if (entry) {
+		r = EMULATE_DONE;
+		*res = entry->data;
+	}
+	spin_unlock(&vcpu->kvm->arch.iocsr_lock);
+	return r;
+}
+
+static int kvm_iocsr_common_set(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		u32 addr, u64 val)
+{
+	int r = EMULATE_FAIL;
+	struct kvm_iocsr_entry *entry;
+
+	spin_lock(&vcpu->kvm->arch.iocsr_lock);
+	entry = _kvm_find_iocsr(vcpu->kvm, addr);
+	if (entry) {
+		r = EMULATE_DONE;
+		entry->data = val;
+	}
+	spin_unlock(&vcpu->kvm->arch.iocsr_lock);
+	return r;
+}
+
+static int kvm_misc_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr,
+		u64 val)
+{
+	if ((val & KVM_IOCSRF_MISC_FUNC_EXT_IOI_EN) && vcpu->vcpu_id == 0)
+		kvm_setup_ls3a_extirq(vcpu->kvm);
+	return kvm_iocsr_common_set(run, vcpu, addr, val);
+}
+
+static int kvm_ipi_get(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr,
+		u64 *res)
+{
+	int ret;
+
+	++vcpu->stat.rdcsr_ipi_access_exits;
+	run->mmio.phys_addr = KVM_IPI_REG_ADDRESS(vcpu->vcpu_id, (addr & 0xff));
+	ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
+			run->mmio.len, res);
+	if (ret) {
+		run->mmio.is_write = 0;
+		vcpu->mmio_needed = 1;
+		vcpu->mmio_is_write = 0;
+		return EMULATE_DO_MMIO;
+	}
+	return EMULATE_DONE;
+}
+
+static int kvm_extioi_isr_get(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		u32 addr, u64 *res)
+{
+	int ret;
+
+	run->mmio.phys_addr =  EXTIOI_PERCORE_ADDR(vcpu->vcpu_id, (addr & 0xff));
+	ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
+			run->mmio.len, res);
+	if (ret) {
+		run->mmio.is_write = 0;
+		vcpu->mmio_needed = 1;
+		vcpu->mmio_is_write = 0;
+		return EMULATE_FAIL;
+	}
+
+	return EMULATE_DONE;
+}
+
+static int kvm_ipi_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr,
+		u64 val)
+{
+	int ret;
+
+	run->mmio.phys_addr = KVM_IPI_REG_ADDRESS(vcpu->vcpu_id, (addr & 0xff));
+	ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
+			run->mmio.len, &val);
+	if (ret < 0) {
+		run->mmio.is_write = 1;
+		vcpu->mmio_needed = 1;
+		vcpu->mmio_is_write = 1;
+		return EMULATE_DO_MMIO;
+	}
+
+	return EMULATE_DONE;
+}
+
+static int kvm_extioi_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr,
+		u64 val)
+{
+	int ret;
+
+	if ((addr & 0x1f00) == KVM_IOCSR_EXTIOI_ISR_BASE) {
+		run->mmio.phys_addr =  EXTIOI_PERCORE_ADDR(vcpu->vcpu_id, (addr & 0xff));
+	} else {
+		run->mmio.phys_addr = EXTIOI_ADDR((addr & 0x1fff));
+	}
+
+	ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
+			run->mmio.len, &val);
+	if (ret < 0) {
+		memcpy(run->mmio.data, &val, run->mmio.len);
+		run->mmio.is_write = 1;
+		vcpu->mmio_needed = 1;
+		vcpu->mmio_is_write = 1;
+		return EMULATE_DO_MMIO;
+	}
+
+	return EMULATE_DONE;
+}
+
+static int kvm_nop_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr,
+		u64 val)
+{
+	return EMULATE_DONE;
+}
+
+/* we put these iocsrs with access frequency, from high to low */
+static struct kvm_iocsr kvm_iocsrs[] = {
+	/* extioi iocsr */
+	{KVM_IOCSR_EXTIOI_EN_BASE, KVM_IOCSR_EXTIOI_EN_BASE + 0x100,
+		NULL, kvm_extioi_set},
+	{KVM_IOCSR_EXTIOI_NODEMAP_BASE, KVM_IOCSR_EXTIOI_NODEMAP_BASE+0x28,
+		NULL, kvm_extioi_set},
+	{KVM_IOCSR_EXTIOI_ROUTE_BASE, KVM_IOCSR_EXTIOI_ROUTE_BASE + 0x100,
+		NULL, kvm_extioi_set},
+	{KVM_IOCSR_EXTIOI_ISR_BASE, KVM_IOCSR_EXTIOI_ISR_BASE + 0x1c,
+		kvm_extioi_isr_get, kvm_extioi_set},
+
+	{KVM_IOCSR_IPI_STATUS, KVM_IOCSR_IPI_STATUS + 0x40,
+		kvm_ipi_get, kvm_ipi_set},
+	{KVM_IOCSR_IPI_SEND, KVM_IOCSR_IPI_SEND + 0x1,
+		NULL, kvm_ipi_set},
+	{KVM_IOCSR_MBUF_SEND, KVM_IOCSR_MBUF_SEND + 0x1,
+		NULL, kvm_ipi_set},
+
+	{KVM_IOCSR_FEATURES, KVM_IOCSR_FEATURES + 0x1,
+		kvm_iocsr_common_get, kvm_nop_set},
+	{KVM_IOCSR_VENDOR, KVM_IOCSR_VENDOR + 0x1,
+		kvm_iocsr_common_get, kvm_nop_set},
+	{KVM_IOCSR_CPUNAME, KVM_IOCSR_CPUNAME + 0x1,
+		kvm_iocsr_common_get, kvm_nop_set},
+	{KVM_IOCSR_NODECNT, KVM_IOCSR_NODECNT + 0x1,
+		kvm_iocsr_common_get, kvm_nop_set},
+	{KVM_IOCSR_MISC_FUNC, KVM_IOCSR_MISC_FUNC + 0x1,
+		kvm_iocsr_common_get, kvm_misc_set},
+};
+
+static int _kvm_emu_iocsr_read(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		u32 addr, u64 *res)
+{
+	enum emulation_result er = EMULATE_FAIL;
+	int i = 0;
+	struct kvm_iocsr *iocsr = NULL;
+
+	if (!irqchip_in_kernel(vcpu->kvm)) {
+		run->iocsr_io.len = run->mmio.len;
+		run->iocsr_io.phys_addr = addr;
+		run->iocsr_io.is_write = 0;
+		return EMULATE_DO_IOCSR;
+	}
+	for (i = 0; i < sizeof(kvm_iocsrs) / sizeof(struct kvm_iocsr); i++) {
+		iocsr = &kvm_iocsrs[i];
+		if (addr >= iocsr->start && addr < iocsr->end) {
+			if (iocsr->get)
+				er = iocsr->get(run, vcpu, addr, res);
+		}
+	}
+
+	if (er != EMULATE_DONE)
+		kvm_debug("%s iocsr 0x%x not support in kvm\n", __func__, addr);
+
+	return er;
+}
+
+static int _kvm_emu_iocsr_write(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		u32 addr, u64 val)
+{
+	enum emulation_result er = EMULATE_FAIL;
+	int i = 0;
+	struct kvm_iocsr *iocsr = NULL;
+
+	if (!irqchip_in_kernel(vcpu->kvm)) {
+		run->iocsr_io.len = run->mmio.len;
+		memcpy(run->iocsr_io.data, &val, run->iocsr_io.len);
+		run->iocsr_io.phys_addr = addr;
+		run->iocsr_io.is_write = 1;
+		return EMULATE_DO_IOCSR;
+	}
+	for (i = 0; i < sizeof(kvm_iocsrs) / sizeof(struct kvm_iocsr); i++) {
+		iocsr = &kvm_iocsrs[i];
+		if (addr >= iocsr->start && addr < iocsr->end) {
+			if (iocsr->set)
+				er = iocsr->set(run, vcpu, addr, val);
+		}
+	}
+	if (er != EMULATE_DONE)
+		kvm_debug("%s iocsr 0x%x not support in kvm\n", __func__, addr);
+
+	return er;
+}
+
+/* all iocsr operation should in kvm, no mmio */
+int _kvm_emu_iocsr(larch_inst inst,
+		struct kvm_run *run, struct kvm_vcpu *vcpu)
+{
+	u32 rd, rj, opcode;
+	u32 val;
+	u64 res = 0;
+	int ret;
+
+	/*
+	 * Each IOCSR with different opcode
+	 */
+	rd = inst.reg2_format.rd;
+	rj = inst.reg2_format.rj;
+	opcode = inst.reg2_format.opcode;
+	val = vcpu->arch.gprs[rj];
+	res = vcpu->arch.gprs[rd];
+	/* LoongArch is Little endian */
+	switch (opcode) {
+	case iocsrrdb_op:
+		run->mmio.len = 1;
+		ret = _kvm_emu_iocsr_read(run, vcpu, val, &res);
+		vcpu->arch.gprs[rd] = (u8) res;
+		break;
+	case iocsrrdh_op:
+		run->mmio.len = 2;
+		ret = _kvm_emu_iocsr_read(run, vcpu, val, &res);
+		vcpu->arch.gprs[rd] = (u16) res;
+		break;
+	case iocsrrdw_op:
+		run->mmio.len = 4;
+		ret = _kvm_emu_iocsr_read(run, vcpu, val, &res);
+		vcpu->arch.gprs[rd] = (u32) res;
+		break;
+	case iocsrrdd_op:
+		run->mmio.len = 8;
+		ret = _kvm_emu_iocsr_read(run, vcpu, val, &res);
+		vcpu->arch.gprs[rd] = res;
+		break;
+	case iocsrwrb_op:
+		run->mmio.len = 1;
+		ret = _kvm_emu_iocsr_write(run, vcpu, val, (u8)res);
+		break;
+	case iocsrwrh_op:
+		run->mmio.len = 2;
+		ret = _kvm_emu_iocsr_write(run, vcpu, val, (u16)res);
+		break;
+	case iocsrwrw_op:
+		run->mmio.len = 4;
+		ret = _kvm_emu_iocsr_write(run, vcpu, val, (u32)res);
+		break;
+	case iocsrwrd_op:
+		run->mmio.len = 8;
+		ret = _kvm_emu_iocsr_write(run, vcpu, val, res);
+		break;
+	default:
+		ret = EMULATE_FAIL;
+		break;
+	}
+
+	if (ret == EMULATE_DO_IOCSR) {
+		vcpu->arch.io_gpr = rd;
+	}
+
+	return ret;
+}
+
+int _kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+	unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
+	enum emulation_result er = EMULATE_DONE;
+
+	switch (run->iocsr_io.len) {
+	case 8:
+		*gpr = *(s64 *)run->iocsr_io.data;
+		break;
+	case 4:
+		*gpr = *(int *)run->iocsr_io.data;
+		break;
+	case 2:
+		*gpr = *(short *)run->iocsr_io.data;
+		break;
+	case 1:
+		*gpr = *(char *) run->iocsr_io.data;
+		break;
+	default:
+		kvm_err("Bad IOCSR length: %d,addr is 0x%lx",
+				run->iocsr_io.len, vcpu->arch.badv);
+		er = EMULATE_FAIL;
+		break;
+	}
+
+	return er;
+}
+
+int _kvm_get_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp)
+{
+	struct kvm_iocsr_entry *entry, tmp;
+	int r = -EFAULT;
+
+	if (copy_from_user(&tmp, argp, sizeof(tmp)))
+		goto out;
+
+	spin_lock(&kvm->arch.iocsr_lock);
+	entry = _kvm_find_iocsr(kvm, tmp.addr);
+	if (entry != NULL)
+		tmp.data = entry->data;
+	spin_unlock(&kvm->arch.iocsr_lock);
+
+	if (entry)
+		r = copy_to_user(argp, &tmp, sizeof(tmp));
+
+out:
+	return r;
+}
+
+int _kvm_set_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp)
+{
+	struct kvm_iocsr_entry *entry, tmp;
+	int r = -EFAULT;
+
+	if (copy_from_user(&tmp, argp, sizeof(tmp)))
+		goto out;
+
+	spin_lock(&kvm->arch.iocsr_lock);
+	entry = _kvm_find_iocsr(kvm, tmp.addr);
+	if (entry != NULL) {
+		r = 0;
+		entry->data = tmp.data;
+	}
+	spin_unlock(&kvm->arch.iocsr_lock);
+
+	if (tmp.addr == KVM_IOCSR_MISC_FUNC)
+		kvm_enable_ls3a_extirq(kvm, tmp.data & KVM_IOCSRF_MISC_FUNC_EXT_IOI_EN);
+
+out:
+	return r;
+}
+
+static struct kvm_iocsr_entry iocsr_array[IOCSR_MAX] = {
+	{KVM_IOCSR_FEATURES, .data = KVM_IOCSRF_NODECNT|KVM_IOCSRF_MSI
+		|KVM_IOCSRF_EXTIOI|KVM_IOCSRF_CSRIPI|KVM_IOCSRF_VM},
+	{KVM_IOCSR_VENDOR, .data = 0x6e6f73676e6f6f4c}, /* Loongson */
+	{KVM_IOCSR_CPUNAME, .data = 0x303030354133},	/* 3A5000 */
+	{KVM_IOCSR_NODECNT, .data = 0x4},
+	{KVM_IOCSR_MISC_FUNC, .data = 0x0},
+};
+
+int _kvm_init_iocsr(struct kvm *kvm)
+{
+	int i = 0;
+
+	spin_lock_init(&kvm->arch.iocsr_lock);
+	for (i = 0; i < IOCSR_MAX; i++) {
+		kvm->arch.iocsr[i].addr = iocsr_array[i].addr;
+		kvm->arch.iocsr[i].data = iocsr_array[i].data;
+	}
+	return 0;
+}

arch/loongarch/kvm/emulate.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/ktime.h>
+#include <linux/kvm_host.h>
+#include <linux/vmalloc.h>
+#include <linux/fs.h>
+#include <linux/random.h>
+#include <asm/page.h>
+#include <asm/cacheflush.h>
+#include <asm/cacheops.h>
+#include <asm/cpu-info.h>
+#include <asm/mmu_context.h>
+#include <asm/tlbflush.h>
+#include <asm/inst.h>
+#include "kvmcpu.h"
+#include "trace.h"
+
+int _kvm_emu_idle(struct kvm_vcpu *vcpu)
+{
+	++vcpu->stat.idle_exits;
+	trace_kvm_exit(vcpu, KVM_TRACE_EXIT_IDLE);
+	if (!vcpu->arch.irq_pending) {
+		kvm_save_timer(vcpu);
+		kvm_vcpu_block(vcpu);
+
+		/*
+		 * We we are runnable, then definitely go off to user space to
+		 * check if any I/O interrupts are pending.
+		 */
+		if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
+			kvm_clear_request(KVM_REQ_UNHALT, vcpu);
+			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
+		}
+	}
+
+	return EMULATE_DONE;
+}
+
+int _kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst)
+{
+	struct kvm_run *run = vcpu->run;
+	unsigned int rd, op8, opcode;
+	unsigned long rd_val = 0;
+	void *data = run->mmio.data;
+	unsigned long curr_pc;
+	int ret = 0;
+
+	/*
+	 * Update PC and hold onto current PC in case there is
+	 * an error and we want to rollback the PC
+	 */
+	curr_pc = vcpu->arch.pc;
+	update_pc(&vcpu->arch);
+
+	op8 = (inst.word >> 24) & 0xff;
+	run->mmio.phys_addr = vcpu->arch.badv;
+	if (run->mmio.phys_addr == KVM_INVALID_ADDR)
+		goto out_fail;
+
+	if (op8 < 0x28) {
+		/* stptrw/d process */
+		rd = inst.reg2i14_format.rd;
+		opcode = inst.reg2i14_format.opcode;
+
+		switch (opcode) {
+		case stptrd_op:
+			run->mmio.len = 8;
+			*(unsigned long *)data = vcpu->arch.gprs[rd];
+			break;
+		case stptrw_op:
+			run->mmio.len = 4;
+			*(unsigned int *)data = vcpu->arch.gprs[rd];
+			break;
+		default:
+			break;
+		}
+	} else if (op8 < 0x30) {
+		/* st.b/h/w/d  process */
+		rd = inst.reg2i12_format.rd;
+		opcode = inst.reg2i12_format.opcode;
+		rd_val = vcpu->arch.gprs[rd];
+
+		switch (opcode) {
+		case std_op:
+			run->mmio.len = 8;
+			*(unsigned long *)data = rd_val;
+			break;
+		case stw_op:
+			run->mmio.len = 4;
+			*(unsigned int *)data = rd_val;
+			break;
+		case sth_op:
+			run->mmio.len = 2;
+			*(unsigned short *)data = rd_val;
+			break;
+		case stb_op:
+			run->mmio.len = 1;
+			*(unsigned char *)data = rd_val;
+			break;
+		default:
+			kvm_err("Store not yet supporded (inst=0x%08x)\n",
+				inst.word);
+			kvm_arch_vcpu_dump_regs(vcpu);
+			goto out_fail;
+		}
+	} else if (op8 == 0x38) {
+		/* stxb/h/w/d process */
+		rd = inst.reg3_format.rd;
+		opcode = inst.reg3_format.opcode;
+
+		switch (opcode) {
+		case stxb_op:
+			run->mmio.len = 1;
+			*(unsigned char *)data = vcpu->arch.gprs[rd];
+			break;
+		case stxh_op:
+			run->mmio.len = 2;
+			*(unsigned short *)data = vcpu->arch.gprs[rd];
+			break;
+		case stxw_op:
+			run->mmio.len = 4;
+			*(unsigned int *)data = vcpu->arch.gprs[rd];
+			break;
+		case stxd_op:
+			run->mmio.len = 8;
+			*(unsigned long *)data = vcpu->arch.gprs[rd];
+			break;
+		default:
+			kvm_err("Store not yet supporded (inst=0x%08x)\n",
+				inst.word);
+			kvm_arch_vcpu_dump_regs(vcpu);
+			goto out_fail;
+		}
+	} else {
+		kvm_err("Store not yet supporded (inst=0x%08x)\n",
+			inst.word);
+		kvm_arch_vcpu_dump_regs(vcpu);
+		goto out_fail;
+	}
+
+	/* All MMIO emulate in kernel go through the common interface */
+	ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
+				run->mmio.len, data);
+	if (!ret) {
+		vcpu->mmio_needed = 0;
+		return EMULATE_DONE;
+	}
+
+	run->mmio.is_write = 1;
+	vcpu->mmio_needed = 1;
+	vcpu->mmio_is_write = 1;
+
+	return EMULATE_DO_MMIO;
+
+out_fail:
+	/* Rollback PC if emulation was unsuccessful */
+	vcpu->arch.pc = curr_pc;
+	return EMULATE_FAIL;
+}
+
+
+int _kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst)
+{
+	unsigned int op8, opcode, rd;
+	int ret = 0;
+	struct kvm_run *run = vcpu->run;
+
+	run->mmio.phys_addr = vcpu->arch.badv;
+	if (run->mmio.phys_addr == KVM_INVALID_ADDR)
+		return EMULATE_FAIL;
+
+	vcpu->mmio_needed = 2;	/* signed */
+	op8 = (inst.word >> 24) & 0xff;
+
+	if (op8 < 0x28) {
+		/* ldptr.w/d process */
+		rd = inst.reg2i14_format.rd;
+		opcode = inst.reg2i14_format.opcode;
+
+		switch (opcode) {
+		case ldptrd_op:
+			run->mmio.len = 8;
+			break;
+		case ldptrw_op:
+			run->mmio.len = 4;
+			break;
+		default:
+			break;
+		}
+	} else if (op8 < 0x2f) {
+		/* ld.b/h/w/d, ld.bu/hu/wu process */
+		rd = inst.reg2i12_format.rd;
+		opcode = inst.reg2i12_format.opcode;
+
+		switch (opcode) {
+		case ldd_op:
+			run->mmio.len = 8;
+			break;
+		case ldwu_op:
+			vcpu->mmio_needed = 1;	/* unsigned */
+			run->mmio.len = 4;
+			break;
+		case ldw_op:
+			run->mmio.len = 4;
+			break;
+		case ldhu_op:
+			vcpu->mmio_needed = 1;	/* unsigned */
+			run->mmio.len = 2;
+			break;
+		case ldh_op:
+			run->mmio.len = 2;
+			break;
+		case ldbu_op:
+			vcpu->mmio_needed = 1;	/* unsigned */
+			run->mmio.len = 1;
+			break;
+		case ldb_op:
+			run->mmio.len = 1;
+			break;
+		default:
+			kvm_err("Load not yet supporded (inst=0x%08x)\n",
+				inst.word);
+			kvm_arch_vcpu_dump_regs(vcpu);
+			vcpu->mmio_needed = 0;
+			return EMULATE_FAIL;
+		}
+	} else if (op8 == 0x38) {
+		/* ldxb/h/w/d, ldxb/h/wu, ldgtb/h/w/d, ldleb/h/w/d process */
+		rd = inst.reg3_format.rd;
+		opcode = inst.reg3_format.opcode;
+
+		switch (opcode) {
+		case ldxb_op:
+			run->mmio.len = 1;
+			break;
+		case ldxbu_op:
+			run->mmio.len = 1;
+			vcpu->mmio_needed = 1;	/* unsigned */
+			break;
+		case ldxh_op:
+			run->mmio.len = 2;
+			break;
+		case ldxhu_op:
+			run->mmio.len = 2;
+			vcpu->mmio_needed = 1;	/* unsigned */
+			break;
+		case ldxw_op:
+			run->mmio.len = 4;
+			break;
+		case ldxwu_op:
+			run->mmio.len = 4;
+			vcpu->mmio_needed = 1;	/* unsigned */
+			break;
+		case ldxd_op:
+			run->mmio.len = 8;
+			break;
+		default:
+			kvm_err("Load not yet supporded (inst=0x%08x)\n",
+				inst.word);
+			kvm_arch_vcpu_dump_regs(vcpu);
+			vcpu->mmio_needed = 0;
+			return EMULATE_FAIL;
+		}
+	} else {
+		kvm_err("Load not yet supporded (inst=0x%08x) @ %lx\n",
+			inst.word, vcpu->arch.pc);
+		vcpu->mmio_needed = 0;
+		return EMULATE_FAIL;
+	}
+
+	/* Set for _kvm_complete_mmio_read use */
+	vcpu->arch.io_gpr = rd;
+	ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
+						run->mmio.len, run->mmio.data);
+	run->mmio.is_write = 0;
+	vcpu->mmio_is_write = 0;
+
+	if (!ret) {
+		_kvm_complete_mmio_read(vcpu, run);
+		vcpu->mmio_needed = 0;
+		return EMULATE_DONE;
+	}
+	return EMULATE_DO_MMIO;
+}
+
+int _kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+	unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
+	enum emulation_result er = EMULATE_DONE;
+
+	/* update with new PC */
+	update_pc(&vcpu->arch);
+	switch (run->mmio.len) {
+	case 8:
+		*gpr = *(s64 *)run->mmio.data;
+		break;
+
+	case 4:
+		if (vcpu->mmio_needed == 2) {
+			*gpr = *(int *)run->mmio.data;
+		} else
+			*gpr = *(unsigned int *)run->mmio.data;
+		break;
+
+	case 2:
+		if (vcpu->mmio_needed == 2)
+			*gpr = *(short *) run->mmio.data;
+		else
+			*gpr = *(unsigned short *)run->mmio.data;
+
+		break;
+	case 1:
+		if (vcpu->mmio_needed == 2)
+			*gpr = *(char *) run->mmio.data;
+		else
+			*gpr = *(unsigned char *) run->mmio.data;
+		break;
+	default:
+		kvm_err("Bad MMIO length: %d,addr is 0x%lx",
+				run->mmio.len, vcpu->arch.badv);
+		er = EMULATE_FAIL;
+		break;
+	}
+
+	return er;
+}

arch/loongarch/kvm/entry.S

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/linkage.h>
+#include <asm/stackframe.h>
+#include <asm/asm.h>
+#include <asm/asmmacro.h>
+#include <asm/regdef.h>
+#include "kvm_compat.h"
+
+#define RESUME_HOST	(1 << 1)
+
+#define GGPR_OFFSET(x)	(KVM_ARCH_GGPR + 8*x)
+#define PT_GPR_OFFSET(x)	(PT_R0 + 8*x)
+
+	.text
+
+.macro kvm_save_guest_gprs base
+	.irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+	KVM_LONG_S	$r\n, \base, GGPR_OFFSET(\n)
+	.endr
+.endm
+
+.macro kvm_restore_guest_gprs base
+	.irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+	KVM_LONG_L	$r\n, \base, GGPR_OFFSET(\n)
+	.endr
+.endm
+
+.macro kvm_save_host_gpr base
+	.irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
+	KVM_LONG_S	$r\n, \base, PT_GPR_OFFSET(\n)
+	.endr
+.endm
+
+.macro kvm_restore_host_gpr base
+	.irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
+	KVM_LONG_L	$r\n, \base, PT_GPR_OFFSET(\n)
+	.endr
+.endm
+
+/*
+ * prepare switch to guest
+ * @param:
+ *  KVM_ARCH: kvm_vcpu_arch, don't touch it until 'ertn'
+ *  GPRNUM: KVM_ARCH gpr number
+ *  tmp, tmp1: temp register
+ */
+.macro kvm_switch_to_guest KVM_ARCH GPRNUM tmp tmp1
+	/* set host excfg.VS=0, all exceptions share one exception entry */
+	csrrd	\tmp, KVM_CSR_ECFG
+	bstrins.w	\tmp, zero, (KVM_ECFG_VS_SHIFT + KVM_ECFG_VS_WIDTH - 1), KVM_ECFG_VS_SHIFT
+	csrwr	\tmp, KVM_CSR_ECFG
+
+	/* Load up the new EENTRY */
+	KVM_LONG_L	\tmp, \KVM_ARCH, KVM_ARCH_GEENTRY
+	csrwr		\tmp, KVM_CSR_EENTRY
+
+	/* Set Guest ERA */
+	KVM_LONG_L	\tmp, \KVM_ARCH, KVM_ARCH_GPC
+	csrwr		\tmp, KVM_CSR_ERA
+
+	/* Save host PGDL */
+	csrrd	\tmp, KVM_CSR_PGDL
+	KVM_LONG_S	\tmp, \KVM_ARCH, KVM_ARCH_HPGD
+
+	/* Switch to kvm */
+	KVM_LONG_L	\tmp1, \KVM_ARCH, KVM_VCPU_KVM - KVM_VCPU_ARCH
+
+	/* Load guest PGDL */
+	lu12i.w \tmp, KVM_GPGD
+	srli.w \tmp, \tmp, 12
+	ldx.d  \tmp, \tmp1, \tmp
+	csrwr	\tmp, KVM_CSR_PGDL
+
+	/* Mix GID and RID */
+	csrrd	\tmp1, KVM_CSR_GSTAT
+	bstrpick.w	\tmp1, \tmp1, (KVM_GSTAT_GID_SHIFT + KVM_GSTAT_GID_WIDTH - 1), KVM_GSTAT_GID_SHIFT
+	csrrd	\tmp, KVM_CSR_GTLBC
+	bstrins.w	\tmp, \tmp1, (KVM_GTLBC_TGID_SHIFT + KVM_GTLBC_TGID_WIDTH - 1), KVM_GTLBC_TGID_SHIFT
+	csrwr	\tmp, KVM_CSR_GTLBC
+
+	/*
+	 * Switch to guest:
+	 *  GSTAT.PGM = 1, ERRCTL.ISERR = 0, TLBRPRMD.ISTLBR = 0
+	 *  ertn
+	 */
+
+	/* Prepare enable Intr before enter guest */
+	ori	\tmp, zero, KVM_PRMD_PIE
+	csrxchg	\tmp, \tmp, KVM_CSR_PRMD
+
+	/* Set PVM bit to setup ertn to guest context */
+	ori	\tmp, zero, KVM_GSTAT_PVM
+	csrxchg	\tmp, \tmp, KVM_CSR_GSTAT
+
+	/* Load Guest gprs */
+	kvm_restore_guest_gprs	\KVM_ARCH
+
+	/* Load KVM_ARCH register */
+	KVM_LONG_L	\KVM_ARCH, \KVM_ARCH, GGPR_OFFSET(\GPRNUM)
+
+	ertn
+.endm
+
+#ifndef	EXCPTION_ENTRY
+#define EXCPTION_ENTRY(name)	\
+	.globl name ASM_NL	\
+	.p2align	12;	\
+	name:			\
+		.cfi_startproc;
+#endif
+#ifndef	EXCPTION_ENDPROC
+#define EXCPTION_ENDPROC(name)	\
+	.cfi_endproc;			\
+	SYM_END(name, SYM_T_FUNC)
+#endif
+
+/* load kvm_vcpu to a2 and store a1 for free use */
+EXCPTION_ENTRY(kvm_exception_entry)
+	csrwr	a2, KVM_TEMP_KS
+	csrrd	a2, KVM_VCPU_KS
+	KVM_LONG_ADDI	a2, a2, KVM_VCPU_ARCH
+
+	/* After save gprs, free to use any gpr */
+	kvm_save_guest_gprs	a2
+	/* Save guest a2 */
+	csrrd	t0, KVM_TEMP_KS
+	KVM_LONG_S	t0, a2, GGPR_OFFSET(REG_A2)
+
+	b	kvm_exit_entry
+EXCPTION_ENDPROC(kvm_exception_entry)
+
+/* a2: kvm_vcpu_arch, a1 is free to use */
+SYM_FUNC_START(kvm_exit_entry)
+	csrrd	s1, KVM_VCPU_KS
+	KVM_LONG_L	s0, s1, KVM_VCPU_RUN
+
+	csrrd		t0, KVM_CSR_ESTAT
+	KVM_LONG_S	t0, a2, KVM_ARCH_HESTAT
+	csrrd		t0, KVM_CSR_ERA
+	KVM_LONG_S	t0, a2, KVM_ARCH_GPC
+	csrrd		t0, KVM_CSR_BADV
+	KVM_LONG_S	t0, a2, KVM_ARCH_HBADV
+	csrrd		t0, KVM_CSR_BADI
+	KVM_LONG_S	t0, a2, KVM_ARCH_HBADI
+
+	/* Restore host excfg.VS */
+	csrrd		t0, KVM_CSR_ECFG
+	KVM_LONG_L	t1, a2, KVM_ARCH_HECFG
+	or		t0, t0, t1
+	csrwr		t0, KVM_CSR_ECFG
+
+	/* Restore host eentry */
+	KVM_LONG_L	t0, a2, KVM_ARCH_HEENTRY
+	csrwr		t0, KVM_CSR_EENTRY
+
+#if defined(CONFIG_CPU_HAS_FPU)
+	/* Save FPU context */
+	csrrd	t0, KVM_CSR_EUEN
+	ori	t1, zero, KVM_EUEN_FPEN | KVM_EUEN_LSXEN | KVM_EUEN_LASXEN
+	and	t2, t0, t1
+	beqz	t2, 1f
+	movfcsr2gr	t3, fcsr0
+	INT_S	t3,	a2, VCPU_FCSR0
+
+	movcf2gr	t3, $fcc0
+	or		t2, t3, zero
+	movcf2gr	t3, $fcc1
+	bstrins.d	t2, t3, 0xf, 0x8
+	movcf2gr	t3, $fcc2
+	bstrins.d	t2, t3, 0x17, 0x10
+	movcf2gr	t3, $fcc3
+	bstrins.d	t2, t3, 0x1f, 0x18
+	movcf2gr	t3, $fcc4
+	bstrins.d	t2, t3, 0x27, 0x20
+	movcf2gr	t3, $fcc5
+	bstrins.d	t2, t3, 0x2f, 0x28
+	movcf2gr	t3, $fcc6
+	bstrins.d	t2, t3, 0x37, 0x30
+	movcf2gr	t3, $fcc7
+	bstrins.d	t2, t3, 0x3f, 0x38
+	KVM_LONG_S	t2, a2, VCPU_FCC
+	movgr2fcsr	fcsr0, zero
+1:
+#endif
+
+	KVM_LONG_L	t0, a2, KVM_ARCH_HPGD
+	csrwr	t0, KVM_CSR_PGDL
+
+	/* Disable PVM bit for keeping from into guest */
+	ori	t0, zero, KVM_GSTAT_PVM
+	csrxchg	zero, t0, KVM_CSR_GSTAT
+
+	/* Clear GTLBC.TGID field */
+	csrrd	t0, KVM_CSR_GTLBC
+	bstrins.w	t0, zero, KVM_GTLBC_TGID_SHIFT + KVM_GTLBC_TGID_WIDTH - 1, KVM_GTLBC_TGID_SHIFT
+	csrwr	t0, KVM_CSR_GTLBC
+
+	/* Enable Address Map mode */
+	ori	t0, zero, (1 << KVM_CRMD_DACM_SHIFT)|(1 << KVM_CRMD_DACF_SHIFT) | KVM_CRMD_PG |PLV_KERN
+	csrwr	t0, KVM_CSR_CRMD
+
+	KVM_LONG_L	tp, a2, KVM_ARCH_HGP
+	KVM_LONG_L	sp, a2, KVM_ARCH_HSTACK
+	/* restore per cpu register */
+	KVM_LONG_L	$r21, a2, KVM_ARCH_HPERCPU
+
+	KVM_LONG_ADDI	sp, sp, -PT_SIZE
+
+	/* Prepare handle exception */
+	or		a0, s0, zero
+	or		a1, s1, zero
+	KVM_LONG_L	t8, a2, KVM_ARCH_HANDLE_EXIT
+	jirl		ra,t8, 0
+
+	ori	t0, zero, KVM_CRMD_IE
+	csrxchg	zero, t0, KVM_CSR_CRMD
+	or	a2, s1, zero
+	KVM_LONG_ADDI	a2, a2, KVM_VCPU_ARCH
+
+	andi	t0, a0, RESUME_HOST
+	bnez	t0, ret_to_host
+	INT_S	zero, a2, KVM_ARCH_ISHYPCALL
+
+ret_to_guest:
+	/* Save per cpu register again, maybe switched to another cpu */
+	KVM_LONG_S	$r21, a2, KVM_ARCH_HPERCPU
+
+	/* Save kvm_vcpu to kscratch */
+	csrwr	s1, KVM_VCPU_KS
+	kvm_switch_to_guest a2 REG_A2 t0 t1
+
+ret_to_host:
+	KVM_LONG_L  a2, a2, KVM_ARCH_HSTACK
+	addi.d  a2, a2, -PT_SIZE
+	srai.w  a3, a0, 2
+	or      a0, a3, zero
+	kvm_restore_host_gpr    a2
+	jirl    zero, ra, 0
+SYM_FUNC_END(kvm_exit_entry)
+
+/*
+ * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
+ *
+ * @register_param:
+ *  a0: kvm_run* run
+ *  a1: kvm_vcpu* vcpu
+ */
+SYM_FUNC_START(kvm_enter_guest)
+	/* allocate space in stack bottom */
+	KVM_LONG_ADDI	a2, sp, -PT_SIZE
+
+	/* save host gprs */
+	kvm_save_host_gpr a2
+
+	/* save host crmd,prmd csr to stack */
+	csrrd		a3, KVM_CSR_CRMD
+	KVM_LONG_S	a3, a2, PT_CRMD
+	csrrd		a3, KVM_CSR_PRMD
+	KVM_LONG_S	a3, a2, PT_PRMD
+
+	KVM_LONG_ADDI	a2, a1, KVM_VCPU_ARCH
+	KVM_LONG_S	sp, a2, KVM_ARCH_HSTACK
+	KVM_LONG_S	tp, a2, KVM_ARCH_HGP
+	/* Save per cpu register */
+	KVM_LONG_S	$r21, a2, KVM_ARCH_HPERCPU
+
+	/* Save kvm_vcpu to kscratch */
+	csrwr	a1, KVM_VCPU_KS
+
+	kvm_switch_to_guest	a2 REG_A2 t0 t1
+
+SYM_FUNC_END(kvm_enter_guest)
+
+SYM_FUNC_START(__kvm_save_fpu)
+	fpu_save_double a0 t1
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_save_fpu)
+
+SYM_FUNC_START(__kvm_restore_fpu)
+	fpu_restore_double a0 t1
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_restore_fpu)
+
+SYM_FUNC_START(__kvm_restore_fcsr)
+	fpu_restore_csr a0 t1
+	fpu_restore_cc  a0 t1 t2
+
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_restore_fcsr)
+
+#ifdef CONFIG_CPU_HAS_LSX
+SYM_FUNC_START(__kvm_save_lsx)
+	lsx_save_data a0 t1
+
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_save_lsx)
+
+SYM_FUNC_START(__kvm_restore_lsx)
+	lsx_restore_data a0 t1
+
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_restore_lsx)
+
+SYM_FUNC_START(__kvm_restore_lsx_upper)
+	lsx_restore_all_upper a0 t0 t1
+
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_restore_lsx_upper)
+#endif
+
+#ifdef CONFIG_CPU_HAS_LASX
+SYM_FUNC_START(__kvm_save_lasx)
+	lasx_save_data a0 t7
+
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_save_lasx)
+
+SYM_FUNC_START(__kvm_restore_lasx)
+	lasx_restore_data a0 t7
+
+	jirl    zero, ra, 0
+SYM_FUNC_END(__kvm_restore_lasx)
+#endif
+

arch/loongarch/kvm/exit.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/preempt.h>
+#include <linux/vmalloc.h>
+#include <asm/cacheflush.h>
+#include <asm/cacheops.h>
+#include <asm/cmpxchg.h>
+#include <asm/fpu.h>
+#include <asm/inst.h>
+#include <asm/mmu_context.h>
+#include <asm/cacheflush.h>
+#include <asm/time.h>
+#include <asm/tlb.h>
+#include <asm/numa.h>
+#include "kvmcpu.h"
+#include <linux/kvm_host.h>
+#include "trace.h"
+#include "kvm_compat.h"
+#include "kvmcsr.h"
+#include "intc/ls3a_ext_irq.h"
+
+/*
+ * Loongarch KVM callback handling for not implemented guest exiting
+ */
+static int _kvm_fault_ni(struct kvm_vcpu *vcpu)
+{
+	unsigned long estat, badv;
+	unsigned int exccode, inst;
+
+	/*
+	 *  Fetch the instruction.
+	 */
+	badv = vcpu->arch.badv;
+	estat = vcpu->arch.host_estat;
+	exccode = (estat & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT;
+	inst = vcpu->arch.badi;
+	kvm_err("Exccode: %d PC=%#lx inst=0x%08x BadVaddr=%#lx estat=%#llx\n",
+		exccode, vcpu->arch.pc, inst, badv, kvm_read_gcsr_estat());
+	kvm_arch_vcpu_dump_regs(vcpu);
+	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+	return RESUME_HOST;
+}
+
+static int _kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst)
+{
+	enum emulation_result er = EMULATE_DONE;
+	unsigned int rd, rj, csrid;
+	unsigned long csr_mask;
+	unsigned long val = 0;
+
+	/*
+	 * CSR value mask imm
+	 * rj = 0 means csrrd
+	 * rj = 1 means csrwr
+	 * rj != 0,1 means csrxchg
+	 */
+	rd = inst.reg2csr_format.rd;
+	rj = inst.reg2csr_format.rj;
+	csrid = inst.reg2csr_format.csr;
+
+	/* Process CSR ops */
+	if (rj == 0) {
+		/* process csrrd */
+		val = _kvm_emu_read_csr(vcpu, csrid);
+		if (er != EMULATE_FAIL)
+			vcpu->arch.gprs[rd] = val;
+	} else if (rj == 1) {
+		/* process csrwr */
+		val = vcpu->arch.gprs[rd];
+		_kvm_emu_write_csr(vcpu, csrid, val);
+	} else {
+		/* process csrxchg */
+		val = vcpu->arch.gprs[rd];
+		csr_mask = vcpu->arch.gprs[rj];
+		_kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val);
+	}
+
+	return er;
+}
+
+static int _kvm_emu_cache(struct kvm_vcpu *vcpu, larch_inst inst)
+{
+	return EMULATE_DONE;
+}
+
+static int _kvm_trap_handle_gspr(struct kvm_vcpu *vcpu)
+{
+	enum emulation_result er = EMULATE_DONE;
+	struct kvm_run *run = vcpu->run;
+	larch_inst inst;
+	unsigned long curr_pc;
+	int rd, rj;
+	unsigned int index;
+
+	/*
+	 *  Fetch the instruction.
+	 */
+	inst.word = vcpu->arch.badi;
+	curr_pc = vcpu->arch.pc;
+	update_pc(&vcpu->arch);
+
+	er = EMULATE_FAIL;
+	switch (((inst.word >> 24) & 0xff)) {
+	case 0x0:
+		/* cpucfg GSPR */
+		if (inst.reg2_format.opcode == 0x1B) {
+			rd = inst.reg2_format.rd;
+			rj = inst.reg2_format.rj;
+			++vcpu->stat.cpucfg_exits;
+			index = vcpu->arch.gprs[rj];
+			vcpu->arch.gprs[rd] = vcpu->kvm->arch.cpucfgs.cpucfg[index];
+			if (vcpu->arch.gprs[rd] == 0) {
+				/*
+				 * Fallback to get host cpucfg info, this is just for
+				 * compatible with older qemu.
+				 */
+				vcpu->arch.gprs[rd] = read_cpucfg(index);
+				/* Nested KVM is not supported */
+				if (index == 2)
+					vcpu->arch.gprs[rd] &= ~CPUCFG2_LVZP;
+			}
+			er = EMULATE_DONE;
+		}
+		break;
+	case 0x4:
+		/* csr GSPR */
+		er = _kvm_handle_csr(vcpu, inst);
+		break;
+	case 0x6:
+		/* iocsr,cache,idle GSPR */
+		switch (((inst.word >> 22) & 0x3ff)) {
+		case 0x18:
+			/* cache GSPR */
+			er = _kvm_emu_cache(vcpu, inst);
+			trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
+			break;
+		case 0x19:
+			/* iocsr/idle GSPR */
+			switch (((inst.word >> 15) & 0x1ffff)) {
+			case 0xc90:
+				/* iocsr GSPR */
+				er = _kvm_emu_iocsr(inst, run, vcpu);
+				break;
+			case 0xc91:
+				/* idle GSPR */
+				er = _kvm_emu_idle(vcpu);
+				break;
+			default:
+				er = EMULATE_FAIL;
+				break;
+			}
+			break;
+		default:
+			er = EMULATE_FAIL;
+			break;
+		}
+		break;
+	default:
+		er = EMULATE_FAIL;
+		break;
+	}
+
+	/* Rollback PC only if emulation was unsuccessful */
+	if (er == EMULATE_FAIL) {
+		kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n",
+			curr_pc, __func__, inst.word);
+
+		kvm_arch_vcpu_dump_regs(vcpu);
+		vcpu->arch.pc = curr_pc;
+	}
+	return er;
+}
+
+static int _kvm_check_hypcall(struct kvm_vcpu *vcpu)
+{
+	enum emulation_result ret;
+	larch_inst inst;
+	unsigned long curr_pc;
+	unsigned int code;
+
+	/*
+	 * Update PC and hold onto current PC in case there is
+	 * an error and we want to rollback the PC
+	 */
+	inst.word = vcpu->arch.badi;
+	code = inst.reg0i15_format.simmediate;
+	curr_pc = vcpu->arch.pc;
+	update_pc(&vcpu->arch);
+
+	ret = EMULATE_DONE;
+	switch (code) {
+	case KVM_HC_CODE_SERIVCE:
+		ret = EMULATE_PV_HYPERCALL;
+		break;
+	case KVM_HC_CODE_SWDBG:
+		/*
+		 * Only SWDBG(SoftWare DeBug) could stop vm
+		 * code other than 0 is ignored.
+		 */
+		ret = EMULATE_DEBUG;
+		break;
+	default:
+		kvm_info("[%#lx] HYPCALL %#03x unsupported\n", vcpu->arch.pc, code);
+		break;
+	}
+
+	if (ret == EMULATE_DEBUG)
+		vcpu->arch.pc = curr_pc;
+
+	return ret;
+}
+
+/* Execute cpucfg instruction will tirggerGSPR,
+ * Also the access to unimplemented csrs 0x15
+ * 0x16, 0x50~0x53, 0x80, 0x81, 0x90~0x95, 0x98
+ * 0xc0~0xff, 0x100~0x109, 0x500~0x502,
+ * cache_op, idle_op iocsr ops the same */
+static int _kvm_handle_gspr(struct kvm_vcpu *vcpu)
+{
+	enum emulation_result er = EMULATE_DONE;
+	int ret = RESUME_GUEST;
+
+	vcpu->arch.is_hypcall = 0;
+
+	er = _kvm_trap_handle_gspr(vcpu);
+
+	if (er == EMULATE_DONE) {
+		ret = RESUME_GUEST;
+	} else if (er == EMULATE_DO_MMIO) {
+		vcpu->run->exit_reason = KVM_EXIT_MMIO;
+		ret = RESUME_HOST;
+	} else if (er == EMULATE_DO_IOCSR) {
+		vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR;
+		ret = RESUME_HOST;
+	} else {
+		kvm_err("%s internal error\n", __func__);
+		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		ret = RESUME_HOST;
+	}
+	return ret;
+}
+
+static int _kvm_handle_hypcall(struct kvm_vcpu *vcpu)
+{
+	enum emulation_result er = EMULATE_DONE;
+	int ret = RESUME_GUEST;
+
+	vcpu->arch.is_hypcall = 0;
+	er = _kvm_check_hypcall(vcpu);
+
+	if (er == EMULATE_PV_HYPERCALL)
+		ret = _kvm_handle_pv_hcall(vcpu);
+	else if (er == EMULATE_DEBUG) {
+		vcpu->run->exit_reason = KVM_EXIT_DEBUG;
+		ret = RESUME_HOST;
+	} else
+		ret = RESUME_GUEST;
+
+	return ret;
+}
+
+static int _kvm_handle_gcm(struct kvm_vcpu *vcpu)
+{
+	int ret, subcode;
+
+	vcpu->arch.is_hypcall = 0;
+	ret = RESUME_GUEST;
+	subcode = (vcpu->arch.host_estat & KVM_ESTAT_ESUBCODE) >> KVM_ESTAT_ESUBCODE_SHIFT;
+	if ((subcode != EXCSUBCODE_GCSC) && (subcode != EXCSUBCODE_GCHC)) {
+		kvm_err("%s internal error\n", __func__);
+		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		ret = RESUME_HOST;
+	}
+
+	return ret;
+}
+
+/**
+ * _kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host
+ * @vcpu:	Virtual CPU context.
+ *
+ * Handle when the guest attempts to use fpu which hasn't been allowed
+ * by the root context.
+ */
+static int _kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu)
+{
+	struct kvm_run *run = vcpu->run;
+
+	/*
+	 * If guest FPU not present, the FPU operation should have been
+	 * treated as a reserved instruction!
+	 * If FPU already in use, we shouldn't get this at all.
+	 */
+	if (WARN_ON(!_kvm_guest_has_fpu(&vcpu->arch) ||
+				vcpu->arch.aux_inuse & KVM_LARCH_FPU)) {
+		kvm_err("%s internal error\n", __func__);
+		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		return RESUME_HOST;
+	}
+
+	kvm_own_fpu(vcpu);
+	return RESUME_GUEST;
+}
+
+/**
+ * _kvm_handle_lsx_disabled() - Guest used LSX while disabled in root.
+ * @vcpu:	Virtual CPU context.
+ *
+ * Handle when the guest attempts to use LSX when it is disabled in the root
+ * context.
+ */
+static int _kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu)
+{
+	struct kvm_run *run = vcpu->run;
+
+	/*
+	 * If LSX not present or not exposed to guest, the LSX operation
+	 * should have been treated as a reserved instruction!
+	 * If LSX already in use, we shouldn't get this at all.
+	 */
+	if (!_kvm_guest_has_lsx(&vcpu->arch) ||
+	    !(kvm_read_gcsr_euen() & KVM_EUEN_LSXEN) ||
+	    vcpu->arch.aux_inuse & KVM_LARCH_LSX) {
+		kvm_err("%s internal error, lsx %d guest euen %llx aux %x",
+			__func__, _kvm_guest_has_lsx(&vcpu->arch),
+			kvm_read_gcsr_euen(), vcpu->arch.aux_inuse);
+		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		return RESUME_HOST;
+	}
+
+#ifdef CONFIG_CPU_HAS_LSX
+	kvm_own_lsx(vcpu);
+#endif
+	return RESUME_GUEST;
+}
+
+bool _kvm_guest_has_lasx(struct kvm_vcpu *vcpu)
+{
+      return cpu_has_lasx && vcpu->arch.lsx_enabled && vcpu->kvm->arch.cpucfg_lasx;
+}
+
+/**
+ * _kvm_handle_lasx_disabled() - Guest used LASX while disabled in root.
+ * @vcpu:	Virtual CPU context.
+ *
+ * Handle when the guest attempts to use LASX when it is disabled in the root
+ * context.
+ */
+static int _kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu)
+{
+	struct kvm_run *run = vcpu->run;
+
+	/*
+	 * If LASX not present or not exposed to guest, the LASX operation
+	 * should have been treated as a reserved instruction!
+	 * If LASX already in use, we shouldn't get this at all.
+	 */
+	if (!_kvm_guest_has_lasx(vcpu) ||
+	    !(kvm_read_gcsr_euen() & KVM_EUEN_LSXEN) ||
+	    !(kvm_read_gcsr_euen() & KVM_EUEN_LASXEN) ||
+	    vcpu->arch.aux_inuse & KVM_LARCH_LASX) {
+		kvm_err("%s internal error, lasx %d guest euen %llx aux %x",
+			__func__, _kvm_guest_has_lasx(vcpu),
+			kvm_read_gcsr_euen(), vcpu->arch.aux_inuse);
+		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		return RESUME_HOST;
+	}
+
+#ifdef CONFIG_CPU_HAS_LASX
+	kvm_own_lasx(vcpu);
+#endif
+	return RESUME_GUEST;
+}
+
+
+static int _kvm_handle_read_fault(struct kvm_vcpu *vcpu)
+{
+	struct kvm_run *run = vcpu->run;
+	ulong badv = vcpu->arch.badv;
+	larch_inst inst;
+	enum emulation_result er = EMULATE_DONE;
+	int ret = RESUME_GUEST;
+
+	if (kvm_handle_mm_fault(vcpu, badv, false)) {
+		/* A code fetch fault doesn't count as an MMIO */
+		if (kvm_is_ifetch_fault(&vcpu->arch)) {
+			kvm_err("%s ifetch error addr:%lx\n", __func__, badv);
+			run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+			return RESUME_HOST;
+		}
+
+		/* Treat as MMIO */
+		inst.word =  vcpu->arch.badi;
+		er = _kvm_emu_mmio_read(vcpu, inst);
+		if (er == EMULATE_FAIL) {
+			kvm_err("Guest Emulate Load failed: PC: %#lx, BadVaddr: %#lx\n",
+				vcpu->arch.pc, badv);
+			run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		}
+	}
+
+	if (er == EMULATE_DONE) {
+		ret = RESUME_GUEST;
+	} else if (er == EMULATE_DO_MMIO) {
+		run->exit_reason = KVM_EXIT_MMIO;
+		ret = RESUME_HOST;
+	} else {
+		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		ret = RESUME_HOST;
+	}
+	return ret;
+}
+
+static int _kvm_handle_write_fault(struct kvm_vcpu *vcpu)
+{
+	struct kvm_run *run = vcpu->run;
+	ulong badv = vcpu->arch.badv;
+	larch_inst inst;
+	enum emulation_result er = EMULATE_DONE;
+	int ret = RESUME_GUEST;
+
+	if (kvm_handle_mm_fault(vcpu, badv, true)) {
+
+		/* Treat as MMIO */
+		inst.word =  vcpu->arch.badi;
+		er = _kvm_emu_mmio_write(vcpu, inst);
+		if (er == EMULATE_FAIL) {
+			kvm_err("Guest Emulate Store failed: PC:  %#lx, BadVaddr: %#lx\n",
+				vcpu->arch.pc, badv);
+			run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		}
+	}
+
+	if (er == EMULATE_DONE) {
+		ret = RESUME_GUEST;
+	} else if (er == EMULATE_DO_MMIO) {
+		run->exit_reason = KVM_EXIT_MMIO;
+		ret = RESUME_HOST;
+	} else {
+		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		ret = RESUME_HOST;
+	}
+	return ret;
+}
+
+static int _kvm_handle_debug(struct kvm_vcpu *vcpu)
+{
+	uint32_t fwps, mwps;
+
+	fwps = kvm_csr_readq(KVM_CSR_FWPS);
+	mwps = kvm_csr_readq(KVM_CSR_MWPS);
+	if (fwps & 0xff)
+		kvm_csr_writeq(fwps, KVM_CSR_FWPS);
+	if (mwps & 0xff)
+		kvm_csr_writeq(mwps, KVM_CSR_MWPS);
+	vcpu->run->debug.arch.exception = KVM_EXCCODE_WATCH;
+	vcpu->run->debug.arch.fwps = fwps;
+	vcpu->run->debug.arch.mwps = mwps;
+	vcpu->run->exit_reason = KVM_EXIT_DEBUG;
+	return RESUME_HOST;
+}
+
+static exit_handle_fn _kvm_fault_tables[KVM_INT_START] = {
+	[KVM_EXCCODE_TLBL]		= _kvm_handle_read_fault,
+	[KVM_EXCCODE_TLBS]		= _kvm_handle_write_fault,
+	[KVM_EXCCODE_TLBI]		= _kvm_handle_read_fault,
+	[KVM_EXCCODE_TLBM]		= _kvm_handle_write_fault,
+	[KVM_EXCCODE_TLBRI]		= _kvm_handle_read_fault,
+	[KVM_EXCCODE_TLBXI]		= _kvm_handle_read_fault,
+	[KVM_EXCCODE_FPDIS]		= _kvm_handle_fpu_disabled,
+	[KVM_EXCCODE_LSXDIS]	= _kvm_handle_lsx_disabled,
+	[KVM_EXCCODE_LASXDIS]	= _kvm_handle_lasx_disabled,
+	[KVM_EXCCODE_WATCH]		= _kvm_handle_debug,
+	[KVM_EXCCODE_GSPR]		= _kvm_handle_gspr,
+	[KVM_EXCCODE_HYP]		= _kvm_handle_hypcall,
+	[KVM_EXCCODE_GCM]		= _kvm_handle_gcm,
+};
+
+int _kvm_handle_fault(struct kvm_vcpu *vcpu, int fault)
+{
+	return _kvm_fault_tables[fault](vcpu);
+}
+
+void _kvm_init_fault(void)
+{
+	int i;
+
+	for (i = 0; i < KVM_INT_START; i++)
+		if (!_kvm_fault_tables[i])
+			_kvm_fault_tables[i] = _kvm_fault_ni;
+}

arch/loongarch/kvm/fpu.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/kernel.h>
+#include <linux/kvm_host.h>
+#include <linux/kvm_para.h>
+#include <asm/fpu.h>
+
+/* FPU/LSX context management */
+void __kvm_save_fpu(struct loongarch_fpu *fpu);
+void __kvm_restore_fpu(struct loongarch_fpu *fpu);
+void __kvm_restore_fcsr(struct loongarch_fpu *fpu);
+
+void kvm_save_fpu(struct kvm_vcpu *cpu)
+{
+	return __kvm_save_fpu(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_save_fpu);
+
+void kvm_restore_fpu(struct kvm_vcpu *cpu)
+{
+	return __kvm_restore_fpu(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_restore_fpu);
+
+void kvm_restore_fcsr(struct kvm_vcpu *cpu)
+{
+	return __kvm_restore_fcsr(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_restore_fcsr);
+
+#ifdef CONFIG_CPU_HAS_LSX
+void __kvm_save_lsx(struct loongarch_fpu *fpu);
+void __kvm_restore_lsx(struct loongarch_fpu *fpu);
+void __kvm_restore_lsx_upper(struct loongarch_fpu *fpu);
+
+void kvm_save_lsx(struct kvm_vcpu *cpu)
+{
+	return __kvm_save_lsx(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_save_lsx);
+
+void kvm_restore_lsx(struct kvm_vcpu *cpu)
+{
+	return __kvm_restore_lsx(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_restore_lsx);
+
+void kvm_restore_lsx_upper(struct kvm_vcpu *cpu)
+{
+	return __kvm_restore_lsx_upper(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_restore_lsx_upper);
+
+#endif
+
+#ifdef CONFIG_CPU_HAS_LSX
+void __kvm_save_lasx(struct loongarch_fpu *fpu);
+void __kvm_restore_lasx(struct loongarch_fpu *fpu);
+void __kvm_restore_lasx_upper(struct loongarch_fpu *fpu);
+
+void kvm_save_lasx(struct kvm_vcpu *cpu)
+{
+	return __kvm_save_lasx(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_save_lasx);
+
+void kvm_restore_lasx(struct kvm_vcpu *cpu)
+{
+	return __kvm_restore_lasx(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_restore_lasx);
+
+void kvm_restore_lasx_upper(struct kvm_vcpu *cpu)
+{
+	return _restore_lasx_upper(&cpu->arch.fpu);
+}
+EXPORT_SYMBOL_GPL(kvm_restore_lasx_upper);
+#endif
+
+EXPORT_SYMBOL_GPL(kvm_enter_guest);
+EXPORT_SYMBOL_GPL(kvm_exception_entry);
+

arch/loongarch/kvm/hypcall.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/kernel.h>
+#include <linux/kvm_host.h>
+#include <linux/sched/stat.h>
+#include <asm/kvm_para.h>
+#include "intc/ls3a_ipi.h"
+#include "kvm_compat.h"
+
+
+int kvm_virt_ipi(struct kvm_vcpu *vcpu)
+{
+	int ret = 0;
+	u64 ipi_bitmap;
+	unsigned int min, action, cpu;
+
+	ipi_bitmap = vcpu->arch.gprs[KVM_REG_A1];
+	min = vcpu->arch.gprs[KVM_REG_A2];
+	action = vcpu->arch.gprs[KVM_REG_A3];
+
+	if (ipi_bitmap) {
+		cpu = find_first_bit((void *)&ipi_bitmap, BITS_PER_LONG);
+		while (cpu < BITS_PER_LONG) {
+			kvm_helper_send_ipi(vcpu, cpu + min, action);
+			cpu = find_next_bit((void *)&ipi_bitmap, BITS_PER_LONG, cpu + 1);
+		}
+	}
+
+	return ret;
+}
+
+int kvm_save_notify(struct kvm_vcpu *vcpu)
+{
+	unsigned long num, id, data;
+
+	int ret = 0;
+
+	num = vcpu->arch.gprs[KVM_REG_A0];
+	id = vcpu->arch.gprs[KVM_REG_A1];
+	data = vcpu->arch.gprs[KVM_REG_A2];
+
+	switch (id) {
+	case KVM_FEATURE_STEAL_TIME:
+		if (!sched_info_on())
+			break;
+		vcpu->arch.st.guest_addr = data;
+		kvm_debug("cpu :%d addr:%lx\n", vcpu->vcpu_id, data);
+		vcpu->arch.st.last_steal = current->sched_info.run_delay;
+		kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
+		break;
+	default:
+		break;
+	};
+
+	return ret;
+};
+
+static int _kvm_pv_feature(struct kvm_vcpu *vcpu)
+{
+	int feature = vcpu->arch.gprs[KVM_REG_A1];
+	int ret = KVM_RET_NOT_SUPPORTED;
+	switch (feature) {
+	case KVM_FEATURE_STEAL_TIME:
+		if (sched_info_on())
+			ret = KVM_RET_SUC;
+		break;
+	case KVM_FEATURE_MULTI_IPI:
+		ret = KVM_RET_SUC;
+		break;
+	default:
+		break;
+	}
+	return ret;
+}
+
+/*
+ * hypcall emulation always return to guest, Caller should check retval.
+ */
+int _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu)
+{
+	unsigned long func = vcpu->arch.gprs[KVM_REG_A0];
+	int hyp_ret = KVM_RET_NOT_SUPPORTED;
+
+	switch (func) {
+	case KVM_HC_FUNC_FEATURE:
+		hyp_ret = _kvm_pv_feature(vcpu);
+		break;
+	case KVM_HC_FUNC_NOTIFY:
+		hyp_ret = kvm_save_notify(vcpu);
+		break;
+	case KVM_HC_FUNC_IPI:
+		hyp_ret = kvm_virt_ipi(vcpu);
+		break;
+	default:
+		kvm_info("[%#lx] hvc func:%#lx unsupported\n", vcpu->arch.pc, func);
+		break;
+	};
+
+	vcpu->arch.gprs[KVM_REG_A0] = hyp_ret;
+
+	return RESUME_GUEST;
+}

arch/loongarch/kvm/intc/irqchip-debug.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/debugfs.h>
+#include <linux/kvm_host.h>
+#include <linux/seq_file.h>
+#include "kvmcpu.h"
+#include "ls3a_ext_irq.h"
+#include "ls7a_irq.h"
+
+#ifdef CONFIG_DEBUG_FS
+static int irqchip_state_show(struct seq_file *m, void *v)
+{
+	struct kvm *kvm = m->private;
+
+	kvm_get_kvm(kvm);
+	kvm_dump_ls3a_extirq_state(m, kvm->arch.v_extirq);
+	kvm_dump_ls7a_ioapic_state(m, kvm->arch.v_ioapic);
+	kvm_put_kvm(kvm);
+
+	return 0;
+}
+
+static int irqchip_debug_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, irqchip_state_show, inode->i_private);
+}
+
+static const struct file_operations irqchip_debug_fops = {
+	.open       = irqchip_debug_open,
+	.read       = seq_read,
+	.llseek     = seq_lseek,
+	.release    = single_release,
+};
+
+void irqchip_debug_init(struct kvm *kvm)
+{
+	debugfs_create_file("irqchip-state", 0444, kvm->debugfs_dentry, kvm,
+			    &irqchip_debug_fops);
+}
+#else
+
+void irqchip_debug_init(struct kvm *kvm) {}
+#endif
+void irqchip_debug_destroy(struct kvm *kvm)
+{
+}

arch/loongarch/kvm/intc/ls3a_ext_irq.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/random.h>
+#include "kvmcpu.h"
+#include "kvm_compat.h"
+#include "ls3a_ipi.h"
+#include "ls7a_irq.h"
+#include "ls3a_ext_irq.h"
+
+#define ls3a_ext_irq_lock(s, flags)	spin_lock_irqsave(&s->lock, flags)
+#define ls3a_ext_irq_unlock(s, flags)	spin_unlock_irqrestore(&s->lock, flags)
+
+extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
+			     struct kvm_loongarch_interrupt *irq);
+void ext_deactive_core_isr(struct kvm *kvm, int irq_num, int vcpu_id)
+{
+	int ipnum;
+	unsigned long found1;
+	struct kvm_loongarch_interrupt irq;
+	struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm);
+	struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq);
+
+	ipnum = state->ext_sw_ipmap[irq_num];
+
+	bitmap_clear((void *)state->ext_isr.reg_u8, irq_num, 1);
+	bitmap_clear((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1);
+
+	bitmap_clear((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1);
+	found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0);
+	kvm_debug("vcpu_id %d irqnum %d found:0x%lx ipnum %d down\n", vcpu_id, irq_num, found1, ipnum);
+	if (found1 == EXTIOI_IRQS) {
+		irq.cpu = vcpu_id;
+		irq.irq = -(ipnum + 2);		/* IP2~IP5 */
+		if (likely(kvm->vcpus[vcpu_id]))
+			kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq);
+		kvm->stat.trigger_ls3a_ext_irq++;
+	}
+}
+
+/**
+ * ext_irq_update_core()
+ * @kvm: KVM structure pointer
+ * @irq_num: 0~256 ext irq num
+ * @level: 0~1 High and low level
+ *
+ * Route the status of the extended interrupt to the host CPU core.
+ *
+ */
+void ext_irq_update_core(struct kvm *kvm, int irq_num, int level)
+{
+	int nrcpus, ipnum, vcpu_id;
+	unsigned long found1;
+	struct kvm_loongarch_interrupt irq;
+	struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm);
+	struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq);
+
+	nrcpus = atomic_read(&kvm->online_vcpus);
+	vcpu_id = state->ext_sw_coremap[irq_num];
+	ipnum = state->ext_sw_ipmap[irq_num];
+
+	if (vcpu_id > (nrcpus - 1)) {
+		vcpu_id = 0;
+	}
+
+	if (level == 1) {
+		if (test_bit(irq_num, (void *)state->ext_en.reg_u8) == false) {
+			return;
+		}
+		if (test_bit(irq_num, (void *)state->ext_isr.reg_u8) == false) {
+			return;
+		}
+		bitmap_set((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1);
+
+		found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0);
+		bitmap_set((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1);
+		kvm_debug("%s:%d  --- vcpu_id %d irqnum %d found1 0x%lx ipnum %d\n",
+					__FUNCTION__, __LINE__, vcpu_id, irq_num, found1, ipnum);
+		if (found1 == EXTIOI_IRQS) {
+			irq.cpu = vcpu_id;
+			irq.irq = ipnum + 2;	/* IP2~IP5 */
+			kvm_debug("%s:%d --- vcpu_id %d ipnum %d raise\n",
+					__FUNCTION__, __LINE__, vcpu_id, ipnum);
+			if (likely(kvm->vcpus[vcpu_id]))
+				kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq);
+			kvm->stat.trigger_ls3a_ext_irq++;
+		}
+	} else {
+		bitmap_clear((void *)state->ext_isr.reg_u8, irq_num, 1);
+		bitmap_clear((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1);
+
+		bitmap_clear((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1);
+		found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0);
+		if (found1 == EXTIOI_IRQS) {
+			irq.cpu = vcpu_id;
+			irq.irq = -(ipnum + 2);		/* IP2~IP5 */
+			if (likely(kvm->vcpus[vcpu_id]))
+				kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq);
+			kvm->stat.trigger_ls3a_ext_irq++;
+		}
+
+	}
+}
+
+void msi_irq_handler(struct kvm *kvm, int irq, int level)
+{
+	unsigned long flags;
+	struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm);
+	struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq);
+
+	if (!atomic64_read(&s->enabled))
+		return;
+
+	kvm_debug("ext_irq_handler:irq = %d,level = %d\n", irq, level);
+
+	ls3a_ext_irq_lock(s, flags);
+	if (level == 1) {
+		if (test_bit(irq, (void *)&state->ext_isr))
+			goto out;
+		__set_bit(irq, (void *)&state->ext_isr);
+	} else {
+		if (!test_bit(irq, (void *)&state->ext_isr))
+			goto out;
+		__clear_bit(irq, (void *)&state->ext_isr);
+	}
+
+	ext_irq_update_core(kvm, irq, level);
+out:
+	ls3a_ext_irq_unlock(s, flags);
+}
+
+static int ls3a_ext_intctl_readb(struct kvm_vcpu *vcpu,
+			      struct kvm_io_device *dev,
+			      gpa_t addr, void *val)
+{
+	uint64_t offset, reg_count;
+	struct ls3a_kvm_extirq *s = NULL;
+	struct kvm_ls3a_extirq_state *state = NULL;
+	int vcpu_id;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	state = &(s->ls3a_ext_irq);
+
+	offset = addr & 0xfffff;
+
+	if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) {
+		reg_count = (offset - EXTIOI_ENABLE_START);
+		*(uint8_t *)val = state->ext_en.reg_u8[reg_count];
+	} else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) {
+		reg_count = (offset - EXTIOI_BOUNCE_START);
+		*(uint8_t *)val = state->bounce.reg_u8[reg_count];
+	} else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) {
+		reg_count = (offset - EXTIOI_ISR_START);
+		*(uint8_t *)val = state->ext_isr.reg_u8[reg_count];
+	} else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) {
+		/* percpu(32 bytes) coreisr reg_count is 0~31 */
+		vcpu_id = (offset >> 8) & 0xff;
+		reg_count = offset & 0xff;
+		*(uint8_t *)val = state->ext_core_isr.reg_u8[vcpu_id][reg_count];
+	} else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) {
+		reg_count = (offset - EXTIOI_IPMAP_START);
+		*(uint8_t *)val = state->ip_map.reg_u8[reg_count];
+	} else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) {
+		reg_count = (offset - EXTIOI_COREMAP_START);
+		*(uint8_t *)val = state->core_map.reg_u8[reg_count];
+	} else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) {
+		reg_count = (offset - EXTIOI_NODETYPE_START);
+		*(uint8_t *)val = state->node_type.reg_u8[reg_count];
+	}
+	kvm_debug("%s: addr=0x%llx,val=0x%x\n",
+				__FUNCTION__, addr, *(uint8_t *)val);
+	return 0;
+}
+
+static int ls3a_ext_intctl_readw(struct kvm_vcpu *vcpu,
+			      struct kvm_io_device *dev,
+			      gpa_t addr, void *val)
+{
+	uint64_t offset, reg_count;
+	struct ls3a_kvm_extirq *s = NULL;
+	struct kvm_ls3a_extirq_state *state = NULL;
+	int vcpu_id;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	state = &(s->ls3a_ext_irq);
+
+	offset = addr & 0xfffff;
+
+	if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) {
+		reg_count = (offset - EXTIOI_ENABLE_START) / 4;
+		*(uint32_t *)val = state->ext_en.reg_u32[reg_count];
+	} else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) {
+		reg_count = (offset - EXTIOI_BOUNCE_START) / 4;
+		*(uint32_t *)val = state->bounce.reg_u32[reg_count];
+	} else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) {
+		reg_count = (offset - EXTIOI_ISR_START) / 4;
+		*(uint32_t *)val = state->ext_isr.reg_u32[reg_count];
+	} else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) {
+		/* percpu(32 bytes) coreisr reg_count is 0~7*/
+		vcpu_id = (offset >> 8) & 0xff;
+		reg_count = (offset & 0xff) / 4;
+		*(uint32_t *)val = state->ext_core_isr.reg_u32[vcpu_id][reg_count];
+	} else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) {
+		reg_count = (offset - EXTIOI_IPMAP_START) / 4;
+		*(uint32_t *)val = state->ip_map.reg_u32[reg_count];
+	} else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) {
+		reg_count = (offset - EXTIOI_COREMAP_START) / 4;
+		*(uint32_t *)val = state->core_map.reg_u32[reg_count];
+	} else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) {
+		reg_count = (offset - EXTIOI_NODETYPE_START) / 4;
+		*(uint32_t *)val = state->node_type.reg_u32[reg_count];
+	}
+	kvm_debug("%s: addr=0x%llx,val=0x%x\n",
+				__FUNCTION__, addr, *(uint32_t *)val);
+
+	return 0;
+}
+
+static int ls3a_ext_intctl_readl(struct kvm_vcpu *vcpu,
+			      struct kvm_io_device *dev,
+			      gpa_t addr, void *val)
+{
+	uint64_t offset, reg_count;
+	struct ls3a_kvm_extirq *s = NULL;
+	struct kvm_ls3a_extirq_state *state = NULL;
+	int vcpu_id;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	state = &(s->ls3a_ext_irq);
+
+	offset = addr & 0xfffff;
+
+	if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) {
+		reg_count = (offset - EXTIOI_ENABLE_START) / 8;
+		*(uint64_t *)val = state->ext_en.reg_u64[reg_count];
+	} else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) {
+		reg_count = (offset - EXTIOI_BOUNCE_START) / 8;
+		*(uint64_t *)val = state->bounce.reg_u64[reg_count];
+	} else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) {
+		reg_count = (offset - EXTIOI_ISR_START) / 8;
+		*(uint64_t *)val = state->ext_isr.reg_u64[reg_count];
+	} else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) {
+		/* percpu(32 bytes) coreisr reg_count is 0~3*/
+		vcpu_id = (offset >> 8) & 0xff;
+		reg_count = (offset & 0xff) / 8;
+
+		*(uint64_t *)val = state->ext_core_isr.reg_u64[vcpu_id][reg_count];
+	} else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) {
+		*(uint64_t *)val = state->ip_map.reg_u64;
+	} else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) {
+		reg_count = (offset - EXTIOI_COREMAP_START) / 8;
+		*(uint64_t *)val = state->core_map.reg_u64[reg_count];
+	} else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) {
+		reg_count = (offset - EXTIOI_NODETYPE_START) / 8;
+		*(uint64_t *)val = state->node_type.reg_u64[reg_count];
+	}
+	kvm_debug("%s: addr=0x%llx,val=0x%llx\n",
+					__FUNCTION__, addr, *(uint64_t *)val);
+	return 0;
+}
+/**
+ * ls3a_ext_intctl_read()
+ * @kvm: KVM structure pointer
+ * @addr: Register address
+ * @size: The width of the register to be read.
+ * @val: The pointer to the read result.
+ *
+ * Analog extended interrupt related register read.
+ *
+ */
+static int ls3a_ext_intctl_read(struct kvm_vcpu *vcpu,
+			      struct kvm_io_device *dev,
+			      gpa_t addr, int size, void *val)
+{
+	struct ls3a_kvm_extirq *s = NULL;
+	unsigned long flags;
+	uint64_t offset;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	offset = addr & 0xfffff;
+	if (offset & (size - 1)) {
+		printk("%s:unaligned address access %llx size %d\n",
+			__FUNCTION__, addr, size);
+		return 0;
+	}
+	addr = (addr & 0xfffff) - EXTIOI_ADDR_OFF;
+	ls3a_ext_irq_lock(s, flags);
+
+	switch (size) {
+	case 1:
+		ls3a_ext_intctl_readb(vcpu, dev, addr, val);
+		break;
+	case 4:
+		ls3a_ext_intctl_readw(vcpu, dev, addr, val);
+		break;
+	case 8:
+		ls3a_ext_intctl_readl(vcpu, dev, addr, val);
+		break;
+	default:
+		WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx, size %d\n",
+						__FUNCTION__, addr, size);
+	}
+	ls3a_ext_irq_unlock(s, flags);
+	kvm_debug("%s(%d):address access %llx size %d\n",
+					__FUNCTION__, __LINE__, offset, size);
+
+	return 0;
+}
+
+static int ls3a_ext_intctl_writeb(struct kvm_vcpu *vcpu,
+			 struct kvm_io_device *dev,
+			 gpa_t addr, const void *__val)
+{
+	uint64_t offset, reg_count;
+	uint8_t val_data_u8, old_data_u8;
+	struct ls3a_kvm_extirq *s = NULL;
+	struct kvm_ls3a_extirq_state *state = NULL;
+	struct kvm *kvm = NULL;
+	int  mask, level, i, irqnum, ipnum;
+	int vcpu_id;
+
+	unsigned long val = *(unsigned long *)__val;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	state = &(s->ls3a_ext_irq);
+	kvm = s->kvm;
+
+	offset = addr & 0xfffff;
+	val_data_u8 = val & 0xffUL;
+
+	kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val);
+
+	if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) {
+		reg_count = (offset - EXTIOI_ENABLE_START);
+		old_data_u8 = state->ext_en.reg_u8[reg_count];
+		if (old_data_u8 != val_data_u8) {
+			state->ext_en.reg_u8[reg_count] = val_data_u8;
+			old_data_u8 = old_data_u8 ^ val_data_u8;
+			mask = 0x1;
+			for (i = 0; i < 8; i++) {
+				if (old_data_u8 & mask) {
+					level = !!(val_data_u8 & (0x1 << i));
+					if (level)
+						ext_irq_update_core(kvm, i + reg_count * 8, level);
+				}
+				mask = mask << 1;
+			}
+		}
+	} else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) {
+		reg_count = (offset - EXTIOI_BOUNCE_START);
+		state->bounce.reg_u8[reg_count] = val_data_u8;
+	} else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) {
+		/*can not be writen*/
+		reg_count = (offset - EXTIOI_ISR_START) & 0x1f;
+		old_data_u8 = state->ext_isr.reg_u8[reg_count];
+		state->ext_isr.reg_u8[reg_count] = old_data_u8 & (~val_data_u8);
+
+		mask = 0x1;
+		for (i = 0; i < 8; i++) {
+			if ((old_data_u8 & mask) && (val_data_u8 & mask)) {
+				ext_irq_update_core(kvm, i + reg_count * 8, 0);
+			}
+			mask = mask << 1;
+		}
+
+	} else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) {
+		int bits;
+		/* percpu(32 bytes) coreisr reg_count is 0~31 */
+		vcpu_id = (offset >> 8) & 0xff;
+		reg_count = offset & 0xff;
+
+		state->ext_core_isr.reg_u8[vcpu_id][reg_count] &= ~val_data_u8;
+
+		bits = sizeof(val_data_u8) * 8;
+		i = find_first_bit((void *)&val_data_u8, bits);
+		while (i < bits) {
+			ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id);
+			bitmap_clear((void *)&val_data_u8, i, 1);
+			i = find_first_bit((void *)&val_data_u8, bits);
+		}
+	} else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) {
+		/*drop arch.core_ip_mask use state->ip_map*/
+		reg_count = (offset - EXTIOI_IPMAP_START);
+		state->ip_map.reg_u8[reg_count] = val_data_u8;
+
+		ipnum = 0;
+
+		for (i = 0; i < 4; i++) {
+			if (val_data_u8 & (0x1 << i)) {
+				ipnum = i;
+				break;
+			}
+		}
+
+		if (val_data_u8) {
+			for (i = 0; i < 32; i++) {
+				irqnum = reg_count * 32 + i;
+				state->ext_sw_ipmap[irqnum] = ipnum;
+			}
+		} else {
+			for (i = 0; i < 32; i++) {
+				irqnum = reg_count * 32 + i;
+				state->ext_sw_ipmap[irqnum] = 0;
+			}
+		}
+	} else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) {
+		reg_count = (offset - EXTIOI_COREMAP_START);
+		state->core_map.reg_u8[reg_count] = val_data_u8;
+		state->ext_sw_coremap[reg_count] = val_data_u8;
+	} else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) {
+		reg_count = (offset - EXTIOI_NODETYPE_START);
+		state->node_type.reg_u8[reg_count] = val_data_u8;
+	} else {
+		WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx\n",
+							__FUNCTION__, addr);
+	}
+
+	return 0;
+}
+
+static int ls3a_ext_intctl_writew(struct kvm_vcpu *vcpu,
+			 struct kvm_io_device *dev,
+			 gpa_t addr, const void *__val)
+{
+	uint64_t offset, reg_count;
+	uint32_t val_data_u32, old_data_u32, mask;
+	struct ls3a_kvm_extirq *s = NULL;
+	struct kvm_ls3a_extirq_state *state = NULL;
+	struct kvm *kvm = NULL;
+	uint8_t tmp_data_u8;
+	int i, level, vcpu_id;
+	unsigned long val;
+
+	val = *(unsigned long *)__val;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	state = &(s->ls3a_ext_irq);
+	kvm = s->kvm;
+
+	offset = addr & 0xfffff;
+	val_data_u32 = val & 0xffffffffUL;
+
+	kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val);
+
+	if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) {
+		reg_count = (offset - EXTIOI_ENABLE_START) / 4;
+		old_data_u32 = state->ext_en.reg_u32[reg_count];
+		if (old_data_u32 != val_data_u32) {
+			state->ext_en.reg_u32[reg_count] = val_data_u32;
+			old_data_u32 = old_data_u32 ^ val_data_u32;
+
+			mask = 0x1;
+			for (i = 0; i < 8 * sizeof(old_data_u32); i++) {
+				if (old_data_u32 & mask) {
+					level = !!(val_data_u32 & (0x1 << i));
+					if (level)
+						ext_irq_update_core(kvm, i + reg_count * 32, level);
+				}
+				mask = mask << 1;
+			}
+		}
+	} else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) {
+		reg_count = (offset - EXTIOI_BOUNCE_START) / 4;
+		state->bounce.reg_u32[reg_count] = val_data_u32;
+	} else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) {
+		/*can not be writen*/
+		reg_count = (offset - EXTIOI_ISR_START) / 4;
+		old_data_u32 = state->ext_isr.reg_u32[reg_count];
+		state->ext_isr.reg_u32[reg_count] = old_data_u32 & (~val_data_u32);
+
+		mask = 0x1;
+		for (i = 0; i < 8 * sizeof(old_data_u32); i++) {
+			if ((old_data_u32 & mask) && (val_data_u32 & mask)) {
+				ext_irq_update_core(kvm, i + reg_count * 32, 0);
+			}
+			mask = mask << 1;
+		}
+	} else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) {
+		int bits;
+		/* percpu(32 bytes) coreisr reg_count is 0~7*/
+		vcpu_id = (offset >> 8) & 0xff;
+		reg_count = (offset & 0xff) / 4;
+
+		/*ext_core_ioisr*/
+		state->ext_core_isr.reg_u32[vcpu_id][reg_count] &= ~val_data_u32;
+
+		bits = sizeof(val_data_u32) * 8;
+		i = find_first_bit((void *)&val_data_u32, bits);
+		while (i < bits) {
+			ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id);
+			bitmap_clear((void *)&val_data_u32, i, 1);
+			i = find_first_bit((void *)&val_data_u32, bits);
+		}
+	} else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) {
+		tmp_data_u8 = val_data_u32 & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u32 >> 8) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u32 >> 16) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u32 >> 24) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8);
+	} else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) {
+		tmp_data_u8 = val_data_u32 & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u32 >> 8) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u32 >> 16) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u32 >> 24) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8);
+		kvm_debug("%s:id:%d addr=0x%llx, offset 0x%llx val 0x%x\n",
+				__FUNCTION__, vcpu->vcpu_id, addr, offset, val_data_u32);
+
+	} else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) {
+		reg_count = (offset - EXTIOI_NODETYPE_START) / 4;
+		state->node_type.reg_u32[reg_count] = val_data_u32;
+	} else {
+		WARN_ONCE(1, "%s:%d Abnormal address access:addr 0x%llx\n",
+							__FUNCTION__, __LINE__, addr);
+	}
+
+	return 0;
+}
+
+static int ls3a_ext_intctl_writel(struct kvm_vcpu *vcpu,
+			 struct kvm_io_device *dev,
+			 gpa_t addr, const void *__val)
+{
+	uint64_t offset, val_data_u64, old_data_u64, reg_count, mask, i;
+	struct ls3a_kvm_extirq *s = NULL;
+	struct kvm_ls3a_extirq_state *state = NULL;
+	struct kvm *kvm = NULL;
+	uint8_t tmp_data_u8;
+	int level, vcpu_id;
+
+	unsigned long val = *(unsigned long *)__val;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	state = &(s->ls3a_ext_irq);
+	kvm = s->kvm;
+
+	offset = addr & 0xfffff;
+	val_data_u64 = val;
+
+	kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val);
+
+	if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) {
+		reg_count = (offset - EXTIOI_ENABLE_START) / 8;
+		old_data_u64 = state->ext_en.reg_u64[reg_count];
+		if (old_data_u64 != val_data_u64) {
+			state->ext_en.reg_u64[reg_count] = val_data_u64;
+			old_data_u64 = old_data_u64 ^ val_data_u64;
+
+			mask = 0x1;
+			for (i = 0; i < 8 * sizeof(old_data_u64); i++) {
+				if (old_data_u64 & mask) {
+					level = !!(val_data_u64 & (0x1 << i));
+					if (level)
+						ext_irq_update_core(kvm, i + reg_count * 64, level);
+				}
+				mask = mask << 1;
+			}
+		}
+	} else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) {
+		reg_count = (offset - EXTIOI_BOUNCE_START) / 8;
+		state->bounce.reg_u64[reg_count] = val_data_u64;
+	} else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) {
+		/*can not be writen*/
+		reg_count = (offset - EXTIOI_ISR_START) / 8;
+		old_data_u64 = state->ext_isr.reg_u64[reg_count];
+		state->ext_isr.reg_u64[reg_count] = old_data_u64 & (~val_data_u64);
+
+		mask = 0x1;
+		for (i = 0; i < 8 * sizeof(old_data_u64); i++) {
+			if ((old_data_u64 & mask) && (val_data_u64 & mask)) {
+				ext_irq_update_core(kvm, i + reg_count * 64, 0);
+			}
+			mask = mask << 1;
+		}
+	} else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) {
+		int bits;
+		vcpu_id = (offset >> 8) & 0xff;
+		reg_count = (offset & 0x1f) / 8;
+
+		/*core_ext_ioisr*/
+		state->ext_core_isr.reg_u64[vcpu_id][reg_count] &= ~val_data_u64;
+
+		bits = sizeof(val_data_u64) * 8;
+		i = find_first_bit((void *)&val_data_u64, bits);
+		while (i < bits) {
+			ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id);
+			bitmap_clear((void *)&val_data_u64, i, 1);
+			i = find_first_bit((void *)&val_data_u64, bits);
+		}
+	} else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) {
+		tmp_data_u8 = val_data_u64 & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 8) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 16) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 24) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8);
+
+		tmp_data_u8 = (val_data_u64 >> 32) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 4, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 40) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 5, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 48) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 6, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 56) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 7, &tmp_data_u8);
+	} else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) {
+		tmp_data_u8 = val_data_u64 & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 8) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 16) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 24) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8);
+
+		tmp_data_u8 = (val_data_u64 >> 32) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 4, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 40) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 5, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 48) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 6, &tmp_data_u8);
+		tmp_data_u8 = (val_data_u64 >> 56) & 0xff;
+		ls3a_ext_intctl_writeb(vcpu, dev, addr + 7, &tmp_data_u8);
+	} else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) {
+		reg_count = (offset - EXTIOI_NODETYPE_START) / 8;
+		state->node_type.reg_u64[reg_count] = val_data_u64;
+	} else {
+		WARN_ONCE(1, "%s:%d Abnormal address access:addr 0x%llx\n",
+					__FUNCTION__, __LINE__, addr);
+	}
+	return 0;
+}
+/**
+ * ls3a_ext_intctl_write()
+ * @kvm: KVM structure pointer
+ * @addr: Register address
+ * @size: The width of the register to be writen.
+ * @val: Value to be written.
+ *
+ * Analog extended interrupt related register write.
+ *
+ */
+static int ls3a_ext_intctl_write(struct kvm_vcpu *vcpu,
+			 struct kvm_io_device *dev,
+			 gpa_t addr, int size, const void *__val)
+{
+	struct ls3a_kvm_extirq *s = NULL;
+	unsigned long flags;
+	uint64_t offset;
+
+	s = container_of(dev, struct ls3a_kvm_extirq, device);
+
+	offset = addr & 0xfffff;
+	if (offset & (size - 1)) {
+		printk("%s(%d):unaligned address access %llx size %d\n",
+			__FUNCTION__, __LINE__, addr, size);
+		return 0;
+	}
+
+	addr = (addr & 0xfffff) - EXTIOI_ADDR_OFF;
+	ls3a_ext_irq_lock(s, flags);
+
+	switch (size) {
+	case 1:
+		ls3a_ext_intctl_writeb(vcpu, dev, addr, __val);
+		break;
+	case 4:
+		ls3a_ext_intctl_writew(vcpu, dev, addr, __val);
+		break;
+	case 8:
+		ls3a_ext_intctl_writel(vcpu, dev, addr, __val);
+		break;
+	default:
+		WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx,size %d\n",
+						__FUNCTION__, addr, size);
+	}
+
+	ls3a_ext_irq_unlock(s, flags);
+
+	kvm_debug("%s(%d):address access %llx size %d\n",
+					__FUNCTION__, __LINE__, offset, size);
+	return 0;
+}
+
+static const struct kvm_io_device_ops kvm_ls3a_ext_irq_ops = {
+	.read     = ls3a_ext_intctl_read,
+	.write    = ls3a_ext_intctl_write,
+};
+
+void kvm_destroy_ls3a_ext_irq(struct kvm *kvm)
+{
+	struct ls3a_kvm_extirq *s = kvm->arch.v_extirq;
+
+	if (!s)
+		return;
+	kvm_io_bus_unregister_dev(s->kvm, KVM_MMIO_BUS, &s->device);
+	kfree(s);
+}
+/*
+ * kvm_create_ls3a_ext_irq()
+ * @kvm KVM structure pointer
+ * Create an extended interrupt resource instance for a virtual machine
+ * Returns: Extended interrupt structure pointer
+ */
+int kvm_create_ls3a_ext_irq(struct kvm *kvm)
+{
+	struct ls3a_kvm_extirq *s;
+	int  ret;
+
+	s = kzalloc(sizeof(struct ls3a_kvm_extirq), GFP_KERNEL);
+	if (!s)
+		return -ENOMEM;
+
+	memset((void *)&s->ls3a_ext_irq, 0x0, sizeof(struct kvm_ls3a_extirq_state));
+
+	spin_lock_init(&s->lock);
+	atomic64_set(&s->enabled, 0);
+	s->kvm = kvm;
+
+	/*
+	 * Initialize MMIO device
+	 */
+	kvm_iodevice_init(&s->device, &kvm_ls3a_ext_irq_ops);
+	mutex_lock(&kvm->slots_lock);
+	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
+					EXTIOI_REG_BASE, EXTIOI_ADDR_SIZE, &s->device);
+	mutex_unlock(&kvm->slots_lock);
+	if (ret < 0) {
+		printk("%s dev_ls3a_ext_irq register error  ret %d\n", __FUNCTION__, ret);
+		goto err_register;
+	}
+
+	kvm->arch.v_extirq = s;
+
+	return 0;
+
+err_register:
+	kfree(s);
+	return -EFAULT;
+}
+
+static int kvm_set_ext_sw_ipmap(struct kvm_ls3a_extirq_state *state)
+{
+	uint8_t val_data_u8;
+	int   i, j, base_irq, irqnum, ipnum;
+
+	ipnum = 0;
+	for (i = 0; i < EXTIOI_IRQS_IPMAP_SIZE; i++) {
+		val_data_u8 = state->ip_map.reg_u8[i];
+		for (j = 0; j < 4; j++) {
+			if (val_data_u8 & (0x1 << j)) {
+				ipnum = j;
+				break;
+			}
+		}
+		kvm_debug("%s:%d ipnum:%d i:%d val_data_u8:0x%x\n", __FUNCTION__, __LINE__,
+									ipnum, i, val_data_u8);
+
+		if (val_data_u8) {
+			for (base_irq = 0; base_irq < EXTIOI_IRQS_PER_GROUP; base_irq++) {
+				irqnum = i * EXTIOI_IRQS_PER_GROUP + base_irq;
+				state->ext_sw_ipmap[irqnum] = ipnum;
+			}
+		} else {
+			for (base_irq = 0; base_irq < EXTIOI_IRQS_PER_GROUP; base_irq++) {
+				irqnum = i * EXTIOI_IRQS_PER_GROUP + base_irq;
+				state->ext_sw_ipmap[irqnum] = 0;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int kvm_set_ext_sw_coremap(struct kvm *kvm, struct kvm_ls3a_extirq_state *state)
+{
+	int reg_count;
+
+	for (reg_count = 0; reg_count < EXTIOI_IRQS; reg_count++) {
+		state->ext_sw_coremap[reg_count] = state->core_map.reg_u8[reg_count];
+
+		kvm_debug("%s:%d  -- reg_count:%d vcpu %d\n",
+				__FUNCTION__, __LINE__, reg_count, state->core_map.reg_u8[reg_count]);
+	}
+
+	return 0;
+}
+
+static int kvm_set_ext_sw_ipisr(struct kvm *kvm, struct kvm_ls3a_extirq_state *state)
+{
+	int ipnum, core, irq_num;
+
+	for (irq_num = 0; irq_num < EXTIOI_IRQS; irq_num++) {
+		core = state->ext_sw_coremap[irq_num];
+		ipnum = state->ext_sw_ipmap[irq_num];
+
+		if (test_bit(irq_num, (void *)state->ext_core_isr.reg_u8[core]) == false) {
+			bitmap_clear((void *)state->ext_sw_ipisr[core][ipnum + 2], irq_num, 1);
+		} else {
+			bitmap_set((void *)state->ext_sw_ipisr[core][ipnum + 2], irq_num, 1);
+		}
+
+	}
+	return 0;
+}
+
+int kvm_get_ls3a_extirq(struct kvm *kvm, struct kvm_loongarch_ls3a_extirq_state *state)
+{
+	struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm);
+	struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq);
+	unsigned long flags;
+	if (!v_extirq)
+		return -EINVAL;
+
+	ls3a_ext_irq_lock(v_extirq, flags);
+	memcpy(state, extirq_state,
+		sizeof(struct kvm_loongarch_ls3a_extirq_state));
+	ls3a_ext_irq_unlock(v_extirq, flags);
+	kvm->stat.get_ls3a_ext_irq++;
+
+	return 0;
+}
+
+int kvm_set_ls3a_extirq(struct kvm *kvm, struct kvm_loongarch_ls3a_extirq_state *state)
+{
+	struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm);
+	struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq);
+	unsigned long flags;
+	if (!v_extirq)
+		return -EINVAL;
+
+	ls3a_ext_irq_lock(v_extirq, flags);
+	memcpy(extirq_state, state,
+		sizeof(struct kvm_loongarch_ls3a_extirq_state));
+	kvm_set_ext_sw_ipmap(extirq_state);
+	kvm_set_ext_sw_coremap(kvm, extirq_state);
+	kvm_set_ext_sw_ipisr(kvm, extirq_state);
+
+	ls3a_ext_irq_unlock(v_extirq, flags);
+	kvm->stat.set_ls3a_ext_irq++;
+
+	return 0;
+}
+
+int kvm_setup_ls3a_extirq(struct kvm *kvm)
+{
+	struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm);
+	struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq);
+	unsigned long flags;
+
+	if (!v_extirq)
+		return -EINVAL;
+
+	ls3a_ext_irq_lock(v_extirq, flags);
+	memset(extirq_state, 0,	sizeof(struct kvm_ls3a_extirq_state));
+	ls3a_ext_irq_unlock(v_extirq, flags);
+
+	atomic64_set(&v_extirq->enabled, 1);
+
+	return 0;
+}
+
+int kvm_enable_ls3a_extirq(struct kvm *kvm, bool enable)
+{
+	struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm);
+
+	if (v_extirq)
+		atomic64_set(&v_extirq->enabled, enable);
+
+	return 0;
+}
+
+void kvm_dump_ls3a_extirq_state(struct seq_file *s,
+		struct ls3a_kvm_extirq *irqchip)
+{
+	struct kvm_ls3a_extirq_state *extirq;
+	int i = 0, j = 0;
+	unsigned long flags;
+
+	seq_puts(s, "LS3A ext irqchip state:\n");
+
+	if (!irqchip)
+		return;
+
+	extirq = &(irqchip->ls3a_ext_irq);
+	ls3a_ext_irq_lock(irqchip, flags);
+	i = (int)atomic64_read(&irqchip->enabled);
+	seq_printf(s, "ext irq enabled:%d", i);
+	seq_puts(s, "\nenabled:(Not Enabled)");
+	for (i = 0; i < EXTIOI_IRQS; i++) {
+		if (!test_bit(i, (void *)&extirq->ext_en))
+			seq_printf(s, "%d ", i);
+	}
+	seq_puts(s, "\nbounce:(Not bounce)");
+	for (i = 0; i < EXTIOI_IRQS; i++) {
+		if (!test_bit(i, (void *)&extirq->bounce))
+			seq_printf(s, "%d ", i);
+	}
+	seq_puts(s, "\next_isr:");
+	for (i = 0; i < EXTIOI_IRQS; i++) {
+		if (test_bit(i, (void *)&extirq->ext_isr))
+			seq_printf(s, "%d ", i);
+	}
+
+	seq_puts(s, "\ncore_isr:");
+	for (i = 0; i < KVM_MAX_VCPUS && kvm_get_vcpu_by_id(irqchip->kvm, i); i++) {
+		seq_printf(s, "\n\t CPU%d:", i);
+		for (j = 0; j < EXTIOI_IRQS; j++) {
+			if (test_bit(j, (void *)&extirq->ext_core_isr.reg_u8[i]))
+				seq_printf(s, "%d ", j);
+		}
+	}
+	seq_printf(s, "\nip_map:%llx", extirq->ip_map.reg_u64);
+	seq_puts(s, "\ncore_map: (only display router to slave cpu)\n");
+	for (i = 0; i < EXTIOI_IRQS_COREMAP_SIZE; i++)
+		if (extirq->core_map.reg_u8[i])
+			seq_printf(s, "\tirq:%d -> cpu:%d\n", i,
+					extirq->core_map.reg_u8[i]);
+	ls3a_ext_irq_unlock(irqchip, flags);
+}

arch/loongarch/kvm/intc/ls3a_ext_irq.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef __LS3A_KVM_EXT_IRQ_H
+#define __LS3A_KVM_EXT_IRQ_H
+
+#include <linux/mm_types.h>
+#include <linux/hrtimer.h>
+#include <linux/kvm_host.h>
+#include <linux/spinlock.h>
+#include <linux/seq_file.h>
+
+#include <kvm/iodev.h>
+
+#define IOCSR_EXTIOI_ADDR		KVM_IOCSR_EXTIOI_NODEMAP_BASE
+
+#define EXTIOI_ADDR_OFF			0x10000
+#define EXTIOI_REG_BASE			(LOONGSON_VIRT_REG_BASE + EXTIOI_ADDR_OFF)
+#define EXTIOI_REG_END			(EXTIOI_REG_BASE + 0x20000)
+#define EXTIOI_ADDR_SIZE		(EXTIOI_REG_END - EXTIOI_REG_BASE)
+#define EXTIOI_PERCORE_REG_OFF		0x10000
+#define EXTIOI_PERCORE_REG_END		(EXTIOI_PERCORE_REG_OFF + 0x10000)
+
+#define EXTIOI_ADDR(off)		(EXTIOI_REG_BASE + (off) - IOCSR_EXTIOI_ADDR)
+#define EXTIOI_PERCORE_ADDR(id, off) \
+	(EXTIOI_REG_BASE + EXTIOI_PERCORE_REG_OFF + ((id) << 8) + (off))
+
+#define EXTIOI_NODETYPE_START		(KVM_IOCSR_EXTIOI_NODEMAP_BASE - IOCSR_EXTIOI_ADDR)
+#define EXTIOI_NODETYPE_END		(EXTIOI_NODETYPE_START + 0x20)
+#define EXTIOI_IPMAP_START		(KVM_IOCSR_EXTIOI_IPMAP_BASE - IOCSR_EXTIOI_ADDR)
+#define EXTIOI_IPMAP_END		(EXTIOI_IPMAP_START + 0x8)
+#define EXTIOI_ENABLE_START		(KVM_IOCSR_EXTIOI_EN_BASE - IOCSR_EXTIOI_ADDR)
+#define EXTIOI_ENABLE_END		(EXTIOI_ENABLE_START + 0x20)
+#define EXTIOI_BOUNCE_START		(KVM_IOCSR_EXTIOI_BOUNCE_BASE - IOCSR_EXTIOI_ADDR)
+#define EXTIOI_BOUNCE_END		(EXTIOI_BOUNCE_START + 0x20)
+#define EXTIOI_ISR_START		(0x1700 - IOCSR_EXTIOI_ADDR)
+#define EXTIOI_ISR_END			(EXTIOI_ISR_START + 0x20)
+#define EXTIOI_COREMAP_START		(KVM_IOCSR_EXTIOI_ROUTE_BASE - IOCSR_EXTIOI_ADDR)
+#define EXTIOI_COREMAP_END		(EXTIOI_COREMAP_START + 0x100)
+#define EXTIOI_COREISR_START		(EXTIOI_PERCORE_REG_OFF)
+#define EXTIOI_COREISR_END		(EXTIOI_PERCORE_REG_END)
+
+#define LS3A_INTC_IP			8
+#define EXTIOI_IRQS			KVM_EXTIOI_IRQS
+#define EXTIOI_IRQS_BITMAP_SIZE		(EXTIOI_IRQS / 8)
+/* map to ipnum per 32 irqs */
+#define EXTIOI_IRQS_IPMAP_SIZE		(EXTIOI_IRQS / 32)
+#define EXTIOI_IRQS_PER_GROUP		KVM_EXTIOI_IRQS_PER_GROUP
+#define EXTIOI_IRQS_COREMAP_SIZE	(EXTIOI_IRQS)
+#define EXTIOI_IRQS_NODETYPE_SIZE	KVM_EXTIOI_IRQS_NODETYPE_SIZE
+
+typedef struct kvm_ls3a_extirq_state {
+	union ext_en {
+		uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE];
+	} ext_en;
+	union bounce {
+		uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE];
+	} bounce;
+	union ext_isr {
+		uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE];
+	} ext_isr;
+	union ext_core_isr {
+		uint64_t reg_u64[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE / 8];
+		uint32_t reg_u32[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE / 4];
+		uint8_t reg_u8[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE];
+	} ext_core_isr;
+	union ip_map {
+		uint64_t reg_u64;
+		uint32_t reg_u32[EXTIOI_IRQS_IPMAP_SIZE / 4];
+		uint8_t reg_u8[EXTIOI_IRQS_IPMAP_SIZE];
+	} ip_map;
+	union core_map {
+		uint64_t reg_u64[EXTIOI_IRQS_COREMAP_SIZE / 8];
+		uint32_t reg_u32[EXTIOI_IRQS_COREMAP_SIZE / 4];
+		uint8_t reg_u8[EXTIOI_IRQS_COREMAP_SIZE];
+	} core_map;
+	union {
+		uint64_t reg_u64[EXTIOI_IRQS_NODETYPE_SIZE / 4];
+		uint32_t reg_u32[EXTIOI_IRQS_NODETYPE_SIZE / 2];
+		uint16_t reg_u16[EXTIOI_IRQS_NODETYPE_SIZE];
+		uint8_t reg_u8[EXTIOI_IRQS_NODETYPE_SIZE * 2];
+	} node_type;
+
+	/*software state */
+	uint8_t ext_sw_ipmap[EXTIOI_IRQS];
+	uint8_t ext_sw_coremap[EXTIOI_IRQS];
+	uint8_t ext_sw_ipisr[KVM_MAX_VCPUS][LS3A_INTC_IP][EXTIOI_IRQS_BITMAP_SIZE];
+} LS3AExtirqState;
+
+struct ls3a_kvm_extirq {
+	spinlock_t lock;
+	struct kvm *kvm;
+	atomic64_t enabled;
+	struct kvm_io_device device;
+	struct kvm_ls3a_extirq_state ls3a_ext_irq;
+};
+
+static inline struct ls3a_kvm_extirq *ls3a_ext_irqchip(struct kvm *kvm)
+{
+	return kvm->arch.v_extirq;
+}
+
+static inline int ls3a_extirq_in_kernel(struct kvm *kvm)
+{
+	int ret;
+
+	ret = (ls3a_ext_irqchip(kvm) != NULL);
+	return ret;
+}
+
+
+void ext_irq_handler(struct kvm *kvm, int irq, int level);
+int kvm_create_ls3a_ext_irq(struct kvm *kvm);
+int kvm_get_ls3a_extirq(struct kvm *kvm,
+			struct kvm_loongarch_ls3a_extirq_state *state);
+int kvm_set_ls3a_extirq(struct kvm *kvm,
+			struct kvm_loongarch_ls3a_extirq_state *state);
+void kvm_destroy_ls3a_ext_irq(struct kvm *kvm);
+void msi_irq_handler(struct kvm *kvm, int irq, int level);
+int kvm_setup_ls3a_extirq(struct kvm *kvm);
+int kvm_enable_ls3a_extirq(struct kvm *kvm, bool enable);
+void kvm_dump_ls3a_extirq_state(struct seq_file *m, struct ls3a_kvm_extirq *irqchip);
+#endif

arch/loongarch/kvm/intc/ls3a_ipi.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include "kvmcpu.h"
+#include "ls3a_ipi.h"
+#include "ls7a_irq.h"
+#include "ls3a_ext_irq.h"
+
+#define ls3a_gipi_lock(s, flags)	spin_lock_irqsave(&s->lock, flags)
+#define ls3a_gipi_unlock(s, flags)	spin_unlock_irqrestore(&s->lock, flags)
+
+extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
+			     struct kvm_loongarch_interrupt *irq);
+int kvm_helper_send_ipi(struct kvm_vcpu *vcpu, unsigned int cpu, unsigned int action)
+{
+	struct kvm *kvm = vcpu->kvm;
+	struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm);
+	gipiState *s = &(ipi->ls3a_gipistate);
+	unsigned long flags;
+	struct kvm_loongarch_interrupt irq;
+
+	kvm->stat.pip_write_exits++;
+
+	ls3a_gipi_lock(ipi, flags);
+	if (s->core[cpu].status == 0) {
+		irq.cpu = cpu;
+		irq.irq = LARCH_INT_IPI;
+		kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq);
+	}
+
+	s->core[cpu].status |= action;
+	ls3a_gipi_unlock(ipi, flags);
+
+	return 0;
+}
+
+static int ls3a_gipi_writel(struct ls3a_kvm_ipi *ipi, gpa_t addr,
+						int len, const void *val)
+{
+	uint64_t data, offset;
+	struct kvm_loongarch_interrupt irq;
+	gipiState *s = &(ipi->ls3a_gipistate);
+	uint32_t cpu, action_data;
+	struct kvm *kvm;
+	void *pbuf;
+	int mailbox, action;
+
+	kvm = ipi->kvm;
+	cpu = (addr >> 8) & 0xff;
+
+	data = *(uint64_t *)val;
+	offset = addr & 0xFF;
+
+	BUG_ON(offset & (len - 1));
+
+	switch (offset) {
+	case CORE0_STATUS_OFF:
+		printk("CORE0_SET_OFF Can't be write\n");
+
+		break;
+	case CORE0_EN_OFF:
+		s->core[cpu].en = data;
+
+		break;
+	case CORE0_IPI_SEND:
+		cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
+		action = (data & 0x1f);
+		action_data = (1 << action);
+
+		if (s->core[cpu].status == 0) {
+			irq.cpu = cpu;
+			irq.irq = LARCH_INT_IPI;
+
+			if (likely(kvm->vcpus[cpu])) {
+				kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq);
+			}
+		}
+		s->core[cpu].status |= action_data;
+		break;
+	case CORE0_SET_OFF:
+		pr_info("CORE0_SET_OFF simulation is required\n");
+		break;
+	case CORE0_CLEAR_OFF:
+		s->core[cpu].status &= ~data;
+		if (!s->core[cpu].status) {
+			irq.cpu = cpu;
+			irq.irq = -LARCH_INT_IPI;
+			if (likely(kvm->vcpus[cpu]))
+				kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq);
+			else
+				kvm_err("Failed lower ipi irq target cpu:%d\n", cpu);
+		}
+
+		break;
+	case CORE0_MAIL_SEND:
+		cpu = ((data & 0xffffffff) >> 16) & 0x3ff;
+		mailbox = ((data & 0xffffffff) >> 2) & 0x7;
+		pbuf =  (void *)s->core[cpu].buf + mailbox * 4;
+
+		*(unsigned int *)pbuf = (unsigned int)(data >> 32);
+		break;
+	case 0x20 ... 0x3c:
+		pbuf =  (void *)s->core[cpu].buf + (offset - 0x20);
+		if (len == 1)
+			*(unsigned char *)pbuf = (unsigned char)data;
+		else if (len == 2)
+			*(unsigned short *)pbuf = (unsigned short)data;
+		else if (len == 4)
+			*(unsigned int *)pbuf = (unsigned int)data;
+		else if (len == 8)
+			*(unsigned long *)pbuf = (unsigned long)data;
+
+		break;
+	default:
+		printk("ls3a_gipi_writel with unknown addr %llx \n", addr);
+		break;
+	}
+	return 0;
+}
+
+static uint64_t ls3a_gipi_readl(struct ls3a_kvm_ipi *ipi,
+				gpa_t addr, int len, void *val)
+{
+	uint64_t offset;
+	uint64_t ret = 0;
+
+	gipiState *s = &(ipi->ls3a_gipistate);
+	uint32_t cpu;
+	void *pbuf;
+
+	cpu = (addr >> 8) & 0xff;
+
+	offset = addr & 0xFF;
+
+	BUG_ON(offset & (len - 1));
+	switch (offset) {
+	case CORE0_STATUS_OFF:
+		ret = s->core[cpu].status;
+		break;
+	case CORE0_EN_OFF:
+		ret = s->core[cpu].en;
+		break;
+	case CORE0_SET_OFF:
+		ret = 0;
+		break;
+	case CORE0_CLEAR_OFF:
+		ret = 0;
+		break;
+	case 0x20 ... 0x3c:
+		pbuf =  (void *)s->core[cpu].buf + (offset - 0x20);
+		if (len == 1)
+			ret  = *(unsigned char *)pbuf;
+		else if (len == 2)
+			ret = *(unsigned short *)pbuf;
+		else if (len == 4)
+			ret = *(unsigned int *)pbuf;
+		else if (len == 8)
+			ret = *(unsigned long *)pbuf;
+		break;
+	default:
+		printk("ls3a_gipi_readl with unknown addr %llx \n", addr);
+		break;
+	}
+
+	*(uint64_t *)val = ret;
+
+	return ret;
+}
+
+static int kvm_ls3a_ipi_write(struct kvm_vcpu *vcpu,
+			      struct kvm_io_device *dev,
+			      gpa_t addr, int len, const void *val)
+{
+	struct ls3a_kvm_ipi *ipi;
+	ipi_io_device *ipi_device;
+	unsigned long flags;
+
+	ipi_device = container_of(dev, ipi_io_device, device);
+	ipi = ipi_device->ipi;
+	ipi->kvm->stat.pip_write_exits++;
+
+	ls3a_gipi_lock(ipi, flags);
+	ls3a_gipi_writel(ipi, addr, len, val);
+	ls3a_gipi_unlock(ipi, flags);
+	return 0;
+}
+
+
+static int kvm_ls3a_ipi_read(struct kvm_vcpu *vcpu,
+			     struct kvm_io_device *dev,
+			     gpa_t addr, int len, void *val)
+{
+	struct ls3a_kvm_ipi *ipi;
+	ipi_io_device *ipi_device;
+	unsigned long flags;
+
+	ipi_device = container_of(dev, ipi_io_device, device);
+	ipi = ipi_device->ipi;
+	ipi->kvm->stat.pip_read_exits++;
+
+	ls3a_gipi_lock(ipi, flags);
+	ls3a_gipi_readl(ipi, addr, len, val);
+	ls3a_gipi_unlock(ipi, flags);
+	return 0;
+}
+
+
+static const struct kvm_io_device_ops kvm_ls3a_ipi_ops = {
+	.read	= kvm_ls3a_ipi_read,
+	.write	= kvm_ls3a_ipi_write,
+};
+
+void kvm_destroy_ls3a_ipi(struct kvm *kvm)
+{
+	struct kvm_io_device *device;
+	struct ls3a_kvm_ipi *vipi = kvm->arch.v_gipi;
+
+	if (!vipi)
+		return;
+	device = &vipi->dev_ls3a_ipi.device;
+	kvm_io_bus_unregister_dev(vipi->kvm, KVM_MMIO_BUS, device);
+	kfree(vipi);
+}
+
+int kvm_create_ls3a_ipi(struct kvm *kvm)
+{
+	struct ls3a_kvm_ipi *s;
+	unsigned long addr;
+	struct kvm_io_device *device;
+	int ret;
+
+	s = kzalloc(sizeof(struct ls3a_kvm_ipi), GFP_KERNEL);
+	if (!s)
+		return -ENOMEM;
+	spin_lock_init(&s->lock);
+	s->kvm = kvm;
+
+	/*
+	 * Initialize MMIO device
+	 */
+	device = &s->dev_ls3a_ipi.device;
+	kvm_iodevice_init(device, &kvm_ls3a_ipi_ops);
+	addr = SMP_MAILBOX;
+	mutex_lock(&kvm->slots_lock);
+	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
+			addr, KVM_IOCSR_IPI_ADDR_SIZE, device);
+	mutex_unlock(&kvm->slots_lock);
+	if (ret < 0) {
+		kvm_err("%s Initialize MMIO dev err ret:%d\n", __func__, ret);
+		goto err;
+	} else {
+		s->dev_ls3a_ipi.ipi = s;
+	}
+
+	kvm->arch.v_gipi = s;
+	return 0;
+
+err:
+	kfree(s);
+	return -EFAULT;
+}
+
+int kvm_get_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state)
+{
+	struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm);
+	gipiState *ipi_state =  &(ipi->ls3a_gipistate);
+	unsigned long flags;
+
+	ls3a_gipi_lock(ipi, flags);
+	memcpy(state, ipi_state, sizeof(gipiState));
+	ls3a_gipi_unlock(ipi, flags);
+	return 0;
+}
+
+int kvm_set_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state)
+{
+	struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm);
+	gipiState *ipi_state =  &(ipi->ls3a_gipistate);
+	unsigned long flags;
+
+	if (!ipi)
+		return -EINVAL;
+
+	ls3a_gipi_lock(ipi, flags);
+	memcpy(ipi_state, state, sizeof(gipiState));
+	ls3a_gipi_unlock(ipi, flags);
+	return 0;
+}

arch/loongarch/kvm/intc/ls3a_ipi.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef __LS3A_KVM_IPI_H
+#define __LS3A_KVM_IPI_H
+
+#include <linux/mm_types.h>
+#include <linux/hrtimer.h>
+#include <linux/kvm_host.h>
+#include <linux/spinlock.h>
+#include <kvm/iodev.h>
+
+typedef struct gipi_single {
+	uint32_t status;
+	uint32_t en;
+	uint32_t set;
+	uint32_t clear;
+	uint64_t buf[4];
+} gipi_single;
+
+typedef struct gipiState {
+	gipi_single core[KVM_MAX_VCPUS];
+} gipiState;
+
+struct ls3a_kvm_ipi;
+
+typedef struct ipi_io_device {
+	struct ls3a_kvm_ipi *ipi;
+	struct kvm_io_device device;
+	int nodeNum;
+} ipi_io_device;
+
+struct ls3a_kvm_ipi {
+	spinlock_t lock;
+	struct kvm *kvm;
+	gipiState ls3a_gipistate;
+	int nodeNum;
+	ipi_io_device dev_ls3a_ipi;
+};
+
+#define SMP_MAILBOX			(LOONGSON_VIRT_REG_BASE + 0x0000)
+#define KVM_IPI_REG_ADDRESS(id, off)	(SMP_MAILBOX | (id << 8) | off)
+#define KVM_IOCSR_IPI_ADDR_SIZE		0x10000
+
+#define CORE0_STATUS_OFF	0x000
+#define CORE0_EN_OFF		0x004
+#define CORE0_SET_OFF		0x008
+#define CORE0_CLEAR_OFF		0x00c
+#define CORE0_BUF_20		0x020
+#define CORE0_BUF_28		0x028
+#define CORE0_BUF_30		0x030
+#define CORE0_BUF_38		0x038
+#define CORE0_IPI_SEND		0x040
+#define CORE0_MAIL_SEND		0x048
+
+static inline struct ls3a_kvm_ipi *ls3a_ipi_irqchip(struct kvm *kvm)
+{
+	return kvm->arch.v_gipi;
+}
+
+static inline int ls3a_ipi_in_kernel(struct kvm *kvm)
+{
+	int ret;
+
+	ret = (ls3a_ipi_irqchip(kvm) != NULL);
+	return ret;
+}
+
+int kvm_create_ls3a_ipi(struct kvm *kvm);
+void kvm_destroy_ls3a_ipi(struct kvm *kvm);
+int kvm_set_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state);
+int kvm_get_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state);
+int kvm_helper_send_ipi(struct kvm_vcpu *vcpu, unsigned int cpu, unsigned int action);
+#endif

arch/loongarch/kvm/intc/ls7a_irq.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/highmem.h>
+#include <linux/mm.h>
+#include "ls3a_ipi.h"
+#include "ls7a_irq.h"
+#include "ls3a_ext_irq.h"
+
+void ls7a_ioapic_lock(struct ls7a_kvm_ioapic *s, unsigned long *flags)
+{
+	unsigned long tmp;
+	spin_lock_irqsave(&s->lock, tmp);
+	*flags = tmp;
+}
+
+void ls7a_ioapic_unlock(struct ls7a_kvm_ioapic *s, unsigned long *flags)
+{
+	unsigned long tmp;
+	tmp = *flags;
+	spin_unlock_irqrestore(&s->lock, tmp);
+}
+
+static void kvm_ls7a_ioapic_raise(struct kvm *kvm, unsigned long mask)
+{
+	unsigned long irqnum, val;
+	struct ls7a_kvm_ioapic *s = ls7a_ioapic_irqchip(kvm);
+	struct kvm_ls7a_ioapic_state *state;
+	struct kvm_loongarch_interrupt irq;
+	int i;
+
+	state = &s->ls7a_ioapic;
+	irq.cpu = -1;
+	val = mask & state->intirr & (~state->int_mask);
+	val &= ~state->intisr;
+	for_each_set_bit(i, &val, 64) {
+		state->intisr |= 0x1ULL << i;
+		irqnum = state->htmsi_vector[i];
+		kvm_debug("msi_irq_handler,%ld,up\n", irqnum);
+		msi_irq_handler(kvm, irqnum, 1);
+	}
+
+	kvm->stat.ls7a_ioapic_update++;
+}
+
+static void kvm_ls7a_ioapic_lower(struct kvm *kvm, unsigned long mask)
+{
+	unsigned long irqnum, val;
+	struct ls7a_kvm_ioapic *s = ls7a_ioapic_irqchip(kvm);
+	struct kvm_ls7a_ioapic_state *state;
+	struct kvm_loongarch_interrupt irq;
+	int i;
+
+	state = &s->ls7a_ioapic;
+	irq.cpu = -1;
+	val = mask & state->intisr;
+	for_each_set_bit(i, &val, 64) {
+		state->intisr &= ~(0x1ULL << i);
+		irqnum = state->htmsi_vector[i];
+		kvm_debug("msi_irq_handler,%ld,down\n", irqnum);
+		msi_irq_handler(kvm, irqnum, 0);
+	}
+
+	kvm->stat.ls7a_ioapic_update++;
+}
+
+int kvm_ls7a_set_msi(struct kvm_kernel_irq_routing_entry *e,
+		struct kvm *kvm, int irq_source_id, int level, bool line_status)
+{
+	if (!level)
+		return -1;
+
+	kvm_debug("msi data is 0x%x\n", e->msi.data);
+	msi_irq_handler(kvm, e->msi.data, 1);
+	return 0;
+}
+
+int kvm_ls7a_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi)
+{
+	struct kvm_kernel_irq_routing_entry route;
+
+	if (msi->flags != 0)
+		return -EINVAL;
+
+	kvm->stat.ls7a_msi_irq++;
+	route.msi.address_lo = msi->address_lo;
+	route.msi.address_hi = msi->address_hi;
+	route.msi.data = msi->data;
+
+	kvm_debug("msi data is 0x%x\n", route.msi.data);
+	return kvm_ls7a_set_msi(&route, kvm,
+				KVM_USERSPACE_IRQ_SOURCE_ID, 1, false);
+
+}
+
+int kvm_ls7a_ioapic_set_irq(struct kvm *kvm, int irq, int level)
+{
+	struct ls7a_kvm_ioapic *s;
+	struct kvm_ls7a_ioapic_state *state;
+	uint64_t mask = 1ULL << irq;
+	s = ls7a_ioapic_irqchip(kvm);
+	state = &s->ls7a_ioapic;
+	BUG_ON(irq < 0 || irq >= LS7A_IOAPIC_NUM_PINS);
+
+	if (state->intedge & mask) {
+		/* edge triggered */
+		if (level) {
+			if ((state->last_intirr & mask) == 0) {
+				state->intirr |= mask;
+				kvm_ls7a_ioapic_raise(kvm, mask);
+			}
+			state->last_intirr |= mask;
+		} else
+			state->last_intirr &= ~mask;
+	} else {
+		/* level triggered */
+		if (!!level) {
+			if ((state->intirr & mask) == 0) {
+				state->intirr |= mask;
+				kvm_ls7a_ioapic_raise(kvm, mask);
+			}
+		} else {
+			if (state->intirr & mask) {
+				state->intirr &= ~mask;
+				kvm_ls7a_ioapic_lower(kvm, mask);
+			}
+		}
+	}
+	kvm->stat.ls7a_ioapic_set_irq++;
+	return 0;
+}
+
+static int ls7a_ioapic_reg_write(struct ls7a_kvm_ioapic *s,
+			  gpa_t addr, int len, const void *val)
+{
+	struct kvm *kvm;
+	struct kvm_ls7a_ioapic_state *state;
+	int64_t offset_tmp;
+	uint64_t offset;
+	uint64_t data, old;
+
+	offset = addr & 0xfff;
+	kvm = s->kvm;
+	state = &(s->ls7a_ioapic);
+
+	if (offset & (len - 1)) {
+		printk("%s(%d):unaligned address access %llx size %d \n",
+			__FUNCTION__, __LINE__, addr, len);
+		return 0;
+	}
+
+	if (8 == len) {
+		data = *(uint64_t *)val;
+		switch (offset) {
+		case LS7A_INT_MASK_OFFSET:
+			old = state->int_mask;
+			state->int_mask = data;
+			if (old & ~data)
+				kvm_ls7a_ioapic_raise(kvm, old & ~data);
+			if (~old & data)
+				kvm_ls7a_ioapic_lower(kvm, ~old & data);
+			break;
+		case LS7A_INT_STATUS_OFFSET:
+			state->intisr = data;
+			break;
+		case LS7A_INT_EDGE_OFFSET:
+			state->intedge = data;
+			break;
+		case LS7A_INT_CLEAR_OFFSET:
+			/*
+			 * only clear edge triggered irq on writing INTCLR reg
+			 * no effect on level triggered irq
+			 */
+			data = data & state->intedge;
+			state->intirr &= ~data;
+			kvm_ls7a_ioapic_lower(kvm, data);
+			state->intisr &= (~data);
+			break;
+		case LS7A_INT_POL_OFFSET:
+			state->int_polarity = data;
+			break;
+		case LS7A_HTMSI_EN_OFFSET:
+			state->htmsi_en = data;
+			break;
+		case LS7A_AUTO_CTRL0_OFFSET:
+		case LS7A_AUTO_CTRL1_OFFSET:
+			break;
+		default:
+			WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len);
+			break;
+		}
+	} else if (1 == len) {
+		data = *(unsigned char *)val;
+		if (offset >= LS7A_HTMSI_VEC_OFFSET) {
+			offset_tmp = offset - LS7A_HTMSI_VEC_OFFSET;
+			if (offset_tmp >= 0 && offset_tmp < 64) {
+				state->htmsi_vector[offset_tmp] =
+					(uint8_t)(data & 0xff);
+			}
+		} else if (offset >=  LS7A_ROUTE_ENTRY_OFFSET) {
+			offset_tmp = offset - LS7A_ROUTE_ENTRY_OFFSET;
+			if (offset_tmp >= 0 && offset_tmp < 64) {
+				state->route_entry[offset_tmp] =
+					(uint8_t)(data & 0xff);
+			}
+		} else {
+			WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len);
+		}
+	} else {
+		WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len);
+	}
+	kvm->stat.ioapic_reg_write++;
+	return 0;
+}
+
+static inline struct ls7a_kvm_ioapic *to_ioapic(struct kvm_io_device *dev)
+{
+	return container_of(dev, struct ls7a_kvm_ioapic, dev_ls7a_ioapic);
+}
+
+static int kvm_ls7a_ioapic_write(struct kvm_vcpu *vcpu,
+				 struct kvm_io_device *this,
+				 gpa_t addr, int len, const void *val)
+{
+	struct ls7a_kvm_ioapic *s = to_ioapic(this);
+	unsigned long flags;
+
+	ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags);
+	ls7a_ioapic_reg_write(s, addr, len, val);
+	ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags);
+
+	return 0;
+}
+
+static int ls7a_ioapic_reg_read(struct ls7a_kvm_ioapic *s,
+		       gpa_t addr, int len, void *val)
+{
+	uint64_t offset, offset_tmp;
+	struct kvm *kvm;
+	struct kvm_ls7a_ioapic_state *state;
+	uint64_t result = 0;
+
+	state = &(s->ls7a_ioapic);
+	kvm = s->kvm;
+	offset = addr & 0xfff;
+	if (offset & (len - 1)) {
+		printk("%s(%d):unaligned address access %llx size %d \n",
+			__FUNCTION__, __LINE__, addr, len);
+		return 0;
+	}
+
+	if (8 == len) {
+		switch (offset) {
+		case LS7A_INT_MASK_OFFSET:
+			result = state->int_mask;
+			break;
+		case LS7A_INT_STATUS_OFFSET:
+			result = state->intisr & (~state->int_mask);
+			break;
+		case LS7A_INT_EDGE_OFFSET:
+			result = state->intedge;
+			break;
+		case LS7A_INT_POL_OFFSET:
+			result = state->int_polarity;
+			break;
+		case LS7A_HTMSI_EN_OFFSET:
+			result = state->htmsi_en;
+			break;
+		case LS7A_AUTO_CTRL0_OFFSET:
+		case LS7A_AUTO_CTRL1_OFFSET:
+			break;
+		case LS7A_INT_ID_OFFSET:
+			result = LS7A_INT_ID_VER;
+			result = (result << 32) + LS7A_INT_ID_VAL;
+			break;
+		default:
+			WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len);
+			break;
+		}
+		if (val != NULL)
+			*(uint64_t *)val = result;
+	} else if (1 == len) {
+		if (offset >= LS7A_HTMSI_VEC_OFFSET) {
+			offset_tmp = offset - LS7A_HTMSI_VEC_OFFSET;
+			if (offset_tmp >= 0 && offset_tmp < 64) {
+				result = state->htmsi_vector[offset_tmp];
+			}
+		} else if (offset >=  LS7A_ROUTE_ENTRY_OFFSET) {
+			offset_tmp = offset - LS7A_ROUTE_ENTRY_OFFSET;
+			if (offset_tmp >= 0 && offset_tmp < 64) {
+				result = state->route_entry[offset_tmp];
+			}
+		} else {
+			WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len);
+		}
+		if (val != NULL)
+			*(unsigned char *)val = result;
+	} else {
+		WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len);
+	}
+	kvm->stat.ioapic_reg_read++;
+	return result;
+}
+
+static int kvm_ls7a_ioapic_read(struct kvm_vcpu *vcpu,
+				struct kvm_io_device *this,
+				gpa_t addr, int len, void *val)
+{
+	struct ls7a_kvm_ioapic *s = to_ioapic(this);
+	unsigned long flags;
+	uint64_t result = 0;
+
+	ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags);
+	result = ls7a_ioapic_reg_read(s, addr, len, val);
+	ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags);
+	return 0;
+}
+
+static const struct kvm_io_device_ops kvm_ls7a_ioapic_ops = {
+	.read     = kvm_ls7a_ioapic_read,
+	.write    = kvm_ls7a_ioapic_write,
+};
+
+static int kvm_ls7a_ioapic_alias_read(struct kvm_vcpu *vcpu,
+		struct kvm_io_device *this, gpa_t addr, int len, void *val)
+{
+	struct ls7a_kvm_ioapic *s;
+	unsigned long flags;
+
+	s = container_of(this, struct ls7a_kvm_ioapic, ls7a_ioapic_alias);
+	ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags);
+	ls7a_ioapic_reg_read(s, addr, len, val);
+	ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags);
+	return 0;
+}
+
+static int kvm_ls7a_ioapic_alias_write(struct kvm_vcpu *vcpu,
+		struct kvm_io_device *this, gpa_t addr, int len, const void *val)
+{
+	struct ls7a_kvm_ioapic *s;
+	unsigned long flags;
+
+	s = container_of(this, struct ls7a_kvm_ioapic, ls7a_ioapic_alias);
+	ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags);
+	ls7a_ioapic_reg_write(s, addr, len, val);
+	ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags);
+
+	return 0;
+}
+
+static const struct kvm_io_device_ops kvm_ls7a_ioapic_ops_alias = {
+	.read     = kvm_ls7a_ioapic_alias_read,
+	.write    = kvm_ls7a_ioapic_alias_write,
+};
+
+int kvm_create_ls7a_ioapic(struct kvm *kvm)
+{
+	struct ls7a_kvm_ioapic *s;
+	int ret;
+	unsigned long ls7a_ioapic_reg_base;
+
+	s = kzalloc(sizeof(struct ls7a_kvm_ioapic), GFP_KERNEL);
+	if (!s)
+		return -ENOMEM;
+	spin_lock_init(&s->lock);
+	s->kvm = kvm;
+
+	ls7a_ioapic_reg_base = LS7A_IOAPIC_GUEST_REG_BASE;
+
+	/*
+	 * Initialize MMIO device
+	 */
+	kvm_iodevice_init(&s->dev_ls7a_ioapic, &kvm_ls7a_ioapic_ops);
+	mutex_lock(&kvm->slots_lock);
+	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ls7a_ioapic_reg_base,
+						   0x1000, &s->dev_ls7a_ioapic);
+	if (ret < 0) {
+		kvm_err("Failed register ioapic, err:%d\n", ret);
+		goto fail_unlock;
+	}
+
+	ls7a_ioapic_reg_base = LS7A_IOAPIC_GUEST_REG_BASE_ALIAS;
+	kvm_iodevice_init(&s->ls7a_ioapic_alias, &kvm_ls7a_ioapic_ops_alias);
+	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ls7a_ioapic_reg_base,
+						0x1000, &s->ls7a_ioapic_alias);
+	if (ret < 0) {
+		kvm_err("Failed register alias ioapic, err:%d\n", ret);
+		kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
+				&s->dev_ls7a_ioapic);
+		goto fail_unlock;
+	}
+	mutex_unlock(&kvm->slots_lock);
+
+	kvm->arch.v_ioapic = s;
+
+	return 0;
+
+fail_unlock:
+	mutex_unlock(&kvm->slots_lock);
+	kfree(s);
+
+	return -EFAULT;
+}
+
+
+int kvm_get_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state)
+{
+	struct ls7a_kvm_ioapic *ls7a_ioapic = ls7a_ioapic_irqchip(kvm);
+	struct kvm_ls7a_ioapic_state *ioapic_state =
+					    &(ls7a_ioapic->ls7a_ioapic);
+	unsigned long flags;
+
+	ls7a_ioapic_lock(ls7a_ioapic, &flags);
+	memcpy(state, ioapic_state, sizeof(struct kvm_ls7a_ioapic_state));
+	ls7a_ioapic_unlock(ls7a_ioapic, &flags);
+	kvm->stat.get_ls7a_ioapic++;
+	return 0;
+}
+
+int kvm_set_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state)
+{
+	struct ls7a_kvm_ioapic *ls7a_ioapic = ls7a_ioapic_irqchip(kvm);
+	struct kvm_ls7a_ioapic_state *ioapic_state =
+					    &(ls7a_ioapic->ls7a_ioapic);
+	unsigned long flags;
+
+	if (!ls7a_ioapic)
+		return -EINVAL;
+
+	ls7a_ioapic_lock(ls7a_ioapic, &flags);
+	memcpy(ioapic_state, state, sizeof(struct kvm_ls7a_ioapic_state));
+	ls7a_ioapic_unlock(ls7a_ioapic, &flags);
+	kvm->stat.set_ls7a_ioapic++;
+	return 0;
+}
+
+void kvm_destroy_ls7a_ioapic(struct kvm *kvm)
+{
+	struct ls7a_kvm_ioapic *vpic = kvm->arch.v_ioapic;
+	if (!vpic)
+		return;
+	kvm_io_bus_unregister_dev(vpic->kvm, KVM_MMIO_BUS,
+					&vpic->ls7a_ioapic_alias);
+	kvm_io_bus_unregister_dev(vpic->kvm, KVM_MMIO_BUS,
+					&vpic->dev_ls7a_ioapic);
+	kfree(vpic);
+}
+
+void kvm_dump_ls7a_ioapic_state(struct seq_file *m,
+		struct ls7a_kvm_ioapic *ioapic)
+{
+	struct kvm_ls7a_ioapic_state *ioapic_state;
+	unsigned long flags;
+	int i = 0;
+
+	if (!ioapic)
+		return;
+
+	seq_puts(m, "\nIOAPIC state:\n");
+	ioapic_state = &(ioapic->ls7a_ioapic);
+
+	ls7a_ioapic_lock(ioapic, &flags);
+	seq_puts(m, "irq masked: ");
+	for (i = 0; i < 64; i++) {
+		if (!test_bit(i, (void *)&ioapic_state->int_mask))
+			seq_printf(m, "%d ", i);
+	}
+	seq_printf(m, "\nhtmsi_en:0x%016llx\n"
+			"intedge:0x%016llx",
+			ioapic_state->htmsi_en,
+			ioapic_state->intedge);
+
+	seq_puts(m, "\nroute_entry: ");
+	for (i = 0; i < 64; i++)
+		seq_printf(m, "%d ", ioapic_state->route_entry[i]);
+
+	seq_puts(m, "\nhtmsi_vector: ");
+	for (i = 0; i < 64; i++)
+		seq_printf(m, "%d ", ioapic_state->htmsi_vector[i]);
+
+	seq_printf(m, "\nintirr:%016llx\n"
+			"intisr:%016llx\n",
+			ioapic_state->intirr,
+			ioapic_state->intisr);
+	ls7a_ioapic_unlock(ioapic, &flags);
+}

arch/loongarch/kvm/intc/ls7a_irq.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef __LS7A_KVM_IRQ_H
+#define __LS7A_KVM_IRQ_H
+
+#include <linux/mm_types.h>
+#include <linux/hrtimer.h>
+#include <linux/kvm_host.h>
+#include <linux/spinlock.h>
+#include <linux/seq_file.h>
+#include "kvmcpu.h"
+
+#include <kvm/iodev.h>
+
+#define LS7A_APIC_NUM_PINS	64
+
+#define LS7A_ROUTE_ENTRY_OFFSET	0x100
+#define LS7A_INT_ID_OFFSET	0x0
+#define LS7A_INT_ID_VAL		0x7000000UL
+#define LS7A_INT_ID_VER		0x1f0001UL
+#define LS7A_INT_MASK_OFFSET	0x20
+#define LS7A_INT_EDGE_OFFSET	0x60
+#define LS7A_INT_CLEAR_OFFSET	0x80
+#define LS7A_INT_STATUS_OFFSET	0x3a0
+#define LS7A_INT_POL_OFFSET	0x3e0
+#define LS7A_HTMSI_EN_OFFSET	0x40
+#define LS7A_HTMSI_VEC_OFFSET	0x200
+#define LS7A_AUTO_CTRL0_OFFSET	0xc0
+#define LS7A_AUTO_CTRL1_OFFSET	0xe0
+
+#define LS7A_IOAPIC_GUEST_REG_BASE		0x10000000UL
+#define LS7A_IOAPIC_GUEST_REG_BASE_ALIAS	0xe0010000000UL
+
+#define LS7A_IOAPIC_NUM_PINS			32
+
+typedef struct kvm_ls7a_ioapic_state {
+	u64 int_id;
+	/* 0x020 interrupt mask register */
+	u64 int_mask;
+	/* 0x040 1=msi */
+	u64 htmsi_en;
+	/* 0x060 edge=1 level  =0 */
+	u64 intedge;
+	/* 0x080 for clean edge int,set 1 clean,set 0 is noused */
+	u64 intclr;
+	/* 0x0c0 */
+	u64 auto_crtl0;
+	/* 0x0e0 */
+	u64 auto_crtl1;
+	/* 0x100 - 0x140 */
+	u8 route_entry[64];
+	/* 0x200 - 0x240 */
+	u8 htmsi_vector[64];
+	/* 0x300 */
+	u64 intisr_chip0;
+	/* 0x320 */
+	u64 intisr_chip1;
+	/* edge detection */
+	u64 last_intirr;
+	/* 0x380 interrupt request register */
+	u64 intirr;
+	/* 0x3a0 interrupt service register */
+	u64 intisr;
+	/* 0x3e0 interrupt level polarity selection register,
+	 * 0 for high level tirgger
+	 */
+	u64 int_polarity;
+} LS7AApicState;
+
+struct ls7a_kvm_ioapic {
+	spinlock_t lock;
+	bool wakeup_needed;
+	unsigned pending_acks;
+	struct kvm *kvm;
+	struct kvm_ls7a_ioapic_state ls7a_ioapic;
+	struct kvm_io_device dev_ls7a_ioapic;
+	struct kvm_io_device ls7a_ioapic_alias;
+	void (*ack_notifier)(void *opaque, int irq);
+	unsigned long irq_states[LS7A_APIC_NUM_PINS];
+};
+
+static inline struct ls7a_kvm_ioapic *ls7a_ioapic_irqchip(struct kvm *kvm)
+{
+	return kvm->arch.v_ioapic;
+}
+
+static inline int ls7a_ioapic_in_kernel(struct kvm *kvm)
+{
+	int ret;
+
+	ret = (ls7a_ioapic_irqchip(kvm) != NULL);
+	return ret;
+}
+
+int kvm_set_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state);
+int kvm_get_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state);
+int kvm_create_ls7a_ioapic(struct kvm *kvm);
+void kvm_destroy_ls7a_ioapic(struct kvm *kvm);
+int kvm_ls7a_ioapic_set_irq(struct kvm *kvm, int irq, int level);
+
+void ls7a_ioapic_lock(struct ls7a_kvm_ioapic *s, unsigned long *flags);
+void ls7a_ioapic_unlock(struct ls7a_kvm_ioapic *s, unsigned long *flags);
+int kvm_ls7a_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
+int kvm_ls7a_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
+					int irq_source_id, int level, bool line_status);
+
+void kvm_dump_ls7a_ioapic_state(struct seq_file *m, struct ls7a_kvm_ioapic *ioapic);
+#endif

arch/loongarch/kvm/interrupt.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/vmalloc.h>
+#include <linux/fs.h>
+#include <asm/page.h>
+#include <asm/cacheflush.h>
+#include "kvmcpu.h"
+#include <linux/kvm_host.h>
+#include "kvm_compat.h"
+
+static u32 int_to_coreint[LOONGARCH_EXC_MAX] = {
+	[LARCH_INT_TIMER]	= CPU_TIMER,
+	[LARCH_INT_IPI]		= CPU_IPI,
+	[LARCH_INT_SIP0]	= CPU_SIP0,
+	[LARCH_INT_SIP1]	= CPU_SIP1,
+	[LARCH_INT_IP0]		= CPU_IP0,
+	[LARCH_INT_IP1]		= CPU_IP1,
+	[LARCH_INT_IP2]		= CPU_IP2,
+	[LARCH_INT_IP3]		= CPU_IP3,
+	[LARCH_INT_IP4]		= CPU_IP4,
+	[LARCH_INT_IP5]		= CPU_IP5,
+	[LARCH_INT_IP6]		= CPU_IP6,
+	[LARCH_INT_IP7]		= CPU_IP7,
+};
+
+static int _kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
+{
+	unsigned int irq = 0;
+
+	clear_bit(priority, &vcpu->arch.irq_pending);
+	if (priority < LOONGARCH_EXC_MAX)
+		irq = int_to_coreint[priority];
+
+	switch (priority) {
+	case LARCH_INT_TIMER:
+	case LARCH_INT_IPI:
+	case LARCH_INT_SIP0:
+	case LARCH_INT_SIP1:
+		kvm_set_gcsr_estat(irq);
+		break;
+
+	case LARCH_INT_IP0:
+	case LARCH_INT_IP1:
+	case LARCH_INT_IP2:
+	case LARCH_INT_IP3:
+	case LARCH_INT_IP4:
+	case LARCH_INT_IP5:
+	case LARCH_INT_IP6:
+	case LARCH_INT_IP7:
+		kvm_set_csr_gintc(irq);
+		break;
+
+	default:
+		break;
+	}
+
+	return 1;
+}
+
+static int _kvm_irq_clear(struct kvm_vcpu *vcpu, unsigned int priority)
+{
+	unsigned int irq = 0;
+
+	clear_bit(priority, &vcpu->arch.irq_clear);
+	if (priority < LOONGARCH_EXC_MAX)
+		irq = int_to_coreint[priority];
+
+	switch (priority) {
+	case LARCH_INT_TIMER:
+	case LARCH_INT_IPI:
+	case LARCH_INT_SIP0:
+	case LARCH_INT_SIP1:
+		kvm_clear_gcsr_estat(irq);
+		break;
+
+	case LARCH_INT_IP0:
+	case LARCH_INT_IP1:
+	case LARCH_INT_IP2:
+	case LARCH_INT_IP3:
+	case LARCH_INT_IP4:
+	case LARCH_INT_IP5:
+	case LARCH_INT_IP6:
+	case LARCH_INT_IP7:
+		kvm_clear_csr_gintc(irq);
+		break;
+
+	default:
+		break;
+	}
+
+	return 1;
+}
+
+void _kvm_deliver_intr(struct kvm_vcpu *vcpu)
+{
+	unsigned long *pending = &vcpu->arch.irq_pending;
+	unsigned long *pending_clr = &vcpu->arch.irq_clear;
+	unsigned int priority;
+
+	if (!(*pending) && !(*pending_clr))
+		return;
+
+	if (*pending_clr) {
+		priority = __ffs(*pending_clr);
+		while (priority <= LOONGARCH_EXC_IPNUM) {
+			_kvm_irq_clear(vcpu, priority);
+			priority = find_next_bit(pending_clr,
+					BITS_PER_BYTE * sizeof(*pending_clr),
+					priority + 1);
+		}
+	}
+
+	if (*pending) {
+		priority = __ffs(*pending);
+		while (priority <= LOONGARCH_EXC_IPNUM) {
+			_kvm_irq_deliver(vcpu, priority);
+			priority = find_next_bit(pending,
+					BITS_PER_BYTE * sizeof(*pending),
+					priority + 1);
+		}
+	}
+
+}
+
+int _kvm_pending_timer(struct kvm_vcpu *vcpu)
+{
+	return test_bit(LARCH_INT_TIMER, &vcpu->arch.irq_pending);
+}

arch/loongarch/kvm/irq.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef __LOONGHARCH_KVM_IRQ_H__
+#define __LOONGHARCH_KVM_IRQ_H__
+
+static inline int irqchip_in_kernel(struct kvm *kvm)
+{
+	return kvm->arch.v_ioapic ? 1 : 0;
+}
+
+#endif

arch/loongarch/kvm/irqfd.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/kvm_host.h>
+#include <trace/events/kvm.h>
+#include "intc/ls7a_irq.h"
+
+static int kvm_ls7a_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e,
+					struct kvm *kvm, int irq_source_id,
+					int level, bool line_status)
+{
+	/* ioapic pin (0 ~ 64) <---> gsi(0 ~ 64) */
+	u32 irq = e->irqchip.pin;
+	unsigned long flags;
+	int ret;
+
+	if (!ls7a_ioapic_in_kernel(kvm))
+		return -ENXIO;
+
+	ls7a_ioapic_lock(ls7a_ioapic_irqchip(kvm), &flags);
+	ret = kvm_ls7a_ioapic_set_irq(kvm, irq, level);
+	ls7a_ioapic_unlock(ls7a_ioapic_irqchip(kvm), &flags);
+
+	return ret;
+}
+
+/**
+ * kvm_set_routing_entry: populate a kvm routing entry
+ * from a user routing entry
+ *
+ * @kvm: the VM this entry is applied to
+ * @e: kvm kernel routing entry handle
+ * @ue: user api routing entry handle
+ * return 0 on success, -EINVAL on errors.
+ */
+int kvm_set_routing_entry(struct kvm *kvm,
+			  struct kvm_kernel_irq_routing_entry *e,
+			  const struct kvm_irq_routing_entry *ue)
+{
+	int r = -EINVAL;
+
+	switch (ue->type) {
+	case KVM_IRQ_ROUTING_IRQCHIP:
+		e->set = kvm_ls7a_set_ioapic_irq;
+
+		e->irqchip.irqchip = ue->u.irqchip.irqchip;
+		e->irqchip.pin = ue->u.irqchip.pin;
+
+		if (e->irqchip.pin >= LS7A_APIC_NUM_PINS)
+			goto out;
+		break;
+	case KVM_IRQ_ROUTING_MSI:
+		e->set = kvm_set_msi;
+		e->msi.address_lo = ue->u.msi.address_lo;
+		e->msi.address_hi = ue->u.msi.address_hi;
+		e->msi.data = ue->u.msi.data;
+		/*
+		 * linux4.19 kernel have flags and devid
+		 * e->msi.flags = ue->flags;
+		 * e->msi.devid = ue->u.msi.devid;
+		 */
+		break;
+	default:
+		goto out;
+	}
+	r = 0;
+out:
+	return r;
+}
+
+int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
+		struct kvm *kvm, int irq_source_id,
+		int level, bool line_status)
+{
+	if (e->type == KVM_IRQ_ROUTING_MSI)
+		return kvm_ls7a_set_msi(e, kvm, irq_source_id, 1, false);
+
+	return -EWOULDBLOCK;
+}
+
+/**
+ * kvm_set_msi: inject the MSI corresponding to the
+ * MSI routing entry
+ *
+ * This is the entry point for irqfd MSI injection
+ * and userspace MSI injection.
+ */
+int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
+		struct kvm *kvm, int irq_source_id,
+		int level, bool line_status)
+{
+	unsigned int ret;
+
+	if (!level)
+		return -1;
+
+	ret = kvm_ls7a_set_msi(e, kvm, irq_source_id, 1, false);
+	return ret;
+}

arch/loongarch/kvm/kvm_compat.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+#include <linux/version.h>
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include "kvm_compat.h"
+
+extern int  _kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+void kvm_arch_check_processor_compat(void *rtn)
+{
+	*(int *)rtn = 0;
+}
+
+void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
+{
+	_kvm_set_spte_hva(kvm, hva, pte);
+	return;
+}
+
+#elif (LINUX_VERSION_CODE == KERNEL_VERSION(5, 4, 0))
+int kvm_arch_check_processor_compat(void)
+{
+	return 0;
+}
+
+int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
+{
+	struct kvm_memslots *slots;
+	struct kvm_memory_slot *memslot;
+	bool is_dirty = false;
+	int r;
+
+	mutex_lock(&kvm->slots_lock);
+
+	r = kvm_clear_dirty_log_protect(kvm, log, &is_dirty);
+
+	if (is_dirty) {
+		slots = kvm_memslots(kvm);
+		memslot = id_to_memslot(slots, log->slot);
+
+		/* Let implementation handle TLB/GVA invalidation */
+		kvm_flush_remote_tlbs(kvm);
+	}
+
+	mutex_unlock(&kvm->slots_lock);
+	return r;
+}
+
+int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
+{
+	return _kvm_set_spte_hva(kvm, hva, pte);
+}
+#else
+int kvm_arch_check_processor_compat(void *opaque)
+{
+	return 0;
+}
+
+int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
+{
+	return _kvm_set_spte_hva(kvm, hva, pte);
+}
+
+void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
+{
+
+}
+
+int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
+{
+	return 0;
+}
+
+void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
+					struct kvm_memory_slot *memslot)
+{
+#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
+	kvm_flush_remote_tlbs (kvm);
+#endif
+}
+#endif

arch/loongarch/kvm/kvm_compat.h

+#ifndef __LOONGARCH_KVM_COMPAT_H__
+#define __LOONGARCH_KVM_COMPAT_H__
+
+#ifdef __ASSEMBLY__
+#define _ULCAST_
+#else
+#define _ULCAST_ (unsigned long)
+#include <linux/version.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+#include <loongson.h>
+#else
+#include <asm/loongarch.h>
+#endif
+#endif
+
+#define KVM_REG_A0		0x4
+#define KVM_REG_A1		0x5
+#define KVM_REG_A2		0x6
+#define KVM_REG_A3		0x7
+/*
+ * ExStatus.ExcCode
+ */
+#define KVM_EXCCODE_RSV		0	/* Reserved */
+#define KVM_EXCCODE_TLBL	1	/* TLB miss on a load */
+#define KVM_EXCCODE_TLBS	2	/* TLB miss on a store */
+#define KVM_EXCCODE_TLBI	3	/* TLB miss on a ifetch */
+#define KVM_EXCCODE_TLBM	4	/* TLB modified fault */
+#define KVM_EXCCODE_TLBRI	5	/* TLB Read-Inhibit exception */
+#define KVM_EXCCODE_TLBXI	6	/* TLB Execution-Inhibit exception */
+#define KVM_EXCCODE_TLBPE	7	/* TLB Privilege Error */
+#define KVM_EXCCODE_ADE		8	/* Address Error */
+#define KVM_EXCCODE_ALE		9	/* Unalign Access */
+#define KVM_EXCCODE_OOB		10	/* Out of bounds */
+#define KVM_EXCCODE_SYS		11	/* System call */
+#define KVM_EXCCODE_BP		12	/* Breakpoint */
+#define KVM_EXCCODE_INE		13	/* Inst. Not Exist */
+#define KVM_EXCCODE_IPE		14	/* Inst. Privileged Error */
+#define KVM_EXCCODE_FPDIS	15	/* FPU Disabled */
+#define KVM_EXCCODE_LSXDIS	16	/* LSX Disabled */
+#define KVM_EXCCODE_LASXDIS	17	/* LASX Disabled */
+#define KVM_EXCCODE_FPE		18	/* Floating Point Exception */
+#define KVM_EXCCODE_WATCH	19	/* Watch address reference */
+#define KVM_EXCCODE_BTDIS	20	/* Binary Trans. Disabled */
+#define KVM_EXCCODE_BTE		21	/* Binary Trans. Exception */
+#define KVM_EXCCODE_GSPR	22	/* Guest Privileged Error */
+#define KVM_EXCCODE_HYP		23	/* Hypercall */
+#define KVM_EXCCODE_GCM		24	/* Guest CSR modified */
+
+#define KVM_INT_START		64
+#define KVM_INT_SIP0		64
+#define KVM_INT_SIP1		65
+#define KVM_INT_IP0		66
+#define KVM_INT_IP1		67
+#define KVM_INT_IP2		68
+#define KVM_INT_IP3		69
+#define KVM_INT_IP4		70
+#define KVM_INT_IP5		71
+#define KVM_INT_IP6		72
+#define KVM_INT_IP7		73
+#define KVM_INT_PC		74 /* Performance Counter */
+#define KVM_INT_TIMER		75
+#define KVM_INT_IPI		76
+#define KVM_INT_NMI		77
+#define KVM_INT_END		78
+#define KVM_INT_NUM		(KVM_INT_END - KVM_INT_START)
+
+#define KVM_CSR_CRMD		0x0	/* Current mode info */
+#define KVM_CRMD_WE_SHIFT	9
+#define KVM_CRMD_WE		(_ULCAST_(0x1) << KVM_CRMD_WE_SHIFT)
+#define KVM_CRMD_DACM_SHIFT	7
+#define KVM_CRMD_DACM_WIDTH	2
+#define KVM_CRMD_DACM		(_ULCAST_(0x3) << KVM_CRMD_DACM_SHIFT)
+#define KVM_CRMD_DACF_SHIFT	5
+#define KVM_CRMD_DACF_WIDTH	2
+#define KVM_CRMD_DACF		(_ULCAST_(0x3) << KVM_CRMD_DACF_SHIFT)
+#define KVM_CRMD_PG_SHIFT	4
+#define KVM_CRMD_PG		(_ULCAST_(0x1) << KVM_CRMD_PG_SHIFT)
+#define KVM_CRMD_DA_SHIFT	3
+#define KVM_CRMD_DA		(_ULCAST_(0x1) << KVM_CRMD_DA_SHIFT)
+#define KVM_CRMD_IE_SHIFT	2
+#define KVM_CRMD_IE		(_ULCAST_(0x1) << KVM_CRMD_IE_SHIFT)
+#define KVM_CRMD_PLV_SHIFT	0
+#define KVM_CRMD_PLV_WIDTH	2
+#define KVM_CRMD_PLV		(_ULCAST_(0x3) << KVM_CRMD_PLV_SHIFT)
+
+#define KVM_CSR_PRMD		0x1	/* Prev-exception mode info */
+#define KVM_PRMD_PIE_SHIFT	2
+#define KVM_PRMD_PWE_SHIFT	3
+#define KVM_PRMD_PIE		(_ULCAST_(0x1) << KVM_PRMD_PIE_SHIFT)
+#define KVM_PRMD_PWE		(_ULCAST_(0x1) << KVM_PRMD_PWE_SHIFT)
+#define KVM_PRMD_PPLV_SHIFT	0
+#define KVM_PRMD_PPLV_WIDTH	2
+#define KVM_PRMD_PPLV		(_ULCAST_(0x3) << KVM_PRMD_PPLV_SHIFT)
+
+#define KVM_CSR_EUEN		0x2	/* Extended unit enable */
+#define KVM_EUEN_LBTEN_SHIFT	3
+#define KVM_EUEN_LBTEN		(_ULCAST_(0x1) << KVM_EUEN_LBTEN_SHIFT)
+#define KVM_EUEN_LASXEN_SHIFT	2
+#define KVM_EUEN_LASXEN		(_ULCAST_(0x1) << KVM_EUEN_LASXEN_SHIFT)
+#define KVM_EUEN_LSXEN_SHIFT	1
+#define KVM_EUEN_LSXEN		(_ULCAST_(0x1) << KVM_EUEN_LSXEN_SHIFT)
+#define KVM_EUEN_FPEN_SHIFT	0
+#define KVM_EUEN_FPEN		(_ULCAST_(0x1) << KVM_EUEN_FPEN_SHIFT)
+
+#define KVM_CSR_MISC		0x3	/* Misc config */
+#define KVM_CSR_ECFG		0x4	/* Exception config */
+#define KVM_ECFG_VS_SHIFT	16
+#define KVM_ECFG_VS_WIDTH	3
+#define KVM_ECFG_VS		(_ULCAST_(0x7) << KVM_ECFG_VS_SHIFT)
+#define KVM_ECFG_IM_SHIFT	0
+#define KVM_ECFG_IM_WIDTH	13
+#define KVM_ECFG_IM		(_ULCAST_(0x1fff) << KVM_ECFG_IM_SHIFT)
+
+#define KVM_CSR_ESTAT			0x5	/* Exception status */
+#define KVM_ESTAT_ESUBCODE_SHIFT	22
+#define KVM_ESTAT_ESUBCODE_WIDTH	9
+#define KVM_ESTAT_ESUBCODE		(_ULCAST_(0x1ff) << KVM_ESTAT_ESUBCODE_SHIFT)
+#define KVM_ESTAT_EXC_SHIFT		16
+#define KVM_ESTAT_EXC_WIDTH		6
+#define KVM_ESTAT_EXC			(_ULCAST_(0x3f) << KVM_ESTAT_EXC_SHIFT)
+#define KVM_ESTAT_IS_SHIFT		0
+#define KVM_ESTAT_IS_WIDTH		15
+#define KVM_ESTAT_IS			(_ULCAST_(0x7fff) << KVM_ESTAT_IS_SHIFT)
+
+#define KVM_CSR_ERA			0x6	/* ERA */
+#define KVM_CSR_BADV			0x7	/* Bad virtual address */
+#define KVM_CSR_BADI			0x8	/* Bad instruction */
+#define KVM_CSR_EENTRY			0xc	/* Exception entry base address */
+#define KVM_CSR_TLBIDX			0x10	/* TLB Index, EHINV, PageSize, NP */
+#define KVM_CSR_TLBEHI			0x11	/* TLB EntryHi */
+#define KVM_CSR_TLBELO0			0x12	/* TLB EntryLo0 */
+#define KVM_CSR_TLBELO1			0x13	/* TLB EntryLo1 */
+#define KVM_CSR_GTLBC			0x15	/* Guest TLB control */
+#define KVM_GTLBC_TGID_SHIFT		16
+#define KVM_GTLBC_TGID_WIDTH		8
+#define KVM_GTLBC_TGID			(_ULCAST_(0xff) << KVM_GTLBC_TGID_SHIFT)
+#define KVM_GTLBC_TOTI_SHIFT		13
+#define KVM_GTLBC_TOTI			(_ULCAST_(0x1) << KVM_GTLBC_TOTI_SHIFT)
+#define KVM_GTLBC_USETGID_SHIFT		12
+#define KVM_GTLBC_USETGID		(_ULCAST_(0x1) << KVM_GTLBC_USETGID_SHIFT)
+#define KVM_GTLBC_GMTLBSZ_SHIFT		0
+#define KVM_GTLBC_GMTLBSZ_WIDTH		6
+#define KVM_GTLBC_GMTLBSZ		(_ULCAST_(0x3f) << KVM_GTLBC_GMTLBSZ_SHIFT)
+
+#define KVM_CSR_TRGP			0x16	/* TLBR read guest info */
+#define KVM_CSR_ASID			0x18	/* ASID */
+#define KVM_CSR_PGDL			0x19	/* Page table base address when VA[47] = 0 */
+#define KVM_CSR_PGDH			0x1a	/* Page table base address when VA[47] = 1 */
+#define KVM_CSR_PGD			0x1b	/* Page table base */
+#define KVM_CSR_PWCTL0			0x1c	/* PWCtl0 */
+#define KVM_CSR_PWCTL1			0x1d	/* PWCtl1 */
+#define KVM_CSR_STLBPGSIZE		0x1e
+#define KVM_CSR_RVACFG			0x1f
+#define KVM_CSR_CPUID			0x20	/* CPU core number */
+#define KVM_CSR_PRCFG1			0x21	/* Config1 */
+#define KVM_CSR_PRCFG2			0x22	/* Config2 */
+#define KVM_CSR_PRCFG3			0x23	/* Config3 */
+#define KVM_CSR_KS0			0x30
+#define KVM_CSR_KS1			0x31
+#define KVM_CSR_KS2			0x32
+#define KVM_CSR_KS3			0x33
+#define KVM_CSR_KS4			0x34
+#define KVM_CSR_KS5			0x35
+#define KVM_CSR_KS6			0x36
+#define KVM_CSR_KS7			0x37
+#define KVM_CSR_KS8			0x38
+#define KVM_CSR_TMID			0x40	/* Timer ID */
+#define KVM_CSR_TCFG			0x41	/* Timer config */
+#define KVM_TCFG_VAL_SHIFT		2
+#define KVM_TCFG_VAL_WIDTH		48
+#define KVM_TCFG_VAL			(_ULCAST_(0x3fffffffffff) << KVM_TCFG_VAL_SHIFT)
+#define KVM_TCFG_PERIOD_SHIFT		1
+#define KVM_TCFG_PERIOD			(_ULCAST_(0x1) << KVM_TCFG_PERIOD_SHIFT)
+#define KVM_TCFG_EN			(_ULCAST_(0x1))
+
+#define KVM_CSR_TVAL			0x42	/* Timer value */
+#define KVM_CSR_CNTC			0x43	/* Timer offset */
+#define KVM_CSR_TINTCLR			0x44	/* Timer interrupt clear */
+#define KVM_CSR_GSTAT			0x50	/* Guest status */
+#define KVM_GSTAT_GID_SHIFT		16
+#define KVM_GSTAT_GID_WIDTH		8
+#define KVM_GSTAT_GID			(_ULCAST_(0xff) << KVM_GSTAT_GID_SHIFT)
+#define KVM_GSTAT_GIDBIT_SHIFT		4
+#define KVM_GSTAT_GIDBIT_WIDTH		6
+#define KVM_GSTAT_GIDBIT		(_ULCAST_(0x3f) << KVM_GSTAT_GIDBIT_SHIFT)
+#define KVM_GSTAT_PVM_SHIFT		1
+#define KVM_GSTAT_PVM			(_ULCAST_(0x1) << KVM_GSTAT_PVM_SHIFT)
+#define KVM_GSTAT_VM_SHIFT		0
+#define KVM_GSTAT_VM			(_ULCAST_(0x1) << KVM_GSTAT_VM_SHIFT)
+
+#define KVM_CSR_GCFG			0x51	/* Guest config */
+#define KVM_GCFG_GPERF_SHIFT		24
+#define KVM_GCFG_GPERF_WIDTH		3
+#define KVM_GCFG_GPERF			(_ULCAST_(0x7) << KVM_GCFG_GPERF_SHIFT)
+#define KVM_GCFG_GCI_SHIFT		20
+#define KVM_GCFG_GCI_WIDTH		2
+#define KVM_GCFG_GCI			(_ULCAST_(0x3) << KVM_GCFG_GCI_SHIFT)
+#define KVM_GCFG_GCI_ALL		(_ULCAST_(0x0) << KVM_GCFG_GCI_SHIFT)
+#define KVM_GCFG_GCI_HIT		(_ULCAST_(0x1) << KVM_GCFG_GCI_SHIFT)
+#define KVM_GCFG_GCI_SECURE		(_ULCAST_(0x2) << KVM_GCFG_GCI_SHIFT)
+#define KVM_GCFG_GCIP_SHIFT		16
+#define KVM_GCFG_GCIP			(_ULCAST_(0xf) << KVM_GCFG_GCIP_SHIFT)
+#define KVM_GCFG_GCIP_ALL		(_ULCAST_(0x1) << KVM_GCFG_GCIP_SHIFT)
+#define KVM_GCFG_GCIP_HIT		(_ULCAST_(0x1) << (KVM_GCFG_GCIP_SHIFT + 1))
+#define KVM_GCFG_GCIP_SECURE		(_ULCAST_(0x1) << (KVM_GCFG_GCIP_SHIFT + 2))
+#define KVM_GCFG_TORU_SHIFT		15
+#define KVM_GCFG_TORU			(_ULCAST_(0x1) << KVM_GCFG_TORU_SHIFT)
+#define KVM_GCFG_TORUP_SHIFT		14
+#define KVM_GCFG_TORUP			(_ULCAST_(0x1) << KVM_GCFG_TORUP_SHIFT)
+#define KVM_GCFG_TOP_SHIFT		13
+#define KVM_GCFG_TOP			(_ULCAST_(0x1) << KVM_GCFG_TOP_SHIFT)
+#define KVM_GCFG_TOPP_SHIFT		12
+#define KVM_GCFG_TOPP			(_ULCAST_(0x1) << KVM_GCFG_TOPP_SHIFT)
+#define KVM_GCFG_TOE_SHIFT		11
+#define KVM_GCFG_TOE			(_ULCAST_(0x1) << KVM_GCFG_TOE_SHIFT)
+#define KVM_GCFG_TOEP_SHIFT		10
+#define KVM_GCFG_TOEP			(_ULCAST_(0x1) << KVM_GCFG_TOEP_SHIFT)
+#define KVM_GCFG_TIT_SHIFT		9
+#define KVM_GCFG_TIT			(_ULCAST_(0x1) << KVM_GCFG_TIT_SHIFT)
+#define KVM_GCFG_TITP_SHIFT		8
+#define KVM_GCFG_TITP			(_ULCAST_(0x1) << KVM_GCFG_TITP_SHIFT)
+#define KVM_GCFG_SIT_SHIFT		7
+#define KVM_GCFG_SIT			(_ULCAST_(0x1) << KVM_GCFG_SIT_SHIFT)
+#define KVM_GCFG_SITP_SHIFT		6
+#define KVM_GCFG_SITP			(_ULCAST_(0x1) << KVM_GCFG_SITP_SHIFT)
+#define KVM_GCFG_MATC_SHITF		4
+#define KVM_GCFG_MATC_WIDTH		2
+#define KVM_GCFG_MATC_MASK		(_ULCAST_(0x3) << KVM_GCFG_MATC_SHITF)
+#define KVM_GCFG_MATC_GUEST		(_ULCAST_(0x0) << KVM_GCFG_MATC_SHITF)
+#define KVM_GCFG_MATC_ROOT		(_ULCAST_(0x1) << KVM_GCFG_MATC_SHITF)
+#define KVM_GCFG_MATC_NEST		(_ULCAST_(0x2) << KVM_GCFG_MATC_SHITF)
+#define KVM_GCFG_MATP_SHITF		0
+#define KVM_GCFG_MATP_WIDTH		4
+#define KVM_GCFG_MATR_MASK		(_ULCAST_(0x3) << KVM_GCFG_MATP_SHITF)
+#define KVM_GCFG_MATP_GUEST		(_ULCAST_(0x0) << KVM_GCFG_MATP_SHITF)
+#define KVM_GCFG_MATP_ROOT		(_ULCAST_(0x1) << KVM_GCFG_MATP_SHITF)
+#define KVM_GCFG_MATP_NEST		(_ULCAST_(0x2) << KVM_GCFG_MATP_SHITF)
+
+#define KVM_CSR_GINTC			0x52	/* Guest interrupt control */
+#define KVM_CSR_GCNTC			0x53	/* Guest timer offset */
+#define KVM_CSR_LLBCTL			0x60	/* LLBit control */
+#define KVM_LLBCTL_ROLLB_SHIFT		0
+#define KVM_LLBCTL_ROLLB		(_ULCAST_(1) << KVM_LLBCTL_ROLLB_SHIFT)
+#define KVM_LLBCTL_WCLLB_SHIFT		1
+#define KVM_LLBCTL_WCLLB		(_ULCAST_(1) << KVM_LLBCTL_WCLLB_SHIFT)
+#define KVM_LLBCTL_KLO_SHIFT		2
+#define KVM_LLBCTL_KLO			(_ULCAST_(1) << KVM_LLBCTL_KLO_SHIFT)
+
+#define KVM_CSR_IMPCTL1		0x80	/* Loongson config1 */
+#define KVM_CSR_IMPCTL2		0x81	/* Loongson config2 */
+#define KVM_CSR_GNMI		0x82
+#define KVM_CSR_TLBRENTRY	0x88	/* TLB refill exception base address */
+#define KVM_CSR_TLBRBADV	0x89	/* TLB refill badvaddr */
+#define KVM_CSR_TLBRERA		0x8a	/* TLB refill ERA */
+#define KVM_CSR_TLBRSAVE	0x8b	/* KScratch for TLB refill exception */
+#define KVM_CSR_TLBRELO0	0x8c	/* TLB refill entrylo0 */
+#define KVM_CSR_TLBRELO1	0x8d	/* TLB refill entrylo1 */
+#define KVM_CSR_TLBREHI		0x8e	/* TLB refill entryhi */
+#define KVM_CSR_TLBRPRMD	0x8f	/* TLB refill mode info */
+#define KVM_CSR_ERRCTL		0x90	/* ERRCTL */
+#define KVM_CSR_ERRINFO1	0x91	/* Error info1 */
+#define KVM_CSR_ERRINFO2	0x92	/* Error info2 */
+#define KVM_CSR_MERRENTRY	0x93	/* Error exception base address */
+#define KVM_CSR_MERRERA		0x94	/* Error exception PC */
+#define KVM_CSR_ERRSAVE		0x95	/* KScratch for machine error exception */
+#define KVM_CSR_CTAG		0x98	/* TagLo + TagHi */
+#define KVM_CSR_DMWIN0		0x180	/* 64 direct map win0: MEM & IF */
+#define KVM_CSR_DMWIN1		0x181	/* 64 direct map win1: MEM & IF */
+#define KVM_CSR_DMWIN2		0x182	/* 64 direct map win2: MEM */
+#define KVM_CSR_DMWIN3		0x183	/* 64 direct map win3: MEM */
+#define KVM_CSR_PERFCTRL0	0x200	/* 32 perf event 0 config */
+#define KVM_CSR_PERFCNTR0	0x201	/* 64 perf event 0 count value */
+#define KVM_CSR_PERFCTRL1	0x202	/* 32 perf event 1 config */
+#define KVM_CSR_PERFCNTR1	0x203	/* 64 perf event 1 count value */
+#define KVM_CSR_PERFCTRL2	0x204	/* 32 perf event 2 config */
+#define KVM_CSR_PERFCNTR2	0x205	/* 64 perf event 2 count value */
+#define KVM_CSR_PERFCTRL3	0x206	/* 32 perf event 3 config */
+#define KVM_CSR_PERFCNTR3	0x207	/* 64 perf event 3 count value */
+#define KVM_PERFCTRL_PLV0	(_ULCAST_(1) << 16)
+#define KVM_PERFCTRL_PLV1	(_ULCAST_(1) << 17)
+#define KVM_PERFCTRL_PLV2	(_ULCAST_(1) << 18)
+#define KVM_PERFCTRL_PLV3	(_ULCAST_(1) << 19)
+#define KVM_PERFCTRL_IE		(_ULCAST_(1) << 20)
+#define KVM_PERFCTRL_GMOD	(_ULCAST_(3) << 21)
+#define KVM_PERFCTRL_EVENT	0x3ff
+
+#define KVM_CSR_MWPC		0x300	/* data breakpoint config */
+#define KVM_CSR_MWPS		0x301	/* data breakpoint status */
+#define KVM_CSR_FWPC		0x380	/* instruction breakpoint config */
+#define KVM_CSR_FWPS		0x381	/* instruction breakpoint status */
+#define KVM_CSR_DEBUG		0x500	/* debug config */
+#define KVM_CSR_DERA		0x501	/* debug era */
+#define KVM_CSR_DESAVE		0x502	/* debug save */
+
+#define KVM_IOCSR_FEATURES			0x8
+#define KVM_IOCSRF_TEMP				BIT_ULL(0)
+#define KVM_IOCSRF_NODECNT			BIT_ULL(1)
+#define KVM_IOCSRF_MSI				BIT_ULL(2)
+#define KVM_IOCSRF_EXTIOI			BIT_ULL(3)
+#define KVM_IOCSRF_CSRIPI			BIT_ULL(4)
+#define KVM_IOCSRF_FREQCSR			BIT_ULL(5)
+#define KVM_IOCSRF_FREQSCALE			BIT_ULL(6)
+#define KVM_IOCSRF_DVFSV1			BIT_ULL(7)
+#define KVM_IOCSRF_EXTIOI_DECODE		BIT_ULL(9)
+#define KVM_IOCSRF_FLATMODE			BIT_ULL(10)
+#define KVM_IOCSRF_VM				BIT_ULL(11)
+
+#define KVM_IOCSR_VENDOR			0x10
+#define KVM_IOCSR_CPUNAME			0x20
+#define KVM_IOCSR_NODECNT			0x408
+
+#define KVM_IOCSR_MISC_FUNC			0x420
+#define KVM_IOCSRF_MISC_FUNC_EXT_IOI_EN		BIT_ULL(48)
+
+/* PerCore CSR, only accessable by local cores */
+#define KVM_IOCSR_IPI_STATUS			0x1000
+#define KVM_IOCSR_IPI_SEND			0x1040
+#define KVM_IOCSR_MBUF_SEND			0x1048
+#define KVM_IOCSR_EXTIOI_NODEMAP_BASE		0x14a0
+#define KVM_IOCSR_EXTIOI_IPMAP_BASE		0x14c0
+#define KVM_IOCSR_EXTIOI_EN_BASE		0x1600
+#define KVM_IOCSR_EXTIOI_BOUNCE_BASE		0x1680
+#define KVM_IOCSR_EXTIOI_ISR_BASE		0x1800
+#define KVM_IOCSR_EXTIOI_ROUTE_BASE		0x1c00
+
+#ifndef __ASSEMBLY__
+
+/* CSR */
+static inline u32 kvm_csr_readl(u32 reg)
+{
+	u32 val;
+
+	asm volatile (
+		"csrrd %[val], %[reg] \n"
+		: [val] "=r" (val)
+		: [reg] "i" (reg)
+		: "memory");
+	return val;
+}
+
+static inline u64 kvm_csr_readq(u32 reg)
+{
+	u64 val;
+
+	asm volatile (
+		"csrrd %[val], %[reg] \n"
+		: [val] "=r" (val)
+		: [reg] "i" (reg)
+		: "memory");
+	return val;
+}
+
+static inline void kvm_csr_writel(u32 val, u32 reg)
+{
+	asm volatile (
+		"csrwr %[val], %[reg] \n"
+		: [val] "+r" (val)
+		: [reg] "i" (reg)
+		: "memory");
+}
+
+static inline void kvm_csr_writeq(u64 val, u32 reg)
+{
+	asm volatile (
+		"csrwr %[val], %[reg] \n"
+		: [val] "+r" (val)
+		: [reg] "i" (reg)
+		: "memory");
+}
+
+static inline u32 kvm_csr_xchgl(u32 val, u32 mask, u32 reg)
+{
+	asm volatile (
+		"csrxchg %[val], %[mask], %[reg] \n"
+		: [val] "+r" (val)
+		: [mask] "r" (mask), [reg] "i" (reg)
+		: "memory");
+	return val;
+}
+
+static inline u64 kvm_csr_xchgq(u64 val, u64 mask, u32 reg)
+{
+	asm volatile (
+		"csrxchg %[val], %[mask], %[reg] \n"
+		: [val] "+r" (val)
+		: [mask] "r" (mask), [reg] "i" (reg)
+		: "memory");
+	return val;
+}
+
+
+/* IOCSR */
+static inline u32 kvm_iocsr_readl(u32 reg)
+{
+	u32 val;
+
+	asm volatile (
+		"iocsrrd.w %[val], %[reg] \n"
+		: [val] "=r" (val)
+		: [reg] "r" (reg)
+		: "memory");
+	return val;
+}
+
+static inline u64 kvm_iocsr_readq(u32 reg)
+{
+	u64 val;
+
+	asm volatile (
+		"iocsrrd.d %[val], %[reg] \n"
+		: [val] "=r" (val)
+		: [reg] "r" (reg)
+		: "memory");
+	return val;
+}
+
+static inline void kvm_iocsr_writeb(u8 val, u32 reg)
+{
+	asm volatile (
+		"iocsrwr.b %[val], %[reg] \n"
+		:
+		: [val] "r" (val), [reg] "r" (reg)
+		: "memory");
+}
+
+static inline void kvm_iocsr_writel(u32 val, u32 reg)
+{
+	asm volatile (
+		"iocsrwr.w %[val], %[reg] \n"
+		:
+		: [val] "r" (val), [reg] "r" (reg)
+		: "memory");
+}
+
+static inline void kvm_iocsr_writeq(u64 val, u32 reg)
+{
+	asm volatile (
+		"iocsrwr.d %[val], %[reg] \n"
+		:
+		: [val] "r" (val), [reg] "r" (reg)
+		: "memory");
+}
+
+
+/* GCSR */
+static inline u64 kvm_gcsr_read(u32 reg)
+{
+	u64 val = 0;
+
+	asm volatile (
+	"parse_r __reg, %[val]	\n"
+	".word 0x5 << 24 | %[reg] << 10 | 0 << 5 | __reg	\n"
+	: [val] "+r" (val)
+	: [reg] "i" (reg)
+	: "memory");
+	return val;
+}
+
+static inline void kvm_gcsr_write(u64 val, u32 reg)
+{
+	asm volatile (
+	"parse_r __reg, %[val]	\n"
+	".word 0x5 << 24 | %[reg] << 10 | 1 << 5 | __reg	\n"
+	: [val] "+r" (val)
+	: [reg] "i" (reg)
+	: "memory");
+}
+
+static inline u64 kvm_gcsr_xchg(u64 val, u64 mask, u32 reg)
+{
+	asm volatile (
+	"parse_r __rd, %[val]	\n"
+	"parse_r __rj, %[mask]	\n"
+	".word 0x5 << 24 | %[reg] << 10 | __rj << 5 | __rd	\n"
+	: [val] "+r" (val)
+	: [mask] "r" (mask), [reg] "i" (reg)
+	: "memory");
+	return val;
+}
+
+#endif /* !__ASSEMBLY__ */
+
+#define kvm_read_csr_euen()		kvm_csr_readq(KVM_CSR_EUEN)
+#define kvm_write_csr_euen(val)		kvm_csr_writeq(val, KVM_CSR_EUEN)
+#define kvm_read_csr_ecfg()		kvm_csr_readq(KVM_CSR_ECFG)
+#define kvm_write_csr_ecfg(val)		kvm_csr_writeq(val, KVM_CSR_ECFG)
+#define kvm_write_csr_perfctrl0(val)	kvm_csr_writeq(val, KVM_CSR_PERFCTRL0)
+#define kvm_write_csr_perfcntr0(val)	kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR0)
+#define kvm_write_csr_perfctrl1(val)	kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL1)
+#define kvm_write_csr_perfcntr1(val)	kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR1)
+#define kvm_write_csr_perfctrl2(val)	kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL2)
+#define kvm_write_csr_perfcntr2(val)	kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR2)
+#define kvm_write_csr_perfctrl3(val)	kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL3)
+#define kvm_write_csr_perfcntr3(val)	kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR3)
+#define kvm_read_csr_impctl1()		kvm_csr_readq(LOONGARCH_CSR_IMPCTL1)
+#define kvm_write_csr_impctl1(val)	kvm_csr_writeq(val, LOONGARCH_CSR_IMPCTL1)
+
+
+/* Guest related CSRS */
+#define kvm_read_csr_gtlbc()		kvm_csr_readq(KVM_CSR_GTLBC)
+#define kvm_write_csr_gtlbc(val)	kvm_csr_writeq(val, KVM_CSR_GTLBC)
+#define kvm_read_csr_trgp()		kvm_csr_readq(KVM_CSR_TRGP)
+#define kvm_read_csr_gcfg()		kvm_csr_readq(KVM_CSR_GCFG)
+#define kvm_write_csr_gcfg(val)		kvm_csr_writeq(val, KVM_CSR_GCFG)
+#define kvm_read_csr_gstat()		kvm_csr_readq(KVM_CSR_GSTAT)
+#define kvm_write_csr_gstat(val)	kvm_csr_writeq(val, KVM_CSR_GSTAT)
+#define kvm_read_csr_gintc()		kvm_csr_readq(KVM_CSR_GINTC)
+#define kvm_write_csr_gintc(val)	kvm_csr_writeq(val, KVM_CSR_GINTC)
+#define kvm_read_csr_gcntc()		kvm_csr_readq(KVM_CSR_GCNTC)
+#define kvm_write_csr_gcntc(val)	kvm_csr_writeq(val, KVM_CSR_GCNTC)
+
+/* Guest CSRS read and write */
+#define kvm_read_gcsr_crmd()		kvm_gcsr_read(KVM_CSR_CRMD)
+#define kvm_write_gcsr_crmd(val)	kvm_gcsr_write(val, KVM_CSR_CRMD)
+#define kvm_read_gcsr_prmd()		kvm_gcsr_read(KVM_CSR_PRMD)
+#define kvm_write_gcsr_prmd(val)	kvm_gcsr_write(val, KVM_CSR_PRMD)
+#define kvm_read_gcsr_euen()		kvm_gcsr_read(KVM_CSR_EUEN)
+#define kvm_write_gcsr_euen(val)	kvm_gcsr_write(val, KVM_CSR_EUEN)
+#define kvm_read_gcsr_misc()		kvm_gcsr_read(KVM_CSR_MISC)
+#define kvm_write_gcsr_misc(val)	kvm_gcsr_write(val, KVM_CSR_MISC)
+#define kvm_read_gcsr_ecfg()		kvm_gcsr_read(KVM_CSR_ECFG)
+#define kvm_write_gcsr_ecfg(val)	kvm_gcsr_write(val, KVM_CSR_ECFG)
+#define kvm_read_gcsr_estat()		kvm_gcsr_read(KVM_CSR_ESTAT)
+#define kvm_write_gcsr_estat(val)	kvm_gcsr_write(val, KVM_CSR_ESTAT)
+#define kvm_read_gcsr_era()		kvm_gcsr_read(KVM_CSR_ERA)
+#define kvm_write_gcsr_era(val)		kvm_gcsr_write(val, KVM_CSR_ERA)
+#define kvm_read_gcsr_badv()		kvm_gcsr_read(KVM_CSR_BADV)
+#define kvm_write_gcsr_badv(val)	kvm_gcsr_write(val, KVM_CSR_BADV)
+#define kvm_read_gcsr_badi()		kvm_gcsr_read(KVM_CSR_BADI)
+#define kvm_write_gcsr_badi(val)	kvm_gcsr_write(val, KVM_CSR_BADI)
+#define kvm_read_gcsr_eentry()		kvm_gcsr_read(KVM_CSR_EENTRY)
+#define kvm_write_gcsr_eentry(val)	kvm_gcsr_write(val, KVM_CSR_EENTRY)
+
+#define kvm_read_gcsr_tlbidx()		kvm_gcsr_read(KVM_CSR_TLBIDX)
+#define kvm_write_gcsr_tlbidx(val)	kvm_gcsr_write(val, KVM_CSR_TLBIDX)
+#define kvm_read_gcsr_tlbhi()		kvm_gcsr_read(KVM_CSR_TLBEHI)
+#define kvm_write_gcsr_tlbhi(val)	kvm_gcsr_write(val, KVM_CSR_TLBEHI)
+#define kvm_read_gcsr_tlblo0()		kvm_gcsr_read(KVM_CSR_TLBELO0)
+#define kvm_write_gcsr_tlblo0(val)	kvm_gcsr_write(val, KVM_CSR_TLBELO0)
+#define kvm_read_gcsr_tlblo1()		kvm_gcsr_read(KVM_CSR_TLBELO1)
+#define kvm_write_gcsr_tlblo1(val)	kvm_gcsr_write(val, KVM_CSR_TLBELO1)
+
+#define kvm_read_gcsr_asid()		kvm_gcsr_read(KVM_CSR_ASID)
+#define kvm_write_gcsr_asid(val)	kvm_gcsr_write(val, KVM_CSR_ASID)
+#define kvm_read_gcsr_pgdl()		kvm_gcsr_read(KVM_CSR_PGDL)
+#define kvm_write_gcsr_pgdl(val)	kvm_gcsr_write(val, KVM_CSR_PGDL)
+#define kvm_read_gcsr_pgdh()		kvm_gcsr_read(KVM_CSR_PGDH)
+#define kvm_write_gcsr_pgdh(val)	kvm_gcsr_write(val, KVM_CSR_PGDH)
+#define kvm_write_gcsr_pgd(val)		kvm_gcsr_write(val, KVM_CSR_PGD)
+#define kvm_read_gcsr_pgd()		kvm_gcsr_read(KVM_CSR_PGD)
+#define kvm_read_gcsr_pwctl0()		kvm_gcsr_read(KVM_CSR_PWCTL0)
+#define kvm_write_gcsr_pwctl0(val)	kvm_gcsr_write(val, KVM_CSR_PWCTL0)
+#define kvm_read_gcsr_pwctl1()		kvm_gcsr_read(KVM_CSR_PWCTL1)
+#define kvm_write_gcsr_pwctl1(val)	kvm_gcsr_write(val, KVM_CSR_PWCTL1)
+#define kvm_read_gcsr_stlbpgsize()	kvm_gcsr_read(KVM_CSR_STLBPGSIZE)
+#define kvm_write_gcsr_stlbpgsize(val)	kvm_gcsr_write(val, KVM_CSR_STLBPGSIZE)
+#define kvm_read_gcsr_rvacfg()		kvm_gcsr_read(KVM_CSR_RVACFG)
+#define kvm_write_gcsr_rvacfg(val)	kvm_gcsr_write(val, KVM_CSR_RVACFG)
+
+#define kvm_read_gcsr_cpuid()		kvm_gcsr_read(KVM_CSR_CPUID)
+#define kvm_write_gcsr_cpuid(val)	kvm_gcsr_write(val, KVM_CSR_CPUID)
+#define kvm_read_gcsr_prcfg1()		kvm_gcsr_read(KVM_CSR_PRCFG1)
+#define kvm_write_gcsr_prcfg1(val)	kvm_gcsr_write(val, KVM_CSR_PRCFG1)
+#define kvm_read_gcsr_prcfg2()		kvm_gcsr_read(KVM_CSR_PRCFG2)
+#define kvm_write_gcsr_prcfg2(val)	kvm_gcsr_write(val, KVM_CSR_PRCFG2)
+#define kvm_read_gcsr_prcfg3()		kvm_gcsr_read(KVM_CSR_PRCFG3)
+#define kvm_write_gcsr_prcfg3(val)	kvm_gcsr_write(val, KVM_CSR_PRCFG3)
+
+#define kvm_read_gcsr_kscratch0()	kvm_gcsr_read(KVM_CSR_KS0)
+#define kvm_write_gcsr_kscratch0(val)	kvm_gcsr_write(val, KVM_CSR_KS0)
+#define kvm_read_gcsr_kscratch1()	kvm_gcsr_read(KVM_CSR_KS1)
+#define kvm_write_gcsr_kscratch1(val)	kvm_gcsr_write(val, KVM_CSR_KS1)
+#define kvm_read_gcsr_kscratch2()	kvm_gcsr_read(KVM_CSR_KS2)
+#define kvm_write_gcsr_kscratch2(val)	kvm_gcsr_write(val, KVM_CSR_KS2)
+#define kvm_read_gcsr_kscratch3()	kvm_gcsr_read(KVM_CSR_KS3)
+#define kvm_write_gcsr_kscratch3(val)	kvm_gcsr_write(val, KVM_CSR_KS3)
+#define kvm_read_gcsr_kscratch4()	kvm_gcsr_read(KVM_CSR_KS4)
+#define kvm_write_gcsr_kscratch4(val)	kvm_gcsr_write(val, KVM_CSR_KS4)
+#define kvm_read_gcsr_kscratch5()	kvm_gcsr_read(KVM_CSR_KS5)
+#define kvm_write_gcsr_kscratch5(val)	kvm_gcsr_write(val, KVM_CSR_KS5)
+#define kvm_read_gcsr_kscratch6()	kvm_gcsr_read(KVM_CSR_KS6)
+#define kvm_write_gcsr_kscratch6(val)	kvm_gcsr_write(val, KVM_CSR_KS6)
+#define kvm_read_gcsr_kscratch7()	kvm_gcsr_read(KVM_CSR_KS7)
+#define kvm_write_gcsr_kscratch7(val)	kvm_gcsr_write(val, KVM_CSR_KS7)
+
+#define kvm_read_gcsr_timerid()		kvm_gcsr_read(KVM_CSR_TMID)
+#define kvm_write_gcsr_timerid(val)	kvm_gcsr_write(val, KVM_CSR_TMID)
+#define kvm_read_gcsr_timercfg()	kvm_gcsr_read(KVM_CSR_TCFG)
+#define kvm_write_gcsr_timercfg(val)	kvm_gcsr_write(val, KVM_CSR_TCFG)
+#define kvm_read_gcsr_timertick()	kvm_gcsr_read(KVM_CSR_TVAL)
+#define kvm_write_gcsr_timertick(val)	kvm_gcsr_write(val, KVM_CSR_TVAL)
+#define kvm_read_gcsr_timeroffset()	kvm_gcsr_read(KVM_CSR_CNTC)
+#define kvm_write_gcsr_timeroffset(val)	kvm_gcsr_write(val, KVM_CSR_CNTC)
+
+#define kvm_read_gcsr_llbctl()		kvm_gcsr_read(KVM_CSR_LLBCTL)
+#define kvm_write_gcsr_llbctl(val)	kvm_gcsr_write(val, KVM_CSR_LLBCTL)
+
+#define kvm_read_gcsr_tlbrentry()	kvm_gcsr_read(KVM_CSR_TLBRENTRY)
+#define kvm_write_gcsr_tlbrentry(val)	kvm_gcsr_write(val, KVM_CSR_TLBRENTRY)
+#define kvm_read_gcsr_tlbrbadv()	kvm_gcsr_read(KVM_CSR_TLBRBADV)
+#define kvm_write_gcsr_tlbrbadv(val)	kvm_gcsr_write(val, KVM_CSR_TLBRBADV)
+#define kvm_read_gcsr_tlbrera()		kvm_gcsr_read(KVM_CSR_TLBRERA)
+#define kvm_write_gcsr_tlbrera(val)	kvm_gcsr_write(val, KVM_CSR_TLBRERA)
+#define kvm_read_gcsr_tlbrsave()	kvm_gcsr_read(KVM_CSR_TLBRSAVE)
+#define kvm_write_gcsr_tlbrsave(val)	kvm_gcsr_write(val, KVM_CSR_TLBRSAVE)
+#define kvm_read_gcsr_tlbrelo0()	kvm_gcsr_read(KVM_CSR_TLBRELO0)
+#define kvm_write_gcsr_tlbrelo0(val)	kvm_gcsr_write(val, KVM_CSR_TLBRELO0)
+#define kvm_read_gcsr_tlbrelo1()	kvm_gcsr_read(KVM_CSR_TLBRELO1)
+#define kvm_write_gcsr_tlbrelo1(val)	kvm_gcsr_write(val, KVM_CSR_TLBRELO1)
+#define kvm_read_gcsr_tlbrehi()		kvm_gcsr_read(KVM_CSR_TLBREHI)
+#define kvm_write_gcsr_tlbrehi(val)	kvm_gcsr_write(val, KVM_CSR_TLBREHI)
+#define kvm_read_gcsr_tlbrprmd()	kvm_gcsr_read(KVM_CSR_TLBRPRMD)
+#define kvm_write_gcsr_tlbrprmd(val)	kvm_gcsr_write(val, KVM_CSR_TLBRPRMD)
+
+#define kvm_read_gcsr_directwin0()	kvm_gcsr_read(KVM_CSR_DMWIN0)
+#define kvm_write_gcsr_directwin0(val)	kvm_gcsr_write(val, KVM_CSR_DMWIN0)
+#define kvm_read_gcsr_directwin1()	kvm_gcsr_read(KVM_CSR_DMWIN1)
+#define kvm_write_gcsr_directwin1(val)	kvm_gcsr_write(val, KVM_CSR_DMWIN1)
+#define kvm_read_gcsr_directwin2()	kvm_gcsr_read(KVM_CSR_DMWIN2)
+#define kvm_write_gcsr_directwin2(val)	kvm_gcsr_write(val, KVM_CSR_DMWIN2)
+#define kvm_read_gcsr_directwin3()	kvm_gcsr_read(KVM_CSR_DMWIN3)
+#define kvm_write_gcsr_directwin3(val)	kvm_gcsr_write(val, KVM_CSR_DMWIN3)
+
+#ifndef __ASSEMBLY__
+
+static inline unsigned long
+kvm_set_csr_gtlbc(unsigned long set)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_gtlbc();
+	new = res | set;
+	kvm_write_csr_gtlbc(new);
+
+	return res;
+}
+
+static inline unsigned long
+kvm_set_csr_euen(unsigned long set)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_euen();
+	new = res | set;
+	kvm_write_csr_euen(new);
+
+	return res;
+}
+
+static inline unsigned long
+kvm_set_csr_gintc(unsigned long set)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_gintc();
+	new = res | set;
+	kvm_write_csr_gintc(new);
+
+	return res;
+}
+
+static inline unsigned long
+kvm_set_gcsr_llbctl(unsigned long set)
+{
+	unsigned long res, new;
+
+	res = kvm_read_gcsr_llbctl();
+	new = res | set;
+	kvm_write_gcsr_llbctl(new);
+
+	return res;
+}
+
+
+static inline unsigned long
+kvm_clear_csr_gtlbc(unsigned long clear)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_gtlbc();
+	new = res & ~clear;
+	kvm_write_csr_gtlbc(new);
+
+	return res;
+}
+
+static inline unsigned long
+kvm_clear_csr_euen(unsigned long clear)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_euen();
+	new = res & ~clear;
+	kvm_write_csr_euen(new);
+
+	return res;
+}
+
+static inline unsigned long
+kvm_clear_csr_gintc(unsigned long clear)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_gintc();
+	new = res & ~clear;
+	kvm_write_csr_gintc(new);
+
+	return res;
+}
+
+static inline unsigned long
+kvm_change_csr_gstat(unsigned long change, unsigned long val)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_gstat();
+	new = res & ~change;
+	new |= (val & change);
+	kvm_write_csr_gstat(new);
+
+	return res;
+}
+
+static inline unsigned long
+kvm_change_csr_gcfg(unsigned long change, unsigned long val)
+{
+	unsigned long res, new;
+
+	res = kvm_read_csr_gcfg();
+	new = res & ~change;
+	new |= (val & change);
+	kvm_write_csr_gcfg(new);
+
+	return res;
+}
+
+
+#define kvm_set_gcsr_estat(val)	\
+	kvm_gcsr_xchg(val, val, KVM_CSR_ESTAT)
+#define kvm_clear_gcsr_estat(val)	\
+	kvm_gcsr_xchg(~(val), val, KVM_CSR_ESTAT)
+
+#endif
+
+/* Device Control API on vcpu fd */
+#define KVM_LARCH_VCPU_PVTIME_CTRL  2
+#define KVM_LARCH_VCPU_PVTIME_IPA   0
+
+#if (_LOONGARCH_SZLONG == 32)
+#define KVM_LONG_ADD	add.w
+#define KVM_LONG_ADDI	addi.w
+#define KVM_LONG_SUB	sub.w
+#define KVM_LONG_L	ld.w
+#define KVM_LONG_S	st.w
+#define KVM_LONG_SLL	slli.w
+#define KVM_LONG_SLLV	sll.w
+#define KVM_LONG_SRL	srli.w
+#define KVM_LONG_SRLV	srl.w
+#define KVM_LONG_SRA	srai.w
+#define KVM_LONG_SRAV	sra.w
+
+#define KVM_LONGSIZE	4
+#define KVM_LONGMASK	3
+#define KVM_LONGLOG	2
+
+/*
+ * How to add/sub/load/store/shift pointers.
+ */
+
+#define KVM_PTR_ADD	add.w
+#define KVM_PTR_ADDI	addi.w
+#define KVM_PTR_SUB	sub.w
+#define KVM_PTR_L	ld.w
+#define KVM_PTR_S	st.w
+#define KVM_PTR_LI	li.w
+#define KVM_PTR_SLL	slli.w
+#define KVM_PTR_SLLV	sll.w
+#define KVM_PTR_SRL	srli.w
+#define KVM_PTR_SRLV	srl.w
+#define KVM_PTR_SRA	srai.w
+#define KVM_PTR_SRAV	sra.w
+
+#define KVM_PTR_SCALESHIFT	2
+
+#define KVM_PTRSIZE	4
+#define KVM_PTRLOG	2
+
+#endif
+
+#if (_LOONGARCH_SZLONG == 64)
+#define KVM_LONG_ADD	add.d
+#define KVM_LONG_ADDI	addi.d
+#define KVM_LONG_SUB	sub.d
+#define KVM_LONG_L	ld.d
+#define KVM_LONG_S	st.d
+#define KVM_LONG_SLL	slli.d
+#define KVM_LONG_SLLV	sll.d
+#define KVM_LONG_SRL	srli.d
+#define KVM_LONG_SRLV	srl.d
+#define KVM_LONG_SRA	sra.w
+#define KVM_LONG_SRAV	sra.d
+
+#define KVM_LONGSIZE	8
+#define KVM_LONGMASK	7
+#define KVM_LONGLOG	3
+
+/*
+ * How to add/sub/load/store/shift pointers.
+ */
+
+#define KVM_PTR_ADD	add.d
+#define KVM_PTR_ADDI	addi.d
+#define KVM_PTR_SUB	sub.d
+#define KVM_PTR_L	ld.d
+#define KVM_PTR_S	st.d
+#define KVM_PTR_LI	li.d
+#define KVM_PTR_SLL	slli.d
+#define KVM_PTR_SLLV	sll.d
+#define KVM_PTR_SRL	srli.d
+#define KVM_PTR_SRLV	srl.d
+#define KVM_PTR_SRA	srai.d
+#define KVM_PTR_SRAV	sra.d
+
+#define KVM_PTR_SCALESHIFT	3
+
+#define KVM_PTRSIZE	8
+#define KVM_PTRLOG	3
+#endif
+
+#endif		/* __LOONGARCH_KVM_COMPAT_H__ */

arch/loongarch/kvm/kvmcpu.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef __ASM_LOONGARCH_KVMCPU_H__
+#define __ASM_LOONGARCH_KVMCPU_H__
+
+#include <linux/kvm_host.h>
+#include <asm/kvm_host.h>
+
+#define LARCH_INT_SIP0			0
+#define LARCH_INT_SIP1			1
+#define LARCH_INT_IP0			2
+#define LARCH_INT_IP1			3
+#define LARCH_INT_IP2			4
+#define LARCH_INT_IP3			5
+#define LARCH_INT_IP4			6
+#define LARCH_INT_IP5			7
+#define LARCH_INT_IP6			8
+#define LARCH_INT_IP7			9
+#define LARCH_INT_PMU			10
+#define LARCH_INT_TIMER			11
+#define LARCH_INT_IPI			12
+#define LOONGARCH_EXC_MAX		(LARCH_INT_IPI + 1)
+#define LOONGARCH_EXC_IPNUM		(LOONGARCH_EXC_MAX)
+
+/* Controlled by 0x5 guest exst */
+#define CPU_SIP0			(_ULCAST_(1))
+#define CPU_SIP1			(_ULCAST_(1) << 1)
+#define CPU_PMU				(_ULCAST_(1) << 10)
+#define CPU_TIMER			(_ULCAST_(1) << 11)
+#define CPU_IPI				(_ULCAST_(1) << 12)
+
+/* Controlled by 0x52 guest exception VIP
+ * aligned to exst bit 5~12
+ */
+#define CPU_IP0				(_ULCAST_(1))
+#define CPU_IP1				(_ULCAST_(1) << 1)
+#define CPU_IP2				(_ULCAST_(1) << 2)
+#define CPU_IP3				(_ULCAST_(1) << 3)
+#define CPU_IP4				(_ULCAST_(1) << 4)
+#define CPU_IP5				(_ULCAST_(1) << 5)
+#define CPU_IP6				(_ULCAST_(1) << 6)
+#define CPU_IP7				(_ULCAST_(1) << 7)
+
+#define MNSEC_PER_SEC			(NSEC_PER_SEC >> 20)
+
+/* KVM_IRQ_LINE irq field index values */
+#define KVM_LOONGSON_IRQ_TYPE_SHIFT		24
+#define KVM_LOONGSON_IRQ_TYPE_MASK		0xff
+#define KVM_LOONGSON_IRQ_VCPU_SHIFT		16
+#define KVM_LOONGSON_IRQ_VCPU_MASK		0xff
+#define KVM_LOONGSON_IRQ_NUM_SHIFT		0
+#define KVM_LOONGSON_IRQ_NUM_MASK		0xffff
+
+/* irq_type field */
+#define KVM_LOONGSON_IRQ_TYPE_CPU_IP		0
+#define KVM_LOONGSON_IRQ_TYPE_CPU_IO		1
+#define KVM_LOONGSON_IRQ_TYPE_HT		2
+#define KVM_LOONGSON_IRQ_TYPE_MSI		3
+#define KVM_LOONGSON_IRQ_TYPE_IOAPIC		4
+#define KVM_LOONGSON_IRQ_TYPE_ROUTE		5
+
+/* out-of-kernel GIC cpu interrupt injection irq_number field */
+#define KVM_LOONGSON_IRQ_CPU_IRQ		0
+#define KVM_LOONGSON_IRQ_CPU_FIQ		1
+#define KVM_LOONGSON_CPU_IP_NUM			8
+
+typedef union loongarch_instruction  larch_inst;
+typedef int (*exit_handle_fn)(struct kvm_vcpu *);
+
+int  _kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst);
+int  _kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst);
+int  _kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run);
+int  _kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run);
+int  _kvm_emu_idle(struct kvm_vcpu *vcpu);
+int  _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu);
+int  _kvm_pending_timer(struct kvm_vcpu *vcpu);
+int  _kvm_handle_fault(struct kvm_vcpu *vcpu, int fault);
+void _kvm_deliver_intr(struct kvm_vcpu *vcpu);
+void irqchip_debug_init(struct kvm *kvm);
+void irqchip_debug_destroy(struct kvm *kvm);
+
+void kvm_own_fpu(struct kvm_vcpu *vcpu);
+void kvm_own_lsx(struct kvm_vcpu *vcpu);
+void kvm_lose_fpu(struct kvm_vcpu *vcpu);
+void kvm_own_lasx(struct kvm_vcpu *vcpu);
+void kvm_save_fpu(struct kvm_vcpu *cpu);
+void kvm_restore_fpu(struct kvm_vcpu *cpu);
+void kvm_restore_fcsr(struct kvm_vcpu *cpu);
+void kvm_save_lsx(struct kvm_vcpu *cpu);
+void kvm_restore_lsx(struct kvm_vcpu *cpu);
+void kvm_restore_lsx_upper(struct kvm_vcpu *cpu);
+void kvm_save_lasx(struct kvm_vcpu *cpu);
+void kvm_restore_lasx(struct kvm_vcpu *cpu);
+void kvm_restore_lasx_upper(struct kvm_vcpu *cpu);
+
+void kvm_lose_hw_perf(struct kvm_vcpu *vcpu);
+void kvm_restore_hw_perf(struct kvm_vcpu *vcpu);
+
+void kvm_acquire_timer(struct kvm_vcpu *vcpu);
+void kvm_reset_timer(struct kvm_vcpu *vcpu);
+enum hrtimer_restart kvm_count_timeout(struct kvm_vcpu *vcpu);
+void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz);
+void kvm_restore_timer(struct kvm_vcpu *vcpu);
+void kvm_save_timer(struct kvm_vcpu *vcpu);
+
+/*
+ * Loongarch KVM guest interrupt handling.
+ */
+static inline void _kvm_queue_irq(struct kvm_vcpu *vcpu, unsigned int irq)
+{
+	set_bit(irq, &vcpu->arch.irq_pending);
+	clear_bit(irq, &vcpu->arch.irq_clear);
+}
+
+static inline void _kvm_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int irq)
+{
+	clear_bit(irq, &vcpu->arch.irq_pending);
+	set_bit(irq, &vcpu->arch.irq_clear);
+}
+
+#endif /* __ASM_LOONGARCH_KVMCPU_H__ */

arch/loongarch/kvm/kvmcsr.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef __LOONGARCH_KVM_CSR_H__
+#define __LOONGARCH_KVM_CSR_H__
+#include <asm/kvm_host.h>
+#include "kvmcpu.h"
+#include <linux/uaccess.h>
+#include <linux/kvm_host.h>
+
+#define kvm_read_hw_gcsr(id)			kvm_gcsr_read(id)
+#define kvm_write_hw_gcsr(csr, id, val)		kvm_gcsr_write(val, id)
+
+int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force);
+int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force);
+unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid);
+void _kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val);
+void _kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid,
+	unsigned long csr_mask, unsigned long val);
+int _kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu);
+
+static inline void kvm_save_hw_gcsr(struct loongarch_csrs *csr, u32 gid)
+{
+	csr->csrs[gid] = kvm_gcsr_read(gid);
+}
+
+static inline void kvm_restore_hw_gcsr(struct loongarch_csrs *csr, u32 gid)
+{
+	kvm_gcsr_write(csr->csrs[gid], gid);
+}
+
+static inline unsigned long kvm_read_sw_gcsr(struct loongarch_csrs *csr, u32 gid)
+{
+	return csr->csrs[gid];
+}
+
+static inline void kvm_write_sw_gcsr(struct loongarch_csrs *csr, u32 gid, unsigned long val)
+{
+	csr->csrs[gid] = val;
+}
+
+static inline void kvm_set_sw_gcsr(struct loongarch_csrs *csr, u32 gid, unsigned long val)
+{
+	csr->csrs[gid] |= val;
+}
+
+static inline void kvm_change_sw_gcsr(struct loongarch_csrs *csr, u32 gid, unsigned mask,
+	unsigned long val)
+{
+	unsigned long _mask = mask;
+	csr->csrs[gid] &= ~_mask;
+	csr->csrs[gid] |= val & _mask;
+}
+
+
+#define GET_HW_GCSR(id, csrid, v)				\
+	do {							\
+		if (csrid == id) {				\
+			*v = (long)kvm_read_hw_gcsr(csrid);	\
+			return 0;				\
+		}						\
+	} while (0)
+
+#define GET_SW_GCSR(csr, id, csrid, v)				\
+	do {							\
+		if (csrid == id) {				\
+			*v = kvm_read_sw_gcsr(csr, id);		\
+			return 0;				\
+		}						\
+	} while (0)
+
+#define SET_HW_GCSR(csr, id, csrid, v)				\
+	do {							\
+		if (csrid == id) {				\
+			kvm_write_hw_gcsr(csr, csrid, *v);	\
+			return 0;				\
+		}						\
+	} while (0)
+
+#define SET_SW_GCSR(csr, id, csrid, v)				\
+	do {							\
+		if (csrid == id) {				\
+			kvm_write_sw_gcsr(csr, csrid, *v);	\
+			return 0;				\
+		}						\
+	} while (0)
+
+int _kvm_init_iocsr(struct kvm *kvm);
+int _kvm_set_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp);
+int _kvm_get_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp);
+
+#define KVM_PMU_PLV_ENABLE      (KVM_PERFCTRL_PLV0 |            \
+					KVM_PERFCTRL_PLV1 |     \
+					KVM_PERFCTRL_PLV2 |     \
+					KVM_PERFCTRL_PLV3)
+
+#define CASE_WRITE_HW_PMU(vcpu, csr, id, csrid, v)                                              \
+	do {                                                                                    \
+		if (csrid == id) {                                                              \
+			if (v & KVM_PMU_PLV_ENABLE) {                                           \
+				kvm_write_csr_gcfg(kvm_read_csr_gcfg() | KVM_GCFG_GPERF);       \
+				kvm_write_hw_gcsr(csr, csrid, v | KVM_PERFCTRL_GMOD);           \
+				vcpu->arch.aux_inuse |= KVM_LARCH_PERF;                         \
+				return ;                                                        \
+			} else {                                                                \
+				kvm_write_sw_gcsr(csr, csrid, v);                               \
+				return;                                                         \
+			}                                                                       \
+		}                                                                               \
+	} while (0)
+
+#endif	/* __LOONGARCH_KVM_CSR_H__ */

arch/loongarch/kvm/loongarch.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/bitops.h>
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/kdebug.h>
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/vmalloc.h>
+#include <linux/sched/signal.h>
+#include <linux/fs.h>
+#include <linux/mod_devicetable.h>
+#include <linux/kvm.h>
+#include <linux/debugfs.h>
+#include <linux/pid.h>
+#include <linux/kvm_host.h>
+#include <linux/sched/stat.h>
+#include <asm/fpu.h>
+#include <asm/page.h>
+#include <asm/cacheflush.h>
+#include <asm/mmu_context.h>
+#include <asm/pgalloc.h>
+#include <asm/pgtable.h>
+#include <asm/cpufeature.h>
+#include "kvmcpu.h"
+#include <asm/setup.h>
+#include <asm/time.h>
+
+#include "intc/ls3a_ipi.h"
+#include "intc/ls7a_irq.h"
+#include "intc/ls3a_ext_irq.h"
+#include "kvm_compat.h"
+#include "kvmcsr.h"
+
+/*
+ * Define loongarch kvm version.
+ * Add version number when qemu/kvm interface changed
+ */
+#define KVM_LOONGARCH_VERSION 1
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+struct kvm_stats_debugfs_item vcpu_debugfs_entries[] = {
+	VCPU_STAT("idle", idle_exits),
+	VCPU_STAT("signal", signal_exits),
+	VCPU_STAT("interrupt", int_exits),
+	VCPU_STAT("rdcsr_cpu_feature", rdcsr_cpu_feature_exits),
+	VCPU_STAT("rdcsr_misc_func", rdcsr_misc_func_exits),
+	VCPU_STAT("rdcsr_ipi_access", rdcsr_ipi_access_exits),
+	VCPU_STAT("cpucfg", cpucfg_exits),
+	VCPU_STAT("huge_dec", huge_dec_exits),
+	VCPU_STAT("huge_thp", huge_thp_exits),
+	VCPU_STAT("huge_adj", huge_adjust_exits),
+	VCPU_STAT("huge_set", huge_set_exits),
+	VCPU_STAT("huge_merg", huge_merge_exits),
+	VCPU_STAT("halt_successful_poll", halt_successful_poll),
+	VCPU_STAT("halt_attempted_poll", halt_attempted_poll),
+	VCPU_STAT("halt_poll_invalid", halt_poll_invalid),
+	VCPU_STAT("halt_wakeup", halt_wakeup),
+	VCPU_STAT("tlbmiss_ld", excep_exits[KVM_EXCCODE_TLBL]),
+	VCPU_STAT("tlbmiss_st", excep_exits[KVM_EXCCODE_TLBS]),
+       	VCPU_STAT("tlb_ifetch", excep_exits[KVM_EXCCODE_TLBI]),	
+	VCPU_STAT("tlbmod", excep_exits[KVM_EXCCODE_TLBM]),	
+	VCPU_STAT("tlbri", excep_exits[KVM_EXCCODE_TLBRI]),	
+	VCPU_STAT("tlbxi", excep_exits[KVM_EXCCODE_TLBXI]),	
+	VCPU_STAT("fp_dis", excep_exits[KVM_EXCCODE_FPDIS]),
+	VCPU_STAT("lsx_dis", excep_exits[KVM_EXCCODE_LSXDIS]),
+	VCPU_STAT("lasx_dis", excep_exits[KVM_EXCCODE_LASXDIS]),
+	VCPU_STAT("fpe", excep_exits[KVM_EXCCODE_FPE]),	
+	VCPU_STAT("watch", excep_exits[KVM_EXCCODE_WATCH]),	
+	VCPU_STAT("gspr", excep_exits[KVM_EXCCODE_GSPR]),	
+	VCPU_STAT("gcm", excep_exits[KVM_EXCCODE_GCM]),	
+	VCPU_STAT("hc", excep_exits[KVM_EXCCODE_HYP]),	
+	{NULL}
+};
+
+struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VM_STAT("remote_tlb_flush", remote_tlb_flush),
+	VM_STAT("pip_read_exits", pip_read_exits),
+	VM_STAT("pip_write_exits", pip_write_exits),
+	VM_STAT("vm_ioctl_irq_line", vm_ioctl_irq_line),
+	VM_STAT("ls7a_ioapic_update", ls7a_ioapic_update),
+	VM_STAT("ls7a_ioapic_set_irq", ls7a_ioapic_set_irq),
+	VM_STAT("ls7a_msi_irq", ls7a_msi_irq),
+	VM_STAT("ioapic_reg_write", ioapic_reg_write),
+	VM_STAT("ioapic_reg_read", ioapic_reg_read),
+	VM_STAT("set_ls7a_ioapic", set_ls7a_ioapic),
+	VM_STAT("get_ls7a_ioapic", get_ls7a_ioapic),
+	VM_STAT("set_ls3a_ext_irq", set_ls3a_ext_irq),
+	VM_STAT("get_ls3a_ext_irq", get_ls3a_ext_irq),
+	VM_STAT("ls3a_ext_irq", trigger_ls3a_ext_irq),
+	{NULL}
+};
+
+bool kvm_trace_guest_mode_change;
+static struct kvm_context __percpu *vmcs;
+
+int kvm_guest_mode_change_trace_reg(void)
+{
+	kvm_trace_guest_mode_change = 1;
+	return 0;
+}
+
+void kvm_guest_mode_change_trace_unreg(void)
+{
+	kvm_trace_guest_mode_change = 0;
+}
+
+/*
+ * XXXKYMA: We are simulatoring a processor that has the WII bit set in
+ * Config7, so we are "runnable" if interrupts are pending
+ */
+int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
+{
+	return !!(vcpu->arch.irq_pending);
+}
+
+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+
+int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
+{
+	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
+}
+
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+void kvm_update_stolen_time(struct kvm_vcpu *vcpu)
+{
+	struct kvm_host_map map;
+	struct kvm_steal_time *st;
+	int ret = 0;
+
+	if (vcpu->arch.st.guest_addr == 0)
+		return;
+
+	ret = kvm_map_gfn(vcpu, vcpu->arch.st.guest_addr >> PAGE_SHIFT,
+				&map, &vcpu->arch.st.cache, false);
+	if (ret) {
+		kvm_info("%s ret:%d\n", __func__, ret);
+		return;
+	}
+	st = map.hva + offset_in_page(vcpu->arch.st.guest_addr);
+	if (st->version & 1)
+		st->version += 1; /* first time write, random junk */
+	st->version += 1;
+	smp_wmb();
+	st->steal += current->sched_info.run_delay -
+		vcpu->arch.st.last_steal;
+	vcpu->arch.st.last_steal = current->sched_info.run_delay;
+	smp_wmb();
+	st->version += 1;
+
+	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false);
+}
+
+bool _kvm_pvtime_supported(void)
+{
+	return !!sched_info_on();
+}
+
+int _kvm_pvtime_set_attr(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr)
+{
+	u64 __user *user = (u64 __user *)attr->addr;
+	struct kvm *kvm = vcpu->kvm;
+	u64 ipa;
+	int ret = 0;
+	int idx;
+
+	if (!_kvm_pvtime_supported() ||
+		attr->attr != KVM_LARCH_VCPU_PVTIME_IPA)
+		return -ENXIO;
+
+	if (get_user(ipa, user))
+		return -EFAULT;
+	if (!IS_ALIGNED(ipa, 64))
+		return -EINVAL;
+
+	/* Check the address is in a valid memslot */
+	idx = srcu_read_lock(&kvm->srcu);
+	if (kvm_is_error_hva(gfn_to_hva(kvm, ipa >> PAGE_SHIFT)))
+		ret = -EINVAL;
+	srcu_read_unlock(&kvm->srcu, idx);
+
+	if (!ret)
+		vcpu->arch.st.guest_addr = ipa;
+
+	return ret;
+}
+
+int _kvm_pvtime_get_attr(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr)
+{
+	u64 __user *user = (u64 __user *)attr->addr;
+	u64 ipa;
+
+	if (!_kvm_pvtime_supported() ||
+		attr->attr != KVM_LARCH_VCPU_PVTIME_IPA)
+		return -ENXIO;
+
+	ipa = vcpu->arch.st.guest_addr;
+
+	if (put_user(ipa, user))
+		return -EFAULT;
+
+	return 0;
+}
+
+int _kvm_pvtime_has_attr(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr)
+{
+	switch (attr->attr) {
+	case KVM_LARCH_VCPU_PVTIME_IPA:
+		if (_kvm_pvtime_supported())
+			return 0;
+	}
+
+	return -ENXIO;
+}
+#endif
+
+int kvm_arch_hardware_enable(void)
+{
+	unsigned long gcfg = 0;
+
+	/* First init gtlbc, gcfg, gstat, gintc. All guest use the same config */
+	kvm_clear_csr_gtlbc(KVM_GTLBC_USETGID | KVM_GTLBC_TOTI);
+	kvm_write_csr_gcfg(0);
+	kvm_write_csr_gstat(0);
+	kvm_write_csr_gintc(0);
+
+	/*
+	 * Enable virtualization features granting guest direct control of
+	 * certain features:
+	 * GCI=2:       Trap on init or unimplement cache instruction.
+	 * TORU=0:      Trap on Root Unimplement.
+	 * CACTRL=1:    Root control cache.
+	 * TOP=0:       Trap on Previlege.
+	 * TOE=0:       Trap on Exception.
+	 * TIT=0:       Trap on Timer.
+	 */
+	gcfg |= KVM_GCFG_GCI_SECURE;
+	gcfg |= KVM_GCFG_MATC_ROOT;
+	gcfg |= KVM_GCFG_TIT;
+	kvm_write_csr_gcfg(gcfg);
+	kvm_flush_tlb_all();
+
+	/* Enable using TGID  */
+	kvm_set_csr_gtlbc(KVM_GTLBC_USETGID);
+	kvm_debug("gtlbc:%llx gintc:%llx gstat:%llx gcfg:%llx",
+			kvm_read_csr_gtlbc(), kvm_read_csr_gintc(),
+			kvm_read_csr_gstat(), kvm_read_csr_gcfg());
+	return 0;
+}
+
+void kvm_arch_hardware_disable(void)
+{
+	kvm_clear_csr_gtlbc(KVM_GTLBC_USETGID | KVM_GTLBC_TOTI);
+	kvm_write_csr_gcfg(0);
+	kvm_write_csr_gstat(0);
+	kvm_write_csr_gintc(0);
+
+	/* Flush any remaining guest TLB entries */
+	kvm_flush_tlb_all();
+}
+
+int kvm_arch_hardware_setup(void *opaque)
+{
+	return 0;
+}
+
+int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
+{
+	/* Allocate page table to map GPA -> RPA */
+	kvm->arch.gpa_mm.pgd = kvm_pgd_alloc();
+	if (!kvm->arch.gpa_mm.pgd)
+		return -ENOMEM;
+
+	kvm->arch.cpucfg_lasx = (read_cpucfg(LOONGARCH_CPUCFG2) &
+					  CPUCFG2_LASX);
+
+	_kvm_init_iocsr(kvm);
+	kvm->arch.vmcs = vmcs;
+
+	return 0;
+}
+
+static void kvm_free_vcpus(struct kvm *kvm)
+{
+	unsigned int i;
+	struct kvm_vcpu *vcpu;
+
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		kvm_vcpu_destroy(vcpu);
+		kvm->vcpus[i] = NULL;
+	}
+
+	atomic_set(&kvm->online_vcpus, 0);
+}
+
+void kvm_arch_destroy_vm(struct kvm *kvm)
+{
+	kvm_destroy_ls3a_ipi(kvm);
+	kvm_destroy_ls7a_ioapic(kvm);
+	kvm_destroy_ls3a_ext_irq(kvm);
+	kvm_free_vcpus(kvm);
+	_kvm_destroy_mm(kvm);
+}
+
+long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl,
+			unsigned long arg)
+{
+	return -ENOIOCTLCMD;
+}
+
+int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
+			    unsigned long npages)
+{
+	return 0;
+}
+
+int kvm_arch_prepare_memory_region(struct kvm *kvm,
+				   struct kvm_memory_slot *memslot,
+				   const struct kvm_userspace_memory_region *mem,
+				   enum kvm_mr_change change)
+{
+	return 0;
+}
+
+static void _kvm_new_vpid(unsigned long cpu, struct kvm_vcpu *vcpu)
+{
+	struct kvm_context *context;
+	unsigned long vpid;
+
+	context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
+	vpid = context->vpid_cache;
+	if (!(++vpid & context->gid_mask)) {
+		if (!vpid)              /* fix version if needed */
+			vpid = context->gid_fisrt_ver;
+
+		++vpid;         /* vpid 0 reserved for root */
+
+		/* start new vpid cycle */
+		kvm_flush_tlb_all();
+	}
+
+	context->vpid_cache = vpid;
+	vcpu->arch.vpid[cpu] = vpid;
+}
+
+/* Returns 1 if the guest TLB may be clobbered */
+static int _kvm_check_requests(struct kvm_vcpu *vcpu, int cpu)
+{
+	int ret = 0;
+	int i;
+
+	if (!kvm_request_pending(vcpu))
+		return 0;
+
+	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
+		/* Drop all vpids for this VCPU */
+		for_each_possible_cpu(i)
+			vcpu->arch.vpid[i] = 0;
+		/* This will clobber guest TLB contents too */
+		ret = 1;
+	}
+
+	return ret;
+}
+
+static void _kvm_update_vmid(struct kvm_vcpu *vcpu, int cpu)
+{
+	struct kvm_context *context;
+	bool migrated;
+	unsigned int gstinfo_gidmask, gstinfo_gid = 0;
+
+	/*
+	 * Are we entering guest context on a different CPU to last time?
+	 * If so, the VCPU's guest TLB state on this CPU may be stale.
+	 */
+	context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
+	migrated = (vcpu->arch.last_exec_cpu != cpu);
+	vcpu->arch.last_exec_cpu = cpu;
+
+	/*
+	 * Check if our vpid is of an older version and thus invalid.
+	 *
+	 * We also discard the stored vpid if we've executed on
+	 * another CPU, as the guest mappings may have changed without
+	 * hypervisor knowledge.
+	 */
+	gstinfo_gidmask = context->gid_mask << KVM_GSTAT_GID_SHIFT;
+	if (migrated ||
+			(vcpu->arch.vpid[cpu] ^ context->vpid_cache) &
+			context->gid_ver_mask) {
+		_kvm_new_vpid(cpu, vcpu);
+		trace_kvm_vpid_change(vcpu, vcpu->arch.vpid[cpu]);
+	}
+	gstinfo_gid = (vcpu->arch.vpid[cpu] & context->gid_mask) <<
+		KVM_GSTAT_GID_SHIFT;
+
+	/* Restore GSTAT(0x50).vpid */
+	kvm_change_csr_gstat(gstinfo_gidmask, gstinfo_gid);
+}
+
+/*
+ * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
+ */
+static int _kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
+{
+	unsigned long exst = vcpu->arch.host_estat;
+	u32 intr = exst & 0x1fff; /* ignore NMI */
+	u32 exccode = (exst & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT;
+	u32 __user *opc = (u32 __user *) vcpu->arch.pc;
+	int ret = RESUME_GUEST, cpu;
+
+	vcpu->mode = OUTSIDE_GUEST_MODE;
+
+	/* Set a default exit reason */
+	run->exit_reason = KVM_EXIT_UNKNOWN;
+	run->ready_for_interrupt_injection = 1;
+
+	/*
+	 * Set the appropriate status bits based on host CPU features,
+	 * before we hit the scheduler
+	 */
+
+	local_irq_enable();
+
+	kvm_debug("%s: exst: %lx, PC: %p, kvm_run: %p, kvm_vcpu: %p\n",
+			__func__, exst, opc, run, vcpu);
+	trace_kvm_exit(vcpu, exccode);
+	if (exccode) {
+		vcpu->stat.excep_exits[exccode]++;
+		ret = _kvm_handle_fault(vcpu, exccode);
+	} else {
+		WARN(!intr, "suspicious vm exiting");
+		++vcpu->stat.int_exits;
+
+		if (need_resched())
+			cond_resched();
+
+		ret = RESUME_GUEST;
+	}
+
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
+		kvm_update_stolen_time(vcpu);
+#endif
+
+	cond_resched();
+
+	local_irq_disable();
+
+	if (ret == RESUME_GUEST)
+		kvm_acquire_timer(vcpu);
+
+	if (!(ret & RESUME_HOST)) {
+		_kvm_deliver_intr(vcpu);
+		/* Only check for signals if not already exiting to userspace */
+		if (signal_pending(current)) {
+			run->exit_reason = KVM_EXIT_INTR;
+			ret = (-EINTR << 2) | RESUME_HOST;
+			++vcpu->stat.signal_exits;
+			trace_kvm_exit(vcpu, KVM_TRACE_EXIT_SIGNAL);
+		}
+	}
+
+	if (ret == RESUME_GUEST) {
+		trace_kvm_reenter(vcpu);
+
+		/*
+		 * Make sure the read of VCPU requests in vcpu_reenter()
+		 * callback is not reordered ahead of the write to vcpu->mode,
+		 * or we could miss a TLB flush request while the requester sees
+		 * the VCPU as outside of guest mode and not needing an IPI.
+		 */
+		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
+
+		cpu = smp_processor_id();
+		_kvm_check_requests(vcpu, cpu);
+		_kvm_update_vmid(vcpu, cpu);
+
+		/*
+		 * If FPU / LSX are enabled (i.e. the guest's FPU / LSX context
+		 * is live), restore FCSR0.
+		 */
+		if (_kvm_guest_has_fpu(&vcpu->arch) &&
+		    kvm_read_csr_euen() & (KVM_EUEN_FPEN | KVM_EUEN_LSXEN)) {
+			kvm_restore_fcsr(vcpu);
+		}
+	}
+
+	return ret;
+}
+
+/* low level hrtimer wake routine */
+static enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer)
+{
+	struct kvm_vcpu *vcpu;
+
+	vcpu = container_of(timer, struct kvm_vcpu, arch.swtimer);
+	_kvm_queue_irq(vcpu, LARCH_INT_TIMER);
+	kvm_vcpu_wake_up(vcpu);
+	return kvm_count_timeout(vcpu);
+}
+
+static void _kvm_vcpu_init(struct kvm_vcpu *vcpu)
+{
+	int i;
+
+	for_each_possible_cpu(i)
+		vcpu->arch.vpid[i] = 0;
+
+	hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC,
+			HRTIMER_MODE_ABS_PINNED);
+	vcpu->arch.swtimer.function = kvm_swtimer_wakeup;
+	vcpu->arch.fpu_enabled = true;
+	vcpu->arch.lsx_enabled = true;
+}
+
+int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
+{
+	vcpu->arch.host_eentry = kvm_csr_readq(KVM_CSR_EENTRY);
+	vcpu->arch.guest_eentry = (unsigned long)kvm_exception_entry;
+	vcpu->arch.vcpu_run = kvm_enter_guest;
+	vcpu->arch.handle_exit = _kvm_handle_exit;
+	vcpu->arch.host_ecfg = (kvm_read_csr_ecfg() & KVM_ECFG_VS);
+
+	/*
+	 * kvm all exceptions share one exception entry, and host <-> guest switch
+	 * also switch excfg.VS field, keep host excfg.VS info here
+	 */
+	vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL);
+	if (!vcpu->arch.csr) {
+		return -ENOMEM;
+	}
+
+	/* Init */
+	vcpu->arch.last_sched_cpu = -1;
+	vcpu->arch.last_exec_cpu = -1;
+	_kvm_vcpu_init(vcpu);
+	return 0;
+}
+
+static void _kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
+{
+	int cpu;
+	struct kvm_context *context;
+
+	/*
+	 * If the VCPU is freed and reused as another VCPU, we don't want the
+	 * matching pointer wrongly hanging around in last_vcpu.
+	 */
+	for_each_possible_cpu(cpu) {
+		context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
+		if (context->last_vcpu == vcpu)
+			context->last_vcpu = NULL;
+	}
+}
+
+void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
+{
+	struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache;
+
+	_kvm_vcpu_uninit(vcpu);
+
+	hrtimer_cancel(&vcpu->arch.swtimer);
+	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
+	if (vcpu->arch.st.guest_addr)
+		kvm_release_pfn(cache->pfn, cache->dirty, cache);
+	kfree(vcpu->arch.csr);
+}
+#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \
+		KVM_GUESTDBG_USE_SW_BP | KVM_GUESTDBG_SINGLESTEP)
+int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
+					struct kvm_guest_debug *dbg)
+{
+	int ret = 0;
+
+	if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) {
+		ret = -EINVAL;
+		goto out;
+	}
+	if (dbg->control & KVM_GUESTDBG_ENABLE) {
+		vcpu->guest_debug = dbg->control;
+		/* No hardware breakpoint */
+	} else {
+		vcpu->guest_debug = 0;
+	}
+out:
+	return ret;
+}
+
+int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
+{
+	int r = -EINTR;
+	int cpu;
+	struct kvm_run *run = vcpu->run;
+
+	vcpu_load(vcpu);
+
+	kvm_sigset_activate(vcpu);
+
+	if (vcpu->mmio_needed) {
+		if (!vcpu->mmio_is_write)
+			_kvm_complete_mmio_read(vcpu, run);
+		vcpu->mmio_needed = 0;
+	} else if (vcpu->arch.is_hypcall) {
+		/* set return value for hypercall v0 register */
+		vcpu->arch.gprs[KVM_REG_A0] = run->hypercall.ret;
+		vcpu->arch.is_hypcall = 0;
+	}
+
+	if (run->exit_reason == KVM_EXIT_LOONGARCH_IOCSR) {
+		if (!run->iocsr_io.is_write)
+			_kvm_complete_iocsr_read(vcpu, run);
+	}
+
+	/* clear exit_reason */
+	run->exit_reason = KVM_EXIT_UNKNOWN;
+	if (run->immediate_exit)
+		goto out;
+
+	lose_fpu(1);
+
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
+		kvm_update_stolen_time(vcpu);
+#endif
+	local_irq_disable();
+	guest_enter_irqoff();
+	trace_kvm_enter(vcpu);
+
+	/*
+	 * Make sure the read of VCPU requests in vcpu_run() callback is not
+	 * reordered ahead of the write to vcpu->mode, or we could miss a TLB
+	 * flush request while the requester sees the VCPU as outside of guest
+	 * mode and not needing an IPI.
+	 */
+	smp_store_mb(vcpu->mode, IN_GUEST_MODE);
+
+	cpu = smp_processor_id();
+	kvm_acquire_timer(vcpu);
+	/* Check if we have any exceptions/interrupts pending */
+	_kvm_deliver_intr(vcpu);
+
+	_kvm_check_requests(vcpu, cpu);
+	_kvm_update_vmid(vcpu, cpu);
+	r = kvm_enter_guest(run, vcpu);
+
+	trace_kvm_out(vcpu);
+	guest_exit_irqoff();
+	local_irq_enable();
+
+out:
+	kvm_sigset_deactivate(vcpu);
+
+	vcpu_put(vcpu);
+	return r;
+}
+
+int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
+			     struct kvm_loongarch_interrupt *irq)
+{
+	int intr = (int)irq->irq;
+
+	if (intr < 0) {
+		_kvm_dequeue_irq(vcpu, -intr);
+		return 0;
+	}
+
+	_kvm_queue_irq(vcpu, intr);
+	kvm_vcpu_kick(vcpu);
+	return 0;
+}
+
+int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
+				    struct kvm_mp_state *mp_state)
+{
+	return -ENOIOCTLCMD;
+}
+
+int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
+				    struct kvm_mp_state *mp_state)
+{
+	return -ENOIOCTLCMD;
+}
+
+/**
+ * kvm_migrate_count() - Migrate timer.
+ * @vcpu:       Virtual CPU.
+ *
+ * Migrate hrtimer to the current CPU by cancelling and restarting it
+ * if it was running prior to being cancelled.
+ *
+ * Must be called when the VCPU is migrated to a different CPU to ensure that
+ * timer expiry during guest execution interrupts the guest and causes the
+ * interrupt to be delivered in a timely manner.
+ */
+static void kvm_migrate_count(struct kvm_vcpu *vcpu)
+{
+	if (hrtimer_cancel(&vcpu->arch.swtimer))
+		hrtimer_restart(&vcpu->arch.swtimer);
+}
+
+static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
+{
+	struct kvm_context *context;
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	bool migrated, all;
+
+	/*
+	 * Have we migrated to a different CPU?
+	 * If so, any old guest TLB state may be stale.
+	 */
+	migrated = (vcpu->arch.last_sched_cpu != cpu);
+
+	/*
+	 * Was this the last VCPU to run on this CPU?
+	 * If not, any old guest state from this VCPU will have been clobbered.
+	 */
+	context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
+	all = migrated || (context->last_vcpu != vcpu);
+	context->last_vcpu = vcpu;
+
+	/*
+	 * Restore timer state regardless
+	 */
+	kvm_restore_timer(vcpu);
+
+	/* Control guest page CCA attribute */
+	kvm_change_csr_gcfg(KVM_GCFG_MATC_MASK, KVM_GCFG_MATC_ROOT);
+	/* Restore hardware perf csr */
+	kvm_restore_hw_perf(vcpu);
+
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
+#endif
+	/* Don't bother restoring registers multiple times unless necessary */
+	if (!all)
+		return 0;
+
+	kvm_write_csr_gcntc((ulong)vcpu->kvm->arch.stablecounter_gftoffset);
+	/*
+	 * Restore guest CSR registers
+	 */
+	kvm_restore_hw_gcsr(csr, KVM_CSR_CRMD);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_PRMD);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_EUEN);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_MISC);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_ECFG);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_ERA);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_BADV);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_BADI);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_EENTRY);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBIDX);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBEHI);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBELO0);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBELO1);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_ASID);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_PGDL);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_PGDH);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_PWCTL0);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_PWCTL1);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_STLBPGSIZE);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_RVACFG);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_CPUID);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS0);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS1);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS2);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS3);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS4);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS5);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS6);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_KS7);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TMID);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_CNTC);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRENTRY);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRBADV);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRERA);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRSAVE);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRELO0);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRELO1);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBREHI);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRPRMD);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN0);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN1);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN2);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN3);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_LLBCTL);
+
+	/* restore Root.Guestexcept from unused Guest guestexcept register */
+	kvm_write_csr_gintc(csr->csrs[KVM_CSR_GINTC]);
+
+	/*
+	 * We should clear linked load bit to break interrupted atomics. This
+	 * prevents a SC on the next VCPU from succeeding by matching a LL on
+	 * the previous VCPU.
+	 */
+	if (vcpu->kvm->created_vcpus > 1)
+		kvm_set_gcsr_llbctl(KVM_LLBCTL_WCLLB);
+
+	return 0;
+}
+
+/* Restore ASID once we are scheduled back after preemption */
+void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	vcpu->cpu = cpu;
+	if (vcpu->arch.last_sched_cpu != cpu) {
+		kvm_debug("[%d->%d]KVM VCPU[%d] switch\n",
+				vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id);
+		/*
+		 * Migrate the timer interrupt to the current CPU so that it
+		 * always interrupts the guest and synchronously triggers a
+		 * guest timer interrupt.
+		 */
+		kvm_migrate_count(vcpu);
+	}
+
+	/* restore guest state to registers */
+	_kvm_vcpu_load(vcpu, cpu);
+	local_irq_restore(flags);
+}
+
+static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+
+	kvm_lose_fpu(vcpu);
+	kvm_lose_hw_perf(vcpu);
+
+	kvm_save_hw_gcsr(csr, KVM_CSR_CRMD);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PRMD);
+	kvm_save_hw_gcsr(csr, KVM_CSR_EUEN);
+	kvm_save_hw_gcsr(csr, KVM_CSR_MISC);
+	kvm_save_hw_gcsr(csr, KVM_CSR_ECFG);
+	kvm_save_hw_gcsr(csr, KVM_CSR_ERA);
+	kvm_save_hw_gcsr(csr, KVM_CSR_BADV);
+	kvm_save_hw_gcsr(csr, KVM_CSR_BADI);
+	kvm_save_hw_gcsr(csr, KVM_CSR_EENTRY);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBIDX);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBEHI);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBELO0);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBELO1);
+	kvm_save_hw_gcsr(csr, KVM_CSR_ASID);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PGDL);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PGDH);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PGD);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PWCTL0);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PWCTL1);
+	kvm_save_hw_gcsr(csr, KVM_CSR_STLBPGSIZE);
+	kvm_save_hw_gcsr(csr, KVM_CSR_RVACFG);
+	kvm_save_hw_gcsr(csr, KVM_CSR_CPUID);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG1);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG2);
+	kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG3);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS0);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS1);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS2);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS3);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS4);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS5);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS6);
+	kvm_save_hw_gcsr(csr, KVM_CSR_KS7);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TMID);
+	kvm_save_hw_gcsr(csr, KVM_CSR_CNTC);
+	kvm_save_hw_gcsr(csr, KVM_CSR_LLBCTL);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBRENTRY);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBRBADV);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBRERA);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBRSAVE);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBRELO0);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBRELO1);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBREHI);
+	kvm_save_hw_gcsr(csr, KVM_CSR_TLBRPRMD);
+	kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN0);
+	kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN1);
+	kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN2);
+	kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN3);
+
+	/* save Root.Guestexcept in unused Guest guestexcept register */
+	kvm_save_timer(vcpu);
+	csr->csrs[KVM_CSR_GINTC] = kvm_read_csr_gintc();
+	return 0;
+}
+
+/* ASID can change if another task is scheduled during preemption */
+void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
+{
+	unsigned long flags;
+	int cpu;
+
+	local_irq_save(flags);
+	cpu = smp_processor_id();
+	vcpu->arch.last_sched_cpu = cpu;
+	vcpu->cpu = -1;
+
+	/* save guest state in registers */
+	_kvm_vcpu_put(vcpu, cpu);
+	local_irq_restore(flags);
+}
+
+static int _kvm_get_one_reg(struct kvm_vcpu *vcpu,
+		const struct kvm_one_reg *reg, s64 *v)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	int reg_idx, ret;
+
+	if ((reg->id & KVM_IOC_CSRID(0)) == KVM_IOC_CSRID(0)) {
+		reg_idx = KVM_GET_IOC_CSRIDX(reg->id);
+		ret = _kvm_getcsr(vcpu, reg_idx, v, 0);
+		if (ret == 0)
+			return ret;
+	}
+
+	switch (reg->id) {
+	case KVM_REG_LOONGARCH_COUNTER:
+		*v = drdtime() + vcpu->kvm->arch.stablecounter_gftoffset;
+		break;
+	default:
+		if ((reg->id & KVM_REG_LOONGARCH_MASK) != KVM_REG_LOONGARCH_CSR)
+			return -EINVAL;
+
+		reg_idx = KVM_GET_IOC_CSRIDX(reg->id);
+		if (reg_idx < CSR_ALL_SIZE)
+			*v = kvm_read_sw_gcsr(csr, reg_idx);
+		else
+			return -EINVAL;
+	}
+	return 0;
+}
+
+static int _kvm_set_one_reg(struct kvm_vcpu *vcpu,
+		const struct kvm_one_reg *reg,
+		s64 v)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	int ret = 0;
+	unsigned long flags;
+	u64 val;
+	int reg_idx;
+
+	val = v;
+	if ((reg->id & KVM_IOC_CSRID(0)) == KVM_IOC_CSRID(0)) {
+		reg_idx = KVM_GET_IOC_CSRIDX(reg->id);
+		ret = _kvm_setcsr(vcpu, reg_idx, &val, 0);
+		if (ret == 0)
+			return ret;
+	}
+
+	switch (reg->id) {
+	case KVM_REG_LOONGARCH_COUNTER:
+		local_irq_save(flags);
+		/*
+		 * gftoffset is relative with board, not vcpu
+		 * only set for the first time for smp system
+		 */
+		if (vcpu->vcpu_id == 0)
+			vcpu->kvm->arch.stablecounter_gftoffset = (signed long)(v - drdtime());
+		kvm_write_csr_gcntc((ulong)vcpu->kvm->arch.stablecounter_gftoffset);
+		local_irq_restore(flags);
+		break;
+	case KVM_REG_LOONGARCH_VCPU_RESET:
+		kvm_reset_timer(vcpu);
+		if (vcpu->vcpu_id == 0)
+			kvm_enable_ls3a_extirq(vcpu->kvm, false);
+		memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending));
+		memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear));
+
+		/* disable pv timer when cpu resetting */
+		vcpu->arch.st.guest_addr = 0;
+		break;
+	default:
+		if ((reg->id & KVM_REG_LOONGARCH_MASK) != KVM_REG_LOONGARCH_CSR)
+			return -EINVAL;
+
+		reg_idx = KVM_GET_IOC_CSRIDX(reg->id);
+		if (reg_idx < CSR_ALL_SIZE)
+			kvm_write_sw_gcsr(csr, reg_idx, v);
+		else
+			return -EINVAL;
+	}
+	return ret;
+}
+
+static int _kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+	int ret;
+	s64 v;
+
+	ret = _kvm_get_one_reg(vcpu, reg, &v);
+	if (ret)
+		return ret;
+
+	ret = -EINVAL;
+	if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
+		u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
+
+		ret = put_user(v, uaddr64);
+	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) {
+		u32 __user *uaddr32 = (u32 __user *)(long)reg->addr;
+		u32 v32 = (u32)v;
+
+		ret = put_user(v32, uaddr32);
+	}
+
+	return ret;
+}
+
+static int _kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+	s64 v;
+	int ret;
+
+	ret = -EINVAL;
+	if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
+		u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
+		ret = get_user(v, uaddr64);
+	} else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) {
+		u32 __user *uaddr32 = (u32 __user *)(long)reg->addr;
+		s32 v32;
+
+		ret = get_user(v32, uaddr32);
+		v = (s64)v32;
+	}
+
+	if (ret)
+		return -EFAULT;
+
+	return _kvm_set_one_reg(vcpu, reg, v);
+}
+
+static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
+				     struct kvm_enable_cap *cap)
+{
+	int r = 0;
+
+	if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap))
+		return -EINVAL;
+	if (cap->flags)
+		return -EINVAL;
+	if (cap->args[0])
+		return -EINVAL;
+
+	switch (cap->cap) {
+	case KVM_CAP_LOONGARCH_FPU:
+	case KVM_CAP_LOONGARCH_LSX:
+		break;
+	default:
+		r = -EINVAL;
+		break;
+	}
+
+	return r;
+}
+
+long kvm_arch_vcpu_async_ioctl(struct file *filp, unsigned int ioctl,
+			       unsigned long arg)
+{
+	struct kvm_vcpu *vcpu = filp->private_data;
+	void __user *argp = (void __user *)arg;
+
+	if (ioctl == KVM_INTERRUPT) {
+		struct kvm_loongarch_interrupt irq;
+
+		if (copy_from_user(&irq, argp, sizeof(irq)))
+			return -EFAULT;
+		kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__,
+			  irq.irq);
+
+		return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
+	}
+
+	return -ENOIOCTLCMD;
+}
+
+int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
+			  bool line_status)
+{
+	u32 irq = irq_level->irq;
+	unsigned int irq_type, vcpu_idx, irq_num, ret;
+	int nrcpus = atomic_read(&kvm->online_vcpus);
+	bool level = irq_level->level;
+	unsigned long flags;
+
+	irq_type = (irq >> KVM_LOONGSON_IRQ_TYPE_SHIFT) & KVM_LOONGSON_IRQ_TYPE_MASK;
+	vcpu_idx = (irq >> KVM_LOONGSON_IRQ_VCPU_SHIFT) & KVM_LOONGSON_IRQ_VCPU_MASK;
+	irq_num = (irq >> KVM_LOONGSON_IRQ_NUM_SHIFT) & KVM_LOONGSON_IRQ_NUM_MASK;
+
+	switch (irq_type) {
+	case KVM_LOONGSON_IRQ_TYPE_IOAPIC:
+		if (!ls7a_ioapic_in_kernel(kvm))
+			return -ENXIO;
+
+		if (vcpu_idx >= nrcpus)
+			return -EINVAL;
+
+		ls7a_ioapic_lock(ls7a_ioapic_irqchip(kvm), &flags);
+		ret = kvm_ls7a_ioapic_set_irq(kvm, irq_num, level);
+		ls7a_ioapic_unlock(ls7a_ioapic_irqchip(kvm), &flags);
+		return ret;
+	}
+	kvm->stat.vm_ioctl_irq_line++;
+
+	return -EINVAL;
+}
+
+static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct loongarch_kvm_irqchip *chip)
+{
+	int r, dlen;
+
+	r = 0;
+	dlen = chip->len - sizeof(struct loongarch_kvm_irqchip);
+	switch (chip->chip_id) {
+	case KVM_IRQCHIP_LS7A_IOAPIC:
+		if (dlen != sizeof(struct kvm_ls7a_ioapic_state)) {
+			kvm_err("get ls7a state err dlen:%d\n", dlen);
+			goto dlen_err;
+		}
+		r = kvm_get_ls7a_ioapic(kvm, (void *)chip->data);
+		break;
+	case KVM_IRQCHIP_LS3A_GIPI:
+		if (dlen != sizeof(gipiState)) {
+			kvm_err("get gipi state err dlen:%d\n", dlen);
+			goto dlen_err;
+		}
+		r = kvm_get_ls3a_ipi(kvm, (void *)chip->data);
+		break;
+	case KVM_IRQCHIP_LS3A_HT_IRQ:
+	case KVM_IRQCHIP_LS3A_ROUTE:
+		break;
+	case KVM_IRQCHIP_LS3A_EXTIRQ:
+		if (dlen != sizeof(struct kvm_loongarch_ls3a_extirq_state)) {
+			kvm_err("get extioi state err dlen:%d\n", dlen);
+			goto dlen_err;
+		}
+		r = kvm_get_ls3a_extirq(kvm, (void *)chip->data);
+		break;
+	case KVM_IRQCHIP_LS3A_IPMASK:
+		break;
+	default:
+		r = -EINVAL;
+		break;
+	}
+	return r;
+dlen_err:
+	r = -EINVAL;
+	return r;
+}
+
+static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct loongarch_kvm_irqchip *chip)
+{
+	int r, dlen;
+
+	r = 0;
+	dlen = chip->len - sizeof(struct loongarch_kvm_irqchip);
+	switch (chip->chip_id) {
+	case KVM_IRQCHIP_LS7A_IOAPIC:
+		if (dlen != sizeof(struct kvm_ls7a_ioapic_state)) {
+			kvm_err("set ls7a state err dlen:%d\n", dlen);
+			goto dlen_err;
+		}
+		r = kvm_set_ls7a_ioapic(kvm, (void *)chip->data);
+		break;
+	case KVM_IRQCHIP_LS3A_GIPI:
+		if (dlen != sizeof(gipiState)) {
+			kvm_err("set gipi state err dlen:%d\n", dlen);
+			goto dlen_err;
+		}
+		r = kvm_set_ls3a_ipi(kvm, (void *)chip->data);
+		break;
+	case KVM_IRQCHIP_LS3A_HT_IRQ:
+	case KVM_IRQCHIP_LS3A_ROUTE:
+		break;
+	case KVM_IRQCHIP_LS3A_EXTIRQ:
+		if (dlen != sizeof(struct kvm_loongarch_ls3a_extirq_state)) {
+			kvm_err("set extioi state err dlen:%d\n", dlen);
+			goto dlen_err;
+		}
+		r = kvm_set_ls3a_extirq(kvm, (void *)chip->data);
+		break;
+	case KVM_IRQCHIP_LS3A_IPMASK:
+		break;
+	default:
+		r = -EINVAL;
+		break;
+	}
+	return r;
+dlen_err:
+	r = -EINVAL;
+	return r;
+}
+
+/*
+ * Read or write a bunch of msrs. All parameters are kernel addresses.
+ *
+ * @return number of msrs set successfully.
+ */
+static int _kvm_csr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
+		struct kvm_csr_entry *entries,
+		int (*do_csr)(struct kvm_vcpu *vcpu,
+			unsigned index, u64 *data, int force))
+{
+	int i;
+
+	for (i = 0; i < msrs->ncsrs; ++i)
+		if (do_csr(vcpu, entries[i].index, &entries[i].data, 1))
+			break;
+
+	return i;
+}
+
+static int kvm_csr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
+		int (*do_csr)(struct kvm_vcpu *vcpu,
+			unsigned index, u64 *data, int force))
+{
+	struct kvm_msrs msrs;
+	struct kvm_csr_entry *entries;
+	int r, n;
+	unsigned size;
+
+	r = -EFAULT;
+	if (copy_from_user(&msrs, user_msrs, sizeof msrs))
+		goto out;
+
+	r = -E2BIG;
+	if (msrs.ncsrs >= CSR_ALL_SIZE)
+		goto out;
+
+	size = sizeof(struct kvm_csr_entry) * msrs.ncsrs;
+	entries = memdup_user(user_msrs->entries, size);
+	if (IS_ERR(entries)) {
+		r = PTR_ERR(entries);
+		goto out;
+	}
+
+	r = n = _kvm_csr_io(vcpu, &msrs, entries, do_csr);
+	if (r < 0)
+		goto out_free;
+
+	r = -EFAULT;
+	if (copy_to_user(user_msrs->entries, entries, size))
+		goto out_free;
+
+	r = n;
+
+out_free:
+	kfree(entries);
+out:
+	return r;
+}
+
+static int _kvm_vcpu_set_attr(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr)
+{
+	int ret = -ENXIO;
+
+	switch (attr->group) {
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	case KVM_LARCH_VCPU_PVTIME_CTRL:
+		ret = _kvm_pvtime_set_attr(vcpu, attr);
+		break;
+#endif
+	default:
+		ret = -ENXIO;
+		break;
+	}
+
+	return ret;
+}
+
+static int _kvm_vcpu_get_attr(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr)
+{
+	int ret = -ENXIO;
+
+	switch (attr->group) {
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	case KVM_LARCH_VCPU_PVTIME_CTRL:
+		ret = _kvm_pvtime_get_attr(vcpu, attr);
+		break;
+#endif
+	default:
+		ret = -ENXIO;
+		break;
+	}
+
+	return ret;
+}
+
+static int _kvm_vcpu_has_attr(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr)
+{
+	int ret = -ENXIO;
+
+	switch (attr->group) {
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	case KVM_LARCH_VCPU_PVTIME_CTRL:
+		ret = _kvm_pvtime_has_attr(vcpu, attr);
+		break;
+#endif
+	default:
+		ret = -ENXIO;
+		break;
+	}
+
+	return ret;
+}
+
+long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
+			 unsigned long arg)
+{
+	struct kvm_vcpu *vcpu = filp->private_data;
+	void __user *argp = (void __user *)arg;
+	struct kvm_device_attr attr;
+	long r;
+
+	vcpu_load(vcpu);
+
+	switch (ioctl) {
+	case KVM_SET_ONE_REG:
+	case KVM_GET_ONE_REG: {
+		struct kvm_one_reg reg;
+
+		r = -EFAULT;
+		if (copy_from_user(&reg, argp, sizeof(reg)))
+			break;
+		if (ioctl == KVM_SET_ONE_REG)
+			r = _kvm_set_reg(vcpu, &reg);
+		else
+			r = _kvm_get_reg(vcpu, &reg);
+		break;
+	}
+	case KVM_ENABLE_CAP: {
+		struct kvm_enable_cap cap;
+
+		r = -EFAULT;
+		if (copy_from_user(&cap, argp, sizeof(cap)))
+			break;
+		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
+		break;
+	}
+	case KVM_CHECK_EXTENSION: {
+		unsigned int ext;
+		if (copy_from_user(&ext, argp, sizeof(ext)))
+			return -EFAULT;
+		switch (ext) {
+		case KVM_CAP_LOONGARCH_FPU:
+			r = !!cpu_has_fpu;
+			break;
+		case KVM_CAP_LOONGARCH_LSX:
+			r = !!cpu_has_lsx;
+			break;
+		default:
+			break;
+		}
+		break;
+	}
+
+	case KVM_LOONGARCH_GET_VCPU_STATE:
+	{
+		int i;
+		struct  kvm_loongarch_vcpu_state vcpu_state;
+		r = -EFAULT;
+
+		vcpu_state.online_vcpus = vcpu->kvm->arch.online_vcpus;
+		vcpu_state.is_migrate = 1;
+		for (i = 0; i < 4; i++)
+			vcpu_state.core_ext_ioisr[i] = vcpu->arch.core_ext_ioisr[i];
+
+		vcpu_state.irq_pending =  vcpu->arch.irq_pending;
+		vcpu_state.irq_clear =  vcpu->arch.irq_clear;
+
+		if (copy_to_user(argp, &vcpu_state, sizeof(struct kvm_loongarch_vcpu_state)))
+			break;
+		r = 0;
+		break;
+	}
+
+	case KVM_LOONGARCH_SET_VCPU_STATE:
+	{
+		int i;
+		struct  kvm_loongarch_vcpu_state vcpu_state;
+		r = -EFAULT;
+
+		if (copy_from_user(&vcpu_state, argp, sizeof(struct kvm_loongarch_vcpu_state)))
+			return -EFAULT;
+
+		vcpu->kvm->arch.online_vcpus = vcpu_state.online_vcpus;
+		vcpu->kvm->arch.is_migrate = vcpu_state.is_migrate;
+		for (i = 0; i < 4; i++)
+			 vcpu->arch.core_ext_ioisr[i] = vcpu_state.core_ext_ioisr[i];
+
+		vcpu->arch.irq_pending = vcpu_state.irq_pending;
+		vcpu->arch.irq_clear = vcpu_state.irq_clear;
+		r = 0;
+		break;
+	}
+	case KVM_GET_MSRS: {
+		r = kvm_csr_io(vcpu, argp, _kvm_getcsr);
+		break;
+	}
+	case KVM_SET_MSRS: {
+		r = kvm_csr_io(vcpu, argp, _kvm_setcsr);
+		break;
+	}
+	case KVM_SET_DEVICE_ATTR: {
+		r = -EFAULT;
+		if (copy_from_user(&attr, argp, sizeof(attr)))
+			break;
+		r = _kvm_vcpu_set_attr(vcpu, &attr);
+		break;
+	}
+	case KVM_GET_DEVICE_ATTR: {
+		r = -EFAULT;
+		if (copy_from_user(&attr, argp, sizeof(attr)))
+			break;
+		r = _kvm_vcpu_get_attr(vcpu, &attr);
+		break;
+	}
+	case KVM_HAS_DEVICE_ATTR: {
+		r = -EFAULT;
+		if (copy_from_user(&attr, argp, sizeof(attr)))
+			break;
+		r = _kvm_vcpu_has_attr(vcpu, &attr);
+		break;
+	}
+	default:
+		r = -ENOIOCTLCMD;
+	}
+
+	vcpu_put(vcpu);
+	return r;
+}
+
+long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
+{
+	struct kvm *kvm = filp->private_data;
+	void __user *argp = (void __user *)arg;
+	long r;
+
+	switch (ioctl) {
+	case KVM_CREATE_IRQCHIP:
+	{
+		mutex_lock(&kvm->lock);
+		r = -EEXIST;
+		if (kvm->arch.v_ioapic)
+			goto create_irqchip_unlock;
+
+		r = kvm_create_ls7a_ioapic(kvm);
+		if (r < 0)
+			goto create_irqchip_unlock;
+		r = kvm_create_ls3a_ipi(kvm);
+		if (r < 0) {
+			mutex_lock(&kvm->slots_lock);
+			kvm_destroy_ls7a_ioapic(kvm);
+			mutex_unlock(&kvm->slots_lock);
+			goto create_irqchip_unlock;
+		}
+		r = kvm_create_ls3a_ext_irq(kvm);
+		if (r < 0) {
+			mutex_lock(&kvm->slots_lock);
+			kvm_destroy_ls3a_ipi(kvm);
+			kvm_destroy_ls7a_ioapic(kvm);
+			mutex_unlock(&kvm->slots_lock);
+		}
+		irqchip_debug_init(kvm);
+		/* Write kvm->irq_routing before kvm->arch.vpic.  */
+		smp_wmb();
+create_irqchip_unlock:
+		mutex_unlock(&kvm->lock);
+		break;
+	}
+	case KVM_GET_IRQCHIP: {
+		struct loongarch_kvm_irqchip *kchip;
+		struct loongarch_kvm_irqchip uchip;
+		if (copy_from_user(&uchip, argp, sizeof(struct loongarch_kvm_irqchip)))
+			goto out;
+		kchip = memdup_user(argp, uchip.len);
+		if (IS_ERR(kchip)) {
+			r = PTR_ERR(kchip);
+			goto out;
+		}
+
+		r = -ENXIO;
+		if (!ls7a_ioapic_in_kernel(kvm))
+			goto get_irqchip_out;
+		r = kvm_vm_ioctl_get_irqchip(kvm, kchip);
+		if (r)
+			goto get_irqchip_out;
+		if (copy_to_user(argp, kchip, kchip->len))
+			goto get_irqchip_out;
+		r = 0;
+get_irqchip_out:
+		kfree(kchip);
+		break;
+	}
+	case KVM_SET_IRQCHIP: {
+		struct loongarch_kvm_irqchip *kchip;
+		struct loongarch_kvm_irqchip uchip;
+		if (copy_from_user(&uchip, argp, sizeof(struct loongarch_kvm_irqchip)))
+			goto out;
+
+		kchip = memdup_user(argp, uchip.len);
+		if (IS_ERR(kchip)) {
+			r = PTR_ERR(kchip);
+			goto out;
+		}
+
+		r = -ENXIO;
+		if (!ls7a_ioapic_in_kernel(kvm))
+			goto set_irqchip_out;
+		r = kvm_vm_ioctl_set_irqchip(kvm, kchip);
+		if (r)
+			goto set_irqchip_out;
+		r = 0;
+set_irqchip_out:
+		kfree(kchip);
+		break;
+	}
+	case KVM_LOONGARCH_GET_IOCSR:
+	{
+		r = _kvm_get_iocsr(kvm, argp);
+		break;
+	}
+	case KVM_LOONGARCH_SET_IOCSR:
+	{
+		r = _kvm_set_iocsr(kvm, argp);
+		break;
+	}
+	case KVM_LOONGARCH_SET_CPUCFG:
+	{
+		r = 0;
+		if (copy_from_user(&kvm->arch.cpucfgs, argp, sizeof(struct kvm_cpucfg)))
+			r = -EFAULT;
+		break;
+	}
+	case KVM_LOONGARCH_GET_CPUCFG:
+	{
+		r = 0;
+		if (copy_to_user(argp, &kvm->arch.cpucfgs, sizeof(struct kvm_cpucfg)))
+		   r = -EFAULT;
+		break;
+	}
+	default:
+		r = -ENOIOCTLCMD;
+	}
+out:
+
+	return r;
+}
+
+int kvm_arch_init(void *opaque)
+{
+	struct kvm_context *context;
+	unsigned long vpid_mask;
+	int cpu;
+
+	vmcs = alloc_percpu(struct kvm_context);
+	if (!vmcs) {
+		printk(KERN_ERR "kvm: failed to allocate percpu kvm_context\n");
+		return -ENOMEM;
+	}
+
+	vpid_mask = kvm_read_csr_gstat();
+	vpid_mask = (vpid_mask & KVM_GSTAT_GIDBIT) >> KVM_GSTAT_GIDBIT_SHIFT;
+	if (vpid_mask)
+		vpid_mask = GENMASK(vpid_mask - 1, 0);
+
+	for_each_possible_cpu(cpu) {
+		context = per_cpu_ptr(vmcs, cpu);
+		context->gid_mask = vpid_mask;
+		context->gid_ver_mask = ~context->gid_mask;
+		context->gid_fisrt_ver = context->gid_mask + 1;
+		context->vpid_cache = context->gid_mask + 1;
+		context->last_vcpu = NULL;
+	}
+
+	_kvm_init_fault();
+	return 0;
+}
+
+void kvm_arch_exit(void)
+{
+	free_percpu(vmcs);
+}
+
+int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
+				  struct kvm_sregs *sregs)
+{
+	return -ENOIOCTLCMD;
+}
+
+int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
+				  struct kvm_sregs *sregs)
+{
+	return -ENOIOCTLCMD;
+}
+
+void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
+{
+}
+
+int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
+{
+	int i = 0;
+
+	/* no need vcpu_load and vcpu_put */
+	fpu->fcsr = vcpu->arch.fpu.fcsr;
+	fpu->fcc = vcpu->arch.fpu.fcc;
+	for (i = 0; i < NUM_FPU_REGS; i++)
+		memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64);
+
+	return 0;
+}
+
+int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
+{
+	int i = 0;
+
+	/* no need vcpu_load and vcpu_put */
+	vcpu->arch.fpu.fcsr = fpu->fcsr;
+	vcpu->arch.fpu.fcc = fpu->fcc;
+	for (i = 0; i < NUM_FPU_REGS; i++)
+		memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64);
+
+	return 0;
+}
+
+vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
+{
+	return VM_FAULT_SIGBUS;
+}
+
+int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
+{
+	int r;
+
+	switch (ext) {
+	case KVM_CAP_ONE_REG:
+	case KVM_CAP_ENABLE_CAP:
+	case KVM_CAP_READONLY_MEM:
+	case KVM_CAP_SYNC_MMU:
+#ifdef CONFIG_HAVE_LS_KVM_MSI
+	case KVM_CAP_SIGNAL_MSI:
+#endif
+	case KVM_CAP_IMMEDIATE_EXIT:
+		r = 1;
+		break;
+	case KVM_CAP_NR_VCPUS:
+		r = num_online_cpus();
+		break;
+	case KVM_CAP_MAX_VCPUS:
+		r = KVM_MAX_VCPUS;
+		break;
+	case KVM_CAP_MAX_VCPU_ID:
+		r = KVM_MAX_VCPU_ID;
+		break;
+	case KVM_CAP_NR_MEMSLOTS:
+		r = KVM_USER_MEM_SLOTS;
+		break;
+	case KVM_CAP_LOONGARCH_FPU:
+		/* We don't handle systems with inconsistent cpu_has_fpu */
+		r = !!cpu_has_fpu;
+		break;
+	case KVM_CAP_LOONGARCH_LSX:
+		/*
+		 * We don't support LSX vector partitioning yet:
+		 * 1) It would require explicit support which can't be tested
+		 *    yet due to lack of support in current hardware.
+		 * 2) It extends the state that would need to be saved/restored
+		 *    by e.g. QEMU for migration.
+		 *
+		 * When vector partitioning hardware becomes available, support
+		 * could be added by requiring a flag when enabling
+		 * KVM_CAP_LOONGARCH_LSX capability to indicate that userland knows
+		 * to save/restore the appropriate extra state.
+		 */
+		r = cpu_has_lsx;
+		break;
+	case KVM_CAP_IRQCHIP:
+	case KVM_CAP_IOEVENTFD:
+		/* we wouldn't be here unless cpu_has_lvz */
+		r = 1;
+		break;
+	case KVM_CAP_LOONGARCH_VZ:
+		/* get user defined kvm version */
+		r = KVM_LOONGARCH_VERSION;
+		break;
+	default:
+		r = 0;
+		break;
+	}
+	return r;
+}
+
+int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
+{
+	return _kvm_pending_timer(vcpu) ||
+		kvm_read_hw_gcsr(KVM_CSR_ESTAT) &
+			(1 << (KVM_INT_TIMER - KVM_INT_START));
+}
+
+int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu)
+{
+	int i;
+	struct loongarch_csrs *csr;
+
+	if (!vcpu)
+		return -1;
+
+	kvm_debug("VCPU Register Dump:\n");
+	kvm_debug("\tpc = 0x%08lx\n", vcpu->arch.pc);
+	kvm_debug("\texceptions: %08lx\n", vcpu->arch.irq_pending);
+
+	for (i = 0; i < 32; i += 4) {
+		kvm_debug("\tgpr%02d: %08lx %08lx %08lx %08lx\n", i,
+		       vcpu->arch.gprs[i],
+		       vcpu->arch.gprs[i + 1],
+		       vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]);
+	}
+
+	csr = vcpu->arch.csr;
+	kvm_debug("\tCRMOD: 0x%08llx, exst: 0x%08llx\n",
+		  kvm_read_hw_gcsr(KVM_CSR_CRMD),
+		  kvm_read_hw_gcsr(KVM_CSR_ESTAT));
+
+	kvm_debug("\tERA: 0x%08llx\n", kvm_read_hw_gcsr(KVM_CSR_ERA));
+
+	return 0;
+}
+
+int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
+{
+	int i;
+
+	vcpu_load(vcpu);
+
+	for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
+		vcpu->arch.gprs[i] = regs->gpr[i];
+	vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */
+	vcpu->arch.pc = regs->pc;
+
+	vcpu_put(vcpu);
+	return 0;
+}
+
+int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
+{
+	int i;
+
+	vcpu_load(vcpu);
+
+	for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
+		regs->gpr[i] = vcpu->arch.gprs[i];
+
+	regs->pc = vcpu->arch.pc;
+
+	vcpu_put(vcpu);
+	return 0;
+}
+
+int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
+				  struct kvm_translation *tr)
+{
+	return 0;
+}
+
+/* Initial guest state */
+int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	unsigned long timer_hz;
+
+	/*
+	 * Initialize guest register state to valid architectural reset state.
+	 */
+	timer_hz = calc_const_freq();
+	kvm_init_timer(vcpu, timer_hz);
+
+	/* Set Initialize mode for GUEST */
+	kvm_write_sw_gcsr(csr, KVM_CSR_CRMD, KVM_CRMD_DA);
+
+	/* Set cpuid */
+	kvm_write_sw_gcsr(csr, KVM_CSR_TMID, vcpu->vcpu_id);
+
+	/* start with no pending virtual guest interrupts */
+	csr->csrs[KVM_CSR_GINTC] = 0;
+	return 0;
+}
+
+/* Enable FPU for guest and restore context */
+void kvm_own_fpu(struct kvm_vcpu *vcpu)
+{
+	unsigned long sr;
+
+	preempt_disable();
+
+	sr = kvm_read_hw_gcsr(KVM_CSR_EUEN);
+
+	/*
+	 * If LSX state is already live, it is undefined how it interacts with
+	 * FR=0 FPU state, and we don't want to hit reserved instruction
+	 * exceptions trying to save the LSX state later when CU=1 && FR=1, so
+	 * play it safe and save it first.
+	 *
+	 * In theory we shouldn't ever hit this case since kvm_lose_fpu() should
+	 * get called when guest CU1 is set, however we can't trust the guest
+	 * not to clobber the status register directly via the commpage.
+	 */
+	if (cpu_has_lsx && sr & KVM_EUEN_FPEN &&
+	    vcpu->arch.aux_inuse & (KVM_LARCH_LSX | KVM_LARCH_LASX))
+		kvm_lose_fpu(vcpu);
+
+	/*
+	 * Enable FPU for guest
+	 * We set FR and FRE according to guest context
+	 */
+	kvm_set_csr_euen(KVM_EUEN_FPEN);
+
+	/* If guest FPU state not active, restore it now */
+	if (!(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) {
+		kvm_restore_fpu(vcpu);
+		vcpu->arch.aux_inuse |= KVM_LARCH_FPU;
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU);
+	} else {
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_FPU);
+	}
+
+	preempt_enable();
+}
+
+#ifdef CONFIG_CPU_HAS_LSX
+/* Enable LSX for guest and restore context */
+void kvm_own_lsx(struct kvm_vcpu *vcpu)
+{
+	preempt_disable();
+
+	/*
+	 * Enable FP if enabled in guest, since we're restoring FP context
+	 * anyway.
+	 */
+	if (_kvm_guest_has_fpu(&vcpu->arch)) {
+
+		kvm_set_csr_euen(KVM_EUEN_FPEN);
+	}
+
+	/* Enable LSX for guest */
+	kvm_set_csr_euen(KVM_EUEN_LSXEN);
+
+	switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU |
+			 KVM_LARCH_LSX | KVM_LARCH_LASX)) {
+		case KVM_LARCH_FPU:
+			/*
+			 * Guest FPU state already loaded,
+			 * only restore upper LSX state
+			 */
+			kvm_restore_lsx_upper(vcpu);
+			vcpu->arch.aux_inuse |= KVM_LARCH_LSX;
+			trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE,
+						KVM_TRACE_AUX_LSX);
+			break;
+		case 0:
+			/* Neither FP or LSX already active,
+			 * restore full LSX state
+			 */
+			kvm_restore_lsx(vcpu);
+			vcpu->arch.aux_inuse |= KVM_LARCH_LSX;
+			if (_kvm_guest_has_fpu(&vcpu->arch))
+				vcpu->arch.aux_inuse |= KVM_LARCH_FPU;
+			trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE,
+					KVM_TRACE_AUX_FPU_LSX);
+		break;
+	default:
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_LSX);
+		break;
+	}
+
+	preempt_enable();
+}
+#endif
+
+#ifdef CONFIG_CPU_HAS_LASX
+/* Enable LASX for guest and restore context */
+void kvm_own_lasx(struct kvm_vcpu *vcpu)
+{
+	preempt_disable();
+
+	/*
+	 * Enable FP if enabled in guest, since we're restoring FP context
+	 * anyway.
+	 */
+	if (_kvm_guest_has_lsx(&vcpu->arch)) {
+		/* Enable LSX for guest */
+		kvm_set_csr_euen(KVM_EUEN_LSXEN);
+	}
+
+	/*
+	 * Enable FPU if enabled in guest, since we're restoring FPU context
+	 * anyway. We set FR and FRE according to guest context.
+	 */
+	if (_kvm_guest_has_fpu(&vcpu->arch)) {
+		kvm_set_csr_euen(KVM_EUEN_FPEN);
+	}
+
+	/* Enable LASX for guest */
+	kvm_set_csr_euen(KVM_EUEN_LASXEN);
+
+	switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU |
+			 KVM_LARCH_LSX | KVM_LARCH_LASX)) {
+	case (KVM_LARCH_LSX | KVM_LARCH_FPU):
+	case KVM_LARCH_LSX:
+		/*
+		 * Guest LSX state already loaded, only restore upper LASX state
+		 */
+		kvm_restore_lasx_upper(vcpu);
+		vcpu->arch.aux_inuse |= KVM_LARCH_LASX;
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX);
+		break;
+	case KVM_LARCH_FPU:
+		/*
+		 * Guest FP state already loaded, only restore 64~256 LASX state
+		 */
+		kvm_restore_lsx_upper(vcpu);
+		kvm_restore_lasx_upper(vcpu);
+		vcpu->arch.aux_inuse |= KVM_LARCH_LASX;
+		if (_kvm_guest_has_lsx(&vcpu->arch))
+			vcpu->arch.aux_inuse |= KVM_LARCH_LSX;
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX);
+		break;
+	case 0:
+		/* Neither FP or LSX already active, restore full LASX state */
+		kvm_restore_lasx(vcpu);
+		vcpu->arch.aux_inuse |= KVM_LARCH_LASX;
+		if (_kvm_guest_has_lsx(&vcpu->arch))
+			vcpu->arch.aux_inuse |= KVM_LARCH_LSX;
+		if (_kvm_guest_has_fpu(&vcpu->arch))
+			vcpu->arch.aux_inuse |= KVM_LARCH_FPU;
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE,
+			      KVM_TRACE_AUX_FPU_LSX_LASX);
+		break;
+	default:
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_LASX);
+		break;
+	}
+
+	preempt_enable();
+}
+#endif
+
+/* Save and disable FPU & LSX & LASX */
+void kvm_lose_fpu(struct kvm_vcpu *vcpu)
+{
+	preempt_disable();
+	if (cpu_has_lasx && (vcpu->arch.aux_inuse & KVM_LARCH_LASX)) {
+
+#ifdef CONFIG_CPU_HAS_LASX
+		kvm_save_lasx(vcpu);
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX_LASX);
+
+		/* Disable LASX & MAS & FPU */
+		disable_lasx();
+		disable_lsx();
+#endif
+
+		if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
+			kvm_clear_csr_euen(KVM_EUEN_FPEN);
+		}
+		vcpu->arch.aux_inuse &= ~(KVM_LARCH_FPU |
+					 KVM_LARCH_LSX | KVM_LARCH_LASX);
+	} else if (cpu_has_lsx && vcpu->arch.aux_inuse & KVM_LARCH_LSX) {
+
+#ifdef CONFIG_CPU_HAS_LASX
+		kvm_save_lsx(vcpu);
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX);
+
+		/* Disable LSX & FPU */
+		disable_lsx();
+#endif
+
+		if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
+			kvm_clear_csr_euen(KVM_EUEN_FPEN);
+		}
+		vcpu->arch.aux_inuse &= ~(KVM_LARCH_FPU | KVM_LARCH_LSX);
+	} else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
+
+		kvm_save_fpu(vcpu);
+		vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU;
+		trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU);
+
+		/* Disable FPU */
+		kvm_clear_csr_euen(KVM_EUEN_FPEN);
+	}
+	preempt_enable();
+}
+
+void kvm_lose_hw_perf(struct kvm_vcpu *vcpu)
+{
+	if (vcpu->arch.aux_inuse & KVM_LARCH_PERF) {
+		struct loongarch_csrs *csr = vcpu->arch.csr;
+		/* save guest pmu csr */
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL0);
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR0);
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL1);
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR1);
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL2);
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR2);
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL3);
+		kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR3);
+		if (((kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL0) |
+			kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL1) |
+			kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL2) |
+			kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL3))
+			& KVM_PMU_PLV_ENABLE) == 0)
+			vcpu->arch.aux_inuse &= ~KVM_LARCH_PERF;
+		/* config host pmu csr */
+		kvm_write_csr_gcfg(kvm_read_csr_gcfg() & ~KVM_GCFG_GPERF);
+		/* TODO: pmu csr used by host and guest at the same time */
+		kvm_write_csr_perfctrl0(0);
+		kvm_write_csr_perfcntr0(0);
+		kvm_write_csr_perfctrl1(0);
+		kvm_write_csr_perfcntr1(0);
+		kvm_write_csr_perfctrl2(0);
+		kvm_write_csr_perfcntr2(0);
+		kvm_write_csr_perfctrl3(0);
+		kvm_write_csr_perfcntr3(0);
+	}
+}
+
+void kvm_restore_hw_perf(struct kvm_vcpu *vcpu)
+{
+	if (vcpu->arch.aux_inuse & KVM_LARCH_PERF) {
+		struct loongarch_csrs *csr = vcpu->arch.csr;
+		/* enable guest pmu */
+		kvm_write_csr_gcfg(kvm_read_csr_gcfg() | KVM_GCFG_GPERF);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL0);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR0);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL1);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR1);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL2);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR2);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL3);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR3);
+	}
+}
+
+static int __init kvm_loongarch_init(void)
+{
+	int ret;
+
+	if (!cpu_has_lvz)
+		return  0;
+
+	ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
+
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void __exit kvm_loongarch_exit(void)
+{
+	kvm_exit();
+}
+
+module_init(kvm_loongarch_init);
+module_exit(kvm_loongarch_exit);
+
+static const struct cpu_feature loongarch_kvm_feature[] = {
+	{ .feature = cpu_feature(LOONGARCH_LVZ) },
+	{},
+};
+MODULE_DEVICE_TABLE(cpu, loongarch_kvm_feature);
+
+EXPORT_TRACEPOINT_SYMBOL(kvm_exit);

arch/loongarch/kvm/mmu.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/highmem.h>
+#include <linux/hugetlb.h>
+#include <linux/page-flags.h>
+#include <linux/kvm_host.h>
+#include <linux/uaccess.h>
+#include <asm/mmu_context.h>
+#include <asm/pgalloc.h>
+#include <asm/tlb.h>
+#include "kvm_compat.h"
+
+/*
+ * KVM_MMU_CACHE_MIN_PAGES is the number of GPA page table translation levels
+ * for which pages need to be cached.
+ */
+#if defined(__PAGETABLE_PMD_FOLDED)
+#define KVM_MMU_CACHE_MIN_PAGES 1
+#else
+#define KVM_MMU_CACHE_MIN_PAGES 2
+#endif
+
+static int kvm_tlb_flush_gpa(struct kvm_vcpu *vcpu, unsigned long gpa)
+{
+	preempt_disable();
+	gpa &= (PAGE_MASK << 1);
+	invtlb(INVTLB_GID_ADDR, kvm_read_csr_gstat() & KVM_GSTAT_GID, gpa);
+	preempt_enable();
+	return 0;
+}
+
+static inline int kvm_pmd_huge(pmd_t pmd)
+{
+#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT
+	return (pmd_val(pmd) & _PAGE_HUGE) != 0;
+#else
+	return 0;
+#endif	
+}
+
+static inline int kvm_pud_huge(pud_t pud)
+{
+#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT
+	return (pud_val(pud) & _PAGE_HUGE) != 0;
+#else
+	return 0;
+#endif	
+}
+
+static inline pmd_t kvm_pmd_mkhuge(pmd_t pmd)
+{
+#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT
+#ifdef	CONFIG_TRANSPARENT_HUGEPAGE
+	return pmd_mkhuge(pmd);
+#else
+	pte_t entry;
+
+	pte_val(entry) = pmd_val(pmd);
+	entry = pte_mkhuge(entry);
+	pmd_val(pmd) = pte_val(entry);
+#endif
+#endif
+	return pmd;
+}
+
+static inline pmd_t kvm_pmd_mkclean(pmd_t pmd)
+{
+#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT
+#ifdef	CONFIG_TRANSPARENT_HUGEPAGE
+	return pmd_mkclean(pmd);
+#else
+	pte_t entry;
+
+	pte_val(entry) = pmd_val(pmd);
+	entry = pte_mkclean(entry);
+	pmd_val(pmd) = pte_val(entry);
+#endif
+#endif
+	return pmd;
+}
+
+static inline pmd_t kvm_pmd_mkold(pmd_t pmd)
+{
+#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT
+#ifdef	CONFIG_TRANSPARENT_HUGEPAGE
+	return pmd_mkold(pmd);
+#else
+	pte_t entry;
+
+	pte_val(entry) = pmd_val(pmd);
+	entry = pte_mkold(entry);
+	pmd_val(pmd) = pte_val(entry);
+#endif
+#endif
+	return pmd;
+}
+
+/**
+ * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory.
+ *
+ * Allocate a blank KVM GPA page directory (PGD) for representing guest physical
+ * to host physical page mappings.
+ *
+ * Returns:	Pointer to new KVM GPA page directory.
+ *		NULL on allocation failure.
+ */
+pgd_t *kvm_pgd_alloc(void)
+{
+	pgd_t *ret;
+	struct page *page;
+
+	page = alloc_pages(GFP_KERNEL, 0);
+	if (!page)
+		return NULL;
+	ret = (pgd_t *) page_address(page);
+	if (ret)
+		pgd_init(ret);
+
+	return ret;
+}
+
+/**
+ * kvm_walk_pgd() - Walk page table with optional allocation.
+ * @pgd:	Page directory pointer.
+ * @addr:	Address to index page table using.
+ * @cache:	MMU page cache to allocate new page tables from, or NULL.
+ *
+ * Walk the page tables pointed to by @pgd to find the PTE corresponding to the
+ * address @addr. If page tables don't exist for @addr, they will be created
+ * from the MMU cache if @cache is not NULL.
+ *
+ * Returns:	Pointer to pte_t corresponding to @addr.
+ *		NULL if a page table doesn't exist for @addr and !@cache.
+ *		NULL if a page table allocation failed.
+ */
+static pte_t *kvm_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache,
+				struct vm_area_struct *vma, unsigned long hva,
+				unsigned long addr)
+{
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+
+	pgd += pgd_index(addr);
+	if (pgd_none(*pgd)) {
+		/* Not used yet */
+		BUG();
+		return NULL;
+	}
+	p4d = p4d_offset(pgd, addr);
+	pud = pud_offset(p4d, addr);
+	if (pud_none(*pud)) {
+		pmd_t *new_pmd;
+
+		if (!cache)
+			return NULL;
+		new_pmd = kvm_mmu_memory_cache_alloc(cache);
+		pmd_init(new_pmd);
+		pud_populate(NULL, pud, new_pmd);
+	}
+	pmd = pmd_offset(pud, addr);
+	if (kvm_pmd_huge(*pmd)) {
+		return (pte_t *)pmd;
+	}
+	if (pmd_none(*pmd)) {
+		pte_t *new_pte;
+
+		if (!cache)
+			return NULL;
+		new_pte = kvm_mmu_memory_cache_alloc(cache);
+		clear_page(new_pte);
+		pmd_populate_kernel(NULL, pmd, new_pte);
+	}
+	return pte_offset_kernel(pmd, addr);
+}
+
+/* Caller must hold kvm->mm_lock */
+static pte_t *kvm_pte_for_gpa(struct kvm *kvm,
+				struct kvm_mmu_memory_cache *cache,
+				struct vm_area_struct *vma, unsigned long hva,
+				unsigned long addr)
+{
+	return kvm_walk_pgd(kvm->arch.gpa_mm.pgd, cache, vma, hva, addr);
+}
+
+#define kvm_pte_for_gpa_fast(kvm, gpa)	kvm_pte_for_gpa(kvm, NULL, NULL, 0, gpa)
+/*
+ * kvm_flush_gpa_{pte,pmd,pud,pgd,pt}.
+ * Flush a range of guest physical address space from the VM's GPA page tables.
+ */
+
+static bool kvm_flush_gpa_pte(pte_t *pte, unsigned long start_gpa,
+				   unsigned long end_gpa, unsigned long *data)
+{
+	int i_min = pte_index(start_gpa);
+	int i_max = pte_index(end_gpa);
+	bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i) {
+		if (!pte_present(pte[i]))
+			continue;
+
+		set_pte(pte + i, __pte(0));
+		if (data)
+			*data = *data + 1;
+	}
+	return safe_to_remove;
+}
+
+static bool kvm_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa,
+				   unsigned long end_gpa, unsigned long *data)
+{
+	pte_t *pte;
+	unsigned long end = ~0ul;
+	int i_min = pmd_index(start_gpa);
+	int i_max = pmd_index(end_gpa);
+	bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
+		if (!pmd_present(pmd[i]))
+			continue;
+
+		if (kvm_pmd_huge(pmd[i]) && pmd_present(pmd[i])) {
+			pmd_clear(pmd + i);
+			if (data)
+				*data += PTRS_PER_PMD;
+			continue;
+		}
+
+		pte = pte_offset_kernel(pmd + i, 0);
+		if (i == i_max)
+			end = end_gpa;
+
+		if (kvm_flush_gpa_pte(pte, start_gpa, end, data)) {
+			pmd_clear(pmd + i);
+			pte_free_kernel(NULL, pte);
+		} else {
+			safe_to_remove = false;
+		}
+	}
+	return safe_to_remove;
+}
+
+static bool kvm_flush_gpa_pud(pud_t *pud, unsigned long start_gpa,
+				   unsigned long end_gpa, unsigned long *data)
+{
+	pmd_t *pmd;
+	unsigned long end = ~0ul;
+	int i_min = pud_index(start_gpa);
+	int i_max = pud_index(end_gpa);
+	bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
+		if (!pud_present(pud[i]))
+			continue;
+
+		pmd = pmd_offset(pud + i, 0);
+		if (i == i_max)
+			end = end_gpa;
+
+		if (kvm_flush_gpa_pmd(pmd, start_gpa, end, data)) {
+			pud_clear(pud + i);
+			pmd_free(NULL, pmd);
+		} else {
+			safe_to_remove = false;
+		}
+	}
+	return safe_to_remove;
+}
+
+static bool kvm_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa,
+				unsigned long end_gpa, unsigned long *data)
+{
+	p4d_t *p4d;
+	pud_t *pud;
+	unsigned long end = ~0ul;
+	int i_min = pgd_index(start_gpa);
+	int i_max = pgd_index(end_gpa);
+	bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
+		if (!pgd_present(pgd[i]))
+			continue;
+
+		p4d = p4d_offset(pgd + i, 0);
+		pud = pud_offset(p4d, 0);
+		if (i == i_max)
+			end = end_gpa;
+
+		if (kvm_flush_gpa_pud(pud, start_gpa, end, data)) {
+			pgd_clear(pgd + i);
+			pud_free(NULL, pud);
+		} else {
+			safe_to_remove = false;
+		}
+	}
+	return safe_to_remove;
+}
+
+/**
+ * kvm_flush_gpa_pt() - Flush a range of guest physical addresses.
+ * @kvm:	KVM pointer.
+ * @start_gfn:	Guest frame number of first page in GPA range to flush.
+ * @end_gfn:	Guest frame number of last page in GPA range to flush.
+ *
+ * Flushes a range of GPA mappings from the GPA page tables.
+ *
+ * The caller must hold the @kvm->mmu_lock spinlock.
+ *
+ * Returns:	Whether its safe to remove the top level page directory because
+ *		all lower levels have been removed.
+ */
+static bool kvm_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn, void *data)
+{
+	return kvm_flush_gpa_pgd(kvm->arch.gpa_mm.pgd,
+				start_gfn << PAGE_SHIFT,
+				end_gfn << PAGE_SHIFT, (unsigned long *)data);
+}
+
+/*
+ * kvm_mkclean_gpa_pt.
+ * Mark a range of guest physical address space clean (writes fault) in the VM's
+ * GPA page table to allow dirty page tracking.
+ */
+
+static int kvm_mkclean_pte(pte_t *pte, unsigned long start, unsigned long end)
+{
+	int ret = 0;
+	int i_min = pte_index(start);
+	int i_max = pte_index(end);
+	int i;
+	pte_t val;
+
+	for (i = i_min; i <= i_max; ++i) {
+		val = pte[i];
+		if (pte_present(val) && pte_dirty(val)) {
+			set_pte(pte + i, pte_mkclean(val));
+			ret = 1;
+		}
+	}
+	return ret;
+}
+
+static int kvm_mkclean_pmd(pmd_t *pmd, unsigned long start, unsigned long end)
+{
+	int ret = 0;
+	pte_t *pte;
+	unsigned long cur_end = ~0ul;
+	int i_min = pmd_index(start);
+	int i_max = pmd_index(end);
+	int i;
+	pmd_t old, new;
+
+	for (i = i_min; i <= i_max; ++i, start = 0) {
+		if (!pmd_present(pmd[i]))
+			continue;
+
+		if (kvm_pmd_huge(pmd[i])) {
+			old = pmd[i];
+			new = kvm_pmd_mkclean(old);
+			if (pmd_val(new) == pmd_val(old))
+				continue;
+			set_pmd(pmd + i, new);
+			ret = 1;
+			continue;
+		}
+
+		pte = pte_offset_kernel(pmd + i, 0);
+		if (i == i_max)
+			cur_end = end;
+
+		ret |= kvm_mkclean_pte(pte, start, cur_end);
+	}
+
+	return ret;
+}
+
+static int kvm_mkclean_pud(pud_t *pud, unsigned long start, unsigned long end)
+{
+	int ret = 0;
+	pmd_t *pmd;
+	unsigned long cur_end = ~0ul;
+	int i_min = pud_index(start);
+	int i_max = pud_index(end);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i, start = 0) {
+		if (!pud_present(pud[i]))
+			continue;
+
+		pmd = pmd_offset(pud + i, 0);
+		if (i == i_max)
+			cur_end = end;
+
+		ret |= kvm_mkclean_pmd(pmd, start, cur_end);
+	}
+	return ret;
+}
+
+static int kvm_mkclean_pgd(pgd_t *pgd, unsigned long start, unsigned long end)
+{
+	int ret = 0;
+	p4d_t *p4d;
+	pud_t *pud;
+	unsigned long cur_end = ~0ul;
+	int i_min = pgd_index(start);
+	int i_max = pgd_index(end);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i, start = 0) {
+		if (!pgd_present(pgd[i]))
+			continue;
+
+		p4d = p4d_offset(pgd + i, 0);
+		pud = pud_offset(p4d, 0);
+		if (i == i_max)
+			cur_end = end;
+
+		ret |= kvm_mkclean_pud(pud, start, cur_end);
+	}
+	return ret;
+}
+
+/**
+ * kvm_mkclean_gpa_pt() - Make a range of guest physical addresses clean.
+ * @kvm:	KVM pointer.
+ * @start_gfn:	Guest frame number of first page in GPA range to flush.
+ * @end_gfn:	Guest frame number of last page in GPA range to flush.
+ *
+ * Make a range of GPA mappings clean so that guest writes will fault and
+ * trigger dirty page logging.
+ *
+ * The caller must hold the @kvm->mmu_lock spinlock.
+ *
+ * Returns:	Whether any GPA mappings were modified, which would require
+ *		derived mappings (GVA page tables & TLB enties) to be
+ *		invalidated.
+ */
+static int kvm_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
+{
+	return kvm_mkclean_pgd(kvm->arch.gpa_mm.pgd, start_gfn << PAGE_SHIFT,
+				end_gfn << PAGE_SHIFT);
+}
+
+/**
+ * kvm_arch_mmu_enable_log_dirty_pt_masked() - write protect dirty pages
+ * @kvm:	The KVM pointer
+ * @slot:	The memory slot associated with mask
+ * @gfn_offset:	The gfn offset in memory slot
+ * @mask:	The mask of dirty pages at offset 'gfn_offset' in this memory
+ *		slot to be write protected
+ *
+ * Walks bits set in mask write protects the associated pte's. Caller must
+ * acquire @kvm->mmu_lock.
+ */
+void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
+		struct kvm_memory_slot *slot,
+		gfn_t gfn_offset, unsigned long mask)
+{
+	gfn_t base_gfn = slot->base_gfn + gfn_offset;
+	gfn_t start = base_gfn +  __ffs(mask);
+	gfn_t end = base_gfn + __fls(mask);
+
+	kvm_mkclean_gpa_pt(kvm, start, end);
+
+	/*
+	 * FIXME: disable THP to improve vm migration success ratio,
+	 * how to know migration failure to enable THP again
+	 */
+	slot->arch.flags |= KVM_MEMSLOT_DISABLE_THP;
+}
+
+void kvm_arch_commit_memory_region(struct kvm *kvm,
+		const struct kvm_userspace_memory_region *mem,
+		struct kvm_memory_slot *old,
+		const struct kvm_memory_slot *new,
+		enum kvm_mr_change change)
+{
+	int needs_flush;
+
+	/*
+	 * If dirty page logging is enabled, write protect all pages in the slot
+	 * ready for dirty logging.
+	 *
+	 * There is no need to do this in any of the following cases:
+	 * CREATE:      No dirty mappings will already exist.
+	 * MOVE/DELETE: The old mappings will already have been cleaned up by
+	 *              kvm_arch_flush_shadow_memslot()
+	 */
+	if (change == KVM_MR_FLAGS_ONLY &&
+			(!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
+			 new->flags & KVM_MEM_LOG_DIRTY_PAGES)) {
+		spin_lock(&kvm->mmu_lock);
+		/* Write protect GPA page table entries */
+		needs_flush = kvm_mkclean_gpa_pt(kvm, new->base_gfn,
+				new->base_gfn + new->npages - 1);
+		/* Let implementation do the rest */
+		if (needs_flush)
+			kvm_flush_remote_tlbs(kvm);
+		spin_unlock(&kvm->mmu_lock);
+	}
+}
+
+void kvm_arch_flush_shadow_all(struct kvm *kvm)
+{
+	/* Flush whole GPA */
+	kvm_flush_gpa_pt(kvm, 0, ~0UL, NULL);
+
+	/* Flush vpid for each VCPU individually */
+	kvm_flush_remote_tlbs(kvm);
+}
+
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+		struct kvm_memory_slot *slot)
+{
+	unsigned long npages;
+
+	/*
+	 * The slot has been made invalid (ready for moving or deletion), so we
+	 * need to ensure that it can no longer be accessed by any guest VCPUs.
+	 */
+
+	npages = 0;
+	spin_lock(&kvm->mmu_lock);
+	/* Flush slot from GPA */
+	kvm_flush_gpa_pt(kvm, slot->base_gfn,
+			slot->base_gfn + slot->npages - 1, &npages);
+	/* Let implementation do the rest */
+	if (npages)
+		kvm_flush_remote_tlbs(kvm);
+	spin_unlock(&kvm->mmu_lock);
+}
+
+void _kvm_destroy_mm(struct kvm *kvm)
+{
+	/* It should always be safe to remove after flushing the whole range */
+	WARN_ON(!kvm_flush_gpa_pt(kvm, 0, ~0UL, NULL));
+	pgd_free(NULL, kvm->arch.gpa_mm.pgd);
+	kvm->arch.gpa_mm.pgd = NULL;
+}
+
+/*
+ * Mark a range of guest physical address space old (all accesses fault) in the
+ * VM's GPA page table to allow detection of commonly used pages.
+ */
+
+static int kvm_mkold_pte(pte_t *pte, unsigned long start,
+				 unsigned long end)
+{
+	int ret = 0;
+	int i_min = pte_index(start);
+	int i_max = pte_index(end);
+	int i;
+	pte_t old, new;
+
+	for (i = i_min; i <= i_max; ++i) {
+		if (!pte_present(pte[i]))
+			continue;
+
+		old = pte[i];
+		new = pte_mkold(old);
+		if (pte_val(new) == pte_val(old))
+			continue;
+		set_pte(pte + i, new);
+		ret = 1;
+	}
+
+	return ret;
+}
+
+static int kvm_mkold_pmd(pmd_t *pmd, unsigned long start, unsigned long end)
+{
+	int ret = 0;
+	pte_t *pte;
+	unsigned long cur_end = ~0ul;
+	int i_min = pmd_index(start);
+	int i_max = pmd_index(end);
+	int i;
+	pmd_t old, new;
+
+	for (i = i_min; i <= i_max; ++i, start = 0) {
+		if (!pmd_present(pmd[i]))
+			continue;
+
+		if (kvm_pmd_huge(pmd[i])) {
+			old = pmd[i];
+			new = kvm_pmd_mkold(old);
+			if (pmd_val(new) == pmd_val(old))
+				continue;
+			set_pmd(pmd + i, new);
+			ret = 1;
+			continue;
+		}
+
+		pte = pte_offset_kernel(pmd + i, 0);
+		if (i == i_max)
+			cur_end = end;
+
+		ret |= kvm_mkold_pte(pte, start, cur_end);
+	}
+
+	return ret;
+}
+
+static int kvm_mkold_pud(pud_t *pud, unsigned long start, unsigned long end)
+{
+	int ret = 0;
+	pmd_t *pmd;
+	unsigned long cur_end = ~0ul;
+	int i_min = pud_index(start);
+	int i_max = pud_index(end);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i, start = 0) {
+		if (!pud_present(pud[i]))
+			continue;
+
+		pmd = pmd_offset(pud + i, 0);
+		if (i == i_max)
+			cur_end = end;
+
+		ret |= kvm_mkold_pmd(pmd, start, cur_end);
+	}
+
+	return ret;
+}
+
+static int kvm_mkold_pgd(pgd_t *pgd, unsigned long start, unsigned long end)
+{
+	int ret = 0;
+	p4d_t *p4d;
+	pud_t *pud;
+	unsigned long cur_end = ~0ul;
+	int i_min = pgd_index(start);
+	int i_max = pgd_index(end);
+	int i;
+
+	for (i = i_min; i <= i_max; ++i, start = 0) {
+		if (!pgd_present(pgd[i]))
+			continue;
+
+		p4d = p4d_offset(pgd + i, 0);
+		pud = pud_offset(p4d, 0);
+		if (i == i_max)
+			cur_end = end;
+
+		ret |= kvm_mkold_pud(pud, start, cur_end);
+	}
+
+	return ret;
+}
+
+static int handle_hva_to_gpa(struct kvm *kvm,
+			     unsigned long start,
+			     unsigned long end,
+			     int (*handler)(struct kvm *kvm, gfn_t gfn,
+					    gpa_t gfn_end,
+					    struct kvm_memory_slot *memslot,
+					    void *data),
+			     void *data)
+{
+	struct kvm_memslots *slots;
+	struct kvm_memory_slot *memslot;
+	int ret = 0;
+
+	slots = kvm_memslots(kvm);
+
+	/* we only care about the pages that the guest sees */
+	kvm_for_each_memslot(memslot, slots) {
+		unsigned long hva_start, hva_end;
+		gfn_t gfn, gfn_end;
+
+		hva_start = max(start, memslot->userspace_addr);
+		hva_end = min(end, memslot->userspace_addr +
+					(memslot->npages << PAGE_SHIFT));
+		if (hva_start >= hva_end)
+			continue;
+
+		/*
+		 * {gfn(page) | page intersects with [hva_start, hva_end)} =
+		 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
+		 */
+		gfn = hva_to_gfn_memslot(hva_start, memslot);
+		gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
+		ret |= handler(kvm, gfn, gfn_end, memslot, data);
+	}
+
+	return ret;
+}
+
+
+static int kvm_unmap_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
+				 struct kvm_memory_slot *memslot, void *data)
+{
+	unsigned long npages;
+
+	npages = 0;
+	kvm_flush_gpa_pt(kvm, gfn, gfn_end - 1, &npages);
+	*(unsigned long *)data = *(unsigned long *)data + npages;
+
+	return npages > 0;
+}
+
+int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable)
+{
+	unsigned long npages;
+
+	npages = 0;
+	return handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, &npages);
+}
+
+static int kvm_set_spte_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
+				struct kvm_memory_slot *memslot, void *data)
+{
+	gpa_t gpa = gfn << PAGE_SHIFT;
+	pte_t hva_pte = *(pte_t *)data;
+	pte_t *gpa_pte = kvm_pte_for_gpa_fast(kvm, gpa);
+	pte_t old_pte;
+
+	if (!gpa_pte)
+		return 0;
+
+	/* Mapping may need adjusting depending on memslot flags */
+	old_pte = *gpa_pte;
+	if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES && !pte_dirty(old_pte))
+		hva_pte = pte_mkclean(hva_pte);
+	else if (memslot->flags & KVM_MEM_READONLY)
+		hva_pte = pte_wrprotect(hva_pte);
+
+	set_pte(gpa_pte, hva_pte);
+
+	/* Replacing an absent or old page doesn't need flushes */
+	if (!pte_present(old_pte) || !pte_young(old_pte))
+		return 0;
+
+	/* Pages swapped, aged, moved, or cleaned require flushes */
+	return !pte_present(hva_pte) ||
+	       !pte_young(hva_pte) ||
+	       pte_pfn(old_pte) != pte_pfn(hva_pte) ||
+	       (pte_dirty(old_pte) && !pte_dirty(hva_pte));
+}
+
+int  _kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
+{
+	unsigned long end = hva + PAGE_SIZE;
+	int ret;
+
+	ret = handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pte);
+	if (ret)
+		/* Flush vpid for each VCPU individually */
+		kvm_flush_remote_tlbs(kvm);
+	return 0;
+}
+
+static int kvm_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
+			       struct kvm_memory_slot *memslot, void *data)
+{
+	return kvm_mkold_pgd(kvm->arch.gpa_mm.pgd, gfn << PAGE_SHIFT,
+				gfn_end << PAGE_SHIFT);
+}
+
+static int kvm_test_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
+				    struct kvm_memory_slot *memslot, void *data)
+{
+	gpa_t gpa = gfn << PAGE_SHIFT;
+	pte_t *gpa_pte = kvm_pte_for_gpa_fast(kvm, gpa);
+
+	if (!gpa_pte)
+		return 0;
+	return pte_young(*gpa_pte);
+}
+
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
+{
+	return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
+}
+
+int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
+{
+	return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
+}
+
+static pud_t *kvm_get_pud(struct kvm *kvm,
+		 struct kvm_mmu_memory_cache *cache, phys_addr_t addr)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+
+	pgd = kvm->arch.gpa_mm.pgd + pgd_index(addr);
+	if (pgd_none(*pgd)) {
+		/* Not used yet */
+		BUG();
+		return NULL;
+	}
+
+	p4d = p4d_offset(pgd, addr);
+	return pud_offset(p4d, addr);
+}
+
+static pmd_t *kvm_get_pmd(struct kvm *kvm,
+			struct vm_area_struct *vma, unsigned long hva,
+			struct kvm_mmu_memory_cache *cache, phys_addr_t addr)
+{
+	pud_t *pud;
+	pmd_t *pmd;
+
+	pud = kvm_get_pud(kvm, cache, addr);
+	if (!pud || kvm_pud_huge(*pud))
+		return NULL;
+
+	if (pud_none(*pud)) {
+		if (!cache)
+			return NULL;
+		pmd = kvm_mmu_memory_cache_alloc(cache);
+		pmd_init(pmd);
+		pud_populate(NULL, pud, pmd);
+	}
+
+	return pmd_offset(pud, addr);
+}
+
+static int kvm_set_pmd_huge(struct kvm_vcpu *vcpu, struct kvm_mmu_memory_cache
+			       *cache, phys_addr_t addr, const pmd_t *new_pmd,
+				struct vm_area_struct *vma, unsigned long hva)
+{
+	pmd_t *pmd, old_pmd;
+
+retry:
+	pmd = kvm_get_pmd(vcpu->kvm, vma, hva, cache, addr);
+	VM_BUG_ON(!pmd);
+
+	old_pmd = *pmd;
+	/*
+	 * Multiple vcpus faulting on the same PMD entry, can
+	 * lead to them sequentially updating the PMD with the
+	 * same value. Following the break-before-make
+	 * (pmd_clear() followed by tlb_flush()) process can
+	 * hinder forward progress due to refaults generated
+	 * on missing translations.
+	 *
+	 * Skip updating the page table if the entry is
+	 * unchanged.
+	 */
+	if (pmd_val(old_pmd) == pmd_val(*new_pmd))
+		return 0;
+
+	if (pmd_present(old_pmd)) {
+		/*
+		 * If we already have PTE level mapping for this block,
+		 * we must unmap it to avoid inconsistent TLB state and
+		 * leaking the table page. We could end up in this situation
+		 * if the memory slot was marked for dirty logging and was
+		 * reverted, leaving PTE level mappings for the pages accessed
+		 * during the period. So, unmap the PTE level mapping for this
+		 * block and retry, as we could have released the upper level
+		 * table in the process.
+		 *
+		 * Normal THP split/merge follows mmu_notifier callbacks and do
+		 * get handled accordingly.
+		 */
+		if (!kvm_pmd_huge(old_pmd)) {
+			++vcpu->stat.huge_merge_exits;
+			kvm_flush_gpa_pt(vcpu->kvm,
+				(addr & PMD_MASK) >> PAGE_SHIFT,
+				((addr & PMD_MASK) + PMD_SIZE - 1) >> PAGE_SHIFT, NULL);
+			goto retry;
+		}
+		/*
+		 * Mapping in huge pages should only happen through a
+		 * fault.  If a page is merged into a transparent huge
+		 * page, the individual subpages of that huge page
+		 * should be unmapped through MMU notifiers before we
+		 * get here.
+		 *
+		 * Merging of CompoundPages is not supported; they
+		 * should become splitting first, unmapped, merged,
+		 * and mapped back in on-demand.
+		 */
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+		WARN_ON_ONCE(pmd_pfn(old_pmd) != pmd_pfn(*new_pmd));
+#endif
+		pmd_clear(pmd);
+	}
+
+	kvm_tlb_flush_gpa(vcpu, addr & PMD_MASK);
+	set_pmd(pmd, *new_pmd);
+	return 0;
+}
+
+/*
+ * Adjust pfn start boundary if support for transparent hugepage
+ */
+static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, unsigned long *gpap)
+{
+	kvm_pfn_t pfn = *pfnp;
+	gfn_t gfn = *gpap >> PAGE_SHIFT;
+	struct page *page = pfn_to_page(pfn);
+
+	/*
+	 * PageTransCompoundMap() returns true for THP and
+	 * hugetlbfs. Make sure the adjustment is done only for THP
+	 * pages.
+	 */
+	if ((!PageHuge(page)) && PageTransCompound(page) &&
+			 (atomic_read(&page->_mapcount) < 0)) {
+		unsigned long mask;
+		/*
+		 * The address we faulted on is backed by a transparent huge
+		 * page.  However, because we map the compound huge page and
+		 * not the individual tail page, we need to transfer the
+		 * refcount to the head page.  We have to be careful that the
+		 * THP doesn't start to split while we are adjusting the
+		 * refcounts.
+		 *
+		 * We are sure this doesn't happen, because mmu_notifier_retry
+		 * was successful and we are holding the mmu_lock, so if this
+		 * THP is trying to split, it will be blocked in the mmu
+		 * notifier before touching any of the pages, specifically
+		 * before being able to call __split_huge_page_refcount().
+		 *
+		 * We can therefore safely transfer the refcount from PG_tail
+		 * to PG_head and switch the pfn from a tail page to the head
+		 * page accordingly.
+		 */
+		mask = PTRS_PER_PMD - 1;
+		VM_BUG_ON((gfn & mask) != (pfn & mask));
+		if (pfn & mask) {
+			*gpap &= PMD_MASK;
+			kvm_release_pfn_clean(pfn);
+			pfn &= ~mask;
+			kvm_get_pfn(pfn);
+			*pfnp = pfn;
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
+static bool fault_supports_huge_mapping(struct kvm_memory_slot *memslot,
+					       unsigned long hva,
+					       unsigned long map_size)
+{
+	gpa_t gpa_start;
+	hva_t uaddr_start, uaddr_end;
+	size_t size;
+
+	if (memslot->arch.flags & KVM_MEMSLOT_DISABLE_THP)
+		return false;
+
+	size = memslot->npages * PAGE_SIZE;
+	gpa_start = memslot->base_gfn << PAGE_SHIFT;
+	uaddr_start = memslot->userspace_addr;
+	uaddr_end = uaddr_start + size;
+
+	/*
+	 * Pages belonging to memslots that don't have the same alignment
+	 * within a PMD/PUD for userspace and GPA cannot be mapped with stage-2
+	 * PMD/PUD entries, because we'll end up mapping the wrong pages.
+	 *
+	 * Consider a layout like the following:
+	 *
+	 *    memslot->userspace_addr:
+	 *    +-----+--------------------+--------------------+---+
+	 *    |abcde|fgh  Stage-1 block  |    Stage-1 block tv|xyz|
+	 *    +-----+--------------------+--------------------+---+
+	 *
+	 *    memslot->base_gfn << PAGE_SIZE:
+	 *      +---+--------------------+--------------------+-----+
+	 *      |abc|def  Stage-2 block  |    Stage-2 block   |tvxyz|
+	 *      +---+--------------------+--------------------+-----+
+	 *
+	 * If we create those stage-2 blocks, we'll end up with this incorrect
+	 * mapping:
+	 *   d -> f
+	 *   e -> g
+	 *   f -> h
+	 */
+	if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1)))
+		return false;
+
+	/*
+	 * Next, let's make sure we're not trying to map anything not covered
+	 * by the memslot. This means we have to prohibit block size mappings
+	 * for the beginning and end of a non-block aligned and non-block sized
+	 * memory slot (illustrated by the head and tail parts of the
+	 * userspace view above containing pages 'abcde' and 'xyz',
+	 * respectively).
+	 *
+	 * Note that it doesn't matter if we do the check using the
+	 * userspace_addr or the base_gfn, as both are equally aligned (per
+	 * the check above) and equally sized.
+	 */
+	return (hva & ~(map_size - 1)) >= uaddr_start &&
+	       (hva & ~(map_size - 1)) + map_size <= uaddr_end;
+}
+
+/**
+ * kvm_map_page_fast() - Fast path GPA fault handler.
+ * @vcpu:		VCPU pointer.
+ * @gpa:		Guest physical address of fault.
+ * @write:	Whether the fault was due to a write.
+ * @out_entry:		New PTE for @gpa (written on success unless NULL).
+ * @out_buddy:		New PTE for @gpa's buddy (written on success unless
+ *			NULL).
+ *
+ * Perform fast path GPA fault handling, doing all that can be done without
+ * calling into KVM. This handles marking old pages young (for idle page
+ * tracking), and dirtying of clean pages (for dirty page logging).
+ *
+ * Returns:	0 on success, in which case we can update derived mappings and
+ *		resume guest execution.
+ *		-EFAULT on failure due to absent GPA mapping or write to
+ *		read-only page, in which case KVM must be consulted.
+ */
+static int kvm_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa,
+				   bool write,
+				   pte_t *out_entry, pte_t *out_buddy)
+{
+	struct kvm *kvm = vcpu->kvm;
+	gfn_t gfn = gpa >> PAGE_SHIFT;
+	pte_t *ptep;
+	kvm_pfn_t pfn = 0;	/* silence bogus GCC warning */
+	bool pfn_valid = false;
+	int ret = 0;
+
+	spin_lock(&kvm->mmu_lock);
+
+	/* Fast path - just check GPA page table for an existing entry */
+	ptep = kvm_pte_for_gpa_fast(kvm, gpa);
+	if (!ptep || !pte_present(*ptep)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	/* Track access to pages marked old */
+	if (!pte_young(*ptep)) {
+		set_pte(ptep, pte_mkyoung(*ptep));
+		pfn = pte_pfn(*ptep);
+		pfn_valid = true;
+		/* call kvm_set_pfn_accessed() after unlock */
+	}
+	if (write && !pte_dirty(*ptep)) {
+		if (!pte_write(*ptep)) {
+			ret = -EFAULT;
+			goto out;
+		}
+
+		/* Track dirtying of writeable pages */
+		set_pte(ptep, pte_mkdirty(*ptep));
+		pfn = pte_pfn(*ptep);
+		if (kvm_pmd_huge(*((pmd_t *)ptep))) {
+			int i;
+			gfn_t base_gfn = (gpa & PMD_MASK) >> PAGE_SHIFT;
+
+			for (i = 0; i < PTRS_PER_PTE; i++)
+				mark_page_dirty(kvm, base_gfn + i);
+		} else
+			mark_page_dirty(kvm, gfn);
+		kvm_set_pfn_dirty(pfn);
+	}
+
+	if (out_entry)
+		*out_entry = *ptep;
+	if (out_buddy)
+		*out_buddy = *ptep_buddy(ptep);
+
+out:
+	spin_unlock(&kvm->mmu_lock);
+	if (pfn_valid)
+		kvm_set_pfn_accessed(pfn);
+	return ret;
+}
+
+/**
+ * kvm_map_page() - Map a guest physical page.
+ * @vcpu:		VCPU pointer.
+ * @gpa:		Guest physical address of fault.
+ * @write:	Whether the fault was due to a write.
+ * @out_entry:		New PTE for @gpa (written on success unless NULL).
+ * @out_buddy:		New PTE for @gpa's buddy (written on success unless
+ *			NULL).
+ *
+ * Handle GPA faults by creating a new GPA mapping (or updating an existing
+ * one).
+ *
+ * This takes care of marking pages young or dirty (idle/dirty page tracking),
+ * asking KVM for the corresponding PFN, and creating a mapping in the GPA page
+ * tables. Derived mappings (GVA page tables and TLBs) must be handled by the
+ * caller.
+ *
+ * Returns:	0 on success, in which case the caller may use the @out_entry
+ *		and @out_buddy PTEs to update derived mappings and resume guest
+ *		execution.
+ *		-EFAULT if there is no memory region at @gpa or a write was
+ *		attempted to a read-only memory region. This is usually handled
+ *		as an MMIO access.
+ */
+static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa,
+			     bool write,
+			     pte_t *out_entry, pte_t *out_buddy)
+{
+	struct kvm *kvm = vcpu->kvm;
+	struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
+	gfn_t gfn = gpa >> PAGE_SHIFT;
+	int srcu_idx, err = 0;
+	kvm_pfn_t pfn;
+	pte_t *ptep;
+	bool writeable;
+	unsigned long prot_bits;
+	unsigned long mmu_seq;
+	u32 exccode = (vcpu->arch.host_estat & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT;
+
+	unsigned long hva;
+	struct kvm_memory_slot *memslot;
+	bool force_pte = false;
+	struct vm_area_struct *vma;
+	unsigned long vma_pagesize;
+	bool writable;
+	int ret, retry_no = 0;
+
+	/* Try the fast path to handle old / clean pages */
+	srcu_idx = srcu_read_lock(&kvm->srcu);
+	if ((exccode != KVM_EXCCODE_TLBRI) && (exccode != KVM_EXCCODE_TLBXI)) {
+		err = kvm_map_page_fast(vcpu, gpa, write, out_entry,
+					      out_buddy);
+		if (!err)
+			goto out;
+	}
+
+	memslot = gfn_to_memslot(kvm, gfn);
+	hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
+	if (kvm_is_error_hva(hva) || (write && !writable))
+		goto out;
+
+	/* Let's check if we will get back a huge page backed by hugetlbfs */
+	mmap_read_lock(current->mm);
+	vma = find_vma_intersection(current->mm, hva, hva + 1);
+	if (unlikely(!vma)) {
+		kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
+		mmap_read_unlock(current->mm);
+		err = -EFAULT;
+		goto out;
+	}
+
+	vma_pagesize = vma_kernel_pagesize(vma);
+
+	if (fault_supports_huge_mapping(memslot, hva, vma_pagesize)) {
+		force_pte = true;
+		vma_pagesize = PAGE_SIZE;
+		++vcpu->stat.huge_dec_exits;
+	}
+
+	/* PMD is not folded, adjust gfn to new boundary */
+	if (vma_pagesize == PMD_SIZE)
+		gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT;
+	mmap_read_unlock(current->mm);
+
+	/* We need a minimum of cached pages ready for page table creation */
+	err = kvm_mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES);
+	if (err)
+		goto out;
+
+retry:
+	/*
+	 * Used to check for invalidations in progress, of the pfn that is
+	 * returned by pfn_to_pfn_prot below.
+	 */
+	mmu_seq = kvm->mmu_notifier_seq;
+	/*
+	 * Ensure the read of mmu_notifier_seq isn't reordered with PTE reads in
+	 * gfn_to_pfn_prot() (which calls get_user_pages()), so that we don't
+	 * risk the page we get a reference to getting unmapped before we have a
+	 * chance to grab the mmu_lock without mmu_notifier_retry() noticing.
+	 *
+	 * This smp_rmb() pairs with the effective smp_wmb() of the combination
+	 * of the pte_unmap_unlock() after the PTE is zapped, and the
+	 * spin_lock() in kvm_mmu_notifier_invalidate_<page|range_end>() before
+	 * mmu_notifier_seq is incremented.
+	 */
+	smp_rmb();
+
+	/* Slow path - ask KVM core whether we can access this GPA */
+	pfn = gfn_to_pfn_prot(kvm, gfn, write, &writeable);
+	if (is_error_noslot_pfn(pfn)) {
+		err = -EFAULT;
+		goto out;
+	}
+
+	spin_lock(&kvm->mmu_lock);
+	/* Check if an invalidation has taken place since we got pfn */
+	if (mmu_notifier_retry(kvm, mmu_seq)) {
+		/*
+		 * This can happen when mappings are changed asynchronously, but
+		 * also synchronously if a COW is triggered by
+		 * gfn_to_pfn_prot().
+		 */
+		spin_unlock(&kvm->mmu_lock);
+		kvm_release_pfn_clean(pfn);
+		if (retry_no > 100) {
+			retry_no = 0;
+			schedule();
+		}
+		retry_no++;
+		goto retry;
+	}
+
+	if (vma_pagesize == PAGE_SIZE && !force_pte) {
+		/*
+		 * Only PMD_SIZE transparent hugepages(THP) are
+		 * currently supported. This code will need to be
+		 * updated to support other THP sizes.
+		 *
+		 * Make sure the host VA and the guest IPA are sufficiently
+		 * aligned and that the block is contained within the memslot.
+		 */
+		++vcpu->stat.huge_thp_exits;
+		if (fault_supports_huge_mapping(memslot, hva, PMD_SIZE) &&
+		    transparent_hugepage_adjust(&pfn, &gpa)) {
+			++vcpu->stat.huge_adjust_exits;
+			vma_pagesize = PMD_SIZE;
+		}
+	}
+
+	/* Set up the prot bits */
+	prot_bits = _PAGE_PRESENT | __READABLE;
+	if (vma->vm_flags & (VM_IO | VM_PFNMAP))
+		prot_bits |= _CACHE_SUC;
+	else
+		prot_bits |= _CACHE_CC;
+
+	if (writeable) {
+		prot_bits |= _PAGE_WRITE;
+		if (write) {
+			prot_bits |= __WRITEABLE | _PAGE_MODIFIED;
+			mark_page_dirty(kvm, gfn);
+			kvm_set_pfn_dirty(pfn);
+		}
+	}
+
+	if (vma_pagesize == PMD_SIZE) {
+		pmd_t new_pmd = pfn_pmd(pfn, __pgprot(prot_bits));
+
+		new_pmd = kvm_pmd_mkhuge(new_pmd);
+
+		if (writeable && write) {
+			int i;
+			gfn_t base_gfn = (gpa & PMD_MASK) >> PAGE_SHIFT;
+
+			for (i = 0; i < PTRS_PER_PTE; i++)
+				mark_page_dirty(kvm, base_gfn + i);
+		}
+		++vcpu->stat.huge_set_exits;
+		ret = kvm_set_pmd_huge(vcpu, memcache, gpa, &new_pmd, vma, hva);
+	} else {
+		pte_t new_pte = pfn_pte(pfn, __pgprot(prot_bits));
+
+		if (writeable && write)
+			mark_page_dirty(kvm, gfn);
+
+		/* Ensure page tables are allocated */
+		ptep = kvm_pte_for_gpa(kvm, memcache, vma, hva, gpa);
+		set_pte(ptep, new_pte);
+
+		err = 0;
+		if (out_entry)
+			*out_entry = new_pte;
+		if (out_buddy)
+			*out_buddy = *ptep_buddy(&new_pte);
+	}
+
+	spin_unlock(&kvm->mmu_lock);
+	kvm_release_pfn_clean(pfn);
+	kvm_set_pfn_accessed(pfn);
+out:
+	srcu_read_unlock(&kvm->srcu, srcu_idx);
+	return err;
+}
+
+int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv,
+				      bool write)
+{
+	int ret;
+
+	ret = kvm_map_page(vcpu, badv, write, NULL, NULL);
+	if (ret)
+		return ret;
+
+	/* Invalidate this entry in the TLB */
+	return kvm_tlb_flush_gpa(vcpu, badv);
+}
+
+/**
+ * kvm_flush_tlb_all() - Flush all root TLB entries for
+ * guests.
+ *
+ * Invalidate all entries including GVA-->GPA and GPA-->HPA mappings.
+ */
+void kvm_flush_tlb_all(void)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	invtlb_all(INVTLB_ALLGID, 0, 0);
+	local_irq_restore(flags);
+}

arch/loongarch/kvm/timer.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/ktime.h>
+#include <linux/kvm_host.h>
+#include <linux/vmalloc.h>
+#include <linux/fs.h>
+#include <linux/random.h>
+#include <asm/page.h>
+#include <asm/cacheflush.h>
+#include <asm/cacheops.h>
+#include <asm/cpu-info.h>
+#include <asm/mmu_context.h>
+#include <asm/tlbflush.h>
+#include <asm/inst.h>
+#include "kvmcpu.h"
+#include "trace.h"
+#include "kvm_compat.h"
+
+/*
+ * ktime_to_tick() - Scale ktime_t to a 64-bit stable timer.
+ *
+ * Caches the dynamic nanosecond bias in vcpu->arch.timer_dyn_bias.
+ */
+static u64 ktime_to_tick(struct kvm_vcpu *vcpu, ktime_t now)
+{
+	s64 now_ns, periods;
+	u64 delta;
+
+	now_ns = ktime_to_ns(now);
+	delta = now_ns + vcpu->arch.timer_dyn_bias;
+
+	if (delta >= vcpu->arch.timer_period) {
+		/* If delta is out of safe range the bias needs adjusting */
+		periods = div64_s64(now_ns, vcpu->arch.timer_period);
+		vcpu->arch.timer_dyn_bias = -periods * vcpu->arch.timer_period;
+		/* Recalculate delta with new bias */
+		delta = now_ns + vcpu->arch.timer_dyn_bias;
+	}
+
+	/*
+	 * We've ensured that:
+	 *   delta < timer_period
+	 */
+	return div_u64(delta * vcpu->arch.timer_mhz, MNSEC_PER_SEC);
+}
+
+/**
+ * kvm_resume_hrtimer() - Resume hrtimer, updating expiry.
+ * @vcpu:	Virtual CPU.
+ * @now:	ktime at point of resume.
+ * @stable_timer:	stable timer at point of resume.
+ *
+ * Resumes the timer and updates the timer expiry based on @now and @count.
+ */
+static void kvm_resume_hrtimer(struct kvm_vcpu *vcpu, ktime_t now, u64 stable_timer)
+{
+	u64 delta;
+	ktime_t expire;
+
+	/* Stable timer decreased to zero or
+	 * initialize to zero, set 4 second timer
+	*/
+	delta = div_u64(stable_timer * MNSEC_PER_SEC, vcpu->arch.timer_mhz);
+	expire = ktime_add_ns(now, delta);
+
+	/* Update hrtimer to use new timeout */
+	hrtimer_cancel(&vcpu->arch.swtimer);
+	hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED);
+}
+
+/**
+ * kvm_init_timer() - Initialise stable timer.
+ * @vcpu:	Virtual CPU.
+ * @timer_hz:	Frequency of timer.
+ *
+ * Initialise the timer to the specified frequency, zero it, and set it going if
+ * it's enabled.
+ */
+void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz)
+{
+	ktime_t now;
+	unsigned long ticks;
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+
+	vcpu->arch.timer_mhz = timer_hz >> 20;
+	vcpu->arch.timer_period = div_u64((u64)MNSEC_PER_SEC * IOCSR_TIMER_MASK, vcpu->arch.timer_mhz);
+	vcpu->arch.timer_dyn_bias = 0;
+
+	/* Starting at 0 */
+	ticks = 0;
+	now = ktime_get();
+	vcpu->arch.timer_bias = ticks - ktime_to_tick(vcpu, now);
+	vcpu->arch.timer_bias &= IOCSR_TIMER_MASK;
+
+	kvm_write_sw_gcsr(csr, KVM_CSR_TVAL, ticks);
+}
+
+/**
+ * kvm_count_timeout() - Push timer forward on timeout.
+ * @vcpu:	Virtual CPU.
+ *
+ * Handle an hrtimer event by push the hrtimer forward a period.
+ *
+ * Returns:	The hrtimer_restart value to return to the hrtimer subsystem.
+ */
+enum hrtimer_restart kvm_count_timeout(struct kvm_vcpu *vcpu)
+{
+	unsigned long timer_cfg;
+
+	/* Add the Count period to the current expiry time */
+	timer_cfg = kvm_read_sw_gcsr(vcpu->arch.csr, KVM_CSR_TCFG);
+	if (timer_cfg & KVM_TCFG_PERIOD) {
+		hrtimer_add_expires_ns(&vcpu->arch.swtimer, timer_cfg & KVM_TCFG_VAL);
+		return HRTIMER_RESTART;
+	} else
+		return HRTIMER_NORESTART;
+}
+
+/*
+ * kvm_restore_timer() - Restore timer state.
+ * @vcpu:       Virtual CPU.
+ *
+ * Restore soft timer state from saved context.
+ */
+void kvm_restore_timer(struct kvm_vcpu *vcpu)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	ktime_t saved_ktime, now;
+	u64 stable_timer, new_timertick = 0;
+	u64 delta = 0;
+	int expired = 0;
+	unsigned long timer_cfg;
+
+	/*
+	 * Set guest stable timer cfg csr
+	 */
+	timer_cfg = kvm_read_sw_gcsr(csr, KVM_CSR_TCFG);
+	kvm_restore_hw_gcsr(csr, KVM_CSR_ESTAT);
+	if (!(timer_cfg & KVM_TCFG_EN)) {
+		kvm_restore_hw_gcsr(csr, KVM_CSR_TCFG);
+		kvm_restore_hw_gcsr(csr, KVM_CSR_TVAL);
+		return;
+	}
+
+	now = ktime_get();
+	saved_ktime = vcpu->arch.stable_ktime_saved;
+	stable_timer = kvm_read_sw_gcsr(csr, KVM_CSR_TVAL);
+
+	/*hrtimer not expire */
+	delta = ktime_to_tick(vcpu, ktime_sub(now, saved_ktime));
+	if (delta >= stable_timer)
+		expired = 1;
+
+	if (expired) {
+		if (timer_cfg & KVM_TCFG_PERIOD) {
+			new_timertick = (delta - stable_timer) % (timer_cfg & KVM_TCFG_VAL);
+		} else {
+			new_timertick = 1;
+		}
+	} else {
+		new_timertick = stable_timer - delta;
+	}
+
+	new_timertick &= KVM_TCFG_VAL;
+	kvm_write_gcsr_timercfg(timer_cfg);
+	kvm_write_gcsr_timertick(new_timertick);
+	if (expired)
+		_kvm_queue_irq(vcpu, LARCH_INT_TIMER);
+}
+
+/*
+ * kvm_acquire_timer() - Switch to hard timer state.
+ * @vcpu:       Virtual CPU.
+ *
+ * Restore hard timer state on top of existing soft timer state if possible.
+ *
+ * Since hard timer won't remain active over preemption, preemption should be
+ * disabled by the caller.
+ */
+void kvm_acquire_timer(struct kvm_vcpu *vcpu)
+{
+	unsigned long flags, guestcfg;
+
+	guestcfg = kvm_read_csr_gcfg();
+	if (!(guestcfg & KVM_GCFG_TIT))
+		return;
+
+	/* enable guest access to hard timer */
+	kvm_write_csr_gcfg(guestcfg & ~KVM_GCFG_TIT);
+
+	/*
+	 * Freeze the soft-timer and sync the guest stable timer with it. We do
+	 * this with interrupts disabled to avoid latency.
+	 */
+	local_irq_save(flags);
+	hrtimer_cancel(&vcpu->arch.swtimer);
+	local_irq_restore(flags);
+}
+
+
+/*
+ * _kvm_save_timer() - Switch to software emulation of guest timer.
+ * @vcpu:       Virtual CPU.
+ *
+ * Save guest timer state and switch to software emulation of guest
+ * timer. The hard timer must already be in use, so preemption should be
+ * disabled.
+ */
+static ktime_t _kvm_save_timer(struct kvm_vcpu *vcpu, u64 *stable_timer)
+{
+	u64 end_stable_timer;
+	ktime_t before_time;
+
+	before_time = ktime_get();
+
+	/*
+	 * Record a final stable timer which we will transfer to the soft-timer.
+	 */
+	end_stable_timer = kvm_read_gcsr_timertick();
+	*stable_timer = end_stable_timer;
+
+	kvm_resume_hrtimer(vcpu, before_time, end_stable_timer);
+	return before_time;
+}
+
+/*
+ * kvm_save_timer() - Save guest timer state.
+ * @vcpu:       Virtual CPU.
+ *
+ * Save guest timer state and switch to soft guest timer if hard timer was in
+ * use.
+ */
+void kvm_save_timer(struct kvm_vcpu *vcpu)
+{
+	struct loongarch_csrs *csr = vcpu->arch.csr;
+	unsigned long guestcfg;
+	u64 stable_timer = 0;
+	ktime_t save_ktime;
+
+	preempt_disable();
+	guestcfg = kvm_read_csr_gcfg();
+	if (!(guestcfg & KVM_GCFG_TIT)) {
+		/* disable guest use of hard timer */
+		kvm_write_csr_gcfg(guestcfg | KVM_GCFG_TIT);
+
+		/* save hard timer state */
+		kvm_save_hw_gcsr(csr, KVM_CSR_TCFG);
+		if (kvm_read_sw_gcsr(csr, KVM_CSR_TCFG) & KVM_TCFG_EN) {
+			save_ktime = _kvm_save_timer(vcpu, &stable_timer);
+			kvm_write_sw_gcsr(csr, KVM_CSR_TVAL, stable_timer);
+			vcpu->arch.stable_ktime_saved = save_ktime;
+			if (stable_timer == IOCSR_TIMER_MASK)
+				_kvm_queue_irq(vcpu, LARCH_INT_TIMER);
+		} else {
+			kvm_save_hw_gcsr(csr, KVM_CSR_TVAL);
+		}
+	}
+
+	/* save timer-related state to VCPU context */
+	kvm_save_hw_gcsr(csr, KVM_CSR_ESTAT);
+	preempt_enable();
+}
+
+void kvm_reset_timer(struct kvm_vcpu *vcpu)
+{
+	kvm_write_gcsr_timercfg(0);
+	kvm_write_sw_gcsr(vcpu->arch.csr, KVM_CSR_TCFG, 0);
+	hrtimer_cancel(&vcpu->arch.swtimer);
+}

arch/loongarch/kvm/trace.h

+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+
+#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_KVM_H
+
+#include <linux/tracepoint.h>
+#include "kvm_compat.h"
+#include "kvmcsr.h"
+
+#undef	TRACE_SYSTEM
+#define TRACE_SYSTEM kvm
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace
+
+/*
+ * arch/loongarch/kvm/loongarch.c
+ */
+extern bool kvm_trace_guest_mode_change;
+int kvm_guest_mode_change_trace_reg(void);
+void kvm_guest_mode_change_trace_unreg(void);
+
+/*
+ * Tracepoints for VM enters
+ */
+DECLARE_EVENT_CLASS(kvm_transition,
+	TP_PROTO(struct kvm_vcpu *vcpu),
+	TP_ARGS(vcpu),
+	TP_STRUCT__entry(
+		__field(unsigned long, pc)
+	),
+
+	TP_fast_assign(
+		__entry->pc = vcpu->arch.pc;
+	),
+
+	TP_printk("PC: 0x%08lx",
+		  __entry->pc)
+);
+
+DEFINE_EVENT(kvm_transition, kvm_enter,
+	     TP_PROTO(struct kvm_vcpu *vcpu),
+	     TP_ARGS(vcpu));
+
+DEFINE_EVENT(kvm_transition, kvm_reenter,
+	     TP_PROTO(struct kvm_vcpu *vcpu),
+	     TP_ARGS(vcpu));
+
+DEFINE_EVENT(kvm_transition, kvm_out,
+	     TP_PROTO(struct kvm_vcpu *vcpu),
+	     TP_ARGS(vcpu));
+
+/* The first 32 exit reasons correspond to Cause.ExcCode */
+#define KVM_TRACE_EXIT_INT	0
+#define KVM_TRACE_EXIT_TLBLD	(KVM_EXCCODE_TLBL)
+#define KVM_TRACE_EXIT_TLBST	(KVM_EXCCODE_TLBS)
+#define KVM_TRACE_EXIT_TLBI	(KVM_EXCCODE_TLBI)
+#define KVM_TRACE_EXIT_TLBMOD	(KVM_EXCCODE_TLBM)
+#define KVM_TRACE_EXIT_TLBRI	(KVM_EXCCODE_TLBRI)
+#define KVM_TRACE_EXIT_TLBXI	(KVM_EXCCODE_TLBXI)
+#define KVM_TRACE_EXIT_TLBPE	(KVM_EXCCODE_TLBPE)
+#define KVM_TRACE_EXIT_ADDE	(KVM_EXCCODE_ADE)
+#define KVM_TRACE_EXIT_UNALIGN	(KVM_EXCCODE_ALE)
+#define KVM_TRACE_EXIT_ODB	(KVM_EXCCODE_OOB)
+#define KVM_TRACE_EXIT_SYSCALL	(KVM_EXCCODE_SYS)
+#define KVM_TRACE_EXIT_BP	(KVM_EXCCODE_BP)
+#define KVM_TRACE_EXIT_INE	(KVM_EXCCODE_INE)
+#define KVM_TRACE_EXIT_IPE	(KVM_EXCCODE_IPE)
+#define KVM_TRACE_EXIT_FPDIS	(KVM_EXCCODE_FPDIS)
+#define KVM_TRACE_EXIT_LSXDIS	(KVM_EXCCODE_LSXDIS)
+#define KVM_TRACE_EXIT_LASXDIS	(KVM_EXCCODE_LASXDIS)
+#define KVM_TRACE_EXIT_FPE	(KVM_EXCCODE_FPE)
+#define KVM_TRACE_EXIT_WATCH	(KVM_EXCCODE_WATCH)
+#define KVM_TRACE_EXIT_GSPR	(KVM_EXCCODE_GSPR)
+#define KVM_TRACE_EXIT_HC	(KVM_EXCCODE_HYP)
+#define KVM_TRACE_EXIT_GCM	(KVM_EXCCODE_GCM)
+
+/* Further exit reasons */
+#define KVM_TRACE_EXIT_IDLE	64
+#define KVM_TRACE_EXIT_CACHE	65
+#define KVM_TRACE_EXIT_SIGNAL	66
+
+/* Tracepoints for VM exits */
+#define kvm_trace_symbol_exit_types				\
+	{ KVM_TRACE_EXIT_INT,		"Interrupt" },		\
+	{ KVM_TRACE_EXIT_TLBLD,		"TLB (LD)" },		\
+	{ KVM_TRACE_EXIT_TLBST,		"TLB (ST)" },		\
+	{ KVM_TRACE_EXIT_TLBI,		"TLB Ifetch" },		\
+	{ KVM_TRACE_EXIT_TLBMOD,	"TLB Mod" },		\
+	{ KVM_TRACE_EXIT_TLBRI,		"TLB RI" },		\
+	{ KVM_TRACE_EXIT_TLBXI,		"TLB XI" },		\
+	{ KVM_TRACE_EXIT_TLBPE,		"TLB Previlege Error" },\
+	{ KVM_TRACE_EXIT_ADDE,		"Address Error" },	\
+	{ KVM_TRACE_EXIT_UNALIGN,	"Address unalign" },	\
+	{ KVM_TRACE_EXIT_ODB,		"Out boundary" },	\
+	{ KVM_TRACE_EXIT_SYSCALL,	"System Call" },	\
+	{ KVM_TRACE_EXIT_BP,		"Breakpoint" },		\
+	{ KVM_TRACE_EXIT_INE,		"Reserved Inst" },	\
+	{ KVM_TRACE_EXIT_IPE,		"Inst prev error" },	\
+	{ KVM_TRACE_EXIT_FPDIS,		"FPU disable" },	\
+	{ KVM_TRACE_EXIT_LSXDIS,	"LSX disable" },	\
+	{ KVM_TRACE_EXIT_LASXDIS,	"LASX disable" },	\
+	{ KVM_TRACE_EXIT_FPE,		"FPE" },		\
+	{ KVM_TRACE_EXIT_WATCH,		"DEBUG" },		\
+	{ KVM_TRACE_EXIT_GSPR,		"GSPR" },		\
+	{ KVM_TRACE_EXIT_HC,		"Hypercall" },		\
+	{ KVM_TRACE_EXIT_GCM,		"CSR Mod" },		\
+	{ KVM_TRACE_EXIT_IDLE,		"IDLE" },		\
+	{ KVM_TRACE_EXIT_CACHE,		"CACHE" },		\
+	{ KVM_TRACE_EXIT_SIGNAL,	"Signal" }
+
+TRACE_EVENT(kvm_exit,
+	    TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason),
+	    TP_ARGS(vcpu, reason),
+	    TP_STRUCT__entry(
+			__field(unsigned long, pc)
+			__field(unsigned int, reason)
+	    ),
+
+	    TP_fast_assign(
+			__entry->pc = vcpu->arch.pc;
+			__entry->reason = reason;
+	    ),
+
+	    TP_printk("[%s]PC: 0x%08lx",
+		      __print_symbolic(__entry->reason,
+				       kvm_trace_symbol_exit_types),
+		      __entry->pc)
+);
+
+#define KVM_TRACE_AUX_RESTORE		0
+#define KVM_TRACE_AUX_SAVE		1
+#define KVM_TRACE_AUX_ENABLE		2
+#define KVM_TRACE_AUX_DISABLE		3
+#define KVM_TRACE_AUX_DISCARD		4
+
+#define KVM_TRACE_AUX_FPU		1
+#define KVM_TRACE_AUX_LSX		2
+#define KVM_TRACE_AUX_FPU_LSX		3
+#define KVM_TRACE_AUX_LASX		4
+#define KVM_TRACE_AUX_FPU_LSX_LASX	7
+
+#define kvm_trace_symbol_aux_op			\
+	{ KVM_TRACE_AUX_RESTORE, "restore" },	\
+	{ KVM_TRACE_AUX_SAVE,    "save" },	\
+	{ KVM_TRACE_AUX_ENABLE,  "enable" },	\
+	{ KVM_TRACE_AUX_DISABLE, "disable" },	\
+	{ KVM_TRACE_AUX_DISCARD, "discard" }
+
+#define kvm_trace_symbol_aux_state		\
+	{ KVM_TRACE_AUX_FPU,     "FPU" },	\
+	{ KVM_TRACE_AUX_LSX,     "LSX" },	\
+	{ KVM_TRACE_AUX_LASX,    "LASX" },	\
+	{ KVM_TRACE_AUX_FPU_LSX, "FPU & LSX" }, \
+	{ KVM_TRACE_AUX_FPU_LSX_LASX, "FPU & LSX & LASX" }
+
+TRACE_EVENT(kvm_aux,
+	    TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op,
+		     unsigned int state),
+	    TP_ARGS(vcpu, op, state),
+	    TP_STRUCT__entry(
+			__field(unsigned long, pc)
+			__field(u8, op)
+			__field(u8, state)
+	    ),
+
+	    TP_fast_assign(
+			__entry->pc = vcpu->arch.pc;
+			__entry->op = op;
+			__entry->state = state;
+	    ),
+
+	    TP_printk("%s %s PC: 0x%08lx",
+		      __print_symbolic(__entry->op,
+				       kvm_trace_symbol_aux_op),
+		      __print_symbolic(__entry->state,
+				       kvm_trace_symbol_aux_state),
+		      __entry->pc)
+);
+
+TRACE_EVENT(kvm_vpid_change,
+	    TP_PROTO(struct kvm_vcpu *vcpu, unsigned long vpid),
+	    TP_ARGS(vcpu, vpid),
+	    TP_STRUCT__entry(
+			__field(unsigned long, vpid)
+	    ),
+
+	    TP_fast_assign(
+			__entry->vpid = vpid;
+	    ),
+
+	    TP_printk("vpid: 0x%08lx",
+		      __entry->vpid)
+);
+
+#endif /* _TRACE_KVM_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>

include/uapi/linux/kvm.h

@@ -251,6 +251,7 @@ struct kvm_hyperv_exit {
 #define KVM_EXIT_X86_RDMSR        29
 #define KVM_EXIT_X86_WRMSR        30
 #define KVM_EXIT_RISCV_SBI        31
+#define KVM_EXIT_LOONGARCH_IOCSR  36
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -320,6 +321,13 @@ struct kvm_run {
 			__u32 len;
 			__u8  is_write;
 		} mmio;
+		/* KVM_EXIT_LOONGARCH_IOCSR */
+		struct {
+			__u64 phys_addr;
+			__u8  data[8];
+			__u32 len;
+			__u8  is_write;
+		} iocsr_io;
 		/* KVM_EXIT_HYPERCALL */
 		struct {
 			__u64 nr;
@@ -703,6 +711,16 @@ struct kvm_s390_irq_state {
 	__u32 reserved[4];  /* will stay unused for compatibility reasons */
 };
 
+struct kvm_loongarch_vcpu_state {
+	__u8 online_vcpus;
+	__u8 is_migrate;
+	__u32 cpu_freq;
+	__u32 count_ctl;
+	__u64 irq_pending;
+	__u64 irq_clear;
+	__u64 core_ext_ioisr[4];
+};
+
 /* for KVM_SET_GUEST_DEBUG */
 
 #define KVM_GUESTDBG_ENABLE		0x00000001
@@ -1064,6 +1082,10 @@ struct kvm_ppc_resize_hpt {
 
 #define KVM_CAP_ARM_CPU_FEATURE 555
 
+#define KVM_CAP_LOONGARCH_FPU 800
+#define KVM_CAP_LOONGARCH_LSX 801
+#define KVM_CAP_LOONGARCH_VZ 802
+
 #ifdef KVM_CAP_IRQ_ROUTING
 
 struct kvm_irq_routing_irqchip {
@@ -1210,6 +1232,7 @@ struct kvm_dirty_tlb {
 #define KVM_REG_ARM64		0x6000000000000000ULL
 #define KVM_REG_MIPS		0x7000000000000000ULL
 #define KVM_REG_RISCV		0x8000000000000000ULL
+#define KVM_REG_LOONGARCH	0x9000000000000000ULL
 
 #define KVM_REG_SIZE_SHIFT	52
 #define KVM_REG_SIZE_MASK	0x00f0000000000000ULL
@@ -1547,6 +1570,14 @@ struct kvm_enc_region {
 #define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
 #define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
 
+/* Add for LOONGSON read nodecounter */
+#define KVM_LOONGARCH_GET_VCPU_STATE	_IOR(KVMIO,   0xc0, struct kvm_loongarch_vcpu_state)
+#define KVM_LOONGARCH_SET_VCPU_STATE	_IOW(KVMIO,   0xc1, struct kvm_loongarch_vcpu_state)
+#define KVM_LOONGARCH_GET_CPUCFG	_IOR(KVMIO,   0xc2, struct kvm_cpucfg)
+#define KVM_LOONGARCH_GET_IOCSR		_IOR(KVMIO,   0xc3, struct kvm_iocsr_entry)
+#define KVM_LOONGARCH_SET_IOCSR		_IOW(KVMIO,   0xc4, struct kvm_iocsr_entry)
+#define KVM_LOONGARCH_SET_CPUCFG	_IOR(KVMIO,   0xc5, struct kvm_cpucfg)
+
 struct kvm_s390_pv_sec_parm {
 	__u64 origin;
 	__u64 length;