diff --git a/arch/loongarch/Kbuild b/arch/loongarch/Kbuild index ab5373d0a24ffa024386c451e553c105bf11da55..7760f9cb9777abd16e8f05e93633f3f1b9ef5f53 100644 --- a/arch/loongarch/Kbuild +++ b/arch/loongarch/Kbuild @@ -2,5 +2,9 @@ obj-y += kernel/ obj-y += mm/ obj-y += vdso/ +ifdef CONFIG_KVM +obj-y += kvm/ +endif + # for cleaning subdir- += boot diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 0436e79e3928ca738e8cb8e34e7e046b2f445b0e..34eff7c52582d8a89bbe5bdfb320c9536836c0a1 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -106,6 +106,7 @@ config LOONGARCH select HAVE_SETUP_PER_CPU_AREA if NUMA select HAVE_SYSCALL_TRACEPOINTS select HAVE_TIF_NOHZ + select HAVE_KVM select HAVE_VIRT_CPU_ACCOUNTING_GEN if !SMP select IRQ_FORCED_THREADING select IRQ_LOONGARCH_CPU @@ -539,3 +540,4 @@ source "drivers/cpufreq/Kconfig" endmenu source "drivers/firmware/Kconfig" +source "arch/loongarch/kvm/Kconfig" diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index 705b8344e5622537a28a04076c2799c3650d8cc9..5a55f27f718cc1f846db3e3733c922144916199a 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -49,6 +49,11 @@ CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_LOONGSON3_ACPI_CPUFREQ=y CONFIG_EFI_CAPSULE_LOADER=m CONFIG_EFI_TEST=m +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=m +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m CONFIG_MODULES=y CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild index a0eed6076c79a4c773a898e7e27739810541ea6b..3e0c3e4e57aeafeef53cdcd8e2f53ca531721b1e 100644 --- a/arch/loongarch/include/asm/Kbuild +++ b/arch/loongarch/include/asm/Kbuild @@ -26,4 +26,3 @@ generic-y += poll.h generic-y += param.h generic-y += posix_types.h generic-y += resource.h -generic-y += kvm_para.h diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index a74e221b73bcc1d7fe435e4d585ba790b67bdeee..b7ebf2aa217a4b98f3c64c5dd6f7845172974b04 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -42,17 +42,18 @@ enum reg1i21_op { }; enum reg2i12_op { - addiw_op = 0x0a, - addid_op = 0x0b, - lu52id_op = 0x0c, - ldb_op = 0xa0, - ldh_op = 0xa1, - ldw_op = 0xa2, - ldd_op = 0xa3, - stb_op = 0xa4, - sth_op = 0xa5, - stw_op = 0xa6, - std_op = 0xa7, + slti_op = 0x8, sltui_op, addiw_op, addid_op, + lu52id_op, cache_op = 0x18, xvldreplb_op = 0xca, + ldb_op = 0xa0, ldh_op, ldw_op, ldd_op, stb_op, sth_op, + stw_op, std_op, ldbu_op, ldhu_op, ldwu_op, preld_op, + flds_op, fsts_op, fldd_op, fstd_op, vld_op, vst_op, xvld_op, + xvst_op, ldlw_op = 0xb8, ldrw_op, ldld_op, ldrd_op, stlw_op, + strw_op, stld_op, strd_op, vldreplb_op = 0xc2, +}; + +enum reg2i14_op { + llw_op = 0x20, scw_op, lld_op, scd_op, ldptrw_op, stptrw_op, + ldptrd_op, stptrd_op, }; enum reg2i16_op { @@ -65,6 +66,49 @@ enum reg2i16_op { bgeu_op = 0x1b, }; +enum reg3_op { + asrtled_op = 0x2, asrtgtd_op, + addw_op = 0x20, addd_op, subw_op, subd_op, + slt_op, sltu_op, maskeqz_op, masknez_op, + nor_op, and_op, or_op, xor_op, orn_op, + andn_op, sllw_op, srlw_op, sraw_op, slld_op, + srld_op, srad_op, rotrb_op, rotrh_op, + rotrw_op, rotrd_op, mulw_op, mulhw_op, + mulhwu_op, muld_op, mulhd_op, mulhdu_op, + mulwdw_op, mulwdwu_op, divw_op, modw_op, + divwu_op, modwu_op, divd_op, modd_op, + divdu_op, moddu_op, crcwbw_op, + crcwhw_op, crcwww_op, crcwdw_op, crccwbw_op, + crccwhw_op, crccwww_op, crccwdw_op, addu12iw_op, + addu12id_op, + adcb_op = 0x60, adch_op, adcw_op, adcd_op, + sbcb_op, sbch_op, sbcw_op, sbcd_op, + rcrb_op, rcrh_op, rcrw_op, rcrd_op, + ldxb_op = 0x7000, ldxh_op = 0x7008, ldxw_op = 0x7010, ldxd_op = 0x7018, + stxb_op = 0x7020, stxh_op = 0x7028, stxw_op = 0x7030, stxd_op = 0x7038, + ldxbu_op = 0x7040, ldxhu_op = 0x7048, ldxwu_op = 0x7050, + preldx_op = 0x7058, fldxs_op = 0x7060, fldxd_op = 0x7068, + fstxs_op = 0x7070, fstxd_op = 0x7078, vldx_op = 0x7080, + vstx_op = 0x7088, xvldx_op = 0x7090, xvstx_op = 0x7098, + amswapw_op = 0x70c0, amswapd_op, amaddw_op, amaddd_op, amandw_op, + amandd_op, amorw_op, amord_op, amxorw_op, amxord_op, ammaxw_op, + ammaxd_op, amminw_op, ammind_op, ammaxwu_op, ammaxdu_op, + amminwu_op, ammindu_op, amswap_dbw_op, amswap_dbd_op, amadd_dbw_op, + amadd_dbd_op, amand_dbw_op, amand_dbd_op, amor_dbw_op, amor_dbd_op, + amxor_dbw_op, amxor_dbd_op, ammax_dbw_op, ammax_dbd_op, ammin_dbw_op, + ammin_dbd_op, ammax_dbwu_op, ammax_dbdu_op, ammin_dbwu_op, + ammin_dbdu_op, fldgts_op = 0x70e8, fldgtd_op, + fldles_op, fldled_op, fstgts_op, fstgtd_op, fstles_op, fstled_op, + ldgtb_op, ldgth_op, ldgtw_op, ldgtd_op, ldleb_op, ldleh_op, ldlew_op, + ldled_op, stgtb_op, stgth_op, stgtw_op, stgtd_op, stleb_op, stleh_op, + stlew_op, stled_op, +}; + +enum reg2_op { + iocsrrdb_op = 0x19200, iocsrrdh_op, iocsrrdw_op, iocsrrdd_op, + iocsrwrb_op, iocsrwrh_op, iocsrwrw_op, iocsrwrd_op, +}; + struct reg0i26_format { unsigned int immediate_h : 10; unsigned int immediate_l : 16; @@ -84,6 +128,12 @@ struct reg1i21_format { unsigned int opcode : 6; }; +struct reg2_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int opcode : 22; +}; + struct reg2i12_format { unsigned int rd : 5; unsigned int rj : 5; @@ -91,6 +141,18 @@ struct reg2i12_format { unsigned int opcode : 10; }; +struct reg2i14_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int simmediate : 14; + unsigned int opcode : 8; +}; + +struct reg0i15_format { + unsigned int simmediate : 15; + unsigned int opcode : 17; +}; + struct reg2i16_format { unsigned int rd : 5; unsigned int rj : 5; @@ -98,13 +160,32 @@ struct reg2i16_format { unsigned int opcode : 6; }; +struct reg3_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int rk : 5; + unsigned int opcode : 17; +}; + +struct reg2csr_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int csr : 14; + unsigned int opcode : 8; +}; + union loongarch_instruction { unsigned int word; struct reg0i26_format reg0i26_format; struct reg1i20_format reg1i20_format; struct reg1i21_format reg1i21_format; + struct reg3_format reg3_format; + struct reg2_format reg2_format; struct reg2i12_format reg2i12_format; + struct reg2i14_format reg2i14_format; struct reg2i16_format reg2i16_format; + struct reg2csr_format reg2csr_format; + struct reg0i15_format reg0i15_format; }; #define LOONGARCH_INSN_SIZE sizeof(union loongarch_instruction) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h new file mode 100644 index 0000000000000000000000000000000000000000..8f181f8cfa8cf467b9f337d8a61becdffc94053f --- /dev/null +++ b/arch/loongarch/include/asm/kvm_host.h @@ -0,0 +1,341 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LOONGARCH_KVM_HOST_H__ +#define __LOONGARCH_KVM_HOST_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Loongarch KVM register ids */ +#define LOONGARCH_CSR_32(_R, _S) \ + (KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U32 | (8 * (_R) + (_S))) + +#define LOONGARCH_CSR_64(_R, _S) \ + (KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U64 | (8 * (_R) + (_S))) + +#define KVM_IOC_CSRID(id) LOONGARCH_CSR_64(id, 0) +#define KVM_GET_IOC_CSRIDX(id) ((id & KVM_CSR_IDX_MASK) >> 3) + +#define LOONGSON_VIRT_REG_BASE 0x1f000000 +#define KVM_MAX_VCPUS 256 +#define KVM_USER_MEM_SLOTS 256 +/* memory slots that does not exposed to userspace */ +#define KVM_PRIVATE_MEM_SLOTS 0 + +#define KVM_HALT_POLL_NS_DEFAULT 500000 + +#define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(1) +#define KVM_INVALID_ADDR 0xdeadbeef +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +struct kvm_vm_stat { + ulong remote_tlb_flush; + u64 vm_ioctl_irq_line; + u64 ls7a_ioapic_update; + u64 ls7a_ioapic_set_irq; + u64 ioapic_reg_write; + u64 ioapic_reg_read; + u64 set_ls7a_ioapic; + u64 get_ls7a_ioapic; + u64 set_ls3a_ext_irq; + u64 get_ls3a_ext_irq; + u64 trigger_ls3a_ext_irq; + u64 pip_read_exits; + u64 pip_write_exits; + u64 ls7a_msi_irq; +}; +struct kvm_vcpu_stat { + u64 excep_exits[EXCCODE_INT_START]; + u64 idle_exits; + u64 signal_exits; + u64 int_exits; + u64 rdcsr_cpu_feature_exits; + u64 rdcsr_misc_func_exits; + u64 rdcsr_ipi_access_exits; + u64 cpucfg_exits; + u64 huge_dec_exits; + u64 huge_thp_exits; + u64 huge_adjust_exits; + u64 huge_set_exits; + u64 huge_merge_exits; + u64 halt_successful_poll; + u64 halt_attempted_poll; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; + u64 halt_poll_invalid; + u64 halt_wakeup; +}; + +#define KVM_MEMSLOT_DISABLE_THP (1UL << 17) +struct kvm_arch_memory_slot { + unsigned int flags; +}; + +enum { + IOCSR_FEATURES, + IOCSR_VENDOR, + IOCSR_CPUNAME, + IOCSR_NODECNT, + IOCSR_MISC_FUNC, + IOCSR_MAX +}; + +struct kvm_context { + unsigned long gid_mask; + unsigned long gid_ver_mask; + unsigned long gid_fisrt_ver; + unsigned long vpid_cache; + struct kvm_vcpu *last_vcpu; +}; + +struct kvm_arch { + /* Guest physical mm */ + struct mm_struct gpa_mm; + /* Mask of CPUs needing GPA ASID flush */ + cpumask_t asid_flush_mask; + + unsigned char online_vcpus; + unsigned char is_migrate; + s64 stablecounter_gftoffset; + u32 cpucfg_lasx; + struct ls7a_kvm_ioapic *v_ioapic; + struct ls3a_kvm_ipi *v_gipi; + struct ls3a_kvm_routerirq *v_routerirq; + struct ls3a_kvm_extirq *v_extirq; + spinlock_t iocsr_lock; + struct kvm_iocsr_entry iocsr[IOCSR_MAX]; + struct kvm_cpucfg cpucfgs; + struct kvm_context __percpu *vmcs; +}; + + +#define LOONGARCH_CSRS 0x100 +#define CSR_UCWIN_BASE 0x100 +#define CSR_UCWIN_SIZE 0x10 +#define CSR_DMWIN_BASE 0x180 +#define CSR_DMWIN_SIZE 0x4 +#define CSR_PERF_BASE 0x200 +#define CSR_PERF_SIZE 0x8 +#define CSR_DEBUG_BASE 0x500 +#define CSR_DEBUG_SIZE 0x3 +#define CSR_ALL_SIZE 0x800 + +struct loongarch_csrs { + unsigned long csrs[CSR_ALL_SIZE]; +}; + +/* Resume Flags */ +#define RESUME_FLAG_DR (1<<0) /* Reload guest nonvolatile state? */ +#define RESUME_FLAG_HOST (1<<1) /* Resume host? */ + +#define RESUME_GUEST 0 +#define RESUME_GUEST_DR RESUME_FLAG_DR +#define RESUME_HOST RESUME_FLAG_HOST + +enum emulation_result { + EMULATE_DONE, /* no further processing */ + EMULATE_DO_MMIO, /* kvm_run filled with MMIO request */ + EMULATE_FAIL, /* can't emulate this instruction */ + EMULATE_WAIT, /* WAIT instruction */ + EMULATE_PRIV_FAIL, + EMULATE_EXCEPT, /* A guest exception has been generated */ + EMULATE_PV_HYPERCALL, /* HYPCALL instruction */ + EMULATE_DEBUG, /* Emulate guest kernel debug */ + EMULATE_DO_IOCSR, /* handle IOCSR request */ +}; + +#define KVM_LARCH_FPU (0x1 << 0) +#define KVM_LARCH_LSX (0x1 << 1) +#define KVM_LARCH_LASX (0x1 << 2) +#define KVM_LARCH_DATA_HWBP (0x1 << 3) +#define KVM_LARCH_INST_HWBP (0x1 << 4) +#define KVM_LARCH_HWBP (KVM_LARCH_DATA_HWBP | KVM_LARCH_INST_HWBP) +#define KVM_LARCH_RESET (0x1 << 5) +#define KVM_LARCH_PERF (0x1 << 6) + +struct kvm_vcpu_arch { + unsigned long guest_eentry; + unsigned long host_eentry; + int (*vcpu_run)(struct kvm_run *run, struct kvm_vcpu *vcpu); + int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); + + /* Host registers preserved across guest mode execution */ + unsigned long host_stack; + unsigned long host_gp; + unsigned long host_pgd; + unsigned long host_pgdhi; + unsigned long host_entryhi; + + /* Host CSR registers used when handling exits from guest */ + unsigned long badv; + unsigned long host_estat; + unsigned long badi; + unsigned long host_ecfg; + unsigned long host_percpu; + + u32 is_hypcall; + /* GPRS */ + unsigned long gprs[32]; + unsigned long pc; + + /* FPU State */ + struct loongarch_fpu fpu FPU_ALIGN; + /* Which auxiliary state is loaded (KVM_LOONGARCH_AUX_*) */ + unsigned int aux_inuse; + + /* CSR State */ + struct loongarch_csrs *csr; + + /* GPR used as IO source/target */ + u32 io_gpr; + + struct hrtimer swtimer; + /* Count timer control KVM register */ + u32 count_ctl; + + /* Bitmask of exceptions that are pending */ + unsigned long irq_pending; + /* Bitmask of pending exceptions to be cleared */ + unsigned long irq_clear; + + /* Cache some mmu pages needed inside spinlock regions */ + struct kvm_mmu_memory_cache mmu_page_cache; + + /* vcpu's vpid is different on each host cpu in an smp system */ + u64 vpid[NR_CPUS]; + + /* Period of stable timer tick in ns */ + u64 timer_period; + /* Frequency of stable timer in Hz */ + u64 timer_mhz; + /* Stable bias from the raw time */ + u64 timer_bias; + /* Dynamic nanosecond bias (multiple of timer_period) to avoid overflow */ + s64 timer_dyn_bias; + /* Save ktime */ + ktime_t stable_ktime_saved; + + u64 core_ext_ioisr[4]; + + /* Last CPU the VCPU state was loaded on */ + int last_sched_cpu; + /* Last CPU the VCPU actually executed guest code on */ + int last_exec_cpu; + + u8 fpu_enabled; + u8 lsx_enabled; + /* paravirt steal time */ + struct { + u64 guest_addr; + u64 last_steal; + struct gfn_to_pfn_cache cache; + } st; + struct kvm_guest_debug_arch guest_debug; + /* save host pmu csr */ + u64 perf_ctrl[4]; + u64 perf_cntr[4]; + +}; + +static inline unsigned long readl_sw_gcsr(struct loongarch_csrs *csr, int reg) +{ + return csr->csrs[reg]; +} + +static inline void writel_sw_gcsr(struct loongarch_csrs *csr, int reg, \ + unsigned long val) +{ + csr->csrs[reg] = val; +} + +/* Helpers */ +static inline bool _kvm_guest_has_fpu(struct kvm_vcpu_arch *arch) +{ + return cpu_has_fpu && arch->fpu_enabled; +} + + +static inline bool _kvm_guest_has_lsx(struct kvm_vcpu_arch *arch) +{ + return cpu_has_lsx && arch->lsx_enabled; +} + +bool _kvm_guest_has_lasx(struct kvm_vcpu *vcpu); +void _kvm_init_fault(void); + +/* Debug: dump vcpu state */ +int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu); + +/* MMU handling */ +int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write); +void kvm_flush_tlb_all(void); +void _kvm_destroy_mm(struct kvm *kvm); +pgd_t *kvm_pgd_alloc(void); +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); + +enum _kvm_fault_result { + KVM_LOONGARCH_MAPPED = 0, + KVM_LOONGARCH_GVA, + KVM_LOONGARCH_GPA, + KVM_LOONGARCH_TLB, + KVM_LOONGARCH_TLBINV, + KVM_LOONGARCH_TLBMOD, +}; + +#define KVM_ARCH_WANT_MMU_NOTIFIER +int kvm_unmap_hva_range(struct kvm *kvm, + unsigned long start, unsigned long end, bool blockable); +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); + +static inline void update_pc(struct kvm_vcpu_arch *arch) +{ + arch->pc += 4; +} + +/** + * kvm_is_ifetch_fault() - Find whether a TLBL exception is due to ifetch fault. + * @vcpu: Virtual CPU. + * + * Returns: Whether the TLBL exception was likely due to an instruction + * fetch fault rather than a data load fault. + */ +static inline bool kvm_is_ifetch_fault(struct kvm_vcpu_arch *arch) +{ + if (arch->pc == arch->badv) + return true; + + return false; +} + +/* Misc */ +static inline void kvm_arch_hardware_unsetup(void) {} +static inline void kvm_arch_sync_events(struct kvm *kvm) {} +static inline void kvm_arch_free_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} + +extern int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu); +extern void kvm_exception_entry(void); +#endif /* __LOONGARCH_KVM_HOST_H__ */ diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h new file mode 100644 index 0000000000000000000000000000000000000000..bd9bb0bf0f70397520df6c8c0c7e1789b62e1d66 --- /dev/null +++ b/arch/loongarch/include/asm/kvm_para.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_KVM_PARA_H +#define _ASM_LOONGARCH_KVM_PARA_H + +/* + * Hypcall code field + */ +#define KVM_HC_CODE_SERIVCE 0x0 +#define KVM_HC_CODE_SWDBG 0x5 +/* + * function id + * 0x00000 ~ 0xfffff Standard Hypervisor Calls + */ +#define KVM_HC_FUNC_FEATURE 0x0 +#define KVM_HC_FUNC_NOTIFY 0x1 +#define KVM_HC_FUNC_IPI 0x2 +/* + * LoongArch support PV feature list + */ +#define KVM_FEATURE_STEAL_TIME 0 +#define KVM_FEATURE_MULTI_IPI 1 +/* + * LoongArch hypcall return code + */ +#define KVM_RET_SUC 1 +#define KVM_RET_NOT_SUPPORTED -1 + +static inline bool kvm_check_and_clear_guest_paused(void) +{ + return false; +} + +static inline unsigned int kvm_arch_para_features(void) +{ + return 0; +} + +static inline unsigned int kvm_arch_para_hints(void) +{ + return 0; +} + +#endif /* _ASM_LOONGARCH_KVM_PARA_H */ diff --git a/arch/loongarch/include/asm/kvm_types.h b/arch/loongarch/include/asm/kvm_types.h new file mode 100644 index 0000000000000000000000000000000000000000..b79845a347f5a2953847e4cc8563286696f46ade --- /dev/null +++ b/arch/loongarch/include/asm/kvm_types.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH64_KVM_TYPES_H +#define _ASM_LOONGARCH64_KVM_TYPES_H + +#define KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 4 + +#endif /* _ASM_LOONGARCH64_KVM_TYPES_H */ + diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h index 71189b28bfb2723b5a066ad6990337848c2a1baf..92cfe170313542c97778e46d1d403e55f7c0dd5d 100644 --- a/arch/loongarch/include/asm/smp.h +++ b/arch/loongarch/include/asm/smp.h @@ -78,16 +78,6 @@ extern void calculate_cpu_foreign_map(void); */ extern void show_ipi_list(struct seq_file *p, int prec); -/* - * This function sends a 'reschedule' IPI to another CPU. - * it goes straight through and wastes no time serializing - * anything. Worst case is that we lose a reschedule ... - */ -static inline void smp_send_reschedule(int cpu) -{ - loongson3_send_ipi_single(cpu, SMP_RESCHEDULE); -} - static inline void arch_send_call_function_single_ipi(int cpu) { loongson3_send_ipi_single(cpu, SMP_CALL_FUNCTION); diff --git a/arch/loongarch/include/uapi/asm/kvm.h b/arch/loongarch/include/uapi/asm/kvm.h new file mode 100644 index 0000000000000000000000000000000000000000..3e785a2c6dd0dd5081a95bc5a2565fe02e17b550 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/kvm.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Loongson Technologies, Inc. All rights reserved. + * Authors: Sanjay Lal + * Authors: Xing Li + */ + +#ifndef __LINUX_KVM_LOONGARCH_H +#define __LINUX_KVM_LOONGARCH_H + +#include +#ifndef __KERNEL__ +#include +#endif + +#define __KVM_HAVE_GUEST_DEBUG +#define KVM_GUESTDBG_USE_SW_BP 0x00010000 +#define KVM_GUESTDBG_USE_HW_BP 0x00020000 +#define KVM_DATA_HW_BREAKPOINT_NUM 8 +#define KVM_INST_HW_BREAKPOINT_NUM 8 + +/* + * KVM Loongarch specific structures and definitions. + * + * Some parts derived from the x86 version of this file. + */ + +#define __KVM_HAVE_READONLY_MEM + +#define KVM_COALESCED_MMIO_PAGE_OFFSET 1 + +/* + * for KVM_GET_REGS and KVM_SET_REGS + */ +struct kvm_regs { + /* out (KVM_GET_REGS) / in (KVM_SET_REGS) */ + __u64 gpr[32]; + __u64 pc; +}; + +/* + * for KVM_GET_CPUCFG + */ +struct kvm_cpucfg { + /* out (KVM_GET_CPUCFG) */ + __u32 cpucfg[64]; +}; + +/* + * for KVM_GET_FPU and KVM_SET_FPU + */ +struct kvm_fpu { + __u32 fcsr; + __u32 none; + __u64 fcc; /* 8x8 */ + struct kvm_fpureg { + __u64 val64[4]; //support max 256 bits + } fpr[32]; +}; + +/* + * For LOONGARCH, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access various + * registers. The id field is broken down as follows: + * + * bits[63..52] - As per linux/kvm.h + * bits[51..32] - Must be zero. + * bits[31..16] - Register set. + * + * Register set = 0: GP registers from kvm_regs (see definitions below). + * + * Register set = 1: CSR registers. + * + * Register set = 2: KVM specific registers (see definitions below). + * + * Register set = 3: FPU / SIMD registers (see definitions below). + * + * Other sets registers may be added in the future. Each set would + * have its own identifier in bits[31..16]. + */ + +#define KVM_REG_LOONGARCH_GP (KVM_REG_LOONGARCH | 0x00000ULL) +#define KVM_REG_LOONGARCH_CSR (KVM_REG_LOONGARCH | 0x10000ULL) +#define KVM_REG_LOONGARCH_KVM (KVM_REG_LOONGARCH | 0x20000ULL) +#define KVM_REG_LOONGARCH_FPU (KVM_REG_LOONGARCH | 0x30000ULL) +#define KVM_REG_LOONGARCH_MASK (KVM_REG_LOONGARCH | 0x30000ULL) +#define KVM_CSR_IDX_MASK (0x10000 - 1) + +/* + * KVM_REG_LOONGARCH_KVM - KVM specific control registers. + */ + +#define KVM_REG_LOONGARCH_COUNTER (KVM_REG_LOONGARCH_KVM | KVM_REG_SIZE_U64 | 3) +#define KVM_REG_LOONGARCH_VCPU_RESET (KVM_REG_LOONGARCH_KVM | KVM_REG_SIZE_U64 | 4) + +#define __KVM_HAVE_IRQ_LINE + +struct kvm_debug_exit_arch { + __u64 era; + __u32 fwps; + __u32 mwps; + __u32 exception; +}; + +/* for KVM_SET_GUEST_DEBUG */ +struct hw_breakpoint { + __u64 addr; + __u64 mask; + __u32 asid; + __u32 ctrl; +}; + +struct kvm_guest_debug_arch { + struct hw_breakpoint data_breakpoint[KVM_DATA_HW_BREAKPOINT_NUM]; + struct hw_breakpoint inst_breakpoint[KVM_INST_HW_BREAKPOINT_NUM]; + int inst_bp_nums, data_bp_nums; +}; + +/* definition of registers in kvm_run */ +struct kvm_sync_regs { +}; + +/* dummy definition */ +struct kvm_sregs { +}; + +struct kvm_iocsr_entry { + __u32 addr; + __u32 pad; + __u64 data; +}; + +struct kvm_csr_entry { + __u32 index; + __u32 reserved; + __u64 data; +}; + +/* for KVM_GET_MSRS and KVM_SET_MSRS */ +struct kvm_msrs { + __u32 ncsrs; /* number of msrs in entries */ + __u32 pad; + + struct kvm_csr_entry entries[0]; +}; + +struct kvm_loongarch_interrupt { + /* in */ + __u32 cpu; + __u32 irq; +}; + +#define KVM_IRQCHIP_LS7A_IOAPIC 0x0 +#define KVM_IRQCHIP_LS3A_GIPI 0x1 +#define KVM_IRQCHIP_LS3A_HT_IRQ 0x2 +#define KVM_IRQCHIP_LS3A_ROUTE 0x3 +#define KVM_IRQCHIP_LS3A_EXTIRQ 0x4 +#define KVM_IRQCHIP_LS3A_IPMASK 0x5 +#define KVM_NR_IRQCHIPS 1 +#define KVM_IRQCHIP_NUM_PINS 64 + +#define KVM_MAX_CORES 256 +#define KVM_EXTIOI_IRQS (256) +#define KVM_EXTIOI_IRQS_BITMAP_SIZE (KVM_EXTIOI_IRQS / 8) +/* map to ipnum per 32 irqs */ +#define KVM_EXTIOI_IRQS_IPMAP_SIZE (KVM_EXTIOI_IRQS / 32) +#define KVM_EXTIOI_IRQS_PER_GROUP 32 +#define KVM_EXTIOI_IRQS_COREMAP_SIZE (KVM_EXTIOI_IRQS) +#define KVM_EXTIOI_IRQS_NODETYPE_SIZE 16 + +struct ls7a_ioapic_state { + /* 0x000 interrupt id register */ + __u64 int_id; + /* 0x020 interrupt mask register */ + __u64 int_mask; + /* 0x040 1=msi */ + __u64 htmsi_en; + /* 0x060 edge=1 level =0 */ + __u64 intedge; + /* 0x080 for clean edge int,set 1 clean,set 0 is noused */ + __u64 intclr; + /* 0x0c0 */ + __u64 auto_crtl0; + /* 0x0e0 */ + __u64 auto_crtl1; + /* 0x100 - 0x140 */ + __u8 route_entry[64]; + /* 0x200 - 0x240 */ + __u8 htmsi_vector[64]; + /* 0x300 */ + __u64 intisr_chip0; + /* 0x320 */ + __u64 intisr_chip1; + /* edge detection */ + __u64 last_intirr; + /* 0x380 interrupt request register */ + __u64 intirr; + /* 0x3a0 interrupt service register */ + __u64 intisr; + /* 0x3e0 interrupt level polarity selection register, + * 0 for high level tirgger + */ + __u64 int_polarity; +}; + +struct loongarch_gipi_single { + __u32 status; + __u32 en; + __u32 set; + __u32 clear; + __u64 buf[4]; +}; + +struct loongarch_gipiState { + struct loongarch_gipi_single core[KVM_MAX_CORES]; +}; + +struct kvm_loongarch_ls3a_extirq_state { + union ext_en_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } ext_en_r; + union bounce_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } bounce_r; + union ext_isr_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } ext_isr_r; + union ext_core_isr_r { + uint64_t reg_u64[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_MAX_CORES][KVM_EXTIOI_IRQS_BITMAP_SIZE]; + } ext_core_isr_r; + union ip_map_r { + uint64_t reg_u64; + uint32_t reg_u32[KVM_EXTIOI_IRQS_IPMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_IPMAP_SIZE]; + } ip_map_r; + union core_map_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_COREMAP_SIZE / 8]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_COREMAP_SIZE / 4]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_COREMAP_SIZE]; + } core_map_r; + union node_type_r { + uint64_t reg_u64[KVM_EXTIOI_IRQS_NODETYPE_SIZE / 4]; + uint32_t reg_u32[KVM_EXTIOI_IRQS_NODETYPE_SIZE / 2]; + uint16_t reg_u16[KVM_EXTIOI_IRQS_NODETYPE_SIZE]; + uint8_t reg_u8[KVM_EXTIOI_IRQS_NODETYPE_SIZE * 2]; + } node_type_r; +}; + +struct loongarch_kvm_irqchip { + __u16 chip_id; + __u16 len; + __u16 vcpu_id; + __u16 reserved; + char data[0]; +}; + +#endif /* __LINUX_KVM_LOONGARCH_H */ diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 8733fc347b3ea412fc3eb6ec06666497b501a0e2..9fdbe934624f6c57cecc0cbaeefae5bd9b0157f2 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -269,3 +270,35 @@ void output_pbe_defines(void) } #endif +void output_kvm_defines(void) +{ + COMMENT(" KVM/LOONGISA Specific offsets. "); + + OFFSET(VCPU_FCSR0, kvm_vcpu_arch, fpu.fcsr); + OFFSET(VCPU_FCC, kvm_vcpu_arch, fpu.fcc); + BLANK(); + + OFFSET(KVM_VCPU_ARCH, kvm_vcpu, arch); + OFFSET(KVM_VCPU_KVM, kvm_vcpu, kvm); + OFFSET(KVM_VCPU_RUN, kvm_vcpu, run); + BLANK(); + + OFFSET(KVM_ARCH_HSTACK, kvm_vcpu_arch, host_stack); + OFFSET(KVM_ARCH_HGP, kvm_vcpu_arch, host_gp); + OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit); + OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd); + OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry); + OFFSET(KVM_ARCH_GPC, kvm_vcpu_arch, pc); + OFFSET(KVM_ARCH_GGPR, kvm_vcpu_arch, gprs); + OFFSET(KVM_ARCH_HESTAT, kvm_vcpu_arch, host_estat); + OFFSET(KVM_ARCH_HBADV, kvm_vcpu_arch, badv); + OFFSET(KVM_ARCH_HBADI, kvm_vcpu_arch, badi); + OFFSET(KVM_ARCH_ISHYPCALL, kvm_vcpu_arch, is_hypcall); + OFFSET(KVM_ARCH_HECFG, kvm_vcpu_arch, host_ecfg); + OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry); + OFFSET(KVM_ARCH_HPERCPU, kvm_vcpu_arch, host_percpu); + + OFFSET(KVM_GPGD, kvm, arch.gpa_mm.pgd); + BLANK(); +} + diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 7e6dde83e6acbf767978ed14be1fe9b713156e3b..4b7b42cf19bdcf89c7ad8a8cf8d05b319cce774e 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -151,6 +151,17 @@ void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action) ipi_write_action(cpu_logical_map(i), (u32)action); } +/* + * This function sends a 'reschedule' IPI to another CPU. + * it goes straight through and wastes no time serializing + * anything. Worst case is that we lose a reschedule ... + */ +void smp_send_reschedule(int cpu) +{ + loongson3_send_ipi_single(cpu, SMP_RESCHEDULE); +} +EXPORT_SYMBOL_GPL(smp_send_reschedule); + irqreturn_t loongson3_ipi_interrupt(int irq, void *dev) { unsigned int action; diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..8cae26f5647fd0a0edf91d7aded39ffd94c5b52b --- /dev/null +++ b/arch/loongarch/kvm/Kconfig @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# KVM configuration +# +source "virt/kvm/Kconfig" + +menuconfig VIRTUALIZATION + bool "Virtualization" + help + Say Y here to get to see options for using your Linux host to run + other operating systems inside virtual machines (guests). + This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if VIRTUALIZATION + +config KVM + tristate "Kernel-based Virtual Machine (KVM) support" + depends on HAVE_KVM + select EXPORT_UASM + select PREEMPT_NOTIFIERS + select ANON_INODES + select KVM_GENERIC_DIRTYLOG_READ_PROTECT + select HAVE_KVM_VCPU_ASYNC_IOCTL + select KVM_MMIO + select MMU_NOTIFIER + select HAVE_KVM_IRQCHIP + select HAVE_KVM_IRQFD + select HAVE_KVM_IRQ_ROUTING + select HAVE_KVM_EVENTFD + select HAVE_KVM_MSI + select SRCU + select KVM_VFIO + help + Support for hosting Guest kernels. + +choice + prompt "Virtualization mode" + depends on KVM + default KVM_LOONGARCH_LVZ + +config KVM_LOONGARCH_LVZ + bool "LOONGARCH Virtualization (VZ) ASE" + help + Use the LOONGARCH Virtualization (VZ) ASE to virtualize guests. This + supports running unmodified guest kernels, but requires hardware + support. + +endchoice + +source "drivers/vhost/Kconfig" + +endif # VIRTUALIZATION diff --git a/arch/loongarch/kvm/Makefile b/arch/loongarch/kvm/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..29dc154609d70ed356ea01d8600f9a033fccee58 --- /dev/null +++ b/arch/loongarch/kvm/Makefile @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for KVM support for LoongArch +# + +OBJECT_FILES_NON_STANDARD_entry.o := y + +common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o \ + irqchip.o eventfd.o) + +KVM := ../../../virt/kvm +common-objs-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o + +EXTRA_CFLAGS += -Ivirt/kvm -Iarch/loongarch/kvm + +kvm-objs := $(common-objs-y) loongarch.o emulate.o interrupt.o +kvm-objs += hypcall.o +kvm-objs += mmu.o +kvm-objs += kvm_compat.o + +kvm-objs += exit.o intc/ls7a_irq.o intc/ls3a_ipi.o intc/irqchip-debug.o\ + timer.o intc/ls3a_ext_irq.o irqfd.o csr.o +obj-$(CONFIG_KVM) += kvm.o +obj-y += entry.o fpu.o diff --git a/arch/loongarch/kvm/csr.c b/arch/loongarch/kvm/csr.c new file mode 100644 index 0000000000000000000000000000000000000000..09ec781fe79aeeed4759be818f450157bf7fe602 --- /dev/null +++ b/arch/loongarch/kvm/csr.c @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include "kvmcpu.h" +#include "intc/ls3a_ipi.h" +#include "intc/ls3a_ext_irq.h" +#include "kvm_compat.h" +#include "kvmcsr.h" +#include "irq.h" + +#define CASE_READ_SW_GCSR(csr, regid, csrid) \ + do { \ + if (regid == csrid) { \ + return kvm_read_sw_gcsr(csr, csrid); \ + } \ + } while (0) + +unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + unsigned long val = 0; + + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRCTL); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO1); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO2); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_MERRERA); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_ERRSAVE); + /* read sw csr when not config pmu to guest */ + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL0); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL1); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL2); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCTRL3); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR0); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR1); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR2); + CASE_READ_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR3); + + val = 0; + if (csrid < 4096) + val = kvm_read_sw_gcsr(csr, csrid); + else + pr_warn_once("Unsupport csrread 0x%x with pc %lx\n", + csrid, vcpu->arch.pc); + return val; +} + +#define CASE_WRITE_SW_GCSR(csr, regid, csrid, val) \ + do { \ + if (regid == csrid) { \ + kvm_write_sw_gcsr(csr, csrid, val); \ + return ; \ + } \ + } while (0) + +void _kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, + unsigned long val) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRCTL, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO1, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO2, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_MERRERA, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_ERRSAVE, val); + + /* give pmu register to guest when config perfctrl */ + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL0, val); + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL1, val); + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL2, val); + CASE_WRITE_HW_PMU(vcpu, csr, csrid, KVM_CSR_PERFCTRL3, val); + /* write sw pmu csr if not config ctrl */ + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR0, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR1, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR2, val); + CASE_WRITE_SW_GCSR(csr, csrid, KVM_CSR_PERFCNTR3, val); + + + if (csrid < 4096) + kvm_write_sw_gcsr(csr, csrid, val); + else + pr_warn_once("Unsupport csrwrite 0x%x with pc %lx\n", + csrid, vcpu->arch.pc); +} + +#define CASE_CHANGE_SW_GCSR(csr, regid, csrid, mask, val) \ + do { \ + if (regid == csrid) { \ + kvm_change_sw_gcsr(csr, csrid, mask, val); \ + return ; \ + } \ + } while (0) + +void _kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, + unsigned long csr_mask, unsigned long val) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_IMPCTL1, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRCTL, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO1, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRINFO2, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRENTRY, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_MERRERA, csr_mask, val); + CASE_CHANGE_SW_GCSR(csr, csrid, KVM_CSR_ERRSAVE, csr_mask, val); + + if (csrid < 4096) { + unsigned long orig; + + orig = kvm_read_sw_gcsr(csr, csrid); + orig &= ~csr_mask; + orig |= val & csr_mask; + kvm_write_sw_gcsr(csr, csrid, orig); + } + pr_warn_once("Unsupport csrxchg 0x%x with pc %lx\n", + csrid, vcpu->arch.pc); +} + +int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + GET_HW_GCSR(id, KVM_CSR_CRMD, v); + GET_HW_GCSR(id, KVM_CSR_PRMD, v); + GET_HW_GCSR(id, KVM_CSR_EUEN, v); + GET_HW_GCSR(id, KVM_CSR_MISC, v); + GET_HW_GCSR(id, KVM_CSR_ECFG, v); + GET_HW_GCSR(id, KVM_CSR_ESTAT, v); + GET_HW_GCSR(id, KVM_CSR_ERA, v); + GET_HW_GCSR(id, KVM_CSR_BADV, v); + GET_HW_GCSR(id, KVM_CSR_BADI, v); + GET_HW_GCSR(id, KVM_CSR_EENTRY, v); + GET_HW_GCSR(id, KVM_CSR_TLBIDX, v); + GET_HW_GCSR(id, KVM_CSR_TLBEHI, v); + GET_HW_GCSR(id, KVM_CSR_TLBELO0, v); + GET_HW_GCSR(id, KVM_CSR_TLBELO1, v); + GET_HW_GCSR(id, KVM_CSR_ASID, v); + GET_HW_GCSR(id, KVM_CSR_PGDL, v); + GET_HW_GCSR(id, KVM_CSR_PGDH, v); + GET_HW_GCSR(id, KVM_CSR_PWCTL0, v); + GET_HW_GCSR(id, KVM_CSR_PWCTL1, v); + GET_HW_GCSR(id, KVM_CSR_STLBPGSIZE, v); + GET_HW_GCSR(id, KVM_CSR_RVACFG, v); + GET_HW_GCSR(id, KVM_CSR_CPUID, v); + GET_HW_GCSR(id, KVM_CSR_PRCFG1, v); + GET_HW_GCSR(id, KVM_CSR_PRCFG2, v); + GET_HW_GCSR(id, KVM_CSR_PRCFG3, v); + GET_HW_GCSR(id, KVM_CSR_KS0, v); + GET_HW_GCSR(id, KVM_CSR_KS1, v); + GET_HW_GCSR(id, KVM_CSR_KS2, v); + GET_HW_GCSR(id, KVM_CSR_KS3, v); + GET_HW_GCSR(id, KVM_CSR_KS4, v); + GET_HW_GCSR(id, KVM_CSR_KS5, v); + GET_HW_GCSR(id, KVM_CSR_KS6, v); + GET_HW_GCSR(id, KVM_CSR_KS7, v); + GET_HW_GCSR(id, KVM_CSR_TMID, v); + GET_HW_GCSR(id, KVM_CSR_TCFG, v); + GET_HW_GCSR(id, KVM_CSR_TVAL, v); + GET_HW_GCSR(id, KVM_CSR_CNTC, v); + GET_HW_GCSR(id, KVM_CSR_LLBCTL, v); + GET_HW_GCSR(id, KVM_CSR_TLBRENTRY, v); + GET_HW_GCSR(id, KVM_CSR_TLBRBADV, v); + GET_HW_GCSR(id, KVM_CSR_TLBRERA, v); + GET_HW_GCSR(id, KVM_CSR_TLBRSAVE, v); + GET_HW_GCSR(id, KVM_CSR_TLBRELO0, v); + GET_HW_GCSR(id, KVM_CSR_TLBRELO1, v); + GET_HW_GCSR(id, KVM_CSR_TLBREHI, v); + GET_HW_GCSR(id, KVM_CSR_TLBRPRMD, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN0, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN1, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN2, v); + GET_HW_GCSR(id, KVM_CSR_DMWIN3, v); + GET_HW_GCSR(id, KVM_CSR_MWPS, v); + GET_HW_GCSR(id, KVM_CSR_FWPS, v); + + GET_SW_GCSR(csr, id, KVM_CSR_IMPCTL1, v); + GET_SW_GCSR(csr, id, KVM_CSR_IMPCTL2, v); + GET_SW_GCSR(csr, id, KVM_CSR_ERRCTL, v); + GET_SW_GCSR(csr, id, KVM_CSR_ERRINFO1, v); + GET_SW_GCSR(csr, id, KVM_CSR_ERRINFO2, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRENTRY, v); + GET_SW_GCSR(csr, id, KVM_CSR_MERRERA, v); + GET_SW_GCSR(csr, id, KVM_CSR_ERRSAVE, v); + GET_SW_GCSR(csr, id, KVM_CSR_CTAG, v); + GET_SW_GCSR(csr, id, KVM_CSR_DEBUG, v); + GET_SW_GCSR(csr, id, KVM_CSR_DERA, v); + GET_SW_GCSR(csr, id, KVM_CSR_DESAVE, v); + + GET_SW_GCSR(csr, id, KVM_CSR_TINTCLR, v); + + if (force && (id < CSR_ALL_SIZE)) { + *v = kvm_read_sw_gcsr(csr, id); + return 0; + } + + return -1; +} + +int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + int ret; + + SET_HW_GCSR(csr, id, KVM_CSR_CRMD, v); + SET_HW_GCSR(csr, id, KVM_CSR_PRMD, v); + SET_HW_GCSR(csr, id, KVM_CSR_EUEN, v); + SET_HW_GCSR(csr, id, KVM_CSR_MISC, v); + SET_HW_GCSR(csr, id, KVM_CSR_ECFG, v); + SET_HW_GCSR(csr, id, KVM_CSR_ERA, v); + SET_HW_GCSR(csr, id, KVM_CSR_BADV, v); + SET_HW_GCSR(csr, id, KVM_CSR_BADI, v); + SET_HW_GCSR(csr, id, KVM_CSR_EENTRY, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBIDX, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBEHI, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBELO0, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBELO1, v); + SET_HW_GCSR(csr, id, KVM_CSR_ASID, v); + SET_HW_GCSR(csr, id, KVM_CSR_PGDL, v); + SET_HW_GCSR(csr, id, KVM_CSR_PGDH, v); + SET_HW_GCSR(csr, id, KVM_CSR_PWCTL0, v); + SET_HW_GCSR(csr, id, KVM_CSR_PWCTL1, v); + SET_HW_GCSR(csr, id, KVM_CSR_STLBPGSIZE, v); + SET_HW_GCSR(csr, id, KVM_CSR_RVACFG, v); + SET_HW_GCSR(csr, id, KVM_CSR_CPUID, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS0, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS1, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS2, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS3, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS4, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS5, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS6, v); + SET_HW_GCSR(csr, id, KVM_CSR_KS7, v); + SET_HW_GCSR(csr, id, KVM_CSR_TMID, v); + SET_HW_GCSR(csr, id, KVM_CSR_TCFG, v); + SET_HW_GCSR(csr, id, KVM_CSR_TVAL, v); + SET_HW_GCSR(csr, id, KVM_CSR_CNTC, v); + SET_HW_GCSR(csr, id, KVM_CSR_LLBCTL, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRENTRY, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRBADV, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRERA, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRSAVE, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRELO0, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRELO1, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBREHI, v); + SET_HW_GCSR(csr, id, KVM_CSR_TLBRPRMD, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN0, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN1, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN2, v); + SET_HW_GCSR(csr, id, KVM_CSR_DMWIN3, v); + SET_HW_GCSR(csr, id, KVM_CSR_MWPS, v); + SET_HW_GCSR(csr, id, KVM_CSR_FWPS, v); + + SET_SW_GCSR(csr, id, KVM_CSR_IMPCTL1, v); + SET_SW_GCSR(csr, id, KVM_CSR_IMPCTL2, v); + SET_SW_GCSR(csr, id, KVM_CSR_ERRCTL, v); + SET_SW_GCSR(csr, id, KVM_CSR_ERRINFO1, v); + SET_SW_GCSR(csr, id, KVM_CSR_ERRINFO2, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRENTRY, v); + SET_SW_GCSR(csr, id, KVM_CSR_MERRERA, v); + SET_SW_GCSR(csr, id, KVM_CSR_ERRSAVE, v); + SET_SW_GCSR(csr, id, KVM_CSR_CTAG, v); + SET_SW_GCSR(csr, id, KVM_CSR_DEBUG, v); + SET_SW_GCSR(csr, id, KVM_CSR_DERA, v); + SET_SW_GCSR(csr, id, KVM_CSR_DESAVE, v); + SET_SW_GCSR(csr, id, KVM_CSR_PRCFG1, v); + SET_SW_GCSR(csr, id, KVM_CSR_PRCFG2, v); + SET_SW_GCSR(csr, id, KVM_CSR_PRCFG3, v); + + SET_SW_GCSR(csr, id, KVM_CSR_PGD, v); + SET_SW_GCSR(csr, id, KVM_CSR_TINTCLR, v); + + ret = -1; + switch (id) { + case KVM_CSR_ESTAT: + kvm_write_gcsr_estat(*v); + /* estat IP0~IP7 inject through guestexcept */ + kvm_write_csr_gintc(((*v) >> 2) & 0xff); + ret = 0; + break; + default: + if (force && (id < CSR_ALL_SIZE)) { + kvm_set_sw_gcsr(csr, id, *v); + ret = 0; + } + break; + } + + return ret; +} + +struct kvm_iocsr { + u32 start, end; + int (*get) (struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, u64 *res); + int (*set) (struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, u64 val); +}; + +static struct kvm_iocsr_entry *_kvm_find_iocsr(struct kvm *kvm, u32 addr) +{ + int i = 0; + + for (i = 0; i < IOCSR_MAX; i++) { + if (addr == kvm->arch.iocsr[i].addr) + return &kvm->arch.iocsr[i]; + } + + return NULL; +} + +static int kvm_iocsr_common_get(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 *res) +{ + int r = EMULATE_FAIL; + struct kvm_iocsr_entry *entry; + + spin_lock(&vcpu->kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(vcpu->kvm, addr); + if (entry) { + r = EMULATE_DONE; + *res = entry->data; + } + spin_unlock(&vcpu->kvm->arch.iocsr_lock); + return r; +} + +static int kvm_iocsr_common_set(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 val) +{ + int r = EMULATE_FAIL; + struct kvm_iocsr_entry *entry; + + spin_lock(&vcpu->kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(vcpu->kvm, addr); + if (entry) { + r = EMULATE_DONE; + entry->data = val; + } + spin_unlock(&vcpu->kvm->arch.iocsr_lock); + return r; +} + +static int kvm_misc_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + if ((val & KVM_IOCSRF_MISC_FUNC_EXT_IOI_EN) && vcpu->vcpu_id == 0) + kvm_setup_ls3a_extirq(vcpu->kvm); + return kvm_iocsr_common_set(run, vcpu, addr, val); +} + +static int kvm_ipi_get(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 *res) +{ + int ret; + + ++vcpu->stat.rdcsr_ipi_access_exits; + run->mmio.phys_addr = KVM_IPI_REG_ADDRESS(vcpu->vcpu_id, (addr & 0xff)); + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, res); + if (ret) { + run->mmio.is_write = 0; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 0; + return EMULATE_DO_MMIO; + } + return EMULATE_DONE; +} + +static int kvm_extioi_isr_get(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 *res) +{ + int ret; + + run->mmio.phys_addr = EXTIOI_PERCORE_ADDR(vcpu->vcpu_id, (addr & 0xff)); + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, res); + if (ret) { + run->mmio.is_write = 0; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 0; + return EMULATE_FAIL; + } + + return EMULATE_DONE; +} + +static int kvm_ipi_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + int ret; + + run->mmio.phys_addr = KVM_IPI_REG_ADDRESS(vcpu->vcpu_id, (addr & 0xff)); + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, &val); + if (ret < 0) { + run->mmio.is_write = 1; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 1; + return EMULATE_DO_MMIO; + } + + return EMULATE_DONE; +} + +static int kvm_extioi_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + int ret; + + if ((addr & 0x1f00) == KVM_IOCSR_EXTIOI_ISR_BASE) { + run->mmio.phys_addr = EXTIOI_PERCORE_ADDR(vcpu->vcpu_id, (addr & 0xff)); + } else { + run->mmio.phys_addr = EXTIOI_ADDR((addr & 0x1fff)); + } + + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, &val); + if (ret < 0) { + memcpy(run->mmio.data, &val, run->mmio.len); + run->mmio.is_write = 1; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 1; + return EMULATE_DO_MMIO; + } + + return EMULATE_DONE; +} + +static int kvm_nop_set(struct kvm_run *run, struct kvm_vcpu *vcpu, u32 addr, + u64 val) +{ + return EMULATE_DONE; +} + +/* we put these iocsrs with access frequency, from high to low */ +static struct kvm_iocsr kvm_iocsrs[] = { + /* extioi iocsr */ + {KVM_IOCSR_EXTIOI_EN_BASE, KVM_IOCSR_EXTIOI_EN_BASE + 0x100, + NULL, kvm_extioi_set}, + {KVM_IOCSR_EXTIOI_NODEMAP_BASE, KVM_IOCSR_EXTIOI_NODEMAP_BASE+0x28, + NULL, kvm_extioi_set}, + {KVM_IOCSR_EXTIOI_ROUTE_BASE, KVM_IOCSR_EXTIOI_ROUTE_BASE + 0x100, + NULL, kvm_extioi_set}, + {KVM_IOCSR_EXTIOI_ISR_BASE, KVM_IOCSR_EXTIOI_ISR_BASE + 0x1c, + kvm_extioi_isr_get, kvm_extioi_set}, + + {KVM_IOCSR_IPI_STATUS, KVM_IOCSR_IPI_STATUS + 0x40, + kvm_ipi_get, kvm_ipi_set}, + {KVM_IOCSR_IPI_SEND, KVM_IOCSR_IPI_SEND + 0x1, + NULL, kvm_ipi_set}, + {KVM_IOCSR_MBUF_SEND, KVM_IOCSR_MBUF_SEND + 0x1, + NULL, kvm_ipi_set}, + + {KVM_IOCSR_FEATURES, KVM_IOCSR_FEATURES + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_VENDOR, KVM_IOCSR_VENDOR + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_CPUNAME, KVM_IOCSR_CPUNAME + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_NODECNT, KVM_IOCSR_NODECNT + 0x1, + kvm_iocsr_common_get, kvm_nop_set}, + {KVM_IOCSR_MISC_FUNC, KVM_IOCSR_MISC_FUNC + 0x1, + kvm_iocsr_common_get, kvm_misc_set}, +}; + +static int _kvm_emu_iocsr_read(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 *res) +{ + enum emulation_result er = EMULATE_FAIL; + int i = 0; + struct kvm_iocsr *iocsr = NULL; + + if (!irqchip_in_kernel(vcpu->kvm)) { + run->iocsr_io.len = run->mmio.len; + run->iocsr_io.phys_addr = addr; + run->iocsr_io.is_write = 0; + return EMULATE_DO_IOCSR; + } + for (i = 0; i < sizeof(kvm_iocsrs) / sizeof(struct kvm_iocsr); i++) { + iocsr = &kvm_iocsrs[i]; + if (addr >= iocsr->start && addr < iocsr->end) { + if (iocsr->get) + er = iocsr->get(run, vcpu, addr, res); + } + } + + if (er != EMULATE_DONE) + kvm_debug("%s iocsr 0x%x not support in kvm\n", __func__, addr); + + return er; +} + +static int _kvm_emu_iocsr_write(struct kvm_run *run, struct kvm_vcpu *vcpu, + u32 addr, u64 val) +{ + enum emulation_result er = EMULATE_FAIL; + int i = 0; + struct kvm_iocsr *iocsr = NULL; + + if (!irqchip_in_kernel(vcpu->kvm)) { + run->iocsr_io.len = run->mmio.len; + memcpy(run->iocsr_io.data, &val, run->iocsr_io.len); + run->iocsr_io.phys_addr = addr; + run->iocsr_io.is_write = 1; + return EMULATE_DO_IOCSR; + } + for (i = 0; i < sizeof(kvm_iocsrs) / sizeof(struct kvm_iocsr); i++) { + iocsr = &kvm_iocsrs[i]; + if (addr >= iocsr->start && addr < iocsr->end) { + if (iocsr->set) + er = iocsr->set(run, vcpu, addr, val); + } + } + if (er != EMULATE_DONE) + kvm_debug("%s iocsr 0x%x not support in kvm\n", __func__, addr); + + return er; +} + +/* all iocsr operation should in kvm, no mmio */ +int _kvm_emu_iocsr(larch_inst inst, + struct kvm_run *run, struct kvm_vcpu *vcpu) +{ + u32 rd, rj, opcode; + u32 val; + u64 res = 0; + int ret; + + /* + * Each IOCSR with different opcode + */ + rd = inst.reg2_format.rd; + rj = inst.reg2_format.rj; + opcode = inst.reg2_format.opcode; + val = vcpu->arch.gprs[rj]; + res = vcpu->arch.gprs[rd]; + /* LoongArch is Little endian */ + switch (opcode) { + case iocsrrdb_op: + run->mmio.len = 1; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = (u8) res; + break; + case iocsrrdh_op: + run->mmio.len = 2; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = (u16) res; + break; + case iocsrrdw_op: + run->mmio.len = 4; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = (u32) res; + break; + case iocsrrdd_op: + run->mmio.len = 8; + ret = _kvm_emu_iocsr_read(run, vcpu, val, &res); + vcpu->arch.gprs[rd] = res; + break; + case iocsrwrb_op: + run->mmio.len = 1; + ret = _kvm_emu_iocsr_write(run, vcpu, val, (u8)res); + break; + case iocsrwrh_op: + run->mmio.len = 2; + ret = _kvm_emu_iocsr_write(run, vcpu, val, (u16)res); + break; + case iocsrwrw_op: + run->mmio.len = 4; + ret = _kvm_emu_iocsr_write(run, vcpu, val, (u32)res); + break; + case iocsrwrd_op: + run->mmio.len = 8; + ret = _kvm_emu_iocsr_write(run, vcpu, val, res); + break; + default: + ret = EMULATE_FAIL; + break; + } + + if (ret == EMULATE_DO_IOCSR) { + vcpu->arch.io_gpr = rd; + } + + return ret; +} + +int _kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; + enum emulation_result er = EMULATE_DONE; + + switch (run->iocsr_io.len) { + case 8: + *gpr = *(s64 *)run->iocsr_io.data; + break; + case 4: + *gpr = *(int *)run->iocsr_io.data; + break; + case 2: + *gpr = *(short *)run->iocsr_io.data; + break; + case 1: + *gpr = *(char *) run->iocsr_io.data; + break; + default: + kvm_err("Bad IOCSR length: %d,addr is 0x%lx", + run->iocsr_io.len, vcpu->arch.badv); + er = EMULATE_FAIL; + break; + } + + return er; +} + +int _kvm_get_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp) +{ + struct kvm_iocsr_entry *entry, tmp; + int r = -EFAULT; + + if (copy_from_user(&tmp, argp, sizeof(tmp))) + goto out; + + spin_lock(&kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(kvm, tmp.addr); + if (entry != NULL) + tmp.data = entry->data; + spin_unlock(&kvm->arch.iocsr_lock); + + if (entry) + r = copy_to_user(argp, &tmp, sizeof(tmp)); + +out: + return r; +} + +int _kvm_set_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp) +{ + struct kvm_iocsr_entry *entry, tmp; + int r = -EFAULT; + + if (copy_from_user(&tmp, argp, sizeof(tmp))) + goto out; + + spin_lock(&kvm->arch.iocsr_lock); + entry = _kvm_find_iocsr(kvm, tmp.addr); + if (entry != NULL) { + r = 0; + entry->data = tmp.data; + } + spin_unlock(&kvm->arch.iocsr_lock); + + if (tmp.addr == KVM_IOCSR_MISC_FUNC) + kvm_enable_ls3a_extirq(kvm, tmp.data & KVM_IOCSRF_MISC_FUNC_EXT_IOI_EN); + +out: + return r; +} + +static struct kvm_iocsr_entry iocsr_array[IOCSR_MAX] = { + {KVM_IOCSR_FEATURES, .data = KVM_IOCSRF_NODECNT|KVM_IOCSRF_MSI + |KVM_IOCSRF_EXTIOI|KVM_IOCSRF_CSRIPI|KVM_IOCSRF_VM}, + {KVM_IOCSR_VENDOR, .data = 0x6e6f73676e6f6f4c}, /* Loongson */ + {KVM_IOCSR_CPUNAME, .data = 0x303030354133}, /* 3A5000 */ + {KVM_IOCSR_NODECNT, .data = 0x4}, + {KVM_IOCSR_MISC_FUNC, .data = 0x0}, +}; + +int _kvm_init_iocsr(struct kvm *kvm) +{ + int i = 0; + + spin_lock_init(&kvm->arch.iocsr_lock); + for (i = 0; i < IOCSR_MAX; i++) { + kvm->arch.iocsr[i].addr = iocsr_array[i].addr; + kvm->arch.iocsr[i].data = iocsr_array[i].data; + } + return 0; +} diff --git a/arch/loongarch/kvm/emulate.c b/arch/loongarch/kvm/emulate.c new file mode 100644 index 0000000000000000000000000000000000000000..01bafcb3abb35b5b4adb4741b3def75e581b2e82 --- /dev/null +++ b/arch/loongarch/kvm/emulate.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include "trace.h" + +int _kvm_emu_idle(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.idle_exits; + trace_kvm_exit(vcpu, KVM_TRACE_EXIT_IDLE); + if (!vcpu->arch.irq_pending) { + kvm_save_timer(vcpu); + kvm_vcpu_block(vcpu); + + /* + * We we are runnable, then definitely go off to user space to + * check if any I/O interrupts are pending. + */ + if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) { + kvm_clear_request(KVM_REQ_UNHALT, vcpu); + vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; + } + } + + return EMULATE_DONE; +} + +int _kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst) +{ + struct kvm_run *run = vcpu->run; + unsigned int rd, op8, opcode; + unsigned long rd_val = 0; + void *data = run->mmio.data; + unsigned long curr_pc; + int ret = 0; + + /* + * Update PC and hold onto current PC in case there is + * an error and we want to rollback the PC + */ + curr_pc = vcpu->arch.pc; + update_pc(&vcpu->arch); + + op8 = (inst.word >> 24) & 0xff; + run->mmio.phys_addr = vcpu->arch.badv; + if (run->mmio.phys_addr == KVM_INVALID_ADDR) + goto out_fail; + + if (op8 < 0x28) { + /* stptrw/d process */ + rd = inst.reg2i14_format.rd; + opcode = inst.reg2i14_format.opcode; + + switch (opcode) { + case stptrd_op: + run->mmio.len = 8; + *(unsigned long *)data = vcpu->arch.gprs[rd]; + break; + case stptrw_op: + run->mmio.len = 4; + *(unsigned int *)data = vcpu->arch.gprs[rd]; + break; + default: + break; + } + } else if (op8 < 0x30) { + /* st.b/h/w/d process */ + rd = inst.reg2i12_format.rd; + opcode = inst.reg2i12_format.opcode; + rd_val = vcpu->arch.gprs[rd]; + + switch (opcode) { + case std_op: + run->mmio.len = 8; + *(unsigned long *)data = rd_val; + break; + case stw_op: + run->mmio.len = 4; + *(unsigned int *)data = rd_val; + break; + case sth_op: + run->mmio.len = 2; + *(unsigned short *)data = rd_val; + break; + case stb_op: + run->mmio.len = 1; + *(unsigned char *)data = rd_val; + break; + default: + kvm_err("Store not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + goto out_fail; + } + } else if (op8 == 0x38) { + /* stxb/h/w/d process */ + rd = inst.reg3_format.rd; + opcode = inst.reg3_format.opcode; + + switch (opcode) { + case stxb_op: + run->mmio.len = 1; + *(unsigned char *)data = vcpu->arch.gprs[rd]; + break; + case stxh_op: + run->mmio.len = 2; + *(unsigned short *)data = vcpu->arch.gprs[rd]; + break; + case stxw_op: + run->mmio.len = 4; + *(unsigned int *)data = vcpu->arch.gprs[rd]; + break; + case stxd_op: + run->mmio.len = 8; + *(unsigned long *)data = vcpu->arch.gprs[rd]; + break; + default: + kvm_err("Store not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + goto out_fail; + } + } else { + kvm_err("Store not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + goto out_fail; + } + + /* All MMIO emulate in kernel go through the common interface */ + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, data); + if (!ret) { + vcpu->mmio_needed = 0; + return EMULATE_DONE; + } + + run->mmio.is_write = 1; + vcpu->mmio_needed = 1; + vcpu->mmio_is_write = 1; + + return EMULATE_DO_MMIO; + +out_fail: + /* Rollback PC if emulation was unsuccessful */ + vcpu->arch.pc = curr_pc; + return EMULATE_FAIL; +} + + +int _kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) +{ + unsigned int op8, opcode, rd; + int ret = 0; + struct kvm_run *run = vcpu->run; + + run->mmio.phys_addr = vcpu->arch.badv; + if (run->mmio.phys_addr == KVM_INVALID_ADDR) + return EMULATE_FAIL; + + vcpu->mmio_needed = 2; /* signed */ + op8 = (inst.word >> 24) & 0xff; + + if (op8 < 0x28) { + /* ldptr.w/d process */ + rd = inst.reg2i14_format.rd; + opcode = inst.reg2i14_format.opcode; + + switch (opcode) { + case ldptrd_op: + run->mmio.len = 8; + break; + case ldptrw_op: + run->mmio.len = 4; + break; + default: + break; + } + } else if (op8 < 0x2f) { + /* ld.b/h/w/d, ld.bu/hu/wu process */ + rd = inst.reg2i12_format.rd; + opcode = inst.reg2i12_format.opcode; + + switch (opcode) { + case ldd_op: + run->mmio.len = 8; + break; + case ldwu_op: + vcpu->mmio_needed = 1; /* unsigned */ + run->mmio.len = 4; + break; + case ldw_op: + run->mmio.len = 4; + break; + case ldhu_op: + vcpu->mmio_needed = 1; /* unsigned */ + run->mmio.len = 2; + break; + case ldh_op: + run->mmio.len = 2; + break; + case ldbu_op: + vcpu->mmio_needed = 1; /* unsigned */ + run->mmio.len = 1; + break; + case ldb_op: + run->mmio.len = 1; + break; + default: + kvm_err("Load not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->mmio_needed = 0; + return EMULATE_FAIL; + } + } else if (op8 == 0x38) { + /* ldxb/h/w/d, ldxb/h/wu, ldgtb/h/w/d, ldleb/h/w/d process */ + rd = inst.reg3_format.rd; + opcode = inst.reg3_format.opcode; + + switch (opcode) { + case ldxb_op: + run->mmio.len = 1; + break; + case ldxbu_op: + run->mmio.len = 1; + vcpu->mmio_needed = 1; /* unsigned */ + break; + case ldxh_op: + run->mmio.len = 2; + break; + case ldxhu_op: + run->mmio.len = 2; + vcpu->mmio_needed = 1; /* unsigned */ + break; + case ldxw_op: + run->mmio.len = 4; + break; + case ldxwu_op: + run->mmio.len = 4; + vcpu->mmio_needed = 1; /* unsigned */ + break; + case ldxd_op: + run->mmio.len = 8; + break; + default: + kvm_err("Load not yet supporded (inst=0x%08x)\n", + inst.word); + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->mmio_needed = 0; + return EMULATE_FAIL; + } + } else { + kvm_err("Load not yet supporded (inst=0x%08x) @ %lx\n", + inst.word, vcpu->arch.pc); + vcpu->mmio_needed = 0; + return EMULATE_FAIL; + } + + /* Set for _kvm_complete_mmio_read use */ + vcpu->arch.io_gpr = rd; + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, run->mmio.data); + run->mmio.is_write = 0; + vcpu->mmio_is_write = 0; + + if (!ret) { + _kvm_complete_mmio_read(vcpu, run); + vcpu->mmio_needed = 0; + return EMULATE_DONE; + } + return EMULATE_DO_MMIO; +} + +int _kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; + enum emulation_result er = EMULATE_DONE; + + /* update with new PC */ + update_pc(&vcpu->arch); + switch (run->mmio.len) { + case 8: + *gpr = *(s64 *)run->mmio.data; + break; + + case 4: + if (vcpu->mmio_needed == 2) { + *gpr = *(int *)run->mmio.data; + } else + *gpr = *(unsigned int *)run->mmio.data; + break; + + case 2: + if (vcpu->mmio_needed == 2) + *gpr = *(short *) run->mmio.data; + else + *gpr = *(unsigned short *)run->mmio.data; + + break; + case 1: + if (vcpu->mmio_needed == 2) + *gpr = *(char *) run->mmio.data; + else + *gpr = *(unsigned char *) run->mmio.data; + break; + default: + kvm_err("Bad MMIO length: %d,addr is 0x%lx", + run->mmio.len, vcpu->arch.badv); + er = EMULATE_FAIL; + break; + } + + return er; +} diff --git a/arch/loongarch/kvm/entry.S b/arch/loongarch/kvm/entry.S new file mode 100644 index 0000000000000000000000000000000000000000..cc0856af60d94fa392cc577c6a52676204981b14 --- /dev/null +++ b/arch/loongarch/kvm/entry.S @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include "kvm_compat.h" + +#define RESUME_HOST (1 << 1) + +#define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x) +#define PT_GPR_OFFSET(x) (PT_R0 + 8*x) + + .text + +.macro kvm_save_guest_gprs base + .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_S $r\n, \base, GGPR_OFFSET(\n) + .endr +.endm + +.macro kvm_restore_guest_gprs base + .irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_L $r\n, \base, GGPR_OFFSET(\n) + .endr +.endm + +.macro kvm_save_host_gpr base + .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_S $r\n, \base, PT_GPR_OFFSET(\n) + .endr +.endm + +.macro kvm_restore_host_gpr base + .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 + KVM_LONG_L $r\n, \base, PT_GPR_OFFSET(\n) + .endr +.endm + +/* + * prepare switch to guest + * @param: + * KVM_ARCH: kvm_vcpu_arch, don't touch it until 'ertn' + * GPRNUM: KVM_ARCH gpr number + * tmp, tmp1: temp register + */ +.macro kvm_switch_to_guest KVM_ARCH GPRNUM tmp tmp1 + /* set host excfg.VS=0, all exceptions share one exception entry */ + csrrd \tmp, KVM_CSR_ECFG + bstrins.w \tmp, zero, (KVM_ECFG_VS_SHIFT + KVM_ECFG_VS_WIDTH - 1), KVM_ECFG_VS_SHIFT + csrwr \tmp, KVM_CSR_ECFG + + /* Load up the new EENTRY */ + KVM_LONG_L \tmp, \KVM_ARCH, KVM_ARCH_GEENTRY + csrwr \tmp, KVM_CSR_EENTRY + + /* Set Guest ERA */ + KVM_LONG_L \tmp, \KVM_ARCH, KVM_ARCH_GPC + csrwr \tmp, KVM_CSR_ERA + + /* Save host PGDL */ + csrrd \tmp, KVM_CSR_PGDL + KVM_LONG_S \tmp, \KVM_ARCH, KVM_ARCH_HPGD + + /* Switch to kvm */ + KVM_LONG_L \tmp1, \KVM_ARCH, KVM_VCPU_KVM - KVM_VCPU_ARCH + + /* Load guest PGDL */ + lu12i.w \tmp, KVM_GPGD + srli.w \tmp, \tmp, 12 + ldx.d \tmp, \tmp1, \tmp + csrwr \tmp, KVM_CSR_PGDL + + /* Mix GID and RID */ + csrrd \tmp1, KVM_CSR_GSTAT + bstrpick.w \tmp1, \tmp1, (KVM_GSTAT_GID_SHIFT + KVM_GSTAT_GID_WIDTH - 1), KVM_GSTAT_GID_SHIFT + csrrd \tmp, KVM_CSR_GTLBC + bstrins.w \tmp, \tmp1, (KVM_GTLBC_TGID_SHIFT + KVM_GTLBC_TGID_WIDTH - 1), KVM_GTLBC_TGID_SHIFT + csrwr \tmp, KVM_CSR_GTLBC + + /* + * Switch to guest: + * GSTAT.PGM = 1, ERRCTL.ISERR = 0, TLBRPRMD.ISTLBR = 0 + * ertn + */ + + /* Prepare enable Intr before enter guest */ + ori \tmp, zero, KVM_PRMD_PIE + csrxchg \tmp, \tmp, KVM_CSR_PRMD + + /* Set PVM bit to setup ertn to guest context */ + ori \tmp, zero, KVM_GSTAT_PVM + csrxchg \tmp, \tmp, KVM_CSR_GSTAT + + /* Load Guest gprs */ + kvm_restore_guest_gprs \KVM_ARCH + + /* Load KVM_ARCH register */ + KVM_LONG_L \KVM_ARCH, \KVM_ARCH, GGPR_OFFSET(\GPRNUM) + + ertn +.endm + +#ifndef EXCPTION_ENTRY +#define EXCPTION_ENTRY(name) \ + .globl name ASM_NL \ + .p2align 12; \ + name: \ + .cfi_startproc; +#endif +#ifndef EXCPTION_ENDPROC +#define EXCPTION_ENDPROC(name) \ + .cfi_endproc; \ + SYM_END(name, SYM_T_FUNC) +#endif + +/* load kvm_vcpu to a2 and store a1 for free use */ +EXCPTION_ENTRY(kvm_exception_entry) + csrwr a2, KVM_TEMP_KS + csrrd a2, KVM_VCPU_KS + KVM_LONG_ADDI a2, a2, KVM_VCPU_ARCH + + /* After save gprs, free to use any gpr */ + kvm_save_guest_gprs a2 + /* Save guest a2 */ + csrrd t0, KVM_TEMP_KS + KVM_LONG_S t0, a2, GGPR_OFFSET(REG_A2) + + b kvm_exit_entry +EXCPTION_ENDPROC(kvm_exception_entry) + +/* a2: kvm_vcpu_arch, a1 is free to use */ +SYM_FUNC_START(kvm_exit_entry) + csrrd s1, KVM_VCPU_KS + KVM_LONG_L s0, s1, KVM_VCPU_RUN + + csrrd t0, KVM_CSR_ESTAT + KVM_LONG_S t0, a2, KVM_ARCH_HESTAT + csrrd t0, KVM_CSR_ERA + KVM_LONG_S t0, a2, KVM_ARCH_GPC + csrrd t0, KVM_CSR_BADV + KVM_LONG_S t0, a2, KVM_ARCH_HBADV + csrrd t0, KVM_CSR_BADI + KVM_LONG_S t0, a2, KVM_ARCH_HBADI + + /* Restore host excfg.VS */ + csrrd t0, KVM_CSR_ECFG + KVM_LONG_L t1, a2, KVM_ARCH_HECFG + or t0, t0, t1 + csrwr t0, KVM_CSR_ECFG + + /* Restore host eentry */ + KVM_LONG_L t0, a2, KVM_ARCH_HEENTRY + csrwr t0, KVM_CSR_EENTRY + +#if defined(CONFIG_CPU_HAS_FPU) + /* Save FPU context */ + csrrd t0, KVM_CSR_EUEN + ori t1, zero, KVM_EUEN_FPEN | KVM_EUEN_LSXEN | KVM_EUEN_LASXEN + and t2, t0, t1 + beqz t2, 1f + movfcsr2gr t3, fcsr0 + INT_S t3, a2, VCPU_FCSR0 + + movcf2gr t3, $fcc0 + or t2, t3, zero + movcf2gr t3, $fcc1 + bstrins.d t2, t3, 0xf, 0x8 + movcf2gr t3, $fcc2 + bstrins.d t2, t3, 0x17, 0x10 + movcf2gr t3, $fcc3 + bstrins.d t2, t3, 0x1f, 0x18 + movcf2gr t3, $fcc4 + bstrins.d t2, t3, 0x27, 0x20 + movcf2gr t3, $fcc5 + bstrins.d t2, t3, 0x2f, 0x28 + movcf2gr t3, $fcc6 + bstrins.d t2, t3, 0x37, 0x30 + movcf2gr t3, $fcc7 + bstrins.d t2, t3, 0x3f, 0x38 + KVM_LONG_S t2, a2, VCPU_FCC + movgr2fcsr fcsr0, zero +1: +#endif + + KVM_LONG_L t0, a2, KVM_ARCH_HPGD + csrwr t0, KVM_CSR_PGDL + + /* Disable PVM bit for keeping from into guest */ + ori t0, zero, KVM_GSTAT_PVM + csrxchg zero, t0, KVM_CSR_GSTAT + + /* Clear GTLBC.TGID field */ + csrrd t0, KVM_CSR_GTLBC + bstrins.w t0, zero, KVM_GTLBC_TGID_SHIFT + KVM_GTLBC_TGID_WIDTH - 1, KVM_GTLBC_TGID_SHIFT + csrwr t0, KVM_CSR_GTLBC + + /* Enable Address Map mode */ + ori t0, zero, (1 << KVM_CRMD_DACM_SHIFT)|(1 << KVM_CRMD_DACF_SHIFT) | KVM_CRMD_PG |PLV_KERN + csrwr t0, KVM_CSR_CRMD + + KVM_LONG_L tp, a2, KVM_ARCH_HGP + KVM_LONG_L sp, a2, KVM_ARCH_HSTACK + /* restore per cpu register */ + KVM_LONG_L $r21, a2, KVM_ARCH_HPERCPU + + KVM_LONG_ADDI sp, sp, -PT_SIZE + + /* Prepare handle exception */ + or a0, s0, zero + or a1, s1, zero + KVM_LONG_L t8, a2, KVM_ARCH_HANDLE_EXIT + jirl ra,t8, 0 + + ori t0, zero, KVM_CRMD_IE + csrxchg zero, t0, KVM_CSR_CRMD + or a2, s1, zero + KVM_LONG_ADDI a2, a2, KVM_VCPU_ARCH + + andi t0, a0, RESUME_HOST + bnez t0, ret_to_host + INT_S zero, a2, KVM_ARCH_ISHYPCALL + +ret_to_guest: + /* Save per cpu register again, maybe switched to another cpu */ + KVM_LONG_S $r21, a2, KVM_ARCH_HPERCPU + + /* Save kvm_vcpu to kscratch */ + csrwr s1, KVM_VCPU_KS + kvm_switch_to_guest a2 REG_A2 t0 t1 + +ret_to_host: + KVM_LONG_L a2, a2, KVM_ARCH_HSTACK + addi.d a2, a2, -PT_SIZE + srai.w a3, a0, 2 + or a0, a3, zero + kvm_restore_host_gpr a2 + jirl zero, ra, 0 +SYM_FUNC_END(kvm_exit_entry) + +/* + * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu) + * + * @register_param: + * a0: kvm_run* run + * a1: kvm_vcpu* vcpu + */ +SYM_FUNC_START(kvm_enter_guest) + /* allocate space in stack bottom */ + KVM_LONG_ADDI a2, sp, -PT_SIZE + + /* save host gprs */ + kvm_save_host_gpr a2 + + /* save host crmd,prmd csr to stack */ + csrrd a3, KVM_CSR_CRMD + KVM_LONG_S a3, a2, PT_CRMD + csrrd a3, KVM_CSR_PRMD + KVM_LONG_S a3, a2, PT_PRMD + + KVM_LONG_ADDI a2, a1, KVM_VCPU_ARCH + KVM_LONG_S sp, a2, KVM_ARCH_HSTACK + KVM_LONG_S tp, a2, KVM_ARCH_HGP + /* Save per cpu register */ + KVM_LONG_S $r21, a2, KVM_ARCH_HPERCPU + + /* Save kvm_vcpu to kscratch */ + csrwr a1, KVM_VCPU_KS + + kvm_switch_to_guest a2 REG_A2 t0 t1 + +SYM_FUNC_END(kvm_enter_guest) + +SYM_FUNC_START(__kvm_save_fpu) + fpu_save_double a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_save_fpu) + +SYM_FUNC_START(__kvm_restore_fpu) + fpu_restore_double a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_fpu) + +SYM_FUNC_START(__kvm_restore_fcsr) + fpu_restore_csr a0 t1 + fpu_restore_cc a0 t1 t2 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_fcsr) + +#ifdef CONFIG_CPU_HAS_LSX +SYM_FUNC_START(__kvm_save_lsx) + lsx_save_data a0 t1 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_save_lsx) + +SYM_FUNC_START(__kvm_restore_lsx) + lsx_restore_data a0 t1 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_lsx) + +SYM_FUNC_START(__kvm_restore_lsx_upper) + lsx_restore_all_upper a0 t0 t1 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_lsx_upper) +#endif + +#ifdef CONFIG_CPU_HAS_LASX +SYM_FUNC_START(__kvm_save_lasx) + lasx_save_data a0 t7 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_save_lasx) + +SYM_FUNC_START(__kvm_restore_lasx) + lasx_restore_data a0 t7 + + jirl zero, ra, 0 +SYM_FUNC_END(__kvm_restore_lasx) +#endif + diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c new file mode 100644 index 0000000000000000000000000000000000000000..5653e082d43ecedfc4b3d40d0f3b6d021c6b8a80 --- /dev/null +++ b/arch/loongarch/kvm/exit.c @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include +#include "trace.h" +#include "kvm_compat.h" +#include "kvmcsr.h" +#include "intc/ls3a_ext_irq.h" + +/* + * Loongarch KVM callback handling for not implemented guest exiting + */ +static int _kvm_fault_ni(struct kvm_vcpu *vcpu) +{ + unsigned long estat, badv; + unsigned int exccode, inst; + + /* + * Fetch the instruction. + */ + badv = vcpu->arch.badv; + estat = vcpu->arch.host_estat; + exccode = (estat & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT; + inst = vcpu->arch.badi; + kvm_err("Exccode: %d PC=%#lx inst=0x%08x BadVaddr=%#lx estat=%#llx\n", + exccode, vcpu->arch.pc, inst, badv, kvm_read_gcsr_estat()); + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; +} + +static int _kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst) +{ + enum emulation_result er = EMULATE_DONE; + unsigned int rd, rj, csrid; + unsigned long csr_mask; + unsigned long val = 0; + + /* + * CSR value mask imm + * rj = 0 means csrrd + * rj = 1 means csrwr + * rj != 0,1 means csrxchg + */ + rd = inst.reg2csr_format.rd; + rj = inst.reg2csr_format.rj; + csrid = inst.reg2csr_format.csr; + + /* Process CSR ops */ + if (rj == 0) { + /* process csrrd */ + val = _kvm_emu_read_csr(vcpu, csrid); + if (er != EMULATE_FAIL) + vcpu->arch.gprs[rd] = val; + } else if (rj == 1) { + /* process csrwr */ + val = vcpu->arch.gprs[rd]; + _kvm_emu_write_csr(vcpu, csrid, val); + } else { + /* process csrxchg */ + val = vcpu->arch.gprs[rd]; + csr_mask = vcpu->arch.gprs[rj]; + _kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val); + } + + return er; +} + +static int _kvm_emu_cache(struct kvm_vcpu *vcpu, larch_inst inst) +{ + return EMULATE_DONE; +} + +static int _kvm_trap_handle_gspr(struct kvm_vcpu *vcpu) +{ + enum emulation_result er = EMULATE_DONE; + struct kvm_run *run = vcpu->run; + larch_inst inst; + unsigned long curr_pc; + int rd, rj; + unsigned int index; + + /* + * Fetch the instruction. + */ + inst.word = vcpu->arch.badi; + curr_pc = vcpu->arch.pc; + update_pc(&vcpu->arch); + + er = EMULATE_FAIL; + switch (((inst.word >> 24) & 0xff)) { + case 0x0: + /* cpucfg GSPR */ + if (inst.reg2_format.opcode == 0x1B) { + rd = inst.reg2_format.rd; + rj = inst.reg2_format.rj; + ++vcpu->stat.cpucfg_exits; + index = vcpu->arch.gprs[rj]; + vcpu->arch.gprs[rd] = vcpu->kvm->arch.cpucfgs.cpucfg[index]; + if (vcpu->arch.gprs[rd] == 0) { + /* + * Fallback to get host cpucfg info, this is just for + * compatible with older qemu. + */ + vcpu->arch.gprs[rd] = read_cpucfg(index); + /* Nested KVM is not supported */ + if (index == 2) + vcpu->arch.gprs[rd] &= ~CPUCFG2_LVZP; + } + er = EMULATE_DONE; + } + break; + case 0x4: + /* csr GSPR */ + er = _kvm_handle_csr(vcpu, inst); + break; + case 0x6: + /* iocsr,cache,idle GSPR */ + switch (((inst.word >> 22) & 0x3ff)) { + case 0x18: + /* cache GSPR */ + er = _kvm_emu_cache(vcpu, inst); + trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); + break; + case 0x19: + /* iocsr/idle GSPR */ + switch (((inst.word >> 15) & 0x1ffff)) { + case 0xc90: + /* iocsr GSPR */ + er = _kvm_emu_iocsr(inst, run, vcpu); + break; + case 0xc91: + /* idle GSPR */ + er = _kvm_emu_idle(vcpu); + break; + default: + er = EMULATE_FAIL; + break; + } + break; + default: + er = EMULATE_FAIL; + break; + } + break; + default: + er = EMULATE_FAIL; + break; + } + + /* Rollback PC only if emulation was unsuccessful */ + if (er == EMULATE_FAIL) { + kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n", + curr_pc, __func__, inst.word); + + kvm_arch_vcpu_dump_regs(vcpu); + vcpu->arch.pc = curr_pc; + } + return er; +} + +static int _kvm_check_hypcall(struct kvm_vcpu *vcpu) +{ + enum emulation_result ret; + larch_inst inst; + unsigned long curr_pc; + unsigned int code; + + /* + * Update PC and hold onto current PC in case there is + * an error and we want to rollback the PC + */ + inst.word = vcpu->arch.badi; + code = inst.reg0i15_format.simmediate; + curr_pc = vcpu->arch.pc; + update_pc(&vcpu->arch); + + ret = EMULATE_DONE; + switch (code) { + case KVM_HC_CODE_SERIVCE: + ret = EMULATE_PV_HYPERCALL; + break; + case KVM_HC_CODE_SWDBG: + /* + * Only SWDBG(SoftWare DeBug) could stop vm + * code other than 0 is ignored. + */ + ret = EMULATE_DEBUG; + break; + default: + kvm_info("[%#lx] HYPCALL %#03x unsupported\n", vcpu->arch.pc, code); + break; + } + + if (ret == EMULATE_DEBUG) + vcpu->arch.pc = curr_pc; + + return ret; +} + +/* Execute cpucfg instruction will tirggerGSPR, + * Also the access to unimplemented csrs 0x15 + * 0x16, 0x50~0x53, 0x80, 0x81, 0x90~0x95, 0x98 + * 0xc0~0xff, 0x100~0x109, 0x500~0x502, + * cache_op, idle_op iocsr ops the same */ +static int _kvm_handle_gspr(struct kvm_vcpu *vcpu) +{ + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + vcpu->arch.is_hypcall = 0; + + er = _kvm_trap_handle_gspr(vcpu); + + if (er == EMULATE_DONE) { + ret = RESUME_GUEST; + } else if (er == EMULATE_DO_MMIO) { + vcpu->run->exit_reason = KVM_EXIT_MMIO; + ret = RESUME_HOST; + } else if (er == EMULATE_DO_IOCSR) { + vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR; + ret = RESUME_HOST; + } else { + kvm_err("%s internal error\n", __func__); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + return ret; +} + +static int _kvm_handle_hypcall(struct kvm_vcpu *vcpu) +{ + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + vcpu->arch.is_hypcall = 0; + er = _kvm_check_hypcall(vcpu); + + if (er == EMULATE_PV_HYPERCALL) + ret = _kvm_handle_pv_hcall(vcpu); + else if (er == EMULATE_DEBUG) { + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + ret = RESUME_HOST; + } else + ret = RESUME_GUEST; + + return ret; +} + +static int _kvm_handle_gcm(struct kvm_vcpu *vcpu) +{ + int ret, subcode; + + vcpu->arch.is_hypcall = 0; + ret = RESUME_GUEST; + subcode = (vcpu->arch.host_estat & KVM_ESTAT_ESUBCODE) >> KVM_ESTAT_ESUBCODE_SHIFT; + if ((subcode != EXCSUBCODE_GCSC) && (subcode != EXCSUBCODE_GCHC)) { + kvm_err("%s internal error\n", __func__); + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + + return ret; +} + +/** + * _kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use fpu which hasn't been allowed + * by the root context. + */ +static int _kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + /* + * If guest FPU not present, the FPU operation should have been + * treated as a reserved instruction! + * If FPU already in use, we shouldn't get this at all. + */ + if (WARN_ON(!_kvm_guest_has_fpu(&vcpu->arch) || + vcpu->arch.aux_inuse & KVM_LARCH_FPU)) { + kvm_err("%s internal error\n", __func__); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + + kvm_own_fpu(vcpu); + return RESUME_GUEST; +} + +/** + * _kvm_handle_lsx_disabled() - Guest used LSX while disabled in root. + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use LSX when it is disabled in the root + * context. + */ +static int _kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + /* + * If LSX not present or not exposed to guest, the LSX operation + * should have been treated as a reserved instruction! + * If LSX already in use, we shouldn't get this at all. + */ + if (!_kvm_guest_has_lsx(&vcpu->arch) || + !(kvm_read_gcsr_euen() & KVM_EUEN_LSXEN) || + vcpu->arch.aux_inuse & KVM_LARCH_LSX) { + kvm_err("%s internal error, lsx %d guest euen %llx aux %x", + __func__, _kvm_guest_has_lsx(&vcpu->arch), + kvm_read_gcsr_euen(), vcpu->arch.aux_inuse); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + +#ifdef CONFIG_CPU_HAS_LSX + kvm_own_lsx(vcpu); +#endif + return RESUME_GUEST; +} + +bool _kvm_guest_has_lasx(struct kvm_vcpu *vcpu) +{ + return cpu_has_lasx && vcpu->arch.lsx_enabled && vcpu->kvm->arch.cpucfg_lasx; +} + +/** + * _kvm_handle_lasx_disabled() - Guest used LASX while disabled in root. + * @vcpu: Virtual CPU context. + * + * Handle when the guest attempts to use LASX when it is disabled in the root + * context. + */ +static int _kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + /* + * If LASX not present or not exposed to guest, the LASX operation + * should have been treated as a reserved instruction! + * If LASX already in use, we shouldn't get this at all. + */ + if (!_kvm_guest_has_lasx(vcpu) || + !(kvm_read_gcsr_euen() & KVM_EUEN_LSXEN) || + !(kvm_read_gcsr_euen() & KVM_EUEN_LASXEN) || + vcpu->arch.aux_inuse & KVM_LARCH_LASX) { + kvm_err("%s internal error, lasx %d guest euen %llx aux %x", + __func__, _kvm_guest_has_lasx(vcpu), + kvm_read_gcsr_euen(), vcpu->arch.aux_inuse); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + +#ifdef CONFIG_CPU_HAS_LASX + kvm_own_lasx(vcpu); +#endif + return RESUME_GUEST; +} + + +static int _kvm_handle_read_fault(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + ulong badv = vcpu->arch.badv; + larch_inst inst; + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + if (kvm_handle_mm_fault(vcpu, badv, false)) { + /* A code fetch fault doesn't count as an MMIO */ + if (kvm_is_ifetch_fault(&vcpu->arch)) { + kvm_err("%s ifetch error addr:%lx\n", __func__, badv); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + return RESUME_HOST; + } + + /* Treat as MMIO */ + inst.word = vcpu->arch.badi; + er = _kvm_emu_mmio_read(vcpu, inst); + if (er == EMULATE_FAIL) { + kvm_err("Guest Emulate Load failed: PC: %#lx, BadVaddr: %#lx\n", + vcpu->arch.pc, badv); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + } + } + + if (er == EMULATE_DONE) { + ret = RESUME_GUEST; + } else if (er == EMULATE_DO_MMIO) { + run->exit_reason = KVM_EXIT_MMIO; + ret = RESUME_HOST; + } else { + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + return ret; +} + +static int _kvm_handle_write_fault(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + ulong badv = vcpu->arch.badv; + larch_inst inst; + enum emulation_result er = EMULATE_DONE; + int ret = RESUME_GUEST; + + if (kvm_handle_mm_fault(vcpu, badv, true)) { + + /* Treat as MMIO */ + inst.word = vcpu->arch.badi; + er = _kvm_emu_mmio_write(vcpu, inst); + if (er == EMULATE_FAIL) { + kvm_err("Guest Emulate Store failed: PC: %#lx, BadVaddr: %#lx\n", + vcpu->arch.pc, badv); + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + } + } + + if (er == EMULATE_DONE) { + ret = RESUME_GUEST; + } else if (er == EMULATE_DO_MMIO) { + run->exit_reason = KVM_EXIT_MMIO; + ret = RESUME_HOST; + } else { + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + ret = RESUME_HOST; + } + return ret; +} + +static int _kvm_handle_debug(struct kvm_vcpu *vcpu) +{ + uint32_t fwps, mwps; + + fwps = kvm_csr_readq(KVM_CSR_FWPS); + mwps = kvm_csr_readq(KVM_CSR_MWPS); + if (fwps & 0xff) + kvm_csr_writeq(fwps, KVM_CSR_FWPS); + if (mwps & 0xff) + kvm_csr_writeq(mwps, KVM_CSR_MWPS); + vcpu->run->debug.arch.exception = KVM_EXCCODE_WATCH; + vcpu->run->debug.arch.fwps = fwps; + vcpu->run->debug.arch.mwps = mwps; + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + return RESUME_HOST; +} + +static exit_handle_fn _kvm_fault_tables[KVM_INT_START] = { + [KVM_EXCCODE_TLBL] = _kvm_handle_read_fault, + [KVM_EXCCODE_TLBS] = _kvm_handle_write_fault, + [KVM_EXCCODE_TLBI] = _kvm_handle_read_fault, + [KVM_EXCCODE_TLBM] = _kvm_handle_write_fault, + [KVM_EXCCODE_TLBRI] = _kvm_handle_read_fault, + [KVM_EXCCODE_TLBXI] = _kvm_handle_read_fault, + [KVM_EXCCODE_FPDIS] = _kvm_handle_fpu_disabled, + [KVM_EXCCODE_LSXDIS] = _kvm_handle_lsx_disabled, + [KVM_EXCCODE_LASXDIS] = _kvm_handle_lasx_disabled, + [KVM_EXCCODE_WATCH] = _kvm_handle_debug, + [KVM_EXCCODE_GSPR] = _kvm_handle_gspr, + [KVM_EXCCODE_HYP] = _kvm_handle_hypcall, + [KVM_EXCCODE_GCM] = _kvm_handle_gcm, +}; + +int _kvm_handle_fault(struct kvm_vcpu *vcpu, int fault) +{ + return _kvm_fault_tables[fault](vcpu); +} + +void _kvm_init_fault(void) +{ + int i; + + for (i = 0; i < KVM_INT_START; i++) + if (!_kvm_fault_tables[i]) + _kvm_fault_tables[i] = _kvm_fault_ni; +} diff --git a/arch/loongarch/kvm/fpu.c b/arch/loongarch/kvm/fpu.c new file mode 100644 index 0000000000000000000000000000000000000000..180e874f04fe7cc2b5c246c415eae19e7c373fb4 --- /dev/null +++ b/arch/loongarch/kvm/fpu.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include + +/* FPU/LSX context management */ +void __kvm_save_fpu(struct loongarch_fpu *fpu); +void __kvm_restore_fpu(struct loongarch_fpu *fpu); +void __kvm_restore_fcsr(struct loongarch_fpu *fpu); + +void kvm_save_fpu(struct kvm_vcpu *cpu) +{ + return __kvm_save_fpu(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_save_fpu); + +void kvm_restore_fpu(struct kvm_vcpu *cpu) +{ + return __kvm_restore_fpu(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_fpu); + +void kvm_restore_fcsr(struct kvm_vcpu *cpu) +{ + return __kvm_restore_fcsr(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_fcsr); + +#ifdef CONFIG_CPU_HAS_LSX +void __kvm_save_lsx(struct loongarch_fpu *fpu); +void __kvm_restore_lsx(struct loongarch_fpu *fpu); +void __kvm_restore_lsx_upper(struct loongarch_fpu *fpu); + +void kvm_save_lsx(struct kvm_vcpu *cpu) +{ + return __kvm_save_lsx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_save_lsx); + +void kvm_restore_lsx(struct kvm_vcpu *cpu) +{ + return __kvm_restore_lsx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lsx); + +void kvm_restore_lsx_upper(struct kvm_vcpu *cpu) +{ + return __kvm_restore_lsx_upper(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lsx_upper); + +#endif + +#ifdef CONFIG_CPU_HAS_LSX +void __kvm_save_lasx(struct loongarch_fpu *fpu); +void __kvm_restore_lasx(struct loongarch_fpu *fpu); +void __kvm_restore_lasx_upper(struct loongarch_fpu *fpu); + +void kvm_save_lasx(struct kvm_vcpu *cpu) +{ + return __kvm_save_lasx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_save_lasx); + +void kvm_restore_lasx(struct kvm_vcpu *cpu) +{ + return __kvm_restore_lasx(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lasx); + +void kvm_restore_lasx_upper(struct kvm_vcpu *cpu) +{ + return _restore_lasx_upper(&cpu->arch.fpu); +} +EXPORT_SYMBOL_GPL(kvm_restore_lasx_upper); +#endif + +EXPORT_SYMBOL_GPL(kvm_enter_guest); +EXPORT_SYMBOL_GPL(kvm_exception_entry); + diff --git a/arch/loongarch/kvm/hypcall.c b/arch/loongarch/kvm/hypcall.c new file mode 100644 index 0000000000000000000000000000000000000000..aaf3a07f23f0f0f253919d8039aa7c561702c10b --- /dev/null +++ b/arch/loongarch/kvm/hypcall.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include "intc/ls3a_ipi.h" +#include "kvm_compat.h" + + +int kvm_virt_ipi(struct kvm_vcpu *vcpu) +{ + int ret = 0; + u64 ipi_bitmap; + unsigned int min, action, cpu; + + ipi_bitmap = vcpu->arch.gprs[KVM_REG_A1]; + min = vcpu->arch.gprs[KVM_REG_A2]; + action = vcpu->arch.gprs[KVM_REG_A3]; + + if (ipi_bitmap) { + cpu = find_first_bit((void *)&ipi_bitmap, BITS_PER_LONG); + while (cpu < BITS_PER_LONG) { + kvm_helper_send_ipi(vcpu, cpu + min, action); + cpu = find_next_bit((void *)&ipi_bitmap, BITS_PER_LONG, cpu + 1); + } + } + + return ret; +} + +int kvm_save_notify(struct kvm_vcpu *vcpu) +{ + unsigned long num, id, data; + + int ret = 0; + + num = vcpu->arch.gprs[KVM_REG_A0]; + id = vcpu->arch.gprs[KVM_REG_A1]; + data = vcpu->arch.gprs[KVM_REG_A2]; + + switch (id) { + case KVM_FEATURE_STEAL_TIME: + if (!sched_info_on()) + break; + vcpu->arch.st.guest_addr = data; + kvm_debug("cpu :%d addr:%lx\n", vcpu->vcpu_id, data); + vcpu->arch.st.last_steal = current->sched_info.run_delay; + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); + break; + default: + break; + }; + + return ret; +}; + +static int _kvm_pv_feature(struct kvm_vcpu *vcpu) +{ + int feature = vcpu->arch.gprs[KVM_REG_A1]; + int ret = KVM_RET_NOT_SUPPORTED; + switch (feature) { + case KVM_FEATURE_STEAL_TIME: + if (sched_info_on()) + ret = KVM_RET_SUC; + break; + case KVM_FEATURE_MULTI_IPI: + ret = KVM_RET_SUC; + break; + default: + break; + } + return ret; +} + +/* + * hypcall emulation always return to guest, Caller should check retval. + */ +int _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu) +{ + unsigned long func = vcpu->arch.gprs[KVM_REG_A0]; + int hyp_ret = KVM_RET_NOT_SUPPORTED; + + switch (func) { + case KVM_HC_FUNC_FEATURE: + hyp_ret = _kvm_pv_feature(vcpu); + break; + case KVM_HC_FUNC_NOTIFY: + hyp_ret = kvm_save_notify(vcpu); + break; + case KVM_HC_FUNC_IPI: + hyp_ret = kvm_virt_ipi(vcpu); + break; + default: + kvm_info("[%#lx] hvc func:%#lx unsupported\n", vcpu->arch.pc, func); + break; + }; + + vcpu->arch.gprs[KVM_REG_A0] = hyp_ret; + + return RESUME_GUEST; +} diff --git a/arch/loongarch/kvm/intc/irqchip-debug.c b/arch/loongarch/kvm/intc/irqchip-debug.c new file mode 100644 index 0000000000000000000000000000000000000000..488b00366b47554603f43e1611516955e9a18526 --- /dev/null +++ b/arch/loongarch/kvm/intc/irqchip-debug.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include "kvmcpu.h" +#include "ls3a_ext_irq.h" +#include "ls7a_irq.h" + +#ifdef CONFIG_DEBUG_FS +static int irqchip_state_show(struct seq_file *m, void *v) +{ + struct kvm *kvm = m->private; + + kvm_get_kvm(kvm); + kvm_dump_ls3a_extirq_state(m, kvm->arch.v_extirq); + kvm_dump_ls7a_ioapic_state(m, kvm->arch.v_ioapic); + kvm_put_kvm(kvm); + + return 0; +} + +static int irqchip_debug_open(struct inode *inode, struct file *file) +{ + return single_open(file, irqchip_state_show, inode->i_private); +} + +static const struct file_operations irqchip_debug_fops = { + .open = irqchip_debug_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void irqchip_debug_init(struct kvm *kvm) +{ + debugfs_create_file("irqchip-state", 0444, kvm->debugfs_dentry, kvm, + &irqchip_debug_fops); +} +#else + +void irqchip_debug_init(struct kvm *kvm) {} +#endif +void irqchip_debug_destroy(struct kvm *kvm) +{ +} diff --git a/arch/loongarch/kvm/intc/ls3a_ext_irq.c b/arch/loongarch/kvm/intc/ls3a_ext_irq.c new file mode 100644 index 0000000000000000000000000000000000000000..33e8b45a3532a780e6889ce842890e2b49cd35d0 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ext_irq.c @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include "kvmcpu.h" +#include "kvm_compat.h" +#include "ls3a_ipi.h" +#include "ls7a_irq.h" +#include "ls3a_ext_irq.h" + +#define ls3a_ext_irq_lock(s, flags) spin_lock_irqsave(&s->lock, flags) +#define ls3a_ext_irq_unlock(s, flags) spin_unlock_irqrestore(&s->lock, flags) + +extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_loongarch_interrupt *irq); +void ext_deactive_core_isr(struct kvm *kvm, int irq_num, int vcpu_id) +{ + int ipnum; + unsigned long found1; + struct kvm_loongarch_interrupt irq; + struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq); + + ipnum = state->ext_sw_ipmap[irq_num]; + + bitmap_clear((void *)state->ext_isr.reg_u8, irq_num, 1); + bitmap_clear((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1); + + bitmap_clear((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1); + found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0); + kvm_debug("vcpu_id %d irqnum %d found:0x%lx ipnum %d down\n", vcpu_id, irq_num, found1, ipnum); + if (found1 == EXTIOI_IRQS) { + irq.cpu = vcpu_id; + irq.irq = -(ipnum + 2); /* IP2~IP5 */ + if (likely(kvm->vcpus[vcpu_id])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq); + kvm->stat.trigger_ls3a_ext_irq++; + } +} + +/** + * ext_irq_update_core() + * @kvm: KVM structure pointer + * @irq_num: 0~256 ext irq num + * @level: 0~1 High and low level + * + * Route the status of the extended interrupt to the host CPU core. + * + */ +void ext_irq_update_core(struct kvm *kvm, int irq_num, int level) +{ + int nrcpus, ipnum, vcpu_id; + unsigned long found1; + struct kvm_loongarch_interrupt irq; + struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq); + + nrcpus = atomic_read(&kvm->online_vcpus); + vcpu_id = state->ext_sw_coremap[irq_num]; + ipnum = state->ext_sw_ipmap[irq_num]; + + if (vcpu_id > (nrcpus - 1)) { + vcpu_id = 0; + } + + if (level == 1) { + if (test_bit(irq_num, (void *)state->ext_en.reg_u8) == false) { + return; + } + if (test_bit(irq_num, (void *)state->ext_isr.reg_u8) == false) { + return; + } + bitmap_set((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1); + + found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0); + bitmap_set((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1); + kvm_debug("%s:%d --- vcpu_id %d irqnum %d found1 0x%lx ipnum %d\n", + __FUNCTION__, __LINE__, vcpu_id, irq_num, found1, ipnum); + if (found1 == EXTIOI_IRQS) { + irq.cpu = vcpu_id; + irq.irq = ipnum + 2; /* IP2~IP5 */ + kvm_debug("%s:%d --- vcpu_id %d ipnum %d raise\n", + __FUNCTION__, __LINE__, vcpu_id, ipnum); + if (likely(kvm->vcpus[vcpu_id])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq); + kvm->stat.trigger_ls3a_ext_irq++; + } + } else { + bitmap_clear((void *)state->ext_isr.reg_u8, irq_num, 1); + bitmap_clear((void *)state->ext_core_isr.reg_u8[vcpu_id], irq_num, 1); + + bitmap_clear((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], irq_num, 1); + found1 = find_next_bit((void *)state->ext_sw_ipisr[vcpu_id][ipnum + 2], EXTIOI_IRQS, 0); + if (found1 == EXTIOI_IRQS) { + irq.cpu = vcpu_id; + irq.irq = -(ipnum + 2); /* IP2~IP5 */ + if (likely(kvm->vcpus[vcpu_id])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[vcpu_id], &irq); + kvm->stat.trigger_ls3a_ext_irq++; + } + + } +} + +void msi_irq_handler(struct kvm *kvm, int irq, int level) +{ + unsigned long flags; + struct ls3a_kvm_extirq *s = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *state = &(s->ls3a_ext_irq); + + if (!atomic64_read(&s->enabled)) + return; + + kvm_debug("ext_irq_handler:irq = %d,level = %d\n", irq, level); + + ls3a_ext_irq_lock(s, flags); + if (level == 1) { + if (test_bit(irq, (void *)&state->ext_isr)) + goto out; + __set_bit(irq, (void *)&state->ext_isr); + } else { + if (!test_bit(irq, (void *)&state->ext_isr)) + goto out; + __clear_bit(irq, (void *)&state->ext_isr); + } + + ext_irq_update_core(kvm, irq, level); +out: + ls3a_ext_irq_unlock(s, flags); +} + +static int ls3a_ext_intctl_readb(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, void *val) +{ + uint64_t offset, reg_count; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + int vcpu_id; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + + offset = addr & 0xfffff; + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START); + *(uint8_t *)val = state->ext_en.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START); + *(uint8_t *)val = state->bounce.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + reg_count = (offset - EXTIOI_ISR_START); + *(uint8_t *)val = state->ext_isr.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + /* percpu(32 bytes) coreisr reg_count is 0~31 */ + vcpu_id = (offset >> 8) & 0xff; + reg_count = offset & 0xff; + *(uint8_t *)val = state->ext_core_isr.reg_u8[vcpu_id][reg_count]; + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + reg_count = (offset - EXTIOI_IPMAP_START); + *(uint8_t *)val = state->ip_map.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START); + *(uint8_t *)val = state->core_map.reg_u8[reg_count]; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START); + *(uint8_t *)val = state->node_type.reg_u8[reg_count]; + } + kvm_debug("%s: addr=0x%llx,val=0x%x\n", + __FUNCTION__, addr, *(uint8_t *)val); + return 0; +} + +static int ls3a_ext_intctl_readw(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, void *val) +{ + uint64_t offset, reg_count; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + int vcpu_id; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + + offset = addr & 0xfffff; + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 4; + *(uint32_t *)val = state->ext_en.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 4; + *(uint32_t *)val = state->bounce.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + reg_count = (offset - EXTIOI_ISR_START) / 4; + *(uint32_t *)val = state->ext_isr.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + /* percpu(32 bytes) coreisr reg_count is 0~7*/ + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0xff) / 4; + *(uint32_t *)val = state->ext_core_isr.reg_u32[vcpu_id][reg_count]; + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + reg_count = (offset - EXTIOI_IPMAP_START) / 4; + *(uint32_t *)val = state->ip_map.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START) / 4; + *(uint32_t *)val = state->core_map.reg_u32[reg_count]; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 4; + *(uint32_t *)val = state->node_type.reg_u32[reg_count]; + } + kvm_debug("%s: addr=0x%llx,val=0x%x\n", + __FUNCTION__, addr, *(uint32_t *)val); + + return 0; +} + +static int ls3a_ext_intctl_readl(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, void *val) +{ + uint64_t offset, reg_count; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + int vcpu_id; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + + offset = addr & 0xfffff; + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 8; + *(uint64_t *)val = state->ext_en.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 8; + *(uint64_t *)val = state->bounce.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + reg_count = (offset - EXTIOI_ISR_START) / 8; + *(uint64_t *)val = state->ext_isr.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + /* percpu(32 bytes) coreisr reg_count is 0~3*/ + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0xff) / 8; + + *(uint64_t *)val = state->ext_core_isr.reg_u64[vcpu_id][reg_count]; + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + *(uint64_t *)val = state->ip_map.reg_u64; + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START) / 8; + *(uint64_t *)val = state->core_map.reg_u64[reg_count]; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 8; + *(uint64_t *)val = state->node_type.reg_u64[reg_count]; + } + kvm_debug("%s: addr=0x%llx,val=0x%llx\n", + __FUNCTION__, addr, *(uint64_t *)val); + return 0; +} +/** + * ls3a_ext_intctl_read() + * @kvm: KVM structure pointer + * @addr: Register address + * @size: The width of the register to be read. + * @val: The pointer to the read result. + * + * Analog extended interrupt related register read. + * + */ +static int ls3a_ext_intctl_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int size, void *val) +{ + struct ls3a_kvm_extirq *s = NULL; + unsigned long flags; + uint64_t offset; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + offset = addr & 0xfffff; + if (offset & (size - 1)) { + printk("%s:unaligned address access %llx size %d\n", + __FUNCTION__, addr, size); + return 0; + } + addr = (addr & 0xfffff) - EXTIOI_ADDR_OFF; + ls3a_ext_irq_lock(s, flags); + + switch (size) { + case 1: + ls3a_ext_intctl_readb(vcpu, dev, addr, val); + break; + case 4: + ls3a_ext_intctl_readw(vcpu, dev, addr, val); + break; + case 8: + ls3a_ext_intctl_readl(vcpu, dev, addr, val); + break; + default: + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx, size %d\n", + __FUNCTION__, addr, size); + } + ls3a_ext_irq_unlock(s, flags); + kvm_debug("%s(%d):address access %llx size %d\n", + __FUNCTION__, __LINE__, offset, size); + + return 0; +} + +static int ls3a_ext_intctl_writeb(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, const void *__val) +{ + uint64_t offset, reg_count; + uint8_t val_data_u8, old_data_u8; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + struct kvm *kvm = NULL; + int mask, level, i, irqnum, ipnum; + int vcpu_id; + + unsigned long val = *(unsigned long *)__val; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + kvm = s->kvm; + + offset = addr & 0xfffff; + val_data_u8 = val & 0xffUL; + + kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val); + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START); + old_data_u8 = state->ext_en.reg_u8[reg_count]; + if (old_data_u8 != val_data_u8) { + state->ext_en.reg_u8[reg_count] = val_data_u8; + old_data_u8 = old_data_u8 ^ val_data_u8; + mask = 0x1; + for (i = 0; i < 8; i++) { + if (old_data_u8 & mask) { + level = !!(val_data_u8 & (0x1 << i)); + if (level) + ext_irq_update_core(kvm, i + reg_count * 8, level); + } + mask = mask << 1; + } + } + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START); + state->bounce.reg_u8[reg_count] = val_data_u8; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + /*can not be writen*/ + reg_count = (offset - EXTIOI_ISR_START) & 0x1f; + old_data_u8 = state->ext_isr.reg_u8[reg_count]; + state->ext_isr.reg_u8[reg_count] = old_data_u8 & (~val_data_u8); + + mask = 0x1; + for (i = 0; i < 8; i++) { + if ((old_data_u8 & mask) && (val_data_u8 & mask)) { + ext_irq_update_core(kvm, i + reg_count * 8, 0); + } + mask = mask << 1; + } + + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + int bits; + /* percpu(32 bytes) coreisr reg_count is 0~31 */ + vcpu_id = (offset >> 8) & 0xff; + reg_count = offset & 0xff; + + state->ext_core_isr.reg_u8[vcpu_id][reg_count] &= ~val_data_u8; + + bits = sizeof(val_data_u8) * 8; + i = find_first_bit((void *)&val_data_u8, bits); + while (i < bits) { + ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id); + bitmap_clear((void *)&val_data_u8, i, 1); + i = find_first_bit((void *)&val_data_u8, bits); + } + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + /*drop arch.core_ip_mask use state->ip_map*/ + reg_count = (offset - EXTIOI_IPMAP_START); + state->ip_map.reg_u8[reg_count] = val_data_u8; + + ipnum = 0; + + for (i = 0; i < 4; i++) { + if (val_data_u8 & (0x1 << i)) { + ipnum = i; + break; + } + } + + if (val_data_u8) { + for (i = 0; i < 32; i++) { + irqnum = reg_count * 32 + i; + state->ext_sw_ipmap[irqnum] = ipnum; + } + } else { + for (i = 0; i < 32; i++) { + irqnum = reg_count * 32 + i; + state->ext_sw_ipmap[irqnum] = 0; + } + } + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + reg_count = (offset - EXTIOI_COREMAP_START); + state->core_map.reg_u8[reg_count] = val_data_u8; + state->ext_sw_coremap[reg_count] = val_data_u8; + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START); + state->node_type.reg_u8[reg_count] = val_data_u8; + } else { + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx\n", + __FUNCTION__, addr); + } + + return 0; +} + +static int ls3a_ext_intctl_writew(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, const void *__val) +{ + uint64_t offset, reg_count; + uint32_t val_data_u32, old_data_u32, mask; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + struct kvm *kvm = NULL; + uint8_t tmp_data_u8; + int i, level, vcpu_id; + unsigned long val; + + val = *(unsigned long *)__val; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + kvm = s->kvm; + + offset = addr & 0xfffff; + val_data_u32 = val & 0xffffffffUL; + + kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val); + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 4; + old_data_u32 = state->ext_en.reg_u32[reg_count]; + if (old_data_u32 != val_data_u32) { + state->ext_en.reg_u32[reg_count] = val_data_u32; + old_data_u32 = old_data_u32 ^ val_data_u32; + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u32); i++) { + if (old_data_u32 & mask) { + level = !!(val_data_u32 & (0x1 << i)); + if (level) + ext_irq_update_core(kvm, i + reg_count * 32, level); + } + mask = mask << 1; + } + } + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 4; + state->bounce.reg_u32[reg_count] = val_data_u32; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + /*can not be writen*/ + reg_count = (offset - EXTIOI_ISR_START) / 4; + old_data_u32 = state->ext_isr.reg_u32[reg_count]; + state->ext_isr.reg_u32[reg_count] = old_data_u32 & (~val_data_u32); + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u32); i++) { + if ((old_data_u32 & mask) && (val_data_u32 & mask)) { + ext_irq_update_core(kvm, i + reg_count * 32, 0); + } + mask = mask << 1; + } + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + int bits; + /* percpu(32 bytes) coreisr reg_count is 0~7*/ + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0xff) / 4; + + /*ext_core_ioisr*/ + state->ext_core_isr.reg_u32[vcpu_id][reg_count] &= ~val_data_u32; + + bits = sizeof(val_data_u32) * 8; + i = find_first_bit((void *)&val_data_u32, bits); + while (i < bits) { + ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id); + bitmap_clear((void *)&val_data_u32, i, 1); + i = find_first_bit((void *)&val_data_u32, bits); + } + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + tmp_data_u8 = val_data_u32 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + tmp_data_u8 = val_data_u32 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u32 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + kvm_debug("%s:id:%d addr=0x%llx, offset 0x%llx val 0x%x\n", + __FUNCTION__, vcpu->vcpu_id, addr, offset, val_data_u32); + + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 4; + state->node_type.reg_u32[reg_count] = val_data_u32; + } else { + WARN_ONCE(1, "%s:%d Abnormal address access:addr 0x%llx\n", + __FUNCTION__, __LINE__, addr); + } + + return 0; +} + +static int ls3a_ext_intctl_writel(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, const void *__val) +{ + uint64_t offset, val_data_u64, old_data_u64, reg_count, mask, i; + struct ls3a_kvm_extirq *s = NULL; + struct kvm_ls3a_extirq_state *state = NULL; + struct kvm *kvm = NULL; + uint8_t tmp_data_u8; + int level, vcpu_id; + + unsigned long val = *(unsigned long *)__val; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + state = &(s->ls3a_ext_irq); + kvm = s->kvm; + + offset = addr & 0xfffff; + val_data_u64 = val; + + kvm_debug("%s: addr=0x%llx,val=0x%lx\n", __FUNCTION__, addr, val); + + if ((offset >= EXTIOI_ENABLE_START) && (offset < EXTIOI_ENABLE_END)) { + reg_count = (offset - EXTIOI_ENABLE_START) / 8; + old_data_u64 = state->ext_en.reg_u64[reg_count]; + if (old_data_u64 != val_data_u64) { + state->ext_en.reg_u64[reg_count] = val_data_u64; + old_data_u64 = old_data_u64 ^ val_data_u64; + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u64); i++) { + if (old_data_u64 & mask) { + level = !!(val_data_u64 & (0x1 << i)); + if (level) + ext_irq_update_core(kvm, i + reg_count * 64, level); + } + mask = mask << 1; + } + } + } else if ((offset >= EXTIOI_BOUNCE_START) && (offset < EXTIOI_BOUNCE_END)) { + reg_count = (offset - EXTIOI_BOUNCE_START) / 8; + state->bounce.reg_u64[reg_count] = val_data_u64; + } else if ((offset >= EXTIOI_ISR_START) && (offset < EXTIOI_ISR_END)) { + /*can not be writen*/ + reg_count = (offset - EXTIOI_ISR_START) / 8; + old_data_u64 = state->ext_isr.reg_u64[reg_count]; + state->ext_isr.reg_u64[reg_count] = old_data_u64 & (~val_data_u64); + + mask = 0x1; + for (i = 0; i < 8 * sizeof(old_data_u64); i++) { + if ((old_data_u64 & mask) && (val_data_u64 & mask)) { + ext_irq_update_core(kvm, i + reg_count * 64, 0); + } + mask = mask << 1; + } + } else if ((offset >= EXTIOI_COREISR_START) && (offset < EXTIOI_COREISR_END)) { + int bits; + vcpu_id = (offset >> 8) & 0xff; + reg_count = (offset & 0x1f) / 8; + + /*core_ext_ioisr*/ + state->ext_core_isr.reg_u64[vcpu_id][reg_count] &= ~val_data_u64; + + bits = sizeof(val_data_u64) * 8; + i = find_first_bit((void *)&val_data_u64, bits); + while (i < bits) { + ext_deactive_core_isr(kvm, i + reg_count * bits, vcpu_id); + bitmap_clear((void *)&val_data_u64, i, 1); + i = find_first_bit((void *)&val_data_u64, bits); + } + } else if ((offset >= EXTIOI_IPMAP_START) && (offset < EXTIOI_IPMAP_END)) { + tmp_data_u8 = val_data_u64 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + + tmp_data_u8 = (val_data_u64 >> 32) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 4, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 40) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 5, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 48) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 6, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 56) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 7, &tmp_data_u8); + } else if ((offset >= EXTIOI_COREMAP_START) && (offset < EXTIOI_COREMAP_END)) { + tmp_data_u8 = val_data_u64 & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 8) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 1, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 16) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 2, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 24) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 3, &tmp_data_u8); + + tmp_data_u8 = (val_data_u64 >> 32) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 4, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 40) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 5, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 48) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 6, &tmp_data_u8); + tmp_data_u8 = (val_data_u64 >> 56) & 0xff; + ls3a_ext_intctl_writeb(vcpu, dev, addr + 7, &tmp_data_u8); + } else if ((offset >= EXTIOI_NODETYPE_START) && (offset < EXTIOI_NODETYPE_END)) { + reg_count = (offset - EXTIOI_NODETYPE_START) / 8; + state->node_type.reg_u64[reg_count] = val_data_u64; + } else { + WARN_ONCE(1, "%s:%d Abnormal address access:addr 0x%llx\n", + __FUNCTION__, __LINE__, addr); + } + return 0; +} +/** + * ls3a_ext_intctl_write() + * @kvm: KVM structure pointer + * @addr: Register address + * @size: The width of the register to be writen. + * @val: Value to be written. + * + * Analog extended interrupt related register write. + * + */ +static int ls3a_ext_intctl_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int size, const void *__val) +{ + struct ls3a_kvm_extirq *s = NULL; + unsigned long flags; + uint64_t offset; + + s = container_of(dev, struct ls3a_kvm_extirq, device); + + offset = addr & 0xfffff; + if (offset & (size - 1)) { + printk("%s(%d):unaligned address access %llx size %d\n", + __FUNCTION__, __LINE__, addr, size); + return 0; + } + + addr = (addr & 0xfffff) - EXTIOI_ADDR_OFF; + ls3a_ext_irq_lock(s, flags); + + switch (size) { + case 1: + ls3a_ext_intctl_writeb(vcpu, dev, addr, __val); + break; + case 4: + ls3a_ext_intctl_writew(vcpu, dev, addr, __val); + break; + case 8: + ls3a_ext_intctl_writel(vcpu, dev, addr, __val); + break; + default: + WARN_ONCE(1, "%s: Abnormal address access:addr 0x%llx,size %d\n", + __FUNCTION__, addr, size); + } + + ls3a_ext_irq_unlock(s, flags); + + kvm_debug("%s(%d):address access %llx size %d\n", + __FUNCTION__, __LINE__, offset, size); + return 0; +} + +static const struct kvm_io_device_ops kvm_ls3a_ext_irq_ops = { + .read = ls3a_ext_intctl_read, + .write = ls3a_ext_intctl_write, +}; + +void kvm_destroy_ls3a_ext_irq(struct kvm *kvm) +{ + struct ls3a_kvm_extirq *s = kvm->arch.v_extirq; + + if (!s) + return; + kvm_io_bus_unregister_dev(s->kvm, KVM_MMIO_BUS, &s->device); + kfree(s); +} +/* + * kvm_create_ls3a_ext_irq() + * @kvm KVM structure pointer + * Create an extended interrupt resource instance for a virtual machine + * Returns: Extended interrupt structure pointer + */ +int kvm_create_ls3a_ext_irq(struct kvm *kvm) +{ + struct ls3a_kvm_extirq *s; + int ret; + + s = kzalloc(sizeof(struct ls3a_kvm_extirq), GFP_KERNEL); + if (!s) + return -ENOMEM; + + memset((void *)&s->ls3a_ext_irq, 0x0, sizeof(struct kvm_ls3a_extirq_state)); + + spin_lock_init(&s->lock); + atomic64_set(&s->enabled, 0); + s->kvm = kvm; + + /* + * Initialize MMIO device + */ + kvm_iodevice_init(&s->device, &kvm_ls3a_ext_irq_ops); + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, + EXTIOI_REG_BASE, EXTIOI_ADDR_SIZE, &s->device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) { + printk("%s dev_ls3a_ext_irq register error ret %d\n", __FUNCTION__, ret); + goto err_register; + } + + kvm->arch.v_extirq = s; + + return 0; + +err_register: + kfree(s); + return -EFAULT; +} + +static int kvm_set_ext_sw_ipmap(struct kvm_ls3a_extirq_state *state) +{ + uint8_t val_data_u8; + int i, j, base_irq, irqnum, ipnum; + + ipnum = 0; + for (i = 0; i < EXTIOI_IRQS_IPMAP_SIZE; i++) { + val_data_u8 = state->ip_map.reg_u8[i]; + for (j = 0; j < 4; j++) { + if (val_data_u8 & (0x1 << j)) { + ipnum = j; + break; + } + } + kvm_debug("%s:%d ipnum:%d i:%d val_data_u8:0x%x\n", __FUNCTION__, __LINE__, + ipnum, i, val_data_u8); + + if (val_data_u8) { + for (base_irq = 0; base_irq < EXTIOI_IRQS_PER_GROUP; base_irq++) { + irqnum = i * EXTIOI_IRQS_PER_GROUP + base_irq; + state->ext_sw_ipmap[irqnum] = ipnum; + } + } else { + for (base_irq = 0; base_irq < EXTIOI_IRQS_PER_GROUP; base_irq++) { + irqnum = i * EXTIOI_IRQS_PER_GROUP + base_irq; + state->ext_sw_ipmap[irqnum] = 0; + } + } + } + + return 0; +} + +static int kvm_set_ext_sw_coremap(struct kvm *kvm, struct kvm_ls3a_extirq_state *state) +{ + int reg_count; + + for (reg_count = 0; reg_count < EXTIOI_IRQS; reg_count++) { + state->ext_sw_coremap[reg_count] = state->core_map.reg_u8[reg_count]; + + kvm_debug("%s:%d -- reg_count:%d vcpu %d\n", + __FUNCTION__, __LINE__, reg_count, state->core_map.reg_u8[reg_count]); + } + + return 0; +} + +static int kvm_set_ext_sw_ipisr(struct kvm *kvm, struct kvm_ls3a_extirq_state *state) +{ + int ipnum, core, irq_num; + + for (irq_num = 0; irq_num < EXTIOI_IRQS; irq_num++) { + core = state->ext_sw_coremap[irq_num]; + ipnum = state->ext_sw_ipmap[irq_num]; + + if (test_bit(irq_num, (void *)state->ext_core_isr.reg_u8[core]) == false) { + bitmap_clear((void *)state->ext_sw_ipisr[core][ipnum + 2], irq_num, 1); + } else { + bitmap_set((void *)state->ext_sw_ipisr[core][ipnum + 2], irq_num, 1); + } + + } + return 0; +} + +int kvm_get_ls3a_extirq(struct kvm *kvm, struct kvm_loongarch_ls3a_extirq_state *state) +{ + struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq); + unsigned long flags; + if (!v_extirq) + return -EINVAL; + + ls3a_ext_irq_lock(v_extirq, flags); + memcpy(state, extirq_state, + sizeof(struct kvm_loongarch_ls3a_extirq_state)); + ls3a_ext_irq_unlock(v_extirq, flags); + kvm->stat.get_ls3a_ext_irq++; + + return 0; +} + +int kvm_set_ls3a_extirq(struct kvm *kvm, struct kvm_loongarch_ls3a_extirq_state *state) +{ + struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq); + unsigned long flags; + if (!v_extirq) + return -EINVAL; + + ls3a_ext_irq_lock(v_extirq, flags); + memcpy(extirq_state, state, + sizeof(struct kvm_loongarch_ls3a_extirq_state)); + kvm_set_ext_sw_ipmap(extirq_state); + kvm_set_ext_sw_coremap(kvm, extirq_state); + kvm_set_ext_sw_ipisr(kvm, extirq_state); + + ls3a_ext_irq_unlock(v_extirq, flags); + kvm->stat.set_ls3a_ext_irq++; + + return 0; +} + +int kvm_setup_ls3a_extirq(struct kvm *kvm) +{ + struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm); + struct kvm_ls3a_extirq_state *extirq_state = &(v_extirq->ls3a_ext_irq); + unsigned long flags; + + if (!v_extirq) + return -EINVAL; + + ls3a_ext_irq_lock(v_extirq, flags); + memset(extirq_state, 0, sizeof(struct kvm_ls3a_extirq_state)); + ls3a_ext_irq_unlock(v_extirq, flags); + + atomic64_set(&v_extirq->enabled, 1); + + return 0; +} + +int kvm_enable_ls3a_extirq(struct kvm *kvm, bool enable) +{ + struct ls3a_kvm_extirq *v_extirq = ls3a_ext_irqchip(kvm); + + if (v_extirq) + atomic64_set(&v_extirq->enabled, enable); + + return 0; +} + +void kvm_dump_ls3a_extirq_state(struct seq_file *s, + struct ls3a_kvm_extirq *irqchip) +{ + struct kvm_ls3a_extirq_state *extirq; + int i = 0, j = 0; + unsigned long flags; + + seq_puts(s, "LS3A ext irqchip state:\n"); + + if (!irqchip) + return; + + extirq = &(irqchip->ls3a_ext_irq); + ls3a_ext_irq_lock(irqchip, flags); + i = (int)atomic64_read(&irqchip->enabled); + seq_printf(s, "ext irq enabled:%d", i); + seq_puts(s, "\nenabled:(Not Enabled)"); + for (i = 0; i < EXTIOI_IRQS; i++) { + if (!test_bit(i, (void *)&extirq->ext_en)) + seq_printf(s, "%d ", i); + } + seq_puts(s, "\nbounce:(Not bounce)"); + for (i = 0; i < EXTIOI_IRQS; i++) { + if (!test_bit(i, (void *)&extirq->bounce)) + seq_printf(s, "%d ", i); + } + seq_puts(s, "\next_isr:"); + for (i = 0; i < EXTIOI_IRQS; i++) { + if (test_bit(i, (void *)&extirq->ext_isr)) + seq_printf(s, "%d ", i); + } + + seq_puts(s, "\ncore_isr:"); + for (i = 0; i < KVM_MAX_VCPUS && kvm_get_vcpu_by_id(irqchip->kvm, i); i++) { + seq_printf(s, "\n\t CPU%d:", i); + for (j = 0; j < EXTIOI_IRQS; j++) { + if (test_bit(j, (void *)&extirq->ext_core_isr.reg_u8[i])) + seq_printf(s, "%d ", j); + } + } + seq_printf(s, "\nip_map:%llx", extirq->ip_map.reg_u64); + seq_puts(s, "\ncore_map: (only display router to slave cpu)\n"); + for (i = 0; i < EXTIOI_IRQS_COREMAP_SIZE; i++) + if (extirq->core_map.reg_u8[i]) + seq_printf(s, "\tirq:%d -> cpu:%d\n", i, + extirq->core_map.reg_u8[i]); + ls3a_ext_irq_unlock(irqchip, flags); +} diff --git a/arch/loongarch/kvm/intc/ls3a_ext_irq.h b/arch/loongarch/kvm/intc/ls3a_ext_irq.h new file mode 100644 index 0000000000000000000000000000000000000000..62fa06239b2811e44a02e8f549fa9ef535e7acaa --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ext_irq.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LS3A_KVM_EXT_IRQ_H +#define __LS3A_KVM_EXT_IRQ_H + +#include +#include +#include +#include +#include + +#include + +#define IOCSR_EXTIOI_ADDR KVM_IOCSR_EXTIOI_NODEMAP_BASE + +#define EXTIOI_ADDR_OFF 0x10000 +#define EXTIOI_REG_BASE (LOONGSON_VIRT_REG_BASE + EXTIOI_ADDR_OFF) +#define EXTIOI_REG_END (EXTIOI_REG_BASE + 0x20000) +#define EXTIOI_ADDR_SIZE (EXTIOI_REG_END - EXTIOI_REG_BASE) +#define EXTIOI_PERCORE_REG_OFF 0x10000 +#define EXTIOI_PERCORE_REG_END (EXTIOI_PERCORE_REG_OFF + 0x10000) + +#define EXTIOI_ADDR(off) (EXTIOI_REG_BASE + (off) - IOCSR_EXTIOI_ADDR) +#define EXTIOI_PERCORE_ADDR(id, off) \ + (EXTIOI_REG_BASE + EXTIOI_PERCORE_REG_OFF + ((id) << 8) + (off)) + +#define EXTIOI_NODETYPE_START (KVM_IOCSR_EXTIOI_NODEMAP_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_NODETYPE_END (EXTIOI_NODETYPE_START + 0x20) +#define EXTIOI_IPMAP_START (KVM_IOCSR_EXTIOI_IPMAP_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_IPMAP_END (EXTIOI_IPMAP_START + 0x8) +#define EXTIOI_ENABLE_START (KVM_IOCSR_EXTIOI_EN_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_ENABLE_END (EXTIOI_ENABLE_START + 0x20) +#define EXTIOI_BOUNCE_START (KVM_IOCSR_EXTIOI_BOUNCE_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_BOUNCE_END (EXTIOI_BOUNCE_START + 0x20) +#define EXTIOI_ISR_START (0x1700 - IOCSR_EXTIOI_ADDR) +#define EXTIOI_ISR_END (EXTIOI_ISR_START + 0x20) +#define EXTIOI_COREMAP_START (KVM_IOCSR_EXTIOI_ROUTE_BASE - IOCSR_EXTIOI_ADDR) +#define EXTIOI_COREMAP_END (EXTIOI_COREMAP_START + 0x100) +#define EXTIOI_COREISR_START (EXTIOI_PERCORE_REG_OFF) +#define EXTIOI_COREISR_END (EXTIOI_PERCORE_REG_END) + +#define LS3A_INTC_IP 8 +#define EXTIOI_IRQS KVM_EXTIOI_IRQS +#define EXTIOI_IRQS_BITMAP_SIZE (EXTIOI_IRQS / 8) +/* map to ipnum per 32 irqs */ +#define EXTIOI_IRQS_IPMAP_SIZE (EXTIOI_IRQS / 32) +#define EXTIOI_IRQS_PER_GROUP KVM_EXTIOI_IRQS_PER_GROUP +#define EXTIOI_IRQS_COREMAP_SIZE (EXTIOI_IRQS) +#define EXTIOI_IRQS_NODETYPE_SIZE KVM_EXTIOI_IRQS_NODETYPE_SIZE + +typedef struct kvm_ls3a_extirq_state { + union ext_en { + uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE]; + } ext_en; + union bounce { + uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE]; + } bounce; + union ext_isr { + uint64_t reg_u64[EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_BITMAP_SIZE]; + } ext_isr; + union ext_core_isr { + uint64_t reg_u64[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE / 8]; + uint32_t reg_u32[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE / 4]; + uint8_t reg_u8[KVM_MAX_VCPUS][EXTIOI_IRQS_BITMAP_SIZE]; + } ext_core_isr; + union ip_map { + uint64_t reg_u64; + uint32_t reg_u32[EXTIOI_IRQS_IPMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_IPMAP_SIZE]; + } ip_map; + union core_map { + uint64_t reg_u64[EXTIOI_IRQS_COREMAP_SIZE / 8]; + uint32_t reg_u32[EXTIOI_IRQS_COREMAP_SIZE / 4]; + uint8_t reg_u8[EXTIOI_IRQS_COREMAP_SIZE]; + } core_map; + union { + uint64_t reg_u64[EXTIOI_IRQS_NODETYPE_SIZE / 4]; + uint32_t reg_u32[EXTIOI_IRQS_NODETYPE_SIZE / 2]; + uint16_t reg_u16[EXTIOI_IRQS_NODETYPE_SIZE]; + uint8_t reg_u8[EXTIOI_IRQS_NODETYPE_SIZE * 2]; + } node_type; + + /*software state */ + uint8_t ext_sw_ipmap[EXTIOI_IRQS]; + uint8_t ext_sw_coremap[EXTIOI_IRQS]; + uint8_t ext_sw_ipisr[KVM_MAX_VCPUS][LS3A_INTC_IP][EXTIOI_IRQS_BITMAP_SIZE]; +} LS3AExtirqState; + +struct ls3a_kvm_extirq { + spinlock_t lock; + struct kvm *kvm; + atomic64_t enabled; + struct kvm_io_device device; + struct kvm_ls3a_extirq_state ls3a_ext_irq; +}; + +static inline struct ls3a_kvm_extirq *ls3a_ext_irqchip(struct kvm *kvm) +{ + return kvm->arch.v_extirq; +} + +static inline int ls3a_extirq_in_kernel(struct kvm *kvm) +{ + int ret; + + ret = (ls3a_ext_irqchip(kvm) != NULL); + return ret; +} + + +void ext_irq_handler(struct kvm *kvm, int irq, int level); +int kvm_create_ls3a_ext_irq(struct kvm *kvm); +int kvm_get_ls3a_extirq(struct kvm *kvm, + struct kvm_loongarch_ls3a_extirq_state *state); +int kvm_set_ls3a_extirq(struct kvm *kvm, + struct kvm_loongarch_ls3a_extirq_state *state); +void kvm_destroy_ls3a_ext_irq(struct kvm *kvm); +void msi_irq_handler(struct kvm *kvm, int irq, int level); +int kvm_setup_ls3a_extirq(struct kvm *kvm); +int kvm_enable_ls3a_extirq(struct kvm *kvm, bool enable); +void kvm_dump_ls3a_extirq_state(struct seq_file *m, struct ls3a_kvm_extirq *irqchip); +#endif diff --git a/arch/loongarch/kvm/intc/ls3a_ipi.c b/arch/loongarch/kvm/intc/ls3a_ipi.c new file mode 100644 index 0000000000000000000000000000000000000000..6c05eaad9f8050823024b185835e5e2be8f1ecd3 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ipi.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include "kvmcpu.h" +#include "ls3a_ipi.h" +#include "ls7a_irq.h" +#include "ls3a_ext_irq.h" + +#define ls3a_gipi_lock(s, flags) spin_lock_irqsave(&s->lock, flags) +#define ls3a_gipi_unlock(s, flags) spin_unlock_irqrestore(&s->lock, flags) + +extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_loongarch_interrupt *irq); +int kvm_helper_send_ipi(struct kvm_vcpu *vcpu, unsigned int cpu, unsigned int action) +{ + struct kvm *kvm = vcpu->kvm; + struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm); + gipiState *s = &(ipi->ls3a_gipistate); + unsigned long flags; + struct kvm_loongarch_interrupt irq; + + kvm->stat.pip_write_exits++; + + ls3a_gipi_lock(ipi, flags); + if (s->core[cpu].status == 0) { + irq.cpu = cpu; + irq.irq = LARCH_INT_IPI; + kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq); + } + + s->core[cpu].status |= action; + ls3a_gipi_unlock(ipi, flags); + + return 0; +} + +static int ls3a_gipi_writel(struct ls3a_kvm_ipi *ipi, gpa_t addr, + int len, const void *val) +{ + uint64_t data, offset; + struct kvm_loongarch_interrupt irq; + gipiState *s = &(ipi->ls3a_gipistate); + uint32_t cpu, action_data; + struct kvm *kvm; + void *pbuf; + int mailbox, action; + + kvm = ipi->kvm; + cpu = (addr >> 8) & 0xff; + + data = *(uint64_t *)val; + offset = addr & 0xFF; + + BUG_ON(offset & (len - 1)); + + switch (offset) { + case CORE0_STATUS_OFF: + printk("CORE0_SET_OFF Can't be write\n"); + + break; + case CORE0_EN_OFF: + s->core[cpu].en = data; + + break; + case CORE0_IPI_SEND: + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; + action = (data & 0x1f); + action_data = (1 << action); + + if (s->core[cpu].status == 0) { + irq.cpu = cpu; + irq.irq = LARCH_INT_IPI; + + if (likely(kvm->vcpus[cpu])) { + kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq); + } + } + s->core[cpu].status |= action_data; + break; + case CORE0_SET_OFF: + pr_info("CORE0_SET_OFF simulation is required\n"); + break; + case CORE0_CLEAR_OFF: + s->core[cpu].status &= ~data; + if (!s->core[cpu].status) { + irq.cpu = cpu; + irq.irq = -LARCH_INT_IPI; + if (likely(kvm->vcpus[cpu])) + kvm_vcpu_ioctl_interrupt(kvm->vcpus[cpu], &irq); + else + kvm_err("Failed lower ipi irq target cpu:%d\n", cpu); + } + + break; + case CORE0_MAIL_SEND: + cpu = ((data & 0xffffffff) >> 16) & 0x3ff; + mailbox = ((data & 0xffffffff) >> 2) & 0x7; + pbuf = (void *)s->core[cpu].buf + mailbox * 4; + + *(unsigned int *)pbuf = (unsigned int)(data >> 32); + break; + case 0x20 ... 0x3c: + pbuf = (void *)s->core[cpu].buf + (offset - 0x20); + if (len == 1) + *(unsigned char *)pbuf = (unsigned char)data; + else if (len == 2) + *(unsigned short *)pbuf = (unsigned short)data; + else if (len == 4) + *(unsigned int *)pbuf = (unsigned int)data; + else if (len == 8) + *(unsigned long *)pbuf = (unsigned long)data; + + break; + default: + printk("ls3a_gipi_writel with unknown addr %llx \n", addr); + break; + } + return 0; +} + +static uint64_t ls3a_gipi_readl(struct ls3a_kvm_ipi *ipi, + gpa_t addr, int len, void *val) +{ + uint64_t offset; + uint64_t ret = 0; + + gipiState *s = &(ipi->ls3a_gipistate); + uint32_t cpu; + void *pbuf; + + cpu = (addr >> 8) & 0xff; + + offset = addr & 0xFF; + + BUG_ON(offset & (len - 1)); + switch (offset) { + case CORE0_STATUS_OFF: + ret = s->core[cpu].status; + break; + case CORE0_EN_OFF: + ret = s->core[cpu].en; + break; + case CORE0_SET_OFF: + ret = 0; + break; + case CORE0_CLEAR_OFF: + ret = 0; + break; + case 0x20 ... 0x3c: + pbuf = (void *)s->core[cpu].buf + (offset - 0x20); + if (len == 1) + ret = *(unsigned char *)pbuf; + else if (len == 2) + ret = *(unsigned short *)pbuf; + else if (len == 4) + ret = *(unsigned int *)pbuf; + else if (len == 8) + ret = *(unsigned long *)pbuf; + break; + default: + printk("ls3a_gipi_readl with unknown addr %llx \n", addr); + break; + } + + *(uint64_t *)val = ret; + + return ret; +} + +static int kvm_ls3a_ipi_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + struct ls3a_kvm_ipi *ipi; + ipi_io_device *ipi_device; + unsigned long flags; + + ipi_device = container_of(dev, ipi_io_device, device); + ipi = ipi_device->ipi; + ipi->kvm->stat.pip_write_exits++; + + ls3a_gipi_lock(ipi, flags); + ls3a_gipi_writel(ipi, addr, len, val); + ls3a_gipi_unlock(ipi, flags); + return 0; +} + + +static int kvm_ls3a_ipi_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + struct ls3a_kvm_ipi *ipi; + ipi_io_device *ipi_device; + unsigned long flags; + + ipi_device = container_of(dev, ipi_io_device, device); + ipi = ipi_device->ipi; + ipi->kvm->stat.pip_read_exits++; + + ls3a_gipi_lock(ipi, flags); + ls3a_gipi_readl(ipi, addr, len, val); + ls3a_gipi_unlock(ipi, flags); + return 0; +} + + +static const struct kvm_io_device_ops kvm_ls3a_ipi_ops = { + .read = kvm_ls3a_ipi_read, + .write = kvm_ls3a_ipi_write, +}; + +void kvm_destroy_ls3a_ipi(struct kvm *kvm) +{ + struct kvm_io_device *device; + struct ls3a_kvm_ipi *vipi = kvm->arch.v_gipi; + + if (!vipi) + return; + device = &vipi->dev_ls3a_ipi.device; + kvm_io_bus_unregister_dev(vipi->kvm, KVM_MMIO_BUS, device); + kfree(vipi); +} + +int kvm_create_ls3a_ipi(struct kvm *kvm) +{ + struct ls3a_kvm_ipi *s; + unsigned long addr; + struct kvm_io_device *device; + int ret; + + s = kzalloc(sizeof(struct ls3a_kvm_ipi), GFP_KERNEL); + if (!s) + return -ENOMEM; + spin_lock_init(&s->lock); + s->kvm = kvm; + + /* + * Initialize MMIO device + */ + device = &s->dev_ls3a_ipi.device; + kvm_iodevice_init(device, &kvm_ls3a_ipi_ops); + addr = SMP_MAILBOX; + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, + addr, KVM_IOCSR_IPI_ADDR_SIZE, device); + mutex_unlock(&kvm->slots_lock); + if (ret < 0) { + kvm_err("%s Initialize MMIO dev err ret:%d\n", __func__, ret); + goto err; + } else { + s->dev_ls3a_ipi.ipi = s; + } + + kvm->arch.v_gipi = s; + return 0; + +err: + kfree(s); + return -EFAULT; +} + +int kvm_get_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state) +{ + struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm); + gipiState *ipi_state = &(ipi->ls3a_gipistate); + unsigned long flags; + + ls3a_gipi_lock(ipi, flags); + memcpy(state, ipi_state, sizeof(gipiState)); + ls3a_gipi_unlock(ipi, flags); + return 0; +} + +int kvm_set_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state) +{ + struct ls3a_kvm_ipi *ipi = ls3a_ipi_irqchip(kvm); + gipiState *ipi_state = &(ipi->ls3a_gipistate); + unsigned long flags; + + if (!ipi) + return -EINVAL; + + ls3a_gipi_lock(ipi, flags); + memcpy(ipi_state, state, sizeof(gipiState)); + ls3a_gipi_unlock(ipi, flags); + return 0; +} diff --git a/arch/loongarch/kvm/intc/ls3a_ipi.h b/arch/loongarch/kvm/intc/ls3a_ipi.h new file mode 100644 index 0000000000000000000000000000000000000000..3d40487177a7b93dea341faa23d628cea8ff323d --- /dev/null +++ b/arch/loongarch/kvm/intc/ls3a_ipi.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LS3A_KVM_IPI_H +#define __LS3A_KVM_IPI_H + +#include +#include +#include +#include +#include + +typedef struct gipi_single { + uint32_t status; + uint32_t en; + uint32_t set; + uint32_t clear; + uint64_t buf[4]; +} gipi_single; + +typedef struct gipiState { + gipi_single core[KVM_MAX_VCPUS]; +} gipiState; + +struct ls3a_kvm_ipi; + +typedef struct ipi_io_device { + struct ls3a_kvm_ipi *ipi; + struct kvm_io_device device; + int nodeNum; +} ipi_io_device; + +struct ls3a_kvm_ipi { + spinlock_t lock; + struct kvm *kvm; + gipiState ls3a_gipistate; + int nodeNum; + ipi_io_device dev_ls3a_ipi; +}; + +#define SMP_MAILBOX (LOONGSON_VIRT_REG_BASE + 0x0000) +#define KVM_IPI_REG_ADDRESS(id, off) (SMP_MAILBOX | (id << 8) | off) +#define KVM_IOCSR_IPI_ADDR_SIZE 0x10000 + +#define CORE0_STATUS_OFF 0x000 +#define CORE0_EN_OFF 0x004 +#define CORE0_SET_OFF 0x008 +#define CORE0_CLEAR_OFF 0x00c +#define CORE0_BUF_20 0x020 +#define CORE0_BUF_28 0x028 +#define CORE0_BUF_30 0x030 +#define CORE0_BUF_38 0x038 +#define CORE0_IPI_SEND 0x040 +#define CORE0_MAIL_SEND 0x048 + +static inline struct ls3a_kvm_ipi *ls3a_ipi_irqchip(struct kvm *kvm) +{ + return kvm->arch.v_gipi; +} + +static inline int ls3a_ipi_in_kernel(struct kvm *kvm) +{ + int ret; + + ret = (ls3a_ipi_irqchip(kvm) != NULL); + return ret; +} + +int kvm_create_ls3a_ipi(struct kvm *kvm); +void kvm_destroy_ls3a_ipi(struct kvm *kvm); +int kvm_set_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state); +int kvm_get_ls3a_ipi(struct kvm *kvm, struct loongarch_gipiState *state); +int kvm_helper_send_ipi(struct kvm_vcpu *vcpu, unsigned int cpu, unsigned int action); +#endif diff --git a/arch/loongarch/kvm/intc/ls7a_irq.c b/arch/loongarch/kvm/intc/ls7a_irq.c new file mode 100644 index 0000000000000000000000000000000000000000..473c2a82652a1ed80cd27ce63be288d5b824ddff --- /dev/null +++ b/arch/loongarch/kvm/intc/ls7a_irq.c @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include "ls3a_ipi.h" +#include "ls7a_irq.h" +#include "ls3a_ext_irq.h" + +void ls7a_ioapic_lock(struct ls7a_kvm_ioapic *s, unsigned long *flags) +{ + unsigned long tmp; + spin_lock_irqsave(&s->lock, tmp); + *flags = tmp; +} + +void ls7a_ioapic_unlock(struct ls7a_kvm_ioapic *s, unsigned long *flags) +{ + unsigned long tmp; + tmp = *flags; + spin_unlock_irqrestore(&s->lock, tmp); +} + +static void kvm_ls7a_ioapic_raise(struct kvm *kvm, unsigned long mask) +{ + unsigned long irqnum, val; + struct ls7a_kvm_ioapic *s = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *state; + struct kvm_loongarch_interrupt irq; + int i; + + state = &s->ls7a_ioapic; + irq.cpu = -1; + val = mask & state->intirr & (~state->int_mask); + val &= ~state->intisr; + for_each_set_bit(i, &val, 64) { + state->intisr |= 0x1ULL << i; + irqnum = state->htmsi_vector[i]; + kvm_debug("msi_irq_handler,%ld,up\n", irqnum); + msi_irq_handler(kvm, irqnum, 1); + } + + kvm->stat.ls7a_ioapic_update++; +} + +static void kvm_ls7a_ioapic_lower(struct kvm *kvm, unsigned long mask) +{ + unsigned long irqnum, val; + struct ls7a_kvm_ioapic *s = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *state; + struct kvm_loongarch_interrupt irq; + int i; + + state = &s->ls7a_ioapic; + irq.cpu = -1; + val = mask & state->intisr; + for_each_set_bit(i, &val, 64) { + state->intisr &= ~(0x1ULL << i); + irqnum = state->htmsi_vector[i]; + kvm_debug("msi_irq_handler,%ld,down\n", irqnum); + msi_irq_handler(kvm, irqnum, 0); + } + + kvm->stat.ls7a_ioapic_update++; +} + +int kvm_ls7a_set_msi(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, int level, bool line_status) +{ + if (!level) + return -1; + + kvm_debug("msi data is 0x%x\n", e->msi.data); + msi_irq_handler(kvm, e->msi.data, 1); + return 0; +} + +int kvm_ls7a_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi) +{ + struct kvm_kernel_irq_routing_entry route; + + if (msi->flags != 0) + return -EINVAL; + + kvm->stat.ls7a_msi_irq++; + route.msi.address_lo = msi->address_lo; + route.msi.address_hi = msi->address_hi; + route.msi.data = msi->data; + + kvm_debug("msi data is 0x%x\n", route.msi.data); + return kvm_ls7a_set_msi(&route, kvm, + KVM_USERSPACE_IRQ_SOURCE_ID, 1, false); + +} + +int kvm_ls7a_ioapic_set_irq(struct kvm *kvm, int irq, int level) +{ + struct ls7a_kvm_ioapic *s; + struct kvm_ls7a_ioapic_state *state; + uint64_t mask = 1ULL << irq; + s = ls7a_ioapic_irqchip(kvm); + state = &s->ls7a_ioapic; + BUG_ON(irq < 0 || irq >= LS7A_IOAPIC_NUM_PINS); + + if (state->intedge & mask) { + /* edge triggered */ + if (level) { + if ((state->last_intirr & mask) == 0) { + state->intirr |= mask; + kvm_ls7a_ioapic_raise(kvm, mask); + } + state->last_intirr |= mask; + } else + state->last_intirr &= ~mask; + } else { + /* level triggered */ + if (!!level) { + if ((state->intirr & mask) == 0) { + state->intirr |= mask; + kvm_ls7a_ioapic_raise(kvm, mask); + } + } else { + if (state->intirr & mask) { + state->intirr &= ~mask; + kvm_ls7a_ioapic_lower(kvm, mask); + } + } + } + kvm->stat.ls7a_ioapic_set_irq++; + return 0; +} + +static int ls7a_ioapic_reg_write(struct ls7a_kvm_ioapic *s, + gpa_t addr, int len, const void *val) +{ + struct kvm *kvm; + struct kvm_ls7a_ioapic_state *state; + int64_t offset_tmp; + uint64_t offset; + uint64_t data, old; + + offset = addr & 0xfff; + kvm = s->kvm; + state = &(s->ls7a_ioapic); + + if (offset & (len - 1)) { + printk("%s(%d):unaligned address access %llx size %d \n", + __FUNCTION__, __LINE__, addr, len); + return 0; + } + + if (8 == len) { + data = *(uint64_t *)val; + switch (offset) { + case LS7A_INT_MASK_OFFSET: + old = state->int_mask; + state->int_mask = data; + if (old & ~data) + kvm_ls7a_ioapic_raise(kvm, old & ~data); + if (~old & data) + kvm_ls7a_ioapic_lower(kvm, ~old & data); + break; + case LS7A_INT_STATUS_OFFSET: + state->intisr = data; + break; + case LS7A_INT_EDGE_OFFSET: + state->intedge = data; + break; + case LS7A_INT_CLEAR_OFFSET: + /* + * only clear edge triggered irq on writing INTCLR reg + * no effect on level triggered irq + */ + data = data & state->intedge; + state->intirr &= ~data; + kvm_ls7a_ioapic_lower(kvm, data); + state->intisr &= (~data); + break; + case LS7A_INT_POL_OFFSET: + state->int_polarity = data; + break; + case LS7A_HTMSI_EN_OFFSET: + state->htmsi_en = data; + break; + case LS7A_AUTO_CTRL0_OFFSET: + case LS7A_AUTO_CTRL1_OFFSET: + break; + default: + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + break; + } + } else if (1 == len) { + data = *(unsigned char *)val; + if (offset >= LS7A_HTMSI_VEC_OFFSET) { + offset_tmp = offset - LS7A_HTMSI_VEC_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + state->htmsi_vector[offset_tmp] = + (uint8_t)(data & 0xff); + } + } else if (offset >= LS7A_ROUTE_ENTRY_OFFSET) { + offset_tmp = offset - LS7A_ROUTE_ENTRY_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + state->route_entry[offset_tmp] = + (uint8_t)(data & 0xff); + } + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + kvm->stat.ioapic_reg_write++; + return 0; +} + +static inline struct ls7a_kvm_ioapic *to_ioapic(struct kvm_io_device *dev) +{ + return container_of(dev, struct ls7a_kvm_ioapic, dev_ls7a_ioapic); +} + +static int kvm_ls7a_ioapic_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, + gpa_t addr, int len, const void *val) +{ + struct ls7a_kvm_ioapic *s = to_ioapic(this); + unsigned long flags; + + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + ls7a_ioapic_reg_write(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + + return 0; +} + +static int ls7a_ioapic_reg_read(struct ls7a_kvm_ioapic *s, + gpa_t addr, int len, void *val) +{ + uint64_t offset, offset_tmp; + struct kvm *kvm; + struct kvm_ls7a_ioapic_state *state; + uint64_t result = 0; + + state = &(s->ls7a_ioapic); + kvm = s->kvm; + offset = addr & 0xfff; + if (offset & (len - 1)) { + printk("%s(%d):unaligned address access %llx size %d \n", + __FUNCTION__, __LINE__, addr, len); + return 0; + } + + if (8 == len) { + switch (offset) { + case LS7A_INT_MASK_OFFSET: + result = state->int_mask; + break; + case LS7A_INT_STATUS_OFFSET: + result = state->intisr & (~state->int_mask); + break; + case LS7A_INT_EDGE_OFFSET: + result = state->intedge; + break; + case LS7A_INT_POL_OFFSET: + result = state->int_polarity; + break; + case LS7A_HTMSI_EN_OFFSET: + result = state->htmsi_en; + break; + case LS7A_AUTO_CTRL0_OFFSET: + case LS7A_AUTO_CTRL1_OFFSET: + break; + case LS7A_INT_ID_OFFSET: + result = LS7A_INT_ID_VER; + result = (result << 32) + LS7A_INT_ID_VAL; + break; + default: + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + break; + } + if (val != NULL) + *(uint64_t *)val = result; + } else if (1 == len) { + if (offset >= LS7A_HTMSI_VEC_OFFSET) { + offset_tmp = offset - LS7A_HTMSI_VEC_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + result = state->htmsi_vector[offset_tmp]; + } + } else if (offset >= LS7A_ROUTE_ENTRY_OFFSET) { + offset_tmp = offset - LS7A_ROUTE_ENTRY_OFFSET; + if (offset_tmp >= 0 && offset_tmp < 64) { + result = state->route_entry[offset_tmp]; + } + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + if (val != NULL) + *(unsigned char *)val = result; + } else { + WARN_ONCE(1, "Abnormal address access:addr 0x%llx,len %d\n", addr, len); + } + kvm->stat.ioapic_reg_read++; + return result; +} + +static int kvm_ls7a_ioapic_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, + gpa_t addr, int len, void *val) +{ + struct ls7a_kvm_ioapic *s = to_ioapic(this); + unsigned long flags; + uint64_t result = 0; + + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + result = ls7a_ioapic_reg_read(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + return 0; +} + +static const struct kvm_io_device_ops kvm_ls7a_ioapic_ops = { + .read = kvm_ls7a_ioapic_read, + .write = kvm_ls7a_ioapic_write, +}; + +static int kvm_ls7a_ioapic_alias_read(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, gpa_t addr, int len, void *val) +{ + struct ls7a_kvm_ioapic *s; + unsigned long flags; + + s = container_of(this, struct ls7a_kvm_ioapic, ls7a_ioapic_alias); + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + ls7a_ioapic_reg_read(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + return 0; +} + +static int kvm_ls7a_ioapic_alias_write(struct kvm_vcpu *vcpu, + struct kvm_io_device *this, gpa_t addr, int len, const void *val) +{ + struct ls7a_kvm_ioapic *s; + unsigned long flags; + + s = container_of(this, struct ls7a_kvm_ioapic, ls7a_ioapic_alias); + ls7a_ioapic_lock(s->kvm->arch.v_ioapic, &flags); + ls7a_ioapic_reg_write(s, addr, len, val); + ls7a_ioapic_unlock(s->kvm->arch.v_ioapic, &flags); + + return 0; +} + +static const struct kvm_io_device_ops kvm_ls7a_ioapic_ops_alias = { + .read = kvm_ls7a_ioapic_alias_read, + .write = kvm_ls7a_ioapic_alias_write, +}; + +int kvm_create_ls7a_ioapic(struct kvm *kvm) +{ + struct ls7a_kvm_ioapic *s; + int ret; + unsigned long ls7a_ioapic_reg_base; + + s = kzalloc(sizeof(struct ls7a_kvm_ioapic), GFP_KERNEL); + if (!s) + return -ENOMEM; + spin_lock_init(&s->lock); + s->kvm = kvm; + + ls7a_ioapic_reg_base = LS7A_IOAPIC_GUEST_REG_BASE; + + /* + * Initialize MMIO device + */ + kvm_iodevice_init(&s->dev_ls7a_ioapic, &kvm_ls7a_ioapic_ops); + mutex_lock(&kvm->slots_lock); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ls7a_ioapic_reg_base, + 0x1000, &s->dev_ls7a_ioapic); + if (ret < 0) { + kvm_err("Failed register ioapic, err:%d\n", ret); + goto fail_unlock; + } + + ls7a_ioapic_reg_base = LS7A_IOAPIC_GUEST_REG_BASE_ALIAS; + kvm_iodevice_init(&s->ls7a_ioapic_alias, &kvm_ls7a_ioapic_ops_alias); + ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ls7a_ioapic_reg_base, + 0x1000, &s->ls7a_ioapic_alias); + if (ret < 0) { + kvm_err("Failed register alias ioapic, err:%d\n", ret); + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, + &s->dev_ls7a_ioapic); + goto fail_unlock; + } + mutex_unlock(&kvm->slots_lock); + + kvm->arch.v_ioapic = s; + + return 0; + +fail_unlock: + mutex_unlock(&kvm->slots_lock); + kfree(s); + + return -EFAULT; +} + + +int kvm_get_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state) +{ + struct ls7a_kvm_ioapic *ls7a_ioapic = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *ioapic_state = + &(ls7a_ioapic->ls7a_ioapic); + unsigned long flags; + + ls7a_ioapic_lock(ls7a_ioapic, &flags); + memcpy(state, ioapic_state, sizeof(struct kvm_ls7a_ioapic_state)); + ls7a_ioapic_unlock(ls7a_ioapic, &flags); + kvm->stat.get_ls7a_ioapic++; + return 0; +} + +int kvm_set_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state) +{ + struct ls7a_kvm_ioapic *ls7a_ioapic = ls7a_ioapic_irqchip(kvm); + struct kvm_ls7a_ioapic_state *ioapic_state = + &(ls7a_ioapic->ls7a_ioapic); + unsigned long flags; + + if (!ls7a_ioapic) + return -EINVAL; + + ls7a_ioapic_lock(ls7a_ioapic, &flags); + memcpy(ioapic_state, state, sizeof(struct kvm_ls7a_ioapic_state)); + ls7a_ioapic_unlock(ls7a_ioapic, &flags); + kvm->stat.set_ls7a_ioapic++; + return 0; +} + +void kvm_destroy_ls7a_ioapic(struct kvm *kvm) +{ + struct ls7a_kvm_ioapic *vpic = kvm->arch.v_ioapic; + if (!vpic) + return; + kvm_io_bus_unregister_dev(vpic->kvm, KVM_MMIO_BUS, + &vpic->ls7a_ioapic_alias); + kvm_io_bus_unregister_dev(vpic->kvm, KVM_MMIO_BUS, + &vpic->dev_ls7a_ioapic); + kfree(vpic); +} + +void kvm_dump_ls7a_ioapic_state(struct seq_file *m, + struct ls7a_kvm_ioapic *ioapic) +{ + struct kvm_ls7a_ioapic_state *ioapic_state; + unsigned long flags; + int i = 0; + + if (!ioapic) + return; + + seq_puts(m, "\nIOAPIC state:\n"); + ioapic_state = &(ioapic->ls7a_ioapic); + + ls7a_ioapic_lock(ioapic, &flags); + seq_puts(m, "irq masked: "); + for (i = 0; i < 64; i++) { + if (!test_bit(i, (void *)&ioapic_state->int_mask)) + seq_printf(m, "%d ", i); + } + seq_printf(m, "\nhtmsi_en:0x%016llx\n" + "intedge:0x%016llx", + ioapic_state->htmsi_en, + ioapic_state->intedge); + + seq_puts(m, "\nroute_entry: "); + for (i = 0; i < 64; i++) + seq_printf(m, "%d ", ioapic_state->route_entry[i]); + + seq_puts(m, "\nhtmsi_vector: "); + for (i = 0; i < 64; i++) + seq_printf(m, "%d ", ioapic_state->htmsi_vector[i]); + + seq_printf(m, "\nintirr:%016llx\n" + "intisr:%016llx\n", + ioapic_state->intirr, + ioapic_state->intisr); + ls7a_ioapic_unlock(ioapic, &flags); +} diff --git a/arch/loongarch/kvm/intc/ls7a_irq.h b/arch/loongarch/kvm/intc/ls7a_irq.h new file mode 100644 index 0000000000000000000000000000000000000000..0c91b63bf88f569b36daa90c0d6d9894fa085391 --- /dev/null +++ b/arch/loongarch/kvm/intc/ls7a_irq.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LS7A_KVM_IRQ_H +#define __LS7A_KVM_IRQ_H + +#include +#include +#include +#include +#include +#include "kvmcpu.h" + +#include + +#define LS7A_APIC_NUM_PINS 64 + +#define LS7A_ROUTE_ENTRY_OFFSET 0x100 +#define LS7A_INT_ID_OFFSET 0x0 +#define LS7A_INT_ID_VAL 0x7000000UL +#define LS7A_INT_ID_VER 0x1f0001UL +#define LS7A_INT_MASK_OFFSET 0x20 +#define LS7A_INT_EDGE_OFFSET 0x60 +#define LS7A_INT_CLEAR_OFFSET 0x80 +#define LS7A_INT_STATUS_OFFSET 0x3a0 +#define LS7A_INT_POL_OFFSET 0x3e0 +#define LS7A_HTMSI_EN_OFFSET 0x40 +#define LS7A_HTMSI_VEC_OFFSET 0x200 +#define LS7A_AUTO_CTRL0_OFFSET 0xc0 +#define LS7A_AUTO_CTRL1_OFFSET 0xe0 + +#define LS7A_IOAPIC_GUEST_REG_BASE 0x10000000UL +#define LS7A_IOAPIC_GUEST_REG_BASE_ALIAS 0xe0010000000UL + +#define LS7A_IOAPIC_NUM_PINS 32 + +typedef struct kvm_ls7a_ioapic_state { + u64 int_id; + /* 0x020 interrupt mask register */ + u64 int_mask; + /* 0x040 1=msi */ + u64 htmsi_en; + /* 0x060 edge=1 level =0 */ + u64 intedge; + /* 0x080 for clean edge int,set 1 clean,set 0 is noused */ + u64 intclr; + /* 0x0c0 */ + u64 auto_crtl0; + /* 0x0e0 */ + u64 auto_crtl1; + /* 0x100 - 0x140 */ + u8 route_entry[64]; + /* 0x200 - 0x240 */ + u8 htmsi_vector[64]; + /* 0x300 */ + u64 intisr_chip0; + /* 0x320 */ + u64 intisr_chip1; + /* edge detection */ + u64 last_intirr; + /* 0x380 interrupt request register */ + u64 intirr; + /* 0x3a0 interrupt service register */ + u64 intisr; + /* 0x3e0 interrupt level polarity selection register, + * 0 for high level tirgger + */ + u64 int_polarity; +} LS7AApicState; + +struct ls7a_kvm_ioapic { + spinlock_t lock; + bool wakeup_needed; + unsigned pending_acks; + struct kvm *kvm; + struct kvm_ls7a_ioapic_state ls7a_ioapic; + struct kvm_io_device dev_ls7a_ioapic; + struct kvm_io_device ls7a_ioapic_alias; + void (*ack_notifier)(void *opaque, int irq); + unsigned long irq_states[LS7A_APIC_NUM_PINS]; +}; + +static inline struct ls7a_kvm_ioapic *ls7a_ioapic_irqchip(struct kvm *kvm) +{ + return kvm->arch.v_ioapic; +} + +static inline int ls7a_ioapic_in_kernel(struct kvm *kvm) +{ + int ret; + + ret = (ls7a_ioapic_irqchip(kvm) != NULL); + return ret; +} + +int kvm_set_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state); +int kvm_get_ls7a_ioapic(struct kvm *kvm, struct ls7a_ioapic_state *state); +int kvm_create_ls7a_ioapic(struct kvm *kvm); +void kvm_destroy_ls7a_ioapic(struct kvm *kvm); +int kvm_ls7a_ioapic_set_irq(struct kvm *kvm, int irq, int level); + +void ls7a_ioapic_lock(struct ls7a_kvm_ioapic *s, unsigned long *flags); +void ls7a_ioapic_unlock(struct ls7a_kvm_ioapic *s, unsigned long *flags); +int kvm_ls7a_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); +int kvm_ls7a_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, + int irq_source_id, int level, bool line_status); + +void kvm_dump_ls7a_ioapic_state(struct seq_file *m, struct ls7a_kvm_ioapic *ioapic); +#endif diff --git a/arch/loongarch/kvm/interrupt.c b/arch/loongarch/kvm/interrupt.c new file mode 100644 index 0000000000000000000000000000000000000000..fb7c4ef7f21f31310546255899c06c48d1ffee54 --- /dev/null +++ b/arch/loongarch/kvm/interrupt.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include +#include "kvm_compat.h" + +static u32 int_to_coreint[LOONGARCH_EXC_MAX] = { + [LARCH_INT_TIMER] = CPU_TIMER, + [LARCH_INT_IPI] = CPU_IPI, + [LARCH_INT_SIP0] = CPU_SIP0, + [LARCH_INT_SIP1] = CPU_SIP1, + [LARCH_INT_IP0] = CPU_IP0, + [LARCH_INT_IP1] = CPU_IP1, + [LARCH_INT_IP2] = CPU_IP2, + [LARCH_INT_IP3] = CPU_IP3, + [LARCH_INT_IP4] = CPU_IP4, + [LARCH_INT_IP5] = CPU_IP5, + [LARCH_INT_IP6] = CPU_IP6, + [LARCH_INT_IP7] = CPU_IP7, +}; + +static int _kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority) +{ + unsigned int irq = 0; + + clear_bit(priority, &vcpu->arch.irq_pending); + if (priority < LOONGARCH_EXC_MAX) + irq = int_to_coreint[priority]; + + switch (priority) { + case LARCH_INT_TIMER: + case LARCH_INT_IPI: + case LARCH_INT_SIP0: + case LARCH_INT_SIP1: + kvm_set_gcsr_estat(irq); + break; + + case LARCH_INT_IP0: + case LARCH_INT_IP1: + case LARCH_INT_IP2: + case LARCH_INT_IP3: + case LARCH_INT_IP4: + case LARCH_INT_IP5: + case LARCH_INT_IP6: + case LARCH_INT_IP7: + kvm_set_csr_gintc(irq); + break; + + default: + break; + } + + return 1; +} + +static int _kvm_irq_clear(struct kvm_vcpu *vcpu, unsigned int priority) +{ + unsigned int irq = 0; + + clear_bit(priority, &vcpu->arch.irq_clear); + if (priority < LOONGARCH_EXC_MAX) + irq = int_to_coreint[priority]; + + switch (priority) { + case LARCH_INT_TIMER: + case LARCH_INT_IPI: + case LARCH_INT_SIP0: + case LARCH_INT_SIP1: + kvm_clear_gcsr_estat(irq); + break; + + case LARCH_INT_IP0: + case LARCH_INT_IP1: + case LARCH_INT_IP2: + case LARCH_INT_IP3: + case LARCH_INT_IP4: + case LARCH_INT_IP5: + case LARCH_INT_IP6: + case LARCH_INT_IP7: + kvm_clear_csr_gintc(irq); + break; + + default: + break; + } + + return 1; +} + +void _kvm_deliver_intr(struct kvm_vcpu *vcpu) +{ + unsigned long *pending = &vcpu->arch.irq_pending; + unsigned long *pending_clr = &vcpu->arch.irq_clear; + unsigned int priority; + + if (!(*pending) && !(*pending_clr)) + return; + + if (*pending_clr) { + priority = __ffs(*pending_clr); + while (priority <= LOONGARCH_EXC_IPNUM) { + _kvm_irq_clear(vcpu, priority); + priority = find_next_bit(pending_clr, + BITS_PER_BYTE * sizeof(*pending_clr), + priority + 1); + } + } + + if (*pending) { + priority = __ffs(*pending); + while (priority <= LOONGARCH_EXC_IPNUM) { + _kvm_irq_deliver(vcpu, priority); + priority = find_next_bit(pending, + BITS_PER_BYTE * sizeof(*pending), + priority + 1); + } + } + +} + +int _kvm_pending_timer(struct kvm_vcpu *vcpu) +{ + return test_bit(LARCH_INT_TIMER, &vcpu->arch.irq_pending); +} diff --git a/arch/loongarch/kvm/irq.h b/arch/loongarch/kvm/irq.h new file mode 100644 index 0000000000000000000000000000000000000000..344ba5ebc4b5eb8127013e1f5a5e77d1602e3962 --- /dev/null +++ b/arch/loongarch/kvm/irq.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LOONGHARCH_KVM_IRQ_H__ +#define __LOONGHARCH_KVM_IRQ_H__ + +static inline int irqchip_in_kernel(struct kvm *kvm) +{ + return kvm->arch.v_ioapic ? 1 : 0; +} + +#endif diff --git a/arch/loongarch/kvm/irqfd.c b/arch/loongarch/kvm/irqfd.c new file mode 100644 index 0000000000000000000000000000000000000000..be2b1bfd07e2ffb6163f46ead977745c0a0b49af --- /dev/null +++ b/arch/loongarch/kvm/irqfd.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include "intc/ls7a_irq.h" + +static int kvm_ls7a_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + /* ioapic pin (0 ~ 64) <---> gsi(0 ~ 64) */ + u32 irq = e->irqchip.pin; + unsigned long flags; + int ret; + + if (!ls7a_ioapic_in_kernel(kvm)) + return -ENXIO; + + ls7a_ioapic_lock(ls7a_ioapic_irqchip(kvm), &flags); + ret = kvm_ls7a_ioapic_set_irq(kvm, irq, level); + ls7a_ioapic_unlock(ls7a_ioapic_irqchip(kvm), &flags); + + return ret; +} + +/** + * kvm_set_routing_entry: populate a kvm routing entry + * from a user routing entry + * + * @kvm: the VM this entry is applied to + * @e: kvm kernel routing entry handle + * @ue: user api routing entry handle + * return 0 on success, -EINVAL on errors. + */ +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + int r = -EINVAL; + + switch (ue->type) { + case KVM_IRQ_ROUTING_IRQCHIP: + e->set = kvm_ls7a_set_ioapic_irq; + + e->irqchip.irqchip = ue->u.irqchip.irqchip; + e->irqchip.pin = ue->u.irqchip.pin; + + if (e->irqchip.pin >= LS7A_APIC_NUM_PINS) + goto out; + break; + case KVM_IRQ_ROUTING_MSI: + e->set = kvm_set_msi; + e->msi.address_lo = ue->u.msi.address_lo; + e->msi.address_hi = ue->u.msi.address_hi; + e->msi.data = ue->u.msi.data; + /* + * linux4.19 kernel have flags and devid + * e->msi.flags = ue->flags; + * e->msi.devid = ue->u.msi.devid; + */ + break; + default: + goto out; + } + r = 0; +out: + return r; +} + +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + if (e->type == KVM_IRQ_ROUTING_MSI) + return kvm_ls7a_set_msi(e, kvm, irq_source_id, 1, false); + + return -EWOULDBLOCK; +} + +/** + * kvm_set_msi: inject the MSI corresponding to the + * MSI routing entry + * + * This is the entry point for irqfd MSI injection + * and userspace MSI injection. + */ +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + unsigned int ret; + + if (!level) + return -1; + + ret = kvm_ls7a_set_msi(e, kvm, irq_source_id, 1, false); + return ret; +} diff --git a/arch/loongarch/kvm/kvm_compat.c b/arch/loongarch/kvm/kvm_compat.c new file mode 100644 index 0000000000000000000000000000000000000000..277d7760aa11a5071c007c37c99058ee86a9a050 --- /dev/null +++ b/arch/loongarch/kvm/kvm_compat.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include "kvm_compat.h" + +extern int _kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +void kvm_arch_check_processor_compat(void *rtn) +{ + *(int *)rtn = 0; +} + +void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + _kvm_set_spte_hva(kvm, hva, pte); + return; +} + +#elif (LINUX_VERSION_CODE == KERNEL_VERSION(5, 4, 0)) +int kvm_arch_check_processor_compat(void) +{ + return 0; +} + +int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + bool is_dirty = false; + int r; + + mutex_lock(&kvm->slots_lock); + + r = kvm_clear_dirty_log_protect(kvm, log, &is_dirty); + + if (is_dirty) { + slots = kvm_memslots(kvm); + memslot = id_to_memslot(slots, log->slot); + + /* Let implementation handle TLB/GVA invalidation */ + kvm_flush_remote_tlbs(kvm); + } + + mutex_unlock(&kvm->slots_lock); + return r; +} + +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + return _kvm_set_spte_hva(kvm, hva, pte); +} +#else +int kvm_arch_check_processor_compat(void *opaque) +{ + return 0; +} + +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + return _kvm_set_spte_hva(kvm, hva, pte); +} + +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + +} + +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) +{ + return 0; +} + +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ +#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL + kvm_flush_remote_tlbs (kvm); +#endif +} +#endif diff --git a/arch/loongarch/kvm/kvm_compat.h b/arch/loongarch/kvm/kvm_compat.h new file mode 100644 index 0000000000000000000000000000000000000000..1da54b2d80ecea367240534b446c8e53dae89b7d --- /dev/null +++ b/arch/loongarch/kvm/kvm_compat.h @@ -0,0 +1,830 @@ +#ifndef __LOONGARCH_KVM_COMPAT_H__ +#define __LOONGARCH_KVM_COMPAT_H__ + +#ifdef __ASSEMBLY__ +#define _ULCAST_ +#else +#define _ULCAST_ (unsigned long) +#include +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) +#include +#else +#include +#endif +#endif + +#define KVM_REG_A0 0x4 +#define KVM_REG_A1 0x5 +#define KVM_REG_A2 0x6 +#define KVM_REG_A3 0x7 +/* + * ExStatus.ExcCode + */ +#define KVM_EXCCODE_RSV 0 /* Reserved */ +#define KVM_EXCCODE_TLBL 1 /* TLB miss on a load */ +#define KVM_EXCCODE_TLBS 2 /* TLB miss on a store */ +#define KVM_EXCCODE_TLBI 3 /* TLB miss on a ifetch */ +#define KVM_EXCCODE_TLBM 4 /* TLB modified fault */ +#define KVM_EXCCODE_TLBRI 5 /* TLB Read-Inhibit exception */ +#define KVM_EXCCODE_TLBXI 6 /* TLB Execution-Inhibit exception */ +#define KVM_EXCCODE_TLBPE 7 /* TLB Privilege Error */ +#define KVM_EXCCODE_ADE 8 /* Address Error */ +#define KVM_EXCCODE_ALE 9 /* Unalign Access */ +#define KVM_EXCCODE_OOB 10 /* Out of bounds */ +#define KVM_EXCCODE_SYS 11 /* System call */ +#define KVM_EXCCODE_BP 12 /* Breakpoint */ +#define KVM_EXCCODE_INE 13 /* Inst. Not Exist */ +#define KVM_EXCCODE_IPE 14 /* Inst. Privileged Error */ +#define KVM_EXCCODE_FPDIS 15 /* FPU Disabled */ +#define KVM_EXCCODE_LSXDIS 16 /* LSX Disabled */ +#define KVM_EXCCODE_LASXDIS 17 /* LASX Disabled */ +#define KVM_EXCCODE_FPE 18 /* Floating Point Exception */ +#define KVM_EXCCODE_WATCH 19 /* Watch address reference */ +#define KVM_EXCCODE_BTDIS 20 /* Binary Trans. Disabled */ +#define KVM_EXCCODE_BTE 21 /* Binary Trans. Exception */ +#define KVM_EXCCODE_GSPR 22 /* Guest Privileged Error */ +#define KVM_EXCCODE_HYP 23 /* Hypercall */ +#define KVM_EXCCODE_GCM 24 /* Guest CSR modified */ + +#define KVM_INT_START 64 +#define KVM_INT_SIP0 64 +#define KVM_INT_SIP1 65 +#define KVM_INT_IP0 66 +#define KVM_INT_IP1 67 +#define KVM_INT_IP2 68 +#define KVM_INT_IP3 69 +#define KVM_INT_IP4 70 +#define KVM_INT_IP5 71 +#define KVM_INT_IP6 72 +#define KVM_INT_IP7 73 +#define KVM_INT_PC 74 /* Performance Counter */ +#define KVM_INT_TIMER 75 +#define KVM_INT_IPI 76 +#define KVM_INT_NMI 77 +#define KVM_INT_END 78 +#define KVM_INT_NUM (KVM_INT_END - KVM_INT_START) + +#define KVM_CSR_CRMD 0x0 /* Current mode info */ +#define KVM_CRMD_WE_SHIFT 9 +#define KVM_CRMD_WE (_ULCAST_(0x1) << KVM_CRMD_WE_SHIFT) +#define KVM_CRMD_DACM_SHIFT 7 +#define KVM_CRMD_DACM_WIDTH 2 +#define KVM_CRMD_DACM (_ULCAST_(0x3) << KVM_CRMD_DACM_SHIFT) +#define KVM_CRMD_DACF_SHIFT 5 +#define KVM_CRMD_DACF_WIDTH 2 +#define KVM_CRMD_DACF (_ULCAST_(0x3) << KVM_CRMD_DACF_SHIFT) +#define KVM_CRMD_PG_SHIFT 4 +#define KVM_CRMD_PG (_ULCAST_(0x1) << KVM_CRMD_PG_SHIFT) +#define KVM_CRMD_DA_SHIFT 3 +#define KVM_CRMD_DA (_ULCAST_(0x1) << KVM_CRMD_DA_SHIFT) +#define KVM_CRMD_IE_SHIFT 2 +#define KVM_CRMD_IE (_ULCAST_(0x1) << KVM_CRMD_IE_SHIFT) +#define KVM_CRMD_PLV_SHIFT 0 +#define KVM_CRMD_PLV_WIDTH 2 +#define KVM_CRMD_PLV (_ULCAST_(0x3) << KVM_CRMD_PLV_SHIFT) + +#define KVM_CSR_PRMD 0x1 /* Prev-exception mode info */ +#define KVM_PRMD_PIE_SHIFT 2 +#define KVM_PRMD_PWE_SHIFT 3 +#define KVM_PRMD_PIE (_ULCAST_(0x1) << KVM_PRMD_PIE_SHIFT) +#define KVM_PRMD_PWE (_ULCAST_(0x1) << KVM_PRMD_PWE_SHIFT) +#define KVM_PRMD_PPLV_SHIFT 0 +#define KVM_PRMD_PPLV_WIDTH 2 +#define KVM_PRMD_PPLV (_ULCAST_(0x3) << KVM_PRMD_PPLV_SHIFT) + +#define KVM_CSR_EUEN 0x2 /* Extended unit enable */ +#define KVM_EUEN_LBTEN_SHIFT 3 +#define KVM_EUEN_LBTEN (_ULCAST_(0x1) << KVM_EUEN_LBTEN_SHIFT) +#define KVM_EUEN_LASXEN_SHIFT 2 +#define KVM_EUEN_LASXEN (_ULCAST_(0x1) << KVM_EUEN_LASXEN_SHIFT) +#define KVM_EUEN_LSXEN_SHIFT 1 +#define KVM_EUEN_LSXEN (_ULCAST_(0x1) << KVM_EUEN_LSXEN_SHIFT) +#define KVM_EUEN_FPEN_SHIFT 0 +#define KVM_EUEN_FPEN (_ULCAST_(0x1) << KVM_EUEN_FPEN_SHIFT) + +#define KVM_CSR_MISC 0x3 /* Misc config */ +#define KVM_CSR_ECFG 0x4 /* Exception config */ +#define KVM_ECFG_VS_SHIFT 16 +#define KVM_ECFG_VS_WIDTH 3 +#define KVM_ECFG_VS (_ULCAST_(0x7) << KVM_ECFG_VS_SHIFT) +#define KVM_ECFG_IM_SHIFT 0 +#define KVM_ECFG_IM_WIDTH 13 +#define KVM_ECFG_IM (_ULCAST_(0x1fff) << KVM_ECFG_IM_SHIFT) + +#define KVM_CSR_ESTAT 0x5 /* Exception status */ +#define KVM_ESTAT_ESUBCODE_SHIFT 22 +#define KVM_ESTAT_ESUBCODE_WIDTH 9 +#define KVM_ESTAT_ESUBCODE (_ULCAST_(0x1ff) << KVM_ESTAT_ESUBCODE_SHIFT) +#define KVM_ESTAT_EXC_SHIFT 16 +#define KVM_ESTAT_EXC_WIDTH 6 +#define KVM_ESTAT_EXC (_ULCAST_(0x3f) << KVM_ESTAT_EXC_SHIFT) +#define KVM_ESTAT_IS_SHIFT 0 +#define KVM_ESTAT_IS_WIDTH 15 +#define KVM_ESTAT_IS (_ULCAST_(0x7fff) << KVM_ESTAT_IS_SHIFT) + +#define KVM_CSR_ERA 0x6 /* ERA */ +#define KVM_CSR_BADV 0x7 /* Bad virtual address */ +#define KVM_CSR_BADI 0x8 /* Bad instruction */ +#define KVM_CSR_EENTRY 0xc /* Exception entry base address */ +#define KVM_CSR_TLBIDX 0x10 /* TLB Index, EHINV, PageSize, NP */ +#define KVM_CSR_TLBEHI 0x11 /* TLB EntryHi */ +#define KVM_CSR_TLBELO0 0x12 /* TLB EntryLo0 */ +#define KVM_CSR_TLBELO1 0x13 /* TLB EntryLo1 */ +#define KVM_CSR_GTLBC 0x15 /* Guest TLB control */ +#define KVM_GTLBC_TGID_SHIFT 16 +#define KVM_GTLBC_TGID_WIDTH 8 +#define KVM_GTLBC_TGID (_ULCAST_(0xff) << KVM_GTLBC_TGID_SHIFT) +#define KVM_GTLBC_TOTI_SHIFT 13 +#define KVM_GTLBC_TOTI (_ULCAST_(0x1) << KVM_GTLBC_TOTI_SHIFT) +#define KVM_GTLBC_USETGID_SHIFT 12 +#define KVM_GTLBC_USETGID (_ULCAST_(0x1) << KVM_GTLBC_USETGID_SHIFT) +#define KVM_GTLBC_GMTLBSZ_SHIFT 0 +#define KVM_GTLBC_GMTLBSZ_WIDTH 6 +#define KVM_GTLBC_GMTLBSZ (_ULCAST_(0x3f) << KVM_GTLBC_GMTLBSZ_SHIFT) + +#define KVM_CSR_TRGP 0x16 /* TLBR read guest info */ +#define KVM_CSR_ASID 0x18 /* ASID */ +#define KVM_CSR_PGDL 0x19 /* Page table base address when VA[47] = 0 */ +#define KVM_CSR_PGDH 0x1a /* Page table base address when VA[47] = 1 */ +#define KVM_CSR_PGD 0x1b /* Page table base */ +#define KVM_CSR_PWCTL0 0x1c /* PWCtl0 */ +#define KVM_CSR_PWCTL1 0x1d /* PWCtl1 */ +#define KVM_CSR_STLBPGSIZE 0x1e +#define KVM_CSR_RVACFG 0x1f +#define KVM_CSR_CPUID 0x20 /* CPU core number */ +#define KVM_CSR_PRCFG1 0x21 /* Config1 */ +#define KVM_CSR_PRCFG2 0x22 /* Config2 */ +#define KVM_CSR_PRCFG3 0x23 /* Config3 */ +#define KVM_CSR_KS0 0x30 +#define KVM_CSR_KS1 0x31 +#define KVM_CSR_KS2 0x32 +#define KVM_CSR_KS3 0x33 +#define KVM_CSR_KS4 0x34 +#define KVM_CSR_KS5 0x35 +#define KVM_CSR_KS6 0x36 +#define KVM_CSR_KS7 0x37 +#define KVM_CSR_KS8 0x38 +#define KVM_CSR_TMID 0x40 /* Timer ID */ +#define KVM_CSR_TCFG 0x41 /* Timer config */ +#define KVM_TCFG_VAL_SHIFT 2 +#define KVM_TCFG_VAL_WIDTH 48 +#define KVM_TCFG_VAL (_ULCAST_(0x3fffffffffff) << KVM_TCFG_VAL_SHIFT) +#define KVM_TCFG_PERIOD_SHIFT 1 +#define KVM_TCFG_PERIOD (_ULCAST_(0x1) << KVM_TCFG_PERIOD_SHIFT) +#define KVM_TCFG_EN (_ULCAST_(0x1)) + +#define KVM_CSR_TVAL 0x42 /* Timer value */ +#define KVM_CSR_CNTC 0x43 /* Timer offset */ +#define KVM_CSR_TINTCLR 0x44 /* Timer interrupt clear */ +#define KVM_CSR_GSTAT 0x50 /* Guest status */ +#define KVM_GSTAT_GID_SHIFT 16 +#define KVM_GSTAT_GID_WIDTH 8 +#define KVM_GSTAT_GID (_ULCAST_(0xff) << KVM_GSTAT_GID_SHIFT) +#define KVM_GSTAT_GIDBIT_SHIFT 4 +#define KVM_GSTAT_GIDBIT_WIDTH 6 +#define KVM_GSTAT_GIDBIT (_ULCAST_(0x3f) << KVM_GSTAT_GIDBIT_SHIFT) +#define KVM_GSTAT_PVM_SHIFT 1 +#define KVM_GSTAT_PVM (_ULCAST_(0x1) << KVM_GSTAT_PVM_SHIFT) +#define KVM_GSTAT_VM_SHIFT 0 +#define KVM_GSTAT_VM (_ULCAST_(0x1) << KVM_GSTAT_VM_SHIFT) + +#define KVM_CSR_GCFG 0x51 /* Guest config */ +#define KVM_GCFG_GPERF_SHIFT 24 +#define KVM_GCFG_GPERF_WIDTH 3 +#define KVM_GCFG_GPERF (_ULCAST_(0x7) << KVM_GCFG_GPERF_SHIFT) +#define KVM_GCFG_GCI_SHIFT 20 +#define KVM_GCFG_GCI_WIDTH 2 +#define KVM_GCFG_GCI (_ULCAST_(0x3) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCI_ALL (_ULCAST_(0x0) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCI_HIT (_ULCAST_(0x1) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCI_SECURE (_ULCAST_(0x2) << KVM_GCFG_GCI_SHIFT) +#define KVM_GCFG_GCIP_SHIFT 16 +#define KVM_GCFG_GCIP (_ULCAST_(0xf) << KVM_GCFG_GCIP_SHIFT) +#define KVM_GCFG_GCIP_ALL (_ULCAST_(0x1) << KVM_GCFG_GCIP_SHIFT) +#define KVM_GCFG_GCIP_HIT (_ULCAST_(0x1) << (KVM_GCFG_GCIP_SHIFT + 1)) +#define KVM_GCFG_GCIP_SECURE (_ULCAST_(0x1) << (KVM_GCFG_GCIP_SHIFT + 2)) +#define KVM_GCFG_TORU_SHIFT 15 +#define KVM_GCFG_TORU (_ULCAST_(0x1) << KVM_GCFG_TORU_SHIFT) +#define KVM_GCFG_TORUP_SHIFT 14 +#define KVM_GCFG_TORUP (_ULCAST_(0x1) << KVM_GCFG_TORUP_SHIFT) +#define KVM_GCFG_TOP_SHIFT 13 +#define KVM_GCFG_TOP (_ULCAST_(0x1) << KVM_GCFG_TOP_SHIFT) +#define KVM_GCFG_TOPP_SHIFT 12 +#define KVM_GCFG_TOPP (_ULCAST_(0x1) << KVM_GCFG_TOPP_SHIFT) +#define KVM_GCFG_TOE_SHIFT 11 +#define KVM_GCFG_TOE (_ULCAST_(0x1) << KVM_GCFG_TOE_SHIFT) +#define KVM_GCFG_TOEP_SHIFT 10 +#define KVM_GCFG_TOEP (_ULCAST_(0x1) << KVM_GCFG_TOEP_SHIFT) +#define KVM_GCFG_TIT_SHIFT 9 +#define KVM_GCFG_TIT (_ULCAST_(0x1) << KVM_GCFG_TIT_SHIFT) +#define KVM_GCFG_TITP_SHIFT 8 +#define KVM_GCFG_TITP (_ULCAST_(0x1) << KVM_GCFG_TITP_SHIFT) +#define KVM_GCFG_SIT_SHIFT 7 +#define KVM_GCFG_SIT (_ULCAST_(0x1) << KVM_GCFG_SIT_SHIFT) +#define KVM_GCFG_SITP_SHIFT 6 +#define KVM_GCFG_SITP (_ULCAST_(0x1) << KVM_GCFG_SITP_SHIFT) +#define KVM_GCFG_MATC_SHITF 4 +#define KVM_GCFG_MATC_WIDTH 2 +#define KVM_GCFG_MATC_MASK (_ULCAST_(0x3) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATC_GUEST (_ULCAST_(0x0) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATC_ROOT (_ULCAST_(0x1) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATC_NEST (_ULCAST_(0x2) << KVM_GCFG_MATC_SHITF) +#define KVM_GCFG_MATP_SHITF 0 +#define KVM_GCFG_MATP_WIDTH 4 +#define KVM_GCFG_MATR_MASK (_ULCAST_(0x3) << KVM_GCFG_MATP_SHITF) +#define KVM_GCFG_MATP_GUEST (_ULCAST_(0x0) << KVM_GCFG_MATP_SHITF) +#define KVM_GCFG_MATP_ROOT (_ULCAST_(0x1) << KVM_GCFG_MATP_SHITF) +#define KVM_GCFG_MATP_NEST (_ULCAST_(0x2) << KVM_GCFG_MATP_SHITF) + +#define KVM_CSR_GINTC 0x52 /* Guest interrupt control */ +#define KVM_CSR_GCNTC 0x53 /* Guest timer offset */ +#define KVM_CSR_LLBCTL 0x60 /* LLBit control */ +#define KVM_LLBCTL_ROLLB_SHIFT 0 +#define KVM_LLBCTL_ROLLB (_ULCAST_(1) << KVM_LLBCTL_ROLLB_SHIFT) +#define KVM_LLBCTL_WCLLB_SHIFT 1 +#define KVM_LLBCTL_WCLLB (_ULCAST_(1) << KVM_LLBCTL_WCLLB_SHIFT) +#define KVM_LLBCTL_KLO_SHIFT 2 +#define KVM_LLBCTL_KLO (_ULCAST_(1) << KVM_LLBCTL_KLO_SHIFT) + +#define KVM_CSR_IMPCTL1 0x80 /* Loongson config1 */ +#define KVM_CSR_IMPCTL2 0x81 /* Loongson config2 */ +#define KVM_CSR_GNMI 0x82 +#define KVM_CSR_TLBRENTRY 0x88 /* TLB refill exception base address */ +#define KVM_CSR_TLBRBADV 0x89 /* TLB refill badvaddr */ +#define KVM_CSR_TLBRERA 0x8a /* TLB refill ERA */ +#define KVM_CSR_TLBRSAVE 0x8b /* KScratch for TLB refill exception */ +#define KVM_CSR_TLBRELO0 0x8c /* TLB refill entrylo0 */ +#define KVM_CSR_TLBRELO1 0x8d /* TLB refill entrylo1 */ +#define KVM_CSR_TLBREHI 0x8e /* TLB refill entryhi */ +#define KVM_CSR_TLBRPRMD 0x8f /* TLB refill mode info */ +#define KVM_CSR_ERRCTL 0x90 /* ERRCTL */ +#define KVM_CSR_ERRINFO1 0x91 /* Error info1 */ +#define KVM_CSR_ERRINFO2 0x92 /* Error info2 */ +#define KVM_CSR_MERRENTRY 0x93 /* Error exception base address */ +#define KVM_CSR_MERRERA 0x94 /* Error exception PC */ +#define KVM_CSR_ERRSAVE 0x95 /* KScratch for machine error exception */ +#define KVM_CSR_CTAG 0x98 /* TagLo + TagHi */ +#define KVM_CSR_DMWIN0 0x180 /* 64 direct map win0: MEM & IF */ +#define KVM_CSR_DMWIN1 0x181 /* 64 direct map win1: MEM & IF */ +#define KVM_CSR_DMWIN2 0x182 /* 64 direct map win2: MEM */ +#define KVM_CSR_DMWIN3 0x183 /* 64 direct map win3: MEM */ +#define KVM_CSR_PERFCTRL0 0x200 /* 32 perf event 0 config */ +#define KVM_CSR_PERFCNTR0 0x201 /* 64 perf event 0 count value */ +#define KVM_CSR_PERFCTRL1 0x202 /* 32 perf event 1 config */ +#define KVM_CSR_PERFCNTR1 0x203 /* 64 perf event 1 count value */ +#define KVM_CSR_PERFCTRL2 0x204 /* 32 perf event 2 config */ +#define KVM_CSR_PERFCNTR2 0x205 /* 64 perf event 2 count value */ +#define KVM_CSR_PERFCTRL3 0x206 /* 32 perf event 3 config */ +#define KVM_CSR_PERFCNTR3 0x207 /* 64 perf event 3 count value */ +#define KVM_PERFCTRL_PLV0 (_ULCAST_(1) << 16) +#define KVM_PERFCTRL_PLV1 (_ULCAST_(1) << 17) +#define KVM_PERFCTRL_PLV2 (_ULCAST_(1) << 18) +#define KVM_PERFCTRL_PLV3 (_ULCAST_(1) << 19) +#define KVM_PERFCTRL_IE (_ULCAST_(1) << 20) +#define KVM_PERFCTRL_GMOD (_ULCAST_(3) << 21) +#define KVM_PERFCTRL_EVENT 0x3ff + +#define KVM_CSR_MWPC 0x300 /* data breakpoint config */ +#define KVM_CSR_MWPS 0x301 /* data breakpoint status */ +#define KVM_CSR_FWPC 0x380 /* instruction breakpoint config */ +#define KVM_CSR_FWPS 0x381 /* instruction breakpoint status */ +#define KVM_CSR_DEBUG 0x500 /* debug config */ +#define KVM_CSR_DERA 0x501 /* debug era */ +#define KVM_CSR_DESAVE 0x502 /* debug save */ + +#define KVM_IOCSR_FEATURES 0x8 +#define KVM_IOCSRF_TEMP BIT_ULL(0) +#define KVM_IOCSRF_NODECNT BIT_ULL(1) +#define KVM_IOCSRF_MSI BIT_ULL(2) +#define KVM_IOCSRF_EXTIOI BIT_ULL(3) +#define KVM_IOCSRF_CSRIPI BIT_ULL(4) +#define KVM_IOCSRF_FREQCSR BIT_ULL(5) +#define KVM_IOCSRF_FREQSCALE BIT_ULL(6) +#define KVM_IOCSRF_DVFSV1 BIT_ULL(7) +#define KVM_IOCSRF_EXTIOI_DECODE BIT_ULL(9) +#define KVM_IOCSRF_FLATMODE BIT_ULL(10) +#define KVM_IOCSRF_VM BIT_ULL(11) + +#define KVM_IOCSR_VENDOR 0x10 +#define KVM_IOCSR_CPUNAME 0x20 +#define KVM_IOCSR_NODECNT 0x408 + +#define KVM_IOCSR_MISC_FUNC 0x420 +#define KVM_IOCSRF_MISC_FUNC_EXT_IOI_EN BIT_ULL(48) + +/* PerCore CSR, only accessable by local cores */ +#define KVM_IOCSR_IPI_STATUS 0x1000 +#define KVM_IOCSR_IPI_SEND 0x1040 +#define KVM_IOCSR_MBUF_SEND 0x1048 +#define KVM_IOCSR_EXTIOI_NODEMAP_BASE 0x14a0 +#define KVM_IOCSR_EXTIOI_IPMAP_BASE 0x14c0 +#define KVM_IOCSR_EXTIOI_EN_BASE 0x1600 +#define KVM_IOCSR_EXTIOI_BOUNCE_BASE 0x1680 +#define KVM_IOCSR_EXTIOI_ISR_BASE 0x1800 +#define KVM_IOCSR_EXTIOI_ROUTE_BASE 0x1c00 + +#ifndef __ASSEMBLY__ + +/* CSR */ +static inline u32 kvm_csr_readl(u32 reg) +{ + u32 val; + + asm volatile ( + "csrrd %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "i" (reg) + : "memory"); + return val; +} + +static inline u64 kvm_csr_readq(u32 reg) +{ + u64 val; + + asm volatile ( + "csrrd %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "i" (reg) + : "memory"); + return val; +} + +static inline void kvm_csr_writel(u32 val, u32 reg) +{ + asm volatile ( + "csrwr %[val], %[reg] \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); +} + +static inline void kvm_csr_writeq(u64 val, u32 reg) +{ + asm volatile ( + "csrwr %[val], %[reg] \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); +} + +static inline u32 kvm_csr_xchgl(u32 val, u32 mask, u32 reg) +{ + asm volatile ( + "csrxchg %[val], %[mask], %[reg] \n" + : [val] "+r" (val) + : [mask] "r" (mask), [reg] "i" (reg) + : "memory"); + return val; +} + +static inline u64 kvm_csr_xchgq(u64 val, u64 mask, u32 reg) +{ + asm volatile ( + "csrxchg %[val], %[mask], %[reg] \n" + : [val] "+r" (val) + : [mask] "r" (mask), [reg] "i" (reg) + : "memory"); + return val; +} + + +/* IOCSR */ +static inline u32 kvm_iocsr_readl(u32 reg) +{ + u32 val; + + asm volatile ( + "iocsrrd.w %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "r" (reg) + : "memory"); + return val; +} + +static inline u64 kvm_iocsr_readq(u32 reg) +{ + u64 val; + + asm volatile ( + "iocsrrd.d %[val], %[reg] \n" + : [val] "=r" (val) + : [reg] "r" (reg) + : "memory"); + return val; +} + +static inline void kvm_iocsr_writeb(u8 val, u32 reg) +{ + asm volatile ( + "iocsrwr.b %[val], %[reg] \n" + : + : [val] "r" (val), [reg] "r" (reg) + : "memory"); +} + +static inline void kvm_iocsr_writel(u32 val, u32 reg) +{ + asm volatile ( + "iocsrwr.w %[val], %[reg] \n" + : + : [val] "r" (val), [reg] "r" (reg) + : "memory"); +} + +static inline void kvm_iocsr_writeq(u64 val, u32 reg) +{ + asm volatile ( + "iocsrwr.d %[val], %[reg] \n" + : + : [val] "r" (val), [reg] "r" (reg) + : "memory"); +} + + +/* GCSR */ +static inline u64 kvm_gcsr_read(u32 reg) +{ + u64 val = 0; + + asm volatile ( + "parse_r __reg, %[val] \n" + ".word 0x5 << 24 | %[reg] << 10 | 0 << 5 | __reg \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); + return val; +} + +static inline void kvm_gcsr_write(u64 val, u32 reg) +{ + asm volatile ( + "parse_r __reg, %[val] \n" + ".word 0x5 << 24 | %[reg] << 10 | 1 << 5 | __reg \n" + : [val] "+r" (val) + : [reg] "i" (reg) + : "memory"); +} + +static inline u64 kvm_gcsr_xchg(u64 val, u64 mask, u32 reg) +{ + asm volatile ( + "parse_r __rd, %[val] \n" + "parse_r __rj, %[mask] \n" + ".word 0x5 << 24 | %[reg] << 10 | __rj << 5 | __rd \n" + : [val] "+r" (val) + : [mask] "r" (mask), [reg] "i" (reg) + : "memory"); + return val; +} + +#endif /* !__ASSEMBLY__ */ + +#define kvm_read_csr_euen() kvm_csr_readq(KVM_CSR_EUEN) +#define kvm_write_csr_euen(val) kvm_csr_writeq(val, KVM_CSR_EUEN) +#define kvm_read_csr_ecfg() kvm_csr_readq(KVM_CSR_ECFG) +#define kvm_write_csr_ecfg(val) kvm_csr_writeq(val, KVM_CSR_ECFG) +#define kvm_write_csr_perfctrl0(val) kvm_csr_writeq(val, KVM_CSR_PERFCTRL0) +#define kvm_write_csr_perfcntr0(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR0) +#define kvm_write_csr_perfctrl1(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL1) +#define kvm_write_csr_perfcntr1(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR1) +#define kvm_write_csr_perfctrl2(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL2) +#define kvm_write_csr_perfcntr2(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR2) +#define kvm_write_csr_perfctrl3(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCTRL3) +#define kvm_write_csr_perfcntr3(val) kvm_csr_writeq(val, LOONGARCH_CSR_PERFCNTR3) +#define kvm_read_csr_impctl1() kvm_csr_readq(LOONGARCH_CSR_IMPCTL1) +#define kvm_write_csr_impctl1(val) kvm_csr_writeq(val, LOONGARCH_CSR_IMPCTL1) + + +/* Guest related CSRS */ +#define kvm_read_csr_gtlbc() kvm_csr_readq(KVM_CSR_GTLBC) +#define kvm_write_csr_gtlbc(val) kvm_csr_writeq(val, KVM_CSR_GTLBC) +#define kvm_read_csr_trgp() kvm_csr_readq(KVM_CSR_TRGP) +#define kvm_read_csr_gcfg() kvm_csr_readq(KVM_CSR_GCFG) +#define kvm_write_csr_gcfg(val) kvm_csr_writeq(val, KVM_CSR_GCFG) +#define kvm_read_csr_gstat() kvm_csr_readq(KVM_CSR_GSTAT) +#define kvm_write_csr_gstat(val) kvm_csr_writeq(val, KVM_CSR_GSTAT) +#define kvm_read_csr_gintc() kvm_csr_readq(KVM_CSR_GINTC) +#define kvm_write_csr_gintc(val) kvm_csr_writeq(val, KVM_CSR_GINTC) +#define kvm_read_csr_gcntc() kvm_csr_readq(KVM_CSR_GCNTC) +#define kvm_write_csr_gcntc(val) kvm_csr_writeq(val, KVM_CSR_GCNTC) + +/* Guest CSRS read and write */ +#define kvm_read_gcsr_crmd() kvm_gcsr_read(KVM_CSR_CRMD) +#define kvm_write_gcsr_crmd(val) kvm_gcsr_write(val, KVM_CSR_CRMD) +#define kvm_read_gcsr_prmd() kvm_gcsr_read(KVM_CSR_PRMD) +#define kvm_write_gcsr_prmd(val) kvm_gcsr_write(val, KVM_CSR_PRMD) +#define kvm_read_gcsr_euen() kvm_gcsr_read(KVM_CSR_EUEN) +#define kvm_write_gcsr_euen(val) kvm_gcsr_write(val, KVM_CSR_EUEN) +#define kvm_read_gcsr_misc() kvm_gcsr_read(KVM_CSR_MISC) +#define kvm_write_gcsr_misc(val) kvm_gcsr_write(val, KVM_CSR_MISC) +#define kvm_read_gcsr_ecfg() kvm_gcsr_read(KVM_CSR_ECFG) +#define kvm_write_gcsr_ecfg(val) kvm_gcsr_write(val, KVM_CSR_ECFG) +#define kvm_read_gcsr_estat() kvm_gcsr_read(KVM_CSR_ESTAT) +#define kvm_write_gcsr_estat(val) kvm_gcsr_write(val, KVM_CSR_ESTAT) +#define kvm_read_gcsr_era() kvm_gcsr_read(KVM_CSR_ERA) +#define kvm_write_gcsr_era(val) kvm_gcsr_write(val, KVM_CSR_ERA) +#define kvm_read_gcsr_badv() kvm_gcsr_read(KVM_CSR_BADV) +#define kvm_write_gcsr_badv(val) kvm_gcsr_write(val, KVM_CSR_BADV) +#define kvm_read_gcsr_badi() kvm_gcsr_read(KVM_CSR_BADI) +#define kvm_write_gcsr_badi(val) kvm_gcsr_write(val, KVM_CSR_BADI) +#define kvm_read_gcsr_eentry() kvm_gcsr_read(KVM_CSR_EENTRY) +#define kvm_write_gcsr_eentry(val) kvm_gcsr_write(val, KVM_CSR_EENTRY) + +#define kvm_read_gcsr_tlbidx() kvm_gcsr_read(KVM_CSR_TLBIDX) +#define kvm_write_gcsr_tlbidx(val) kvm_gcsr_write(val, KVM_CSR_TLBIDX) +#define kvm_read_gcsr_tlbhi() kvm_gcsr_read(KVM_CSR_TLBEHI) +#define kvm_write_gcsr_tlbhi(val) kvm_gcsr_write(val, KVM_CSR_TLBEHI) +#define kvm_read_gcsr_tlblo0() kvm_gcsr_read(KVM_CSR_TLBELO0) +#define kvm_write_gcsr_tlblo0(val) kvm_gcsr_write(val, KVM_CSR_TLBELO0) +#define kvm_read_gcsr_tlblo1() kvm_gcsr_read(KVM_CSR_TLBELO1) +#define kvm_write_gcsr_tlblo1(val) kvm_gcsr_write(val, KVM_CSR_TLBELO1) + +#define kvm_read_gcsr_asid() kvm_gcsr_read(KVM_CSR_ASID) +#define kvm_write_gcsr_asid(val) kvm_gcsr_write(val, KVM_CSR_ASID) +#define kvm_read_gcsr_pgdl() kvm_gcsr_read(KVM_CSR_PGDL) +#define kvm_write_gcsr_pgdl(val) kvm_gcsr_write(val, KVM_CSR_PGDL) +#define kvm_read_gcsr_pgdh() kvm_gcsr_read(KVM_CSR_PGDH) +#define kvm_write_gcsr_pgdh(val) kvm_gcsr_write(val, KVM_CSR_PGDH) +#define kvm_write_gcsr_pgd(val) kvm_gcsr_write(val, KVM_CSR_PGD) +#define kvm_read_gcsr_pgd() kvm_gcsr_read(KVM_CSR_PGD) +#define kvm_read_gcsr_pwctl0() kvm_gcsr_read(KVM_CSR_PWCTL0) +#define kvm_write_gcsr_pwctl0(val) kvm_gcsr_write(val, KVM_CSR_PWCTL0) +#define kvm_read_gcsr_pwctl1() kvm_gcsr_read(KVM_CSR_PWCTL1) +#define kvm_write_gcsr_pwctl1(val) kvm_gcsr_write(val, KVM_CSR_PWCTL1) +#define kvm_read_gcsr_stlbpgsize() kvm_gcsr_read(KVM_CSR_STLBPGSIZE) +#define kvm_write_gcsr_stlbpgsize(val) kvm_gcsr_write(val, KVM_CSR_STLBPGSIZE) +#define kvm_read_gcsr_rvacfg() kvm_gcsr_read(KVM_CSR_RVACFG) +#define kvm_write_gcsr_rvacfg(val) kvm_gcsr_write(val, KVM_CSR_RVACFG) + +#define kvm_read_gcsr_cpuid() kvm_gcsr_read(KVM_CSR_CPUID) +#define kvm_write_gcsr_cpuid(val) kvm_gcsr_write(val, KVM_CSR_CPUID) +#define kvm_read_gcsr_prcfg1() kvm_gcsr_read(KVM_CSR_PRCFG1) +#define kvm_write_gcsr_prcfg1(val) kvm_gcsr_write(val, KVM_CSR_PRCFG1) +#define kvm_read_gcsr_prcfg2() kvm_gcsr_read(KVM_CSR_PRCFG2) +#define kvm_write_gcsr_prcfg2(val) kvm_gcsr_write(val, KVM_CSR_PRCFG2) +#define kvm_read_gcsr_prcfg3() kvm_gcsr_read(KVM_CSR_PRCFG3) +#define kvm_write_gcsr_prcfg3(val) kvm_gcsr_write(val, KVM_CSR_PRCFG3) + +#define kvm_read_gcsr_kscratch0() kvm_gcsr_read(KVM_CSR_KS0) +#define kvm_write_gcsr_kscratch0(val) kvm_gcsr_write(val, KVM_CSR_KS0) +#define kvm_read_gcsr_kscratch1() kvm_gcsr_read(KVM_CSR_KS1) +#define kvm_write_gcsr_kscratch1(val) kvm_gcsr_write(val, KVM_CSR_KS1) +#define kvm_read_gcsr_kscratch2() kvm_gcsr_read(KVM_CSR_KS2) +#define kvm_write_gcsr_kscratch2(val) kvm_gcsr_write(val, KVM_CSR_KS2) +#define kvm_read_gcsr_kscratch3() kvm_gcsr_read(KVM_CSR_KS3) +#define kvm_write_gcsr_kscratch3(val) kvm_gcsr_write(val, KVM_CSR_KS3) +#define kvm_read_gcsr_kscratch4() kvm_gcsr_read(KVM_CSR_KS4) +#define kvm_write_gcsr_kscratch4(val) kvm_gcsr_write(val, KVM_CSR_KS4) +#define kvm_read_gcsr_kscratch5() kvm_gcsr_read(KVM_CSR_KS5) +#define kvm_write_gcsr_kscratch5(val) kvm_gcsr_write(val, KVM_CSR_KS5) +#define kvm_read_gcsr_kscratch6() kvm_gcsr_read(KVM_CSR_KS6) +#define kvm_write_gcsr_kscratch6(val) kvm_gcsr_write(val, KVM_CSR_KS6) +#define kvm_read_gcsr_kscratch7() kvm_gcsr_read(KVM_CSR_KS7) +#define kvm_write_gcsr_kscratch7(val) kvm_gcsr_write(val, KVM_CSR_KS7) + +#define kvm_read_gcsr_timerid() kvm_gcsr_read(KVM_CSR_TMID) +#define kvm_write_gcsr_timerid(val) kvm_gcsr_write(val, KVM_CSR_TMID) +#define kvm_read_gcsr_timercfg() kvm_gcsr_read(KVM_CSR_TCFG) +#define kvm_write_gcsr_timercfg(val) kvm_gcsr_write(val, KVM_CSR_TCFG) +#define kvm_read_gcsr_timertick() kvm_gcsr_read(KVM_CSR_TVAL) +#define kvm_write_gcsr_timertick(val) kvm_gcsr_write(val, KVM_CSR_TVAL) +#define kvm_read_gcsr_timeroffset() kvm_gcsr_read(KVM_CSR_CNTC) +#define kvm_write_gcsr_timeroffset(val) kvm_gcsr_write(val, KVM_CSR_CNTC) + +#define kvm_read_gcsr_llbctl() kvm_gcsr_read(KVM_CSR_LLBCTL) +#define kvm_write_gcsr_llbctl(val) kvm_gcsr_write(val, KVM_CSR_LLBCTL) + +#define kvm_read_gcsr_tlbrentry() kvm_gcsr_read(KVM_CSR_TLBRENTRY) +#define kvm_write_gcsr_tlbrentry(val) kvm_gcsr_write(val, KVM_CSR_TLBRENTRY) +#define kvm_read_gcsr_tlbrbadv() kvm_gcsr_read(KVM_CSR_TLBRBADV) +#define kvm_write_gcsr_tlbrbadv(val) kvm_gcsr_write(val, KVM_CSR_TLBRBADV) +#define kvm_read_gcsr_tlbrera() kvm_gcsr_read(KVM_CSR_TLBRERA) +#define kvm_write_gcsr_tlbrera(val) kvm_gcsr_write(val, KVM_CSR_TLBRERA) +#define kvm_read_gcsr_tlbrsave() kvm_gcsr_read(KVM_CSR_TLBRSAVE) +#define kvm_write_gcsr_tlbrsave(val) kvm_gcsr_write(val, KVM_CSR_TLBRSAVE) +#define kvm_read_gcsr_tlbrelo0() kvm_gcsr_read(KVM_CSR_TLBRELO0) +#define kvm_write_gcsr_tlbrelo0(val) kvm_gcsr_write(val, KVM_CSR_TLBRELO0) +#define kvm_read_gcsr_tlbrelo1() kvm_gcsr_read(KVM_CSR_TLBRELO1) +#define kvm_write_gcsr_tlbrelo1(val) kvm_gcsr_write(val, KVM_CSR_TLBRELO1) +#define kvm_read_gcsr_tlbrehi() kvm_gcsr_read(KVM_CSR_TLBREHI) +#define kvm_write_gcsr_tlbrehi(val) kvm_gcsr_write(val, KVM_CSR_TLBREHI) +#define kvm_read_gcsr_tlbrprmd() kvm_gcsr_read(KVM_CSR_TLBRPRMD) +#define kvm_write_gcsr_tlbrprmd(val) kvm_gcsr_write(val, KVM_CSR_TLBRPRMD) + +#define kvm_read_gcsr_directwin0() kvm_gcsr_read(KVM_CSR_DMWIN0) +#define kvm_write_gcsr_directwin0(val) kvm_gcsr_write(val, KVM_CSR_DMWIN0) +#define kvm_read_gcsr_directwin1() kvm_gcsr_read(KVM_CSR_DMWIN1) +#define kvm_write_gcsr_directwin1(val) kvm_gcsr_write(val, KVM_CSR_DMWIN1) +#define kvm_read_gcsr_directwin2() kvm_gcsr_read(KVM_CSR_DMWIN2) +#define kvm_write_gcsr_directwin2(val) kvm_gcsr_write(val, KVM_CSR_DMWIN2) +#define kvm_read_gcsr_directwin3() kvm_gcsr_read(KVM_CSR_DMWIN3) +#define kvm_write_gcsr_directwin3(val) kvm_gcsr_write(val, KVM_CSR_DMWIN3) + +#ifndef __ASSEMBLY__ + +static inline unsigned long +kvm_set_csr_gtlbc(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_csr_gtlbc(); + new = res | set; + kvm_write_csr_gtlbc(new); + + return res; +} + +static inline unsigned long +kvm_set_csr_euen(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_csr_euen(); + new = res | set; + kvm_write_csr_euen(new); + + return res; +} + +static inline unsigned long +kvm_set_csr_gintc(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_csr_gintc(); + new = res | set; + kvm_write_csr_gintc(new); + + return res; +} + +static inline unsigned long +kvm_set_gcsr_llbctl(unsigned long set) +{ + unsigned long res, new; + + res = kvm_read_gcsr_llbctl(); + new = res | set; + kvm_write_gcsr_llbctl(new); + + return res; +} + + +static inline unsigned long +kvm_clear_csr_gtlbc(unsigned long clear) +{ + unsigned long res, new; + + res = kvm_read_csr_gtlbc(); + new = res & ~clear; + kvm_write_csr_gtlbc(new); + + return res; +} + +static inline unsigned long +kvm_clear_csr_euen(unsigned long clear) +{ + unsigned long res, new; + + res = kvm_read_csr_euen(); + new = res & ~clear; + kvm_write_csr_euen(new); + + return res; +} + +static inline unsigned long +kvm_clear_csr_gintc(unsigned long clear) +{ + unsigned long res, new; + + res = kvm_read_csr_gintc(); + new = res & ~clear; + kvm_write_csr_gintc(new); + + return res; +} + +static inline unsigned long +kvm_change_csr_gstat(unsigned long change, unsigned long val) +{ + unsigned long res, new; + + res = kvm_read_csr_gstat(); + new = res & ~change; + new |= (val & change); + kvm_write_csr_gstat(new); + + return res; +} + +static inline unsigned long +kvm_change_csr_gcfg(unsigned long change, unsigned long val) +{ + unsigned long res, new; + + res = kvm_read_csr_gcfg(); + new = res & ~change; + new |= (val & change); + kvm_write_csr_gcfg(new); + + return res; +} + + +#define kvm_set_gcsr_estat(val) \ + kvm_gcsr_xchg(val, val, KVM_CSR_ESTAT) +#define kvm_clear_gcsr_estat(val) \ + kvm_gcsr_xchg(~(val), val, KVM_CSR_ESTAT) + +#endif + +/* Device Control API on vcpu fd */ +#define KVM_LARCH_VCPU_PVTIME_CTRL 2 +#define KVM_LARCH_VCPU_PVTIME_IPA 0 + +#if (_LOONGARCH_SZLONG == 32) +#define KVM_LONG_ADD add.w +#define KVM_LONG_ADDI addi.w +#define KVM_LONG_SUB sub.w +#define KVM_LONG_L ld.w +#define KVM_LONG_S st.w +#define KVM_LONG_SLL slli.w +#define KVM_LONG_SLLV sll.w +#define KVM_LONG_SRL srli.w +#define KVM_LONG_SRLV srl.w +#define KVM_LONG_SRA srai.w +#define KVM_LONG_SRAV sra.w + +#define KVM_LONGSIZE 4 +#define KVM_LONGMASK 3 +#define KVM_LONGLOG 2 + +/* + * How to add/sub/load/store/shift pointers. + */ + +#define KVM_PTR_ADD add.w +#define KVM_PTR_ADDI addi.w +#define KVM_PTR_SUB sub.w +#define KVM_PTR_L ld.w +#define KVM_PTR_S st.w +#define KVM_PTR_LI li.w +#define KVM_PTR_SLL slli.w +#define KVM_PTR_SLLV sll.w +#define KVM_PTR_SRL srli.w +#define KVM_PTR_SRLV srl.w +#define KVM_PTR_SRA srai.w +#define KVM_PTR_SRAV sra.w + +#define KVM_PTR_SCALESHIFT 2 + +#define KVM_PTRSIZE 4 +#define KVM_PTRLOG 2 + +#endif + +#if (_LOONGARCH_SZLONG == 64) +#define KVM_LONG_ADD add.d +#define KVM_LONG_ADDI addi.d +#define KVM_LONG_SUB sub.d +#define KVM_LONG_L ld.d +#define KVM_LONG_S st.d +#define KVM_LONG_SLL slli.d +#define KVM_LONG_SLLV sll.d +#define KVM_LONG_SRL srli.d +#define KVM_LONG_SRLV srl.d +#define KVM_LONG_SRA sra.w +#define KVM_LONG_SRAV sra.d + +#define KVM_LONGSIZE 8 +#define KVM_LONGMASK 7 +#define KVM_LONGLOG 3 + +/* + * How to add/sub/load/store/shift pointers. + */ + +#define KVM_PTR_ADD add.d +#define KVM_PTR_ADDI addi.d +#define KVM_PTR_SUB sub.d +#define KVM_PTR_L ld.d +#define KVM_PTR_S st.d +#define KVM_PTR_LI li.d +#define KVM_PTR_SLL slli.d +#define KVM_PTR_SLLV sll.d +#define KVM_PTR_SRL srli.d +#define KVM_PTR_SRLV srl.d +#define KVM_PTR_SRA srai.d +#define KVM_PTR_SRAV sra.d + +#define KVM_PTR_SCALESHIFT 3 + +#define KVM_PTRSIZE 8 +#define KVM_PTRLOG 3 +#endif + +#endif /* __LOONGARCH_KVM_COMPAT_H__ */ diff --git a/arch/loongarch/kvm/kvmcpu.h b/arch/loongarch/kvm/kvmcpu.h new file mode 100644 index 0000000000000000000000000000000000000000..0962becea1b5da47ec3aaae616401e1f29820567 --- /dev/null +++ b/arch/loongarch/kvm/kvmcpu.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __ASM_LOONGARCH_KVMCPU_H__ +#define __ASM_LOONGARCH_KVMCPU_H__ + +#include +#include + +#define LARCH_INT_SIP0 0 +#define LARCH_INT_SIP1 1 +#define LARCH_INT_IP0 2 +#define LARCH_INT_IP1 3 +#define LARCH_INT_IP2 4 +#define LARCH_INT_IP3 5 +#define LARCH_INT_IP4 6 +#define LARCH_INT_IP5 7 +#define LARCH_INT_IP6 8 +#define LARCH_INT_IP7 9 +#define LARCH_INT_PMU 10 +#define LARCH_INT_TIMER 11 +#define LARCH_INT_IPI 12 +#define LOONGARCH_EXC_MAX (LARCH_INT_IPI + 1) +#define LOONGARCH_EXC_IPNUM (LOONGARCH_EXC_MAX) + +/* Controlled by 0x5 guest exst */ +#define CPU_SIP0 (_ULCAST_(1)) +#define CPU_SIP1 (_ULCAST_(1) << 1) +#define CPU_PMU (_ULCAST_(1) << 10) +#define CPU_TIMER (_ULCAST_(1) << 11) +#define CPU_IPI (_ULCAST_(1) << 12) + +/* Controlled by 0x52 guest exception VIP + * aligned to exst bit 5~12 + */ +#define CPU_IP0 (_ULCAST_(1)) +#define CPU_IP1 (_ULCAST_(1) << 1) +#define CPU_IP2 (_ULCAST_(1) << 2) +#define CPU_IP3 (_ULCAST_(1) << 3) +#define CPU_IP4 (_ULCAST_(1) << 4) +#define CPU_IP5 (_ULCAST_(1) << 5) +#define CPU_IP6 (_ULCAST_(1) << 6) +#define CPU_IP7 (_ULCAST_(1) << 7) + +#define MNSEC_PER_SEC (NSEC_PER_SEC >> 20) + +/* KVM_IRQ_LINE irq field index values */ +#define KVM_LOONGSON_IRQ_TYPE_SHIFT 24 +#define KVM_LOONGSON_IRQ_TYPE_MASK 0xff +#define KVM_LOONGSON_IRQ_VCPU_SHIFT 16 +#define KVM_LOONGSON_IRQ_VCPU_MASK 0xff +#define KVM_LOONGSON_IRQ_NUM_SHIFT 0 +#define KVM_LOONGSON_IRQ_NUM_MASK 0xffff + +/* irq_type field */ +#define KVM_LOONGSON_IRQ_TYPE_CPU_IP 0 +#define KVM_LOONGSON_IRQ_TYPE_CPU_IO 1 +#define KVM_LOONGSON_IRQ_TYPE_HT 2 +#define KVM_LOONGSON_IRQ_TYPE_MSI 3 +#define KVM_LOONGSON_IRQ_TYPE_IOAPIC 4 +#define KVM_LOONGSON_IRQ_TYPE_ROUTE 5 + +/* out-of-kernel GIC cpu interrupt injection irq_number field */ +#define KVM_LOONGSON_IRQ_CPU_IRQ 0 +#define KVM_LOONGSON_IRQ_CPU_FIQ 1 +#define KVM_LOONGSON_CPU_IP_NUM 8 + +typedef union loongarch_instruction larch_inst; +typedef int (*exit_handle_fn)(struct kvm_vcpu *); + +int _kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst); +int _kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst); +int _kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run); +int _kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run); +int _kvm_emu_idle(struct kvm_vcpu *vcpu); +int _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu); +int _kvm_pending_timer(struct kvm_vcpu *vcpu); +int _kvm_handle_fault(struct kvm_vcpu *vcpu, int fault); +void _kvm_deliver_intr(struct kvm_vcpu *vcpu); +void irqchip_debug_init(struct kvm *kvm); +void irqchip_debug_destroy(struct kvm *kvm); + +void kvm_own_fpu(struct kvm_vcpu *vcpu); +void kvm_own_lsx(struct kvm_vcpu *vcpu); +void kvm_lose_fpu(struct kvm_vcpu *vcpu); +void kvm_own_lasx(struct kvm_vcpu *vcpu); +void kvm_save_fpu(struct kvm_vcpu *cpu); +void kvm_restore_fpu(struct kvm_vcpu *cpu); +void kvm_restore_fcsr(struct kvm_vcpu *cpu); +void kvm_save_lsx(struct kvm_vcpu *cpu); +void kvm_restore_lsx(struct kvm_vcpu *cpu); +void kvm_restore_lsx_upper(struct kvm_vcpu *cpu); +void kvm_save_lasx(struct kvm_vcpu *cpu); +void kvm_restore_lasx(struct kvm_vcpu *cpu); +void kvm_restore_lasx_upper(struct kvm_vcpu *cpu); + +void kvm_lose_hw_perf(struct kvm_vcpu *vcpu); +void kvm_restore_hw_perf(struct kvm_vcpu *vcpu); + +void kvm_acquire_timer(struct kvm_vcpu *vcpu); +void kvm_reset_timer(struct kvm_vcpu *vcpu); +enum hrtimer_restart kvm_count_timeout(struct kvm_vcpu *vcpu); +void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz); +void kvm_restore_timer(struct kvm_vcpu *vcpu); +void kvm_save_timer(struct kvm_vcpu *vcpu); + +/* + * Loongarch KVM guest interrupt handling. + */ +static inline void _kvm_queue_irq(struct kvm_vcpu *vcpu, unsigned int irq) +{ + set_bit(irq, &vcpu->arch.irq_pending); + clear_bit(irq, &vcpu->arch.irq_clear); +} + +static inline void _kvm_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int irq) +{ + clear_bit(irq, &vcpu->arch.irq_pending); + set_bit(irq, &vcpu->arch.irq_clear); +} + +#endif /* __ASM_LOONGARCH_KVMCPU_H__ */ diff --git a/arch/loongarch/kvm/kvmcsr.h b/arch/loongarch/kvm/kvmcsr.h new file mode 100644 index 0000000000000000000000000000000000000000..24a84a3f72cd19a4e6df37cff1861496039c9ae8 --- /dev/null +++ b/arch/loongarch/kvm/kvmcsr.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#ifndef __LOONGARCH_KVM_CSR_H__ +#define __LOONGARCH_KVM_CSR_H__ +#include +#include "kvmcpu.h" +#include +#include + +#define kvm_read_hw_gcsr(id) kvm_gcsr_read(id) +#define kvm_write_hw_gcsr(csr, id, val) kvm_gcsr_write(val, id) + +int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force); +int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *v, int force); +unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid); +void _kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val); +void _kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, + unsigned long csr_mask, unsigned long val); +int _kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu); + +static inline void kvm_save_hw_gcsr(struct loongarch_csrs *csr, u32 gid) +{ + csr->csrs[gid] = kvm_gcsr_read(gid); +} + +static inline void kvm_restore_hw_gcsr(struct loongarch_csrs *csr, u32 gid) +{ + kvm_gcsr_write(csr->csrs[gid], gid); +} + +static inline unsigned long kvm_read_sw_gcsr(struct loongarch_csrs *csr, u32 gid) +{ + return csr->csrs[gid]; +} + +static inline void kvm_write_sw_gcsr(struct loongarch_csrs *csr, u32 gid, unsigned long val) +{ + csr->csrs[gid] = val; +} + +static inline void kvm_set_sw_gcsr(struct loongarch_csrs *csr, u32 gid, unsigned long val) +{ + csr->csrs[gid] |= val; +} + +static inline void kvm_change_sw_gcsr(struct loongarch_csrs *csr, u32 gid, unsigned mask, + unsigned long val) +{ + unsigned long _mask = mask; + csr->csrs[gid] &= ~_mask; + csr->csrs[gid] |= val & _mask; +} + + +#define GET_HW_GCSR(id, csrid, v) \ + do { \ + if (csrid == id) { \ + *v = (long)kvm_read_hw_gcsr(csrid); \ + return 0; \ + } \ + } while (0) + +#define GET_SW_GCSR(csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + *v = kvm_read_sw_gcsr(csr, id); \ + return 0; \ + } \ + } while (0) + +#define SET_HW_GCSR(csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + kvm_write_hw_gcsr(csr, csrid, *v); \ + return 0; \ + } \ + } while (0) + +#define SET_SW_GCSR(csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + kvm_write_sw_gcsr(csr, csrid, *v); \ + return 0; \ + } \ + } while (0) + +int _kvm_init_iocsr(struct kvm *kvm); +int _kvm_set_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp); +int _kvm_get_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp); + +#define KVM_PMU_PLV_ENABLE (KVM_PERFCTRL_PLV0 | \ + KVM_PERFCTRL_PLV1 | \ + KVM_PERFCTRL_PLV2 | \ + KVM_PERFCTRL_PLV3) + +#define CASE_WRITE_HW_PMU(vcpu, csr, id, csrid, v) \ + do { \ + if (csrid == id) { \ + if (v & KVM_PMU_PLV_ENABLE) { \ + kvm_write_csr_gcfg(kvm_read_csr_gcfg() | KVM_GCFG_GPERF); \ + kvm_write_hw_gcsr(csr, csrid, v | KVM_PERFCTRL_GMOD); \ + vcpu->arch.aux_inuse |= KVM_LARCH_PERF; \ + return ; \ + } else { \ + kvm_write_sw_gcsr(csr, csrid, v); \ + return; \ + } \ + } \ + } while (0) + +#endif /* __LOONGARCH_KVM_CSR_H__ */ diff --git a/arch/loongarch/kvm/loongarch.c b/arch/loongarch/kvm/loongarch.c new file mode 100644 index 0000000000000000000000000000000000000000..03da5959aaebebfd9023ffd4a7ab3c7dfa58f30c --- /dev/null +++ b/arch/loongarch/kvm/loongarch.c @@ -0,0 +1,2085 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include +#include + +#include "intc/ls3a_ipi.h" +#include "intc/ls7a_irq.h" +#include "intc/ls3a_ext_irq.h" +#include "kvm_compat.h" +#include "kvmcsr.h" + +/* + * Define loongarch kvm version. + * Add version number when qemu/kvm interface changed + */ +#define KVM_LOONGARCH_VERSION 1 +#define CREATE_TRACE_POINTS +#include "trace.h" +struct kvm_stats_debugfs_item vcpu_debugfs_entries[] = { + VCPU_STAT("idle", idle_exits), + VCPU_STAT("signal", signal_exits), + VCPU_STAT("interrupt", int_exits), + VCPU_STAT("rdcsr_cpu_feature", rdcsr_cpu_feature_exits), + VCPU_STAT("rdcsr_misc_func", rdcsr_misc_func_exits), + VCPU_STAT("rdcsr_ipi_access", rdcsr_ipi_access_exits), + VCPU_STAT("cpucfg", cpucfg_exits), + VCPU_STAT("huge_dec", huge_dec_exits), + VCPU_STAT("huge_thp", huge_thp_exits), + VCPU_STAT("huge_adj", huge_adjust_exits), + VCPU_STAT("huge_set", huge_set_exits), + VCPU_STAT("huge_merg", huge_merge_exits), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("tlbmiss_ld", excep_exits[KVM_EXCCODE_TLBL]), + VCPU_STAT("tlbmiss_st", excep_exits[KVM_EXCCODE_TLBS]), + VCPU_STAT("tlb_ifetch", excep_exits[KVM_EXCCODE_TLBI]), + VCPU_STAT("tlbmod", excep_exits[KVM_EXCCODE_TLBM]), + VCPU_STAT("tlbri", excep_exits[KVM_EXCCODE_TLBRI]), + VCPU_STAT("tlbxi", excep_exits[KVM_EXCCODE_TLBXI]), + VCPU_STAT("fp_dis", excep_exits[KVM_EXCCODE_FPDIS]), + VCPU_STAT("lsx_dis", excep_exits[KVM_EXCCODE_LSXDIS]), + VCPU_STAT("lasx_dis", excep_exits[KVM_EXCCODE_LASXDIS]), + VCPU_STAT("fpe", excep_exits[KVM_EXCCODE_FPE]), + VCPU_STAT("watch", excep_exits[KVM_EXCCODE_WATCH]), + VCPU_STAT("gspr", excep_exits[KVM_EXCCODE_GSPR]), + VCPU_STAT("gcm", excep_exits[KVM_EXCCODE_GCM]), + VCPU_STAT("hc", excep_exits[KVM_EXCCODE_HYP]), + {NULL} +}; + +struct kvm_stats_debugfs_item debugfs_entries[] = { + VM_STAT("remote_tlb_flush", remote_tlb_flush), + VM_STAT("pip_read_exits", pip_read_exits), + VM_STAT("pip_write_exits", pip_write_exits), + VM_STAT("vm_ioctl_irq_line", vm_ioctl_irq_line), + VM_STAT("ls7a_ioapic_update", ls7a_ioapic_update), + VM_STAT("ls7a_ioapic_set_irq", ls7a_ioapic_set_irq), + VM_STAT("ls7a_msi_irq", ls7a_msi_irq), + VM_STAT("ioapic_reg_write", ioapic_reg_write), + VM_STAT("ioapic_reg_read", ioapic_reg_read), + VM_STAT("set_ls7a_ioapic", set_ls7a_ioapic), + VM_STAT("get_ls7a_ioapic", get_ls7a_ioapic), + VM_STAT("set_ls3a_ext_irq", set_ls3a_ext_irq), + VM_STAT("get_ls3a_ext_irq", get_ls3a_ext_irq), + VM_STAT("ls3a_ext_irq", trigger_ls3a_ext_irq), + {NULL} +}; + +bool kvm_trace_guest_mode_change; +static struct kvm_context __percpu *vmcs; + +int kvm_guest_mode_change_trace_reg(void) +{ + kvm_trace_guest_mode_change = 1; + return 0; +} + +void kvm_guest_mode_change_trace_unreg(void) +{ + kvm_trace_guest_mode_change = 0; +} + +/* + * XXXKYMA: We are simulatoring a processor that has the WII bit set in + * Config7, so we are "runnable" if interrupts are pending + */ +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return !!(vcpu->arch.irq_pending); +} + +bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) +{ + return false; +} + +int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; +} + +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING +void kvm_update_stolen_time(struct kvm_vcpu *vcpu) +{ + struct kvm_host_map map; + struct kvm_steal_time *st; + int ret = 0; + + if (vcpu->arch.st.guest_addr == 0) + return; + + ret = kvm_map_gfn(vcpu, vcpu->arch.st.guest_addr >> PAGE_SHIFT, + &map, &vcpu->arch.st.cache, false); + if (ret) { + kvm_info("%s ret:%d\n", __func__, ret); + return; + } + st = map.hva + offset_in_page(vcpu->arch.st.guest_addr); + if (st->version & 1) + st->version += 1; /* first time write, random junk */ + st->version += 1; + smp_wmb(); + st->steal += current->sched_info.run_delay - + vcpu->arch.st.last_steal; + vcpu->arch.st.last_steal = current->sched_info.run_delay; + smp_wmb(); + st->version += 1; + + kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false); +} + +bool _kvm_pvtime_supported(void) +{ + return !!sched_info_on(); +} + +int _kvm_pvtime_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + u64 __user *user = (u64 __user *)attr->addr; + struct kvm *kvm = vcpu->kvm; + u64 ipa; + int ret = 0; + int idx; + + if (!_kvm_pvtime_supported() || + attr->attr != KVM_LARCH_VCPU_PVTIME_IPA) + return -ENXIO; + + if (get_user(ipa, user)) + return -EFAULT; + if (!IS_ALIGNED(ipa, 64)) + return -EINVAL; + + /* Check the address is in a valid memslot */ + idx = srcu_read_lock(&kvm->srcu); + if (kvm_is_error_hva(gfn_to_hva(kvm, ipa >> PAGE_SHIFT))) + ret = -EINVAL; + srcu_read_unlock(&kvm->srcu, idx); + + if (!ret) + vcpu->arch.st.guest_addr = ipa; + + return ret; +} + +int _kvm_pvtime_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + u64 __user *user = (u64 __user *)attr->addr; + u64 ipa; + + if (!_kvm_pvtime_supported() || + attr->attr != KVM_LARCH_VCPU_PVTIME_IPA) + return -ENXIO; + + ipa = vcpu->arch.st.guest_addr; + + if (put_user(ipa, user)) + return -EFAULT; + + return 0; +} + +int _kvm_pvtime_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + switch (attr->attr) { + case KVM_LARCH_VCPU_PVTIME_IPA: + if (_kvm_pvtime_supported()) + return 0; + } + + return -ENXIO; +} +#endif + +int kvm_arch_hardware_enable(void) +{ + unsigned long gcfg = 0; + + /* First init gtlbc, gcfg, gstat, gintc. All guest use the same config */ + kvm_clear_csr_gtlbc(KVM_GTLBC_USETGID | KVM_GTLBC_TOTI); + kvm_write_csr_gcfg(0); + kvm_write_csr_gstat(0); + kvm_write_csr_gintc(0); + + /* + * Enable virtualization features granting guest direct control of + * certain features: + * GCI=2: Trap on init or unimplement cache instruction. + * TORU=0: Trap on Root Unimplement. + * CACTRL=1: Root control cache. + * TOP=0: Trap on Previlege. + * TOE=0: Trap on Exception. + * TIT=0: Trap on Timer. + */ + gcfg |= KVM_GCFG_GCI_SECURE; + gcfg |= KVM_GCFG_MATC_ROOT; + gcfg |= KVM_GCFG_TIT; + kvm_write_csr_gcfg(gcfg); + kvm_flush_tlb_all(); + + /* Enable using TGID */ + kvm_set_csr_gtlbc(KVM_GTLBC_USETGID); + kvm_debug("gtlbc:%llx gintc:%llx gstat:%llx gcfg:%llx", + kvm_read_csr_gtlbc(), kvm_read_csr_gintc(), + kvm_read_csr_gstat(), kvm_read_csr_gcfg()); + return 0; +} + +void kvm_arch_hardware_disable(void) +{ + kvm_clear_csr_gtlbc(KVM_GTLBC_USETGID | KVM_GTLBC_TOTI); + kvm_write_csr_gcfg(0); + kvm_write_csr_gstat(0); + kvm_write_csr_gintc(0); + + /* Flush any remaining guest TLB entries */ + kvm_flush_tlb_all(); +} + +int kvm_arch_hardware_setup(void *opaque) +{ + return 0; +} + +int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) +{ + /* Allocate page table to map GPA -> RPA */ + kvm->arch.gpa_mm.pgd = kvm_pgd_alloc(); + if (!kvm->arch.gpa_mm.pgd) + return -ENOMEM; + + kvm->arch.cpucfg_lasx = (read_cpucfg(LOONGARCH_CPUCFG2) & + CPUCFG2_LASX); + + _kvm_init_iocsr(kvm); + kvm->arch.vmcs = vmcs; + + return 0; +} + +static void kvm_free_vcpus(struct kvm *kvm) +{ + unsigned int i; + struct kvm_vcpu *vcpu; + + kvm_for_each_vcpu(i, vcpu, kvm) { + kvm_vcpu_destroy(vcpu); + kvm->vcpus[i] = NULL; + } + + atomic_set(&kvm->online_vcpus, 0); +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kvm_destroy_ls3a_ipi(kvm); + kvm_destroy_ls7a_ioapic(kvm); + kvm_destroy_ls3a_ext_irq(kvm); + kvm_free_vcpus(kvm); + _kvm_destroy_mm(kvm); +} + +long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned long npages) +{ + return 0; +} + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) +{ + return 0; +} + +static void _kvm_new_vpid(unsigned long cpu, struct kvm_vcpu *vcpu) +{ + struct kvm_context *context; + unsigned long vpid; + + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + vpid = context->vpid_cache; + if (!(++vpid & context->gid_mask)) { + if (!vpid) /* fix version if needed */ + vpid = context->gid_fisrt_ver; + + ++vpid; /* vpid 0 reserved for root */ + + /* start new vpid cycle */ + kvm_flush_tlb_all(); + } + + context->vpid_cache = vpid; + vcpu->arch.vpid[cpu] = vpid; +} + +/* Returns 1 if the guest TLB may be clobbered */ +static int _kvm_check_requests(struct kvm_vcpu *vcpu, int cpu) +{ + int ret = 0; + int i; + + if (!kvm_request_pending(vcpu)) + return 0; + + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { + /* Drop all vpids for this VCPU */ + for_each_possible_cpu(i) + vcpu->arch.vpid[i] = 0; + /* This will clobber guest TLB contents too */ + ret = 1; + } + + return ret; +} + +static void _kvm_update_vmid(struct kvm_vcpu *vcpu, int cpu) +{ + struct kvm_context *context; + bool migrated; + unsigned int gstinfo_gidmask, gstinfo_gid = 0; + + /* + * Are we entering guest context on a different CPU to last time? + * If so, the VCPU's guest TLB state on this CPU may be stale. + */ + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + migrated = (vcpu->arch.last_exec_cpu != cpu); + vcpu->arch.last_exec_cpu = cpu; + + /* + * Check if our vpid is of an older version and thus invalid. + * + * We also discard the stored vpid if we've executed on + * another CPU, as the guest mappings may have changed without + * hypervisor knowledge. + */ + gstinfo_gidmask = context->gid_mask << KVM_GSTAT_GID_SHIFT; + if (migrated || + (vcpu->arch.vpid[cpu] ^ context->vpid_cache) & + context->gid_ver_mask) { + _kvm_new_vpid(cpu, vcpu); + trace_kvm_vpid_change(vcpu, vcpu->arch.vpid[cpu]); + } + gstinfo_gid = (vcpu->arch.vpid[cpu] & context->gid_mask) << + KVM_GSTAT_GID_SHIFT; + + /* Restore GSTAT(0x50).vpid */ + kvm_change_csr_gstat(gstinfo_gidmask, gstinfo_gid); +} + +/* + * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV) + */ +static int _kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) +{ + unsigned long exst = vcpu->arch.host_estat; + u32 intr = exst & 0x1fff; /* ignore NMI */ + u32 exccode = (exst & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT; + u32 __user *opc = (u32 __user *) vcpu->arch.pc; + int ret = RESUME_GUEST, cpu; + + vcpu->mode = OUTSIDE_GUEST_MODE; + + /* Set a default exit reason */ + run->exit_reason = KVM_EXIT_UNKNOWN; + run->ready_for_interrupt_injection = 1; + + /* + * Set the appropriate status bits based on host CPU features, + * before we hit the scheduler + */ + + local_irq_enable(); + + kvm_debug("%s: exst: %lx, PC: %p, kvm_run: %p, kvm_vcpu: %p\n", + __func__, exst, opc, run, vcpu); + trace_kvm_exit(vcpu, exccode); + if (exccode) { + vcpu->stat.excep_exits[exccode]++; + ret = _kvm_handle_fault(vcpu, exccode); + } else { + WARN(!intr, "suspicious vm exiting"); + ++vcpu->stat.int_exits; + + if (need_resched()) + cond_resched(); + + ret = RESUME_GUEST; + } + +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) + kvm_update_stolen_time(vcpu); +#endif + + cond_resched(); + + local_irq_disable(); + + if (ret == RESUME_GUEST) + kvm_acquire_timer(vcpu); + + if (!(ret & RESUME_HOST)) { + _kvm_deliver_intr(vcpu); + /* Only check for signals if not already exiting to userspace */ + if (signal_pending(current)) { + run->exit_reason = KVM_EXIT_INTR; + ret = (-EINTR << 2) | RESUME_HOST; + ++vcpu->stat.signal_exits; + trace_kvm_exit(vcpu, KVM_TRACE_EXIT_SIGNAL); + } + } + + if (ret == RESUME_GUEST) { + trace_kvm_reenter(vcpu); + + /* + * Make sure the read of VCPU requests in vcpu_reenter() + * callback is not reordered ahead of the write to vcpu->mode, + * or we could miss a TLB flush request while the requester sees + * the VCPU as outside of guest mode and not needing an IPI. + */ + smp_store_mb(vcpu->mode, IN_GUEST_MODE); + + cpu = smp_processor_id(); + _kvm_check_requests(vcpu, cpu); + _kvm_update_vmid(vcpu, cpu); + + /* + * If FPU / LSX are enabled (i.e. the guest's FPU / LSX context + * is live), restore FCSR0. + */ + if (_kvm_guest_has_fpu(&vcpu->arch) && + kvm_read_csr_euen() & (KVM_EUEN_FPEN | KVM_EUEN_LSXEN)) { + kvm_restore_fcsr(vcpu); + } + } + + return ret; +} + +/* low level hrtimer wake routine */ +static enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer) +{ + struct kvm_vcpu *vcpu; + + vcpu = container_of(timer, struct kvm_vcpu, arch.swtimer); + _kvm_queue_irq(vcpu, LARCH_INT_TIMER); + kvm_vcpu_wake_up(vcpu); + return kvm_count_timeout(vcpu); +} + +static void _kvm_vcpu_init(struct kvm_vcpu *vcpu) +{ + int i; + + for_each_possible_cpu(i) + vcpu->arch.vpid[i] = 0; + + hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC, + HRTIMER_MODE_ABS_PINNED); + vcpu->arch.swtimer.function = kvm_swtimer_wakeup; + vcpu->arch.fpu_enabled = true; + vcpu->arch.lsx_enabled = true; +} + +int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) +{ + vcpu->arch.host_eentry = kvm_csr_readq(KVM_CSR_EENTRY); + vcpu->arch.guest_eentry = (unsigned long)kvm_exception_entry; + vcpu->arch.vcpu_run = kvm_enter_guest; + vcpu->arch.handle_exit = _kvm_handle_exit; + vcpu->arch.host_ecfg = (kvm_read_csr_ecfg() & KVM_ECFG_VS); + + /* + * kvm all exceptions share one exception entry, and host <-> guest switch + * also switch excfg.VS field, keep host excfg.VS info here + */ + vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL); + if (!vcpu->arch.csr) { + return -ENOMEM; + } + + /* Init */ + vcpu->arch.last_sched_cpu = -1; + vcpu->arch.last_exec_cpu = -1; + _kvm_vcpu_init(vcpu); + return 0; +} + +static void _kvm_vcpu_uninit(struct kvm_vcpu *vcpu) +{ + int cpu; + struct kvm_context *context; + + /* + * If the VCPU is freed and reused as another VCPU, we don't want the + * matching pointer wrongly hanging around in last_vcpu. + */ + for_each_possible_cpu(cpu) { + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + if (context->last_vcpu == vcpu) + context->last_vcpu = NULL; + } +} + +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache; + + _kvm_vcpu_uninit(vcpu); + + hrtimer_cancel(&vcpu->arch.swtimer); + kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); + if (vcpu->arch.st.guest_addr) + kvm_release_pfn(cache->pfn, cache->dirty, cache); + kfree(vcpu->arch.csr); +} +#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ + KVM_GUESTDBG_USE_SW_BP | KVM_GUESTDBG_SINGLESTEP) +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + int ret = 0; + + if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) { + ret = -EINVAL; + goto out; + } + if (dbg->control & KVM_GUESTDBG_ENABLE) { + vcpu->guest_debug = dbg->control; + /* No hardware breakpoint */ + } else { + vcpu->guest_debug = 0; + } +out: + return ret; +} + +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) +{ + int r = -EINTR; + int cpu; + struct kvm_run *run = vcpu->run; + + vcpu_load(vcpu); + + kvm_sigset_activate(vcpu); + + if (vcpu->mmio_needed) { + if (!vcpu->mmio_is_write) + _kvm_complete_mmio_read(vcpu, run); + vcpu->mmio_needed = 0; + } else if (vcpu->arch.is_hypcall) { + /* set return value for hypercall v0 register */ + vcpu->arch.gprs[KVM_REG_A0] = run->hypercall.ret; + vcpu->arch.is_hypcall = 0; + } + + if (run->exit_reason == KVM_EXIT_LOONGARCH_IOCSR) { + if (!run->iocsr_io.is_write) + _kvm_complete_iocsr_read(vcpu, run); + } + + /* clear exit_reason */ + run->exit_reason = KVM_EXIT_UNKNOWN; + if (run->immediate_exit) + goto out; + + lose_fpu(1); + +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) + kvm_update_stolen_time(vcpu); +#endif + local_irq_disable(); + guest_enter_irqoff(); + trace_kvm_enter(vcpu); + + /* + * Make sure the read of VCPU requests in vcpu_run() callback is not + * reordered ahead of the write to vcpu->mode, or we could miss a TLB + * flush request while the requester sees the VCPU as outside of guest + * mode and not needing an IPI. + */ + smp_store_mb(vcpu->mode, IN_GUEST_MODE); + + cpu = smp_processor_id(); + kvm_acquire_timer(vcpu); + /* Check if we have any exceptions/interrupts pending */ + _kvm_deliver_intr(vcpu); + + _kvm_check_requests(vcpu, cpu); + _kvm_update_vmid(vcpu, cpu); + r = kvm_enter_guest(run, vcpu); + + trace_kvm_out(vcpu); + guest_exit_irqoff(); + local_irq_enable(); + +out: + kvm_sigset_deactivate(vcpu); + + vcpu_put(vcpu); + return r; +} + +int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_loongarch_interrupt *irq) +{ + int intr = (int)irq->irq; + + if (intr < 0) { + _kvm_dequeue_irq(vcpu, -intr); + return 0; + } + + _kvm_queue_irq(vcpu, intr); + kvm_vcpu_kick(vcpu); + return 0; +} + +int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -ENOIOCTLCMD; +} + +/** + * kvm_migrate_count() - Migrate timer. + * @vcpu: Virtual CPU. + * + * Migrate hrtimer to the current CPU by cancelling and restarting it + * if it was running prior to being cancelled. + * + * Must be called when the VCPU is migrated to a different CPU to ensure that + * timer expiry during guest execution interrupts the guest and causes the + * interrupt to be delivered in a timely manner. + */ +static void kvm_migrate_count(struct kvm_vcpu *vcpu) +{ + if (hrtimer_cancel(&vcpu->arch.swtimer)) + hrtimer_restart(&vcpu->arch.swtimer); +} + +static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + struct kvm_context *context; + struct loongarch_csrs *csr = vcpu->arch.csr; + bool migrated, all; + + /* + * Have we migrated to a different CPU? + * If so, any old guest TLB state may be stale. + */ + migrated = (vcpu->arch.last_sched_cpu != cpu); + + /* + * Was this the last VCPU to run on this CPU? + * If not, any old guest state from this VCPU will have been clobbered. + */ + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + all = migrated || (context->last_vcpu != vcpu); + context->last_vcpu = vcpu; + + /* + * Restore timer state regardless + */ + kvm_restore_timer(vcpu); + + /* Control guest page CCA attribute */ + kvm_change_csr_gcfg(KVM_GCFG_MATC_MASK, KVM_GCFG_MATC_ROOT); + /* Restore hardware perf csr */ + kvm_restore_hw_perf(vcpu); + +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +#endif + /* Don't bother restoring registers multiple times unless necessary */ + if (!all) + return 0; + + kvm_write_csr_gcntc((ulong)vcpu->kvm->arch.stablecounter_gftoffset); + /* + * Restore guest CSR registers + */ + kvm_restore_hw_gcsr(csr, KVM_CSR_CRMD); + kvm_restore_hw_gcsr(csr, KVM_CSR_PRMD); + kvm_restore_hw_gcsr(csr, KVM_CSR_EUEN); + kvm_restore_hw_gcsr(csr, KVM_CSR_MISC); + kvm_restore_hw_gcsr(csr, KVM_CSR_ECFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_ERA); + kvm_restore_hw_gcsr(csr, KVM_CSR_BADV); + kvm_restore_hw_gcsr(csr, KVM_CSR_BADI); + kvm_restore_hw_gcsr(csr, KVM_CSR_EENTRY); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBIDX); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBEHI); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBELO0); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBELO1); + kvm_restore_hw_gcsr(csr, KVM_CSR_ASID); + kvm_restore_hw_gcsr(csr, KVM_CSR_PGDL); + kvm_restore_hw_gcsr(csr, KVM_CSR_PGDH); + kvm_restore_hw_gcsr(csr, KVM_CSR_PWCTL0); + kvm_restore_hw_gcsr(csr, KVM_CSR_PWCTL1); + kvm_restore_hw_gcsr(csr, KVM_CSR_STLBPGSIZE); + kvm_restore_hw_gcsr(csr, KVM_CSR_RVACFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_CPUID); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS0); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS1); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS2); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS3); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS4); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS5); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS6); + kvm_restore_hw_gcsr(csr, KVM_CSR_KS7); + kvm_restore_hw_gcsr(csr, KVM_CSR_TMID); + kvm_restore_hw_gcsr(csr, KVM_CSR_CNTC); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRENTRY); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRBADV); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRERA); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRSAVE); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRELO0); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRELO1); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBREHI); + kvm_restore_hw_gcsr(csr, KVM_CSR_TLBRPRMD); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN0); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN1); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN2); + kvm_restore_hw_gcsr(csr, KVM_CSR_DMWIN3); + kvm_restore_hw_gcsr(csr, KVM_CSR_LLBCTL); + + /* restore Root.Guestexcept from unused Guest guestexcept register */ + kvm_write_csr_gintc(csr->csrs[KVM_CSR_GINTC]); + + /* + * We should clear linked load bit to break interrupted atomics. This + * prevents a SC on the next VCPU from succeeding by matching a LL on + * the previous VCPU. + */ + if (vcpu->kvm->created_vcpus > 1) + kvm_set_gcsr_llbctl(KVM_LLBCTL_WCLLB); + + return 0; +} + +/* Restore ASID once we are scheduled back after preemption */ +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + unsigned long flags; + + local_irq_save(flags); + vcpu->cpu = cpu; + if (vcpu->arch.last_sched_cpu != cpu) { + kvm_debug("[%d->%d]KVM VCPU[%d] switch\n", + vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id); + /* + * Migrate the timer interrupt to the current CPU so that it + * always interrupts the guest and synchronously triggers a + * guest timer interrupt. + */ + kvm_migrate_count(vcpu); + } + + /* restore guest state to registers */ + _kvm_vcpu_load(vcpu, cpu); + local_irq_restore(flags); +} + +static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + + kvm_lose_fpu(vcpu); + kvm_lose_hw_perf(vcpu); + + kvm_save_hw_gcsr(csr, KVM_CSR_CRMD); + kvm_save_hw_gcsr(csr, KVM_CSR_PRMD); + kvm_save_hw_gcsr(csr, KVM_CSR_EUEN); + kvm_save_hw_gcsr(csr, KVM_CSR_MISC); + kvm_save_hw_gcsr(csr, KVM_CSR_ECFG); + kvm_save_hw_gcsr(csr, KVM_CSR_ERA); + kvm_save_hw_gcsr(csr, KVM_CSR_BADV); + kvm_save_hw_gcsr(csr, KVM_CSR_BADI); + kvm_save_hw_gcsr(csr, KVM_CSR_EENTRY); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBIDX); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBEHI); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBELO0); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBELO1); + kvm_save_hw_gcsr(csr, KVM_CSR_ASID); + kvm_save_hw_gcsr(csr, KVM_CSR_PGDL); + kvm_save_hw_gcsr(csr, KVM_CSR_PGDH); + kvm_save_hw_gcsr(csr, KVM_CSR_PGD); + kvm_save_hw_gcsr(csr, KVM_CSR_PWCTL0); + kvm_save_hw_gcsr(csr, KVM_CSR_PWCTL1); + kvm_save_hw_gcsr(csr, KVM_CSR_STLBPGSIZE); + kvm_save_hw_gcsr(csr, KVM_CSR_RVACFG); + kvm_save_hw_gcsr(csr, KVM_CSR_CPUID); + kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG1); + kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG2); + kvm_save_hw_gcsr(csr, KVM_CSR_PRCFG3); + kvm_save_hw_gcsr(csr, KVM_CSR_KS0); + kvm_save_hw_gcsr(csr, KVM_CSR_KS1); + kvm_save_hw_gcsr(csr, KVM_CSR_KS2); + kvm_save_hw_gcsr(csr, KVM_CSR_KS3); + kvm_save_hw_gcsr(csr, KVM_CSR_KS4); + kvm_save_hw_gcsr(csr, KVM_CSR_KS5); + kvm_save_hw_gcsr(csr, KVM_CSR_KS6); + kvm_save_hw_gcsr(csr, KVM_CSR_KS7); + kvm_save_hw_gcsr(csr, KVM_CSR_TMID); + kvm_save_hw_gcsr(csr, KVM_CSR_CNTC); + kvm_save_hw_gcsr(csr, KVM_CSR_LLBCTL); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRENTRY); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRBADV); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRERA); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRSAVE); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRELO0); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRELO1); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBREHI); + kvm_save_hw_gcsr(csr, KVM_CSR_TLBRPRMD); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN0); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN1); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN2); + kvm_save_hw_gcsr(csr, KVM_CSR_DMWIN3); + + /* save Root.Guestexcept in unused Guest guestexcept register */ + kvm_save_timer(vcpu); + csr->csrs[KVM_CSR_GINTC] = kvm_read_csr_gintc(); + return 0; +} + +/* ASID can change if another task is scheduled during preemption */ +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + unsigned long flags; + int cpu; + + local_irq_save(flags); + cpu = smp_processor_id(); + vcpu->arch.last_sched_cpu = cpu; + vcpu->cpu = -1; + + /* save guest state in registers */ + _kvm_vcpu_put(vcpu, cpu); + local_irq_restore(flags); +} + +static int _kvm_get_one_reg(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, s64 *v) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + int reg_idx, ret; + + if ((reg->id & KVM_IOC_CSRID(0)) == KVM_IOC_CSRID(0)) { + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + ret = _kvm_getcsr(vcpu, reg_idx, v, 0); + if (ret == 0) + return ret; + } + + switch (reg->id) { + case KVM_REG_LOONGARCH_COUNTER: + *v = drdtime() + vcpu->kvm->arch.stablecounter_gftoffset; + break; + default: + if ((reg->id & KVM_REG_LOONGARCH_MASK) != KVM_REG_LOONGARCH_CSR) + return -EINVAL; + + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + if (reg_idx < CSR_ALL_SIZE) + *v = kvm_read_sw_gcsr(csr, reg_idx); + else + return -EINVAL; + } + return 0; +} + +static int _kvm_set_one_reg(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg, + s64 v) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + int ret = 0; + unsigned long flags; + u64 val; + int reg_idx; + + val = v; + if ((reg->id & KVM_IOC_CSRID(0)) == KVM_IOC_CSRID(0)) { + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + ret = _kvm_setcsr(vcpu, reg_idx, &val, 0); + if (ret == 0) + return ret; + } + + switch (reg->id) { + case KVM_REG_LOONGARCH_COUNTER: + local_irq_save(flags); + /* + * gftoffset is relative with board, not vcpu + * only set for the first time for smp system + */ + if (vcpu->vcpu_id == 0) + vcpu->kvm->arch.stablecounter_gftoffset = (signed long)(v - drdtime()); + kvm_write_csr_gcntc((ulong)vcpu->kvm->arch.stablecounter_gftoffset); + local_irq_restore(flags); + break; + case KVM_REG_LOONGARCH_VCPU_RESET: + kvm_reset_timer(vcpu); + if (vcpu->vcpu_id == 0) + kvm_enable_ls3a_extirq(vcpu->kvm, false); + memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending)); + memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear)); + + /* disable pv timer when cpu resetting */ + vcpu->arch.st.guest_addr = 0; + break; + default: + if ((reg->id & KVM_REG_LOONGARCH_MASK) != KVM_REG_LOONGARCH_CSR) + return -EINVAL; + + reg_idx = KVM_GET_IOC_CSRIDX(reg->id); + if (reg_idx < CSR_ALL_SIZE) + kvm_write_sw_gcsr(csr, reg_idx, v); + else + return -EINVAL; + } + return ret; +} + +static int _kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + int ret; + s64 v; + + ret = _kvm_get_one_reg(vcpu, reg, &v); + if (ret) + return ret; + + ret = -EINVAL; + if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { + u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; + + ret = put_user(v, uaddr64); + } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) { + u32 __user *uaddr32 = (u32 __user *)(long)reg->addr; + u32 v32 = (u32)v; + + ret = put_user(v32, uaddr32); + } + + return ret; +} + +static int _kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + s64 v; + int ret; + + ret = -EINVAL; + if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { + u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; + ret = get_user(v, uaddr64); + } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) { + u32 __user *uaddr32 = (u32 __user *)(long)reg->addr; + s32 v32; + + ret = get_user(v32, uaddr32); + v = (s64)v32; + } + + if (ret) + return -EFAULT; + + return _kvm_set_one_reg(vcpu, reg, v); +} + +static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, + struct kvm_enable_cap *cap) +{ + int r = 0; + + if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap)) + return -EINVAL; + if (cap->flags) + return -EINVAL; + if (cap->args[0]) + return -EINVAL; + + switch (cap->cap) { + case KVM_CAP_LOONGARCH_FPU: + case KVM_CAP_LOONGARCH_LSX: + break; + default: + r = -EINVAL; + break; + } + + return r; +} + +long kvm_arch_vcpu_async_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + + if (ioctl == KVM_INTERRUPT) { + struct kvm_loongarch_interrupt irq; + + if (copy_from_user(&irq, argp, sizeof(irq))) + return -EFAULT; + kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, + irq.irq); + + return kvm_vcpu_ioctl_interrupt(vcpu, &irq); + } + + return -ENOIOCTLCMD; +} + +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, + bool line_status) +{ + u32 irq = irq_level->irq; + unsigned int irq_type, vcpu_idx, irq_num, ret; + int nrcpus = atomic_read(&kvm->online_vcpus); + bool level = irq_level->level; + unsigned long flags; + + irq_type = (irq >> KVM_LOONGSON_IRQ_TYPE_SHIFT) & KVM_LOONGSON_IRQ_TYPE_MASK; + vcpu_idx = (irq >> KVM_LOONGSON_IRQ_VCPU_SHIFT) & KVM_LOONGSON_IRQ_VCPU_MASK; + irq_num = (irq >> KVM_LOONGSON_IRQ_NUM_SHIFT) & KVM_LOONGSON_IRQ_NUM_MASK; + + switch (irq_type) { + case KVM_LOONGSON_IRQ_TYPE_IOAPIC: + if (!ls7a_ioapic_in_kernel(kvm)) + return -ENXIO; + + if (vcpu_idx >= nrcpus) + return -EINVAL; + + ls7a_ioapic_lock(ls7a_ioapic_irqchip(kvm), &flags); + ret = kvm_ls7a_ioapic_set_irq(kvm, irq_num, level); + ls7a_ioapic_unlock(ls7a_ioapic_irqchip(kvm), &flags); + return ret; + } + kvm->stat.vm_ioctl_irq_line++; + + return -EINVAL; +} + +static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct loongarch_kvm_irqchip *chip) +{ + int r, dlen; + + r = 0; + dlen = chip->len - sizeof(struct loongarch_kvm_irqchip); + switch (chip->chip_id) { + case KVM_IRQCHIP_LS7A_IOAPIC: + if (dlen != sizeof(struct kvm_ls7a_ioapic_state)) { + kvm_err("get ls7a state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_get_ls7a_ioapic(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_GIPI: + if (dlen != sizeof(gipiState)) { + kvm_err("get gipi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_get_ls3a_ipi(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_HT_IRQ: + case KVM_IRQCHIP_LS3A_ROUTE: + break; + case KVM_IRQCHIP_LS3A_EXTIRQ: + if (dlen != sizeof(struct kvm_loongarch_ls3a_extirq_state)) { + kvm_err("get extioi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_get_ls3a_extirq(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_IPMASK: + break; + default: + r = -EINVAL; + break; + } + return r; +dlen_err: + r = -EINVAL; + return r; +} + +static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct loongarch_kvm_irqchip *chip) +{ + int r, dlen; + + r = 0; + dlen = chip->len - sizeof(struct loongarch_kvm_irqchip); + switch (chip->chip_id) { + case KVM_IRQCHIP_LS7A_IOAPIC: + if (dlen != sizeof(struct kvm_ls7a_ioapic_state)) { + kvm_err("set ls7a state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_set_ls7a_ioapic(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_GIPI: + if (dlen != sizeof(gipiState)) { + kvm_err("set gipi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_set_ls3a_ipi(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_HT_IRQ: + case KVM_IRQCHIP_LS3A_ROUTE: + break; + case KVM_IRQCHIP_LS3A_EXTIRQ: + if (dlen != sizeof(struct kvm_loongarch_ls3a_extirq_state)) { + kvm_err("set extioi state err dlen:%d\n", dlen); + goto dlen_err; + } + r = kvm_set_ls3a_extirq(kvm, (void *)chip->data); + break; + case KVM_IRQCHIP_LS3A_IPMASK: + break; + default: + r = -EINVAL; + break; + } + return r; +dlen_err: + r = -EINVAL; + return r; +} + +/* + * Read or write a bunch of msrs. All parameters are kernel addresses. + * + * @return number of msrs set successfully. + */ +static int _kvm_csr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, + struct kvm_csr_entry *entries, + int (*do_csr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data, int force)) +{ + int i; + + for (i = 0; i < msrs->ncsrs; ++i) + if (do_csr(vcpu, entries[i].index, &entries[i].data, 1)) + break; + + return i; +} + +static int kvm_csr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, + int (*do_csr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data, int force)) +{ + struct kvm_msrs msrs; + struct kvm_csr_entry *entries; + int r, n; + unsigned size; + + r = -EFAULT; + if (copy_from_user(&msrs, user_msrs, sizeof msrs)) + goto out; + + r = -E2BIG; + if (msrs.ncsrs >= CSR_ALL_SIZE) + goto out; + + size = sizeof(struct kvm_csr_entry) * msrs.ncsrs; + entries = memdup_user(user_msrs->entries, size); + if (IS_ERR(entries)) { + r = PTR_ERR(entries); + goto out; + } + + r = n = _kvm_csr_io(vcpu, &msrs, entries, do_csr); + if (r < 0) + goto out_free; + + r = -EFAULT; + if (copy_to_user(user_msrs->entries, entries, size)) + goto out_free; + + r = n; + +out_free: + kfree(entries); +out: + return r; +} + +static int _kvm_vcpu_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + case KVM_LARCH_VCPU_PVTIME_CTRL: + ret = _kvm_pvtime_set_attr(vcpu, attr); + break; +#endif + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int _kvm_vcpu_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + case KVM_LARCH_VCPU_PVTIME_CTRL: + ret = _kvm_pvtime_get_attr(vcpu, attr); + break; +#endif + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int _kvm_vcpu_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret = -ENXIO; + + switch (attr->group) { +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + case KVM_LARCH_VCPU_PVTIME_CTRL: + ret = _kvm_pvtime_has_attr(vcpu, attr); + break; +#endif + default: + ret = -ENXIO; + break; + } + + return ret; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + struct kvm_device_attr attr; + long r; + + vcpu_load(vcpu); + + switch (ioctl) { + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: { + struct kvm_one_reg reg; + + r = -EFAULT; + if (copy_from_user(®, argp, sizeof(reg))) + break; + if (ioctl == KVM_SET_ONE_REG) + r = _kvm_set_reg(vcpu, ®); + else + r = _kvm_get_reg(vcpu, ®); + break; + } + case KVM_ENABLE_CAP: { + struct kvm_enable_cap cap; + + r = -EFAULT; + if (copy_from_user(&cap, argp, sizeof(cap))) + break; + r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); + break; + } + case KVM_CHECK_EXTENSION: { + unsigned int ext; + if (copy_from_user(&ext, argp, sizeof(ext))) + return -EFAULT; + switch (ext) { + case KVM_CAP_LOONGARCH_FPU: + r = !!cpu_has_fpu; + break; + case KVM_CAP_LOONGARCH_LSX: + r = !!cpu_has_lsx; + break; + default: + break; + } + break; + } + + case KVM_LOONGARCH_GET_VCPU_STATE: + { + int i; + struct kvm_loongarch_vcpu_state vcpu_state; + r = -EFAULT; + + vcpu_state.online_vcpus = vcpu->kvm->arch.online_vcpus; + vcpu_state.is_migrate = 1; + for (i = 0; i < 4; i++) + vcpu_state.core_ext_ioisr[i] = vcpu->arch.core_ext_ioisr[i]; + + vcpu_state.irq_pending = vcpu->arch.irq_pending; + vcpu_state.irq_clear = vcpu->arch.irq_clear; + + if (copy_to_user(argp, &vcpu_state, sizeof(struct kvm_loongarch_vcpu_state))) + break; + r = 0; + break; + } + + case KVM_LOONGARCH_SET_VCPU_STATE: + { + int i; + struct kvm_loongarch_vcpu_state vcpu_state; + r = -EFAULT; + + if (copy_from_user(&vcpu_state, argp, sizeof(struct kvm_loongarch_vcpu_state))) + return -EFAULT; + + vcpu->kvm->arch.online_vcpus = vcpu_state.online_vcpus; + vcpu->kvm->arch.is_migrate = vcpu_state.is_migrate; + for (i = 0; i < 4; i++) + vcpu->arch.core_ext_ioisr[i] = vcpu_state.core_ext_ioisr[i]; + + vcpu->arch.irq_pending = vcpu_state.irq_pending; + vcpu->arch.irq_clear = vcpu_state.irq_clear; + r = 0; + break; + } + case KVM_GET_MSRS: { + r = kvm_csr_io(vcpu, argp, _kvm_getcsr); + break; + } + case KVM_SET_MSRS: { + r = kvm_csr_io(vcpu, argp, _kvm_setcsr); + break; + } + case KVM_SET_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = _kvm_vcpu_set_attr(vcpu, &attr); + break; + } + case KVM_GET_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = _kvm_vcpu_get_attr(vcpu, &attr); + break; + } + case KVM_HAS_DEVICE_ATTR: { + r = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + r = _kvm_vcpu_has_attr(vcpu, &attr); + break; + } + default: + r = -ENOIOCTLCMD; + } + + vcpu_put(vcpu); + return r; +} + +long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = filp->private_data; + void __user *argp = (void __user *)arg; + long r; + + switch (ioctl) { + case KVM_CREATE_IRQCHIP: + { + mutex_lock(&kvm->lock); + r = -EEXIST; + if (kvm->arch.v_ioapic) + goto create_irqchip_unlock; + + r = kvm_create_ls7a_ioapic(kvm); + if (r < 0) + goto create_irqchip_unlock; + r = kvm_create_ls3a_ipi(kvm); + if (r < 0) { + mutex_lock(&kvm->slots_lock); + kvm_destroy_ls7a_ioapic(kvm); + mutex_unlock(&kvm->slots_lock); + goto create_irqchip_unlock; + } + r = kvm_create_ls3a_ext_irq(kvm); + if (r < 0) { + mutex_lock(&kvm->slots_lock); + kvm_destroy_ls3a_ipi(kvm); + kvm_destroy_ls7a_ioapic(kvm); + mutex_unlock(&kvm->slots_lock); + } + irqchip_debug_init(kvm); + /* Write kvm->irq_routing before kvm->arch.vpic. */ + smp_wmb(); +create_irqchip_unlock: + mutex_unlock(&kvm->lock); + break; + } + case KVM_GET_IRQCHIP: { + struct loongarch_kvm_irqchip *kchip; + struct loongarch_kvm_irqchip uchip; + if (copy_from_user(&uchip, argp, sizeof(struct loongarch_kvm_irqchip))) + goto out; + kchip = memdup_user(argp, uchip.len); + if (IS_ERR(kchip)) { + r = PTR_ERR(kchip); + goto out; + } + + r = -ENXIO; + if (!ls7a_ioapic_in_kernel(kvm)) + goto get_irqchip_out; + r = kvm_vm_ioctl_get_irqchip(kvm, kchip); + if (r) + goto get_irqchip_out; + if (copy_to_user(argp, kchip, kchip->len)) + goto get_irqchip_out; + r = 0; +get_irqchip_out: + kfree(kchip); + break; + } + case KVM_SET_IRQCHIP: { + struct loongarch_kvm_irqchip *kchip; + struct loongarch_kvm_irqchip uchip; + if (copy_from_user(&uchip, argp, sizeof(struct loongarch_kvm_irqchip))) + goto out; + + kchip = memdup_user(argp, uchip.len); + if (IS_ERR(kchip)) { + r = PTR_ERR(kchip); + goto out; + } + + r = -ENXIO; + if (!ls7a_ioapic_in_kernel(kvm)) + goto set_irqchip_out; + r = kvm_vm_ioctl_set_irqchip(kvm, kchip); + if (r) + goto set_irqchip_out; + r = 0; +set_irqchip_out: + kfree(kchip); + break; + } + case KVM_LOONGARCH_GET_IOCSR: + { + r = _kvm_get_iocsr(kvm, argp); + break; + } + case KVM_LOONGARCH_SET_IOCSR: + { + r = _kvm_set_iocsr(kvm, argp); + break; + } + case KVM_LOONGARCH_SET_CPUCFG: + { + r = 0; + if (copy_from_user(&kvm->arch.cpucfgs, argp, sizeof(struct kvm_cpucfg))) + r = -EFAULT; + break; + } + case KVM_LOONGARCH_GET_CPUCFG: + { + r = 0; + if (copy_to_user(argp, &kvm->arch.cpucfgs, sizeof(struct kvm_cpucfg))) + r = -EFAULT; + break; + } + default: + r = -ENOIOCTLCMD; + } +out: + + return r; +} + +int kvm_arch_init(void *opaque) +{ + struct kvm_context *context; + unsigned long vpid_mask; + int cpu; + + vmcs = alloc_percpu(struct kvm_context); + if (!vmcs) { + printk(KERN_ERR "kvm: failed to allocate percpu kvm_context\n"); + return -ENOMEM; + } + + vpid_mask = kvm_read_csr_gstat(); + vpid_mask = (vpid_mask & KVM_GSTAT_GIDBIT) >> KVM_GSTAT_GIDBIT_SHIFT; + if (vpid_mask) + vpid_mask = GENMASK(vpid_mask - 1, 0); + + for_each_possible_cpu(cpu) { + context = per_cpu_ptr(vmcs, cpu); + context->gid_mask = vpid_mask; + context->gid_ver_mask = ~context->gid_mask; + context->gid_fisrt_ver = context->gid_mask + 1; + context->vpid_cache = context->gid_mask + 1; + context->last_vcpu = NULL; + } + + _kvm_init_fault(); + return 0; +} + +void kvm_arch_exit(void) +{ + free_percpu(vmcs); +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -ENOIOCTLCMD; +} + +void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) +{ +} + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + int i = 0; + + /* no need vcpu_load and vcpu_put */ + fpu->fcsr = vcpu->arch.fpu.fcsr; + fpu->fcc = vcpu->arch.fpu.fcc; + for (i = 0; i < NUM_FPU_REGS; i++) + memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + int i = 0; + + /* no need vcpu_load and vcpu_put */ + vcpu->arch.fpu.fcsr = fpu->fcsr; + vcpu->arch.fpu.fcc = fpu->fcc; + for (i = 0; i < NUM_FPU_REGS; i++) + memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64); + + return 0; +} + +vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + +int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) +{ + int r; + + switch (ext) { + case KVM_CAP_ONE_REG: + case KVM_CAP_ENABLE_CAP: + case KVM_CAP_READONLY_MEM: + case KVM_CAP_SYNC_MMU: +#ifdef CONFIG_HAVE_LS_KVM_MSI + case KVM_CAP_SIGNAL_MSI: +#endif + case KVM_CAP_IMMEDIATE_EXIT: + r = 1; + break; + case KVM_CAP_NR_VCPUS: + r = num_online_cpus(); + break; + case KVM_CAP_MAX_VCPUS: + r = KVM_MAX_VCPUS; + break; + case KVM_CAP_MAX_VCPU_ID: + r = KVM_MAX_VCPU_ID; + break; + case KVM_CAP_NR_MEMSLOTS: + r = KVM_USER_MEM_SLOTS; + break; + case KVM_CAP_LOONGARCH_FPU: + /* We don't handle systems with inconsistent cpu_has_fpu */ + r = !!cpu_has_fpu; + break; + case KVM_CAP_LOONGARCH_LSX: + /* + * We don't support LSX vector partitioning yet: + * 1) It would require explicit support which can't be tested + * yet due to lack of support in current hardware. + * 2) It extends the state that would need to be saved/restored + * by e.g. QEMU for migration. + * + * When vector partitioning hardware becomes available, support + * could be added by requiring a flag when enabling + * KVM_CAP_LOONGARCH_LSX capability to indicate that userland knows + * to save/restore the appropriate extra state. + */ + r = cpu_has_lsx; + break; + case KVM_CAP_IRQCHIP: + case KVM_CAP_IOEVENTFD: + /* we wouldn't be here unless cpu_has_lvz */ + r = 1; + break; + case KVM_CAP_LOONGARCH_VZ: + /* get user defined kvm version */ + r = KVM_LOONGARCH_VERSION; + break; + default: + r = 0; + break; + } + return r; +} + +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return _kvm_pending_timer(vcpu) || + kvm_read_hw_gcsr(KVM_CSR_ESTAT) & + (1 << (KVM_INT_TIMER - KVM_INT_START)); +} + +int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) +{ + int i; + struct loongarch_csrs *csr; + + if (!vcpu) + return -1; + + kvm_debug("VCPU Register Dump:\n"); + kvm_debug("\tpc = 0x%08lx\n", vcpu->arch.pc); + kvm_debug("\texceptions: %08lx\n", vcpu->arch.irq_pending); + + for (i = 0; i < 32; i += 4) { + kvm_debug("\tgpr%02d: %08lx %08lx %08lx %08lx\n", i, + vcpu->arch.gprs[i], + vcpu->arch.gprs[i + 1], + vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]); + } + + csr = vcpu->arch.csr; + kvm_debug("\tCRMOD: 0x%08llx, exst: 0x%08llx\n", + kvm_read_hw_gcsr(KVM_CSR_CRMD), + kvm_read_hw_gcsr(KVM_CSR_ESTAT)); + + kvm_debug("\tERA: 0x%08llx\n", kvm_read_hw_gcsr(KVM_CSR_ERA)); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + int i; + + vcpu_load(vcpu); + + for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++) + vcpu->arch.gprs[i] = regs->gpr[i]; + vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */ + vcpu->arch.pc = regs->pc; + + vcpu_put(vcpu); + return 0; +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + int i; + + vcpu_load(vcpu); + + for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++) + regs->gpr[i] = vcpu->arch.gprs[i]; + + regs->pc = vcpu->arch.pc; + + vcpu_put(vcpu); + return 0; +} + +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + return 0; +} + +/* Initial guest state */ +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + unsigned long timer_hz; + + /* + * Initialize guest register state to valid architectural reset state. + */ + timer_hz = calc_const_freq(); + kvm_init_timer(vcpu, timer_hz); + + /* Set Initialize mode for GUEST */ + kvm_write_sw_gcsr(csr, KVM_CSR_CRMD, KVM_CRMD_DA); + + /* Set cpuid */ + kvm_write_sw_gcsr(csr, KVM_CSR_TMID, vcpu->vcpu_id); + + /* start with no pending virtual guest interrupts */ + csr->csrs[KVM_CSR_GINTC] = 0; + return 0; +} + +/* Enable FPU for guest and restore context */ +void kvm_own_fpu(struct kvm_vcpu *vcpu) +{ + unsigned long sr; + + preempt_disable(); + + sr = kvm_read_hw_gcsr(KVM_CSR_EUEN); + + /* + * If LSX state is already live, it is undefined how it interacts with + * FR=0 FPU state, and we don't want to hit reserved instruction + * exceptions trying to save the LSX state later when CU=1 && FR=1, so + * play it safe and save it first. + * + * In theory we shouldn't ever hit this case since kvm_lose_fpu() should + * get called when guest CU1 is set, however we can't trust the guest + * not to clobber the status register directly via the commpage. + */ + if (cpu_has_lsx && sr & KVM_EUEN_FPEN && + vcpu->arch.aux_inuse & (KVM_LARCH_LSX | KVM_LARCH_LASX)) + kvm_lose_fpu(vcpu); + + /* + * Enable FPU for guest + * We set FR and FRE according to guest context + */ + kvm_set_csr_euen(KVM_EUEN_FPEN); + + /* If guest FPU state not active, restore it now */ + if (!(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) { + kvm_restore_fpu(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_FPU; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU); + } else { + trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_FPU); + } + + preempt_enable(); +} + +#ifdef CONFIG_CPU_HAS_LSX +/* Enable LSX for guest and restore context */ +void kvm_own_lsx(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + + /* + * Enable FP if enabled in guest, since we're restoring FP context + * anyway. + */ + if (_kvm_guest_has_fpu(&vcpu->arch)) { + + kvm_set_csr_euen(KVM_EUEN_FPEN); + } + + /* Enable LSX for guest */ + kvm_set_csr_euen(KVM_EUEN_LSXEN); + + switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | + KVM_LARCH_LSX | KVM_LARCH_LASX)) { + case KVM_LARCH_FPU: + /* + * Guest FPU state already loaded, + * only restore upper LSX state + */ + kvm_restore_lsx_upper(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_LSX; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, + KVM_TRACE_AUX_LSX); + break; + case 0: + /* Neither FP or LSX already active, + * restore full LSX state + */ + kvm_restore_lsx(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_LSX; + if (_kvm_guest_has_fpu(&vcpu->arch)) + vcpu->arch.aux_inuse |= KVM_LARCH_FPU; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, + KVM_TRACE_AUX_FPU_LSX); + break; + default: + trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_LSX); + break; + } + + preempt_enable(); +} +#endif + +#ifdef CONFIG_CPU_HAS_LASX +/* Enable LASX for guest and restore context */ +void kvm_own_lasx(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + + /* + * Enable FP if enabled in guest, since we're restoring FP context + * anyway. + */ + if (_kvm_guest_has_lsx(&vcpu->arch)) { + /* Enable LSX for guest */ + kvm_set_csr_euen(KVM_EUEN_LSXEN); + } + + /* + * Enable FPU if enabled in guest, since we're restoring FPU context + * anyway. We set FR and FRE according to guest context. + */ + if (_kvm_guest_has_fpu(&vcpu->arch)) { + kvm_set_csr_euen(KVM_EUEN_FPEN); + } + + /* Enable LASX for guest */ + kvm_set_csr_euen(KVM_EUEN_LASXEN); + + switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | + KVM_LARCH_LSX | KVM_LARCH_LASX)) { + case (KVM_LARCH_LSX | KVM_LARCH_FPU): + case KVM_LARCH_LSX: + /* + * Guest LSX state already loaded, only restore upper LASX state + */ + kvm_restore_lasx_upper(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_LASX; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); + break; + case KVM_LARCH_FPU: + /* + * Guest FP state already loaded, only restore 64~256 LASX state + */ + kvm_restore_lsx_upper(vcpu); + kvm_restore_lasx_upper(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_LASX; + if (_kvm_guest_has_lsx(&vcpu->arch)) + vcpu->arch.aux_inuse |= KVM_LARCH_LSX; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); + break; + case 0: + /* Neither FP or LSX already active, restore full LASX state */ + kvm_restore_lasx(vcpu); + vcpu->arch.aux_inuse |= KVM_LARCH_LASX; + if (_kvm_guest_has_lsx(&vcpu->arch)) + vcpu->arch.aux_inuse |= KVM_LARCH_LSX; + if (_kvm_guest_has_fpu(&vcpu->arch)) + vcpu->arch.aux_inuse |= KVM_LARCH_FPU; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, + KVM_TRACE_AUX_FPU_LSX_LASX); + break; + default: + trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_LASX); + break; + } + + preempt_enable(); +} +#endif + +/* Save and disable FPU & LSX & LASX */ +void kvm_lose_fpu(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + if (cpu_has_lasx && (vcpu->arch.aux_inuse & KVM_LARCH_LASX)) { + +#ifdef CONFIG_CPU_HAS_LASX + kvm_save_lasx(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX_LASX); + + /* Disable LASX & MAS & FPU */ + disable_lasx(); + disable_lsx(); +#endif + + if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + kvm_clear_csr_euen(KVM_EUEN_FPEN); + } + vcpu->arch.aux_inuse &= ~(KVM_LARCH_FPU | + KVM_LARCH_LSX | KVM_LARCH_LASX); + } else if (cpu_has_lsx && vcpu->arch.aux_inuse & KVM_LARCH_LSX) { + +#ifdef CONFIG_CPU_HAS_LASX + kvm_save_lsx(vcpu); + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX); + + /* Disable LSX & FPU */ + disable_lsx(); +#endif + + if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + kvm_clear_csr_euen(KVM_EUEN_FPEN); + } + vcpu->arch.aux_inuse &= ~(KVM_LARCH_FPU | KVM_LARCH_LSX); + } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { + + kvm_save_fpu(vcpu); + vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU; + trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); + + /* Disable FPU */ + kvm_clear_csr_euen(KVM_EUEN_FPEN); + } + preempt_enable(); +} + +void kvm_lose_hw_perf(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.aux_inuse & KVM_LARCH_PERF) { + struct loongarch_csrs *csr = vcpu->arch.csr; + /* save guest pmu csr */ + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL0); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR0); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL1); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR1); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL2); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR2); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCTRL3); + kvm_save_hw_gcsr(csr, KVM_CSR_PERFCNTR3); + if (((kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL0) | + kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL1) | + kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL2) | + kvm_read_sw_gcsr(csr, KVM_CSR_PERFCTRL3)) + & KVM_PMU_PLV_ENABLE) == 0) + vcpu->arch.aux_inuse &= ~KVM_LARCH_PERF; + /* config host pmu csr */ + kvm_write_csr_gcfg(kvm_read_csr_gcfg() & ~KVM_GCFG_GPERF); + /* TODO: pmu csr used by host and guest at the same time */ + kvm_write_csr_perfctrl0(0); + kvm_write_csr_perfcntr0(0); + kvm_write_csr_perfctrl1(0); + kvm_write_csr_perfcntr1(0); + kvm_write_csr_perfctrl2(0); + kvm_write_csr_perfcntr2(0); + kvm_write_csr_perfctrl3(0); + kvm_write_csr_perfcntr3(0); + } +} + +void kvm_restore_hw_perf(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.aux_inuse & KVM_LARCH_PERF) { + struct loongarch_csrs *csr = vcpu->arch.csr; + /* enable guest pmu */ + kvm_write_csr_gcfg(kvm_read_csr_gcfg() | KVM_GCFG_GPERF); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL0); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR0); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL1); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR1); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL2); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR2); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCTRL3); + kvm_restore_hw_gcsr(csr, KVM_CSR_PERFCNTR3); + } +} + +static int __init kvm_loongarch_init(void) +{ + int ret; + + if (!cpu_has_lvz) + return 0; + + ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (ret) + return ret; + + return 0; +} + +static void __exit kvm_loongarch_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_loongarch_init); +module_exit(kvm_loongarch_exit); + +static const struct cpu_feature loongarch_kvm_feature[] = { + { .feature = cpu_feature(LOONGARCH_LVZ) }, + {}, +}; +MODULE_DEVICE_TABLE(cpu, loongarch_kvm_feature); + +EXPORT_TRACEPOINT_SYMBOL(kvm_exit); diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c new file mode 100644 index 0000000000000000000000000000000000000000..ba9fa7895d2566767532d8f6dfc10ed4ffd0a45b --- /dev/null +++ b/arch/loongarch/kvm/mmu.c @@ -0,0 +1,1314 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvm_compat.h" + +/* + * KVM_MMU_CACHE_MIN_PAGES is the number of GPA page table translation levels + * for which pages need to be cached. + */ +#if defined(__PAGETABLE_PMD_FOLDED) +#define KVM_MMU_CACHE_MIN_PAGES 1 +#else +#define KVM_MMU_CACHE_MIN_PAGES 2 +#endif + +static int kvm_tlb_flush_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) +{ + preempt_disable(); + gpa &= (PAGE_MASK << 1); + invtlb(INVTLB_GID_ADDR, kvm_read_csr_gstat() & KVM_GSTAT_GID, gpa); + preempt_enable(); + return 0; +} + +static inline int kvm_pmd_huge(pmd_t pmd) +{ +#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT + return (pmd_val(pmd) & _PAGE_HUGE) != 0; +#else + return 0; +#endif +} + +static inline int kvm_pud_huge(pud_t pud) +{ +#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT + return (pud_val(pud) & _PAGE_HUGE) != 0; +#else + return 0; +#endif +} + +static inline pmd_t kvm_pmd_mkhuge(pmd_t pmd) +{ +#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + return pmd_mkhuge(pmd); +#else + pte_t entry; + + pte_val(entry) = pmd_val(pmd); + entry = pte_mkhuge(entry); + pmd_val(pmd) = pte_val(entry); +#endif +#endif + return pmd; +} + +static inline pmd_t kvm_pmd_mkclean(pmd_t pmd) +{ +#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + return pmd_mkclean(pmd); +#else + pte_t entry; + + pte_val(entry) = pmd_val(pmd); + entry = pte_mkclean(entry); + pmd_val(pmd) = pte_val(entry); +#endif +#endif + return pmd; +} + +static inline pmd_t kvm_pmd_mkold(pmd_t pmd) +{ +#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + return pmd_mkold(pmd); +#else + pte_t entry; + + pte_val(entry) = pmd_val(pmd); + entry = pte_mkold(entry); + pmd_val(pmd) = pte_val(entry); +#endif +#endif + return pmd; +} + +/** + * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory. + * + * Allocate a blank KVM GPA page directory (PGD) for representing guest physical + * to host physical page mappings. + * + * Returns: Pointer to new KVM GPA page directory. + * NULL on allocation failure. + */ +pgd_t *kvm_pgd_alloc(void) +{ + pgd_t *ret; + struct page *page; + + page = alloc_pages(GFP_KERNEL, 0); + if (!page) + return NULL; + ret = (pgd_t *) page_address(page); + if (ret) + pgd_init(ret); + + return ret; +} + +/** + * kvm_walk_pgd() - Walk page table with optional allocation. + * @pgd: Page directory pointer. + * @addr: Address to index page table using. + * @cache: MMU page cache to allocate new page tables from, or NULL. + * + * Walk the page tables pointed to by @pgd to find the PTE corresponding to the + * address @addr. If page tables don't exist for @addr, they will be created + * from the MMU cache if @cache is not NULL. + * + * Returns: Pointer to pte_t corresponding to @addr. + * NULL if a page table doesn't exist for @addr and !@cache. + * NULL if a page table allocation failed. + */ +static pte_t *kvm_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, + struct vm_area_struct *vma, unsigned long hva, + unsigned long addr) +{ + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + pgd += pgd_index(addr); + if (pgd_none(*pgd)) { + /* Not used yet */ + BUG(); + return NULL; + } + p4d = p4d_offset(pgd, addr); + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + pmd_t *new_pmd; + + if (!cache) + return NULL; + new_pmd = kvm_mmu_memory_cache_alloc(cache); + pmd_init(new_pmd); + pud_populate(NULL, pud, new_pmd); + } + pmd = pmd_offset(pud, addr); + if (kvm_pmd_huge(*pmd)) { + return (pte_t *)pmd; + } + if (pmd_none(*pmd)) { + pte_t *new_pte; + + if (!cache) + return NULL; + new_pte = kvm_mmu_memory_cache_alloc(cache); + clear_page(new_pte); + pmd_populate_kernel(NULL, pmd, new_pte); + } + return pte_offset_kernel(pmd, addr); +} + +/* Caller must hold kvm->mm_lock */ +static pte_t *kvm_pte_for_gpa(struct kvm *kvm, + struct kvm_mmu_memory_cache *cache, + struct vm_area_struct *vma, unsigned long hva, + unsigned long addr) +{ + return kvm_walk_pgd(kvm->arch.gpa_mm.pgd, cache, vma, hva, addr); +} + +#define kvm_pte_for_gpa_fast(kvm, gpa) kvm_pte_for_gpa(kvm, NULL, NULL, 0, gpa) +/* + * kvm_flush_gpa_{pte,pmd,pud,pgd,pt}. + * Flush a range of guest physical address space from the VM's GPA page tables. + */ + +static bool kvm_flush_gpa_pte(pte_t *pte, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + int i_min = pte_index(start_gpa); + int i_max = pte_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1); + int i; + + for (i = i_min; i <= i_max; ++i) { + if (!pte_present(pte[i])) + continue; + + set_pte(pte + i, __pte(0)); + if (data) + *data = *data + 1; + } + return safe_to_remove; +} + +static bool kvm_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + pte_t *pte; + unsigned long end = ~0ul; + int i_min = pmd_index(start_gpa); + int i_max = pmd_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1); + int i; + + for (i = i_min; i <= i_max; ++i, start_gpa = 0) { + if (!pmd_present(pmd[i])) + continue; + + if (kvm_pmd_huge(pmd[i]) && pmd_present(pmd[i])) { + pmd_clear(pmd + i); + if (data) + *data += PTRS_PER_PMD; + continue; + } + + pte = pte_offset_kernel(pmd + i, 0); + if (i == i_max) + end = end_gpa; + + if (kvm_flush_gpa_pte(pte, start_gpa, end, data)) { + pmd_clear(pmd + i); + pte_free_kernel(NULL, pte); + } else { + safe_to_remove = false; + } + } + return safe_to_remove; +} + +static bool kvm_flush_gpa_pud(pud_t *pud, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + pmd_t *pmd; + unsigned long end = ~0ul; + int i_min = pud_index(start_gpa); + int i_max = pud_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1); + int i; + + for (i = i_min; i <= i_max; ++i, start_gpa = 0) { + if (!pud_present(pud[i])) + continue; + + pmd = pmd_offset(pud + i, 0); + if (i == i_max) + end = end_gpa; + + if (kvm_flush_gpa_pmd(pmd, start_gpa, end, data)) { + pud_clear(pud + i); + pmd_free(NULL, pmd); + } else { + safe_to_remove = false; + } + } + return safe_to_remove; +} + +static bool kvm_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa, + unsigned long end_gpa, unsigned long *data) +{ + p4d_t *p4d; + pud_t *pud; + unsigned long end = ~0ul; + int i_min = pgd_index(start_gpa); + int i_max = pgd_index(end_gpa); + bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1); + int i; + + for (i = i_min; i <= i_max; ++i, start_gpa = 0) { + if (!pgd_present(pgd[i])) + continue; + + p4d = p4d_offset(pgd + i, 0); + pud = pud_offset(p4d, 0); + if (i == i_max) + end = end_gpa; + + if (kvm_flush_gpa_pud(pud, start_gpa, end, data)) { + pgd_clear(pgd + i); + pud_free(NULL, pud); + } else { + safe_to_remove = false; + } + } + return safe_to_remove; +} + +/** + * kvm_flush_gpa_pt() - Flush a range of guest physical addresses. + * @kvm: KVM pointer. + * @start_gfn: Guest frame number of first page in GPA range to flush. + * @end_gfn: Guest frame number of last page in GPA range to flush. + * + * Flushes a range of GPA mappings from the GPA page tables. + * + * The caller must hold the @kvm->mmu_lock spinlock. + * + * Returns: Whether its safe to remove the top level page directory because + * all lower levels have been removed. + */ +static bool kvm_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn, void *data) +{ + return kvm_flush_gpa_pgd(kvm->arch.gpa_mm.pgd, + start_gfn << PAGE_SHIFT, + end_gfn << PAGE_SHIFT, (unsigned long *)data); +} + +/* + * kvm_mkclean_gpa_pt. + * Mark a range of guest physical address space clean (writes fault) in the VM's + * GPA page table to allow dirty page tracking. + */ + +static int kvm_mkclean_pte(pte_t *pte, unsigned long start, unsigned long end) +{ + int ret = 0; + int i_min = pte_index(start); + int i_max = pte_index(end); + int i; + pte_t val; + + for (i = i_min; i <= i_max; ++i) { + val = pte[i]; + if (pte_present(val) && pte_dirty(val)) { + set_pte(pte + i, pte_mkclean(val)); + ret = 1; + } + } + return ret; +} + +static int kvm_mkclean_pmd(pmd_t *pmd, unsigned long start, unsigned long end) +{ + int ret = 0; + pte_t *pte; + unsigned long cur_end = ~0ul; + int i_min = pmd_index(start); + int i_max = pmd_index(end); + int i; + pmd_t old, new; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pmd_present(pmd[i])) + continue; + + if (kvm_pmd_huge(pmd[i])) { + old = pmd[i]; + new = kvm_pmd_mkclean(old); + if (pmd_val(new) == pmd_val(old)) + continue; + set_pmd(pmd + i, new); + ret = 1; + continue; + } + + pte = pte_offset_kernel(pmd + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkclean_pte(pte, start, cur_end); + } + + return ret; +} + +static int kvm_mkclean_pud(pud_t *pud, unsigned long start, unsigned long end) +{ + int ret = 0; + pmd_t *pmd; + unsigned long cur_end = ~0ul; + int i_min = pud_index(start); + int i_max = pud_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pud_present(pud[i])) + continue; + + pmd = pmd_offset(pud + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkclean_pmd(pmd, start, cur_end); + } + return ret; +} + +static int kvm_mkclean_pgd(pgd_t *pgd, unsigned long start, unsigned long end) +{ + int ret = 0; + p4d_t *p4d; + pud_t *pud; + unsigned long cur_end = ~0ul; + int i_min = pgd_index(start); + int i_max = pgd_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pgd_present(pgd[i])) + continue; + + p4d = p4d_offset(pgd + i, 0); + pud = pud_offset(p4d, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkclean_pud(pud, start, cur_end); + } + return ret; +} + +/** + * kvm_mkclean_gpa_pt() - Make a range of guest physical addresses clean. + * @kvm: KVM pointer. + * @start_gfn: Guest frame number of first page in GPA range to flush. + * @end_gfn: Guest frame number of last page in GPA range to flush. + * + * Make a range of GPA mappings clean so that guest writes will fault and + * trigger dirty page logging. + * + * The caller must hold the @kvm->mmu_lock spinlock. + * + * Returns: Whether any GPA mappings were modified, which would require + * derived mappings (GVA page tables & TLB enties) to be + * invalidated. + */ +static int kvm_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn) +{ + return kvm_mkclean_pgd(kvm->arch.gpa_mm.pgd, start_gfn << PAGE_SHIFT, + end_gfn << PAGE_SHIFT); +} + +/** + * kvm_arch_mmu_enable_log_dirty_pt_masked() - write protect dirty pages + * @kvm: The KVM pointer + * @slot: The memory slot associated with mask + * @gfn_offset: The gfn offset in memory slot + * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory + * slot to be write protected + * + * Walks bits set in mask write protects the associated pte's. Caller must + * acquire @kvm->mmu_lock. + */ +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + gfn_t base_gfn = slot->base_gfn + gfn_offset; + gfn_t start = base_gfn + __ffs(mask); + gfn_t end = base_gfn + __fls(mask); + + kvm_mkclean_gpa_pt(kvm, start, end); + + /* + * FIXME: disable THP to improve vm migration success ratio, + * how to know migration failure to enable THP again + */ + slot->arch.flags |= KVM_MEMSLOT_DISABLE_THP; +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + int needs_flush; + + /* + * If dirty page logging is enabled, write protect all pages in the slot + * ready for dirty logging. + * + * There is no need to do this in any of the following cases: + * CREATE: No dirty mappings will already exist. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot() + */ + if (change == KVM_MR_FLAGS_ONLY && + (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) && + new->flags & KVM_MEM_LOG_DIRTY_PAGES)) { + spin_lock(&kvm->mmu_lock); + /* Write protect GPA page table entries */ + needs_flush = kvm_mkclean_gpa_pt(kvm, new->base_gfn, + new->base_gfn + new->npages - 1); + /* Let implementation do the rest */ + if (needs_flush) + kvm_flush_remote_tlbs(kvm); + spin_unlock(&kvm->mmu_lock); + } +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ + /* Flush whole GPA */ + kvm_flush_gpa_pt(kvm, 0, ~0UL, NULL); + + /* Flush vpid for each VCPU individually */ + kvm_flush_remote_tlbs(kvm); +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ + unsigned long npages; + + /* + * The slot has been made invalid (ready for moving or deletion), so we + * need to ensure that it can no longer be accessed by any guest VCPUs. + */ + + npages = 0; + spin_lock(&kvm->mmu_lock); + /* Flush slot from GPA */ + kvm_flush_gpa_pt(kvm, slot->base_gfn, + slot->base_gfn + slot->npages - 1, &npages); + /* Let implementation do the rest */ + if (npages) + kvm_flush_remote_tlbs(kvm); + spin_unlock(&kvm->mmu_lock); +} + +void _kvm_destroy_mm(struct kvm *kvm) +{ + /* It should always be safe to remove after flushing the whole range */ + WARN_ON(!kvm_flush_gpa_pt(kvm, 0, ~0UL, NULL)); + pgd_free(NULL, kvm->arch.gpa_mm.pgd); + kvm->arch.gpa_mm.pgd = NULL; +} + +/* + * Mark a range of guest physical address space old (all accesses fault) in the + * VM's GPA page table to allow detection of commonly used pages. + */ + +static int kvm_mkold_pte(pte_t *pte, unsigned long start, + unsigned long end) +{ + int ret = 0; + int i_min = pte_index(start); + int i_max = pte_index(end); + int i; + pte_t old, new; + + for (i = i_min; i <= i_max; ++i) { + if (!pte_present(pte[i])) + continue; + + old = pte[i]; + new = pte_mkold(old); + if (pte_val(new) == pte_val(old)) + continue; + set_pte(pte + i, new); + ret = 1; + } + + return ret; +} + +static int kvm_mkold_pmd(pmd_t *pmd, unsigned long start, unsigned long end) +{ + int ret = 0; + pte_t *pte; + unsigned long cur_end = ~0ul; + int i_min = pmd_index(start); + int i_max = pmd_index(end); + int i; + pmd_t old, new; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pmd_present(pmd[i])) + continue; + + if (kvm_pmd_huge(pmd[i])) { + old = pmd[i]; + new = kvm_pmd_mkold(old); + if (pmd_val(new) == pmd_val(old)) + continue; + set_pmd(pmd + i, new); + ret = 1; + continue; + } + + pte = pte_offset_kernel(pmd + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkold_pte(pte, start, cur_end); + } + + return ret; +} + +static int kvm_mkold_pud(pud_t *pud, unsigned long start, unsigned long end) +{ + int ret = 0; + pmd_t *pmd; + unsigned long cur_end = ~0ul; + int i_min = pud_index(start); + int i_max = pud_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pud_present(pud[i])) + continue; + + pmd = pmd_offset(pud + i, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkold_pmd(pmd, start, cur_end); + } + + return ret; +} + +static int kvm_mkold_pgd(pgd_t *pgd, unsigned long start, unsigned long end) +{ + int ret = 0; + p4d_t *p4d; + pud_t *pud; + unsigned long cur_end = ~0ul; + int i_min = pgd_index(start); + int i_max = pgd_index(end); + int i; + + for (i = i_min; i <= i_max; ++i, start = 0) { + if (!pgd_present(pgd[i])) + continue; + + p4d = p4d_offset(pgd + i, 0); + pud = pud_offset(p4d, 0); + if (i == i_max) + cur_end = end; + + ret |= kvm_mkold_pud(pud, start, cur_end); + } + + return ret; +} + +static int handle_hva_to_gpa(struct kvm *kvm, + unsigned long start, + unsigned long end, + int (*handler)(struct kvm *kvm, gfn_t gfn, + gpa_t gfn_end, + struct kvm_memory_slot *memslot, + void *data), + void *data) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int ret = 0; + + slots = kvm_memslots(kvm); + + /* we only care about the pages that the guest sees */ + kvm_for_each_memslot(memslot, slots) { + unsigned long hva_start, hva_end; + gfn_t gfn, gfn_end; + + hva_start = max(start, memslot->userspace_addr); + hva_end = min(end, memslot->userspace_addr + + (memslot->npages << PAGE_SHIFT)); + if (hva_start >= hva_end) + continue; + + /* + * {gfn(page) | page intersects with [hva_start, hva_end)} = + * {gfn_start, gfn_start+1, ..., gfn_end-1}. + */ + gfn = hva_to_gfn_memslot(hva_start, memslot); + gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); + ret |= handler(kvm, gfn, gfn_end, memslot, data); + } + + return ret; +} + + +static int kvm_unmap_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + unsigned long npages; + + npages = 0; + kvm_flush_gpa_pt(kvm, gfn, gfn_end - 1, &npages); + *(unsigned long *)data = *(unsigned long *)data + npages; + + return npages > 0; +} + +int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable) +{ + unsigned long npages; + + npages = 0; + return handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, &npages); +} + +static int kvm_set_spte_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + gpa_t gpa = gfn << PAGE_SHIFT; + pte_t hva_pte = *(pte_t *)data; + pte_t *gpa_pte = kvm_pte_for_gpa_fast(kvm, gpa); + pte_t old_pte; + + if (!gpa_pte) + return 0; + + /* Mapping may need adjusting depending on memslot flags */ + old_pte = *gpa_pte; + if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES && !pte_dirty(old_pte)) + hva_pte = pte_mkclean(hva_pte); + else if (memslot->flags & KVM_MEM_READONLY) + hva_pte = pte_wrprotect(hva_pte); + + set_pte(gpa_pte, hva_pte); + + /* Replacing an absent or old page doesn't need flushes */ + if (!pte_present(old_pte) || !pte_young(old_pte)) + return 0; + + /* Pages swapped, aged, moved, or cleaned require flushes */ + return !pte_present(hva_pte) || + !pte_young(hva_pte) || + pte_pfn(old_pte) != pte_pfn(hva_pte) || + (pte_dirty(old_pte) && !pte_dirty(hva_pte)); +} + +int _kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + unsigned long end = hva + PAGE_SIZE; + int ret; + + ret = handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pte); + if (ret) + /* Flush vpid for each VCPU individually */ + kvm_flush_remote_tlbs(kvm); + return 0; +} + +static int kvm_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + return kvm_mkold_pgd(kvm->arch.gpa_mm.pgd, gfn << PAGE_SHIFT, + gfn_end << PAGE_SHIFT); +} + +static int kvm_test_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, + struct kvm_memory_slot *memslot, void *data) +{ + gpa_t gpa = gfn << PAGE_SHIFT; + pte_t *gpa_pte = kvm_pte_for_gpa_fast(kvm, gpa); + + if (!gpa_pte) + return 0; + return pte_young(*gpa_pte); +} + +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) +{ + return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); +} + +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) +{ + return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); +} + +static pud_t *kvm_get_pud(struct kvm *kvm, + struct kvm_mmu_memory_cache *cache, phys_addr_t addr) +{ + pgd_t *pgd; + p4d_t *p4d; + + pgd = kvm->arch.gpa_mm.pgd + pgd_index(addr); + if (pgd_none(*pgd)) { + /* Not used yet */ + BUG(); + return NULL; + } + + p4d = p4d_offset(pgd, addr); + return pud_offset(p4d, addr); +} + +static pmd_t *kvm_get_pmd(struct kvm *kvm, + struct vm_area_struct *vma, unsigned long hva, + struct kvm_mmu_memory_cache *cache, phys_addr_t addr) +{ + pud_t *pud; + pmd_t *pmd; + + pud = kvm_get_pud(kvm, cache, addr); + if (!pud || kvm_pud_huge(*pud)) + return NULL; + + if (pud_none(*pud)) { + if (!cache) + return NULL; + pmd = kvm_mmu_memory_cache_alloc(cache); + pmd_init(pmd); + pud_populate(NULL, pud, pmd); + } + + return pmd_offset(pud, addr); +} + +static int kvm_set_pmd_huge(struct kvm_vcpu *vcpu, struct kvm_mmu_memory_cache + *cache, phys_addr_t addr, const pmd_t *new_pmd, + struct vm_area_struct *vma, unsigned long hva) +{ + pmd_t *pmd, old_pmd; + +retry: + pmd = kvm_get_pmd(vcpu->kvm, vma, hva, cache, addr); + VM_BUG_ON(!pmd); + + old_pmd = *pmd; + /* + * Multiple vcpus faulting on the same PMD entry, can + * lead to them sequentially updating the PMD with the + * same value. Following the break-before-make + * (pmd_clear() followed by tlb_flush()) process can + * hinder forward progress due to refaults generated + * on missing translations. + * + * Skip updating the page table if the entry is + * unchanged. + */ + if (pmd_val(old_pmd) == pmd_val(*new_pmd)) + return 0; + + if (pmd_present(old_pmd)) { + /* + * If we already have PTE level mapping for this block, + * we must unmap it to avoid inconsistent TLB state and + * leaking the table page. We could end up in this situation + * if the memory slot was marked for dirty logging and was + * reverted, leaving PTE level mappings for the pages accessed + * during the period. So, unmap the PTE level mapping for this + * block and retry, as we could have released the upper level + * table in the process. + * + * Normal THP split/merge follows mmu_notifier callbacks and do + * get handled accordingly. + */ + if (!kvm_pmd_huge(old_pmd)) { + ++vcpu->stat.huge_merge_exits; + kvm_flush_gpa_pt(vcpu->kvm, + (addr & PMD_MASK) >> PAGE_SHIFT, + ((addr & PMD_MASK) + PMD_SIZE - 1) >> PAGE_SHIFT, NULL); + goto retry; + } + /* + * Mapping in huge pages should only happen through a + * fault. If a page is merged into a transparent huge + * page, the individual subpages of that huge page + * should be unmapped through MMU notifiers before we + * get here. + * + * Merging of CompoundPages is not supported; they + * should become splitting first, unmapped, merged, + * and mapped back in on-demand. + */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + WARN_ON_ONCE(pmd_pfn(old_pmd) != pmd_pfn(*new_pmd)); +#endif + pmd_clear(pmd); + } + + kvm_tlb_flush_gpa(vcpu, addr & PMD_MASK); + set_pmd(pmd, *new_pmd); + return 0; +} + +/* + * Adjust pfn start boundary if support for transparent hugepage + */ +static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, unsigned long *gpap) +{ + kvm_pfn_t pfn = *pfnp; + gfn_t gfn = *gpap >> PAGE_SHIFT; + struct page *page = pfn_to_page(pfn); + + /* + * PageTransCompoundMap() returns true for THP and + * hugetlbfs. Make sure the adjustment is done only for THP + * pages. + */ + if ((!PageHuge(page)) && PageTransCompound(page) && + (atomic_read(&page->_mapcount) < 0)) { + unsigned long mask; + /* + * The address we faulted on is backed by a transparent huge + * page. However, because we map the compound huge page and + * not the individual tail page, we need to transfer the + * refcount to the head page. We have to be careful that the + * THP doesn't start to split while we are adjusting the + * refcounts. + * + * We are sure this doesn't happen, because mmu_notifier_retry + * was successful and we are holding the mmu_lock, so if this + * THP is trying to split, it will be blocked in the mmu + * notifier before touching any of the pages, specifically + * before being able to call __split_huge_page_refcount(). + * + * We can therefore safely transfer the refcount from PG_tail + * to PG_head and switch the pfn from a tail page to the head + * page accordingly. + */ + mask = PTRS_PER_PMD - 1; + VM_BUG_ON((gfn & mask) != (pfn & mask)); + if (pfn & mask) { + *gpap &= PMD_MASK; + kvm_release_pfn_clean(pfn); + pfn &= ~mask; + kvm_get_pfn(pfn); + *pfnp = pfn; + } + + return true; + } + + return false; +} + +static bool fault_supports_huge_mapping(struct kvm_memory_slot *memslot, + unsigned long hva, + unsigned long map_size) +{ + gpa_t gpa_start; + hva_t uaddr_start, uaddr_end; + size_t size; + + if (memslot->arch.flags & KVM_MEMSLOT_DISABLE_THP) + return false; + + size = memslot->npages * PAGE_SIZE; + gpa_start = memslot->base_gfn << PAGE_SHIFT; + uaddr_start = memslot->userspace_addr; + uaddr_end = uaddr_start + size; + + /* + * Pages belonging to memslots that don't have the same alignment + * within a PMD/PUD for userspace and GPA cannot be mapped with stage-2 + * PMD/PUD entries, because we'll end up mapping the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SIZE: + * +---+--------------------+--------------------+-----+ + * |abc|def Stage-2 block | Stage-2 block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those stage-2 blocks, we'll end up with this incorrect + * mapping: + * d -> f + * e -> g + * f -> h + */ + if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) + return false; + + /* + * Next, let's make sure we're not trying to map anything not covered + * by the memslot. This means we have to prohibit block size mappings + * for the beginning and end of a non-block aligned and non-block sized + * memory slot (illustrated by the head and tail parts of the + * userspace view above containing pages 'abcde' and 'xyz', + * respectively). + * + * Note that it doesn't matter if we do the check using the + * userspace_addr or the base_gfn, as both are equally aligned (per + * the check above) and equally sized. + */ + return (hva & ~(map_size - 1)) >= uaddr_start && + (hva & ~(map_size - 1)) + map_size <= uaddr_end; +} + +/** + * kvm_map_page_fast() - Fast path GPA fault handler. + * @vcpu: VCPU pointer. + * @gpa: Guest physical address of fault. + * @write: Whether the fault was due to a write. + * @out_entry: New PTE for @gpa (written on success unless NULL). + * @out_buddy: New PTE for @gpa's buddy (written on success unless + * NULL). + * + * Perform fast path GPA fault handling, doing all that can be done without + * calling into KVM. This handles marking old pages young (for idle page + * tracking), and dirtying of clean pages (for dirty page logging). + * + * Returns: 0 on success, in which case we can update derived mappings and + * resume guest execution. + * -EFAULT on failure due to absent GPA mapping or write to + * read-only page, in which case KVM must be consulted. + */ +static int kvm_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa, + bool write, + pte_t *out_entry, pte_t *out_buddy) +{ + struct kvm *kvm = vcpu->kvm; + gfn_t gfn = gpa >> PAGE_SHIFT; + pte_t *ptep; + kvm_pfn_t pfn = 0; /* silence bogus GCC warning */ + bool pfn_valid = false; + int ret = 0; + + spin_lock(&kvm->mmu_lock); + + /* Fast path - just check GPA page table for an existing entry */ + ptep = kvm_pte_for_gpa_fast(kvm, gpa); + if (!ptep || !pte_present(*ptep)) { + ret = -EFAULT; + goto out; + } + + /* Track access to pages marked old */ + if (!pte_young(*ptep)) { + set_pte(ptep, pte_mkyoung(*ptep)); + pfn = pte_pfn(*ptep); + pfn_valid = true; + /* call kvm_set_pfn_accessed() after unlock */ + } + if (write && !pte_dirty(*ptep)) { + if (!pte_write(*ptep)) { + ret = -EFAULT; + goto out; + } + + /* Track dirtying of writeable pages */ + set_pte(ptep, pte_mkdirty(*ptep)); + pfn = pte_pfn(*ptep); + if (kvm_pmd_huge(*((pmd_t *)ptep))) { + int i; + gfn_t base_gfn = (gpa & PMD_MASK) >> PAGE_SHIFT; + + for (i = 0; i < PTRS_PER_PTE; i++) + mark_page_dirty(kvm, base_gfn + i); + } else + mark_page_dirty(kvm, gfn); + kvm_set_pfn_dirty(pfn); + } + + if (out_entry) + *out_entry = *ptep; + if (out_buddy) + *out_buddy = *ptep_buddy(ptep); + +out: + spin_unlock(&kvm->mmu_lock); + if (pfn_valid) + kvm_set_pfn_accessed(pfn); + return ret; +} + +/** + * kvm_map_page() - Map a guest physical page. + * @vcpu: VCPU pointer. + * @gpa: Guest physical address of fault. + * @write: Whether the fault was due to a write. + * @out_entry: New PTE for @gpa (written on success unless NULL). + * @out_buddy: New PTE for @gpa's buddy (written on success unless + * NULL). + * + * Handle GPA faults by creating a new GPA mapping (or updating an existing + * one). + * + * This takes care of marking pages young or dirty (idle/dirty page tracking), + * asking KVM for the corresponding PFN, and creating a mapping in the GPA page + * tables. Derived mappings (GVA page tables and TLBs) must be handled by the + * caller. + * + * Returns: 0 on success, in which case the caller may use the @out_entry + * and @out_buddy PTEs to update derived mappings and resume guest + * execution. + * -EFAULT if there is no memory region at @gpa or a write was + * attempted to a read-only memory region. This is usually handled + * as an MMIO access. + */ +static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, + bool write, + pte_t *out_entry, pte_t *out_buddy) +{ + struct kvm *kvm = vcpu->kvm; + struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; + gfn_t gfn = gpa >> PAGE_SHIFT; + int srcu_idx, err = 0; + kvm_pfn_t pfn; + pte_t *ptep; + bool writeable; + unsigned long prot_bits; + unsigned long mmu_seq; + u32 exccode = (vcpu->arch.host_estat & KVM_ESTAT_EXC) >> KVM_ESTAT_EXC_SHIFT; + + unsigned long hva; + struct kvm_memory_slot *memslot; + bool force_pte = false; + struct vm_area_struct *vma; + unsigned long vma_pagesize; + bool writable; + int ret, retry_no = 0; + + /* Try the fast path to handle old / clean pages */ + srcu_idx = srcu_read_lock(&kvm->srcu); + if ((exccode != KVM_EXCCODE_TLBRI) && (exccode != KVM_EXCCODE_TLBXI)) { + err = kvm_map_page_fast(vcpu, gpa, write, out_entry, + out_buddy); + if (!err) + goto out; + } + + memslot = gfn_to_memslot(kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); + if (kvm_is_error_hva(hva) || (write && !writable)) + goto out; + + /* Let's check if we will get back a huge page backed by hugetlbfs */ + mmap_read_lock(current->mm); + vma = find_vma_intersection(current->mm, hva, hva + 1); + if (unlikely(!vma)) { + kvm_err("Failed to find VMA for hva 0x%lx\n", hva); + mmap_read_unlock(current->mm); + err = -EFAULT; + goto out; + } + + vma_pagesize = vma_kernel_pagesize(vma); + + if (fault_supports_huge_mapping(memslot, hva, vma_pagesize)) { + force_pte = true; + vma_pagesize = PAGE_SIZE; + ++vcpu->stat.huge_dec_exits; + } + + /* PMD is not folded, adjust gfn to new boundary */ + if (vma_pagesize == PMD_SIZE) + gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; + mmap_read_unlock(current->mm); + + /* We need a minimum of cached pages ready for page table creation */ + err = kvm_mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES); + if (err) + goto out; + +retry: + /* + * Used to check for invalidations in progress, of the pfn that is + * returned by pfn_to_pfn_prot below. + */ + mmu_seq = kvm->mmu_notifier_seq; + /* + * Ensure the read of mmu_notifier_seq isn't reordered with PTE reads in + * gfn_to_pfn_prot() (which calls get_user_pages()), so that we don't + * risk the page we get a reference to getting unmapped before we have a + * chance to grab the mmu_lock without mmu_notifier_retry() noticing. + * + * This smp_rmb() pairs with the effective smp_wmb() of the combination + * of the pte_unmap_unlock() after the PTE is zapped, and the + * spin_lock() in kvm_mmu_notifier_invalidate_() before + * mmu_notifier_seq is incremented. + */ + smp_rmb(); + + /* Slow path - ask KVM core whether we can access this GPA */ + pfn = gfn_to_pfn_prot(kvm, gfn, write, &writeable); + if (is_error_noslot_pfn(pfn)) { + err = -EFAULT; + goto out; + } + + spin_lock(&kvm->mmu_lock); + /* Check if an invalidation has taken place since we got pfn */ + if (mmu_notifier_retry(kvm, mmu_seq)) { + /* + * This can happen when mappings are changed asynchronously, but + * also synchronously if a COW is triggered by + * gfn_to_pfn_prot(). + */ + spin_unlock(&kvm->mmu_lock); + kvm_release_pfn_clean(pfn); + if (retry_no > 100) { + retry_no = 0; + schedule(); + } + retry_no++; + goto retry; + } + + if (vma_pagesize == PAGE_SIZE && !force_pte) { + /* + * Only PMD_SIZE transparent hugepages(THP) are + * currently supported. This code will need to be + * updated to support other THP sizes. + * + * Make sure the host VA and the guest IPA are sufficiently + * aligned and that the block is contained within the memslot. + */ + ++vcpu->stat.huge_thp_exits; + if (fault_supports_huge_mapping(memslot, hva, PMD_SIZE) && + transparent_hugepage_adjust(&pfn, &gpa)) { + ++vcpu->stat.huge_adjust_exits; + vma_pagesize = PMD_SIZE; + } + } + + /* Set up the prot bits */ + prot_bits = _PAGE_PRESENT | __READABLE; + if (vma->vm_flags & (VM_IO | VM_PFNMAP)) + prot_bits |= _CACHE_SUC; + else + prot_bits |= _CACHE_CC; + + if (writeable) { + prot_bits |= _PAGE_WRITE; + if (write) { + prot_bits |= __WRITEABLE | _PAGE_MODIFIED; + mark_page_dirty(kvm, gfn); + kvm_set_pfn_dirty(pfn); + } + } + + if (vma_pagesize == PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, __pgprot(prot_bits)); + + new_pmd = kvm_pmd_mkhuge(new_pmd); + + if (writeable && write) { + int i; + gfn_t base_gfn = (gpa & PMD_MASK) >> PAGE_SHIFT; + + for (i = 0; i < PTRS_PER_PTE; i++) + mark_page_dirty(kvm, base_gfn + i); + } + ++vcpu->stat.huge_set_exits; + ret = kvm_set_pmd_huge(vcpu, memcache, gpa, &new_pmd, vma, hva); + } else { + pte_t new_pte = pfn_pte(pfn, __pgprot(prot_bits)); + + if (writeable && write) + mark_page_dirty(kvm, gfn); + + /* Ensure page tables are allocated */ + ptep = kvm_pte_for_gpa(kvm, memcache, vma, hva, gpa); + set_pte(ptep, new_pte); + + err = 0; + if (out_entry) + *out_entry = new_pte; + if (out_buddy) + *out_buddy = *ptep_buddy(&new_pte); + } + + spin_unlock(&kvm->mmu_lock); + kvm_release_pfn_clean(pfn); + kvm_set_pfn_accessed(pfn); +out: + srcu_read_unlock(&kvm->srcu, srcu_idx); + return err; +} + +int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, + bool write) +{ + int ret; + + ret = kvm_map_page(vcpu, badv, write, NULL, NULL); + if (ret) + return ret; + + /* Invalidate this entry in the TLB */ + return kvm_tlb_flush_gpa(vcpu, badv); +} + +/** + * kvm_flush_tlb_all() - Flush all root TLB entries for + * guests. + * + * Invalidate all entries including GVA-->GPA and GPA-->HPA mappings. + */ +void kvm_flush_tlb_all(void) +{ + unsigned long flags; + + local_irq_save(flags); + invtlb_all(INVTLB_ALLGID, 0, 0); + local_irq_restore(flags); +} diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c new file mode 100644 index 0000000000000000000000000000000000000000..6fa063434ae2f55c5447e2d0372851aad1af25a6 --- /dev/null +++ b/arch/loongarch/kvm/timer.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kvmcpu.h" +#include "trace.h" +#include "kvm_compat.h" + +/* + * ktime_to_tick() - Scale ktime_t to a 64-bit stable timer. + * + * Caches the dynamic nanosecond bias in vcpu->arch.timer_dyn_bias. + */ +static u64 ktime_to_tick(struct kvm_vcpu *vcpu, ktime_t now) +{ + s64 now_ns, periods; + u64 delta; + + now_ns = ktime_to_ns(now); + delta = now_ns + vcpu->arch.timer_dyn_bias; + + if (delta >= vcpu->arch.timer_period) { + /* If delta is out of safe range the bias needs adjusting */ + periods = div64_s64(now_ns, vcpu->arch.timer_period); + vcpu->arch.timer_dyn_bias = -periods * vcpu->arch.timer_period; + /* Recalculate delta with new bias */ + delta = now_ns + vcpu->arch.timer_dyn_bias; + } + + /* + * We've ensured that: + * delta < timer_period + */ + return div_u64(delta * vcpu->arch.timer_mhz, MNSEC_PER_SEC); +} + +/** + * kvm_resume_hrtimer() - Resume hrtimer, updating expiry. + * @vcpu: Virtual CPU. + * @now: ktime at point of resume. + * @stable_timer: stable timer at point of resume. + * + * Resumes the timer and updates the timer expiry based on @now and @count. + */ +static void kvm_resume_hrtimer(struct kvm_vcpu *vcpu, ktime_t now, u64 stable_timer) +{ + u64 delta; + ktime_t expire; + + /* Stable timer decreased to zero or + * initialize to zero, set 4 second timer + */ + delta = div_u64(stable_timer * MNSEC_PER_SEC, vcpu->arch.timer_mhz); + expire = ktime_add_ns(now, delta); + + /* Update hrtimer to use new timeout */ + hrtimer_cancel(&vcpu->arch.swtimer); + hrtimer_start(&vcpu->arch.swtimer, expire, HRTIMER_MODE_ABS_PINNED); +} + +/** + * kvm_init_timer() - Initialise stable timer. + * @vcpu: Virtual CPU. + * @timer_hz: Frequency of timer. + * + * Initialise the timer to the specified frequency, zero it, and set it going if + * it's enabled. + */ +void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long timer_hz) +{ + ktime_t now; + unsigned long ticks; + struct loongarch_csrs *csr = vcpu->arch.csr; + + vcpu->arch.timer_mhz = timer_hz >> 20; + vcpu->arch.timer_period = div_u64((u64)MNSEC_PER_SEC * IOCSR_TIMER_MASK, vcpu->arch.timer_mhz); + vcpu->arch.timer_dyn_bias = 0; + + /* Starting at 0 */ + ticks = 0; + now = ktime_get(); + vcpu->arch.timer_bias = ticks - ktime_to_tick(vcpu, now); + vcpu->arch.timer_bias &= IOCSR_TIMER_MASK; + + kvm_write_sw_gcsr(csr, KVM_CSR_TVAL, ticks); +} + +/** + * kvm_count_timeout() - Push timer forward on timeout. + * @vcpu: Virtual CPU. + * + * Handle an hrtimer event by push the hrtimer forward a period. + * + * Returns: The hrtimer_restart value to return to the hrtimer subsystem. + */ +enum hrtimer_restart kvm_count_timeout(struct kvm_vcpu *vcpu) +{ + unsigned long timer_cfg; + + /* Add the Count period to the current expiry time */ + timer_cfg = kvm_read_sw_gcsr(vcpu->arch.csr, KVM_CSR_TCFG); + if (timer_cfg & KVM_TCFG_PERIOD) { + hrtimer_add_expires_ns(&vcpu->arch.swtimer, timer_cfg & KVM_TCFG_VAL); + return HRTIMER_RESTART; + } else + return HRTIMER_NORESTART; +} + +/* + * kvm_restore_timer() - Restore timer state. + * @vcpu: Virtual CPU. + * + * Restore soft timer state from saved context. + */ +void kvm_restore_timer(struct kvm_vcpu *vcpu) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + ktime_t saved_ktime, now; + u64 stable_timer, new_timertick = 0; + u64 delta = 0; + int expired = 0; + unsigned long timer_cfg; + + /* + * Set guest stable timer cfg csr + */ + timer_cfg = kvm_read_sw_gcsr(csr, KVM_CSR_TCFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_ESTAT); + if (!(timer_cfg & KVM_TCFG_EN)) { + kvm_restore_hw_gcsr(csr, KVM_CSR_TCFG); + kvm_restore_hw_gcsr(csr, KVM_CSR_TVAL); + return; + } + + now = ktime_get(); + saved_ktime = vcpu->arch.stable_ktime_saved; + stable_timer = kvm_read_sw_gcsr(csr, KVM_CSR_TVAL); + + /*hrtimer not expire */ + delta = ktime_to_tick(vcpu, ktime_sub(now, saved_ktime)); + if (delta >= stable_timer) + expired = 1; + + if (expired) { + if (timer_cfg & KVM_TCFG_PERIOD) { + new_timertick = (delta - stable_timer) % (timer_cfg & KVM_TCFG_VAL); + } else { + new_timertick = 1; + } + } else { + new_timertick = stable_timer - delta; + } + + new_timertick &= KVM_TCFG_VAL; + kvm_write_gcsr_timercfg(timer_cfg); + kvm_write_gcsr_timertick(new_timertick); + if (expired) + _kvm_queue_irq(vcpu, LARCH_INT_TIMER); +} + +/* + * kvm_acquire_timer() - Switch to hard timer state. + * @vcpu: Virtual CPU. + * + * Restore hard timer state on top of existing soft timer state if possible. + * + * Since hard timer won't remain active over preemption, preemption should be + * disabled by the caller. + */ +void kvm_acquire_timer(struct kvm_vcpu *vcpu) +{ + unsigned long flags, guestcfg; + + guestcfg = kvm_read_csr_gcfg(); + if (!(guestcfg & KVM_GCFG_TIT)) + return; + + /* enable guest access to hard timer */ + kvm_write_csr_gcfg(guestcfg & ~KVM_GCFG_TIT); + + /* + * Freeze the soft-timer and sync the guest stable timer with it. We do + * this with interrupts disabled to avoid latency. + */ + local_irq_save(flags); + hrtimer_cancel(&vcpu->arch.swtimer); + local_irq_restore(flags); +} + + +/* + * _kvm_save_timer() - Switch to software emulation of guest timer. + * @vcpu: Virtual CPU. + * + * Save guest timer state and switch to software emulation of guest + * timer. The hard timer must already be in use, so preemption should be + * disabled. + */ +static ktime_t _kvm_save_timer(struct kvm_vcpu *vcpu, u64 *stable_timer) +{ + u64 end_stable_timer; + ktime_t before_time; + + before_time = ktime_get(); + + /* + * Record a final stable timer which we will transfer to the soft-timer. + */ + end_stable_timer = kvm_read_gcsr_timertick(); + *stable_timer = end_stable_timer; + + kvm_resume_hrtimer(vcpu, before_time, end_stable_timer); + return before_time; +} + +/* + * kvm_save_timer() - Save guest timer state. + * @vcpu: Virtual CPU. + * + * Save guest timer state and switch to soft guest timer if hard timer was in + * use. + */ +void kvm_save_timer(struct kvm_vcpu *vcpu) +{ + struct loongarch_csrs *csr = vcpu->arch.csr; + unsigned long guestcfg; + u64 stable_timer = 0; + ktime_t save_ktime; + + preempt_disable(); + guestcfg = kvm_read_csr_gcfg(); + if (!(guestcfg & KVM_GCFG_TIT)) { + /* disable guest use of hard timer */ + kvm_write_csr_gcfg(guestcfg | KVM_GCFG_TIT); + + /* save hard timer state */ + kvm_save_hw_gcsr(csr, KVM_CSR_TCFG); + if (kvm_read_sw_gcsr(csr, KVM_CSR_TCFG) & KVM_TCFG_EN) { + save_ktime = _kvm_save_timer(vcpu, &stable_timer); + kvm_write_sw_gcsr(csr, KVM_CSR_TVAL, stable_timer); + vcpu->arch.stable_ktime_saved = save_ktime; + if (stable_timer == IOCSR_TIMER_MASK) + _kvm_queue_irq(vcpu, LARCH_INT_TIMER); + } else { + kvm_save_hw_gcsr(csr, KVM_CSR_TVAL); + } + } + + /* save timer-related state to VCPU context */ + kvm_save_hw_gcsr(csr, KVM_CSR_ESTAT); + preempt_enable(); +} + +void kvm_reset_timer(struct kvm_vcpu *vcpu) +{ + kvm_write_gcsr_timercfg(0); + kvm_write_sw_gcsr(vcpu->arch.csr, KVM_CSR_TCFG, 0); + hrtimer_cancel(&vcpu->arch.swtimer); +} diff --git a/arch/loongarch/kvm/trace.h b/arch/loongarch/kvm/trace.h new file mode 100644 index 0000000000000000000000000000000000000000..ae0693531a82eef2c874aab1ef15a0057b69071d --- /dev/null +++ b/arch/loongarch/kvm/trace.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_KVM_H + +#include +#include "kvm_compat.h" +#include "kvmcsr.h" + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace + +/* + * arch/loongarch/kvm/loongarch.c + */ +extern bool kvm_trace_guest_mode_change; +int kvm_guest_mode_change_trace_reg(void); +void kvm_guest_mode_change_trace_unreg(void); + +/* + * Tracepoints for VM enters + */ +DECLARE_EVENT_CLASS(kvm_transition, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu), + TP_STRUCT__entry( + __field(unsigned long, pc) + ), + + TP_fast_assign( + __entry->pc = vcpu->arch.pc; + ), + + TP_printk("PC: 0x%08lx", + __entry->pc) +); + +DEFINE_EVENT(kvm_transition, kvm_enter, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu)); + +DEFINE_EVENT(kvm_transition, kvm_reenter, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu)); + +DEFINE_EVENT(kvm_transition, kvm_out, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu)); + +/* The first 32 exit reasons correspond to Cause.ExcCode */ +#define KVM_TRACE_EXIT_INT 0 +#define KVM_TRACE_EXIT_TLBLD (KVM_EXCCODE_TLBL) +#define KVM_TRACE_EXIT_TLBST (KVM_EXCCODE_TLBS) +#define KVM_TRACE_EXIT_TLBI (KVM_EXCCODE_TLBI) +#define KVM_TRACE_EXIT_TLBMOD (KVM_EXCCODE_TLBM) +#define KVM_TRACE_EXIT_TLBRI (KVM_EXCCODE_TLBRI) +#define KVM_TRACE_EXIT_TLBXI (KVM_EXCCODE_TLBXI) +#define KVM_TRACE_EXIT_TLBPE (KVM_EXCCODE_TLBPE) +#define KVM_TRACE_EXIT_ADDE (KVM_EXCCODE_ADE) +#define KVM_TRACE_EXIT_UNALIGN (KVM_EXCCODE_ALE) +#define KVM_TRACE_EXIT_ODB (KVM_EXCCODE_OOB) +#define KVM_TRACE_EXIT_SYSCALL (KVM_EXCCODE_SYS) +#define KVM_TRACE_EXIT_BP (KVM_EXCCODE_BP) +#define KVM_TRACE_EXIT_INE (KVM_EXCCODE_INE) +#define KVM_TRACE_EXIT_IPE (KVM_EXCCODE_IPE) +#define KVM_TRACE_EXIT_FPDIS (KVM_EXCCODE_FPDIS) +#define KVM_TRACE_EXIT_LSXDIS (KVM_EXCCODE_LSXDIS) +#define KVM_TRACE_EXIT_LASXDIS (KVM_EXCCODE_LASXDIS) +#define KVM_TRACE_EXIT_FPE (KVM_EXCCODE_FPE) +#define KVM_TRACE_EXIT_WATCH (KVM_EXCCODE_WATCH) +#define KVM_TRACE_EXIT_GSPR (KVM_EXCCODE_GSPR) +#define KVM_TRACE_EXIT_HC (KVM_EXCCODE_HYP) +#define KVM_TRACE_EXIT_GCM (KVM_EXCCODE_GCM) + +/* Further exit reasons */ +#define KVM_TRACE_EXIT_IDLE 64 +#define KVM_TRACE_EXIT_CACHE 65 +#define KVM_TRACE_EXIT_SIGNAL 66 + +/* Tracepoints for VM exits */ +#define kvm_trace_symbol_exit_types \ + { KVM_TRACE_EXIT_INT, "Interrupt" }, \ + { KVM_TRACE_EXIT_TLBLD, "TLB (LD)" }, \ + { KVM_TRACE_EXIT_TLBST, "TLB (ST)" }, \ + { KVM_TRACE_EXIT_TLBI, "TLB Ifetch" }, \ + { KVM_TRACE_EXIT_TLBMOD, "TLB Mod" }, \ + { KVM_TRACE_EXIT_TLBRI, "TLB RI" }, \ + { KVM_TRACE_EXIT_TLBXI, "TLB XI" }, \ + { KVM_TRACE_EXIT_TLBPE, "TLB Previlege Error" },\ + { KVM_TRACE_EXIT_ADDE, "Address Error" }, \ + { KVM_TRACE_EXIT_UNALIGN, "Address unalign" }, \ + { KVM_TRACE_EXIT_ODB, "Out boundary" }, \ + { KVM_TRACE_EXIT_SYSCALL, "System Call" }, \ + { KVM_TRACE_EXIT_BP, "Breakpoint" }, \ + { KVM_TRACE_EXIT_INE, "Reserved Inst" }, \ + { KVM_TRACE_EXIT_IPE, "Inst prev error" }, \ + { KVM_TRACE_EXIT_FPDIS, "FPU disable" }, \ + { KVM_TRACE_EXIT_LSXDIS, "LSX disable" }, \ + { KVM_TRACE_EXIT_LASXDIS, "LASX disable" }, \ + { KVM_TRACE_EXIT_FPE, "FPE" }, \ + { KVM_TRACE_EXIT_WATCH, "DEBUG" }, \ + { KVM_TRACE_EXIT_GSPR, "GSPR" }, \ + { KVM_TRACE_EXIT_HC, "Hypercall" }, \ + { KVM_TRACE_EXIT_GCM, "CSR Mod" }, \ + { KVM_TRACE_EXIT_IDLE, "IDLE" }, \ + { KVM_TRACE_EXIT_CACHE, "CACHE" }, \ + { KVM_TRACE_EXIT_SIGNAL, "Signal" } + +TRACE_EVENT(kvm_exit, + TP_PROTO(struct kvm_vcpu *vcpu, unsigned int reason), + TP_ARGS(vcpu, reason), + TP_STRUCT__entry( + __field(unsigned long, pc) + __field(unsigned int, reason) + ), + + TP_fast_assign( + __entry->pc = vcpu->arch.pc; + __entry->reason = reason; + ), + + TP_printk("[%s]PC: 0x%08lx", + __print_symbolic(__entry->reason, + kvm_trace_symbol_exit_types), + __entry->pc) +); + +#define KVM_TRACE_AUX_RESTORE 0 +#define KVM_TRACE_AUX_SAVE 1 +#define KVM_TRACE_AUX_ENABLE 2 +#define KVM_TRACE_AUX_DISABLE 3 +#define KVM_TRACE_AUX_DISCARD 4 + +#define KVM_TRACE_AUX_FPU 1 +#define KVM_TRACE_AUX_LSX 2 +#define KVM_TRACE_AUX_FPU_LSX 3 +#define KVM_TRACE_AUX_LASX 4 +#define KVM_TRACE_AUX_FPU_LSX_LASX 7 + +#define kvm_trace_symbol_aux_op \ + { KVM_TRACE_AUX_RESTORE, "restore" }, \ + { KVM_TRACE_AUX_SAVE, "save" }, \ + { KVM_TRACE_AUX_ENABLE, "enable" }, \ + { KVM_TRACE_AUX_DISABLE, "disable" }, \ + { KVM_TRACE_AUX_DISCARD, "discard" } + +#define kvm_trace_symbol_aux_state \ + { KVM_TRACE_AUX_FPU, "FPU" }, \ + { KVM_TRACE_AUX_LSX, "LSX" }, \ + { KVM_TRACE_AUX_LASX, "LASX" }, \ + { KVM_TRACE_AUX_FPU_LSX, "FPU & LSX" }, \ + { KVM_TRACE_AUX_FPU_LSX_LASX, "FPU & LSX & LASX" } + +TRACE_EVENT(kvm_aux, + TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op, + unsigned int state), + TP_ARGS(vcpu, op, state), + TP_STRUCT__entry( + __field(unsigned long, pc) + __field(u8, op) + __field(u8, state) + ), + + TP_fast_assign( + __entry->pc = vcpu->arch.pc; + __entry->op = op; + __entry->state = state; + ), + + TP_printk("%s %s PC: 0x%08lx", + __print_symbolic(__entry->op, + kvm_trace_symbol_aux_op), + __print_symbolic(__entry->state, + kvm_trace_symbol_aux_state), + __entry->pc) +); + +TRACE_EVENT(kvm_vpid_change, + TP_PROTO(struct kvm_vcpu *vcpu, unsigned long vpid), + TP_ARGS(vcpu, vpid), + TP_STRUCT__entry( + __field(unsigned long, vpid) + ), + + TP_fast_assign( + __entry->vpid = vpid; + ), + + TP_printk("vpid: 0x%08lx", + __entry->vpid) +); + +#endif /* _TRACE_KVM_H */ + +/* This part must be outside protection */ +#include diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index aac92756317a69b63613e84bba314055eb4e7229..90b2e4a3198d34e56a3048dc9910eb132253d9aa 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -252,6 +252,7 @@ struct kvm_hyperv_exit { #define KVM_EXIT_X86_WRMSR 30 #define KVM_EXIT_RISCV_SBI 31 #define KVM_EXIT_X86_BUS_LOCK 33 +#define KVM_EXIT_LOONGARCH_IOCSR 36 #define KVM_EXIT_NOTIFY 37 /* For KVM_EXIT_INTERNAL_ERROR */ @@ -322,6 +323,13 @@ struct kvm_run { __u32 len; __u8 is_write; } mmio; + /* KVM_EXIT_LOONGARCH_IOCSR */ + struct { + __u64 phys_addr; + __u8 data[8]; + __u32 len; + __u8 is_write; + } iocsr_io; /* KVM_EXIT_HYPERCALL */ struct { __u64 nr; @@ -710,6 +718,16 @@ struct kvm_s390_irq_state { __u32 reserved[4]; /* will stay unused for compatibility reasons */ }; +struct kvm_loongarch_vcpu_state { + __u8 online_vcpus; + __u8 is_migrate; + __u32 cpu_freq; + __u32 count_ctl; + __u64 irq_pending; + __u64 irq_clear; + __u64 core_ext_ioisr[4]; +}; + /* for KVM_SET_GUEST_DEBUG */ #define KVM_GUESTDBG_ENABLE 0x00000001 @@ -1077,6 +1095,10 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_ARM_CPU_FEATURE 555 +#define KVM_CAP_LOONGARCH_FPU 800 +#define KVM_CAP_LOONGARCH_LSX 801 +#define KVM_CAP_LOONGARCH_VZ 802 + #ifdef KVM_CAP_IRQ_ROUTING struct kvm_irq_routing_irqchip { @@ -1223,6 +1245,7 @@ struct kvm_dirty_tlb { #define KVM_REG_ARM64 0x6000000000000000ULL #define KVM_REG_MIPS 0x7000000000000000ULL #define KVM_REG_RISCV 0x8000000000000000ULL +#define KVM_REG_LOONGARCH 0x9000000000000000ULL #define KVM_REG_SIZE_SHIFT 52 #define KVM_REG_SIZE_MASK 0x00f0000000000000ULL @@ -1560,6 +1583,14 @@ struct kvm_enc_region { #define KVM_S390_NORMAL_RESET _IO(KVMIO, 0xc3) #define KVM_S390_CLEAR_RESET _IO(KVMIO, 0xc4) +/* Add for LOONGSON read nodecounter */ +#define KVM_LOONGARCH_GET_VCPU_STATE _IOR(KVMIO, 0xc0, struct kvm_loongarch_vcpu_state) +#define KVM_LOONGARCH_SET_VCPU_STATE _IOW(KVMIO, 0xc1, struct kvm_loongarch_vcpu_state) +#define KVM_LOONGARCH_GET_CPUCFG _IOR(KVMIO, 0xc2, struct kvm_cpucfg) +#define KVM_LOONGARCH_GET_IOCSR _IOR(KVMIO, 0xc3, struct kvm_iocsr_entry) +#define KVM_LOONGARCH_SET_IOCSR _IOW(KVMIO, 0xc4, struct kvm_iocsr_entry) +#define KVM_LOONGARCH_SET_CPUCFG _IOR(KVMIO, 0xc5, struct kvm_cpucfg) + /* Available with KVM_CAP_XSAVE2 */ #define KVM_GET_XSAVE2 _IOR(KVMIO, 0xcf, struct kvm_xsave)