diff --git a/arch/loongarch/Kbuild b/arch/loongarch/Kbuild index 30d9afffc83758354c0c544a862cfcb9a89906b7..7760f9cb9777abd16e8f05e93633f3f1b9ef5f53 100644 --- a/arch/loongarch/Kbuild +++ b/arch/loongarch/Kbuild @@ -1,7 +1,10 @@ obj-y += kernel/ obj-y += mm/ obj-y += vdso/ -obj-$(CONFIG_KVM) += kvm/ + +ifdef CONFIG_KVM +obj-y += kvm/ +endif # for cleaning subdir- += boot diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index 705b8344e5622537a28a04076c2799c3650d8cc9..5a55f27f718cc1f846db3e3733c922144916199a 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -49,6 +49,11 @@ CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_LOONGSON3_ACPI_CPUFREQ=y CONFIG_EFI_CAPSULE_LOADER=m CONFIG_EFI_TEST=m +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=m +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m CONFIG_MODULES=y CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild index a0eed6076c79a4c773a898e7e27739810541ea6b..3e0c3e4e57aeafeef53cdcd8e2f53ca531721b1e 100644 --- a/arch/loongarch/include/asm/Kbuild +++ b/arch/loongarch/include/asm/Kbuild @@ -26,4 +26,3 @@ generic-y += poll.h generic-y += param.h generic-y += posix_types.h generic-y += resource.h -generic-y += kvm_para.h diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index a74e221b73bcc1d7fe435e4d585ba790b67bdeee..b7ebf2aa217a4b98f3c64c5dd6f7845172974b04 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -42,17 +42,18 @@ enum reg1i21_op { }; enum reg2i12_op { - addiw_op = 0x0a, - addid_op = 0x0b, - lu52id_op = 0x0c, - ldb_op = 0xa0, - ldh_op = 0xa1, - ldw_op = 0xa2, - ldd_op = 0xa3, - stb_op = 0xa4, - sth_op = 0xa5, - stw_op = 0xa6, - std_op = 0xa7, + slti_op = 0x8, sltui_op, addiw_op, addid_op, + lu52id_op, cache_op = 0x18, xvldreplb_op = 0xca, + ldb_op = 0xa0, ldh_op, ldw_op, ldd_op, stb_op, sth_op, + stw_op, std_op, ldbu_op, ldhu_op, ldwu_op, preld_op, + flds_op, fsts_op, fldd_op, fstd_op, vld_op, vst_op, xvld_op, + xvst_op, ldlw_op = 0xb8, ldrw_op, ldld_op, ldrd_op, stlw_op, + strw_op, stld_op, strd_op, vldreplb_op = 0xc2, +}; + +enum reg2i14_op { + llw_op = 0x20, scw_op, lld_op, scd_op, ldptrw_op, stptrw_op, + ldptrd_op, stptrd_op, }; enum reg2i16_op { @@ -65,6 +66,49 @@ enum reg2i16_op { bgeu_op = 0x1b, }; +enum reg3_op { + asrtled_op = 0x2, asrtgtd_op, + addw_op = 0x20, addd_op, subw_op, subd_op, + slt_op, sltu_op, maskeqz_op, masknez_op, + nor_op, and_op, or_op, xor_op, orn_op, + andn_op, sllw_op, srlw_op, sraw_op, slld_op, + srld_op, srad_op, rotrb_op, rotrh_op, + rotrw_op, rotrd_op, mulw_op, mulhw_op, + mulhwu_op, muld_op, mulhd_op, mulhdu_op, + mulwdw_op, mulwdwu_op, divw_op, modw_op, + divwu_op, modwu_op, divd_op, modd_op, + divdu_op, moddu_op, crcwbw_op, + crcwhw_op, crcwww_op, crcwdw_op, crccwbw_op, + crccwhw_op, crccwww_op, crccwdw_op, addu12iw_op, + addu12id_op, + adcb_op = 0x60, adch_op, adcw_op, adcd_op, + sbcb_op, sbch_op, sbcw_op, sbcd_op, + rcrb_op, rcrh_op, rcrw_op, rcrd_op, + ldxb_op = 0x7000, ldxh_op = 0x7008, ldxw_op = 0x7010, ldxd_op = 0x7018, + stxb_op = 0x7020, stxh_op = 0x7028, stxw_op = 0x7030, stxd_op = 0x7038, + ldxbu_op = 0x7040, ldxhu_op = 0x7048, ldxwu_op = 0x7050, + preldx_op = 0x7058, fldxs_op = 0x7060, fldxd_op = 0x7068, + fstxs_op = 0x7070, fstxd_op = 0x7078, vldx_op = 0x7080, + vstx_op = 0x7088, xvldx_op = 0x7090, xvstx_op = 0x7098, + amswapw_op = 0x70c0, amswapd_op, amaddw_op, amaddd_op, amandw_op, + amandd_op, amorw_op, amord_op, amxorw_op, amxord_op, ammaxw_op, + ammaxd_op, amminw_op, ammind_op, ammaxwu_op, ammaxdu_op, + amminwu_op, ammindu_op, amswap_dbw_op, amswap_dbd_op, amadd_dbw_op, + amadd_dbd_op, amand_dbw_op, amand_dbd_op, amor_dbw_op, amor_dbd_op, + amxor_dbw_op, amxor_dbd_op, ammax_dbw_op, ammax_dbd_op, ammin_dbw_op, + ammin_dbd_op, ammax_dbwu_op, ammax_dbdu_op, ammin_dbwu_op, + ammin_dbdu_op, fldgts_op = 0x70e8, fldgtd_op, + fldles_op, fldled_op, fstgts_op, fstgtd_op, fstles_op, fstled_op, + ldgtb_op, ldgth_op, ldgtw_op, ldgtd_op, ldleb_op, ldleh_op, ldlew_op, + ldled_op, stgtb_op, stgth_op, stgtw_op, stgtd_op, stleb_op, stleh_op, + stlew_op, stled_op, +}; + +enum reg2_op { + iocsrrdb_op = 0x19200, iocsrrdh_op, iocsrrdw_op, iocsrrdd_op, + iocsrwrb_op, iocsrwrh_op, iocsrwrw_op, iocsrwrd_op, +}; + struct reg0i26_format { unsigned int immediate_h : 10; unsigned int immediate_l : 16; @@ -84,6 +128,12 @@ struct reg1i21_format { unsigned int opcode : 6; }; +struct reg2_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int opcode : 22; +}; + struct reg2i12_format { unsigned int rd : 5; unsigned int rj : 5; @@ -91,6 +141,18 @@ struct reg2i12_format { unsigned int opcode : 10; }; +struct reg2i14_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int simmediate : 14; + unsigned int opcode : 8; +}; + +struct reg0i15_format { + unsigned int simmediate : 15; + unsigned int opcode : 17; +}; + struct reg2i16_format { unsigned int rd : 5; unsigned int rj : 5; @@ -98,13 +160,32 @@ struct reg2i16_format { unsigned int opcode : 6; }; +struct reg3_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int rk : 5; + unsigned int opcode : 17; +}; + +struct reg2csr_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int csr : 14; + unsigned int opcode : 8; +}; + union loongarch_instruction { unsigned int word; struct reg0i26_format reg0i26_format; struct reg1i20_format reg1i20_format; struct reg1i21_format reg1i21_format; + struct reg3_format reg3_format; + struct reg2_format reg2_format; struct reg2i12_format reg2i12_format; + struct reg2i14_format reg2i14_format; struct reg2i16_format reg2i16_format; + struct reg2csr_format reg2csr_format; + struct reg0i15_format reg0i15_format; }; #define LOONGARCH_INSN_SIZE sizeof(union loongarch_instruction) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 56319843ddfda5142c48ef8c90aa5d87fa082857..8f181f8cfa8cf467b9f337d8a61becdffc94053f 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -76,6 +76,8 @@ struct kvm_vcpu_stat { u64 huge_merge_exits; u64 halt_successful_poll; u64 halt_attempted_poll; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; u64 halt_poll_invalid; u64 halt_wakeup; }; @@ -158,23 +160,6 @@ enum emulation_result { EMULATE_DO_IOCSR, /* handle IOCSR request */ }; -#define KVM_NR_MEM_OBJS 4 -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ -struct kvm_mmu_memory_cache { - int nobjs; - void *objects[KVM_NR_MEM_OBJS]; -}; - -#if defined(CONFIG_CPU_HAS_LASX) -#define FPU_ALIGN __aligned(32) -#elif defined(CONFIG_CPU_HAS_LSX) -#define FPU_ALIGN __aligned(16) -#else -#define FPU_ALIGN -#endif #define KVM_LARCH_FPU (0x1 << 0) #define KVM_LARCH_LSX (0x1 << 1) #define KVM_LARCH_LASX (0x1 << 2) @@ -316,7 +301,7 @@ enum _kvm_fault_result { #define KVM_ARCH_WANT_MMU_NOTIFIER int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable); -void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); @@ -344,7 +329,7 @@ static inline bool kvm_is_ifetch_fault(struct kvm_vcpu_arch *arch) static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, - struct kvm_memory_slot *free, struct kvm_memory_slot *dont) {} + struct kvm_memory_slot *slot) {} static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h new file mode 100644 index 0000000000000000000000000000000000000000..bd9bb0bf0f70397520df6c8c0c7e1789b62e1d66 --- /dev/null +++ b/arch/loongarch/include/asm/kvm_para.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_LOONGARCH_KVM_PARA_H +#define _ASM_LOONGARCH_KVM_PARA_H + +/* + * Hypcall code field + */ +#define KVM_HC_CODE_SERIVCE 0x0 +#define KVM_HC_CODE_SWDBG 0x5 +/* + * function id + * 0x00000 ~ 0xfffff Standard Hypervisor Calls + */ +#define KVM_HC_FUNC_FEATURE 0x0 +#define KVM_HC_FUNC_NOTIFY 0x1 +#define KVM_HC_FUNC_IPI 0x2 +/* + * LoongArch support PV feature list + */ +#define KVM_FEATURE_STEAL_TIME 0 +#define KVM_FEATURE_MULTI_IPI 1 +/* + * LoongArch hypcall return code + */ +#define KVM_RET_SUC 1 +#define KVM_RET_NOT_SUPPORTED -1 + +static inline bool kvm_check_and_clear_guest_paused(void) +{ + return false; +} + +static inline unsigned int kvm_arch_para_features(void) +{ + return 0; +} + +static inline unsigned int kvm_arch_para_hints(void) +{ + return 0; +} + +#endif /* _ASM_LOONGARCH_KVM_PARA_H */ diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 8733fc347b3ea412fc3eb6ec06666497b501a0e2..9fdbe934624f6c57cecc0cbaeefae5bd9b0157f2 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -269,3 +270,35 @@ void output_pbe_defines(void) } #endif +void output_kvm_defines(void) +{ + COMMENT(" KVM/LOONGISA Specific offsets. "); + + OFFSET(VCPU_FCSR0, kvm_vcpu_arch, fpu.fcsr); + OFFSET(VCPU_FCC, kvm_vcpu_arch, fpu.fcc); + BLANK(); + + OFFSET(KVM_VCPU_ARCH, kvm_vcpu, arch); + OFFSET(KVM_VCPU_KVM, kvm_vcpu, kvm); + OFFSET(KVM_VCPU_RUN, kvm_vcpu, run); + BLANK(); + + OFFSET(KVM_ARCH_HSTACK, kvm_vcpu_arch, host_stack); + OFFSET(KVM_ARCH_HGP, kvm_vcpu_arch, host_gp); + OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit); + OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd); + OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry); + OFFSET(KVM_ARCH_GPC, kvm_vcpu_arch, pc); + OFFSET(KVM_ARCH_GGPR, kvm_vcpu_arch, gprs); + OFFSET(KVM_ARCH_HESTAT, kvm_vcpu_arch, host_estat); + OFFSET(KVM_ARCH_HBADV, kvm_vcpu_arch, badv); + OFFSET(KVM_ARCH_HBADI, kvm_vcpu_arch, badi); + OFFSET(KVM_ARCH_ISHYPCALL, kvm_vcpu_arch, is_hypcall); + OFFSET(KVM_ARCH_HECFG, kvm_vcpu_arch, host_ecfg); + OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry); + OFFSET(KVM_ARCH_HPERCPU, kvm_vcpu_arch, host_percpu); + + OFFSET(KVM_GPGD, kvm, arch.gpa_mm.pgd); + BLANK(); +} + diff --git a/arch/loongarch/kvm/csr.c b/arch/loongarch/kvm/csr.c index 763b00719469d40ec7029ee2ef8ce23c3ac6634a..09ec781fe79aeeed4759be818f450157bf7fe602 100644 --- a/arch/loongarch/kvm/csr.c +++ b/arch/loongarch/kvm/csr.c @@ -9,15 +9,15 @@ #include "kvmcpu.h" #include "intc/ls3a_ipi.h" #include "intc/ls3a_ext_irq.h" -#include "ls_irq.h" #include "kvm_compat.h" #include "kvmcsr.h" +#include "irq.h" -#define CASE_READ_SW_GCSR(csr, regid, csrid) \ - do { \ - if (regid == csrid) { \ - return kvm_read_sw_gcsr(csr, csrid); \ - } \ +#define CASE_READ_SW_GCSR(csr, regid, csrid) \ + do { \ + if (regid == csrid) { \ + return kvm_read_sw_gcsr(csr, csrid); \ + } \ } while (0) unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid) @@ -50,11 +50,11 @@ unsigned long _kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid) return val; } -#define CASE_WRITE_SW_GCSR(csr, regid, csrid, val) \ - do { \ +#define CASE_WRITE_SW_GCSR(csr, regid, csrid, val) \ + do { \ if (regid == csrid) { \ - kvm_write_sw_gcsr(csr, csrid, val); \ - return ; \ + kvm_write_sw_gcsr(csr, csrid, val); \ + return ; \ } \ } while (0) @@ -89,12 +89,12 @@ void _kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, csrid, vcpu->arch.pc); } -#define CASE_CHANGE_SW_GCSR(csr, regid, csrid, mask, val) \ - do { \ - if (regid == csrid) { \ - kvm_change_sw_gcsr(csr, csrid, mask, val); \ - return ; \ - } \ +#define CASE_CHANGE_SW_GCSR(csr, regid, csrid, mask, val) \ + do { \ + if (regid == csrid) { \ + kvm_change_sw_gcsr(csr, csrid, mask, val); \ + return ; \ + } \ } while (0) void _kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, diff --git a/arch/loongarch/kvm/entry.S b/arch/loongarch/kvm/entry.S index 5a51a1b026e9c325ca93b39bd6d69e39c9d4c2c7..cc0856af60d94fa392cc577c6a52676204981b14 100644 --- a/arch/loongarch/kvm/entry.S +++ b/arch/loongarch/kvm/entry.S @@ -221,7 +221,7 @@ SYM_FUNC_START(kvm_exit_entry) or a2, s1, zero KVM_LONG_ADDI a2, a2, KVM_VCPU_ARCH - andi t0, v0, RESUME_HOST + andi t0, a0, RESUME_HOST bnez t0, ret_to_host INT_S zero, a2, KVM_ARCH_ISHYPCALL @@ -236,8 +236,8 @@ ret_to_guest: ret_to_host: KVM_LONG_L a2, a2, KVM_ARCH_HSTACK addi.d a2, a2, -PT_SIZE - srai.w a3, v0, 2 - or v0, a3, zero + srai.w a3, a0, 2 + or a0, a3, zero kvm_restore_host_gpr a2 jirl zero, ra, 0 SYM_FUNC_END(kvm_exit_entry) diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index c6abcc1e884fadfe14a0a000de16db00c2d6cba5..5653e082d43ecedfc4b3d40d0f3b6d021c6b8a80 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -18,11 +18,8 @@ #include #include #include -#include #include "kvmcpu.h" #include -#include - #include "trace.h" #include "kvm_compat.h" #include "kvmcsr.h" @@ -337,7 +334,9 @@ static int _kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu) return RESUME_HOST; } +#ifdef CONFIG_CPU_HAS_LSX kvm_own_lsx(vcpu); +#endif return RESUME_GUEST; } @@ -373,8 +372,9 @@ static int _kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu) return RESUME_HOST; } +#ifdef CONFIG_CPU_HAS_LASX kvm_own_lasx(vcpu); - +#endif return RESUME_GUEST; } @@ -459,27 +459,27 @@ static int _kvm_handle_debug(struct kvm_vcpu *vcpu) kvm_csr_writeq(fwps, KVM_CSR_FWPS); if (mwps & 0xff) kvm_csr_writeq(mwps, KVM_CSR_MWPS); - vcpu->run->debug.arch.exception = EXCCODE_WATCH; + vcpu->run->debug.arch.exception = KVM_EXCCODE_WATCH; vcpu->run->debug.arch.fwps = fwps; vcpu->run->debug.arch.mwps = mwps; vcpu->run->exit_reason = KVM_EXIT_DEBUG; return RESUME_HOST; } -static exit_handle_fn _kvm_fault_tables[EXCCODE_INT_START] = { - [EXCCODE_TLBL] = _kvm_handle_read_fault, - [EXCCODE_TLBS] = _kvm_handle_write_fault, - [EXCCODE_TLBI] = _kvm_handle_read_fault, - [EXCCODE_TLBM] = _kvm_handle_write_fault, - [EXCCODE_TLBRI] = _kvm_handle_read_fault, - [EXCCODE_TLBXI] = _kvm_handle_read_fault, - [EXCCODE_FPDIS] = _kvm_handle_fpu_disabled, - [EXCCODE_LSXDIS] = _kvm_handle_lsx_disabled, - [EXCCODE_LASXDIS] = _kvm_handle_lasx_disabled, - [EXCCODE_WATCH] = _kvm_handle_debug, - [EXCCODE_GSPR] = _kvm_handle_gspr, - [EXCCODE_HYP] = _kvm_handle_hypcall, - [EXCCODE_GCM] = _kvm_handle_gcm, +static exit_handle_fn _kvm_fault_tables[KVM_INT_START] = { + [KVM_EXCCODE_TLBL] = _kvm_handle_read_fault, + [KVM_EXCCODE_TLBS] = _kvm_handle_write_fault, + [KVM_EXCCODE_TLBI] = _kvm_handle_read_fault, + [KVM_EXCCODE_TLBM] = _kvm_handle_write_fault, + [KVM_EXCCODE_TLBRI] = _kvm_handle_read_fault, + [KVM_EXCCODE_TLBXI] = _kvm_handle_read_fault, + [KVM_EXCCODE_FPDIS] = _kvm_handle_fpu_disabled, + [KVM_EXCCODE_LSXDIS] = _kvm_handle_lsx_disabled, + [KVM_EXCCODE_LASXDIS] = _kvm_handle_lasx_disabled, + [KVM_EXCCODE_WATCH] = _kvm_handle_debug, + [KVM_EXCCODE_GSPR] = _kvm_handle_gspr, + [KVM_EXCCODE_HYP] = _kvm_handle_hypcall, + [KVM_EXCCODE_GCM] = _kvm_handle_gcm, }; int _kvm_handle_fault(struct kvm_vcpu *vcpu, int fault) @@ -491,7 +491,7 @@ void _kvm_init_fault(void) { int i; - for (i = 0; i < EXCCODE_INT_START; i++) + for (i = 0; i < KVM_INT_START; i++) if (!_kvm_fault_tables[i]) _kvm_fault_tables[i] = _kvm_fault_ni; } diff --git a/arch/loongarch/kvm/hypcall.c b/arch/loongarch/kvm/hypcall.c index 180d110a08a7137087205d1069870cd7587ee9e6..aaf3a07f23f0f0f253919d8039aa7c561702c10b 100644 --- a/arch/loongarch/kvm/hypcall.c +++ b/arch/loongarch/kvm/hypcall.c @@ -7,8 +7,9 @@ #include #include #include -#include #include "intc/ls3a_ipi.h" +#include "kvm_compat.h" + int kvm_virt_ipi(struct kvm_vcpu *vcpu) { @@ -16,9 +17,9 @@ int kvm_virt_ipi(struct kvm_vcpu *vcpu) u64 ipi_bitmap; unsigned int min, action, cpu; - ipi_bitmap = vcpu->arch.gprs[REG_A1]; - min = vcpu->arch.gprs[REG_A2]; - action = vcpu->arch.gprs[REG_A3]; + ipi_bitmap = vcpu->arch.gprs[KVM_REG_A1]; + min = vcpu->arch.gprs[KVM_REG_A2]; + action = vcpu->arch.gprs[KVM_REG_A3]; if (ipi_bitmap) { cpu = find_first_bit((void *)&ipi_bitmap, BITS_PER_LONG); @@ -37,9 +38,9 @@ int kvm_save_notify(struct kvm_vcpu *vcpu) int ret = 0; - num = vcpu->arch.gprs[REG_A0]; - id = vcpu->arch.gprs[REG_A1]; - data = vcpu->arch.gprs[REG_A2]; + num = vcpu->arch.gprs[KVM_REG_A0]; + id = vcpu->arch.gprs[KVM_REG_A1]; + data = vcpu->arch.gprs[KVM_REG_A2]; switch (id) { case KVM_FEATURE_STEAL_TIME: @@ -59,7 +60,7 @@ int kvm_save_notify(struct kvm_vcpu *vcpu) static int _kvm_pv_feature(struct kvm_vcpu *vcpu) { - int feature = vcpu->arch.gprs[REG_A1]; + int feature = vcpu->arch.gprs[KVM_REG_A1]; int ret = KVM_RET_NOT_SUPPORTED; switch (feature) { case KVM_FEATURE_STEAL_TIME: @@ -80,7 +81,7 @@ static int _kvm_pv_feature(struct kvm_vcpu *vcpu) */ int _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu) { - unsigned long func = vcpu->arch.gprs[REG_A0]; + unsigned long func = vcpu->arch.gprs[KVM_REG_A0]; int hyp_ret = KVM_RET_NOT_SUPPORTED; switch (func) { @@ -98,7 +99,7 @@ int _kvm_handle_pv_hcall(struct kvm_vcpu *vcpu) break; }; - vcpu->arch.gprs[REG_V0] = hyp_ret; + vcpu->arch.gprs[KVM_REG_A0] = hyp_ret; return RESUME_GUEST; } diff --git a/arch/loongarch/kvm/intc/ls7a_irq.h b/arch/loongarch/kvm/intc/ls7a_irq.h index fad85d55240dbe6a8606a3b721edfbdc66a0c6ad..0c91b63bf88f569b36daa90c0d6d9894fa085391 100644 --- a/arch/loongarch/kvm/intc/ls7a_irq.h +++ b/arch/loongarch/kvm/intc/ls7a_irq.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include "kvmcpu.h" diff --git a/arch/loongarch/kvm/ls_irq.h b/arch/loongarch/kvm/irq.h similarity index 100% rename from arch/loongarch/kvm/ls_irq.h rename to arch/loongarch/kvm/irq.h diff --git a/arch/loongarch/kvm/kvm_compat.c b/arch/loongarch/kvm/kvm_compat.c index a8931cf0ab47f8e092f5b6f2c917ca6c289d67cf..277d7760aa11a5071c007c37c99058ee86a9a050 100644 --- a/arch/loongarch/kvm/kvm_compat.c +++ b/arch/loongarch/kvm/kvm_compat.c @@ -21,7 +21,7 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) return; } -#else +#elif (LINUX_VERSION_CODE == KERNEL_VERSION(5, 4, 0)) int kvm_arch_check_processor_compat(void) { return 0; @@ -54,5 +54,32 @@ int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) { return _kvm_set_spte_hva(kvm, hva, pte); } -#endif +#else +int kvm_arch_check_processor_compat(void *opaque) +{ + return 0; +} +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + return _kvm_set_spte_hva(kvm, hva, pte); +} + +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + +} + +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) +{ + return 0; +} + +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ +#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL + kvm_flush_remote_tlbs (kvm); +#endif +} +#endif diff --git a/arch/loongarch/kvm/kvm_compat.h b/arch/loongarch/kvm/kvm_compat.h index ff68ab1d8aba8bf3e6aaf9567ecbd2417dabfbec..1da54b2d80ecea367240534b446c8e53dae89b7d 100644 --- a/arch/loongarch/kvm/kvm_compat.h +++ b/arch/loongarch/kvm/kvm_compat.h @@ -8,9 +8,62 @@ #include #if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) #include +#else +#include #endif #endif +#define KVM_REG_A0 0x4 +#define KVM_REG_A1 0x5 +#define KVM_REG_A2 0x6 +#define KVM_REG_A3 0x7 +/* + * ExStatus.ExcCode + */ +#define KVM_EXCCODE_RSV 0 /* Reserved */ +#define KVM_EXCCODE_TLBL 1 /* TLB miss on a load */ +#define KVM_EXCCODE_TLBS 2 /* TLB miss on a store */ +#define KVM_EXCCODE_TLBI 3 /* TLB miss on a ifetch */ +#define KVM_EXCCODE_TLBM 4 /* TLB modified fault */ +#define KVM_EXCCODE_TLBRI 5 /* TLB Read-Inhibit exception */ +#define KVM_EXCCODE_TLBXI 6 /* TLB Execution-Inhibit exception */ +#define KVM_EXCCODE_TLBPE 7 /* TLB Privilege Error */ +#define KVM_EXCCODE_ADE 8 /* Address Error */ +#define KVM_EXCCODE_ALE 9 /* Unalign Access */ +#define KVM_EXCCODE_OOB 10 /* Out of bounds */ +#define KVM_EXCCODE_SYS 11 /* System call */ +#define KVM_EXCCODE_BP 12 /* Breakpoint */ +#define KVM_EXCCODE_INE 13 /* Inst. Not Exist */ +#define KVM_EXCCODE_IPE 14 /* Inst. Privileged Error */ +#define KVM_EXCCODE_FPDIS 15 /* FPU Disabled */ +#define KVM_EXCCODE_LSXDIS 16 /* LSX Disabled */ +#define KVM_EXCCODE_LASXDIS 17 /* LASX Disabled */ +#define KVM_EXCCODE_FPE 18 /* Floating Point Exception */ +#define KVM_EXCCODE_WATCH 19 /* Watch address reference */ +#define KVM_EXCCODE_BTDIS 20 /* Binary Trans. Disabled */ +#define KVM_EXCCODE_BTE 21 /* Binary Trans. Exception */ +#define KVM_EXCCODE_GSPR 22 /* Guest Privileged Error */ +#define KVM_EXCCODE_HYP 23 /* Hypercall */ +#define KVM_EXCCODE_GCM 24 /* Guest CSR modified */ + +#define KVM_INT_START 64 +#define KVM_INT_SIP0 64 +#define KVM_INT_SIP1 65 +#define KVM_INT_IP0 66 +#define KVM_INT_IP1 67 +#define KVM_INT_IP2 68 +#define KVM_INT_IP3 69 +#define KVM_INT_IP4 70 +#define KVM_INT_IP5 71 +#define KVM_INT_IP6 72 +#define KVM_INT_IP7 73 +#define KVM_INT_PC 74 /* Performance Counter */ +#define KVM_INT_TIMER 75 +#define KVM_INT_IPI 76 +#define KVM_INT_NMI 77 +#define KVM_INT_END 78 +#define KVM_INT_NUM (KVM_INT_END - KVM_INT_START) + #define KVM_CSR_CRMD 0x0 /* Current mode info */ #define KVM_CRMD_WE_SHIFT 9 #define KVM_CRMD_WE (_ULCAST_(0x1) << KVM_CRMD_WE_SHIFT) @@ -223,6 +276,14 @@ #define KVM_CSR_PERFCNTR2 0x205 /* 64 perf event 2 count value */ #define KVM_CSR_PERFCTRL3 0x206 /* 32 perf event 3 config */ #define KVM_CSR_PERFCNTR3 0x207 /* 64 perf event 3 count value */ +#define KVM_PERFCTRL_PLV0 (_ULCAST_(1) << 16) +#define KVM_PERFCTRL_PLV1 (_ULCAST_(1) << 17) +#define KVM_PERFCTRL_PLV2 (_ULCAST_(1) << 18) +#define KVM_PERFCTRL_PLV3 (_ULCAST_(1) << 19) +#define KVM_PERFCTRL_IE (_ULCAST_(1) << 20) +#define KVM_PERFCTRL_GMOD (_ULCAST_(3) << 21) +#define KVM_PERFCTRL_EVENT 0x3ff + #define KVM_CSR_MWPC 0x300 /* data breakpoint config */ #define KVM_CSR_MWPS 0x301 /* data breakpoint status */ #define KVM_CSR_FWPC 0x380 /* instruction breakpoint config */ @@ -652,10 +713,10 @@ kvm_change_csr_gstat(unsigned long change, unsigned long val) { unsigned long res, new; - res = read_csr_gstat(); + res = kvm_read_csr_gstat(); new = res & ~change; new |= (val & change); - write_csr_gstat(new); + kvm_write_csr_gstat(new); return res; } @@ -665,10 +726,10 @@ kvm_change_csr_gcfg(unsigned long change, unsigned long val) { unsigned long res, new; - res = read_csr_gcfg(); + res = kvm_read_csr_gcfg(); new = res & ~change; new |= (val & change); - write_csr_gcfg(new); + kvm_write_csr_gcfg(new); return res; } @@ -681,6 +742,9 @@ kvm_change_csr_gcfg(unsigned long change, unsigned long val) #endif +/* Device Control API on vcpu fd */ +#define KVM_LARCH_VCPU_PVTIME_CTRL 2 +#define KVM_LARCH_VCPU_PVTIME_IPA 0 #if (_LOONGARCH_SZLONG == 32) #define KVM_LONG_ADD add.w diff --git a/arch/loongarch/kvm/kvmcsr.h b/arch/loongarch/kvm/kvmcsr.h index 303303a8a2456f1e5ae53e7c409b925c6dbd871b..24a84a3f72cd19a4e6df37cff1861496039c9ae8 100644 --- a/arch/loongarch/kvm/kvmcsr.h +++ b/arch/loongarch/kvm/kvmcsr.h @@ -7,7 +7,6 @@ #define __LOONGARCH_KVM_CSR_H__ #include #include "kvmcpu.h" -#include #include #include @@ -92,17 +91,17 @@ int _kvm_init_iocsr(struct kvm *kvm); int _kvm_set_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp); int _kvm_get_iocsr(struct kvm *kvm, struct kvm_iocsr_entry *__user argp); -#define KVM_PMU_PLV_ENABLE (CSR_PERFCTRL_PLV0 | \ - CSR_PERFCTRL_PLV1 | \ - CSR_PERFCTRL_PLV2 | \ - CSR_PERFCTRL_PLV3) +#define KVM_PMU_PLV_ENABLE (KVM_PERFCTRL_PLV0 | \ + KVM_PERFCTRL_PLV1 | \ + KVM_PERFCTRL_PLV2 | \ + KVM_PERFCTRL_PLV3) #define CASE_WRITE_HW_PMU(vcpu, csr, id, csrid, v) \ do { \ if (csrid == id) { \ if (v & KVM_PMU_PLV_ENABLE) { \ - kvm_write_csr_gcfg(kvm_read_csr_gcfg() | CSR_GCFG_GPERF); \ - kvm_write_hw_gcsr(csr, csrid, v | CSR_PERFCTRL_GMOD); \ + kvm_write_csr_gcfg(kvm_read_csr_gcfg() | KVM_GCFG_GPERF); \ + kvm_write_hw_gcsr(csr, csrid, v | KVM_PERFCTRL_GMOD); \ vcpu->arch.aux_inuse |= KVM_LARCH_PERF; \ return ; \ } else { \ diff --git a/arch/loongarch/kvm/loongarch.c b/arch/loongarch/kvm/loongarch.c index 2268176f5cc6d925cfc8b7f700bc4f30f4832886..03da5959aaebebfd9023ffd4a7ab3c7dfa58f30c 100644 --- a/arch/loongarch/kvm/loongarch.c +++ b/arch/loongarch/kvm/loongarch.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -29,7 +28,6 @@ #include "kvmcpu.h" #include #include -#include #include "intc/ls3a_ipi.h" #include "intc/ls7a_irq.h" @@ -44,59 +42,55 @@ #define KVM_LOONGARCH_VERSION 1 #define CREATE_TRACE_POINTS #include "trace.h" -#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x) -#define VM_STAT(x) offsetof(struct kvm, stat.x) struct kvm_stats_debugfs_item vcpu_debugfs_entries[] = { - { "idle", VCPU_STAT(idle_exits), KVM_STAT_VCPU}, - { "signal", VCPU_STAT(signal_exits), KVM_STAT_VCPU}, - { "interrupt", VCPU_STAT(int_exits), KVM_STAT_VCPU}, - { "tlbmiss_ld", VCPU_STAT(excep_exits[EXCCODE_TLBL]), KVM_STAT_VCPU}, - { "tlbmiss_st", VCPU_STAT(excep_exits[EXCCODE_TLBS]), KVM_STAT_VCPU}, - { "tlb_ifetch", VCPU_STAT(excep_exits[EXCCODE_TLBI]), KVM_STAT_VCPU}, - { "tlbmod", VCPU_STAT(excep_exits[EXCCODE_TLBM]), KVM_STAT_VCPU}, - { "tlbri", VCPU_STAT(excep_exits[EXCCODE_TLBRI]), KVM_STAT_VCPU}, - { "tlbxi", VCPU_STAT(excep_exits[EXCCODE_TLBXI]), KVM_STAT_VCPU}, - { "fp_dis", VCPU_STAT(excep_exits[EXCCODE_FPDIS]), KVM_STAT_VCPU}, - { "lsx_dis", VCPU_STAT(excep_exits[EXCCODE_LSXDIS]), KVM_STAT_VCPU}, - { "lasx_dis", VCPU_STAT(excep_exits[EXCCODE_LASXDIS]), KVM_STAT_VCPU}, - { "fpe", VCPU_STAT(excep_exits[EXCCODE_FPE]), KVM_STAT_VCPU}, - { "watch", VCPU_STAT(excep_exits[EXCCODE_WATCH]), KVM_STAT_VCPU}, - { "gspr", VCPU_STAT(excep_exits[EXCCODE_GSPR]), KVM_STAT_VCPU}, - { "gcm", VCPU_STAT(excep_exits[EXCCODE_GCM]), KVM_STAT_VCPU}, - { "hc", VCPU_STAT(excep_exits[EXCCODE_HYP]), KVM_STAT_VCPU}, - - { "rdcsr_cpu_feature", VCPU_STAT(rdcsr_cpu_feature_exits), KVM_STAT_VCPU }, - { "rdcsr_misc_func", VCPU_STAT(rdcsr_misc_func_exits), KVM_STAT_VCPU }, - { "rdcsr_ipi_access", VCPU_STAT(rdcsr_ipi_access_exits), KVM_STAT_VCPU }, - { "cpucfg", VCPU_STAT(cpucfg_exits), KVM_STAT_VCPU }, - { "huge_dec", VCPU_STAT(huge_dec_exits), KVM_STAT_VCPU }, - { "huge_thp", VCPU_STAT(huge_thp_exits), KVM_STAT_VCPU }, - { "huge_adj", VCPU_STAT(huge_adjust_exits), KVM_STAT_VCPU }, - { "huge_set", VCPU_STAT(huge_set_exits), KVM_STAT_VCPU }, - { "huge_merg", VCPU_STAT(huge_merge_exits), KVM_STAT_VCPU }, - - { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU }, - { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU }, - { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid), KVM_STAT_VCPU }, - { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU }, + VCPU_STAT("idle", idle_exits), + VCPU_STAT("signal", signal_exits), + VCPU_STAT("interrupt", int_exits), + VCPU_STAT("rdcsr_cpu_feature", rdcsr_cpu_feature_exits), + VCPU_STAT("rdcsr_misc_func", rdcsr_misc_func_exits), + VCPU_STAT("rdcsr_ipi_access", rdcsr_ipi_access_exits), + VCPU_STAT("cpucfg", cpucfg_exits), + VCPU_STAT("huge_dec", huge_dec_exits), + VCPU_STAT("huge_thp", huge_thp_exits), + VCPU_STAT("huge_adj", huge_adjust_exits), + VCPU_STAT("huge_set", huge_set_exits), + VCPU_STAT("huge_merg", huge_merge_exits), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("tlbmiss_ld", excep_exits[KVM_EXCCODE_TLBL]), + VCPU_STAT("tlbmiss_st", excep_exits[KVM_EXCCODE_TLBS]), + VCPU_STAT("tlb_ifetch", excep_exits[KVM_EXCCODE_TLBI]), + VCPU_STAT("tlbmod", excep_exits[KVM_EXCCODE_TLBM]), + VCPU_STAT("tlbri", excep_exits[KVM_EXCCODE_TLBRI]), + VCPU_STAT("tlbxi", excep_exits[KVM_EXCCODE_TLBXI]), + VCPU_STAT("fp_dis", excep_exits[KVM_EXCCODE_FPDIS]), + VCPU_STAT("lsx_dis", excep_exits[KVM_EXCCODE_LSXDIS]), + VCPU_STAT("lasx_dis", excep_exits[KVM_EXCCODE_LASXDIS]), + VCPU_STAT("fpe", excep_exits[KVM_EXCCODE_FPE]), + VCPU_STAT("watch", excep_exits[KVM_EXCCODE_WATCH]), + VCPU_STAT("gspr", excep_exits[KVM_EXCCODE_GSPR]), + VCPU_STAT("gcm", excep_exits[KVM_EXCCODE_GCM]), + VCPU_STAT("hc", excep_exits[KVM_EXCCODE_HYP]), {NULL} }; struct kvm_stats_debugfs_item debugfs_entries[] = { - { "remote_tlb_flush", VM_STAT(remote_tlb_flush), KVM_STAT_VM }, - { "pip_read_exits", VM_STAT(pip_read_exits), KVM_STAT_VM }, - { "pip_write_exits", VM_STAT(pip_write_exits), KVM_STAT_VM }, - { "vm_ioctl_irq_line", VM_STAT(vm_ioctl_irq_line), KVM_STAT_VM }, - { "ls7a_ioapic_update", VM_STAT(ls7a_ioapic_update), KVM_STAT_VM }, - { "ls7a_ioapic_set_irq", VM_STAT(ls7a_ioapic_set_irq), KVM_STAT_VM }, - { "ls7a_msi_irq", VM_STAT(ls7a_msi_irq), KVM_STAT_VM }, - { "ioapic_reg_write", VM_STAT(ioapic_reg_write), KVM_STAT_VM }, - { "ioapic_reg_read", VM_STAT(ioapic_reg_read), KVM_STAT_VM }, - { "set_ls7a_ioapic", VM_STAT(set_ls7a_ioapic), KVM_STAT_VM }, - { "get_ls7a_ioapic", VM_STAT(get_ls7a_ioapic), KVM_STAT_VM }, - { "set_ls3a_ext_irq", VM_STAT(set_ls3a_ext_irq), KVM_STAT_VM }, - { "get_ls3a_ext_irq", VM_STAT(get_ls3a_ext_irq), KVM_STAT_VM }, - { "ls3a_ext_irq", VM_STAT(trigger_ls3a_ext_irq), KVM_STAT_VM }, + VM_STAT("remote_tlb_flush", remote_tlb_flush), + VM_STAT("pip_read_exits", pip_read_exits), + VM_STAT("pip_write_exits", pip_write_exits), + VM_STAT("vm_ioctl_irq_line", vm_ioctl_irq_line), + VM_STAT("ls7a_ioapic_update", ls7a_ioapic_update), + VM_STAT("ls7a_ioapic_set_irq", ls7a_ioapic_set_irq), + VM_STAT("ls7a_msi_irq", ls7a_msi_irq), + VM_STAT("ioapic_reg_write", ioapic_reg_write), + VM_STAT("ioapic_reg_read", ioapic_reg_read), + VM_STAT("set_ls7a_ioapic", set_ls7a_ioapic), + VM_STAT("get_ls7a_ioapic", get_ls7a_ioapic), + VM_STAT("set_ls3a_ext_irq", set_ls3a_ext_irq), + VM_STAT("get_ls3a_ext_irq", get_ls3a_ext_irq), + VM_STAT("ls3a_ext_irq", trigger_ls3a_ext_irq), {NULL} }; @@ -133,7 +127,7 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; } -#ifdef CONFIG_PARAVIRT +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING void kvm_update_stolen_time(struct kvm_vcpu *vcpu) { struct kvm_host_map map; @@ -249,14 +243,10 @@ int kvm_arch_hardware_enable(void) * TOE=0: Trap on Exception. * TIT=0: Trap on Timer. */ - if (cpu_has_gcip_all) - gcfg |= KVM_GCFG_GCI_SECURE; - if (cpu_has_matc_root) - gcfg |= KVM_GCFG_MATC_ROOT; - + gcfg |= KVM_GCFG_GCI_SECURE; + gcfg |= KVM_GCFG_MATC_ROOT; gcfg |= KVM_GCFG_TIT; kvm_write_csr_gcfg(gcfg); - kvm_flush_tlb_all(); /* Enable using TGID */ @@ -278,7 +268,7 @@ void kvm_arch_hardware_disable(void) kvm_flush_tlb_all(); } -int kvm_arch_hardware_setup(void) +int kvm_arch_hardware_setup(void *opaque) { return 0; } @@ -299,56 +289,17 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return 0; } -static int lvcpu_stat_get(void *address, u64 *val) -{ - *val = *(u64 *)address; - return 0; -} -DEFINE_SIMPLE_ATTRIBUTE(lvcpu_stat_fops, lvcpu_stat_get, NULL, "%llu\n"); - -static int vcpu_pid_get(void *arg, u64 *val) -{ - struct kvm_vcpu *vcpu = (struct kvm_vcpu *)arg; - if (vcpu) - *val = pid_vnr(vcpu->pid); - return 0; -} -DEFINE_SIMPLE_ATTRIBUTE(vcpu_pid_fops, vcpu_pid_get, NULL, "%llu\n"); - -bool kvm_arch_has_vcpu_debugfs(void) -{ - return true; -} - -int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu) -{ - struct kvm_stats_debugfs_item *p; - debugfs_create_file("pid", 0444, vcpu->debugfs_dentry, vcpu, &vcpu_pid_fops); - for (p = vcpu_debugfs_entries; p->name && p->kind == KVM_STAT_VCPU; ++p) { - debugfs_create_file(p->name, 0444, vcpu->debugfs_dentry, - (void *)vcpu + p->offset, &lvcpu_stat_fops); - } - - return 0; -} - static void kvm_free_vcpus(struct kvm *kvm) { unsigned int i; struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) { - kvm_arch_vcpu_free(vcpu); - } - - mutex_lock(&kvm->lock); - - for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) + kvm_vcpu_destroy(vcpu); kvm->vcpus[i] = NULL; + } atomic_set(&kvm->online_vcpus, 0); - - mutex_unlock(&kvm->lock); } void kvm_arch_destroy_vm(struct kvm *kvm) @@ -496,7 +447,7 @@ static int _kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) ret = RESUME_GUEST; } -#ifdef CONFIG_PARAVIRT +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) kvm_update_stolen_time(vcpu); #endif @@ -547,71 +498,82 @@ static int _kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) return ret; } -struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) +/* low level hrtimer wake routine */ +static enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer) { - int err; - struct kvm_vcpu *vcpu = kzalloc(sizeof(struct kvm_vcpu), GFP_KERNEL); + struct kvm_vcpu *vcpu; - if (!vcpu) { - err = -ENOMEM; - goto out; - } + vcpu = container_of(timer, struct kvm_vcpu, arch.swtimer); + _kvm_queue_irq(vcpu, LARCH_INT_TIMER); + kvm_vcpu_wake_up(vcpu); + return kvm_count_timeout(vcpu); +} - err = kvm_vcpu_init(vcpu, kvm, id); +static void _kvm_vcpu_init(struct kvm_vcpu *vcpu) +{ + int i; - if (err) - goto out_free_cpu; + for_each_possible_cpu(i) + vcpu->arch.vpid[i] = 0; - kvm->arch.online_vcpus = id + 1; + hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC, + HRTIMER_MODE_ABS_PINNED); + vcpu->arch.swtimer.function = kvm_swtimer_wakeup; + vcpu->arch.fpu_enabled = true; + vcpu->arch.lsx_enabled = true; +} +int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) +{ vcpu->arch.host_eentry = kvm_csr_readq(KVM_CSR_EENTRY); vcpu->arch.guest_eentry = (unsigned long)kvm_exception_entry; vcpu->arch.vcpu_run = kvm_enter_guest; vcpu->arch.handle_exit = _kvm_handle_exit; - vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL); + vcpu->arch.host_ecfg = (kvm_read_csr_ecfg() & KVM_ECFG_VS); + /* * kvm all exceptions share one exception entry, and host <-> guest switch * also switch excfg.VS field, keep host excfg.VS info here */ - vcpu->arch.host_ecfg = (kvm_read_csr_ecfg() & KVM_ECFG_VS); - + vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL); if (!vcpu->arch.csr) { - err = -ENOMEM; - goto out_uninit_cpu; + return -ENOMEM; } /* Init */ vcpu->arch.last_sched_cpu = -1; vcpu->arch.last_exec_cpu = -1; + _kvm_vcpu_init(vcpu); + return 0; +} - return vcpu; - -out_uninit_cpu: - kvm_vcpu_uninit(vcpu); - -out_free_cpu: - kfree(vcpu); +static void _kvm_vcpu_uninit(struct kvm_vcpu *vcpu) +{ + int cpu; + struct kvm_context *context; -out: - return ERR_PTR(err); + /* + * If the VCPU is freed and reused as another VCPU, we don't want the + * matching pointer wrongly hanging around in last_vcpu. + */ + for_each_possible_cpu(cpu) { + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + if (context->last_vcpu == vcpu) + context->last_vcpu = NULL; + } } -void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache; - hrtimer_cancel(&vcpu->arch.swtimer); + _kvm_vcpu_uninit(vcpu); - kvm_vcpu_uninit(vcpu); - kvm_mmu_free_memory_caches(vcpu); - kvm_release_pfn(cache->pfn, cache->dirty, cache); + hrtimer_cancel(&vcpu->arch.swtimer); + kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); + if (vcpu->arch.st.guest_addr) + kvm_release_pfn(cache->pfn, cache->dirty, cache); kfree(vcpu->arch.csr); - kfree(vcpu); -} - -void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) -{ - kvm_arch_vcpu_free(vcpu); } #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ KVM_GUESTDBG_USE_SW_BP | KVM_GUESTDBG_SINGLESTEP) @@ -634,10 +596,11 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return ret; } -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { int r = -EINTR; int cpu; + struct kvm_run *run = vcpu->run; vcpu_load(vcpu); @@ -649,7 +612,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) vcpu->mmio_needed = 0; } else if (vcpu->arch.is_hypcall) { /* set return value for hypercall v0 register */ - vcpu->arch.gprs[REG_V0] = run->hypercall.ret; + vcpu->arch.gprs[KVM_REG_A0] = run->hypercall.ret; vcpu->arch.is_hypcall = 0; } @@ -665,7 +628,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) lose_fpu(1); -#ifdef CONFIG_PARAVIRT +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) kvm_update_stolen_time(vcpu); #endif @@ -775,7 +738,7 @@ static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) /* Restore hardware perf csr */ kvm_restore_hw_perf(vcpu); -#ifdef CONFIG_PARAVIRT +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); #endif /* Don't bother restoring registers multiple times unless necessary */ @@ -1302,7 +1265,7 @@ static int _kvm_vcpu_set_attr(struct kvm_vcpu *vcpu, int ret = -ENXIO; switch (attr->group) { -#ifdef CONFIG_PARAVIRT +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING case KVM_LARCH_VCPU_PVTIME_CTRL: ret = _kvm_pvtime_set_attr(vcpu, attr); break; @@ -1321,7 +1284,7 @@ static int _kvm_vcpu_get_attr(struct kvm_vcpu *vcpu, int ret = -ENXIO; switch (attr->group) { -#ifdef CONFIG_PARAVIRT +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING case KVM_LARCH_VCPU_PVTIME_CTRL: ret = _kvm_pvtime_get_attr(vcpu, attr); break; @@ -1340,7 +1303,7 @@ static int _kvm_vcpu_has_attr(struct kvm_vcpu *vcpu, int ret = -ENXIO; switch (attr->group) { -#ifdef CONFIG_PARAVIRT +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING case KVM_LARCH_VCPU_PVTIME_CTRL: ret = _kvm_pvtime_has_attr(vcpu, attr); break; @@ -1479,54 +1442,6 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, return r; } -/** - * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot - * @kvm: kvm instance - * @log: slot id and address to which we copy the log - * - * Steps 1-4 below provide general overview of dirty page logging. See - * kvm_get_dirty_log_protect() function description for additional details. - * - * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we - * always flush the TLB (step 4) even if previous step failed and the dirty - * bitmap may be corrupt. Regardless of previous outcome the KVM logging API - * does not preclude user space subsequent dirty log read. Flushing TLB ensures - * writes will be marked dirty for next log read. - * - * 1. Take a snapshot of the bit and clear it if needed. - * 2. Write protect the corresponding page. - * 3. Copy the snapshot to the userspace. - * 4. Flush TLB's if needed. - */ -int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) -{ - struct kvm_memslots *slots; - struct kvm_memory_slot *memslot; - bool is_dirty = false; - int r; - - mutex_lock(&kvm->slots_lock); - - r = kvm_get_dirty_log_protect(kvm, log, &is_dirty); - - if (is_dirty) { - slots = kvm_memslots(kvm); - memslot = id_to_memslot(slots, log->slot); - - /* - * FIXME: disable THP to improve vm migration success ratio, - * how to know migration failure to enable THP again - */ - memslot->arch.flags |= KVM_MEMSLOT_DISABLE_THP; - - /* Let implementation handle TLB/GVA invalidation */ - kvm_flush_remote_tlbs(kvm); - } - - mutex_unlock(&kvm->slots_lock); - return r; -} - long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm *kvm = filp->private_data; @@ -1792,7 +1707,7 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { return _kvm_pending_timer(vcpu) || kvm_read_hw_gcsr(KVM_CSR_ESTAT) & - (1 << (EXCCODE_TIMER - EXCCODE_INT_START)); + (1 << (KVM_INT_TIMER - KVM_INT_START)); } int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) @@ -1854,56 +1769,6 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) return 0; } -static void kvm_swtimer_func(unsigned long data) -{ - struct kvm_vcpu *vcpu = (struct kvm_vcpu *)data; - - _kvm_queue_irq(vcpu, LARCH_INT_TIMER); - if (swq_has_sleeper(&vcpu->wq)) - swake_up_one(&vcpu->wq); -} - -/* low level hrtimer wake routine */ -static enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer) -{ - struct kvm_vcpu *vcpu; - - vcpu = container_of(timer, struct kvm_vcpu, arch.swtimer); - kvm_swtimer_func((unsigned long) vcpu); - return kvm_count_timeout(vcpu); -} - -int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) -{ - int i; - - for_each_possible_cpu(i) - vcpu->arch.vpid[i] = 0; - - hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC, - HRTIMER_MODE_ABS_PINNED); - vcpu->arch.swtimer.function = kvm_swtimer_wakeup; - vcpu->arch.fpu_enabled = true; - vcpu->arch.lsx_enabled = true; - return 0; -} - -void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) -{ - int cpu; - struct kvm_context *context; - - /* - * If the VCPU is freed and reused as another VCPU, we don't want the - * matching pointer wrongly hanging around in last_vcpu. - */ - for_each_possible_cpu(cpu) { - context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); - if (context->last_vcpu == vcpu) - context->last_vcpu = NULL; - } -} - int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, struct kvm_translation *tr) { @@ -2098,12 +1963,14 @@ void kvm_lose_fpu(struct kvm_vcpu *vcpu) preempt_disable(); if (cpu_has_lasx && (vcpu->arch.aux_inuse & KVM_LARCH_LASX)) { +#ifdef CONFIG_CPU_HAS_LASX kvm_save_lasx(vcpu); trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX_LASX); /* Disable LASX & MAS & FPU */ disable_lasx(); disable_lsx(); +#endif if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { kvm_clear_csr_euen(KVM_EUEN_FPEN); @@ -2112,11 +1979,14 @@ void kvm_lose_fpu(struct kvm_vcpu *vcpu) KVM_LARCH_LSX | KVM_LARCH_LASX); } else if (cpu_has_lsx && vcpu->arch.aux_inuse & KVM_LARCH_LSX) { +#ifdef CONFIG_CPU_HAS_LASX kvm_save_lsx(vcpu); trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_LSX); /* Disable LSX & FPU */ disable_lsx(); +#endif + if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { kvm_clear_csr_euen(KVM_EUEN_FPEN); } diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index 8ddfb0e8a89af97d5f63eda18b44aa64cfe8884b..ba9fa7895d2566767532d8f6dfc10ed4ffd0a45b 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c @@ -32,6 +32,24 @@ static int kvm_tlb_flush_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) return 0; } +static inline int kvm_pmd_huge(pmd_t pmd) +{ +#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT + return (pmd_val(pmd) & _PAGE_HUGE) != 0; +#else + return 0; +#endif +} + +static inline int kvm_pud_huge(pud_t pud) +{ +#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT + return (pud_val(pud) & _PAGE_HUGE) != 0; +#else + return 0; +#endif +} + static inline pmd_t kvm_pmd_mkhuge(pmd_t pmd) { #ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT @@ -80,54 +98,6 @@ static inline pmd_t kvm_pmd_mkold(pmd_t pmd) return pmd; } -static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, - int min, int max) -{ - void *page; - - BUG_ON(max > KVM_NR_MEM_OBJS); - if (cache->nobjs >= min) - return 0; - while (cache->nobjs < max) { - page = (void *)__get_free_page(GFP_KERNEL); - if (!page) - return -ENOMEM; - cache->objects[cache->nobjs++] = page; - } - return 0; -} - -static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) -{ - while (mc->nobjs) - free_page((unsigned long)mc->objects[--mc->nobjs]); -} - -static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, - struct vm_area_struct *vma, unsigned long hva) -{ - void *p = NULL; - struct mempolicy *pol = __get_vma_policy(vma, hva); - struct page *page; - - - BUG_ON(!mc || !mc->nobjs); - if (pol) { - page = alloc_page_vma(GFP_KERNEL_ACCOUNT | GFP_ATOMIC, vma, hva); - if (page) - p = page_address(page); - } - if (!p) - p = mc->objects[--mc->nobjs]; - - return p; -} - -void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) -{ - mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); -} - /** * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory. * @@ -170,6 +140,7 @@ static pte_t *kvm_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, struct vm_area_struct *vma, unsigned long hva, unsigned long addr) { + p4d_t *p4d; pud_t *pud; pmd_t *pmd; @@ -179,32 +150,31 @@ static pte_t *kvm_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, BUG(); return NULL; } - pud = pud_offset(pgd, addr); + p4d = p4d_offset(pgd, addr); + pud = pud_offset(p4d, addr); if (pud_none(*pud)) { pmd_t *new_pmd; if (!cache) return NULL; - new_pmd = mmu_memory_cache_alloc(cache, vma, hva); + new_pmd = kvm_mmu_memory_cache_alloc(cache); pmd_init(new_pmd); pud_populate(NULL, pud, new_pmd); } pmd = pmd_offset(pud, addr); -#ifdef CONFIG_LOONGARCH_HUGE_TLB_SUPPORT - if (pmd_huge(*pmd)) { + if (kvm_pmd_huge(*pmd)) { return (pte_t *)pmd; } -#endif if (pmd_none(*pmd)) { pte_t *new_pte; if (!cache) return NULL; - new_pte = mmu_memory_cache_alloc(cache, vma, hva); + new_pte = kvm_mmu_memory_cache_alloc(cache); clear_page(new_pte); pmd_populate_kernel(NULL, pmd, new_pte); } - return pte_offset(pmd, addr); + return pte_offset_kernel(pmd, addr); } /* Caller must hold kvm->mm_lock */ @@ -225,8 +195,8 @@ static pte_t *kvm_pte_for_gpa(struct kvm *kvm, static bool kvm_flush_gpa_pte(pte_t *pte, unsigned long start_gpa, unsigned long end_gpa, unsigned long *data) { - int i_min = __pte_offset(start_gpa); - int i_max = __pte_offset(end_gpa); + int i_min = pte_index(start_gpa); + int i_max = pte_index(end_gpa); bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1); int i; @@ -246,8 +216,8 @@ static bool kvm_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa, { pte_t *pte; unsigned long end = ~0ul; - int i_min = __pmd_offset(start_gpa); - int i_max = __pmd_offset(end_gpa); + int i_min = pmd_index(start_gpa); + int i_max = pmd_index(end_gpa); bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1); int i; @@ -255,14 +225,14 @@ static bool kvm_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa, if (!pmd_present(pmd[i])) continue; - if (pmd_huge(pmd[i]) && pmd_present(pmd[i])) { + if (kvm_pmd_huge(pmd[i]) && pmd_present(pmd[i])) { pmd_clear(pmd + i); if (data) *data += PTRS_PER_PMD; continue; } - pte = pte_offset(pmd + i, 0); + pte = pte_offset_kernel(pmd + i, 0); if (i == i_max) end = end_gpa; @@ -281,8 +251,8 @@ static bool kvm_flush_gpa_pud(pud_t *pud, unsigned long start_gpa, { pmd_t *pmd; unsigned long end = ~0ul; - int i_min = __pud_offset(start_gpa); - int i_max = __pud_offset(end_gpa); + int i_min = pud_index(start_gpa); + int i_max = pud_index(end_gpa); bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1); int i; @@ -307,6 +277,7 @@ static bool kvm_flush_gpa_pud(pud_t *pud, unsigned long start_gpa, static bool kvm_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa, unsigned long end_gpa, unsigned long *data) { + p4d_t *p4d; pud_t *pud; unsigned long end = ~0ul; int i_min = pgd_index(start_gpa); @@ -318,7 +289,8 @@ static bool kvm_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa, if (!pgd_present(pgd[i])) continue; - pud = pud_offset(pgd + i, 0); + p4d = p4d_offset(pgd + i, 0); + pud = pud_offset(p4d, 0); if (i == i_max) end = end_gpa; @@ -361,8 +333,8 @@ static bool kvm_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn, vo static int kvm_mkclean_pte(pte_t *pte, unsigned long start, unsigned long end) { int ret = 0; - int i_min = __pte_offset(start); - int i_max = __pte_offset(end); + int i_min = pte_index(start); + int i_max = pte_index(end); int i; pte_t val; @@ -381,8 +353,8 @@ static int kvm_mkclean_pmd(pmd_t *pmd, unsigned long start, unsigned long end) int ret = 0; pte_t *pte; unsigned long cur_end = ~0ul; - int i_min = __pmd_offset(start); - int i_max = __pmd_offset(end); + int i_min = pmd_index(start); + int i_max = pmd_index(end); int i; pmd_t old, new; @@ -390,7 +362,7 @@ static int kvm_mkclean_pmd(pmd_t *pmd, unsigned long start, unsigned long end) if (!pmd_present(pmd[i])) continue; - if (pmd_huge(pmd[i])) { + if (kvm_pmd_huge(pmd[i])) { old = pmd[i]; new = kvm_pmd_mkclean(old); if (pmd_val(new) == pmd_val(old)) @@ -400,7 +372,7 @@ static int kvm_mkclean_pmd(pmd_t *pmd, unsigned long start, unsigned long end) continue; } - pte = pte_offset(pmd + i, 0); + pte = pte_offset_kernel(pmd + i, 0); if (i == i_max) cur_end = end; @@ -415,8 +387,8 @@ static int kvm_mkclean_pud(pud_t *pud, unsigned long start, unsigned long end) int ret = 0; pmd_t *pmd; unsigned long cur_end = ~0ul; - int i_min = __pud_offset(start); - int i_max = __pud_offset(end); + int i_min = pud_index(start); + int i_max = pud_index(end); int i; for (i = i_min; i <= i_max; ++i, start = 0) { @@ -435,6 +407,7 @@ static int kvm_mkclean_pud(pud_t *pud, unsigned long start, unsigned long end) static int kvm_mkclean_pgd(pgd_t *pgd, unsigned long start, unsigned long end) { int ret = 0; + p4d_t *p4d; pud_t *pud; unsigned long cur_end = ~0ul; int i_min = pgd_index(start); @@ -445,7 +418,8 @@ static int kvm_mkclean_pgd(pgd_t *pgd, unsigned long start, unsigned long end) if (!pgd_present(pgd[i])) continue; - pud = pud_offset(pgd + i, 0); + p4d = p4d_offset(pgd + i, 0); + pud = pud_offset(p4d, 0); if (i == i_max) cur_end = end; @@ -495,20 +469,22 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, gfn_t end = base_gfn + __fls(mask); kvm_mkclean_gpa_pt(kvm, start, end); + + /* + * FIXME: disable THP to improve vm migration success ratio, + * how to know migration failure to enable THP again + */ + slot->arch.flags |= KVM_MEMSLOT_DISABLE_THP; } void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { int needs_flush; - kvm_debug("%s: kvm: %p slot: %d, GPA: %llx, size: %llx, QVA: %llx\n", - __func__, kvm, mem->slot, mem->guest_phys_addr, - mem->memory_size, mem->userspace_addr); - /* * If dirty page logging is enabled, write protect all pages in the slot * ready for dirty logging. @@ -579,8 +555,8 @@ static int kvm_mkold_pte(pte_t *pte, unsigned long start, unsigned long end) { int ret = 0; - int i_min = __pte_offset(start); - int i_max = __pte_offset(end); + int i_min = pte_index(start); + int i_max = pte_index(end); int i; pte_t old, new; @@ -604,8 +580,8 @@ static int kvm_mkold_pmd(pmd_t *pmd, unsigned long start, unsigned long end) int ret = 0; pte_t *pte; unsigned long cur_end = ~0ul; - int i_min = __pmd_offset(start); - int i_max = __pmd_offset(end); + int i_min = pmd_index(start); + int i_max = pmd_index(end); int i; pmd_t old, new; @@ -613,7 +589,7 @@ static int kvm_mkold_pmd(pmd_t *pmd, unsigned long start, unsigned long end) if (!pmd_present(pmd[i])) continue; - if (pmd_huge(pmd[i])) { + if (kvm_pmd_huge(pmd[i])) { old = pmd[i]; new = kvm_pmd_mkold(old); if (pmd_val(new) == pmd_val(old)) @@ -623,7 +599,7 @@ static int kvm_mkold_pmd(pmd_t *pmd, unsigned long start, unsigned long end) continue; } - pte = pte_offset(pmd + i, 0); + pte = pte_offset_kernel(pmd + i, 0); if (i == i_max) cur_end = end; @@ -638,8 +614,8 @@ static int kvm_mkold_pud(pud_t *pud, unsigned long start, unsigned long end) int ret = 0; pmd_t *pmd; unsigned long cur_end = ~0ul; - int i_min = __pud_offset(start); - int i_max = __pud_offset(end); + int i_min = pud_index(start); + int i_max = pud_index(end); int i; for (i = i_min; i <= i_max; ++i, start = 0) { @@ -659,6 +635,7 @@ static int kvm_mkold_pud(pud_t *pud, unsigned long start, unsigned long end) static int kvm_mkold_pgd(pgd_t *pgd, unsigned long start, unsigned long end) { int ret = 0; + p4d_t *p4d; pud_t *pud; unsigned long cur_end = ~0ul; int i_min = pgd_index(start); @@ -669,7 +646,8 @@ static int kvm_mkold_pgd(pgd_t *pgd, unsigned long start, unsigned long end) if (!pgd_present(pgd[i])) continue; - pud = pud_offset(pgd + i, 0); + p4d = p4d_offset(pgd + i, 0); + pud = pud_offset(p4d, 0); if (i == i_max) cur_end = end; @@ -813,6 +791,7 @@ static pud_t *kvm_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, phys_addr_t addr) { pgd_t *pgd; + p4d_t *p4d; pgd = kvm->arch.gpa_mm.pgd + pgd_index(addr); if (pgd_none(*pgd)) { @@ -821,7 +800,8 @@ static pud_t *kvm_get_pud(struct kvm *kvm, return NULL; } - return pud_offset(pgd, addr); + p4d = p4d_offset(pgd, addr); + return pud_offset(p4d, addr); } static pmd_t *kvm_get_pmd(struct kvm *kvm, @@ -832,13 +812,13 @@ static pmd_t *kvm_get_pmd(struct kvm *kvm, pmd_t *pmd; pud = kvm_get_pud(kvm, cache, addr); - if (!pud || pud_huge(*pud)) + if (!pud || kvm_pud_huge(*pud)) return NULL; if (pud_none(*pud)) { if (!cache) return NULL; - pmd = mmu_memory_cache_alloc(cache, vma, hva); + pmd = kvm_mmu_memory_cache_alloc(cache); pmd_init(pmd); pud_populate(NULL, pud, pmd); } @@ -885,7 +865,7 @@ static int kvm_set_pmd_huge(struct kvm_vcpu *vcpu, struct kvm_mmu_memory_cache * Normal THP split/merge follows mmu_notifier callbacks and do * get handled accordingly. */ - if (!pmd_huge(old_pmd)) { + if (!kvm_pmd_huge(old_pmd)) { ++vcpu->stat.huge_merge_exits; kvm_flush_gpa_pt(vcpu->kvm, (addr & PMD_MASK) >> PAGE_SHIFT, @@ -1077,7 +1057,7 @@ static int kvm_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa, /* Track dirtying of writeable pages */ set_pte(ptep, pte_mkdirty(*ptep)); pfn = pte_pfn(*ptep); - if (pmd_huge(*((pmd_t *)ptep))) { + if (kvm_pmd_huge(*((pmd_t *)ptep))) { int i; gfn_t base_gfn = (gpa & PMD_MASK) >> PAGE_SHIFT; @@ -1149,7 +1129,7 @@ static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, /* Try the fast path to handle old / clean pages */ srcu_idx = srcu_read_lock(&kvm->srcu); - if ((exccode != EXCCODE_TLBRI) && (exccode != EXCCODE_TLBXI)) { + if ((exccode != KVM_EXCCODE_TLBRI) && (exccode != KVM_EXCCODE_TLBXI)) { err = kvm_map_page_fast(vcpu, gpa, write, out_entry, out_buddy); if (!err) @@ -1162,11 +1142,11 @@ static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, goto out; /* Let's check if we will get back a huge page backed by hugetlbfs */ - down_read(¤t->mm->mmap_sem); + mmap_read_lock(current->mm); vma = find_vma_intersection(current->mm, hva, hva + 1); if (unlikely(!vma)) { kvm_err("Failed to find VMA for hva 0x%lx\n", hva); - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); err = -EFAULT; goto out; } @@ -1182,12 +1162,10 @@ static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, /* PMD is not folded, adjust gfn to new boundary */ if (vma_pagesize == PMD_SIZE) gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; - - up_read(¤t->mm->mmap_sem); + mmap_read_unlock(current->mm); /* We need a minimum of cached pages ready for page table creation */ - err = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, - KVM_NR_MEM_OBJS); + err = kvm_mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES); if (err) goto out; @@ -1262,7 +1240,7 @@ static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, if (writeable) { prot_bits |= _PAGE_WRITE; if (write) { - prot_bits |= __WRITEABLE; + prot_bits |= __WRITEABLE | _PAGE_MODIFIED; mark_page_dirty(kvm, gfn); kvm_set_pfn_dirty(pfn); } diff --git a/arch/loongarch/kvm/trace.h b/arch/loongarch/kvm/trace.h index d72d10cb39fd6eeec17fa9ccde626eaabd4dbf1c..ae0693531a82eef2c874aab1ef15a0057b69071d 100644 --- a/arch/loongarch/kvm/trace.h +++ b/arch/loongarch/kvm/trace.h @@ -54,28 +54,28 @@ DEFINE_EVENT(kvm_transition, kvm_out, /* The first 32 exit reasons correspond to Cause.ExcCode */ #define KVM_TRACE_EXIT_INT 0 -#define KVM_TRACE_EXIT_TLBLD (EXCCODE_TLBL) -#define KVM_TRACE_EXIT_TLBST (EXCCODE_TLBS) -#define KVM_TRACE_EXIT_TLBI (EXCCODE_TLBI) -#define KVM_TRACE_EXIT_TLBMOD (EXCCODE_TLBM) -#define KVM_TRACE_EXIT_TLBRI (EXCCODE_TLBRI) -#define KVM_TRACE_EXIT_TLBXI (EXCCODE_TLBXI) -#define KVM_TRACE_EXIT_TLBPE (EXCCODE_TLBPE) -#define KVM_TRACE_EXIT_ADDE (EXCCODE_ADE) -#define KVM_TRACE_EXIT_UNALIGN (EXCCODE_ALE) -#define KVM_TRACE_EXIT_ODB (EXCCODE_OOB) -#define KVM_TRACE_EXIT_SYSCALL (EXCCODE_SYS) -#define KVM_TRACE_EXIT_BP (EXCCODE_BP) -#define KVM_TRACE_EXIT_INE (EXCCODE_INE) -#define KVM_TRACE_EXIT_IPE (EXCCODE_IPE) -#define KVM_TRACE_EXIT_FPDIS (EXCCODE_FPDIS) -#define KVM_TRACE_EXIT_LSXDIS (EXCCODE_LSXDIS) -#define KVM_TRACE_EXIT_LASXDIS (EXCCODE_LASXDIS) -#define KVM_TRACE_EXIT_FPE (EXCCODE_FPE) -#define KVM_TRACE_EXIT_WATCH (EXCCODE_WATCH) -#define KVM_TRACE_EXIT_GSPR (EXCCODE_GSPR) -#define KVM_TRACE_EXIT_HC (EXCCODE_HYP) -#define KVM_TRACE_EXIT_GCM (EXCCODE_GCM) +#define KVM_TRACE_EXIT_TLBLD (KVM_EXCCODE_TLBL) +#define KVM_TRACE_EXIT_TLBST (KVM_EXCCODE_TLBS) +#define KVM_TRACE_EXIT_TLBI (KVM_EXCCODE_TLBI) +#define KVM_TRACE_EXIT_TLBMOD (KVM_EXCCODE_TLBM) +#define KVM_TRACE_EXIT_TLBRI (KVM_EXCCODE_TLBRI) +#define KVM_TRACE_EXIT_TLBXI (KVM_EXCCODE_TLBXI) +#define KVM_TRACE_EXIT_TLBPE (KVM_EXCCODE_TLBPE) +#define KVM_TRACE_EXIT_ADDE (KVM_EXCCODE_ADE) +#define KVM_TRACE_EXIT_UNALIGN (KVM_EXCCODE_ALE) +#define KVM_TRACE_EXIT_ODB (KVM_EXCCODE_OOB) +#define KVM_TRACE_EXIT_SYSCALL (KVM_EXCCODE_SYS) +#define KVM_TRACE_EXIT_BP (KVM_EXCCODE_BP) +#define KVM_TRACE_EXIT_INE (KVM_EXCCODE_INE) +#define KVM_TRACE_EXIT_IPE (KVM_EXCCODE_IPE) +#define KVM_TRACE_EXIT_FPDIS (KVM_EXCCODE_FPDIS) +#define KVM_TRACE_EXIT_LSXDIS (KVM_EXCCODE_LSXDIS) +#define KVM_TRACE_EXIT_LASXDIS (KVM_EXCCODE_LASXDIS) +#define KVM_TRACE_EXIT_FPE (KVM_EXCCODE_FPE) +#define KVM_TRACE_EXIT_WATCH (KVM_EXCCODE_WATCH) +#define KVM_TRACE_EXIT_GSPR (KVM_EXCCODE_GSPR) +#define KVM_TRACE_EXIT_HC (KVM_EXCCODE_HYP) +#define KVM_TRACE_EXIT_GCM (KVM_EXCCODE_GCM) /* Further exit reasons */ #define KVM_TRACE_EXIT_IDLE 64 @@ -83,7 +83,7 @@ DEFINE_EVENT(kvm_transition, kvm_out, #define KVM_TRACE_EXIT_SIGNAL 66 /* Tracepoints for VM exits */ -#define kvm_trace_symbol_exit_types \ +#define kvm_trace_symbol_exit_types \ { KVM_TRACE_EXIT_INT, "Interrupt" }, \ { KVM_TRACE_EXIT_TLBLD, "TLB (LD)" }, \ { KVM_TRACE_EXIT_TLBST, "TLB (ST)" }, \ @@ -142,7 +142,7 @@ TRACE_EVENT(kvm_exit, #define KVM_TRACE_AUX_LASX 4 #define KVM_TRACE_AUX_FPU_LSX_LASX 7 -#define kvm_trace_symbol_aux_op \ +#define kvm_trace_symbol_aux_op \ { KVM_TRACE_AUX_RESTORE, "restore" }, \ { KVM_TRACE_AUX_SAVE, "save" }, \ { KVM_TRACE_AUX_ENABLE, "enable" }, \ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 3af8b0164f1e6b2fa07469778aa31f47d1e39645..49fd23cc6637f2146fe437d3a606920a08b47417 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -251,6 +251,7 @@ struct kvm_hyperv_exit { #define KVM_EXIT_X86_RDMSR 29 #define KVM_EXIT_X86_WRMSR 30 #define KVM_EXIT_RISCV_SBI 31 +#define KVM_EXIT_LOONGARCH_IOCSR 36 /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ @@ -320,6 +321,13 @@ struct kvm_run { __u32 len; __u8 is_write; } mmio; + /* KVM_EXIT_LOONGARCH_IOCSR */ + struct { + __u64 phys_addr; + __u8 data[8]; + __u32 len; + __u8 is_write; + } iocsr_io; /* KVM_EXIT_HYPERCALL */ struct { __u64 nr; @@ -703,6 +711,16 @@ struct kvm_s390_irq_state { __u32 reserved[4]; /* will stay unused for compatibility reasons */ }; +struct kvm_loongarch_vcpu_state { + __u8 online_vcpus; + __u8 is_migrate; + __u32 cpu_freq; + __u32 count_ctl; + __u64 irq_pending; + __u64 irq_clear; + __u64 core_ext_ioisr[4]; +}; + /* for KVM_SET_GUEST_DEBUG */ #define KVM_GUESTDBG_ENABLE 0x00000001 @@ -1064,6 +1082,10 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_ARM_CPU_FEATURE 555 +#define KVM_CAP_LOONGARCH_FPU 800 +#define KVM_CAP_LOONGARCH_LSX 801 +#define KVM_CAP_LOONGARCH_VZ 802 + #ifdef KVM_CAP_IRQ_ROUTING struct kvm_irq_routing_irqchip { @@ -1210,6 +1232,7 @@ struct kvm_dirty_tlb { #define KVM_REG_ARM64 0x6000000000000000ULL #define KVM_REG_MIPS 0x7000000000000000ULL #define KVM_REG_RISCV 0x8000000000000000ULL +#define KVM_REG_LOONGARCH 0x9000000000000000ULL #define KVM_REG_SIZE_SHIFT 52 #define KVM_REG_SIZE_MASK 0x00f0000000000000ULL @@ -1547,6 +1570,14 @@ struct kvm_enc_region { #define KVM_S390_NORMAL_RESET _IO(KVMIO, 0xc3) #define KVM_S390_CLEAR_RESET _IO(KVMIO, 0xc4) +/* Add for LOONGSON read nodecounter */ +#define KVM_LOONGARCH_GET_VCPU_STATE _IOR(KVMIO, 0xc0, struct kvm_loongarch_vcpu_state) +#define KVM_LOONGARCH_SET_VCPU_STATE _IOW(KVMIO, 0xc1, struct kvm_loongarch_vcpu_state) +#define KVM_LOONGARCH_GET_CPUCFG _IOR(KVMIO, 0xc2, struct kvm_cpucfg) +#define KVM_LOONGARCH_GET_IOCSR _IOR(KVMIO, 0xc3, struct kvm_iocsr_entry) +#define KVM_LOONGARCH_SET_IOCSR _IOW(KVMIO, 0xc4, struct kvm_iocsr_entry) +#define KVM_LOONGARCH_SET_CPUCFG _IOR(KVMIO, 0xc5, struct kvm_cpucfg) + struct kvm_s390_pv_sec_parm { __u64 origin; __u64 length;