diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 4b69be9bea5535127b6e1e83e354fb602652ae6a..aa7e808b9d1e15a0d12c8bdcb74a20bb1101072a 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -259,19 +259,6 @@ int fpu_clone(struct task_struct *dst) return 0; } -/* - * Activate the current task's in-memory FPU context, - * if it has not been used before: - */ -static void fpu__initialize(struct fpu *fpu) -{ - WARN_ON_FPU(fpu != ¤t->thread.fpu); - - set_thread_flag(TIF_NEED_FPU_LOAD); - fpstate_init(&fpu->state); - trace_x86_fpu_init_state(fpu); -} - /* * Drops current FPU state: deactivates the fpregs and * the fpstate. NOTE: it still leaves previous contents @@ -314,47 +301,99 @@ static inline void restore_fpregs_from_init_fpstate(u64 features_mask) pkru_write_default(); } +static inline unsigned int init_fpstate_copy_size(void) +{ + if (!use_xsave()) + return fpu_kernel_xstate_size; + + /* XSAVE(S) just needs the legacy and the xstate header part */ + return sizeof(init_fpstate.xsave); +} + +/* Temporary workaround. Will be removed once PKRU and XSTATE are untangled. */ +static inline void pkru_set_default_in_xstate(struct xregs_state *xsave) +{ + struct pkru_state *pk; + + if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) + return; + /* + * Force XFEATURE_PKRU to be set in the header otherwise + * get_xsave_addr() does not work and it also needs to be set to + * make XRSTOR(S) load it. + */ + xsave->header.xfeatures |= XFEATURE_MASK_PKRU; + pk = get_xsave_addr(xsave, XFEATURE_PKRU); + pk->pkru = pkru_get_init_value(); +} + /* - * Clear the FPU state back to init state. - * - * Called by sys_execve(), by the signal handler code and by various - * error paths. + * Reset current->fpu memory state to the init values. + */ +static void fpu_reset_fpstate(void) +{ + struct fpu *fpu = ¤t->thread.fpu; + + fpregs_lock(); + fpu__drop(fpu); + /* + * This does not change the actual hardware registers. It just + * resets the memory image and sets TIF_NEED_FPU_LOAD so a + * subsequent return to usermode will reload the registers from the + * task's memory image. + * + * Do not use fpstate_init() here. Just copy init_fpstate which has + * the correct content already except for PKRU. + */ + memcpy(&fpu->state, &init_fpstate, init_fpstate_copy_size()); + pkru_set_default_in_xstate(&fpu->state.xsave); + set_thread_flag(TIF_NEED_FPU_LOAD); + fpregs_unlock(); +} + +/* + * Reset current's user FPU states to the init states. current's + * supervisor states, if any, are not modified by this function. The + * caller guarantees that the XSTATE header in memory is intact. */ -static void fpu__clear(struct fpu *fpu, bool user_only) +void fpu__clear_user_states(struct fpu *fpu) { WARN_ON_FPU(fpu != ¤t->thread.fpu); - if (!static_cpu_has(X86_FEATURE_FPU)) { - fpu__drop(fpu); - fpu__initialize(fpu); + fpregs_lock(); + if (!cpu_feature_enabled(X86_FEATURE_FPU)) { + fpu_reset_fpstate(); + fpregs_unlock(); return; } - fpregs_lock(); - - if (user_only) { - if (!fpregs_state_valid(fpu, smp_processor_id()) && - xfeatures_mask_supervisor()) - os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor()); - restore_fpregs_from_init_fpstate(xfeatures_mask_user()); - } else { - restore_fpregs_from_init_fpstate(xfeatures_mask_all); + /* + * Ensure that current's supervisor states are loaded into their + * corresponding registers. + */ + if (xfeatures_mask_supervisor() && + !fpregs_state_valid(fpu, smp_processor_id())) { + os_xrstor(&fpu->state.xsave, xfeatures_mask_supervisor()); } + /* Reset user states in registers. */ + restore_fpregs_from_init_fpstate(xfeatures_mask_user()); + + /* + * Now all FPU registers have their desired values. Inform the FPU + * state machine that current's FPU registers are in the hardware + * registers. The memory image does not need to be updated because + * any operation relying on it has to save the registers first when + * current's FPU is marked active. + */ fpregs_mark_activate(); fpregs_unlock(); } -void fpu__clear_user_states(struct fpu *fpu) -{ - fpu__clear(fpu, true); -} - void fpu_flush_thread(void) { - fpu__clear(¤t->thread.fpu, false); + fpu_reset_fpstate(); } - /* * Load FPU context before returning to userspace. */ diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index b9b0ddcc2b62144d08fae2bda092425e4a21b796..d20963c3bb6d015f2f1a8bb70b9c478e3dc56280 100755 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -188,6 +188,15 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, return ret; } +static void pkru_flush_thread(void) +{ + /* + * If PKRU is enabled the default PKRU value has to be loaded into + * the hardware right here (similar to context switch). + */ + pkru_write_default(); +} + void flush_thread(void) { struct task_struct *tsk = current; @@ -196,6 +205,7 @@ void flush_thread(void) memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); fpu_flush_thread(); + pkru_flush_thread(); } void disable_TSC(void)