Skip to content

K
Kernel

openeuler / Kernel
大约 1 年 前同步成功

通知 6
Star 0
Fork 0
K
Kernel

体验新版 GitCode，发现更多精彩内容 >>

提交 e75c73ad 编写于 作者: L Linus Torvalds
浏览文件
操作

Merge branch 'x86-fpu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 FPU updates from Ingo Molnar:
 "This tree contains two main changes:

   - The big FPU code rewrite: wide reaching cleanups and reorganization
     that pulls all the FPU code together into a clean base in
     arch/x86/fpu/.

     The resulting code is leaner and faster, and much easier to
     understand.  This enables future work to further simplify the FPU
     code (such as removing lazy FPU restores).

     By its nature these changes have a substantial regression risk: FPU
     code related bugs are long lived, because races are often subtle
     and bugs mask as user-space failures that are difficult to track
     back to kernel side backs.  I'm aware of no unfixed (or even
     suspected) FPU related regression so far.

   - MPX support rework/fixes.  As this is still not a released CPU
     feature, there were some buglets in the code - should be much more
     robust now (Dave Hansen)"

* 'x86-fpu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (250 commits)
  x86/fpu: Fix double-increment in setup_xstate_features()
  x86/mpx: Allow 32-bit binaries on 64-bit kernels again
  x86/mpx: Do not count MPX VMAs as neighbors when unmapping
  x86/mpx: Rewrite the unmap code
  x86/mpx: Support 32-bit binaries on 64-bit kernels
  x86/mpx: Use 32-bit-only cmpxchg() for 32-bit apps
  x86/mpx: Introduce new 'directory entry' to 'addr' helper function
  x86/mpx: Add temporary variable to reduce masking
  x86: Make is_64bit_mm() widely available
  x86/mpx: Trace allocation of new bounds tables
  x86/mpx: Trace the attempts to find bounds tables
  x86/mpx: Trace entry to bounds exception paths
  x86/mpx: Trace #BR exceptions
  x86/mpx: Introduce a boot-time disable flag
  x86/mpx: Restrict the mmap() size check to bounds tables
  x86/mpx: Remove redundant MPX_BNDCFG_ADDR_MASK
  x86/mpx: Clean up the code by not passing a task pointer around when unnecessary
  x86/mpx: Use the new get_xsave_field_ptr()API
  x86/fpu/xstate: Wrap get_xsave_addr() to make it safer
  x86/fpu/xstate: Fix up bad get_xsave_addr() assumptions
  ...
上级 cfe3eceb a8424003
展开全部 隐藏空白更改
内联 并排
Showing with 4082 addition and 1207 deletion
Documentation/kernel-parameters.txt
浏览文件 @ e75c73ad
......@@ -937,6 +937,10 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
Enable debug messages at boot time. See
Documentation/dynamic-debug-howto.txt for details.
nompx [X86] Disables Intel Memory Protection Extensions.
See Documentation/x86/intel_mpx.txt for more
information about the feature.
eagerfpu= [X86]
on enable eager fpu restore
off disable eager fpu restore
......
Documentation/preempt-locking.txt
浏览文件 @ e75c73ad
......@@ -48,7 +48,7 @@ preemption must be disabled around such regions.
Note, some FPU functions are already explicitly preempt safe. For example,
kernel_fpu_begin and kernel_fpu_end will disable and enable preemption.
However, math_state_restore must be called with preemption disabled.
However, fpu__restore() must be called with preemption disabled.
RULE #3: Lock acquire and release must be performed by same task
......
arch/x86/Kconfig.debug
浏览文件 @ e75c73ad
......@@ -332,4 +332,16 @@ config X86_DEBUG_STATIC_CPU_HAS
If unsure, say N.
config X86_DEBUG_FPU
bool "Debug the x86 FPU code"
depends on DEBUG_KERNEL
default y
---help---
If this option is enabled then there will be extra sanity
checks and (boot time) debug printouts added to the kernel.
This debugging adds some small amount of runtime overhead
to the kernel.
If unsure, say N.
endmenu
arch/x86/crypto/aesni-intel_glue.c
浏览文件 @ e75c73ad
......@@ -32,7 +32,7 @@
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/cpu_device_id.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
#include <asm/crypto/aes.h>
#include <crypto/ablk_helper.h>
#include <crypto/scatterwalk.h>
......
arch/x86/crypto/camellia_aesni_avx2_glue.c
浏览文件 @ e75c73ad
......@@ -19,8 +19,7 @@
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>
......@@ -561,16 +560,15 @@ static struct crypto_alg cmll_algs[10] = { {
static int __init camellia_aesni_init(void)
{
u64 xcr0;
const char *feature_name;
if (!cpu_has_avx2 || !cpu_has_avx || !cpu_has_aes || !cpu_has_osxsave) {
pr_info("AVX2 or AES-NI instructions are not detected.\n");
return -ENODEV;
}
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX2 detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
pr_info("CPU feature '%s' is not supported.\n", feature_name);
return -ENODEV;
}
......
arch/x86/crypto/camellia_aesni_avx_glue.c
浏览文件 @ e75c73ad
......@@ -19,8 +19,7 @@
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>
......@@ -553,16 +552,10 @@ static struct crypto_alg cmll_algs[10] = { {
static int __init camellia_aesni_init(void)
{
u64 xcr0;
const char *feature_name;
if (!cpu_has_avx || !cpu_has_aes || !cpu_has_osxsave) {
pr_info("AVX or AES-NI instructions are not detected.\n");
return -ENODEV;
}
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
pr_info("CPU feature '%s' is not supported.\n", feature_name);
return -ENODEV;
}
......
arch/x86/crypto/cast5_avx_glue.c
浏览文件 @ e75c73ad
......@@ -31,8 +31,7 @@
#include <crypto/cast5.h>
#include <crypto/cryptd.h>
#include <crypto/ctr.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <asm/crypto/glue_helper.h>
#define CAST5_PARALLEL_BLOCKS 16
......@@ -468,16 +467,10 @@ static struct crypto_alg cast5_algs[6] = { {
static int __init cast5_init(void)
{
u64 xcr0;
const char *feature_name;
if (!cpu_has_avx || !cpu_has_osxsave) {
pr_info("AVX instructions are not detected.\n");
return -ENODEV;
}
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
pr_info("CPU feature '%s' is not supported.\n", feature_name);
return -ENODEV;
}
......
arch/x86/crypto/cast6_avx_glue.c
浏览文件 @ e75c73ad
......@@ -36,8 +36,7 @@
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <asm/crypto/glue_helper.h>
#define CAST6_PARALLEL_BLOCKS 8
......@@ -590,16 +589,10 @@ static struct crypto_alg cast6_algs[10] = { {
static int __init cast6_init(void)
{
u64 xcr0;
const char *feature_name;
if (!cpu_has_avx || !cpu_has_osxsave) {
pr_info("AVX instructions are not detected.\n");
return -ENODEV;
}
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
pr_info("CPU feature '%s' is not supported.\n", feature_name);
return -ENODEV;
}
......
arch/x86/crypto/crc32-pclmul_glue.c
浏览文件 @ e75c73ad
......@@ -35,7 +35,7 @@
#include <asm/cpufeature.h>
#include <asm/cpu_device_id.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
#define CHKSUM_BLOCK_SIZE 1
#define CHKSUM_DIGEST_SIZE 4
......
arch/x86/crypto/crc32c-intel_glue.c
浏览文件 @ e75c73ad
......@@ -32,8 +32,7 @@
#include <asm/cpufeature.h>
#include <asm/cpu_device_id.h>
#include <asm/i387.h>
#include <asm/fpu-internal.h>
#include <asm/fpu/internal.h>
#define CHKSUM_BLOCK_SIZE 1
#define CHKSUM_DIGEST_SIZE 4
......
arch/x86/crypto/crct10dif-pclmul_glue.c
浏览文件 @ e75c73ad
......@@ -29,7 +29,7 @@
#include <linux/init.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
#include <asm/cpufeature.h>
#include <asm/cpu_device_id.h>
......
arch/x86/crypto/fpu.c
浏览文件 @ e75c73ad
......@@ -18,7 +18,7 @@
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/crypto.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
struct crypto_fpu_ctx {
struct crypto_blkcipher *child;
......
arch/x86/crypto/ghash-clmulni-intel_glue.c
浏览文件 @ e75c73ad
......@@ -19,7 +19,7 @@
#include <crypto/cryptd.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/hash.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>
#define GHASH_BLOCK_SIZE 16
......
arch/x86/crypto/serpent_avx2_glue.c
浏览文件 @ e75c73ad
......@@ -20,8 +20,7 @@
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <crypto/serpent.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <asm/crypto/serpent-avx.h>
#include <asm/crypto/glue_helper.h>
......@@ -537,16 +536,14 @@ static struct crypto_alg srp_algs[10] = { {
static int __init init(void)
{
u64 xcr0;
const char *feature_name;
if (!cpu_has_avx2 || !cpu_has_osxsave) {
pr_info("AVX2 instructions are not detected.\n");
return -ENODEV;
}
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
pr_info("CPU feature '%s' is not supported.\n", feature_name);
return -ENODEV;
}
......
arch/x86/crypto/serpent_avx_glue.c
浏览文件 @ e75c73ad
......@@ -36,8 +36,7 @@
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <asm/crypto/serpent-avx.h>
#include <asm/crypto/glue_helper.h>
......@@ -596,16 +595,10 @@ static struct crypto_alg serpent_algs[10] = { {
static int __init serpent_init(void)
{
u64 xcr0;
const char *feature_name;
if (!cpu_has_avx || !cpu_has_osxsave) {
printk(KERN_INFO "AVX instructions are not detected.\n");
return -ENODEV;
}
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
printk(KERN_INFO "AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
pr_info("CPU feature '%s' is not supported.\n", feature_name);
return -ENODEV;
}
......
arch/x86/crypto/sha-mb/sha1_mb.c
浏览文件 @ e75c73ad
......@@ -65,11 +65,8 @@
#include <crypto/mcryptd.h>
#include <crypto/crypto_wq.h>
#include <asm/byteorder.h>
#include <asm/i387.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <linux/hardirq.h>
#include <asm/fpu-internal.h>
#include <asm/fpu/api.h>
#include "sha_mb_ctx.h"
#define FLUSH_INTERVAL 1000 /* in usec */
......
arch/x86/crypto/sha1_ssse3_glue.c
浏览文件 @ e75c73ad
......@@ -29,9 +29,7 @@
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha1_base.h>
#include <asm/i387.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
......@@ -123,15 +121,9 @@ static struct shash_alg alg = {
#ifdef CONFIG_AS_AVX
static bool __init avx_usable(void)
{
u64 xcr0;
if (!cpu_has_avx || !cpu_has_osxsave)
return false;
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) {
if (cpu_has_avx)
pr_info("AVX detected but unusable.\n");
return false;
}
......
arch/x86/crypto/sha256_ssse3_glue.c
浏览文件 @ e75c73ad
......@@ -37,9 +37,7 @@
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha256_base.h>
#include <asm/i387.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <linux/string.h>
asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data,
......@@ -132,15 +130,9 @@ static struct shash_alg algs[] = { {
#ifdef CONFIG_AS_AVX
static bool __init avx_usable(void)
{
u64 xcr0;
if (!cpu_has_avx || !cpu_has_osxsave)
return false;
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) {
if (cpu_has_avx)
pr_info("AVX detected but unusable.\n");
return false;
}
......
arch/x86/crypto/sha512_ssse3_glue.c
浏览文件 @ e75c73ad
......@@ -35,9 +35,7 @@
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha512_base.h>
#include <asm/i387.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <linux/string.h>
......@@ -131,15 +129,9 @@ static struct shash_alg algs[] = { {
#ifdef CONFIG_AS_AVX
static bool __init avx_usable(void)
{
u64 xcr0;
if (!cpu_has_avx || !cpu_has_osxsave)
return false;
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
pr_info("AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) {
if (cpu_has_avx)
pr_info("AVX detected but unusable.\n");
return false;
}
......
arch/x86/crypto/twofish_avx_glue.c
浏览文件 @ e75c73ad
......@@ -36,9 +36,7 @@
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/i387.h>
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <asm/fpu/api.h>
#include <asm/crypto/twofish.h>
#include <asm/crypto/glue_helper.h>
#include <crypto/scatterwalk.h>
......@@ -558,16 +556,10 @@ static struct crypto_alg twofish_algs[10] = { {
static int __init twofish_init(void)
{
u64 xcr0;
const char *feature_name;
if (!cpu_has_avx || !cpu_has_osxsave) {
printk(KERN_INFO "AVX instructions are not detected.\n");
return -ENODEV;
}
xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
printk(KERN_INFO "AVX detected but unusable.\n");
if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
pr_info("CPU feature '%s' is not supported.\n", feature_name);
return -ENODEV;
}
......
arch/x86/ia32/ia32_signal.c
浏览文件 @ e75c73ad
......@@ -21,8 +21,8 @@
#include <linux/binfmts.h>
#include <asm/ucontext.h>
#include <asm/uaccess.h>
#include <asm/i387.h>
#include <asm/fpu-internal.h>
#include <asm/fpu/internal.h>
#include <asm/fpu/signal.h>
#include <asm/ptrace.h>
#include <asm/ia32_unistd.h>
#include <asm/user32.h>
......@@ -198,7 +198,7 @@ static int ia32_restore_sigcontext(struct pt_regs *regs,
buf = compat_ptr(tmp);
} get_user_catch(err);
err |= restore_xstate_sig(buf, 1);
err |= fpu__restore_sig(buf, 1);
force_iret();
......@@ -308,6 +308,7 @@ static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
size_t frame_size,
void __user **fpstate)
{
struct fpu *fpu = &current->thread.fpu;
unsigned long sp;
/* Default to using normal stack */
......@@ -322,12 +323,12 @@ static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs,
ksig->ka.sa.sa_restorer)
sp = (unsigned long) ksig->ka.sa.sa_restorer;
if (used_math()) {
if (fpu->fpstate_active) {
unsigned long fx_aligned, math_size;
sp = alloc_mathframe(sp, 1, &fx_aligned, &math_size);
sp = fpu__alloc_mathframe(sp, 1, &fx_aligned, &math_size);
*fpstate = (struct _fpstate_ia32 __user *) sp;
if (save_xstate_sig(*fpstate, (void __user *)fx_aligned,
if (copy_fpstate_to_sigframe(*fpstate, (void __user *)fx_aligned,
math_size) < 0)
return (void __user *) -1L;
}
......
arch/x86/include/asm/alternative.h
浏览文件 @ e75c73ad
......@@ -52,6 +52,12 @@ struct alt_instr {
u8 padlen; /* length of build-time padding */
} __packed;
/*
* Debug flag that can be tested to see whether alternative
* instructions were patched in already:
*/
extern int alternatives_patched;
extern void alternative_instructions(void);
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
......
arch/x86/include/asm/crypto/glue_helper.h
浏览文件 @ e75c73ad
......@@ -7,7 +7,7 @@
#include <linux/kernel.h>
#include <linux/crypto.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
#include <crypto/b128ops.h>
typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src);
......
arch/x86/include/asm/efi.h
浏览文件 @ e75c73ad
#ifndef _ASM_X86_EFI_H
#define _ASM_X86_EFI_H
#include <asm/i387.h>
#include <asm/fpu/api.h>
#include <asm/pgtable.h>
/*
......
arch/x86/include/asm/i387.h → arch/x86/include/asm/fpu/api.h
浏览文件 @ e75c73ad
......@@ -7,23 +7,8 @@
* x86-64 work by Andi Kleen 2002
*/
#ifndef _ASM_X86_I387_H
#define _ASM_X86_I387_H
#ifndef __ASSEMBLY__
#include <linux/sched.h>
#include <linux/hardirq.h>
struct pt_regs;
struct user_i387_struct;
extern int init_fpu(struct task_struct *child);
extern void fpu_finit(struct fpu *fpu);
extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
extern void math_state_restore(void);
extern bool irq_fpu_usable(void);
#ifndef _ASM_X86_FPU_API_H
#define _ASM_X86_FPU_API_H
/*
* Careful: __kernel_fpu_begin/end() must be called with preempt disabled
......@@ -37,23 +22,9 @@ extern bool irq_fpu_usable(void);
*/
extern void __kernel_fpu_begin(void);
extern void __kernel_fpu_end(void);
static inline void kernel_fpu_begin(void)
{
preempt_disable();
WARN_ON_ONCE(!irq_fpu_usable());
__kernel_fpu_begin();
}
static inline void kernel_fpu_end(void)
{
__kernel_fpu_end();
preempt_enable();
}
/* Must be called with preempt disabled */
extern void kernel_fpu_disable(void);
extern void kernel_fpu_enable(void);
extern void kernel_fpu_begin(void);
extern void kernel_fpu_end(void);
extern bool irq_fpu_usable(void);
/*
* Some instructions like VIA's padlock instructions generate a spurious
......@@ -62,47 +33,16 @@ extern void kernel_fpu_enable(void);
* in interrupt context interacting wrongly with other user/kernel fpu usage, we
* should use them only in the context of irq_ts_save/restore()
*/
static inline int irq_ts_save(void)
{
/*
* If in process context and not atomic, we can take a spurious DNA fault.
* Otherwise, doing clts() in process context requires disabling preemption
* or some heavy lifting like kernel_fpu_begin()
*/
if (!in_atomic())
return 0;
if (read_cr0() & X86_CR0_TS) {
clts();
return 1;
}
return 0;
}
static inline void irq_ts_restore(int TS_state)
{
if (TS_state)
stts();
}
extern int irq_ts_save(void);
extern void irq_ts_restore(int TS_state);
/*
* The question "does this thread have fpu access?"
* is slightly racy, since preemption could come in
* and revoke it immediately after the test.
* Query the presence of one or more xfeatures. Works on any legacy CPU as well.
*
* However, even in that very unlikely scenario,
* we can just assume we have FPU access - typically
* to save the FP state - we'll just take a #NM
* fault and get the FPU access back.
* If 'feature_name' is set then put a human-readable description of
* the feature there as well - this can be used to print error (or success)
* messages.
*/
static inline int user_has_fpu(void)
{
return current->thread.fpu.has_fpu;
}
extern void unlazy_fpu(struct task_struct *tsk);
#endif /* __ASSEMBLY__ */
extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);
#endif /* _ASM_X86_I387_H */
#endif /* _ASM_X86_FPU_API_H */
arch/x86/include/asm/fpu/regset.h 0 → 100644
浏览文件 @ e75c73ad
/*
* FPU regset handling methods:
*/
#ifndef _ASM_X86_FPU_REGSET_H
#define _ASM_X86_FPU_REGSET_H
#include <linux/regset.h>
extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;
extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
xstateregs_get;
extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
xstateregs_set;
/*
* xstateregs_active == regset_fpregs_active. Please refer to the comment
* at the definition of regset_fpregs_active.
*/
#define xstateregs_active regset_fpregs_active
#endif /* _ASM_X86_FPU_REGSET_H */
arch/x86/include/asm/fpu/signal.h 0 → 100644
浏览文件 @ e75c73ad
/*
* x86 FPU signal frame handling methods:
*/
#ifndef _ASM_X86_FPU_SIGNAL_H
#define _ASM_X86_FPU_SIGNAL_H
#ifdef CONFIG_X86_64
# include <asm/sigcontext32.h>
# include <asm/user32.h>
struct ksignal;
int ia32_setup_rt_frame(int sig, struct ksignal *ksig,
compat_sigset_t *set, struct pt_regs *regs);
int ia32_setup_frame(int sig, struct ksignal *ksig,
compat_sigset_t *set, struct pt_regs *regs);
#else
# define user_i387_ia32_struct user_i387_struct
# define user32_fxsr_struct user_fxsr_struct
# define ia32_setup_frame __setup_frame
# define ia32_setup_rt_frame __setup_rt_frame
#endif
extern void convert_from_fxsr(struct user_i387_ia32_struct *env,
struct task_struct *tsk);
extern void convert_to_fxsr(struct task_struct *tsk,
const struct user_i387_ia32_struct *env);
unsigned long
fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
unsigned long *buf_fx, unsigned long *size);
extern void fpu__init_prepare_fx_sw_frame(void);
#endif /* _ASM_X86_FPU_SIGNAL_H */
arch/x86/include/asm/fpu/types.h 0 → 100644
浏览文件 @ e75c73ad
/*
* FPU data structures:
*/
#ifndef _ASM_X86_FPU_H
#define _ASM_X86_FPU_H
/*
* The legacy x87 FPU state format, as saved by FSAVE and
* restored by the FRSTOR instructions:
*/
struct fregs_state {
u32 cwd; /* FPU Control Word */
u32 swd; /* FPU Status Word */
u32 twd; /* FPU Tag Word */
u32 fip; /* FPU IP Offset */
u32 fcs; /* FPU IP Selector */
u32 foo; /* FPU Operand Pointer Offset */
u32 fos; /* FPU Operand Pointer Selector */
/* 8*10 bytes for each FP-reg = 80 bytes: */
u32 st_space[20];
/* Software status information [not touched by FSAVE]: */
u32 status;
};
/*
* The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
* restored by the FXRSTOR instructions. It's similar to the FSAVE
* format, but differs in some areas, plus has extensions at
* the end for the XMM registers.
*/
struct fxregs_state {
u16 cwd; /* Control Word */
u16 swd; /* Status Word */
u16 twd; /* Tag Word */
u16 fop; /* Last Instruction Opcode */
union {
struct {
u64 rip; /* Instruction Pointer */
u64 rdp; /* Data Pointer */
};
struct {
u32 fip; /* FPU IP Offset */
u32 fcs; /* FPU IP Selector */
u32 foo; /* FPU Operand Offset */
u32 fos; /* FPU Operand Selector */
};
};
u32 mxcsr; /* MXCSR Register State */
u32 mxcsr_mask; /* MXCSR Mask */
/* 8*16 bytes for each FP-reg = 128 bytes: */
u32 st_space[32];
/* 16*16 bytes for each XMM-reg = 256 bytes: */
u32 xmm_space[64];
u32 padding[12];
union {
u32 padding1[12];
u32 sw_reserved[12];
};
} __attribute__((aligned(16)));
/* Default value for fxregs_state.mxcsr: */
#define MXCSR_DEFAULT 0x1f80
/*
* Software based FPU emulation state. This is arbitrary really,
* it matches the x87 format to make it easier to understand:
*/
struct swregs_state {
u32 cwd;
u32 swd;
u32 twd;
u32 fip;
u32 fcs;
u32 foo;
u32 fos;
/* 8*10 bytes for each FP-reg = 80 bytes: */
u32 st_space[20];
u8 ftop;
u8 changed;
u8 lookahead;
u8 no_update;
u8 rm;
u8 alimit;
struct math_emu_info *info;
u32 entry_eip;
};
/*
* List of XSAVE features Linux knows about:
*/
enum xfeature_bit {
XSTATE_BIT_FP,
XSTATE_BIT_SSE,
XSTATE_BIT_YMM,
XSTATE_BIT_BNDREGS,
XSTATE_BIT_BNDCSR,
XSTATE_BIT_OPMASK,
XSTATE_BIT_ZMM_Hi256,
XSTATE_BIT_Hi16_ZMM,
XFEATURES_NR_MAX,
};
#define XSTATE_FP (1 << XSTATE_BIT_FP)
#define XSTATE_SSE (1 << XSTATE_BIT_SSE)
#define XSTATE_YMM (1 << XSTATE_BIT_YMM)
#define XSTATE_BNDREGS (1 << XSTATE_BIT_BNDREGS)
#define XSTATE_BNDCSR (1 << XSTATE_BIT_BNDCSR)
#define XSTATE_OPMASK (1 << XSTATE_BIT_OPMASK)
#define XSTATE_ZMM_Hi256 (1 << XSTATE_BIT_ZMM_Hi256)
#define XSTATE_Hi16_ZMM (1 << XSTATE_BIT_Hi16_ZMM)
#define XSTATE_FPSSE (XSTATE_FP | XSTATE_SSE)
#define XSTATE_AVX512 (XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM)
/*
* There are 16x 256-bit AVX registers named YMM0-YMM15.
* The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
* and are stored in 'struct fxregs_state::xmm_space[]'.
*
* The high 128 bits are stored here:
* 16x 128 bits == 256 bytes.
*/
struct ymmh_struct {
u8 ymmh_space[256];
};
/* We don't support LWP yet: */
struct lwp_struct {
u8 reserved[128];
};
/* Intel MPX support: */
struct bndreg {
u64 lower_bound;
u64 upper_bound;
} __packed;
struct bndcsr {
u64 bndcfgu;
u64 bndstatus;
} __packed;
struct mpx_struct {
struct bndreg bndreg[4];
struct bndcsr bndcsr;
};
struct xstate_header {
u64 xfeatures;
u64 xcomp_bv;
u64 reserved[6];
} __attribute__((packed));
/* New processor state extensions should be added here: */
#define XSTATE_RESERVE (sizeof(struct ymmh_struct) + \
sizeof(struct lwp_struct) + \
sizeof(struct mpx_struct) )
/*
* This is our most modern FPU state format, as saved by the XSAVE
* and restored by the XRSTOR instructions.
*
* It consists of a legacy fxregs portion, an xstate header and
* subsequent fixed size areas as defined by the xstate header.
* Not all CPUs support all the extensions.
*/
struct xregs_state {
struct fxregs_state i387;
struct xstate_header header;
u8 __reserved[XSTATE_RESERVE];
} __attribute__ ((packed, aligned (64)));
/*
* This is a union of all the possible FPU state formats
* put together, so that we can pick the right one runtime.
*
* The size of the structure is determined by the largest
* member - which is the xsave area:
*/
union fpregs_state {
struct fregs_state fsave;
struct fxregs_state fxsave;
struct swregs_state soft;
struct xregs_state xsave;
};
/*
* Highest level per task FPU state data structure that
* contains the FPU register state plus various FPU
* state fields:
*/
struct fpu {
/*
* @state:
*
* In-memory copy of all FPU registers that we save/restore
* over context switches. If the task is using the FPU then
* the registers in the FPU are more recent than this state
* copy. If the task context-switches away then they get
* saved here and represent the FPU state.
*
* After context switches there may be a (short) time period
* during which the in-FPU hardware registers are unchanged
* and still perfectly match this state, if the tasks
* scheduled afterwards are not using the FPU.
*
* This is the 'lazy restore' window of optimization, which
* we track though 'fpu_fpregs_owner_ctx' and 'fpu->last_cpu'.
*
* We detect whether a subsequent task uses the FPU via setting
* CR0::TS to 1, which causes any FPU use to raise a #NM fault.
*
* During this window, if the task gets scheduled again, we
* might be able to skip having to do a restore from this
* memory buffer to the hardware registers - at the cost of
* incurring the overhead of #NM fault traps.
*
* Note that on modern CPUs that support the XSAVEOPT (or other
* optimized XSAVE instructions), we don't use #NM traps anymore,
* as the hardware can track whether FPU registers need saving
* or not. On such CPUs we activate the non-lazy ('eagerfpu')
* logic, which unconditionally saves/restores all FPU state
* across context switches. (if FPU state exists.)
*/
union fpregs_state state;
/*
* @last_cpu:
*
* Records the last CPU on which this context was loaded into
* FPU registers. (In the lazy-restore case we might be
* able to reuse FPU registers across multiple context switches
* this way, if no intermediate task used the FPU.)
*
* A value of -1 is used to indicate that the FPU state in context
* memory is newer than the FPU state in registers, and that the
* FPU state should be reloaded next time the task is run.
*/
unsigned int last_cpu;
/*
* @fpstate_active:
*
* This flag indicates whether this context is active: if the task
* is not running then we can restore from this context, if the task
* is running then we should save into this context.
*/
unsigned char fpstate_active;
/*
* @fpregs_active:
*
* This flag determines whether a given context is actively
* loaded into the FPU's registers and that those registers
* represent the task's current FPU state.
*
* Note the interaction with fpstate_active:
*
* # task does not use the FPU:
* fpstate_active == 0
*
* # task uses the FPU and regs are active:
* fpstate_active == 1 && fpregs_active == 1
*
* # the regs are inactive but still match fpstate:
* fpstate_active == 1 && fpregs_active == 0 && fpregs_owner == fpu
*
* The third state is what we use for the lazy restore optimization
* on lazy-switching CPUs.
*/
unsigned char fpregs_active;
/*
* @counter:
*
* This counter contains the number of consecutive context switches
* during which the FPU stays used. If this is over a threshold, the
* lazy FPU restore logic becomes eager, to save the trap overhead.
* This is an unsigned char so that after 256 iterations the counter
* wraps and the context switch behavior turns lazy again; this is to
* deal with bursty apps that only use the FPU for a short time:
*/
unsigned char counter;
};
#endif /* _ASM_X86_FPU_H */
arch/x86/include/asm/fpu/xstate.h 0 → 100644
浏览文件 @ e75c73ad
#ifndef __ASM_X86_XSAVE_H
#define __ASM_X86_XSAVE_H
#include <linux/types.h>
#include <asm/processor.h>
#include <linux/uaccess.h>
/* Bit 63 of XCR0 is reserved for future expansion */
#define XSTATE_EXTEND_MASK (~(XSTATE_FPSSE | (1ULL << 63)))
#define XSTATE_CPUID 0x0000000d
#define FXSAVE_SIZE 512
#define XSAVE_HDR_SIZE 64
#define XSAVE_HDR_OFFSET FXSAVE_SIZE
#define XSAVE_YMM_SIZE 256
#define XSAVE_YMM_OFFSET (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)
/* Supported features which support lazy state saving */
#define XSTATE_LAZY (XSTATE_FP | XSTATE_SSE | XSTATE_YMM \
| XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM)
/* Supported features which require eager state saving */
#define XSTATE_EAGER (XSTATE_BNDREGS | XSTATE_BNDCSR)
/* All currently supported features */
#define XCNTXT_MASK (XSTATE_LAZY | XSTATE_EAGER)
#ifdef CONFIG_X86_64
#define REX_PREFIX "0x48, "
#else
#define REX_PREFIX
#endif
extern unsigned int xstate_size;
extern u64 xfeatures_mask;
extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
void *get_xsave_addr(struct xregs_state *xsave, int xstate);
const void *get_xsave_field_ptr(int xstate_field);
#endif
arch/x86/include/asm/kvm_host.h
浏览文件 @ e75c73ad
......@@ -1002,8 +1002,6 @@ void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
void kvm_inject_nmi(struct kvm_vcpu *vcpu);
int fx_init(struct kvm_vcpu *vcpu);
void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
const u8 *new, int bytes);
int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn);
......
arch/x86/include/asm/mmu_context.h
浏览文件 @ e75c73ad
......@@ -142,6 +142,19 @@ static inline void arch_exit_mmap(struct mm_struct *mm)
paravirt_arch_exit_mmap(mm);
}
#ifdef CONFIG_X86_64
static inline bool is_64bit_mm(struct mm_struct *mm)
{
return !config_enabled(CONFIG_IA32_EMULATION) ||
!(mm->context.ia32_compat == TIF_IA32);
}
#else
static inline bool is_64bit_mm(struct mm_struct *mm)
{
return false;
}
#endif
static inline void arch_bprm_mm_init(struct mm_struct *mm,
struct vm_area_struct *vma)
{
......
arch/x86/include/asm/mpx.h
浏览文件 @ e75c73ad
......@@ -13,55 +13,50 @@
#define MPX_BNDCFG_ENABLE_FLAG 0x1
#define MPX_BD_ENTRY_VALID_FLAG 0x1
#ifdef CONFIG_X86_64
/* upper 28 bits [47:20] of the virtual address in 64-bit used to
* index into bounds directory (BD).
*/
#define MPX_BD_ENTRY_OFFSET 28
#define MPX_BD_ENTRY_SHIFT 3
/* bits [19:3] of the virtual address in 64-bit used to index into
* bounds table (BT).
/*
* The upper 28 bits [47:20] of the virtual address in 64-bit
* are used to index into bounds directory (BD).
*
* The directory is 2G (2^31) in size, and with 8-byte entries
* it has 2^28 entries.
*/
#define MPX_BT_ENTRY_OFFSET 17
#define MPX_BT_ENTRY_SHIFT 5
#define MPX_IGN_BITS 3
#define MPX_BD_ENTRY_TAIL 3
#define MPX_BD_SIZE_BYTES_64 (1UL<<31)
#define MPX_BD_ENTRY_BYTES_64 8
#define MPX_BD_NR_ENTRIES_64 (MPX_BD_SIZE_BYTES_64/MPX_BD_ENTRY_BYTES_64)
#else
#define MPX_BD_ENTRY_OFFSET 20
#define MPX_BD_ENTRY_SHIFT 2
#define MPX_BT_ENTRY_OFFSET 10
#define MPX_BT_ENTRY_SHIFT 4
#define MPX_IGN_BITS 2
#define MPX_BD_ENTRY_TAIL 2
/*
* The 32-bit directory is 4MB (2^22) in size, and with 4-byte
* entries it has 2^20 entries.
*/
#define MPX_BD_SIZE_BYTES_32 (1UL<<22)
#define MPX_BD_ENTRY_BYTES_32 4
#define MPX_BD_NR_ENTRIES_32 (MPX_BD_SIZE_BYTES_32/MPX_BD_ENTRY_BYTES_32)
#endif
/*
* A 64-bit table is 4MB total in size, and an entry is
* 4 64-bit pointers in size.
*/
#define MPX_BT_SIZE_BYTES_64 (1UL<<22)
#define MPX_BT_ENTRY_BYTES_64 32
#define MPX_BT_NR_ENTRIES_64 (MPX_BT_SIZE_BYTES_64/MPX_BT_ENTRY_BYTES_64)
#define MPX_BD_SIZE_BYTES (1UL<<(MPX_BD_ENTRY_OFFSET+MPX_BD_ENTRY_SHIFT))
#define MPX_BT_SIZE_BYTES (1UL<<(MPX_BT_ENTRY_OFFSET+MPX_BT_ENTRY_SHIFT))
/*
* A 32-bit table is 16kB total in size, and an entry is
* 4 32-bit pointers in size.
*/
#define MPX_BT_SIZE_BYTES_32 (1UL<<14)
#define MPX_BT_ENTRY_BYTES_32 16
#define MPX_BT_NR_ENTRIES_32 (MPX_BT_SIZE_BYTES_32/MPX_BT_ENTRY_BYTES_32)
#define MPX_BNDSTA_TAIL 2
#define MPX_BNDCFG_TAIL 12
#define MPX_BNDSTA_ADDR_MASK (~((1UL<<MPX_BNDSTA_TAIL)-1))
#define MPX_BNDCFG_ADDR_MASK (~((1UL<<MPX_BNDCFG_TAIL)-1))
#define MPX_BT_ADDR_MASK (~((1UL<<MPX_BD_ENTRY_TAIL)-1))
#define MPX_BNDCFG_ADDR_MASK (~((1UL<<MPX_BNDCFG_TAIL)-1))
#define MPX_BNDSTA_ERROR_CODE 0x3
#define MPX_BD_ENTRY_MASK ((1<<MPX_BD_ENTRY_OFFSET)-1)
#define MPX_BT_ENTRY_MASK ((1<<MPX_BT_ENTRY_OFFSET)-1)
#define MPX_GET_BD_ENTRY_OFFSET(addr) ((((addr)>>(MPX_BT_ENTRY_OFFSET+ \
MPX_IGN_BITS)) & MPX_BD_ENTRY_MASK) << MPX_BD_ENTRY_SHIFT)
#define MPX_GET_BT_ENTRY_OFFSET(addr) ((((addr)>>MPX_IGN_BITS) & \
MPX_BT_ENTRY_MASK) << MPX_BT_ENTRY_SHIFT)
#ifdef CONFIG_X86_INTEL_MPX
siginfo_t *mpx_generate_siginfo(struct pt_regs *regs,
struct xsave_struct *xsave_buf);
int mpx_handle_bd_fault(struct xsave_struct *xsave_buf);
siginfo_t *mpx_generate_siginfo(struct pt_regs *regs);
int mpx_handle_bd_fault(void);
static inline int kernel_managing_mpx_tables(struct mm_struct *mm)
{
return (mm->bd_addr != MPX_INVALID_BOUNDS_DIR);
......@@ -77,12 +72,11 @@ static inline void mpx_mm_init(struct mm_struct *mm)
void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long start, unsigned long end);
#else
static inline siginfo_t *mpx_generate_siginfo(struct pt_regs *regs,
struct xsave_struct *xsave_buf)
static inline siginfo_t *mpx_generate_siginfo(struct pt_regs *regs)
{
return NULL;
}
static inline int mpx_handle_bd_fault(struct xsave_struct *xsave_buf)
static inline int mpx_handle_bd_fault(void)
{
return -EINVAL;
}
......
arch/x86/include/asm/processor.h
浏览文件 @ e75c73ad
......@@ -21,6 +21,7 @@ struct mm_struct;
#include <asm/desc_defs.h>
#include <asm/nops.h>
#include <asm/special_insns.h>
#include <asm/fpu/types.h>
#include <linux/personality.h>
#include <linux/cpumask.h>
......@@ -52,11 +53,16 @@ static inline void *current_text_addr(void)
return pc;
}
/*
* These alignment constraints are for performance in the vSMP case,
* but in the task_struct case we must also meet hardware imposed
* alignment requirements of the FPU state:
*/
#ifdef CONFIG_X86_VSMP
# define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT)
#else
# define ARCH_MIN_TASKALIGN 16
# define ARCH_MIN_TASKALIGN __alignof__(union fpregs_state)
# define ARCH_MIN_MMSTRUCT_ALIGN 0
#endif
......@@ -166,7 +172,6 @@ extern const struct seq_operations cpuinfo_op;
#define cache_line_size() (boot_cpu_data.x86_cache_alignment)
extern void cpu_detect(struct cpuinfo_x86 *c);
extern void fpu_detect(struct cpuinfo_x86 *c);
extern void early_cpu_init(void);
extern void identify_boot_cpu(void);
......@@ -313,128 +318,6 @@ struct orig_ist {
unsigned long ist[7];
};
#define MXCSR_DEFAULT 0x1f80
struct i387_fsave_struct {
u32 cwd; /* FPU Control Word */
u32 swd; /* FPU Status Word */
u32 twd; /* FPU Tag Word */
u32 fip; /* FPU IP Offset */
u32 fcs; /* FPU IP Selector */
u32 foo; /* FPU Operand Pointer Offset */
u32 fos; /* FPU Operand Pointer Selector */
/* 8*10 bytes for each FP-reg = 80 bytes: */
u32 st_space[20];
/* Software status information [not touched by FSAVE ]: */
u32 status;
};
struct i387_fxsave_struct {
u16 cwd; /* Control Word */
u16 swd; /* Status Word */
u16 twd; /* Tag Word */
u16 fop; /* Last Instruction Opcode */
union {
struct {
u64 rip; /* Instruction Pointer */
u64 rdp; /* Data Pointer */
};
struct {
u32 fip; /* FPU IP Offset */
u32 fcs; /* FPU IP Selector */
u32 foo; /* FPU Operand Offset */
u32 fos; /* FPU Operand Selector */
};
};
u32 mxcsr; /* MXCSR Register State */
u32 mxcsr_mask; /* MXCSR Mask */
/* 8*16 bytes for each FP-reg = 128 bytes: */
u32 st_space[32];
/* 16*16 bytes for each XMM-reg = 256 bytes: */
u32 xmm_space[64];
u32 padding[12];
union {
u32 padding1[12];
u32 sw_reserved[12];
};
} __attribute__((aligned(16)));
struct i387_soft_struct {
u32 cwd;
u32 swd;
u32 twd;
u32 fip;
u32 fcs;
u32 foo;
u32 fos;
/* 8*10 bytes for each FP-reg = 80 bytes: */
u32 st_space[20];
u8 ftop;
u8 changed;
u8 lookahead;
u8 no_update;
u8 rm;
u8 alimit;
struct math_emu_info *info;
u32 entry_eip;
};
struct ymmh_struct {
/* 16 * 16 bytes for each YMMH-reg = 256 bytes */
u32 ymmh_space[64];
};
/* We don't support LWP yet: */
struct lwp_struct {
u8 reserved[128];
};
struct bndreg {
u64 lower_bound;
u64 upper_bound;
} __packed;
struct bndcsr {
u64 bndcfgu;
u64 bndstatus;
} __packed;
struct xsave_hdr_struct {
u64 xstate_bv;
u64 xcomp_bv;
u64 reserved[6];
} __attribute__((packed));
struct xsave_struct {
struct i387_fxsave_struct i387;
struct xsave_hdr_struct xsave_hdr;
struct ymmh_struct ymmh;
struct lwp_struct lwp;
struct bndreg bndreg[4];
struct bndcsr bndcsr;
/* new processor state extensions will go here */
} __attribute__ ((packed, aligned (64)));
union thread_xstate {
struct i387_fsave_struct fsave;
struct i387_fxsave_struct fxsave;
struct i387_soft_struct soft;
struct xsave_struct xsave;
};
struct fpu {
unsigned int last_cpu;
unsigned int has_fpu;
union thread_xstate *state;
};
#ifdef CONFIG_X86_64
DECLARE_PER_CPU(struct orig_ist, orig_ist);
......@@ -483,8 +366,6 @@ DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
#endif /* X86_64 */
extern unsigned int xstate_size;
extern void free_thread_xstate(struct task_struct *);
extern struct kmem_cache *task_xstate_cachep;
struct perf_event;
......@@ -508,6 +389,10 @@ struct thread_struct {
unsigned long fs;
#endif
unsigned long gs;
/* Floating point and extended processor state */
struct fpu fpu;
/* Save middle states of ptrace breakpoints */
struct perf_event *ptrace_bps[HBP_NUM];
/* Debug status used for traps, single steps, etc... */
......@@ -518,8 +403,6 @@ struct thread_struct {
unsigned long cr2;
unsigned long trap_nr;
unsigned long error_code;
/* floating point and extended processor state */
struct fpu fpu;
#ifdef CONFIG_X86_32
/* Virtual 86 mode info */
struct vm86_struct __user *vm86_info;
......@@ -535,15 +418,6 @@ struct thread_struct {
unsigned long iopl;
/* Max allowed port in the bitmap, in bytes: */
unsigned io_bitmap_max;
/*
* fpu_counter contains the number of consecutive context switches
* that the FPU is used. If this is over a threshold, the lazy fpu
* saving becomes unlazy to save the trap. This is an unsigned char
* so that after 256 times the counter wraps and the behavior turns
* lazy again; this to deal with bursty apps that only use FPU for
* a short time
*/
unsigned char fpu_counter;
};
/*
......@@ -928,18 +802,18 @@ extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);
/* Register/unregister a process' MPX related resource */
#define MPX_ENABLE_MANAGEMENT(tsk) mpx_enable_management((tsk))
#define MPX_DISABLE_MANAGEMENT(tsk) mpx_disable_management((tsk))
#define MPX_ENABLE_MANAGEMENT() mpx_enable_management()
#define MPX_DISABLE_MANAGEMENT() mpx_disable_management()
#ifdef CONFIG_X86_INTEL_MPX
extern int mpx_enable_management(struct task_struct *tsk);
extern int mpx_disable_management(struct task_struct *tsk);
extern int mpx_enable_management(void);
extern int mpx_disable_management(void);
#else
static inline int mpx_enable_management(struct task_struct *tsk)
static inline int mpx_enable_management(void)
{
return -EINVAL;
}
static inline int mpx_disable_management(struct task_struct *tsk)
static inline int mpx_disable_management(void)
{
return -EINVAL;
}
......
arch/x86/include/asm/simd.h
浏览文件 @ e75c73ad
#include <asm/i387.h>
#include <asm/fpu/api.h>
/*
* may_use_simd - whether it is allowable at this time to issue SIMD
......
arch/x86/include/asm/stackprotector.h
浏览文件 @ e75c73ad
......@@ -39,7 +39,9 @@
#include <asm/processor.h>
#include <asm/percpu.h>
#include <asm/desc.h>
#include <linux/random.h>
#include <linux/sched.h>
/*
* 24 byte read-only segment initializer for stack canary. Linker
......
arch/x86/include/asm/suspend_32.h
浏览文件 @ e75c73ad
......@@ -7,7 +7,7 @@
#define _ASM_X86_SUSPEND_32_H
#include <asm/desc.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
/* image of the saved processor state */
struct saved_context {
......
arch/x86/include/asm/suspend_64.h
浏览文件 @ e75c73ad
......@@ -7,7 +7,7 @@
#define _ASM_X86_SUSPEND_64_H
#include <asm/desc.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
/*
* Image of the saved processor state, used by the low level ACPI suspend to
......
arch/x86/include/asm/trace/mpx.h 0 → 100644
浏览文件 @ e75c73ad
#undef TRACE_SYSTEM
#define TRACE_SYSTEM mpx
#if !defined(_TRACE_MPX_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_MPX_H
#include <linux/tracepoint.h>
#ifdef CONFIG_X86_INTEL_MPX
TRACE_EVENT(mpx_bounds_register_exception,
TP_PROTO(void *addr_referenced,
const struct bndreg *bndreg),
TP_ARGS(addr_referenced, bndreg),
TP_STRUCT__entry(
__field(void *, addr_referenced)
__field(u64, lower_bound)
__field(u64, upper_bound)
),
TP_fast_assign(
__entry->addr_referenced = addr_referenced;
__entry->lower_bound = bndreg->lower_bound;
__entry->upper_bound = bndreg->upper_bound;
),
/*
* Note that we are printing out the '~' of the upper
* bounds register here. It is actually stored in its
* one's complement form so that its 'init' state
* corresponds to all 0's. But, that looks like
* gibberish when printed out, so print out the 1's
* complement instead of the actual value here. Note
* though that you still need to specify filters for the
* actual value, not the displayed one.
*/
TP_printk("address referenced: 0x%p bounds: lower: 0x%llx ~upper: 0x%llx",
__entry->addr_referenced,
__entry->lower_bound,
~__entry->upper_bound
)
);
TRACE_EVENT(bounds_exception_mpx,
TP_PROTO(const struct bndcsr *bndcsr),
TP_ARGS(bndcsr),
TP_STRUCT__entry(
__field(u64, bndcfgu)
__field(u64, bndstatus)
),
TP_fast_assign(
/* need to get rid of the 'const' on bndcsr */
__entry->bndcfgu = (u64)bndcsr->bndcfgu;
__entry->bndstatus = (u64)bndcsr->bndstatus;
),
TP_printk("bndcfgu:0x%llx bndstatus:0x%llx",
__entry->bndcfgu,
__entry->bndstatus)
);
DECLARE_EVENT_CLASS(mpx_range_trace,
TP_PROTO(unsigned long start,
unsigned long end),
TP_ARGS(start, end),
TP_STRUCT__entry(
__field(unsigned long, start)
__field(unsigned long, end)
),
TP_fast_assign(
__entry->start = start;
__entry->end = end;
),
TP_printk("[0x%p:0x%p]",
(void *)__entry->start,
(void *)__entry->end
)
);
DEFINE_EVENT(mpx_range_trace, mpx_unmap_zap,
TP_PROTO(unsigned long start, unsigned long end),
TP_ARGS(start, end)
);
DEFINE_EVENT(mpx_range_trace, mpx_unmap_search,
TP_PROTO(unsigned long start, unsigned long end),
TP_ARGS(start, end)
);
TRACE_EVENT(mpx_new_bounds_table,
TP_PROTO(unsigned long table_vaddr),
TP_ARGS(table_vaddr),
TP_STRUCT__entry(
__field(unsigned long, table_vaddr)
),
TP_fast_assign(
__entry->table_vaddr = table_vaddr;
),
TP_printk("table vaddr:%p", (void *)__entry->table_vaddr)
);
#else
/*
* This gets used outside of MPX-specific code, so we need a stub.
*/
static inline void trace_bounds_exception_mpx(const struct bndcsr *bndcsr)
{
}
#endif /* CONFIG_X86_INTEL_MPX */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH asm/trace/
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE mpx
#endif /* _TRACE_MPX_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
arch/x86/include/asm/user.h
浏览文件 @ e75c73ad
......@@ -14,8 +14,8 @@ struct user_ymmh_regs {
__u32 ymmh_space[64];
};
struct user_xsave_hdr {
__u64 xstate_bv;
struct user_xstate_header {
__u64 xfeatures;
__u64 reserved1[2];
__u64 reserved2[5];
};
......@@ -41,11 +41,11 @@ struct user_xsave_hdr {
* particular process/thread.
*
* Also when the user modifies certain state FP/SSE/etc through the
* ptrace interface, they must ensure that the xsave_hdr.xstate_bv
* ptrace interface, they must ensure that the header.xfeatures
* bytes[512..519] of the memory layout are updated correspondingly.
* i.e., for example when FP state is modified to a non-init state,
* xsave_hdr.xstate_bv's bit 0 must be set to '1', when SSE is modified to
* non-init state, xsave_hdr.xstate_bv's bit 1 must to be set to '1', etc.
* header.xfeatures's bit 0 must be set to '1', when SSE is modified to
* non-init state, header.xfeatures's bit 1 must to be set to '1', etc.
*/
#define USER_XSTATE_FX_SW_WORDS 6
#define USER_XSTATE_XCR0_WORD 0
......@@ -55,7 +55,7 @@ struct user_xstateregs {
__u64 fpx_space[58];
__u64 xstate_fx_sw[USER_XSTATE_FX_SW_WORDS];
} i387;
struct user_xsave_hdr xsave_hdr;
struct user_xstate_header header;
struct user_ymmh_regs ymmh;
/* further processor state extensions go here */
};
......
arch/x86/include/asm/xcr.h 已删除 100644 → 0
浏览文件 @ cfe3eceb
/* -*- linux-c -*- ------------------------------------------------------- *
*
* Copyright 2008 rPath, Inc. - All Rights Reserved
*
* This file is part of the Linux kernel, and is made available under
* the terms of the GNU General Public License version 2 or (at your
* option) any later version; incorporated herein by reference.
*
* ----------------------------------------------------------------------- */
/*
* asm-x86/xcr.h
*
* Definitions for the eXtended Control Register instructions
*/
#ifndef _ASM_X86_XCR_H
#define _ASM_X86_XCR_H
#define XCR_XFEATURE_ENABLED_MASK 0x00000000
#ifdef __KERNEL__
# ifndef __ASSEMBLY__
#include <linux/types.h>
static inline u64 xgetbv(u32 index)
{
u32 eax, edx;
asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
: "=a" (eax), "=d" (edx)
: "c" (index));
return eax + ((u64)edx << 32);
}
static inline void xsetbv(u32 index, u64 value)
{
u32 eax = value;
u32 edx = value >> 32;
asm volatile(".byte 0x0f,0x01,0xd1" /* xsetbv */
: : "a" (eax), "d" (edx), "c" (index));
}
# endif /* __ASSEMBLY__ */
#endif /* __KERNEL__ */
#endif /* _ASM_X86_XCR_H */
arch/x86/include/asm/xor.h
浏览文件 @ e75c73ad
......@@ -36,7 +36,7 @@
* no advantages to be gotten from x86-64 here anyways.
*/
#include <asm/i387.h>
#include <asm/fpu/api.h>
#ifdef CONFIG_X86_32
/* reduce register pressure */
......
arch/x86/include/asm/xor_32.h
浏览文件 @ e75c73ad
......@@ -26,7 +26,7 @@
#define XO3(x, y) " pxor 8*("#x")(%4), %%mm"#y" ;\n"
#define XO4(x, y) " pxor 8*("#x")(%5), %%mm"#y" ;\n"
#include <asm/i387.h>
#include <asm/fpu/api.h>
static void
xor_pII_mmx_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
......
arch/x86/include/asm/xor_avx.h
浏览文件 @ e75c73ad
......@@ -18,7 +18,7 @@
#ifdef CONFIG_AS_AVX
#include <linux/compiler.h>
#include <asm/i387.h>
#include <asm/fpu/api.h>
#define BLOCK4(i) \
BLOCK(32 * i, 0) \
......
arch/x86/include/asm/xsave.h 已删除 100644 → 0
浏览文件 @ cfe3eceb
#ifndef __ASM_X86_XSAVE_H
#define __ASM_X86_XSAVE_H
#include <linux/types.h>
#include <asm/processor.h>
#define XSTATE_CPUID 0x0000000d
#define XSTATE_FP 0x1
#define XSTATE_SSE 0x2
#define XSTATE_YMM 0x4
#define XSTATE_BNDREGS 0x8
#define XSTATE_BNDCSR 0x10
#define XSTATE_OPMASK 0x20
#define XSTATE_ZMM_Hi256 0x40
#define XSTATE_Hi16_ZMM 0x80
#define XSTATE_FPSSE (XSTATE_FP | XSTATE_SSE)
#define XSTATE_AVX512 (XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM)
/* Bit 63 of XCR0 is reserved for future expansion */
#define XSTATE_EXTEND_MASK (~(XSTATE_FPSSE | (1ULL << 63)))
#define FXSAVE_SIZE 512
#define XSAVE_HDR_SIZE 64
#define XSAVE_HDR_OFFSET FXSAVE_SIZE
#define XSAVE_YMM_SIZE 256
#define XSAVE_YMM_OFFSET (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)
/* Supported features which support lazy state saving */
#define XSTATE_LAZY (XSTATE_FP | XSTATE_SSE | XSTATE_YMM \
| XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM)
/* Supported features which require eager state saving */
#define XSTATE_EAGER (XSTATE_BNDREGS | XSTATE_BNDCSR)
/* All currently supported features */
#define XCNTXT_MASK (XSTATE_LAZY | XSTATE_EAGER)
#ifdef CONFIG_X86_64
#define REX_PREFIX "0x48, "
#else
#define REX_PREFIX
#endif
extern unsigned int xstate_size;
extern u64 pcntxt_mask;
extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
extern struct xsave_struct *init_xstate_buf;
extern void xsave_init(void);
extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
extern int init_fpu(struct task_struct *child);
/* These macros all use (%edi)/(%rdi) as the single memory argument. */
#define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27"
#define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37"
#define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f"
#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f"
#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f"
#define xstate_fault ".section .fixup,\"ax\"\n" \
"3: movl $-1,%[err]\n" \
" jmp 2b\n" \
".previous\n" \
_ASM_EXTABLE(1b, 3b) \
: [err] "=r" (err)
/*
* This function is called only during boot time when x86 caps are not set
* up and alternative can not be used yet.
*/
static inline int xsave_state_booting(struct xsave_struct *fx, u64 mask)
{
u32 lmask = mask;
u32 hmask = mask >> 32;
int err = 0;
WARN_ON(system_state != SYSTEM_BOOTING);
if (boot_cpu_has(X86_FEATURE_XSAVES))
asm volatile("1:"XSAVES"\n\t"
"2:\n\t"
xstate_fault
: "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
: "memory");
else
asm volatile("1:"XSAVE"\n\t"
"2:\n\t"
xstate_fault
: "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
: "memory");
return err;
}
/*
* This function is called only during boot time when x86 caps are not set
* up and alternative can not be used yet.
*/
static inline int xrstor_state_booting(struct xsave_struct *fx, u64 mask)
{
u32 lmask = mask;
u32 hmask = mask >> 32;
int err = 0;
WARN_ON(system_state != SYSTEM_BOOTING);
if (boot_cpu_has(X86_FEATURE_XSAVES))
asm volatile("1:"XRSTORS"\n\t"
"2:\n\t"
xstate_fault
: "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
: "memory");
else
asm volatile("1:"XRSTOR"\n\t"
"2:\n\t"
xstate_fault
: "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
: "memory");
return err;
}
/*
* Save processor xstate to xsave area.
*/
static inline int xsave_state(struct xsave_struct *fx, u64 mask)
{
u32 lmask = mask;
u32 hmask = mask >> 32;
int err = 0;
/*
* If xsaves is enabled, xsaves replaces xsaveopt because
* it supports compact format and supervisor states in addition to
* modified optimization in xsaveopt.
*
* Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave
* because xsaveopt supports modified optimization which is not
* supported by xsave.
*
* If none of xsaves and xsaveopt is enabled, use xsave.
*/
alternative_input_2(
"1:"XSAVE,
XSAVEOPT,
X86_FEATURE_XSAVEOPT,
XSAVES,
X86_FEATURE_XSAVES,
[fx] "D" (fx), "a" (lmask), "d" (hmask) :
"memory");
asm volatile("2:\n\t"
xstate_fault
: "0" (0)
: "memory");
return err;
}
/*
* Restore processor xstate from xsave area.
*/
static inline int xrstor_state(struct xsave_struct *fx, u64 mask)
{
int err = 0;
u32 lmask = mask;
u32 hmask = mask >> 32;
/*
* Use xrstors to restore context if it is enabled. xrstors supports
* compacted format of xsave area which is not supported by xrstor.
*/
alternative_input(
"1: " XRSTOR,
XRSTORS,
X86_FEATURE_XSAVES,
"D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
: "memory");
asm volatile("2:\n"
xstate_fault
: "0" (0)
: "memory");
return err;
}
/*
* Save xstate context for old process during context switch.
*/
static inline void fpu_xsave(struct fpu *fpu)
{
xsave_state(&fpu->state->xsave, -1);
}
/*
* Restore xstate context for new process during context switch.
*/
static inline int fpu_xrstor_checking(struct xsave_struct *fx)
{
return xrstor_state(fx, -1);
}
/*
* Save xstate to user space xsave area.
*
* We don't use modified optimization because xrstor/xrstors might track
* a different application.
*
* We don't use compacted format xsave area for
* backward compatibility for old applications which don't understand
* compacted format of xsave area.
*/
static inline int xsave_user(struct xsave_struct __user *buf)
{
int err;
/*
* Clear the xsave header first, so that reserved fields are
* initialized to zero.
*/
err = __clear_user(&buf->xsave_hdr, sizeof(buf->xsave_hdr));
if (unlikely(err))
return -EFAULT;
__asm__ __volatile__(ASM_STAC "\n"
"1:"XSAVE"\n"
"2: " ASM_CLAC "\n"
xstate_fault
: "D" (buf), "a" (-1), "d" (-1), "0" (0)
: "memory");
return err;
}
/*
* Restore xstate from user space xsave area.
*/
static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask)
{
int err = 0;
struct xsave_struct *xstate = ((__force struct xsave_struct *)buf);
u32 lmask = mask;
u32 hmask = mask >> 32;
__asm__ __volatile__(ASM_STAC "\n"
"1:"XRSTOR"\n"
"2: " ASM_CLAC "\n"
xstate_fault
: "D" (xstate), "a" (lmask), "d" (hmask), "0" (0)
: "memory"); /* memory required? */
return err;
}
void *get_xsave_addr(struct xsave_struct *xsave, int xstate);
void setup_xstate_comp(void);
#endif
arch/x86/include/uapi/asm/sigcontext.h
浏览文件 @ e75c73ad
......@@ -25,7 +25,7 @@ struct _fpx_sw_bytes {
__u32 extended_size; /* total size of the layout referred by
* fpstate pointer in the sigcontext.
*/
__u64 xstate_bv;
__u64 xfeatures;
/* feature bit mask (including fp/sse/extended
* state) that is present in the memory
* layout.
......@@ -209,8 +209,8 @@ struct sigcontext {
#endif /* !__i386__ */
struct _xsave_hdr {
__u64 xstate_bv;
struct _header {
__u64 xfeatures;
__u64 reserved1[2];
__u64 reserved2[5];
};
......@@ -228,7 +228,7 @@ struct _ymmh_state {
*/
struct _xstate {
struct _fpstate fpstate;
struct _xsave_hdr xstate_hdr;
struct _header xstate_hdr;
struct _ymmh_state ymmh;
/* new processor state extensions go here */
};
......
arch/x86/kernel/Makefile
浏览文件 @ e75c73ad
......@@ -44,7 +44,7 @@ obj-y += pci-iommu_table.o
obj-y += resource.o
obj-y += process.o
obj-y += i387.o xsave.o
obj-y += fpu/
obj-y += ptrace.o
obj-$(CONFIG_X86_32) += tls.o
obj-$(CONFIG_IA32_EMULATION) += tls.o
......
arch/x86/kernel/alternative.c
浏览文件 @ e75c73ad
......@@ -21,6 +21,10 @@
#include <asm/io.h>
#include <asm/fixmap.h>
int __read_mostly alternatives_patched;
EXPORT_SYMBOL_GPL(alternatives_patched);
#define MAX_PATCH_LEN (255-1)
static int __initdata_or_module debug_alternative;
......@@ -627,6 +631,7 @@ void __init alternative_instructions(void)
apply_paravirt(__parainstructions, __parainstructions_end);
restart_nmi();
alternatives_patched = 1;
}
/**
......
arch/x86/kernel/cpu/bugs.c
浏览文件 @ e75c73ad
......@@ -12,57 +12,11 @@
#include <asm/bugs.h>
#include <asm/processor.h>
#include <asm/processor-flags.h>
#include <asm/i387.h>
#include <asm/fpu/internal.h>
#include <asm/msr.h>
#include <asm/paravirt.h>
#include <asm/alternative.h>
static double __initdata x = 4195835.0;
static double __initdata y = 3145727.0;
/*
* This used to check for exceptions..
* However, it turns out that to support that,
* the XMM trap handlers basically had to
* be buggy. So let's have a correct XMM trap
* handler, and forget about printing out
* some status at boot.
*
* We should really only care about bugs here
* anyway. Not features.
*/
static void __init check_fpu(void)
{
s32 fdiv_bug;
kernel_fpu_begin();
/*
* trap_init() enabled FXSR and company _before_ testing for FP
* problems here.
*
* Test for the divl bug: http://en.wikipedia.org/wiki/Fdiv_bug
*/
__asm__("fninit\n\t"
"fldl %1\n\t"
"fdivl %2\n\t"
"fmull %2\n\t"
"fldl %1\n\t"
"fsubp %%st,%%st(1)\n\t"
"fistpl %0\n\t"
"fwait\n\t"
"fninit"
: "=m" (*&fdiv_bug)
: "m" (*&x), "m" (*&y));
kernel_fpu_end();
if (fdiv_bug) {
set_cpu_bug(&boot_cpu_data, X86_BUG_FDIV);
pr_warn("Hmm, FPU with FDIV bug\n");
}
}
void __init check_bugs(void)
{
identify_boot_cpu();
......@@ -85,10 +39,5 @@ void __init check_bugs(void)
'0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
alternative_instructions();
/*
* kernel_fpu_begin/end() in check_fpu() relies on the patched
* alternative instructions.
*/
if (cpu_has_fpu)
check_fpu();
fpu__init_check_bugs();
}
arch/x86/kernel/cpu/common.c
浏览文件 @ e75c73ad
......@@ -32,8 +32,7 @@
#include <asm/setup.h>
#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/i387.h>
#include <asm/fpu-internal.h>
#include <asm/fpu/internal.h>
#include <asm/mtrr.h>
#include <linux/numa.h>
#include <asm/asm.h>
......@@ -146,32 +145,21 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
} };
EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
static int __init x86_xsave_setup(char *s)
static int __init x86_mpx_setup(char *s)
{
/* require an exact match without trailing characters */
if (strlen(s))
return 0;
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
setup_clear_cpu_cap(X86_FEATURE_XSAVES);
setup_clear_cpu_cap(X86_FEATURE_AVX);
setup_clear_cpu_cap(X86_FEATURE_AVX2);
return 1;
}
__setup("noxsave", x86_xsave_setup);
static int __init x86_xsaveopt_setup(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
return 1;
}
__setup("noxsaveopt", x86_xsaveopt_setup);
/* do not emit a message if the feature is not present */
if (!boot_cpu_has(X86_FEATURE_MPX))
return 1;
static int __init x86_xsaves_setup(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_XSAVES);
setup_clear_cpu_cap(X86_FEATURE_MPX);
pr_info("nompx: Intel Memory Protection Extensions (MPX) disabled\n");
return 1;
}
__setup("noxsaves", x86_xsaves_setup);
__setup("nompx", x86_mpx_setup);
#ifdef CONFIG_X86_32
static int cachesize_override = -1;
......@@ -184,14 +172,6 @@ static int __init cachesize_setup(char *str)
}
__setup("cachesize=", cachesize_setup);
static int __init x86_fxsr_setup(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_FXSR);
setup_clear_cpu_cap(X86_FEATURE_XMM);
return 1;
}
__setup("nofxsr", x86_fxsr_setup);
static int __init x86_sep_setup(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_SEP);
......@@ -762,7 +742,7 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
cpu_detect(c);
get_cpu_vendor(c);
get_cpu_cap(c);
fpu_detect(c);
fpu__init_system(c);
if (this_cpu->c_early_init)
this_cpu->c_early_init(c);
......@@ -1186,8 +1166,6 @@ DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
/*
* Special IST stacks which the CPU switches to when it calls
* an IST-marked descriptor entry. Up to 7 stacks (hardware
......@@ -1278,7 +1256,6 @@ DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
/*
* On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
......@@ -1442,7 +1419,7 @@ void cpu_init(void)
clear_all_debug_regs();
dbg_restore_debug_regs();
fpu_init();
fpu__init_cpu();
if (is_uv_system())
uv_cpu_init();
......@@ -1498,7 +1475,7 @@ void cpu_init(void)
clear_all_debug_regs();
dbg_restore_debug_regs();
fpu_init();
fpu__init_cpu();
}
#endif
......
arch/x86/kernel/fpu/Makefile 0 → 100644
浏览文件 @ e75c73ad
#
# Build rules for the FPU support code:
#
obj-y += init.o bugs.o core.o regset.o signal.o xstate.o
arch/x86/kernel/fpu/bugs.c 0 → 100644
浏览文件 @ e75c73ad
/*
* x86 FPU bug checks:
*/
#include <asm/fpu/internal.h>
/*
* Boot time CPU/FPU FDIV bug detection code:
*/
static double __initdata x = 4195835.0;
static double __initdata y = 3145727.0;
/*
* This used to check for exceptions..
* However, it turns out that to support that,
* the XMM trap handlers basically had to
* be buggy. So let's have a correct XMM trap
* handler, and forget about printing out
* some status at boot.
*
* We should really only care about bugs here
* anyway. Not features.
*/
static void __init check_fpu(void)
{
u32 cr0_saved;
s32 fdiv_bug;
/* We might have CR0::TS set already, clear it: */
cr0_saved = read_cr0();
write_cr0(cr0_saved & ~X86_CR0_TS);
kernel_fpu_begin();
/*
* trap_init() enabled FXSR and company _before_ testing for FP
* problems here.
*
* Test for the divl bug: http://en.wikipedia.org/wiki/Fdiv_bug
*/
__asm__("fninit\n\t"
"fldl %1\n\t"
"fdivl %2\n\t"
"fmull %2\n\t"
"fldl %1\n\t"
"fsubp %%st,%%st(1)\n\t"
"fistpl %0\n\t"
"fwait\n\t"
"fninit"
: "=m" (*&fdiv_bug)
: "m" (*&x), "m" (*&y));
kernel_fpu_end();
write_cr0(cr0_saved);
if (fdiv_bug) {
set_cpu_bug(&boot_cpu_data, X86_BUG_FDIV);
pr_warn("Hmm, FPU with FDIV bug\n");
}
}
void __init fpu__init_check_bugs(void)
{
/*
* kernel_fpu_begin/end() in check_fpu() relies on the patched
* alternative instructions.
*/
if (cpu_has_fpu)
check_fpu();
}
arch/x86/kernel/fpu/core.c 0 → 100644
浏览文件 @ e75c73ad
/*
* Copyright (C) 1994 Linus Torvalds
*
* Pentium III FXSR, SSE support
* General FPU state handling cleanups
* Gareth Hughes <gareth@valinux.com>, May 2000
*/
#include <asm/fpu/internal.h>
#include <asm/fpu/regset.h>
#include <asm/fpu/signal.h>
#include <asm/traps.h>
#include <linux/hardirq.h>
/*
* Represents the initial FPU state. It's mostly (but not completely) zeroes,
* depending on the FPU hardware format:
*/
union fpregs_state init_fpstate __read_mostly;
/*
* Track whether the kernel is using the FPU state
* currently.
*
* This flag is used:
*
* - by IRQ context code to potentially use the FPU
* if it's unused.
*
* - to debug kernel_fpu_begin()/end() correctness
*/
static DEFINE_PER_CPU(bool, in_kernel_fpu);
/*
* Track which context is using the FPU on the CPU:
*/
DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
static void kernel_fpu_disable(void)
{
WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
this_cpu_write(in_kernel_fpu, true);
}
static void kernel_fpu_enable(void)
{
WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
this_cpu_write(in_kernel_fpu, false);
}
static bool kernel_fpu_disabled(void)
{
return this_cpu_read(in_kernel_fpu);
}
/*
* Were we in an interrupt that interrupted kernel mode?
*
* On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
* pair does nothing at all: the thread must not have fpu (so
* that we don't try to save the FPU state), and TS must
* be set (so that the clts/stts pair does nothing that is
* visible in the interrupted kernel thread).
*
* Except for the eagerfpu case when we return true; in the likely case
* the thread has FPU but we are not going to set/clear TS.
*/
static bool interrupted_kernel_fpu_idle(void)
{
if (kernel_fpu_disabled())
return false;
if (use_eager_fpu())
return true;
return !current->thread.fpu.fpregs_active && (read_cr0() & X86_CR0_TS);
}
/*
* Were we in user mode (or vm86 mode) when we were
* interrupted?
*
* Doing kernel_fpu_begin/end() is ok if we are running
* in an interrupt context from user mode - we'll just
* save the FPU state as required.
*/
static bool interrupted_user_mode(void)
{
struct pt_regs *regs = get_irq_regs();
return regs && user_mode(regs);
}
/*
* Can we use the FPU in kernel mode with the
* whole "kernel_fpu_begin/end()" sequence?
*
* It's always ok in process context (ie "not interrupt")
* but it is sometimes ok even from an irq.
*/
bool irq_fpu_usable(void)
{
return !in_interrupt() ||
interrupted_user_mode() ||
interrupted_kernel_fpu_idle();
}
EXPORT_SYMBOL(irq_fpu_usable);
void __kernel_fpu_begin(void)
{
struct fpu *fpu = &current->thread.fpu;
WARN_ON_FPU(!irq_fpu_usable());
kernel_fpu_disable();
if (fpu->fpregs_active) {
copy_fpregs_to_fpstate(fpu);
} else {
this_cpu_write(fpu_fpregs_owner_ctx, NULL);
__fpregs_activate_hw();
}
}
EXPORT_SYMBOL(__kernel_fpu_begin);
void __kernel_fpu_end(void)
{
struct fpu *fpu = &current->thread.fpu;
if (fpu->fpregs_active)
copy_kernel_to_fpregs(&fpu->state);
else
__fpregs_deactivate_hw();
kernel_fpu_enable();
}
EXPORT_SYMBOL(__kernel_fpu_end);
void kernel_fpu_begin(void)
{
preempt_disable();
__kernel_fpu_begin();
}
EXPORT_SYMBOL_GPL(kernel_fpu_begin);
void kernel_fpu_end(void)
{
__kernel_fpu_end();
preempt_enable();
}
EXPORT_SYMBOL_GPL(kernel_fpu_end);
/*
* CR0::TS save/restore functions:
*/
int irq_ts_save(void)
{
/*
* If in process context and not atomic, we can take a spurious DNA fault.
* Otherwise, doing clts() in process context requires disabling preemption
* or some heavy lifting like kernel_fpu_begin()
*/
if (!in_atomic())
return 0;
if (read_cr0() & X86_CR0_TS) {
clts();
return 1;
}
return 0;
}
EXPORT_SYMBOL_GPL(irq_ts_save);
void irq_ts_restore(int TS_state)
{
if (TS_state)
stts();
}
EXPORT_SYMBOL_GPL(irq_ts_restore);
/*
* Save the FPU state (mark it for reload if necessary):
*
* This only ever gets called for the current task.
*/
void fpu__save(struct fpu *fpu)
{
WARN_ON_FPU(fpu != &current->thread.fpu);
preempt_disable();
if (fpu->fpregs_active) {
if (!copy_fpregs_to_fpstate(fpu))
fpregs_deactivate(fpu);
}
preempt_enable();
}
EXPORT_SYMBOL_GPL(fpu__save);
/*
* Legacy x87 fpstate state init:
*/
static inline void fpstate_init_fstate(struct fregs_state *fp)
{
fp->cwd = 0xffff037fu;
fp->swd = 0xffff0000u;
fp->twd = 0xffffffffu;
fp->fos = 0xffff0000u;
}
void fpstate_init(union fpregs_state *state)
{
if (!cpu_has_fpu) {
fpstate_init_soft(&state->soft);
return;
}
memset(state, 0, xstate_size);
if (cpu_has_fxsr)
fpstate_init_fxstate(&state->fxsave);
else
fpstate_init_fstate(&state->fsave);
}
EXPORT_SYMBOL_GPL(fpstate_init);
/*
* Copy the current task's FPU state to a new task's FPU context.
*
* In both the 'eager' and the 'lazy' case we save hardware registers
* directly to the destination buffer.
*/
static void fpu_copy(struct fpu *dst_fpu, struct fpu *src_fpu)
{
WARN_ON_FPU(src_fpu != &current->thread.fpu);
/*
* Don't let 'init optimized' areas of the XSAVE area
* leak into the child task:
*/
if (use_eager_fpu())
memset(&dst_fpu->state.xsave, 0, xstate_size);
/*
* Save current FPU registers directly into the child
* FPU context, without any memory-to-memory copying.
*
* If the FPU context got destroyed in the process (FNSAVE
* done on old CPUs) then copy it back into the source
* context and mark the current task for lazy restore.
*
* We have to do all this with preemption disabled,
* mostly because of the FNSAVE case, because in that
* case we must not allow preemption in the window
* between the FNSAVE and us marking the context lazy.
*
* It shouldn't be an issue as even FNSAVE is plenty
* fast in terms of critical section length.
*/
preempt_disable();
if (!copy_fpregs_to_fpstate(dst_fpu)) {
memcpy(&src_fpu->state, &dst_fpu->state, xstate_size);
fpregs_deactivate(src_fpu);
}
preempt_enable();
}
int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
{
dst_fpu->counter = 0;
dst_fpu->fpregs_active = 0;
dst_fpu->last_cpu = -1;
if (src_fpu->fpstate_active)
fpu_copy(dst_fpu, src_fpu);
return 0;
}
/*
* Activate the current task's in-memory FPU context,
* if it has not been used before:
*/
void fpu__activate_curr(struct fpu *fpu)
{
WARN_ON_FPU(fpu != &current->thread.fpu);
if (!fpu->fpstate_active) {
fpstate_init(&fpu->state);
/* Safe to do for the current task: */
fpu->fpstate_active = 1;
}
}
EXPORT_SYMBOL_GPL(fpu__activate_curr);
/*
* This function must be called before we read a task's fpstate.
*
* If the task has not used the FPU before then initialize its
* fpstate.
*
* If the task has used the FPU before then save it.
*/
void fpu__activate_fpstate_read(struct fpu *fpu)
{
/*
* If fpregs are active (in the current CPU), then
* copy them to the fpstate:
*/
if (fpu->fpregs_active) {
fpu__save(fpu);
} else {
if (!fpu->fpstate_active) {
fpstate_init(&fpu->state);
/* Safe to do for current and for stopped child tasks: */
fpu->fpstate_active = 1;
}
}
}
/*
* This function must be called before we write a task's fpstate.
*
* If the task has used the FPU before then unlazy it.
* If the task has not used the FPU before then initialize its fpstate.
*
* After this function call, after registers in the fpstate are
* modified and the child task has woken up, the child task will
* restore the modified FPU state from the modified context. If we
* didn't clear its lazy status here then the lazy in-registers
* state pending on its former CPU could be restored, corrupting
* the modifications.
*/
void fpu__activate_fpstate_write(struct fpu *fpu)
{
/*
* Only stopped child tasks can be used to modify the FPU
* state in the fpstate buffer:
*/
WARN_ON_FPU(fpu == &current->thread.fpu);
if (fpu->fpstate_active) {
/* Invalidate any lazy state: */
fpu->last_cpu = -1;
} else {
fpstate_init(&fpu->state);
/* Safe to do for stopped child tasks: */
fpu->fpstate_active = 1;
}
}
/*
* 'fpu__restore()' is called to copy FPU registers from
* the FPU fpstate to the live hw registers and to activate
* access to the hardware registers, so that FPU instructions
* can be used afterwards.
*
* Must be called with kernel preemption disabled (for example
* with local interrupts disabled, as it is in the case of
* do_device_not_available()).
*/
void fpu__restore(struct fpu *fpu)
{
fpu__activate_curr(fpu);
/* Avoid __kernel_fpu_begin() right after fpregs_activate() */
kernel_fpu_disable();
fpregs_activate(fpu);
copy_kernel_to_fpregs(&fpu->state);
fpu->counter++;
kernel_fpu_enable();
}
EXPORT_SYMBOL_GPL(fpu__restore);
/*
* Drops current FPU state: deactivates the fpregs and
* the fpstate. NOTE: it still leaves previous contents
* in the fpregs in the eager-FPU case.
*
* This function can be used in cases where we know that
* a state-restore is coming: either an explicit one,
* or a reschedule.
*/
void fpu__drop(struct fpu *fpu)
{
preempt_disable();
fpu->counter = 0;
if (fpu->fpregs_active) {
/* Ignore delayed exceptions from user space */
asm volatile("1: fwait\n"
"2:\n"
_ASM_EXTABLE(1b, 2b));
fpregs_deactivate(fpu);
}
fpu->fpstate_active = 0;
preempt_enable();
}
/*
* Clear FPU registers by setting them up from
* the init fpstate:
*/
static inline void copy_init_fpstate_to_fpregs(void)
{
if (use_xsave())
copy_kernel_to_xregs(&init_fpstate.xsave, -1);
else
copy_kernel_to_fxregs(&init_fpstate.fxsave);
}
/*
* Clear the FPU state back to init state.
*
* Called by sys_execve(), by the signal handler code and by various
* error paths.
*/
void fpu__clear(struct fpu *fpu)
{
WARN_ON_FPU(fpu != &current->thread.fpu); /* Almost certainly an anomaly */
if (!use_eager_fpu()) {
/* FPU state will be reallocated lazily at the first use. */
fpu__drop(fpu);
} else {
if (!fpu->fpstate_active) {
fpu__activate_curr(fpu);
user_fpu_begin();
}
copy_init_fpstate_to_fpregs();
}
}
/*
* x87 math exception handling:
*/
static inline unsigned short get_fpu_cwd(struct fpu *fpu)
{
if (cpu_has_fxsr) {
return fpu->state.fxsave.cwd;
} else {
return (unsigned short)fpu->state.fsave.cwd;
}
}
static inline unsigned short get_fpu_swd(struct fpu *fpu)
{
if (cpu_has_fxsr) {
return fpu->state.fxsave.swd;
} else {
return (unsigned short)fpu->state.fsave.swd;
}
}
static inline unsigned short get_fpu_mxcsr(struct fpu *fpu)
{
if (cpu_has_xmm) {
return fpu->state.fxsave.mxcsr;
} else {
return MXCSR_DEFAULT;
}
}
int fpu__exception_code(struct fpu *fpu, int trap_nr)
{
int err;
if (trap_nr == X86_TRAP_MF) {
unsigned short cwd, swd;
/*
* (~cwd & swd) will mask out exceptions that are not set to unmasked
* status. 0x3f is the exception bits in these regs, 0x200 is the
* C1 reg you need in case of a stack fault, 0x040 is the stack
* fault bit. We should only be taking one exception at a time,
* so if this combination doesn't produce any single exception,
* then we have a bad program that isn't synchronizing its FPU usage
* and it will suffer the consequences since we won't be able to
* fully reproduce the context of the exception
*/
cwd = get_fpu_cwd(fpu);
swd = get_fpu_swd(fpu);
err = swd & ~cwd;
} else {
/*
* The SIMD FPU exceptions are handled a little differently, as there
* is only a single status/control register. Thus, to determine which
* unmasked exception was caught we must mask the exception mask bits
* at 0x1f80, and then use these to mask the exception bits at 0x3f.
*/
unsigned short mxcsr = get_fpu_mxcsr(fpu);
err = ~(mxcsr >> 7) & mxcsr;
}
if (err & 0x001) { /* Invalid op */
/*
* swd & 0x240 == 0x040: Stack Underflow
* swd & 0x240 == 0x240: Stack Overflow
* User must clear the SF bit (0x40) if set
*/
return FPE_FLTINV;
} else if (err & 0x004) { /* Divide by Zero */
return FPE_FLTDIV;
} else if (err & 0x008) { /* Overflow */
return FPE_FLTOVF;
} else if (err & 0x012) { /* Denormal, Underflow */
return FPE_FLTUND;
} else if (err & 0x020) { /* Precision */
return FPE_FLTRES;
}
/*
* If we're using IRQ 13, or supposedly even some trap
* X86_TRAP_MF implementations, it's possible
* we get a spurious trap, which is not an error.
*/
return 0;
}
arch/x86/kernel/fpu/init.c 0 → 100644
浏览文件 @ e75c73ad
/*
* x86 FPU boot time init code:
*/
#include <asm/fpu/internal.h>
#include <asm/tlbflush.h>
/*
* Initialize the TS bit in CR0 according to the style of context-switches
* we are using:
*/
static void fpu__init_cpu_ctx_switch(void)
{
if (!cpu_has_eager_fpu)
stts();
else
clts();
}
/*
* Initialize the registers found in all CPUs, CR0 and CR4:
*/
static void fpu__init_cpu_generic(void)
{
unsigned long cr0;
unsigned long cr4_mask = 0;
if (cpu_has_fxsr)
cr4_mask |= X86_CR4_OSFXSR;
if (cpu_has_xmm)
cr4_mask |= X86_CR4_OSXMMEXCPT;
if (cr4_mask)
cr4_set_bits(cr4_mask);
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
if (!cpu_has_fpu)
cr0 |= X86_CR0_EM;
write_cr0(cr0);
/* Flush out any pending x87 state: */
asm volatile ("fninit");
}
/*
* Enable all supported FPU features. Called when a CPU is brought online:
*/
void fpu__init_cpu(void)
{
fpu__init_cpu_generic();
fpu__init_cpu_xstate();
fpu__init_cpu_ctx_switch();
}
/*
* The earliest FPU detection code.
*
* Set the X86_FEATURE_FPU CPU-capability bit based on
* trying to execute an actual sequence of FPU instructions:
*/
static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
{
unsigned long cr0;
u16 fsw, fcw;
fsw = fcw = 0xffff;
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
write_cr0(cr0);
asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
: "+m" (fsw), "+m" (fcw));
if (fsw == 0 && (fcw & 0x103f) == 0x003f)
set_cpu_cap(c, X86_FEATURE_FPU);
else
clear_cpu_cap(c, X86_FEATURE_FPU);
#ifndef CONFIG_MATH_EMULATION
if (!cpu_has_fpu) {
pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
for (;;)
asm volatile("hlt");
}
#endif
}
/*
* Boot time FPU feature detection code:
*/
unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
static void __init fpu__init_system_mxcsr(void)
{
unsigned int mask = 0;
if (cpu_has_fxsr) {
struct fxregs_state fx_tmp __aligned(32) = { };
asm volatile("fxsave %0" : "+m" (fx_tmp));
mask = fx_tmp.mxcsr_mask;
/*
* If zero then use the default features mask,
* which has all features set, except the
* denormals-are-zero feature bit:
*/
if (mask == 0)
mask = 0x0000ffbf;
}
mxcsr_feature_mask &= mask;
}
/*
* Once per bootup FPU initialization sequences that will run on most x86 CPUs:
*/
static void __init fpu__init_system_generic(void)
{
/*
* Set up the legacy init FPU context. (xstate init might overwrite this
* with a more modern format, if the CPU supports it.)
*/
fpstate_init_fxstate(&init_fpstate.fxsave);
fpu__init_system_mxcsr();
}
/*
* Size of the FPU context state. All tasks in the system use the
* same context size, regardless of what portion they use.
* This is inherent to the XSAVE architecture which puts all state
* components into a single, continuous memory block:
*/
unsigned int xstate_size;
EXPORT_SYMBOL_GPL(xstate_size);
/*
* Set up the xstate_size based on the legacy FPU context size.
*
* We set this up first, and later it will be overwritten by
* fpu__init_system_xstate() if the CPU knows about xstates.
*/
static void __init fpu__init_system_xstate_size_legacy(void)
{
static int on_boot_cpu = 1;
WARN_ON_FPU(!on_boot_cpu);
on_boot_cpu = 0;
/*
* Note that xstate_size might be overwriten later during
* fpu__init_system_xstate().
*/
if (!cpu_has_fpu) {
/*
* Disable xsave as we do not support it if i387
* emulation is enabled.
*/
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
xstate_size = sizeof(struct swregs_state);
} else {
if (cpu_has_fxsr)
xstate_size = sizeof(struct fxregs_state);
else
xstate_size = sizeof(struct fregs_state);
}
/*
* Quirk: we don't yet handle the XSAVES* instructions
* correctly, as we don't correctly convert between
* standard and compacted format when interfacing
* with user-space - so disable it for now.
*
* The difference is small: with recent CPUs the
* compacted format is only marginally smaller than
* the standard FPU state format.
*
* ( This is easy to backport while we are fixing
* XSAVES* support. )
*/
setup_clear_cpu_cap(X86_FEATURE_XSAVES);
}
/*
* FPU context switching strategies:
*
* Against popular belief, we don't do lazy FPU saves, due to the
* task migration complications it brings on SMP - we only do
* lazy FPU restores.
*
* 'lazy' is the traditional strategy, which is based on setting
* CR0::TS to 1 during context-switch (instead of doing a full
* restore of the FPU state), which causes the first FPU instruction
* after the context switch (whenever it is executed) to fault - at
* which point we lazily restore the FPU state into FPU registers.
*
* Tasks are of course under no obligation to execute FPU instructions,
* so it can easily happen that another context-switch occurs without
* a single FPU instruction being executed. If we eventually switch
* back to the original task (that still owns the FPU) then we have
* not only saved the restores along the way, but we also have the
* FPU ready to be used for the original task.
*
* 'eager' switching is used on modern CPUs, there we switch the FPU
* state during every context switch, regardless of whether the task
* has used FPU instructions in that time slice or not. This is done
* because modern FPU context saving instructions are able to optimize
* state saving and restoration in hardware: they can detect both
* unused and untouched FPU state and optimize accordingly.
*
* [ Note that even in 'lazy' mode we might optimize context switches
* to use 'eager' restores, if we detect that a task is using the FPU
* frequently. See the fpu->counter logic in fpu/internal.h for that. ]
*/
static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO;
static int __init eager_fpu_setup(char *s)
{
if (!strcmp(s, "on"))
eagerfpu = ENABLE;
else if (!strcmp(s, "off"))
eagerfpu = DISABLE;
else if (!strcmp(s, "auto"))
eagerfpu = AUTO;
return 1;
}
__setup("eagerfpu=", eager_fpu_setup);
/*
* Pick the FPU context switching strategy:
*/
static void __init fpu__init_system_ctx_switch(void)
{
static bool on_boot_cpu = 1;
WARN_ON_FPU(!on_boot_cpu);
on_boot_cpu = 0;
WARN_ON_FPU(current->thread.fpu.fpstate_active);
current_thread_info()->status = 0;
/* Auto enable eagerfpu for xsaveopt */
if (cpu_has_xsaveopt && eagerfpu != DISABLE)
eagerfpu = ENABLE;
if (xfeatures_mask & XSTATE_EAGER) {
if (eagerfpu == DISABLE) {
pr_err("x86/fpu: eagerfpu switching disabled, disabling the following xstate features: 0x%llx.\n",
xfeatures_mask & XSTATE_EAGER);
xfeatures_mask &= ~XSTATE_EAGER;
} else {
eagerfpu = ENABLE;
}
}
if (eagerfpu == ENABLE)
setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
printk(KERN_INFO "x86/fpu: Using '%s' FPU context switches.\n", eagerfpu == ENABLE ? "eager" : "lazy");
}
/*
* Called on the boot CPU once per system bootup, to set up the initial
* FPU state that is later cloned into all processes:
*/
void __init fpu__init_system(struct cpuinfo_x86 *c)
{
fpu__init_system_early_generic(c);
/*
* The FPU has to be operational for some of the
* later FPU init activities:
*/
fpu__init_cpu();
/*
* But don't leave CR0::TS set yet, as some of the FPU setup
* methods depend on being able to execute FPU instructions
* that will fault on a set TS, such as the FXSAVE in
* fpu__init_system_mxcsr().
*/
clts();
fpu__init_system_generic();
fpu__init_system_xstate_size_legacy();
fpu__init_system_xstate();
fpu__init_system_ctx_switch();
}
/*
* Boot parameter to turn off FPU support and fall back to math-emu:
*/
static int __init no_387(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_FPU);
return 1;
}
__setup("no387", no_387);
/*
* Disable all xstate CPU features:
*/
static int __init x86_noxsave_setup(char *s)
{
if (strlen(s))
return 0;
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
setup_clear_cpu_cap(X86_FEATURE_XSAVES);
setup_clear_cpu_cap(X86_FEATURE_AVX);
setup_clear_cpu_cap(X86_FEATURE_AVX2);
return 1;
}
__setup("noxsave", x86_noxsave_setup);
/*
* Disable the XSAVEOPT instruction specifically:
*/
static int __init x86_noxsaveopt_setup(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
return 1;
}
__setup("noxsaveopt", x86_noxsaveopt_setup);
/*
* Disable the XSAVES instruction:
*/
static int __init x86_noxsaves_setup(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_XSAVES);
return 1;
}
__setup("noxsaves", x86_noxsaves_setup);
/*
* Disable FX save/restore and SSE support:
*/
static int __init x86_nofxsr_setup(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_FXSR);
setup_clear_cpu_cap(X86_FEATURE_FXSR_OPT);
setup_clear_cpu_cap(X86_FEATURE_XMM);
return 1;
}
__setup("nofxsr", x86_nofxsr_setup);
arch/x86/kernel/i387.c → arch/x86/kernel/fpu/regset.c
浏览文件 @ e75c73ad
/*
* Copyright (C) 1994 Linus Torvalds
*
* Pentium III FXSR, SSE support
* General FPU state handling cleanups
* Gareth Hughes <gareth@valinux.com>, May 2000
* FPU register's regset abstraction, for ptrace, core dumps, etc.
*/
#include <linux/module.h>
#include <linux/regset.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <asm/sigcontext.h>
#include <asm/processor.h>
#include <asm/math_emu.h>
#include <asm/tlbflush.h>
#include <asm/uaccess.h>
#include <asm/ptrace.h>
#include <asm/i387.h>
#include <asm/fpu-internal.h>
#include <asm/user.h>
static DEFINE_PER_CPU(bool, in_kernel_fpu);
void kernel_fpu_disable(void)
{
WARN_ON(this_cpu_read(in_kernel_fpu));
this_cpu_write(in_kernel_fpu, true);
}
void kernel_fpu_enable(void)
{
this_cpu_write(in_kernel_fpu, false);
}
/*
* Were we in an interrupt that interrupted kernel mode?
*
* On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
* pair does nothing at all: the thread must not have fpu (so
* that we don't try to save the FPU state), and TS must
* be set (so that the clts/stts pair does nothing that is
* visible in the interrupted kernel thread).
*
* Except for the eagerfpu case when we return true; in the likely case
* the thread has FPU but we are not going to set/clear TS.
*/
static inline bool interrupted_kernel_fpu_idle(void)
{
if (this_cpu_read(in_kernel_fpu))
return false;
if (use_eager_fpu())
return true;
return !__thread_has_fpu(current) &&
(read_cr0() & X86_CR0_TS);
}
/*
* Were we in user mode (or vm86 mode) when we were
* interrupted?
*
* Doing kernel_fpu_begin/end() is ok if we are running
* in an interrupt context from user mode - we'll just
* save the FPU state as required.
*/
static inline bool interrupted_user_mode(void)
{
struct pt_regs *regs = get_irq_regs();
return regs && user_mode(regs);
}
#include <asm/fpu/internal.h>
#include <asm/fpu/signal.h>
#include <asm/fpu/regset.h>
/*
* Can we use the FPU in kernel mode with the
* whole "kernel_fpu_begin/end()" sequence?
*
* It's always ok in process context (ie "not interrupt")
* but it is sometimes ok even from an irq.
*/
bool irq_fpu_usable(void)
{
return !in_interrupt() ||
interrupted_user_mode() ||
interrupted_kernel_fpu_idle();
}
EXPORT_SYMBOL(irq_fpu_usable);
void __kernel_fpu_begin(void)
{
struct task_struct *me = current;
this_cpu_write(in_kernel_fpu, true);
if (__thread_has_fpu(me)) {
__save_init_fpu(me);
} else {
this_cpu_write(fpu_owner_task, NULL);
if (!use_eager_fpu())
clts();
}
}
EXPORT_SYMBOL(__kernel_fpu_begin);
void __kernel_fpu_end(void)
{
struct task_struct *me = current;
if (__thread_has_fpu(me)) {
if (WARN_ON(restore_fpu_checking(me)))
fpu_reset_state(me);
} else if (!use_eager_fpu()) {
stts();
}
this_cpu_write(in_kernel_fpu, false);
}
EXPORT_SYMBOL(__kernel_fpu_end);
void unlazy_fpu(struct task_struct *tsk)
{
preempt_disable();
if (__thread_has_fpu(tsk)) {
if (use_eager_fpu()) {
__save_fpu(tsk);
} else {
__save_init_fpu(tsk);
__thread_fpu_end(tsk);
}
}
preempt_enable();
}
EXPORT_SYMBOL(unlazy_fpu);
unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
unsigned int xstate_size;
EXPORT_SYMBOL_GPL(xstate_size);
static struct i387_fxsave_struct fx_scratch;
static void mxcsr_feature_mask_init(void)
{
unsigned long mask = 0;
if (cpu_has_fxsr) {
memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
asm volatile("fxsave %0" : "+m" (fx_scratch));
mask = fx_scratch.mxcsr_mask;
if (mask == 0)
mask = 0x0000ffbf;
}
mxcsr_feature_mask &= mask;
}
static void init_thread_xstate(void)
{
/*
* Note that xstate_size might be overwriten later during
* xsave_init().
*/
if (!cpu_has_fpu) {
/*
* Disable xsave as we do not support it if i387
* emulation is enabled.
*/
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
xstate_size = sizeof(struct i387_soft_struct);
return;
}
if (cpu_has_fxsr)
xstate_size = sizeof(struct i387_fxsave_struct);
else
xstate_size = sizeof(struct i387_fsave_struct);
/*
* Quirk: we don't yet handle the XSAVES* instructions
* correctly, as we don't correctly convert between
* standard and compacted format when interfacing
* with user-space - so disable it for now.
*
* The difference is small: with recent CPUs the
* compacted format is only marginally smaller than
* the standard FPU state format.
*
* ( This is easy to backport while we are fixing
* XSAVES* support. )
*/
setup_clear_cpu_cap(X86_FEATURE_XSAVES);
}
/*
* Called at bootup to set up the initial FPU state that is later cloned
* into all processes.
*/
void fpu_init(void)
{
unsigned long cr0;
unsigned long cr4_mask = 0;
#ifndef CONFIG_MATH_EMULATION
if (!cpu_has_fpu) {
pr_emerg("No FPU found and no math emulation present\n");
pr_emerg("Giving up\n");
for (;;)
asm volatile("hlt");
}
#endif
if (cpu_has_fxsr)
cr4_mask |= X86_CR4_OSFXSR;
if (cpu_has_xmm)
cr4_mask |= X86_CR4_OSXMMEXCPT;
if (cr4_mask)
cr4_set_bits(cr4_mask);
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
if (!cpu_has_fpu)
cr0 |= X86_CR0_EM;
write_cr0(cr0);
/*
* init_thread_xstate is only called once to avoid overriding
* xstate_size during boot time or during CPU hotplug.
*/
if (xstate_size == 0)
init_thread_xstate();
mxcsr_feature_mask_init();
xsave_init();
eager_fpu_init();
}
void fpu_finit(struct fpu *fpu)
{
if (!cpu_has_fpu) {
finit_soft_fpu(&fpu->state->soft);
return;
}
memset(fpu->state, 0, xstate_size);
if (cpu_has_fxsr) {
fx_finit(&fpu->state->fxsave);
} else {
struct i387_fsave_struct *fp = &fpu->state->fsave;
fp->cwd = 0xffff037fu;
fp->swd = 0xffff0000u;
fp->twd = 0xffffffffu;
fp->fos = 0xffff0000u;
}
}
EXPORT_SYMBOL_GPL(fpu_finit);
/*
* The _current_ task is using the FPU for the first time
* so initialize it and set the mxcsr to its default
* value at reset if we support XMM instructions and then
* remember the current task has used the FPU.
*/
int init_fpu(struct task_struct *tsk)
{
int ret;
if (tsk_used_math(tsk)) {
if (cpu_has_fpu && tsk == current)
unlazy_fpu(tsk);
task_disable_lazy_fpu_restore(tsk);
return 0;
}
/*
* Memory allocation at the first usage of the FPU and other state.
*/
ret = fpu_alloc(&tsk->thread.fpu);
if (ret)
return ret;
fpu_finit(&tsk->thread.fpu);
set_stopped_child_used_math(tsk);
return 0;
}
EXPORT_SYMBOL_GPL(init_fpu);
/*
* The xstateregs_active() routine is the same as the fpregs_active() routine,
* The xstateregs_active() routine is the same as the regset_fpregs_active() routine,
* as the "regset->n" for the xstate regset will be updated based on the feature
* capabilites supported by the xsave.
*/
int fpregs_active(struct task_struct *target, const struct user_regset *regset)
int regset_fpregs_active(struct task_struct *target, const struct user_regset *regset)
{
return tsk_used_math(target) ? regset->n : 0;
struct fpu *target_fpu = &target->thread.fpu;
return target_fpu->fpstate_active ? regset->n : 0;
}
int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
int regset_xregset_fpregs_active(struct task_struct *target, const struct user_regset *regset)
{
return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
struct fpu *target_fpu = &target->thread.fpu;
return (cpu_has_fxsr && target_fpu->fpstate_active) ? regset->n : 0;
}
int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
void *kbuf, void __user *ubuf)
{
int ret;
struct fpu *fpu = &target->thread.fpu;
if (!cpu_has_fxsr)
return -ENODEV;
ret = init_fpu(target);
if (ret)
return ret;
sanitize_i387_state(target);
fpu__activate_fpstate_read(fpu);
fpstate_sanitize_xstate(fpu);
return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.fpu.state->fxsave, 0, -1);
&fpu->state.fxsave, 0, -1);
}
int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf)
{
struct fpu *fpu = &target->thread.fpu;
int ret;
if (!cpu_has_fxsr)
return -ENODEV;
ret = init_fpu(target);
if (ret)
return ret;
sanitize_i387_state(target);
fpu__activate_fpstate_write(fpu);
fpstate_sanitize_xstate(fpu);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.fpu.state->fxsave, 0, -1);
&fpu->state.fxsave, 0, -1);
/*
* mxcsr reserved bits must be masked to zero for security reasons.
*/
target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
fpu->state.fxsave.mxcsr &= mxcsr_feature_mask;
/*
* update the header bits in the xsave header, indicating the
* presence of FP and SSE state.
*/
if (cpu_has_xsave)
target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
fpu->state.xsave.header.xfeatures |= XSTATE_FPSSE;
return ret;
}
......@@ -356,17 +75,16 @@ int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
void *kbuf, void __user *ubuf)
{
struct xsave_struct *xsave;
struct fpu *fpu = &target->thread.fpu;
struct xregs_state *xsave;
int ret;
if (!cpu_has_xsave)
return -ENODEV;
ret = init_fpu(target);
if (ret)
return ret;
fpu__activate_fpstate_read(fpu);
xsave = &target->thread.fpu.state->xsave;
xsave = &fpu->state.xsave;
/*
* Copy the 48bytes defined by the software first into the xstate
......@@ -386,28 +104,28 @@ int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf)
{
struct xsave_struct *xsave;
struct fpu *fpu = &target->thread.fpu;
struct xregs_state *xsave;
int ret;
if (!cpu_has_xsave)
return -ENODEV;
ret = init_fpu(target);
if (ret)
return ret;
fpu__activate_fpstate_write(fpu);
xsave = &target->thread.fpu.state->xsave;
xsave = &fpu->state.xsave;
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, xsave, 0, -1);
/*
* mxcsr reserved bits must be masked to zero for security reasons.
*/
xsave->i387.mxcsr &= mxcsr_feature_mask;
xsave->xsave_hdr.xstate_bv &= pcntxt_mask;
xsave->header.xfeatures &= xfeatures_mask;
/*
* These bits must be zero.
*/
memset(&xsave->xsave_hdr.reserved, 0, 48);
memset(&xsave->header.reserved, 0, 48);
return ret;
}
......@@ -438,7 +156,7 @@ static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
#define FP_EXP_TAG_SPECIAL 2
#define FP_EXP_TAG_EMPTY 3
static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
static inline u32 twd_fxsr_to_i387(struct fxregs_state *fxsave)
{
struct _fpxreg *st;
u32 tos = (fxsave->swd >> 11) & 7;
......@@ -486,7 +204,7 @@ static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
void
convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
{
struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
struct fxregs_state *fxsave = &tsk->thread.fpu.state.fxsave;
struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
int i;
......@@ -524,7 +242,7 @@ void convert_to_fxsr(struct task_struct *tsk,
const struct user_i387_ia32_struct *env)
{
struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
struct fxregs_state *fxsave = &tsk->thread.fpu.state.fxsave;
struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
int i;
......@@ -552,22 +270,20 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
void *kbuf, void __user *ubuf)
{
struct fpu *fpu = &target->thread.fpu;
struct user_i387_ia32_struct env;
int ret;
ret = init_fpu(target);
if (ret)
return ret;
fpu__activate_fpstate_read(fpu);
if (!static_cpu_has(X86_FEATURE_FPU))
return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
if (!cpu_has_fxsr)
return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.fpu.state->fsave, 0,
&fpu->state.fsave, 0,
-1);
sanitize_i387_state(target);
fpstate_sanitize_xstate(fpu);
if (kbuf && pos == 0 && count == sizeof(env)) {
convert_from_fxsr(kbuf, target);
......@@ -583,21 +299,19 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf)
{
struct fpu *fpu = &target->thread.fpu;
struct user_i387_ia32_struct env;
int ret;
ret = init_fpu(target);
if (ret)
return ret;
sanitize_i387_state(target);
fpu__activate_fpstate_write(fpu);
fpstate_sanitize_xstate(fpu);
if (!static_cpu_has(X86_FEATURE_FPU))
return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
if (!cpu_has_fxsr)
return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.fpu.state->fsave, 0,
&fpu->state.fsave, 0,
-1);
if (pos > 0 || count < sizeof(env))
......@@ -612,7 +326,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
* presence of FP.
*/
if (cpu_has_xsave)
target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
fpu->state.xsave.header.xfeatures |= XSTATE_FP;
return ret;
}
......@@ -623,49 +337,20 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
* struct user_i387_struct) but is in fact only used for 32-bit
* dumps, so on 64-bit it is really struct user_i387_ia32_struct.
*/
int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
int dump_fpu(struct pt_regs *regs, struct user_i387_struct *ufpu)
{
struct task_struct *tsk = current;
struct fpu *fpu = &tsk->thread.fpu;
int fpvalid;
fpvalid = !!used_math();
fpvalid = fpu->fpstate_active;
if (fpvalid)
fpvalid = !fpregs_get(tsk, NULL,
0, sizeof(struct user_i387_ia32_struct),
fpu, NULL);
ufpu, NULL);
return fpvalid;
}
EXPORT_SYMBOL(dump_fpu);
#endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */
static int __init no_387(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_FPU);
return 1;
}
__setup("no387", no_387);
void fpu_detect(struct cpuinfo_x86 *c)
{
unsigned long cr0;
u16 fsw, fcw;
fsw = fcw = 0xffff;
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
write_cr0(cr0);
asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
: "+m" (fsw), "+m" (fcw));
if (fsw == 0 && (fcw & 0x103f) == 0x003f)
set_cpu_cap(c, X86_FEATURE_FPU);
else
clear_cpu_cap(c, X86_FEATURE_FPU);
/* The final cr0 value is set in fpu_init() */
}
arch/x86/kernel/fpu/xstate.c 0 → 100644
浏览文件 @ e75c73ad
/*
* xsave/xrstor support.
*
* Author: Suresh Siddha <suresh.b.siddha@intel.com>
*/
#include <linux/compat.h>
#include <linux/cpu.h>
#include <asm/fpu/api.h>
#include <asm/fpu/internal.h>
#include <asm/fpu/signal.h>
#include <asm/fpu/regset.h>
#include <asm/tlbflush.h>
static const char *xfeature_names[] =
{
"x87 floating point registers" ,
"SSE registers" ,
"AVX registers" ,
"MPX bounds registers" ,
"MPX CSR" ,
"AVX-512 opmask" ,
"AVX-512 Hi256" ,
"AVX-512 ZMM_Hi256" ,
"unknown xstate feature" ,
};
/*
* Mask of xstate features supported by the CPU and the kernel:
*/
u64 xfeatures_mask __read_mostly;
static unsigned int xstate_offsets[XFEATURES_NR_MAX] = { [ 0 ... XFEATURES_NR_MAX - 1] = -1};
static unsigned int xstate_sizes[XFEATURES_NR_MAX] = { [ 0 ... XFEATURES_NR_MAX - 1] = -1};
static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8];
/* The number of supported xfeatures in xfeatures_mask: */
static unsigned int xfeatures_nr;
/*
* Return whether the system supports a given xfeature.
*
* Also return the name of the (most advanced) feature that the caller requested:
*/
int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
{
u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask;
if (unlikely(feature_name)) {
long xfeature_idx, max_idx;
u64 xfeatures_print;
/*
* So we use FLS here to be able to print the most advanced
* feature that was requested but is missing. So if a driver
* asks about "XSTATE_SSE | XSTATE_YMM" we'll print the
* missing AVX feature - this is the most informative message
* to users:
*/
if (xfeatures_missing)
xfeatures_print = xfeatures_missing;
else
xfeatures_print = xfeatures_needed;
xfeature_idx = fls64(xfeatures_print)-1;
max_idx = ARRAY_SIZE(xfeature_names)-1;
xfeature_idx = min(xfeature_idx, max_idx);
*feature_name = xfeature_names[xfeature_idx];
}
if (xfeatures_missing)
return 0;
return 1;
}
EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
/*
* When executing XSAVEOPT (or other optimized XSAVE instructions), if
* a processor implementation detects that an FPU state component is still
* (or is again) in its initialized state, it may clear the corresponding
* bit in the header.xfeatures field, and can skip the writeout of registers
* to the corresponding memory layout.
*
* This means that when the bit is zero, the state component might still contain
* some previous - non-initialized register state.
*
* Before writing xstate information to user-space we sanitize those components,
* to always ensure that the memory layout of a feature will be in the init state
* if the corresponding header bit is zero. This is to ensure that user-space doesn't
* see some stale state in the memory layout during signal handling, debugging etc.
*/
void fpstate_sanitize_xstate(struct fpu *fpu)
{
struct fxregs_state *fx = &fpu->state.fxsave;
int feature_bit;
u64 xfeatures;
if (!use_xsaveopt())
return;
xfeatures = fpu->state.xsave.header.xfeatures;
/*
* None of the feature bits are in init state. So nothing else
* to do for us, as the memory layout is up to date.
*/
if ((xfeatures & xfeatures_mask) == xfeatures_mask)
return;
/*
* FP is in init state
*/
if (!(xfeatures & XSTATE_FP)) {
fx->cwd = 0x37f;
fx->swd = 0;
fx->twd = 0;
fx->fop = 0;
fx->rip = 0;
fx->rdp = 0;
memset(&fx->st_space[0], 0, 128);
}
/*
* SSE is in init state
*/
if (!(xfeatures & XSTATE_SSE))
memset(&fx->xmm_space[0], 0, 256);
/*
* First two features are FPU and SSE, which above we handled
* in a special way already:
*/
feature_bit = 0x2;
xfeatures = (xfeatures_mask & ~xfeatures) >> 2;
/*
* Update all the remaining memory layouts according to their
* standard xstate layout, if their header bit is in the init
* state:
*/
while (xfeatures) {
if (xfeatures & 0x1) {
int offset = xstate_offsets[feature_bit];
int size = xstate_sizes[feature_bit];
memcpy((void *)fx + offset,
(void *)&init_fpstate.xsave + offset,
size);
}
xfeatures >>= 1;
feature_bit++;
}
}
/*
* Enable the extended processor state save/restore feature.
* Called once per CPU onlining.
*/
void fpu__init_cpu_xstate(void)
{
if (!cpu_has_xsave || !xfeatures_mask)
return;
cr4_set_bits(X86_CR4_OSXSAVE);
xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
}
/*
* Record the offsets and sizes of various xstates contained
* in the XSAVE state memory layout.
*
* ( Note that certain features might be non-present, for them
* we'll have 0 offset and 0 size. )
*/
static void __init setup_xstate_features(void)
{
u32 eax, ebx, ecx, edx, leaf;
xfeatures_nr = fls64(xfeatures_mask);
for (leaf = 2; leaf < xfeatures_nr; leaf++) {
cpuid_count(XSTATE_CPUID, leaf, &eax, &ebx, &ecx, &edx);
xstate_offsets[leaf] = ebx;
xstate_sizes[leaf] = eax;
printk(KERN_INFO "x86/fpu: xstate_offset[%d]: %04x, xstate_sizes[%d]: %04x\n", leaf, ebx, leaf, eax);
}
}
static void __init print_xstate_feature(u64 xstate_mask)
{
const char *feature_name;
if (cpu_has_xfeatures(xstate_mask, &feature_name))
pr_info("x86/fpu: Supporting XSAVE feature 0x%02Lx: '%s'\n", xstate_mask, feature_name);
}
/*
* Print out all the supported xstate features:
*/
static void __init print_xstate_features(void)
{
print_xstate_feature(XSTATE_FP);
print_xstate_feature(XSTATE_SSE);
print_xstate_feature(XSTATE_YMM);
print_xstate_feature(XSTATE_BNDREGS);
print_xstate_feature(XSTATE_BNDCSR);
print_xstate_feature(XSTATE_OPMASK);
print_xstate_feature(XSTATE_ZMM_Hi256);
print_xstate_feature(XSTATE_Hi16_ZMM);
}
/*
* This function sets up offsets and sizes of all extended states in
* xsave area. This supports both standard format and compacted format
* of the xsave aread.
*/
static void __init setup_xstate_comp(void)
{
unsigned int xstate_comp_sizes[sizeof(xfeatures_mask)*8];
int i;
/*
* The FP xstates and SSE xstates are legacy states. They are always
* in the fixed offsets in the xsave area in either compacted form
* or standard form.
*/
xstate_comp_offsets[0] = 0;
xstate_comp_offsets[1] = offsetof(struct fxregs_state, xmm_space);
if (!cpu_has_xsaves) {
for (i = 2; i < xfeatures_nr; i++) {
if (test_bit(i, (unsigned long *)&xfeatures_mask)) {
xstate_comp_offsets[i] = xstate_offsets[i];
xstate_comp_sizes[i] = xstate_sizes[i];
}
}
return;
}
xstate_comp_offsets[2] = FXSAVE_SIZE + XSAVE_HDR_SIZE;
for (i = 2; i < xfeatures_nr; i++) {
if (test_bit(i, (unsigned long *)&xfeatures_mask))
xstate_comp_sizes[i] = xstate_sizes[i];
else
xstate_comp_sizes[i] = 0;
if (i > 2)
xstate_comp_offsets[i] = xstate_comp_offsets[i-1]
+ xstate_comp_sizes[i-1];
}
}
/*
* setup the xstate image representing the init state
*/
static void __init setup_init_fpu_buf(void)
{
static int on_boot_cpu = 1;
WARN_ON_FPU(!on_boot_cpu);
on_boot_cpu = 0;
if (!cpu_has_xsave)
return;
setup_xstate_features();
print_xstate_features();
if (cpu_has_xsaves) {
init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
init_fpstate.xsave.header.xfeatures = xfeatures_mask;
}
/*
* Init all the features state with header_bv being 0x0
*/
copy_kernel_to_xregs_booting(&init_fpstate.xsave);
/*
* Dump the init state again. This is to identify the init state
* of any feature which is not represented by all zero's.
*/
copy_xregs_to_kernel_booting(&init_fpstate.xsave);
}
/*
* Calculate total size of enabled xstates in XCR0/xfeatures_mask.
*/
static void __init init_xstate_size(void)
{
unsigned int eax, ebx, ecx, edx;
int i;
if (!cpu_has_xsaves) {
cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
xstate_size = ebx;
return;
}
xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
for (i = 2; i < 64; i++) {
if (test_bit(i, (unsigned long *)&xfeatures_mask)) {
cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
xstate_size += eax;
}
}
}
/*
* Enable and initialize the xsave feature.
* Called once per system bootup.
*/
void __init fpu__init_system_xstate(void)
{
unsigned int eax, ebx, ecx, edx;
static int on_boot_cpu = 1;
WARN_ON_FPU(!on_boot_cpu);
on_boot_cpu = 0;
if (!cpu_has_xsave) {
pr_info("x86/fpu: Legacy x87 FPU detected.\n");
return;
}
if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
WARN_ON_FPU(1);
return;
}
cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
xfeatures_mask = eax + ((u64)edx << 32);
if ((xfeatures_mask & XSTATE_FPSSE) != XSTATE_FPSSE) {
pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask);
BUG();
}
/* Support only the state known to the OS: */
xfeatures_mask = xfeatures_mask & XCNTXT_MASK;
/* Enable xstate instructions to be able to continue with initialization: */
fpu__init_cpu_xstate();
/* Recompute the context size for enabled features: */
init_xstate_size();
update_regset_xstate_info(xstate_size, xfeatures_mask);
fpu__init_prepare_fx_sw_frame();
setup_init_fpu_buf();
setup_xstate_comp();
pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is 0x%x bytes, using '%s' format.\n",
xfeatures_mask,
xstate_size,
cpu_has_xsaves ? "compacted" : "standard");
}
/*
* Restore minimal FPU state after suspend:
*/
void fpu__resume_cpu(void)
{
/*
* Restore XCR0 on xsave capable CPUs:
*/
if (cpu_has_xsave)
xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
}
/*
* Given the xsave area and a state inside, this function returns the
* address of the state.
*
* This is the API that is called to get xstate address in either
* standard format or compacted format of xsave area.
*
* Note that if there is no data for the field in the xsave buffer
* this will return NULL.
*
* Inputs:
* xstate: the thread's storage area for all FPU data
* xstate_feature: state which is defined in xsave.h (e.g.
* XSTATE_FP, XSTATE_SSE, etc...)
* Output:
* address of the state in the xsave area, or NULL if the
* field is not present in the xsave buffer.
*/
void *get_xsave_addr(struct xregs_state *xsave, int xstate_feature)
{
int feature_nr = fls64(xstate_feature) - 1;
/*
* Do we even *have* xsave state?
*/
if (!boot_cpu_has(X86_FEATURE_XSAVE))
return NULL;
xsave = &current->thread.fpu.state.xsave;
/*
* We should not ever be requesting features that we
* have not enabled. Remember that pcntxt_mask is
* what we write to the XCR0 register.
*/
WARN_ONCE(!(xfeatures_mask & xstate_feature),
"get of unsupported state");
/*
* This assumes the last 'xsave*' instruction to
* have requested that 'xstate_feature' be saved.
* If it did not, we might be seeing and old value
* of the field in the buffer.
*
* This can happen because the last 'xsave' did not
* request that this feature be saved (unlikely)
* or because the "init optimization" caused it
* to not be saved.
*/
if (!(xsave->header.xfeatures & xstate_feature))
return NULL;
return (void *)xsave + xstate_comp_offsets[feature_nr];
}
EXPORT_SYMBOL_GPL(get_xsave_addr);
/*
* This wraps up the common operations that need to occur when retrieving
* data from xsave state. It first ensures that the current task was
* using the FPU and retrieves the data in to a buffer. It then calculates
* the offset of the requested field in the buffer.
*
* This function is safe to call whether the FPU is in use or not.
*
* Note that this only works on the current task.
*
* Inputs:
* @xsave_state: state which is defined in xsave.h (e.g. XSTATE_FP,
* XSTATE_SSE, etc...)
* Output:
* address of the state in the xsave area or NULL if the state
* is not present or is in its 'init state'.
*/
const void *get_xsave_field_ptr(int xsave_state)
{
struct fpu *fpu = &current->thread.fpu;
if (!fpu->fpstate_active)
return NULL;
/*
* fpu__save() takes the CPU's xstate registers
* and saves them off to the 'fpu memory buffer.
*/
fpu__save(fpu);
return get_xsave_addr(&fpu->state.xsave, xsave_state);
}
arch/x86/kernel/process.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kernel/process_32.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kernel/process_64.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kernel/ptrace.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kernel/signal.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kernel/smpboot.c
浏览文件 @ e75c73ad
......@@ -68,8 +68,7 @@
#include <asm/mwait.h>
#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/i387.h>
#include <asm/fpu-internal.h>
#include <asm/fpu/internal.h>
#include <asm/setup.h>
#include <asm/uv/uv.h>
#include <linux/mc146818rtc.h>
......
arch/x86/kernel/traps.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kernel/uprobes.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kvm/cpuid.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kvm/vmx.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/kvm/x86.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/lguest/boot.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/lib/mmx_32.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/math-emu/fpu_aux.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/math-emu/fpu_entry.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/math-emu/fpu_system.h
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/mm/mpx.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/power/cpu.c
浏览文件 @ e75c73ad
此差异已折叠。
arch/x86/xen/enlighten.c
浏览文件 @ e75c73ad
此差异已折叠。
drivers/char/hw_random/via-rng.c
浏览文件 @ e75c73ad
此差异已折叠。
drivers/crypto/padlock-aes.c
浏览文件 @ e75c73ad
此差异已折叠。
drivers/crypto/padlock-sha.c
浏览文件 @ e75c73ad
此差异已折叠。
drivers/lguest/x86/core.c
浏览文件 @ e75c73ad
此差异已折叠。
kernel/sys.c
浏览文件 @ e75c73ad
此差异已折叠。
lib/raid6/x86.h
浏览文件 @ e75c73ad
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或