processor.h 24.0 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
H
H. Peter Anvin 已提交
2 3
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H
4

5 6
#include <asm/processor-flags.h>

7 8 9
/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;
10
struct vm86;
11

12 13 14
#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
15
#include <uapi/asm/sigcontext.h>
16
#include <asm/current.h>
17
#include <asm/cpufeatures.h>
18
#include <asm/page.h>
19
#include <asm/pgtable_types.h>
20
#include <asm/percpu.h>
21 22
#include <asm/msr.h>
#include <asm/desc_defs.h>
23
#include <asm/nops.h>
24
#include <asm/special_insns.h>
25
#include <asm/fpu/types.h>
26
#include <asm/unwind_hints.h>
27

28
#include <linux/personality.h>
29
#include <linux/cache.h>
30
#include <linux/threads.h>
31
#include <linux/math64.h>
32
#include <linux/err.h>
33
#include <linux/irqflags.h>
34
#include <linux/mem_encrypt.h>
35 36 37 38 39 40 41 42

/*
 * We handle most unaligned accesses in hardware.  On the other hand
 * unaligned DMA can be quite expensive on some Nehalem processors.
 *
 * Based on this we disable the IP header alignment in network drivers.
 */
#define NET_IP_ALIGN	0
43

44
#define HBP_NUM 4
45 46 47 48 49 50 51
/*
 * Default implementation of macro that returns current
 * instruction pointer ("program counter").
 */
static inline void *current_text_addr(void)
{
	void *pc;
52 53 54

	asm volatile("mov $1f, %0; 1:":"=r" (pc));

55 56 57
	return pc;
}

58 59 60 61 62
/*
 * These alignment constraints are for performance in the vSMP case,
 * but in the task_struct case we must also meet hardware imposed
 * alignment requirements of the FPU state:
 */
63
#ifdef CONFIG_X86_VSMP
64 65
# define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
66
#else
67
# define ARCH_MIN_TASKALIGN		__alignof__(union fpregs_state)
68
# define ARCH_MIN_MMSTRUCT_ALIGN	0
69 70
#endif

71 72 73 74 75 76 77 78 79 80 81
enum tlb_infos {
	ENTRIES,
	NR_INFO
};

extern u16 __read_mostly tlb_lli_4k[NR_INFO];
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
82
extern u16 __read_mostly tlb_lld_1g[NR_INFO];
83

84 85
/*
 *  CPU type and hardware bug flags. Kept separately for each CPU.
86
 *  Members of this structure are referenced in head_32.S, so think twice
87 88 89 90
 *  before touching them. [mj]
 */

struct cpuinfo_x86 {
91 92 93 94
	__u8			x86;		/* CPU family */
	__u8			x86_vendor;	/* CPU vendor */
	__u8			x86_model;
	__u8			x86_mask;
95
#ifdef CONFIG_X86_64
96
	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
97
	int			x86_tlbsize;
98
#endif
99 100 101 102
	__u8			x86_virt_bits;
	__u8			x86_phys_bits;
	/* CPUID returned core id bits: */
	__u8			x86_coreid_bits;
103
	__u8			cu_id;
104 105 106 107
	/* Max extended CPUID function supported: */
	__u32			extended_cpuid_level;
	/* Maximum supported CPUID level, -1=no CPUID: */
	int			cpuid_level;
108
	__u32			x86_capability[NCAPINTS + NBUGINTS];
109 110 111 112 113
	char			x86_vendor_id[16];
	char			x86_model_id[64];
	/* in KB - valid for CPUS which support this call: */
	int			x86_cache_size;
	int			x86_cache_alignment;	/* In bytes */
114 115 116
	/* Cache QoS architectural values: */
	int			x86_cache_max_rmid;	/* max index */
	int			x86_cache_occ_scale;	/* scale to bytes */
117 118 119 120 121
	int			x86_power;
	unsigned long		loops_per_jiffy;
	/* cpuid returned max cores value: */
	u16			 x86_max_cores;
	u16			apicid;
Y
Yinghai Lu 已提交
122
	u16			initial_apicid;
123 124 125 126 127
	u16			x86_clflush_size;
	/* number of cores as seen by the OS: */
	u16			booted_cores;
	/* Physical processor id: */
	u16			phys_proc_id;
128 129
	/* Logical processor id: */
	u16			logical_proc_id;
130 131 132 133
	/* Core id: */
	u16			cpu_core_id;
	/* Index into per_cpu list: */
	u16			cpu_index;
134
	u32			microcode;
135
} __randomize_layout;
136

137 138 139 140 141 142 143 144 145 146 147
struct cpuid_regs {
	u32 eax, ebx, ecx, edx;
};

enum cpuid_regs_idx {
	CPUID_EAX = 0,
	CPUID_EBX,
	CPUID_ECX,
	CPUID_EDX,
};

148 149 150 151 152 153 154 155 156 157
#define X86_VENDOR_INTEL	0
#define X86_VENDOR_CYRIX	1
#define X86_VENDOR_AMD		2
#define X86_VENDOR_UMC		3
#define X86_VENDOR_CENTAUR	5
#define X86_VENDOR_TRANSMETA	7
#define X86_VENDOR_NSC		8
#define X86_VENDOR_NUM		9

#define X86_VENDOR_UNKNOWN	0xff
158

159 160 161
/*
 * capabilities of CPUs
 */
162 163 164
extern struct cpuinfo_x86	boot_cpu_data;
extern struct cpuinfo_x86	new_cpu_data;

165
extern struct x86_hw_tss	doublefault_tss;
166 167
extern __u32			cpu_caps_cleared[NCAPINTS];
extern __u32			cpu_caps_set[NCAPINTS];
168 169

#ifdef CONFIG_SMP
170
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
171 172
#define cpu_data(cpu)		per_cpu(cpu_info, cpu)
#else
173
#define cpu_info		boot_cpu_data
174 175 176
#define cpu_data(cpu)		boot_cpu_data
#endif

177 178
extern const struct seq_operations cpuinfo_op;

179 180 181
#define cache_line_size()	(boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);
182

183
extern void early_cpu_init(void);
184 185
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
186
extern void print_cpu_info(struct cpuinfo_x86 *);
187
void print_cpu_msr(struct cpuinfo_x86 *);
188
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
189 190 191
extern u32 get_scattered_cpuid_leaf(unsigned int level,
				    unsigned int sub_leaf,
				    enum cpuid_regs_idx reg);
192
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
193
extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
194

195
extern void detect_extended_topology(struct cpuinfo_x86 *c);
196 197
extern void detect_ht(struct cpuinfo_x86 *c);

198 199 200 201 202 203 204 205
#ifdef CONFIG_X86_32
extern int have_cpuid_p(void);
#else
static inline int have_cpuid_p(void)
{
	return 1;
}
#endif
206
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
207
				unsigned int *ecx, unsigned int *edx)
208 209
{
	/* ecx is often an input as well as an output. */
210
	asm volatile("cpuid"
211 212 213 214
	    : "=a" (*eax),
	      "=b" (*ebx),
	      "=c" (*ecx),
	      "=d" (*edx)
215 216
	    : "0" (*eax), "2" (*ecx)
	    : "memory");
217 218
}

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
#define native_cpuid_reg(reg)					\
static inline unsigned int native_cpuid_##reg(unsigned int op)	\
{								\
	unsigned int eax = op, ebx, ecx = 0, edx;		\
								\
	native_cpuid(&eax, &ebx, &ecx, &edx);			\
								\
	return reg;						\
}

/*
 * Native CPUID functions returning a single datum.
 */
native_cpuid_reg(eax)
native_cpuid_reg(ebx)
native_cpuid_reg(ecx)
native_cpuid_reg(edx)

237 238 239 240 241 242 243 244
/*
 * Friendlier CR3 helpers.
 */
static inline unsigned long read_cr3_pa(void)
{
	return __read_cr3() & CR3_ADDR_MASK;
}

245 246 247 248 249
static inline unsigned long native_read_cr3_pa(void)
{
	return __native_read_cr3() & CR3_ADDR_MASK;
}

250 251
static inline void load_cr3(pgd_t *pgdir)
{
252
	write_cr3(__sme_pa(pgdir));
253
}
254

255 256 257 258 259
/*
 * Note that while the legacy 'TSS' name comes from 'Task State Segment',
 * on modern x86 CPUs the TSS also holds information important to 64-bit mode,
 * unrelated to the task-switch mechanism:
 */
260 261 262
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
263 264 265
	unsigned short		back_link, __blh;
	unsigned long		sp0;
	unsigned short		ss0, __ss0h;
266
	unsigned long		sp1;
267 268

	/*
269 270 271 272 273 274
	 * We don't use ring 1, so ss1 is a convenient scratch space in
	 * the same cacheline as sp0.  We use ss1 to cache the value in
	 * MSR_IA32_SYSENTER_CS.  When we context switch
	 * MSR_IA32_SYSENTER_CS, we first check if the new value being
	 * written matches ss1, and, if it's not, then we wrmsr the new
	 * value and update ss1.
275
	 *
276 277 278 279
	 * The only reason we context switch MSR_IA32_SYSENTER_CS is
	 * that we set it to zero in vm86 tasks to avoid corrupting the
	 * stack if we were to go through the sysenter path from vm86
	 * mode.
280 281 282 283
	 */
	unsigned short		ss1;	/* MSR_IA32_SYSENTER_CS */

	unsigned short		__ss1h;
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
	unsigned long		sp2;
	unsigned short		ss2, __ss2h;
	unsigned long		__cr3;
	unsigned long		ip;
	unsigned long		flags;
	unsigned long		ax;
	unsigned long		cx;
	unsigned long		dx;
	unsigned long		bx;
	unsigned long		sp;
	unsigned long		bp;
	unsigned long		si;
	unsigned long		di;
	unsigned short		es, __esh;
	unsigned short		cs, __csh;
	unsigned short		ss, __ssh;
	unsigned short		ds, __dsh;
	unsigned short		fs, __fsh;
	unsigned short		gs, __gsh;
	unsigned short		ldt, __ldth;
	unsigned short		trace;
	unsigned short		io_bitmap_base;

307 308 309
} __attribute__((packed));
#else
struct x86_hw_tss {
310 311
	u32			reserved1;
	u64			sp0;
312 313 314 315 316

	/*
	 * We store cpu_current_top_of_stack in sp1 so it's always accessible.
	 * Linux does not use ring 1, so sp1 is not otherwise needed.
	 */
317
	u64			sp1;
318

319 320 321 322 323 324 325 326
	u64			sp2;
	u64			reserved2;
	u64			ist[7];
	u32			reserved3;
	u32			reserved4;
	u16			reserved5;
	u16			io_bitmap_base;

327
} __attribute__((packed));
328 329 330
#endif

/*
331
 * IO-bitmap sizes:
332
 */
333 334 335
#define IO_BITMAP_BITS			65536
#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
336
#define IO_BITMAP_OFFSET		(offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss))
337
#define INVALID_IO_BITMAP_OFFSET	0x8000
338 339

struct tss_struct {
340
	/*
341 342 343 344 345 346 347 348 349
	 * Space for the temporary SYSENTER stack, used for SYSENTER
	 * and the entry trampoline as well.
	 */
	unsigned long		SYSENTER_stack[64];

	/*
	 * The fixed hardware portion.  This must not cross a page boundary
	 * at risk of violating the SDM's advice and potentially triggering
	 * errata.
350 351
	 */
	struct x86_hw_tss	x86_tss;
352 353 354 355 356 357 358

	/*
	 * The extra 1 is there because the CPU will access an
	 * additional byte beyond the end of the IO permission
	 * bitmap. The extra byte must be all 1 bits, and must
	 * be within the limit.
	 */
359
	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];
360
} __aligned(PAGE_SIZE);
361

362
DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss);
363

364 365 366 367 368 369 370 371 372 373
/*
 * sizeof(unsigned long) coming from an extra "long" at the end
 * of the iobitmap.
 *
 * -1? seg base+limit should be pointing to the address of the
 * last valid byte
 */
#define __KERNEL_TSS_LIMIT	\
	(IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)

374 375
#ifdef CONFIG_X86_32
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
376 377
#else
#define cpu_current_top_of_stack cpu_tss.x86_tss.sp1
378 379
#endif

380 381 382
/*
 * Save the original ist values for checking stack pointers during debugging
 */
383
struct orig_ist {
384
	unsigned long		ist[7];
385 386
};

387
#ifdef CONFIG_X86_64
388
DECLARE_PER_CPU(struct orig_ist, orig_ist);
389

390 391 392 393 394 395 396 397 398 399 400 401 402
union irq_stack_union {
	char irq_stack[IRQ_STACK_SIZE];
	/*
	 * GCC hardcodes the stack canary as %gs:40.  Since the
	 * irq_stack is the object at %gs:0, we reserve the bottom
	 * 48 bytes of the irq stack for the canary.
	 */
	struct {
		char gs_base[40];
		unsigned long stack_canary;
	};
};

403
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
404 405
DECLARE_INIT_PER_CPU(irq_stack_union);

406
DECLARE_PER_CPU(char *, irq_stack_ptr);
407 408
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
409 410
#else	/* X86_64 */
#ifdef CONFIG_CC_STACKPROTECTOR
411 412 413 414 415 416 417 418 419 420
/*
 * Make sure stack canary segment base is cached-aligned:
 *   "For Intel Atom processors, avoid non zero segment base address
 *    that is not aligned to cache line boundary at all cost."
 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
 */
struct stack_canary {
	char __pad[20];		/* canary at %gs:20 */
	unsigned long canary;
};
421
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
422
#endif
423 424 425 426 427 428 429 430 431
/*
 * per-CPU IRQ handling stacks
 */
struct irq_stack {
	u32                     stack[THREAD_SIZE/sizeof(u32)];
} __aligned(THREAD_SIZE);

DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
432
#endif	/* X86_64 */
433

434
extern unsigned int fpu_kernel_xstate_size;
435
extern unsigned int fpu_user_xstate_size;
436

437 438
struct perf_event;

439 440 441 442
typedef struct {
	unsigned long		seg;
} mm_segment_t;

443
struct thread_struct {
444 445
	/* Cached TLS descriptors: */
	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
446
#ifdef CONFIG_X86_32
447
	unsigned long		sp0;
448
#endif
449
	unsigned long		sp;
450
#ifdef CONFIG_X86_32
451
	unsigned long		sysenter_cs;
452
#else
453 454 455 456
	unsigned short		es;
	unsigned short		ds;
	unsigned short		fsindex;
	unsigned short		gsindex;
457
#endif
458 459 460

	u32			status;		/* thread synchronous flags */

461
#ifdef CONFIG_X86_64
462 463 464 465 466 467 468 469 470
	unsigned long		fsbase;
	unsigned long		gsbase;
#else
	/*
	 * XXX: this could presumably be unsigned short.  Alternatively,
	 * 32-bit kernels could be taught to use fsindex instead.
	 */
	unsigned long fs;
	unsigned long gs;
471
#endif
472

473 474 475 476
	/* Save middle states of ptrace breakpoints */
	struct perf_event	*ptrace_bps[HBP_NUM];
	/* Debug status used for traps, single steps, etc... */
	unsigned long           debugreg6;
477 478
	/* Keep track of the exact dr7 value set by the user */
	unsigned long           ptrace_dr7;
479 480
	/* Fault info: */
	unsigned long		cr2;
481
	unsigned long		trap_nr;
482
	unsigned long		error_code;
483
#ifdef CONFIG_VM86
484
	/* Virtual 86 mode info */
485
	struct vm86		*vm86;
486
#endif
487 488 489 490 491
	/* IO permissions: */
	unsigned long		*io_bitmap_ptr;
	unsigned long		iopl;
	/* Max allowed port in the bitmap, in bytes: */
	unsigned		io_bitmap_max;
492

493 494
	mm_segment_t		addr_limit;

495
	unsigned int		sig_on_uaccess_err:1;
496 497
	unsigned int		uaccess_err:1;	/* uaccess failed */

498 499 500 501 502 503
	/* Floating point and extended processor state */
	struct fpu		fpu;
	/*
	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
	 * the end.
	 */
504 505
};

506 507 508 509 510 511 512 513 514
/*
 * Thread-synchronous status.
 *
 * This is different from the flags in that nobody else
 * ever touches our thread-synchronous status, so we don't
 * have to worry about atomic accesses.
 */
#define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/

515 516 517 518 519 520 521
/*
 * Set IOPL bits in EFLAGS from given mask
 */
static inline void native_set_iopl_mask(unsigned mask)
{
#ifdef CONFIG_X86_32
	unsigned int reg;
522

523 524 525 526 527 528 529 530
	asm volatile ("pushfl;"
		      "popl %0;"
		      "andl %1, %0;"
		      "orl %2, %0;"
		      "pushl %0;"
		      "popfl"
		      : "=&r" (reg)
		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
531 532 533
#endif
}

534
static inline void
535
native_load_sp0(unsigned long sp0)
536
{
537
	this_cpu_write(cpu_tss.x86_tss.sp0, sp0);
538
}
539

540 541 542 543 544 545 546
static inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
	asm volatile("swapgs" ::: "memory");
#endif
}

547
static inline unsigned long current_top_of_stack(void)
548
{
549 550 551 552 553
	/*
	 *  We can't read directly from tss.sp0: sp0 on x86_32 is special in
	 *  and around vm86 mode and sp0 on x86_64 is special because of the
	 *  entry trampoline.
	 */
554
	return this_cpu_read_stable(cpu_current_top_of_stack);
555 556
}

557 558 559 560 561 562
static inline bool on_thread_stack(void)
{
	return (unsigned long)(current_top_of_stack() -
			       current_stack_pointer) < THREAD_SIZE;
}

563 564 565
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
566
#define __cpuid			native_cpuid
567

568
static inline void load_sp0(unsigned long sp0)
569
{
570
	native_load_sp0(sp0);
571 572
}

573
#define set_iopl_mask native_set_iopl_mask
574 575
#endif /* CONFIG_PARAVIRT */

576 577 578 579
/* Free all resources held by a thread. */
extern void release_thread(struct task_struct *);

unsigned long get_wchan(struct task_struct *p);
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612

/*
 * Generic CPUID function
 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
 * resulting in stale register contents being returned.
 */
static inline void cpuid(unsigned int op,
			 unsigned int *eax, unsigned int *ebx,
			 unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = 0;
	__cpuid(eax, ebx, ecx, edx);
}

/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(unsigned int op, int count,
			       unsigned int *eax, unsigned int *ebx,
			       unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = count;
	__cpuid(eax, ebx, ecx, edx);
}

/*
 * CPUID functions returning a single datum
 */
static inline unsigned int cpuid_eax(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
613

614 615
	return eax;
}
616

617 618 619 620 621
static inline unsigned int cpuid_ebx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
622

623 624
	return ebx;
}
625

626 627 628 629 630
static inline unsigned int cpuid_ecx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
631

632 633
	return ecx;
}
634

635 636 637 638 639
static inline unsigned int cpuid_edx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
640

641 642 643
	return edx;
}

644
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
645
static __always_inline void rep_nop(void)
646
{
647
	asm volatile("rep; nop" ::: "memory");
648 649
}

650
static __always_inline void cpu_relax(void)
651 652 653 654
{
	rep_nop();
}

655 656 657 658 659 660 661 662 663 664 665 666 667 668
/*
 * This function forces the icache and prefetched instruction stream to
 * catch up with reality in two very specific cases:
 *
 *  a) Text was modified using one virtual address and is about to be executed
 *     from the same physical page at a different virtual address.
 *
 *  b) Text was modified on a different CPU, may subsequently be
 *     executed on this CPU, and you want to make sure the new version
 *     gets executed.  This generally means you're calling this in a IPI.
 *
 * If you're calling this for a different reason, you're probably doing
 * it wrong.
 */
669 670
static inline void sync_core(void)
{
671
	/*
672 673 674 675 676 677 678 679 680 681 682 683
	 * There are quite a few ways to do this.  IRET-to-self is nice
	 * because it works on every CPU, at any CPL (so it's compatible
	 * with paravirtualization), and it never exits to a hypervisor.
	 * The only down sides are that it's a bit slow (it seems to be
	 * a bit more than 2x slower than the fastest options) and that
	 * it unmasks NMIs.  The "push %cs" is needed because, in
	 * paravirtual environments, __KERNEL_CS may not be a valid CS
	 * value when we do IRET directly.
	 *
	 * In case NMI unmasking or performance ever becomes a problem,
	 * the next best option appears to be MOV-to-CR2 and an
	 * unconditional jump.  That sequence also works on all CPUs,
684
	 * but it will fault at CPL3 (i.e. Xen PV).
685 686 687 688 689 690 691
	 *
	 * CPUID is the conventional way, but it's nasty: it doesn't
	 * exist on some 486-like CPUs, and it usually exits to a
	 * hypervisor.
	 *
	 * Like all of Linux's memory ordering operations, this is a
	 * compiler barrier as well.
692
	 */
693 694 695 696 697 698 699
#ifdef CONFIG_X86_32
	asm volatile (
		"pushfl\n\t"
		"pushl %%cs\n\t"
		"pushl $1f\n\t"
		"iret\n\t"
		"1:"
700
		: ASM_CALL_CONSTRAINT : : "memory");
701
#else
702 703 704
	unsigned int tmp;

	asm volatile (
705
		UNWIND_HINT_SAVE
706 707 708 709 710 711 712 713 714
		"mov %%ss, %0\n\t"
		"pushq %q0\n\t"
		"pushq %%rsp\n\t"
		"addq $8, (%%rsp)\n\t"
		"pushfq\n\t"
		"mov %%cs, %0\n\t"
		"pushq %q0\n\t"
		"pushq $1f\n\t"
		"iretq\n\t"
715
		UNWIND_HINT_RESTORE
716
		"1:"
717
		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
718
#endif
719 720 721
}

extern void select_idle_routine(const struct cpuinfo_x86 *c);
722
extern void amd_e400_c1e_apic_setup(void);
723

724
extern unsigned long		boot_option_idle_override;
725

726
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
727
			 IDLE_POLL};
728

729 730 731
extern void enable_sep_cpu(void);
extern int sysenter_setup(void);

732
extern void early_trap_init(void);
733
void early_trap_pf_init(void);
734

735
/* Defined in head.S */
736
extern struct desc_ptr		early_gdt_descr;
737 738

extern void cpu_set_gdt(int);
739
extern void switch_to_new_gdt(int);
740
extern void load_direct_gdt(int);
741
extern void load_fixmap_gdt(int);
742
extern void load_percpu_segment(int);
743 744
extern void cpu_init(void);

745 746
static inline unsigned long get_debugctlmsr(void)
{
P
Peter Zijlstra 已提交
747
	unsigned long debugctlmsr = 0;
748 749 750 751 752 753 754

#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return 0;
#endif
	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);

P
Peter Zijlstra 已提交
755
	return debugctlmsr;
756 757
}

758 759 760 761 762 763 764 765 766
static inline void update_debugctlmsr(unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return;
#endif
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
}

767 768
extern void set_task_blockstep(struct task_struct *task, bool on);

769 770
/* Boot loader type from the setup header: */
extern int			bootloader_type;
771
extern int			bootloader_version;
772

773
extern char			ignore_fpu_irq;
774 775 776 777 778

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#define ARCH_HAS_SPINLOCK_PREFETCH

779
#ifdef CONFIG_X86_32
780
# define BASE_PREFETCH		""
781
# define ARCH_HAS_PREFETCH
782
#else
783
# define BASE_PREFETCH		"prefetcht0 %P1"
784 785
#endif

786 787 788 789 790 791
/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
792 793
static inline void prefetch(const void *x)
{
794
	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
795
			  X86_FEATURE_XMM,
796
			  "m" (*(const char *)x));
797 798
}

799 800 801 802 803
/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
804 805
static inline void prefetchw(const void *x)
{
806 807 808
	alternative_input(BASE_PREFETCH, "prefetchw %P1",
			  X86_FEATURE_3DNOWPREFETCH,
			  "m" (*(const char *)x));
809 810
}

811 812 813 814 815
static inline void spin_lock_prefetch(const void *x)
{
	prefetchw(x);
}

816 817 818
#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
			   TOP_OF_KERNEL_STACK_PADDING)

819 820
#define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))

821 822 823 824 825 826 827
#define task_pt_regs(task) \
({									\
	unsigned long __ptr = (unsigned long)task_stack_page(task);	\
	__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;		\
	((struct pt_regs *)__ptr) - 1;					\
})

828 829 830 831
#ifdef CONFIG_X86_32
/*
 * User space process size: 3GB (default).
 */
832
#define IA32_PAGE_OFFSET	PAGE_OFFSET
833
#define TASK_SIZE		PAGE_OFFSET
834
#define TASK_SIZE_LOW		TASK_SIZE
835
#define TASK_SIZE_MAX		TASK_SIZE
836
#define DEFAULT_MAP_WINDOW	TASK_SIZE
837 838 839 840
#define STACK_TOP		TASK_SIZE
#define STACK_TOP_MAX		STACK_TOP

#define INIT_THREAD  {							  \
841
	.sp0			= TOP_OF_INIT_STACK,			  \
842 843
	.sysenter_cs		= __KERNEL_CS,				  \
	.io_bitmap_ptr		= NULL,					  \
844
	.addr_limit		= KERNEL_DS,				  \
845 846
}

847
#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
848 849 850

#else
/*
851 852 853 854 855 856 857
 * User space process size. 47bits minus one guard page.  The guard
 * page is necessary on Intel CPUs: if a SYSCALL instruction is at
 * the highest possible canonical userspace address, then that
 * syscall will enter the kernel with a non-canonical return
 * address, and SYSRET will explode dangerously.  We avoid this
 * particular problem by preventing anything from being mapped
 * at the maximum canonical address.
858
 */
859
#define TASK_SIZE_MAX	((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)
860

861
#define DEFAULT_MAP_WINDOW	((1UL << 47) - PAGE_SIZE)
862 863 864 865

/* This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
866 867
#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
					0xc0000000 : 0xFFFFe000)
868

869 870
#define TASK_SIZE_LOW		(test_thread_flag(TIF_ADDR32) ? \
					IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
871
#define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
872
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
873
#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
874
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
875

876
#define STACK_TOP		TASK_SIZE_LOW
877
#define STACK_TOP_MAX		TASK_SIZE_MAX
878

879 880
#define INIT_THREAD  {						\
	.addr_limit		= KERNEL_DS,			\
881 882
}

883
extern unsigned long KSTK_ESP(struct task_struct *task);
884

885 886
#endif /* CONFIG_X86_64 */

I
Ingo Molnar 已提交
887 888 889
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
					       unsigned long new_sp);

890 891
/*
 * This decides where the kernel will search for a free chunk of vm
892 893
 * space during mmap's.
 */
894
#define __TASK_UNMAPPED_BASE(task_size)	(PAGE_ALIGN(task_size / 3))
895
#define TASK_UNMAPPED_BASE		__TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
896

897
#define KSTK_EIP(task)		(task_pt_regs(task)->ip)
898

899 900 901 902 903 904 905
/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)	get_tsc_mode((adr))
#define SET_TSC_CTL(val)	set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

906 907
DECLARE_PER_CPU(u64, msr_misc_features_shadow);

908
/* Register/unregister a process' MPX related resource */
909 910
#define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()
#define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()
911 912

#ifdef CONFIG_X86_INTEL_MPX
913 914
extern int mpx_enable_management(void);
extern int mpx_disable_management(void);
915
#else
916
static inline int mpx_enable_management(void)
917 918 919
{
	return -EINVAL;
}
920
static inline int mpx_disable_management(void)
921 922 923 924 925
{
	return -EINVAL;
}
#endif /* CONFIG_X86_INTEL_MPX */

926
#ifdef CONFIG_CPU_SUP_AMD
927
extern u16 amd_get_nb_id(int cpu);
928
extern u32 amd_get_nodes_per_socket(void);
929 930 931 932
#else
static inline u16 amd_get_nb_id(int cpu)		{ return 0; }
static inline u32 amd_get_nodes_per_socket(void)	{ return 0; }
#endif
933

934 935 936 937 938 939 940 941 942 943 944 945 946 947 948
static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
{
	uint32_t base, eax, signature[3];

	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);

		if (!memcmp(sig, signature, 12) &&
		    (leaves == 0 || ((eax - base) >= leaves)))
			return base;
	}

	return 0;
}

949 950 951 952
extern unsigned long arch_align_stack(unsigned long sp);
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);

void default_idle(void);
953 954 955 956 957
#ifdef	CONFIG_XEN
bool xen_set_default_idle(void);
#else
#define xen_set_default_idle 0
#endif
958 959

void stop_this_cpu(void *dummy);
960
void df_debug(struct pt_regs *regs, long error_code);
H
H. Peter Anvin 已提交
961
#endif /* _ASM_X86_PROCESSOR_H */