processor.h 24.0 KB
Newer Older
H
H. Peter Anvin 已提交
1 2
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H
3

4 5
#include <asm/processor-flags.h>

6 7 8 9
/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;

10 11 12 13 14 15 16 17
#include <asm/vm86.h>
#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
#include <asm/sigcontext.h>
#include <asm/current.h>
#include <asm/cpufeature.h>
#include <asm/page.h>
18
#include <asm/pgtable_types.h>
19
#include <asm/percpu.h>
20 21
#include <asm/msr.h>
#include <asm/desc_defs.h>
22
#include <asm/nops.h>
23
#include <asm/special_insns.h>
24

25
#include <linux/personality.h>
26 27
#include <linux/cpumask.h>
#include <linux/cache.h>
28
#include <linux/threads.h>
29
#include <linux/math64.h>
30
#include <linux/err.h>
31 32 33 34 35 36 37 38 39
#include <linux/irqflags.h>

/*
 * We handle most unaligned accesses in hardware.  On the other hand
 * unaligned DMA can be quite expensive on some Nehalem processors.
 *
 * Based on this we disable the IP header alignment in network drivers.
 */
#define NET_IP_ALIGN	0
40

41
#define HBP_NUM 4
42 43 44 45 46 47 48
/*
 * Default implementation of macro that returns current
 * instruction pointer ("program counter").
 */
static inline void *current_text_addr(void)
{
	void *pc;
49 50 51

	asm volatile("mov $1f, %0; 1:":"=r" (pc));

52 53 54
	return pc;
}

55
#ifdef CONFIG_X86_VSMP
56 57
# define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
58
#else
59 60
# define ARCH_MIN_TASKALIGN		16
# define ARCH_MIN_MMSTRUCT_ALIGN	0
61 62
#endif

63 64 65 66 67 68 69 70 71 72 73
enum tlb_infos {
	ENTRIES,
	NR_INFO
};

extern u16 __read_mostly tlb_lli_4k[NR_INFO];
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
74
extern u16 __read_mostly tlb_lld_1g[NR_INFO];
75

76 77 78 79 80 81 82
/*
 *  CPU type and hardware bug flags. Kept separately for each CPU.
 *  Members of this structure are referenced in head.S, so think twice
 *  before touching them. [mj]
 */

struct cpuinfo_x86 {
83 84 85 86
	__u8			x86;		/* CPU family */
	__u8			x86_vendor;	/* CPU vendor */
	__u8			x86_model;
	__u8			x86_mask;
87
#ifdef CONFIG_X86_32
88 89 90 91 92
	char			wp_works_ok;	/* It doesn't on 386's */

	/* Problems on some 486Dx4's and old 386's: */
	char			rfu;
	char			pad0;
93
	char			pad1;
94
#else
95
	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
96
	int			x86_tlbsize;
97
#endif
98 99 100 101 102 103 104 105
	__u8			x86_virt_bits;
	__u8			x86_phys_bits;
	/* CPUID returned core id bits: */
	__u8			x86_coreid_bits;
	/* Max extended CPUID function supported: */
	__u32			extended_cpuid_level;
	/* Maximum supported CPUID level, -1=no CPUID: */
	int			cpuid_level;
106
	__u32			x86_capability[NCAPINTS + NBUGINTS];
107 108 109 110 111 112 113 114 115 116
	char			x86_vendor_id[16];
	char			x86_model_id[64];
	/* in KB - valid for CPUS which support this call: */
	int			x86_cache_size;
	int			x86_cache_alignment;	/* In bytes */
	int			x86_power;
	unsigned long		loops_per_jiffy;
	/* cpuid returned max cores value: */
	u16			 x86_max_cores;
	u16			apicid;
Y
Yinghai Lu 已提交
117
	u16			initial_apicid;
118 119 120 121 122 123 124
	u16			x86_clflush_size;
	/* number of cores as seen by the OS: */
	u16			booted_cores;
	/* Physical processor id: */
	u16			phys_proc_id;
	/* Core id: */
	u16			cpu_core_id;
125 126
	/* Compute unit id */
	u8			compute_unit_id;
127 128
	/* Index into per_cpu list: */
	u16			cpu_index;
129
	u32			microcode;
130
};
131

132 133 134 135 136 137 138 139 140 141
#define X86_VENDOR_INTEL	0
#define X86_VENDOR_CYRIX	1
#define X86_VENDOR_AMD		2
#define X86_VENDOR_UMC		3
#define X86_VENDOR_CENTAUR	5
#define X86_VENDOR_TRANSMETA	7
#define X86_VENDOR_NSC		8
#define X86_VENDOR_NUM		9

#define X86_VENDOR_UNKNOWN	0xff
142

143 144 145
/*
 * capabilities of CPUs
 */
146 147 148 149
extern struct cpuinfo_x86	boot_cpu_data;
extern struct cpuinfo_x86	new_cpu_data;

extern struct tss_struct	doublefault_tss;
150 151
extern __u32			cpu_caps_cleared[NCAPINTS];
extern __u32			cpu_caps_set[NCAPINTS];
152 153

#ifdef CONFIG_SMP
154
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
155 156
#define cpu_data(cpu)		per_cpu(cpu_info, cpu)
#else
157
#define cpu_info		boot_cpu_data
158 159 160
#define cpu_data(cpu)		boot_cpu_data
#endif

161 162
extern const struct seq_operations cpuinfo_op;

163 164 165
#define cache_line_size()	(boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);
166
extern void fpu_detect(struct cpuinfo_x86 *c);
167

168
extern void early_cpu_init(void);
169 170
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
171
extern void print_cpu_info(struct cpuinfo_x86 *);
172
void print_cpu_msr(struct cpuinfo_x86 *);
173 174
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
175
extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
176

177
extern void detect_extended_topology(struct cpuinfo_x86 *c);
178 179
extern void detect_ht(struct cpuinfo_x86 *c);

180 181 182 183 184 185 186 187
#ifdef CONFIG_X86_32
extern int have_cpuid_p(void);
#else
static inline int have_cpuid_p(void)
{
	return 1;
}
#endif
188
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
189
				unsigned int *ecx, unsigned int *edx)
190 191
{
	/* ecx is often an input as well as an output. */
192
	asm volatile("cpuid"
193 194 195 196
	    : "=a" (*eax),
	      "=b" (*ebx),
	      "=c" (*ecx),
	      "=d" (*edx)
197 198
	    : "0" (*eax), "2" (*ecx)
	    : "memory");
199 200
}

201 202 203 204
static inline void load_cr3(pgd_t *pgdir)
{
	write_cr3(__pa(pgdir));
}
205

206 207 208
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
	unsigned short		back_link, __blh;
	unsigned long		sp0;
	unsigned short		ss0, __ss0h;
	unsigned long		sp1;
	/* ss1 caches MSR_IA32_SYSENTER_CS: */
	unsigned short		ss1, __ss1h;
	unsigned long		sp2;
	unsigned short		ss2, __ss2h;
	unsigned long		__cr3;
	unsigned long		ip;
	unsigned long		flags;
	unsigned long		ax;
	unsigned long		cx;
	unsigned long		dx;
	unsigned long		bx;
	unsigned long		sp;
	unsigned long		bp;
	unsigned long		si;
	unsigned long		di;
	unsigned short		es, __esh;
	unsigned short		cs, __csh;
	unsigned short		ss, __ssh;
	unsigned short		ds, __dsh;
	unsigned short		fs, __fsh;
	unsigned short		gs, __gsh;
	unsigned short		ldt, __ldth;
	unsigned short		trace;
	unsigned short		io_bitmap_base;

238 239 240
} __attribute__((packed));
#else
struct x86_hw_tss {
241 242 243 244 245 246 247 248 249 250 251
	u32			reserved1;
	u64			sp0;
	u64			sp1;
	u64			sp2;
	u64			reserved2;
	u64			ist[7];
	u32			reserved3;
	u32			reserved4;
	u16			reserved5;
	u16			io_bitmap_base;

252 253 254 255
} __attribute__((packed)) ____cacheline_aligned;
#endif

/*
256
 * IO-bitmap sizes:
257
 */
258 259 260 261 262
#define IO_BITMAP_BITS			65536
#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
#define IO_BITMAP_OFFSET		offsetof(struct tss_struct, io_bitmap)
#define INVALID_IO_BITMAP_OFFSET	0x8000
263 264

struct tss_struct {
265 266 267 268
	/*
	 * The hardware state:
	 */
	struct x86_hw_tss	x86_tss;
269 270 271 272 273 274 275

	/*
	 * The extra 1 is there because the CPU will access an
	 * additional byte beyond the end of the IO permission
	 * bitmap. The extra byte must be all 1 bits, and must
	 * be within the limit.
	 */
276 277
	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];

278
	/*
279
	 * .. and then another 0x100 bytes for the emergency kernel stack:
280
	 */
281 282
	unsigned long		stack[64];

283
} ____cacheline_aligned;
284

285
DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, init_tss);
286

287 288 289
/*
 * Save the original ist values for checking stack pointers during debugging
 */
290
struct orig_ist {
291
	unsigned long		ist[7];
292 293
};

R
Roland McGrath 已提交
294
#define	MXCSR_DEFAULT		0x1f80
295

R
Roland McGrath 已提交
296
struct i387_fsave_struct {
I
Ingo Molnar 已提交
297 298 299 300 301 302 303 304 305
	u32			cwd;	/* FPU Control Word		*/
	u32			swd;	/* FPU Status Word		*/
	u32			twd;	/* FPU Tag Word			*/
	u32			fip;	/* FPU IP Offset		*/
	u32			fcs;	/* FPU IP Selector		*/
	u32			foo;	/* FPU Operand Pointer Offset	*/
	u32			fos;	/* FPU Operand Pointer Selector	*/

	/* 8*10 bytes for each FP-reg = 80 bytes:			*/
306
	u32			st_space[20];
I
Ingo Molnar 已提交
307 308

	/* Software status information [not touched by FSAVE ]:		*/
309
	u32			status;
310 311 312
};

struct i387_fxsave_struct {
I
Ingo Molnar 已提交
313 314 315 316
	u16			cwd; /* Control Word			*/
	u16			swd; /* Status Word			*/
	u16			twd; /* Tag Word			*/
	u16			fop; /* Last Instruction Opcode		*/
R
Roland McGrath 已提交
317 318
	union {
		struct {
I
Ingo Molnar 已提交
319 320
			u64	rip; /* Instruction Pointer		*/
			u64	rdp; /* Data Pointer			*/
R
Roland McGrath 已提交
321 322
		};
		struct {
I
Ingo Molnar 已提交
323 324 325 326
			u32	fip; /* FPU IP Offset			*/
			u32	fcs; /* FPU IP Selector			*/
			u32	foo; /* FPU Operand Offset		*/
			u32	fos; /* FPU Operand Selector		*/
R
Roland McGrath 已提交
327 328
		};
	};
I
Ingo Molnar 已提交
329 330 331 332
	u32			mxcsr;		/* MXCSR Register State */
	u32			mxcsr_mask;	/* MXCSR Mask		*/

	/* 8*16 bytes for each FP-reg = 128 bytes:			*/
333
	u32			st_space[32];
I
Ingo Molnar 已提交
334 335

	/* 16*16 bytes for each XMM-reg = 256 bytes:			*/
336
	u32			xmm_space[64];
I
Ingo Molnar 已提交
337

338 339 340 341 342 343
	u32			padding[12];

	union {
		u32		padding1[12];
		u32		sw_reserved[12];
	};
344

345 346
} __attribute__((aligned(16)));

R
Roland McGrath 已提交
347
struct i387_soft_struct {
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
	u32			cwd;
	u32			swd;
	u32			twd;
	u32			fip;
	u32			fcs;
	u32			foo;
	u32			fos;
	/* 8*10 bytes for each FP-reg = 80 bytes: */
	u32			st_space[20];
	u8			ftop;
	u8			changed;
	u8			lookahead;
	u8			no_update;
	u8			rm;
	u8			alimit;
T
Tejun Heo 已提交
363
	struct math_emu_info	*info;
364
	u32			entry_eip;
R
Roland McGrath 已提交
365 366
};

367 368 369 370 371
struct ymmh_struct {
	/* 16 * 16 bytes for each YMMH-reg = 256 bytes */
	u32 ymmh_space[64];
};

372
/* We don't support LWP yet: */
373
struct lwp_struct {
374
	u8 reserved[128];
375 376
};

377 378 379
struct bndreg {
	u64 lower_bound;
	u64 upper_bound;
380 381
} __packed;

382 383 384
struct bndcsr {
	u64 bndcfgu;
	u64 bndstatus;
385 386
} __packed;

387 388
struct xsave_hdr_struct {
	u64 xstate_bv;
389 390
	u64 xcomp_bv;
	u64 reserved[6];
391 392 393 394 395
} __attribute__((packed));

struct xsave_struct {
	struct i387_fxsave_struct i387;
	struct xsave_hdr_struct xsave_hdr;
396
	struct ymmh_struct ymmh;
397
	struct lwp_struct lwp;
398
	struct bndreg bndreg[4];
399
	struct bndcsr bndcsr;
400 401 402
	/* new processor state extensions will go here */
} __attribute__ ((packed, aligned (64)));

403
union thread_xstate {
R
Roland McGrath 已提交
404
	struct i387_fsave_struct	fsave;
405
	struct i387_fxsave_struct	fxsave;
406
	struct i387_soft_struct		soft;
407
	struct xsave_struct		xsave;
408 409
};

410
struct fpu {
411 412
	unsigned int last_cpu;
	unsigned int has_fpu;
413 414 415
	union thread_xstate *state;
};

416
#ifdef CONFIG_X86_64
417
DECLARE_PER_CPU(struct orig_ist, orig_ist);
418

419 420 421 422 423 424 425 426 427 428 429 430 431
union irq_stack_union {
	char irq_stack[IRQ_STACK_SIZE];
	/*
	 * GCC hardcodes the stack canary as %gs:40.  Since the
	 * irq_stack is the object at %gs:0, we reserve the bottom
	 * 48 bytes of the irq stack for the canary.
	 */
	struct {
		char gs_base[40];
		unsigned long stack_canary;
	};
};

432
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
433 434
DECLARE_INIT_PER_CPU(irq_stack_union);

435
DECLARE_PER_CPU(char *, irq_stack_ptr);
436 437
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
438 439
#else	/* X86_64 */
#ifdef CONFIG_CC_STACKPROTECTOR
440 441 442 443 444 445 446 447 448 449
/*
 * Make sure stack canary segment base is cached-aligned:
 *   "For Intel Atom processors, avoid non zero segment base address
 *    that is not aligned to cache line boundary at all cost."
 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
 */
struct stack_canary {
	char __pad[20];		/* canary at %gs:20 */
	unsigned long canary;
};
450
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
451
#endif
452 453 454 455 456 457 458 459 460
/*
 * per-CPU IRQ handling stacks
 */
struct irq_stack {
	u32                     stack[THREAD_SIZE/sizeof(u32)];
} __aligned(THREAD_SIZE);

DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
461
#endif	/* X86_64 */
462

463
extern unsigned int xstate_size;
464 465
extern void free_thread_xstate(struct task_struct *);
extern struct kmem_cache *task_xstate_cachep;
466

467 468
struct perf_event;

469
struct thread_struct {
470 471 472 473
	/* Cached TLS descriptors: */
	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
	unsigned long		sp0;
	unsigned long		sp;
474
#ifdef CONFIG_X86_32
475
	unsigned long		sysenter_cs;
476
#else
477 478 479 480 481
	unsigned long		usersp;	/* Copy from PDA */
	unsigned short		es;
	unsigned short		ds;
	unsigned short		fsindex;
	unsigned short		gsindex;
482
#endif
483
#ifdef CONFIG_X86_32
484
	unsigned long		ip;
485
#endif
486
#ifdef CONFIG_X86_64
487
	unsigned long		fs;
488
#endif
489
	unsigned long		gs;
490 491 492 493
	/* Save middle states of ptrace breakpoints */
	struct perf_event	*ptrace_bps[HBP_NUM];
	/* Debug status used for traps, single steps, etc... */
	unsigned long           debugreg6;
494 495
	/* Keep track of the exact dr7 value set by the user */
	unsigned long           ptrace_dr7;
496 497
	/* Fault info: */
	unsigned long		cr2;
498
	unsigned long		trap_nr;
499
	unsigned long		error_code;
500
	/* floating point and extended processor state */
501
	struct fpu		fpu;
502
#ifdef CONFIG_X86_32
503
	/* Virtual 86 mode info */
504 505
	struct vm86_struct __user *vm86_info;
	unsigned long		screen_bitmap;
506 507 508 509 510
	unsigned long		v86flags;
	unsigned long		v86mask;
	unsigned long		saved_sp0;
	unsigned int		saved_fs;
	unsigned int		saved_gs;
511
#endif
512 513 514 515 516
	/* IO permissions: */
	unsigned long		*io_bitmap_ptr;
	unsigned long		iopl;
	/* Max allowed port in the bitmap, in bytes: */
	unsigned		io_bitmap_max;
517 518 519 520 521 522 523 524 525
	/*
	 * fpu_counter contains the number of consecutive context switches
	 * that the FPU is used. If this is over a threshold, the lazy fpu
	 * saving becomes unlazy to save the trap. This is an unsigned char
	 * so that after 256 times the counter wraps and the behavior turns
	 * lazy again; this to deal with bursty apps that only use FPU for
	 * a short time
	 */
	unsigned char fpu_counter;
526 527
};

528 529 530 531 532 533 534
/*
 * Set IOPL bits in EFLAGS from given mask
 */
static inline void native_set_iopl_mask(unsigned mask)
{
#ifdef CONFIG_X86_32
	unsigned int reg;
535

536 537 538 539 540 541 542 543
	asm volatile ("pushfl;"
		      "popl %0;"
		      "andl %1, %0;"
		      "orl %2, %0;"
		      "pushl %0;"
		      "popfl"
		      : "=&r" (reg)
		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
544 545 546
#endif
}

547 548
static inline void
native_load_sp0(struct tss_struct *tss, struct thread_struct *thread)
549 550 551
{
	tss->x86_tss.sp0 = thread->sp0;
#ifdef CONFIG_X86_32
552
	/* Only happens when SEP is enabled, no need to test "SEP"arately: */
553 554 555 556 557 558
	if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) {
		tss->x86_tss.ss1 = thread->sysenter_cs;
		wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
	}
#endif
}
559

560 561 562 563 564 565 566
static inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
	asm volatile("swapgs" ::: "memory");
#endif
}

567 568 569 570 571
static inline unsigned long this_cpu_sp0(void)
{
	return this_cpu_read_stable(init_tss.x86_tss.sp0);
}

572 573 574
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
575 576
#define __cpuid			native_cpuid
#define paravirt_enabled()	0
577

578 579
static inline void load_sp0(struct tss_struct *tss,
			    struct thread_struct *thread)
580 581 582 583
{
	native_load_sp0(tss, thread);
}

584
#define set_iopl_mask native_set_iopl_mask
585 586
#endif /* CONFIG_PARAVIRT */

587
typedef struct {
588
	unsigned long		seg;
589 590 591
} mm_segment_t;


592 593 594 595
/* Free all resources held by a thread. */
extern void release_thread(struct task_struct *);

unsigned long get_wchan(struct task_struct *p);
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628

/*
 * Generic CPUID function
 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
 * resulting in stale register contents being returned.
 */
static inline void cpuid(unsigned int op,
			 unsigned int *eax, unsigned int *ebx,
			 unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = 0;
	__cpuid(eax, ebx, ecx, edx);
}

/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(unsigned int op, int count,
			       unsigned int *eax, unsigned int *ebx,
			       unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = count;
	__cpuid(eax, ebx, ecx, edx);
}

/*
 * CPUID functions returning a single datum
 */
static inline unsigned int cpuid_eax(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
629

630 631
	return eax;
}
632

633 634 635 636 637
static inline unsigned int cpuid_ebx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
638

639 640
	return ebx;
}
641

642 643 644 645 646
static inline unsigned int cpuid_ecx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
647

648 649
	return ecx;
}
650

651 652 653 654 655
static inline unsigned int cpuid_edx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
656

657 658 659
	return edx;
}

660 661 662
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
static inline void rep_nop(void)
{
663
	asm volatile("rep; nop" ::: "memory");
664 665
}

666 667 668 669 670
static inline void cpu_relax(void)
{
	rep_nop();
}

671 672
#define cpu_relax_lowlatency() cpu_relax()

673
/* Stop speculative execution and prefetching of modified code. */
674 675 676
static inline void sync_core(void)
{
	int tmp;
677

678
#ifdef CONFIG_M486
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
	/*
	 * Do a CPUID if available, otherwise do a jump.  The jump
	 * can conveniently enough be the jump around CPUID.
	 */
	asm volatile("cmpl %2,%1\n\t"
		     "jl 1f\n\t"
		     "cpuid\n"
		     "1:"
		     : "=a" (tmp)
		     : "rm" (boot_cpu_data.cpuid_level), "ri" (0), "0" (1)
		     : "ebx", "ecx", "edx", "memory");
#else
	/*
	 * CPUID is a barrier to speculative execution.
	 * Prefetched instructions are automatically
	 * invalidated when modified.
	 */
	asm volatile("cpuid"
		     : "=a" (tmp)
		     : "0" (1)
		     : "ebx", "ecx", "edx", "memory");
700
#endif
701 702 703
}

extern void select_idle_routine(const struct cpuinfo_x86 *c);
704
extern void init_amd_e400_c1e_mask(void);
705

706
extern unsigned long		boot_option_idle_override;
707
extern bool			amd_e400_c1e_detected;
708

709
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
710
			 IDLE_POLL};
711

712 713 714
extern void enable_sep_cpu(void);
extern int sysenter_setup(void);

715
extern void early_trap_init(void);
716
void early_trap_pf_init(void);
717

718
/* Defined in head.S */
719
extern struct desc_ptr		early_gdt_descr;
720 721

extern void cpu_set_gdt(int);
722
extern void switch_to_new_gdt(int);
723
extern void load_percpu_segment(int);
724 725
extern void cpu_init(void);

726 727
static inline unsigned long get_debugctlmsr(void)
{
P
Peter Zijlstra 已提交
728
	unsigned long debugctlmsr = 0;
729 730 731 732 733 734 735

#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return 0;
#endif
	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);

P
Peter Zijlstra 已提交
736
	return debugctlmsr;
737 738
}

739 740 741 742 743 744 745 746 747
static inline void update_debugctlmsr(unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return;
#endif
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
}

748 749
extern void set_task_blockstep(struct task_struct *task, bool on);

750 751 752 753 754 755 756
/*
 * from system description table in BIOS. Mostly for MCA use, but
 * others may find it useful:
 */
extern unsigned int		machine_id;
extern unsigned int		machine_submodel_id;
extern unsigned int		BIOS_revision;
757

758 759
/* Boot loader type from the setup header: */
extern int			bootloader_type;
760
extern int			bootloader_version;
761

762
extern char			ignore_fpu_irq;
763 764 765 766 767

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#define ARCH_HAS_SPINLOCK_PREFETCH

768
#ifdef CONFIG_X86_32
769
# define BASE_PREFETCH		""
770
# define ARCH_HAS_PREFETCH
771
#else
772
# define BASE_PREFETCH		"prefetcht0 %P1"
773 774
#endif

775 776 777 778 779 780
/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
781 782
static inline void prefetch(const void *x)
{
783
	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
784
			  X86_FEATURE_XMM,
785
			  "m" (*(const char *)x));
786 787
}

788 789 790 791 792
/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
793 794
static inline void prefetchw(const void *x)
{
795 796 797
	alternative_input(BASE_PREFETCH, "prefetchw %P1",
			  X86_FEATURE_3DNOWPREFETCH,
			  "m" (*(const char *)x));
798 799
}

800 801 802 803 804
static inline void spin_lock_prefetch(const void *x)
{
	prefetchw(x);
}

805 806 807 808
#ifdef CONFIG_X86_32
/*
 * User space process size: 3GB (default).
 */
809
#define TASK_SIZE		PAGE_OFFSET
810
#define TASK_SIZE_MAX		TASK_SIZE
811 812 813 814 815 816 817 818
#define STACK_TOP		TASK_SIZE
#define STACK_TOP_MAX		STACK_TOP

#define INIT_THREAD  {							  \
	.sp0			= sizeof(init_stack) + (long)&init_stack, \
	.vm86_info		= NULL,					  \
	.sysenter_cs		= __KERNEL_CS,				  \
	.io_bitmap_ptr		= NULL,					  \
819 820 821 822 823 824 825 826
}

/*
 * Note that the .io_bitmap member must be extra-big. This is because
 * the CPU will access an additional byte beyond the end of the IO
 * permission bitmap. The extra byte must be all 1 bits, and must
 * be within the limit.
 */
827 828
#define INIT_TSS  {							  \
	.x86_tss = {							  \
829
		.sp0		= sizeof(init_stack) + (long)&init_stack, \
830 831 832 833 834
		.ss0		= __KERNEL_DS,				  \
		.ss1		= __KERNEL_CS,				  \
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,		  \
	 },								  \
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },	  \
835 836 837 838 839 840 841 842 843 844 845 846 847 848
}

extern unsigned long thread_saved_pc(struct task_struct *tsk);

#define THREAD_SIZE_LONGS      (THREAD_SIZE/sizeof(unsigned long))
#define KSTK_TOP(info)                                                 \
({                                                                     \
       unsigned long *__ptr = (unsigned long *)(info);                 \
       (unsigned long)(&__ptr[THREAD_SIZE_LONGS]);                     \
})

/*
 * The below -8 is to reserve 8 bytes on top of the ring0 stack.
 * This is necessary to guarantee that the entire "struct pt_regs"
849
 * is accessible even if the CPU haven't stored the SS/ESP registers
850 851 852 853 854 855 856 857 858 859 860 861 862
 * on the stack (interrupt gate does not save these registers
 * when switching to the same priv ring).
 * Therefore beware: accessing the ss/esp fields of the
 * "struct pt_regs" is possible, but they may contain the
 * completely wrong values.
 */
#define task_pt_regs(task)                                             \
({                                                                     \
       struct pt_regs *__regs__;                                       \
       __regs__ = (struct pt_regs *)(KSTK_TOP(task_stack_page(task))-8); \
       __regs__ - 1;                                                   \
})

863
#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
864 865 866

#else
/*
867 868 869 870 871 872 873
 * User space process size. 47bits minus one guard page.  The guard
 * page is necessary on Intel CPUs: if a SYSCALL instruction is at
 * the highest possible canonical userspace address, then that
 * syscall will enter the kernel with a non-canonical return
 * address, and SYSRET will explode dangerously.  We avoid this
 * particular problem by preventing anything from being mapped
 * at the maximum canonical address.
874
 */
875
#define TASK_SIZE_MAX	((1UL << 47) - PAGE_SIZE)
876 877 878 879

/* This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
880 881
#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
					0xc0000000 : 0xFFFFe000)
882

883
#define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
884
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
885
#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
886
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
887

888
#define STACK_TOP		TASK_SIZE
889
#define STACK_TOP_MAX		TASK_SIZE_MAX
890

891 892 893 894 895 896 897 898 899 900 901 902
#define INIT_THREAD  { \
	.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \
}

#define INIT_TSS  { \
	.x86_tss.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \
}

/*
 * Return saved PC of a blocked thread.
 * What is this good for? it will be always the scheduler or ret_from_fork.
 */
903
#define thread_saved_pc(t)	(*(unsigned long *)((t)->thread.sp - 8))
904

905
#define task_pt_regs(tsk)	((struct pt_regs *)(tsk)->thread.sp0 - 1)
906
extern unsigned long KSTK_ESP(struct task_struct *task);
907 908 909 910 911 912

/*
 * User space RSP while inside the SYSCALL fast path
 */
DECLARE_PER_CPU(unsigned long, old_rsp);

913 914
#endif /* CONFIG_X86_64 */

I
Ingo Molnar 已提交
915 916 917
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
					       unsigned long new_sp);

918 919
/*
 * This decides where the kernel will search for a free chunk of vm
920 921 922 923
 * space during mmap's.
 */
#define TASK_UNMAPPED_BASE	(PAGE_ALIGN(TASK_SIZE / 3))

924
#define KSTK_EIP(task)		(task_pt_regs(task)->ip)
925

926 927 928 929 930 931 932
/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)	get_tsc_mode((adr))
#define SET_TSC_CTL(val)	set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
/* Register/unregister a process' MPX related resource */
#define MPX_ENABLE_MANAGEMENT(tsk)	mpx_enable_management((tsk))
#define MPX_DISABLE_MANAGEMENT(tsk)	mpx_disable_management((tsk))

#ifdef CONFIG_X86_INTEL_MPX
extern int mpx_enable_management(struct task_struct *tsk);
extern int mpx_disable_management(struct task_struct *tsk);
#else
static inline int mpx_enable_management(struct task_struct *tsk)
{
	return -EINVAL;
}
static inline int mpx_disable_management(struct task_struct *tsk)
{
	return -EINVAL;
}
#endif /* CONFIG_X86_INTEL_MPX */

951
extern u16 amd_get_nb_id(int cpu);
952

953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
{
	uint32_t base, eax, signature[3];

	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);

		if (!memcmp(sig, signature, 12) &&
		    (leaves == 0 || ((eax - base) >= leaves)))
			return base;
	}

	return 0;
}

968 969 970 971
extern unsigned long arch_align_stack(unsigned long sp);
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);

void default_idle(void);
972 973 974 975 976
#ifdef	CONFIG_XEN
bool xen_set_default_idle(void);
#else
#define xen_set_default_idle 0
#endif
977 978

void stop_this_cpu(void *dummy);
979
void df_debug(struct pt_regs *regs, long error_code);
H
H. Peter Anvin 已提交
980
#endif /* _ASM_X86_PROCESSOR_H */