processor.h 23.7 KB
Newer Older
H
H. Peter Anvin 已提交
1 2
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H
3

4 5
#include <asm/processor-flags.h>

6 7 8 9
/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;

10 11 12 13 14 15 16
#include <asm/vm86.h>
#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
#include <asm/sigcontext.h>
#include <asm/current.h>
#include <asm/cpufeature.h>
17
#include <asm/system.h>
18
#include <asm/page.h>
19
#include <asm/pgtable_types.h>
20
#include <asm/percpu.h>
21 22
#include <asm/msr.h>
#include <asm/desc_defs.h>
23
#include <asm/nops.h>
M
Markus Metzger 已提交
24
#include <asm/ds.h>
25

26
#include <linux/personality.h>
27 28
#include <linux/cpumask.h>
#include <linux/cache.h>
29 30
#include <linux/threads.h>
#include <linux/init.h>
31

32 33 34 35 36 37 38
/*
 * Default implementation of macro that returns current
 * instruction pointer ("program counter").
 */
static inline void *current_text_addr(void)
{
	void *pc;
39 40 41

	asm volatile("mov $1f, %0; 1:":"=r" (pc));

42 43 44
	return pc;
}

45
#ifdef CONFIG_X86_VSMP
46 47
# define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
48
#else
49 50
# define ARCH_MIN_TASKALIGN		16
# define ARCH_MIN_MMSTRUCT_ALIGN	0
51 52
#endif

53 54 55 56 57 58 59
/*
 *  CPU type and hardware bug flags. Kept separately for each CPU.
 *  Members of this structure are referenced in head.S, so think twice
 *  before touching them. [mj]
 */

struct cpuinfo_x86 {
60 61 62 63
	__u8			x86;		/* CPU family */
	__u8			x86_vendor;	/* CPU vendor */
	__u8			x86_model;
	__u8			x86_mask;
64
#ifdef CONFIG_X86_32
65 66 67 68 69 70 71 72 73 74
	char			wp_works_ok;	/* It doesn't on 386's */

	/* Problems on some 486Dx4's and old 386's: */
	char			hlt_works_ok;
	char			hard_math;
	char			rfu;
	char			fdiv_bug;
	char			f00f_bug;
	char			coma_bug;
	char			pad0;
75
#else
76
	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
77
	int			x86_tlbsize;
78
#endif
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
	__u8			x86_virt_bits;
	__u8			x86_phys_bits;
	/* CPUID returned core id bits: */
	__u8			x86_coreid_bits;
	/* Max extended CPUID function supported: */
	__u32			extended_cpuid_level;
	/* Maximum supported CPUID level, -1=no CPUID: */
	int			cpuid_level;
	__u32			x86_capability[NCAPINTS];
	char			x86_vendor_id[16];
	char			x86_model_id[64];
	/* in KB - valid for CPUS which support this call: */
	int			x86_cache_size;
	int			x86_cache_alignment;	/* In bytes */
	int			x86_power;
	unsigned long		loops_per_jiffy;
95
#ifdef CONFIG_SMP
96
	/* cpus sharing the last level cache: */
97
	cpumask_var_t		llc_shared_map;
98
#endif
99 100 101
	/* cpuid returned max cores value: */
	u16			 x86_max_cores;
	u16			apicid;
Y
Yinghai Lu 已提交
102
	u16			initial_apicid;
103
	u16			x86_clflush_size;
104
#ifdef CONFIG_SMP
105 106 107 108 109 110 111 112
	/* number of cores as seen by the OS: */
	u16			booted_cores;
	/* Physical processor id: */
	u16			phys_proc_id;
	/* Core id: */
	u16			cpu_core_id;
	/* Index into per_cpu list: */
	u16			cpu_index;
113
#endif
114
	unsigned int		x86_hyper_vendor;
115 116
} __attribute__((__aligned__(SMP_CACHE_BYTES)));

117 118 119 120 121 122 123 124 125 126
#define X86_VENDOR_INTEL	0
#define X86_VENDOR_CYRIX	1
#define X86_VENDOR_AMD		2
#define X86_VENDOR_UMC		3
#define X86_VENDOR_CENTAUR	5
#define X86_VENDOR_TRANSMETA	7
#define X86_VENDOR_NSC		8
#define X86_VENDOR_NUM		9

#define X86_VENDOR_UNKNOWN	0xff
127

128 129 130
#define X86_HYPER_VENDOR_NONE  0
#define X86_HYPER_VENDOR_VMWARE 1

131 132 133
/*
 * capabilities of CPUs
 */
134 135 136 137 138
extern struct cpuinfo_x86	boot_cpu_data;
extern struct cpuinfo_x86	new_cpu_data;

extern struct tss_struct	doublefault_tss;
extern __u32			cleared_cpu_caps[NCAPINTS];
139 140

#ifdef CONFIG_SMP
141
DECLARE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
142
#define cpu_data(cpu)		per_cpu(cpu_info, cpu)
143
#define current_cpu_data	__get_cpu_var(cpu_info)
144 145 146 147 148
#else
#define cpu_data(cpu)		boot_cpu_data
#define current_cpu_data	boot_cpu_data
#endif

149 150
extern const struct seq_operations cpuinfo_op;

G
Glauber Costa 已提交
151 152 153 154 155 156 157 158 159
static inline int hlt_works(int cpu)
{
#ifdef CONFIG_X86_32
	return cpu_data(cpu).hlt_works_ok;
#else
	return 1;
#endif
}

160 161 162
#define cache_line_size()	(boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);
163

164 165
extern struct pt_regs *idle_regs(struct pt_regs *);

166
extern void early_cpu_init(void);
167 168
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
169 170 171 172 173
extern void print_cpu_info(struct cpuinfo_x86 *);
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
extern unsigned short num_cache_leaves;

174
extern void detect_extended_topology(struct cpuinfo_x86 *c);
175 176
extern void detect_ht(struct cpuinfo_x86 *c);

177
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
178
				unsigned int *ecx, unsigned int *edx)
179 180
{
	/* ecx is often an input as well as an output. */
181 182 183 184 185 186
	asm("cpuid"
	    : "=a" (*eax),
	      "=b" (*ebx),
	      "=c" (*ecx),
	      "=d" (*edx)
	    : "0" (*eax), "2" (*ecx));
187 188
}

189 190 191 192
static inline void load_cr3(pgd_t *pgdir)
{
	write_cr3(__pa(pgdir));
}
193

194 195 196
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
	unsigned short		back_link, __blh;
	unsigned long		sp0;
	unsigned short		ss0, __ss0h;
	unsigned long		sp1;
	/* ss1 caches MSR_IA32_SYSENTER_CS: */
	unsigned short		ss1, __ss1h;
	unsigned long		sp2;
	unsigned short		ss2, __ss2h;
	unsigned long		__cr3;
	unsigned long		ip;
	unsigned long		flags;
	unsigned long		ax;
	unsigned long		cx;
	unsigned long		dx;
	unsigned long		bx;
	unsigned long		sp;
	unsigned long		bp;
	unsigned long		si;
	unsigned long		di;
	unsigned short		es, __esh;
	unsigned short		cs, __csh;
	unsigned short		ss, __ssh;
	unsigned short		ds, __dsh;
	unsigned short		fs, __fsh;
	unsigned short		gs, __gsh;
	unsigned short		ldt, __ldth;
	unsigned short		trace;
	unsigned short		io_bitmap_base;

226 227 228
} __attribute__((packed));
#else
struct x86_hw_tss {
229 230 231 232 233 234 235 236 237 238 239
	u32			reserved1;
	u64			sp0;
	u64			sp1;
	u64			sp2;
	u64			reserved2;
	u64			ist[7];
	u32			reserved3;
	u32			reserved4;
	u16			reserved5;
	u16			io_bitmap_base;

240 241 242 243
} __attribute__((packed)) ____cacheline_aligned;
#endif

/*
244
 * IO-bitmap sizes:
245
 */
246 247 248 249 250
#define IO_BITMAP_BITS			65536
#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
#define IO_BITMAP_OFFSET		offsetof(struct tss_struct, io_bitmap)
#define INVALID_IO_BITMAP_OFFSET	0x8000
251 252

struct tss_struct {
253 254 255 256
	/*
	 * The hardware state:
	 */
	struct x86_hw_tss	x86_tss;
257 258 259 260 261 262 263

	/*
	 * The extra 1 is there because the CPU will access an
	 * additional byte beyond the end of the IO permission
	 * bitmap. The extra byte must be all 1 bits, and must
	 * be within the limit.
	 */
264 265
	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];

266
	/*
267
	 * .. and then another 0x100 bytes for the emergency kernel stack:
268
	 */
269 270
	unsigned long		stack[64];

271
} ____cacheline_aligned;
272

273
DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, init_tss);
274

275 276 277
/*
 * Save the original ist values for checking stack pointers during debugging
 */
278
struct orig_ist {
279
	unsigned long		ist[7];
280 281
};

R
Roland McGrath 已提交
282
#define	MXCSR_DEFAULT		0x1f80
283

R
Roland McGrath 已提交
284
struct i387_fsave_struct {
I
Ingo Molnar 已提交
285 286 287 288 289 290 291 292 293
	u32			cwd;	/* FPU Control Word		*/
	u32			swd;	/* FPU Status Word		*/
	u32			twd;	/* FPU Tag Word			*/
	u32			fip;	/* FPU IP Offset		*/
	u32			fcs;	/* FPU IP Selector		*/
	u32			foo;	/* FPU Operand Pointer Offset	*/
	u32			fos;	/* FPU Operand Pointer Selector	*/

	/* 8*10 bytes for each FP-reg = 80 bytes:			*/
294
	u32			st_space[20];
I
Ingo Molnar 已提交
295 296

	/* Software status information [not touched by FSAVE ]:		*/
297
	u32			status;
298 299 300
};

struct i387_fxsave_struct {
I
Ingo Molnar 已提交
301 302 303 304
	u16			cwd; /* Control Word			*/
	u16			swd; /* Status Word			*/
	u16			twd; /* Tag Word			*/
	u16			fop; /* Last Instruction Opcode		*/
R
Roland McGrath 已提交
305 306
	union {
		struct {
I
Ingo Molnar 已提交
307 308
			u64	rip; /* Instruction Pointer		*/
			u64	rdp; /* Data Pointer			*/
R
Roland McGrath 已提交
309 310
		};
		struct {
I
Ingo Molnar 已提交
311 312 313 314
			u32	fip; /* FPU IP Offset			*/
			u32	fcs; /* FPU IP Selector			*/
			u32	foo; /* FPU Operand Offset		*/
			u32	fos; /* FPU Operand Selector		*/
R
Roland McGrath 已提交
315 316
		};
	};
I
Ingo Molnar 已提交
317 318 319 320
	u32			mxcsr;		/* MXCSR Register State */
	u32			mxcsr_mask;	/* MXCSR Mask		*/

	/* 8*16 bytes for each FP-reg = 128 bytes:			*/
321
	u32			st_space[32];
I
Ingo Molnar 已提交
322 323

	/* 16*16 bytes for each XMM-reg = 256 bytes:			*/
324
	u32			xmm_space[64];
I
Ingo Molnar 已提交
325

326 327 328 329 330 331
	u32			padding[12];

	union {
		u32		padding1[12];
		u32		sw_reserved[12];
	};
332

333 334
} __attribute__((aligned(16)));

R
Roland McGrath 已提交
335
struct i387_soft_struct {
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
	u32			cwd;
	u32			swd;
	u32			twd;
	u32			fip;
	u32			fcs;
	u32			foo;
	u32			fos;
	/* 8*10 bytes for each FP-reg = 80 bytes: */
	u32			st_space[20];
	u8			ftop;
	u8			changed;
	u8			lookahead;
	u8			no_update;
	u8			rm;
	u8			alimit;
T
Tejun Heo 已提交
351
	struct math_emu_info	*info;
352
	u32			entry_eip;
R
Roland McGrath 已提交
353 354
};

355 356 357 358 359
struct ymmh_struct {
	/* 16 * 16 bytes for each YMMH-reg = 256 bytes */
	u32 ymmh_space[64];
};

360 361 362 363 364 365 366 367 368
struct xsave_hdr_struct {
	u64 xstate_bv;
	u64 reserved1[2];
	u64 reserved2[5];
} __attribute__((packed));

struct xsave_struct {
	struct i387_fxsave_struct i387;
	struct xsave_hdr_struct xsave_hdr;
369
	struct ymmh_struct ymmh;
370 371 372
	/* new processor state extensions will go here */
} __attribute__ ((packed, aligned (64)));

373
union thread_xstate {
R
Roland McGrath 已提交
374
	struct i387_fsave_struct	fsave;
375
	struct i387_fxsave_struct	fxsave;
376
	struct i387_soft_struct		soft;
377
	struct xsave_struct		xsave;
378 379
};

380
#ifdef CONFIG_X86_64
381
DECLARE_PER_CPU(struct orig_ist, orig_ist);
382

383 384 385 386 387 388 389 390 391 392 393 394 395
union irq_stack_union {
	char irq_stack[IRQ_STACK_SIZE];
	/*
	 * GCC hardcodes the stack canary as %gs:40.  Since the
	 * irq_stack is the object at %gs:0, we reserve the bottom
	 * 48 bytes of the irq stack for the canary.
	 */
	struct {
		char gs_base[40];
		unsigned long stack_canary;
	};
};

396
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union);
397 398
DECLARE_INIT_PER_CPU(irq_stack_union);

399
DECLARE_PER_CPU(char *, irq_stack_ptr);
400 401 402
DECLARE_PER_CPU(unsigned int, irq_count);
extern unsigned long kernel_eflags;
extern asmlinkage void ignore_sysret(void);
403 404 405
#else	/* X86_64 */
#ifdef CONFIG_CC_STACKPROTECTOR
DECLARE_PER_CPU(unsigned long, stack_canary);
406
#endif
407
#endif	/* X86_64 */
408

409
extern unsigned int xstate_size;
410 411
extern void free_thread_xstate(struct task_struct *);
extern struct kmem_cache *task_xstate_cachep;
412 413 414 415
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
extern unsigned short num_cache_leaves;

416
struct thread_struct {
417 418 419 420
	/* Cached TLS descriptors: */
	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
	unsigned long		sp0;
	unsigned long		sp;
421
#ifdef CONFIG_X86_32
422
	unsigned long		sysenter_cs;
423
#else
424 425 426 427 428
	unsigned long		usersp;	/* Copy from PDA */
	unsigned short		es;
	unsigned short		ds;
	unsigned short		fsindex;
	unsigned short		gsindex;
429
#endif
430 431 432 433 434 435 436 437 438 439 440 441 442 443
	unsigned long		ip;
	unsigned long		fs;
	unsigned long		gs;
	/* Hardware debugging registers: */
	unsigned long		debugreg0;
	unsigned long		debugreg1;
	unsigned long		debugreg2;
	unsigned long		debugreg3;
	unsigned long		debugreg6;
	unsigned long		debugreg7;
	/* Fault info: */
	unsigned long		cr2;
	unsigned long		trap_no;
	unsigned long		error_code;
444 445
	/* floating point and extended processor state */
	union thread_xstate	*xstate;
446
#ifdef CONFIG_X86_32
447
	/* Virtual 86 mode info */
448 449
	struct vm86_struct __user *vm86_info;
	unsigned long		screen_bitmap;
450 451 452 453 454
	unsigned long		v86flags;
	unsigned long		v86mask;
	unsigned long		saved_sp0;
	unsigned int		saved_fs;
	unsigned int		saved_gs;
455
#endif
456 457 458 459 460
	/* IO permissions: */
	unsigned long		*io_bitmap_ptr;
	unsigned long		iopl;
	/* Max allowed port in the bitmap, in bytes: */
	unsigned		io_bitmap_max;
461 462
/* MSR_IA32_DEBUGCTLMSR value to switch in if TIF_DEBUGCTLMSR is set.  */
	unsigned long	debugctlmsr;
M
Markus Metzger 已提交
463
	/* Debug Store context; see asm/ds.h */
M
Markus Metzger 已提交
464
	struct ds_context	*ds_ctx;
465 466
};

467 468
static inline unsigned long native_get_debugreg(int regno)
{
469
	unsigned long val = 0;	/* Damn you, gcc! */
470 471 472

	switch (regno) {
	case 0:
473 474
		asm("mov %%db0, %0" :"=r" (val));
		break;
475
	case 1:
476 477
		asm("mov %%db1, %0" :"=r" (val));
		break;
478
	case 2:
479 480
		asm("mov %%db2, %0" :"=r" (val));
		break;
481
	case 3:
482 483
		asm("mov %%db3, %0" :"=r" (val));
		break;
484
	case 6:
485 486
		asm("mov %%db6, %0" :"=r" (val));
		break;
487
	case 7:
488 489
		asm("mov %%db7, %0" :"=r" (val));
		break;
490 491 492 493 494 495 496 497 498 499
	default:
		BUG();
	}
	return val;
}

static inline void native_set_debugreg(int regno, unsigned long value)
{
	switch (regno) {
	case 0:
500
		asm("mov %0, %%db0"	::"r" (value));
501 502
		break;
	case 1:
503
		asm("mov %0, %%db1"	::"r" (value));
504 505
		break;
	case 2:
506
		asm("mov %0, %%db2"	::"r" (value));
507 508
		break;
	case 3:
509
		asm("mov %0, %%db3"	::"r" (value));
510 511
		break;
	case 6:
512
		asm("mov %0, %%db6"	::"r" (value));
513 514
		break;
	case 7:
515
		asm("mov %0, %%db7"	::"r" (value));
516 517 518 519 520 521
		break;
	default:
		BUG();
	}
}

522 523 524 525 526 527 528
/*
 * Set IOPL bits in EFLAGS from given mask
 */
static inline void native_set_iopl_mask(unsigned mask)
{
#ifdef CONFIG_X86_32
	unsigned int reg;
529

530 531 532 533 534 535 536 537
	asm volatile ("pushfl;"
		      "popl %0;"
		      "andl %1, %0;"
		      "orl %2, %0;"
		      "pushl %0;"
		      "popfl"
		      : "=&r" (reg)
		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
538 539 540
#endif
}

541 542
static inline void
native_load_sp0(struct tss_struct *tss, struct thread_struct *thread)
543 544 545
{
	tss->x86_tss.sp0 = thread->sp0;
#ifdef CONFIG_X86_32
546
	/* Only happens when SEP is enabled, no need to test "SEP"arately: */
547 548 549 550 551 552
	if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) {
		tss->x86_tss.ss1 = thread->sysenter_cs;
		wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
	}
#endif
}
553

554 555 556 557 558 559 560
static inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
	asm volatile("swapgs" ::: "memory");
#endif
}

561 562 563
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
564 565
#define __cpuid			native_cpuid
#define paravirt_enabled()	0
566 567 568 569 570 571 572 573 574

/*
 * These special macros can be used to get or set a debugging register
 */
#define get_debugreg(var, register)				\
	(var) = native_get_debugreg(register)
#define set_debugreg(value, register)				\
	native_set_debugreg(register, value)

575 576
static inline void load_sp0(struct tss_struct *tss,
			    struct thread_struct *thread)
577 578 579 580
{
	native_load_sp0(tss, thread);
}

581
#define set_iopl_mask native_set_iopl_mask
582 583 584 585 586 587 588 589
#endif /* CONFIG_PARAVIRT */

/*
 * Save the cr4 feature set we're using (ie
 * Pentium 4MB enable and PPro Global page
 * enable), so that any CPU's that boot up
 * after us can get the correct flags.
 */
590
extern unsigned long		mmu_cr4_features;
591 592 593 594

static inline void set_in_cr4(unsigned long mask)
{
	unsigned cr4;
595

596 597 598 599 600 601 602 603 604
	mmu_cr4_features |= mask;
	cr4 = read_cr4();
	cr4 |= mask;
	write_cr4(cr4);
}

static inline void clear_in_cr4(unsigned long mask)
{
	unsigned cr4;
605

606 607 608 609 610 611
	mmu_cr4_features &= ~mask;
	cr4 = read_cr4();
	cr4 &= ~mask;
	write_cr4(cr4);
}

612
typedef struct {
613
	unsigned long		seg;
614 615 616
} mm_segment_t;


617 618 619 620 621 622 623 624
/*
 * create a kernel thread without removing it from tasklists
 */
extern int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);

/* Free all resources held by a thread. */
extern void release_thread(struct task_struct *);

625
/* Prepare to copy thread state - unlazy all lazy state */
626
extern void prepare_to_copy(struct task_struct *tsk);
627

628
unsigned long get_wchan(struct task_struct *p);
629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661

/*
 * Generic CPUID function
 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
 * resulting in stale register contents being returned.
 */
static inline void cpuid(unsigned int op,
			 unsigned int *eax, unsigned int *ebx,
			 unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = 0;
	__cpuid(eax, ebx, ecx, edx);
}

/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(unsigned int op, int count,
			       unsigned int *eax, unsigned int *ebx,
			       unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = count;
	__cpuid(eax, ebx, ecx, edx);
}

/*
 * CPUID functions returning a single datum
 */
static inline unsigned int cpuid_eax(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
662

663 664
	return eax;
}
665

666 667 668 669 670
static inline unsigned int cpuid_ebx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
671

672 673
	return ebx;
}
674

675 676 677 678 679
static inline unsigned int cpuid_ecx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
680

681 682
	return ecx;
}
683

684 685 686 687 688
static inline unsigned int cpuid_edx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
689

690 691 692
	return edx;
}

693 694 695
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
static inline void rep_nop(void)
{
696
	asm volatile("rep; nop" ::: "memory");
697 698
}

699 700 701 702 703 704
static inline void cpu_relax(void)
{
	rep_nop();
}

/* Stop speculative execution: */
705 706 707
static inline void sync_core(void)
{
	int tmp;
708

709
	asm volatile("cpuid" : "=a" (tmp) : "0" (1)
710
		     : "ebx", "ecx", "edx", "memory");
711 712
}

713 714
static inline void __monitor(const void *eax, unsigned long ecx,
			     unsigned long edx)
715
{
716
	/* "monitor %eax, %ecx, %edx;" */
717 718
	asm volatile(".byte 0x0f, 0x01, 0xc8;"
		     :: "a" (eax), "c" (ecx), "d"(edx));
719 720 721 722
}

static inline void __mwait(unsigned long eax, unsigned long ecx)
{
723
	/* "mwait %eax, %ecx;" */
724 725
	asm volatile(".byte 0x0f, 0x01, 0xc9;"
		     :: "a" (eax), "c" (ecx));
726 727 728 729
}

static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
{
730
	trace_hardirqs_on();
731
	/* "mwait %eax, %ecx;" */
732 733
	asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
		     :: "a" (eax), "c" (ecx));
734 735 736 737 738
}

extern void mwait_idle_with_hints(unsigned long eax, unsigned long ecx);

extern void select_idle_routine(const struct cpuinfo_x86 *c);
739
extern void init_c1e_mask(void);
740

741
extern unsigned long		boot_option_idle_override;
Z
Zhao Yakui 已提交
742
extern unsigned long		idle_halt;
743
extern unsigned long		idle_nomwait;
744

745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
/*
 * on systems with caches, caches must be flashed as the absolute
 * last instruction before going into a suspended halt.  Otherwise,
 * dirty data can linger in the cache and become stale on resume,
 * leading to strange errors.
 *
 * perform a variety of operations to guarantee that the compiler
 * will not reorder instructions.  wbinvd itself is serializing
 * so the processor will not reorder.
 *
 * Systems without cache can just go into halt.
 */
static inline void wbinvd_halt(void)
{
	mb();
	/* check for clflush to determine if wbinvd is legal */
	if (cpu_has_clflush)
		asm volatile("cli; wbinvd; 1: hlt; jmp 1b" : : : "memory");
	else
		while (1)
			halt();
}

768 769 770 771
extern void enable_sep_cpu(void);
extern int sysenter_setup(void);

/* Defined in head.S */
772
extern struct desc_ptr		early_gdt_descr;
773 774

extern void cpu_set_gdt(int);
775
extern void switch_to_new_gdt(int);
776
extern void load_percpu_segment(int);
777 778
extern void cpu_init(void);

779 780 781 782 783 784 785 786 787 788 789 790 791
static inline unsigned long get_debugctlmsr(void)
{
    unsigned long debugctlmsr = 0;

#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return 0;
#endif
	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);

    return debugctlmsr;
}

792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
static inline unsigned long get_debugctlmsr_on_cpu(int cpu)
{
	u64 debugctlmsr = 0;
	u32 val1, val2;

#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return 0;
#endif
	rdmsr_on_cpu(cpu, MSR_IA32_DEBUGCTLMSR, &val1, &val2);
	debugctlmsr = val1 | ((u64)val2 << 32);

	return debugctlmsr;
}

807 808 809 810 811 812 813 814 815
static inline void update_debugctlmsr(unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return;
#endif
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
}

816 817 818 819 820 821 822 823 824 825 826 827
static inline void update_debugctlmsr_on_cpu(int cpu,
					     unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return;
#endif
	wrmsr_on_cpu(cpu, MSR_IA32_DEBUGCTLMSR,
		     (u32)((u64)debugctlmsr),
		     (u32)((u64)debugctlmsr >> 32));
}

828 829 830 831 832 833 834
/*
 * from system description table in BIOS. Mostly for MCA use, but
 * others may find it useful:
 */
extern unsigned int		machine_id;
extern unsigned int		machine_submodel_id;
extern unsigned int		BIOS_revision;
835

836 837
/* Boot loader type from the setup header: */
extern int			bootloader_type;
838

839
extern char			ignore_fpu_irq;
840 841 842 843 844

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#define ARCH_HAS_SPINLOCK_PREFETCH

845
#ifdef CONFIG_X86_32
846 847
# define BASE_PREFETCH		ASM_NOP4
# define ARCH_HAS_PREFETCH
848
#else
849
# define BASE_PREFETCH		"prefetcht0 (%1)"
850 851
#endif

852 853 854 855 856 857
/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
858 859 860 861 862 863 864 865
static inline void prefetch(const void *x)
{
	alternative_input(BASE_PREFETCH,
			  "prefetchnta (%1)",
			  X86_FEATURE_XMM,
			  "r" (x));
}

866 867 868 869 870
/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
871 872 873 874 875 876 877 878
static inline void prefetchw(const void *x)
{
	alternative_input(BASE_PREFETCH,
			  "prefetchw (%1)",
			  X86_FEATURE_3DNOW,
			  "r" (x));
}

879 880 881 882 883
static inline void spin_lock_prefetch(const void *x)
{
	prefetchw(x);
}

884 885 886 887
#ifdef CONFIG_X86_32
/*
 * User space process size: 3GB (default).
 */
888
#define TASK_SIZE		PAGE_OFFSET
889
#define TASK_SIZE_MAX		TASK_SIZE
890 891 892 893 894 895 896 897 898
#define STACK_TOP		TASK_SIZE
#define STACK_TOP_MAX		STACK_TOP

#define INIT_THREAD  {							  \
	.sp0			= sizeof(init_stack) + (long)&init_stack, \
	.vm86_info		= NULL,					  \
	.sysenter_cs		= __KERNEL_CS,				  \
	.io_bitmap_ptr		= NULL,					  \
	.fs			= __KERNEL_PERCPU,			  \
899 900 901 902 903 904 905 906
}

/*
 * Note that the .io_bitmap member must be extra-big. This is because
 * the CPU will access an additional byte beyond the end of the IO
 * permission bitmap. The extra byte must be all 1 bits, and must
 * be within the limit.
 */
907 908
#define INIT_TSS  {							  \
	.x86_tss = {							  \
909
		.sp0		= sizeof(init_stack) + (long)&init_stack, \
910 911 912 913 914
		.ss0		= __KERNEL_DS,				  \
		.ss1		= __KERNEL_CS,				  \
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,		  \
	 },								  \
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },	  \
915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
}

extern unsigned long thread_saved_pc(struct task_struct *tsk);

#define THREAD_SIZE_LONGS      (THREAD_SIZE/sizeof(unsigned long))
#define KSTK_TOP(info)                                                 \
({                                                                     \
       unsigned long *__ptr = (unsigned long *)(info);                 \
       (unsigned long)(&__ptr[THREAD_SIZE_LONGS]);                     \
})

/*
 * The below -8 is to reserve 8 bytes on top of the ring0 stack.
 * This is necessary to guarantee that the entire "struct pt_regs"
 * is accessable even if the CPU haven't stored the SS/ESP registers
 * on the stack (interrupt gate does not save these registers
 * when switching to the same priv ring).
 * Therefore beware: accessing the ss/esp fields of the
 * "struct pt_regs" is possible, but they may contain the
 * completely wrong values.
 */
#define task_pt_regs(task)                                             \
({                                                                     \
       struct pt_regs *__regs__;                                       \
       __regs__ = (struct pt_regs *)(KSTK_TOP(task_stack_page(task))-8); \
       __regs__ - 1;                                                   \
})

943
#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
944 945 946 947 948

#else
/*
 * User space process size. 47bits minus one guard page.
 */
949
#define TASK_SIZE_MAX	((1UL << 47) - PAGE_SIZE)
950 951 952 953

/* This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
954 955
#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
					0xc0000000 : 0xFFFFe000)
956

957
#define TASK_SIZE		(test_thread_flag(TIF_IA32) ? \
958
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
959
#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_IA32)) ? \
960
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
961

962
#define STACK_TOP		TASK_SIZE
963
#define STACK_TOP_MAX		TASK_SIZE_MAX
964

965 966 967 968 969 970 971 972 973 974 975 976
#define INIT_THREAD  { \
	.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \
}

#define INIT_TSS  { \
	.x86_tss.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \
}

/*
 * Return saved PC of a blocked thread.
 * What is this good for? it will be always the scheduler or ret_from_fork.
 */
977
#define thread_saved_pc(t)	(*(unsigned long *)((t)->thread.sp - 8))
978

979 980
#define task_pt_regs(tsk)	((struct pt_regs *)(tsk)->thread.sp0 - 1)
#define KSTK_ESP(tsk)		-1 /* sorry. doesn't work for syscall. */
981 982
#endif /* CONFIG_X86_64 */

I
Ingo Molnar 已提交
983 984 985
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
					       unsigned long new_sp);

986 987
/*
 * This decides where the kernel will search for a free chunk of vm
988 989 990 991
 * space during mmap's.
 */
#define TASK_UNMAPPED_BASE	(PAGE_ALIGN(TASK_SIZE / 3))

992
#define KSTK_EIP(task)		(task_pt_regs(task)->ip)
993

994 995 996 997 998 999 1000
/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)	get_tsc_mode((adr))
#define SET_TSC_CTL(val)	set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

H
H. Peter Anvin 已提交
1001
#endif /* _ASM_X86_PROCESSOR_H */