processor.h 20.6 KB
Newer Older
H
H. Peter Anvin 已提交
1 2
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H
3

4 5
#include <asm/processor-flags.h>

6 7 8
/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;
9
struct vm86;
10

11 12 13 14 15 16 17
#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
#include <asm/sigcontext.h>
#include <asm/current.h>
#include <asm/cpufeature.h>
#include <asm/page.h>
18
#include <asm/pgtable_types.h>
19
#include <asm/percpu.h>
20 21
#include <asm/msr.h>
#include <asm/desc_defs.h>
22
#include <asm/nops.h>
23
#include <asm/special_insns.h>
24
#include <asm/fpu/types.h>
25

26
#include <linux/personality.h>
27 28
#include <linux/cpumask.h>
#include <linux/cache.h>
29
#include <linux/threads.h>
30
#include <linux/math64.h>
31
#include <linux/err.h>
32 33 34 35 36 37 38 39 40
#include <linux/irqflags.h>

/*
 * We handle most unaligned accesses in hardware.  On the other hand
 * unaligned DMA can be quite expensive on some Nehalem processors.
 *
 * Based on this we disable the IP header alignment in network drivers.
 */
#define NET_IP_ALIGN	0
41

42
#define HBP_NUM 4
43 44 45 46 47 48 49
/*
 * Default implementation of macro that returns current
 * instruction pointer ("program counter").
 */
static inline void *current_text_addr(void)
{
	void *pc;
50 51 52

	asm volatile("mov $1f, %0; 1:":"=r" (pc));

53 54 55
	return pc;
}

56 57 58 59 60
/*
 * These alignment constraints are for performance in the vSMP case,
 * but in the task_struct case we must also meet hardware imposed
 * alignment requirements of the FPU state:
 */
61
#ifdef CONFIG_X86_VSMP
62 63
# define ARCH_MIN_TASKALIGN		(1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN	(1 << INTERNODE_CACHE_SHIFT)
64
#else
65
# define ARCH_MIN_TASKALIGN		__alignof__(union fpregs_state)
66
# define ARCH_MIN_MMSTRUCT_ALIGN	0
67 68
#endif

69 70 71 72 73 74 75 76 77 78 79
enum tlb_infos {
	ENTRIES,
	NR_INFO
};

extern u16 __read_mostly tlb_lli_4k[NR_INFO];
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
80
extern u16 __read_mostly tlb_lld_1g[NR_INFO];
81

82 83 84 85 86 87 88
/*
 *  CPU type and hardware bug flags. Kept separately for each CPU.
 *  Members of this structure are referenced in head.S, so think twice
 *  before touching them. [mj]
 */

struct cpuinfo_x86 {
89 90 91 92
	__u8			x86;		/* CPU family */
	__u8			x86_vendor;	/* CPU vendor */
	__u8			x86_model;
	__u8			x86_mask;
93
#ifdef CONFIG_X86_32
94 95 96 97 98
	char			wp_works_ok;	/* It doesn't on 386's */

	/* Problems on some 486Dx4's and old 386's: */
	char			rfu;
	char			pad0;
99
	char			pad1;
100
#else
101
	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
102
	int			x86_tlbsize;
103
#endif
104 105 106 107 108 109 110 111
	__u8			x86_virt_bits;
	__u8			x86_phys_bits;
	/* CPUID returned core id bits: */
	__u8			x86_coreid_bits;
	/* Max extended CPUID function supported: */
	__u32			extended_cpuid_level;
	/* Maximum supported CPUID level, -1=no CPUID: */
	int			cpuid_level;
112
	__u32			x86_capability[NCAPINTS + NBUGINTS];
113 114 115 116 117
	char			x86_vendor_id[16];
	char			x86_model_id[64];
	/* in KB - valid for CPUS which support this call: */
	int			x86_cache_size;
	int			x86_cache_alignment;	/* In bytes */
118 119 120
	/* Cache QoS architectural values: */
	int			x86_cache_max_rmid;	/* max index */
	int			x86_cache_occ_scale;	/* scale to bytes */
121 122 123 124 125
	int			x86_power;
	unsigned long		loops_per_jiffy;
	/* cpuid returned max cores value: */
	u16			 x86_max_cores;
	u16			apicid;
Y
Yinghai Lu 已提交
126
	u16			initial_apicid;
127 128 129 130 131 132 133
	u16			x86_clflush_size;
	/* number of cores as seen by the OS: */
	u16			booted_cores;
	/* Physical processor id: */
	u16			phys_proc_id;
	/* Core id: */
	u16			cpu_core_id;
134 135
	/* Compute unit id */
	u8			compute_unit_id;
136 137
	/* Index into per_cpu list: */
	u16			cpu_index;
138
	u32			microcode;
139
};
140

141 142 143 144 145 146 147 148 149 150
#define X86_VENDOR_INTEL	0
#define X86_VENDOR_CYRIX	1
#define X86_VENDOR_AMD		2
#define X86_VENDOR_UMC		3
#define X86_VENDOR_CENTAUR	5
#define X86_VENDOR_TRANSMETA	7
#define X86_VENDOR_NSC		8
#define X86_VENDOR_NUM		9

#define X86_VENDOR_UNKNOWN	0xff
151

152 153 154
/*
 * capabilities of CPUs
 */
155 156 157 158
extern struct cpuinfo_x86	boot_cpu_data;
extern struct cpuinfo_x86	new_cpu_data;

extern struct tss_struct	doublefault_tss;
159 160
extern __u32			cpu_caps_cleared[NCAPINTS];
extern __u32			cpu_caps_set[NCAPINTS];
161 162

#ifdef CONFIG_SMP
163
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
164 165
#define cpu_data(cpu)		per_cpu(cpu_info, cpu)
#else
166
#define cpu_info		boot_cpu_data
167 168 169
#define cpu_data(cpu)		boot_cpu_data
#endif

170 171
extern const struct seq_operations cpuinfo_op;

172 173 174
#define cache_line_size()	(boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);
175

176
extern void early_cpu_init(void);
177 178
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
179
extern void print_cpu_info(struct cpuinfo_x86 *);
180
void print_cpu_msr(struct cpuinfo_x86 *);
181 182
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c);
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c);
183
extern void init_amd_cacheinfo(struct cpuinfo_x86 *c);
184

185
extern void detect_extended_topology(struct cpuinfo_x86 *c);
186 187
extern void detect_ht(struct cpuinfo_x86 *c);

188 189 190 191 192 193 194 195
#ifdef CONFIG_X86_32
extern int have_cpuid_p(void);
#else
static inline int have_cpuid_p(void)
{
	return 1;
}
#endif
196
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
197
				unsigned int *ecx, unsigned int *edx)
198 199
{
	/* ecx is often an input as well as an output. */
200
	asm volatile("cpuid"
201 202 203 204
	    : "=a" (*eax),
	      "=b" (*ebx),
	      "=c" (*ecx),
	      "=d" (*edx)
205 206
	    : "0" (*eax), "2" (*ecx)
	    : "memory");
207 208
}

209 210 211 212
static inline void load_cr3(pgd_t *pgdir)
{
	write_cr3(__pa(pgdir));
}
213

214 215 216
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
217 218 219
	unsigned short		back_link, __blh;
	unsigned long		sp0;
	unsigned short		ss0, __ss0h;
220
	unsigned long		sp1;
221 222

	/*
223 224 225 226 227 228
	 * We don't use ring 1, so ss1 is a convenient scratch space in
	 * the same cacheline as sp0.  We use ss1 to cache the value in
	 * MSR_IA32_SYSENTER_CS.  When we context switch
	 * MSR_IA32_SYSENTER_CS, we first check if the new value being
	 * written matches ss1, and, if it's not, then we wrmsr the new
	 * value and update ss1.
229
	 *
230 231 232 233
	 * The only reason we context switch MSR_IA32_SYSENTER_CS is
	 * that we set it to zero in vm86 tasks to avoid corrupting the
	 * stack if we were to go through the sysenter path from vm86
	 * mode.
234 235 236 237
	 */
	unsigned short		ss1;	/* MSR_IA32_SYSENTER_CS */

	unsigned short		__ss1h;
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
	unsigned long		sp2;
	unsigned short		ss2, __ss2h;
	unsigned long		__cr3;
	unsigned long		ip;
	unsigned long		flags;
	unsigned long		ax;
	unsigned long		cx;
	unsigned long		dx;
	unsigned long		bx;
	unsigned long		sp;
	unsigned long		bp;
	unsigned long		si;
	unsigned long		di;
	unsigned short		es, __esh;
	unsigned short		cs, __csh;
	unsigned short		ss, __ssh;
	unsigned short		ds, __dsh;
	unsigned short		fs, __fsh;
	unsigned short		gs, __gsh;
	unsigned short		ldt, __ldth;
	unsigned short		trace;
	unsigned short		io_bitmap_base;

261 262 263
} __attribute__((packed));
#else
struct x86_hw_tss {
264 265 266 267 268 269 270 271 272 273 274
	u32			reserved1;
	u64			sp0;
	u64			sp1;
	u64			sp2;
	u64			reserved2;
	u64			ist[7];
	u32			reserved3;
	u32			reserved4;
	u16			reserved5;
	u16			io_bitmap_base;

275 276 277 278
} __attribute__((packed)) ____cacheline_aligned;
#endif

/*
279
 * IO-bitmap sizes:
280
 */
281 282 283 284 285
#define IO_BITMAP_BITS			65536
#define IO_BITMAP_BYTES			(IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS			(IO_BITMAP_BYTES/sizeof(long))
#define IO_BITMAP_OFFSET		offsetof(struct tss_struct, io_bitmap)
#define INVALID_IO_BITMAP_OFFSET	0x8000
286 287

struct tss_struct {
288 289 290 291
	/*
	 * The hardware state:
	 */
	struct x86_hw_tss	x86_tss;
292 293 294 295 296 297 298

	/*
	 * The extra 1 is there because the CPU will access an
	 * additional byte beyond the end of the IO permission
	 * bitmap. The extra byte must be all 1 bits, and must
	 * be within the limit.
	 */
299 300
	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];

301
	/*
302
	 * Space for the temporary SYSENTER stack:
303
	 */
304
	unsigned long		SYSENTER_stack[64];
305

306
} ____cacheline_aligned;
307

308
DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss);
309

310 311 312 313
#ifdef CONFIG_X86_32
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
#endif

314 315 316
/*
 * Save the original ist values for checking stack pointers during debugging
 */
317
struct orig_ist {
318
	unsigned long		ist[7];
319 320
};

321
#ifdef CONFIG_X86_64
322
DECLARE_PER_CPU(struct orig_ist, orig_ist);
323

324 325 326 327 328 329 330 331 332 333 334 335 336
union irq_stack_union {
	char irq_stack[IRQ_STACK_SIZE];
	/*
	 * GCC hardcodes the stack canary as %gs:40.  Since the
	 * irq_stack is the object at %gs:0, we reserve the bottom
	 * 48 bytes of the irq stack for the canary.
	 */
	struct {
		char gs_base[40];
		unsigned long stack_canary;
	};
};

337
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
338 339
DECLARE_INIT_PER_CPU(irq_stack_union);

340
DECLARE_PER_CPU(char *, irq_stack_ptr);
341 342
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
343 344
#else	/* X86_64 */
#ifdef CONFIG_CC_STACKPROTECTOR
345 346 347 348 349 350 351 352 353 354
/*
 * Make sure stack canary segment base is cached-aligned:
 *   "For Intel Atom processors, avoid non zero segment base address
 *    that is not aligned to cache line boundary at all cost."
 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
 */
struct stack_canary {
	char __pad[20];		/* canary at %gs:20 */
	unsigned long canary;
};
355
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
356
#endif
357 358 359 360 361 362 363 364 365
/*
 * per-CPU IRQ handling stacks
 */
struct irq_stack {
	u32                     stack[THREAD_SIZE/sizeof(u32)];
} __aligned(THREAD_SIZE);

DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
366
#endif	/* X86_64 */
367

368
extern unsigned int xstate_size;
369

370 371
struct perf_event;

372
struct thread_struct {
373 374 375 376
	/* Cached TLS descriptors: */
	struct desc_struct	tls_array[GDT_ENTRY_TLS_ENTRIES];
	unsigned long		sp0;
	unsigned long		sp;
377
#ifdef CONFIG_X86_32
378
	unsigned long		sysenter_cs;
379
#else
380 381 382 383
	unsigned short		es;
	unsigned short		ds;
	unsigned short		fsindex;
	unsigned short		gsindex;
384
#endif
385
#ifdef CONFIG_X86_32
386
	unsigned long		ip;
387
#endif
388
#ifdef CONFIG_X86_64
389
	unsigned long		fs;
390
#endif
391
	unsigned long		gs;
392

393 394 395 396
	/* Save middle states of ptrace breakpoints */
	struct perf_event	*ptrace_bps[HBP_NUM];
	/* Debug status used for traps, single steps, etc... */
	unsigned long           debugreg6;
397 398
	/* Keep track of the exact dr7 value set by the user */
	unsigned long           ptrace_dr7;
399 400
	/* Fault info: */
	unsigned long		cr2;
401
	unsigned long		trap_nr;
402
	unsigned long		error_code;
403
#ifdef CONFIG_VM86
404
	/* Virtual 86 mode info */
405
	struct vm86		*vm86;
406
#endif
407 408 409 410 411
	/* IO permissions: */
	unsigned long		*io_bitmap_ptr;
	unsigned long		iopl;
	/* Max allowed port in the bitmap, in bytes: */
	unsigned		io_bitmap_max;
412 413 414 415 416 417 418

	/* Floating point and extended processor state */
	struct fpu		fpu;
	/*
	 * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
	 * the end.
	 */
419 420
};

421 422 423 424 425 426 427
/*
 * Set IOPL bits in EFLAGS from given mask
 */
static inline void native_set_iopl_mask(unsigned mask)
{
#ifdef CONFIG_X86_32
	unsigned int reg;
428

429 430 431 432 433 434 435 436
	asm volatile ("pushfl;"
		      "popl %0;"
		      "andl %1, %0;"
		      "orl %2, %0;"
		      "pushl %0;"
		      "popfl"
		      : "=&r" (reg)
		      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
437 438 439
#endif
}

440 441
static inline void
native_load_sp0(struct tss_struct *tss, struct thread_struct *thread)
442 443 444
{
	tss->x86_tss.sp0 = thread->sp0;
#ifdef CONFIG_X86_32
445
	/* Only happens when SEP is enabled, no need to test "SEP"arately: */
446 447 448 449 450 451
	if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) {
		tss->x86_tss.ss1 = thread->sysenter_cs;
		wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0);
	}
#endif
}
452

453 454 455 456 457 458 459
static inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
	asm volatile("swapgs" ::: "memory");
#endif
}

460
static inline unsigned long current_top_of_stack(void)
461
{
462
#ifdef CONFIG_X86_64
463
	return this_cpu_read_stable(cpu_tss.x86_tss.sp0);
464 465 466 467
#else
	/* sp0 on x86_32 is special in and around vm86 mode. */
	return this_cpu_read_stable(cpu_current_top_of_stack);
#endif
468 469
}

470 471 472
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
473 474
#define __cpuid			native_cpuid
#define paravirt_enabled()	0
475

476 477
static inline void load_sp0(struct tss_struct *tss,
			    struct thread_struct *thread)
478 479 480 481
{
	native_load_sp0(tss, thread);
}

482
#define set_iopl_mask native_set_iopl_mask
483 484
#endif /* CONFIG_PARAVIRT */

485
typedef struct {
486
	unsigned long		seg;
487 488 489
} mm_segment_t;


490 491 492 493
/* Free all resources held by a thread. */
extern void release_thread(struct task_struct *);

unsigned long get_wchan(struct task_struct *p);
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526

/*
 * Generic CPUID function
 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
 * resulting in stale register contents being returned.
 */
static inline void cpuid(unsigned int op,
			 unsigned int *eax, unsigned int *ebx,
			 unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = 0;
	__cpuid(eax, ebx, ecx, edx);
}

/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(unsigned int op, int count,
			       unsigned int *eax, unsigned int *ebx,
			       unsigned int *ecx, unsigned int *edx)
{
	*eax = op;
	*ecx = count;
	__cpuid(eax, ebx, ecx, edx);
}

/*
 * CPUID functions returning a single datum
 */
static inline unsigned int cpuid_eax(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
527

528 529
	return eax;
}
530

531 532 533 534 535
static inline unsigned int cpuid_ebx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
536

537 538
	return ebx;
}
539

540 541 542 543 544
static inline unsigned int cpuid_ecx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
545

546 547
	return ecx;
}
548

549 550 551 552 553
static inline unsigned int cpuid_edx(unsigned int op)
{
	unsigned int eax, ebx, ecx, edx;

	cpuid(op, &eax, &ebx, &ecx, &edx);
554

555 556 557
	return edx;
}

558 559 560
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
static inline void rep_nop(void)
{
561
	asm volatile("rep; nop" ::: "memory");
562 563
}

564 565 566 567 568
static inline void cpu_relax(void)
{
	rep_nop();
}

569 570
#define cpu_relax_lowlatency() cpu_relax()

571
/* Stop speculative execution and prefetching of modified code. */
572 573 574
static inline void sync_core(void)
{
	int tmp;
575

576
#ifdef CONFIG_M486
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
	/*
	 * Do a CPUID if available, otherwise do a jump.  The jump
	 * can conveniently enough be the jump around CPUID.
	 */
	asm volatile("cmpl %2,%1\n\t"
		     "jl 1f\n\t"
		     "cpuid\n"
		     "1:"
		     : "=a" (tmp)
		     : "rm" (boot_cpu_data.cpuid_level), "ri" (0), "0" (1)
		     : "ebx", "ecx", "edx", "memory");
#else
	/*
	 * CPUID is a barrier to speculative execution.
	 * Prefetched instructions are automatically
	 * invalidated when modified.
	 */
	asm volatile("cpuid"
		     : "=a" (tmp)
		     : "0" (1)
		     : "ebx", "ecx", "edx", "memory");
598
#endif
599 600 601
}

extern void select_idle_routine(const struct cpuinfo_x86 *c);
602
extern void init_amd_e400_c1e_mask(void);
603

604
extern unsigned long		boot_option_idle_override;
605
extern bool			amd_e400_c1e_detected;
606

607
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
608
			 IDLE_POLL};
609

610 611 612
extern void enable_sep_cpu(void);
extern int sysenter_setup(void);

613
extern void early_trap_init(void);
614
void early_trap_pf_init(void);
615

616
/* Defined in head.S */
617
extern struct desc_ptr		early_gdt_descr;
618 619

extern void cpu_set_gdt(int);
620
extern void switch_to_new_gdt(int);
621
extern void load_percpu_segment(int);
622 623
extern void cpu_init(void);

624 625
static inline unsigned long get_debugctlmsr(void)
{
P
Peter Zijlstra 已提交
626
	unsigned long debugctlmsr = 0;
627 628 629 630 631 632 633

#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return 0;
#endif
	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);

P
Peter Zijlstra 已提交
634
	return debugctlmsr;
635 636
}

637 638 639 640 641 642 643 644 645
static inline void update_debugctlmsr(unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
	if (boot_cpu_data.x86 < 6)
		return;
#endif
	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
}

646 647
extern void set_task_blockstep(struct task_struct *task, bool on);

648 649
/* Boot loader type from the setup header: */
extern int			bootloader_type;
650
extern int			bootloader_version;
651

652
extern char			ignore_fpu_irq;
653 654 655 656 657

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#define ARCH_HAS_SPINLOCK_PREFETCH

658
#ifdef CONFIG_X86_32
659
# define BASE_PREFETCH		""
660
# define ARCH_HAS_PREFETCH
661
#else
662
# define BASE_PREFETCH		"prefetcht0 %P1"
663 664
#endif

665 666 667 668 669 670
/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
671 672
static inline void prefetch(const void *x)
{
673
	alternative_input(BASE_PREFETCH, "prefetchnta %P1",
674
			  X86_FEATURE_XMM,
675
			  "m" (*(const char *)x));
676 677
}

678 679 680 681 682
/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
683 684
static inline void prefetchw(const void *x)
{
685 686 687
	alternative_input(BASE_PREFETCH, "prefetchw %P1",
			  X86_FEATURE_3DNOWPREFETCH,
			  "m" (*(const char *)x));
688 689
}

690 691 692 693 694
static inline void spin_lock_prefetch(const void *x)
{
	prefetchw(x);
}

695 696 697
#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
			   TOP_OF_KERNEL_STACK_PADDING)

698 699 700 701
#ifdef CONFIG_X86_32
/*
 * User space process size: 3GB (default).
 */
702
#define TASK_SIZE		PAGE_OFFSET
703
#define TASK_SIZE_MAX		TASK_SIZE
704 705 706 707
#define STACK_TOP		TASK_SIZE
#define STACK_TOP_MAX		STACK_TOP

#define INIT_THREAD  {							  \
708
	.sp0			= TOP_OF_INIT_STACK,			  \
709 710
	.sysenter_cs		= __KERNEL_CS,				  \
	.io_bitmap_ptr		= NULL,					  \
711 712 713 714 715
}

extern unsigned long thread_saved_pc(struct task_struct *tsk);

/*
716
 * TOP_OF_KERNEL_STACK_PADDING reserves 8 bytes on top of the ring0 stack.
717
 * This is necessary to guarantee that the entire "struct pt_regs"
718
 * is accessible even if the CPU haven't stored the SS/ESP registers
719 720 721 722 723 724
 * on the stack (interrupt gate does not save these registers
 * when switching to the same priv ring).
 * Therefore beware: accessing the ss/esp fields of the
 * "struct pt_regs" is possible, but they may contain the
 * completely wrong values.
 */
725 726 727 728 729
#define task_pt_regs(task) \
({									\
	unsigned long __ptr = (unsigned long)task_stack_page(task);	\
	__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;		\
	((struct pt_regs *)__ptr) - 1;					\
730 731
})

732
#define KSTK_ESP(task)		(task_pt_regs(task)->sp)
733 734 735

#else
/*
736 737 738 739 740 741 742
 * User space process size. 47bits minus one guard page.  The guard
 * page is necessary on Intel CPUs: if a SYSCALL instruction is at
 * the highest possible canonical userspace address, then that
 * syscall will enter the kernel with a non-canonical return
 * address, and SYSRET will explode dangerously.  We avoid this
 * particular problem by preventing anything from being mapped
 * at the maximum canonical address.
743
 */
744
#define TASK_SIZE_MAX	((1UL << 47) - PAGE_SIZE)
745 746 747 748

/* This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
749 750
#define IA32_PAGE_OFFSET	((current->personality & ADDR_LIMIT_3GB) ? \
					0xc0000000 : 0xFFFFe000)
751

752
#define TASK_SIZE		(test_thread_flag(TIF_ADDR32) ? \
753
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
754
#define TASK_SIZE_OF(child)	((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
755
					IA32_PAGE_OFFSET : TASK_SIZE_MAX)
756

757
#define STACK_TOP		TASK_SIZE
758
#define STACK_TOP_MAX		TASK_SIZE_MAX
759

760
#define INIT_THREAD  { \
761
	.sp0 = TOP_OF_INIT_STACK \
762 763 764 765 766 767
}

/*
 * Return saved PC of a blocked thread.
 * What is this good for? it will be always the scheduler or ret_from_fork.
 */
768
#define thread_saved_pc(t)	(*(unsigned long *)((t)->thread.sp - 8))
769

770
#define task_pt_regs(tsk)	((struct pt_regs *)(tsk)->thread.sp0 - 1)
771
extern unsigned long KSTK_ESP(struct task_struct *task);
772

773 774
#endif /* CONFIG_X86_64 */

I
Ingo Molnar 已提交
775 776 777
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
					       unsigned long new_sp);

778 779
/*
 * This decides where the kernel will search for a free chunk of vm
780 781 782 783
 * space during mmap's.
 */
#define TASK_UNMAPPED_BASE	(PAGE_ALIGN(TASK_SIZE / 3))

784
#define KSTK_EIP(task)		(task_pt_regs(task)->ip)
785

786 787 788 789 790 791 792
/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)	get_tsc_mode((adr))
#define SET_TSC_CTL(val)	set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

793
/* Register/unregister a process' MPX related resource */
794 795
#define MPX_ENABLE_MANAGEMENT()	mpx_enable_management()
#define MPX_DISABLE_MANAGEMENT()	mpx_disable_management()
796 797

#ifdef CONFIG_X86_INTEL_MPX
798 799
extern int mpx_enable_management(void);
extern int mpx_disable_management(void);
800
#else
801
static inline int mpx_enable_management(void)
802 803 804
{
	return -EINVAL;
}
805
static inline int mpx_disable_management(void)
806 807 808 809 810
{
	return -EINVAL;
}
#endif /* CONFIG_X86_INTEL_MPX */

811
extern u16 amd_get_nb_id(int cpu);
812
extern u32 amd_get_nodes_per_socket(void);
813

814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
{
	uint32_t base, eax, signature[3];

	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
		cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);

		if (!memcmp(sig, signature, 12) &&
		    (leaves == 0 || ((eax - base) >= leaves)))
			return base;
	}

	return 0;
}

829 830 831 832
extern unsigned long arch_align_stack(unsigned long sp);
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);

void default_idle(void);
833 834 835 836 837
#ifdef	CONFIG_XEN
bool xen_set_default_idle(void);
#else
#define xen_set_default_idle 0
#endif
838 839

void stop_this_cpu(void *dummy);
840
void df_debug(struct pt_regs *regs, long error_code);
H
H. Peter Anvin 已提交
841
#endif /* _ASM_X86_PROCESSOR_H */