process_64.c 16.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Pentium III FXSR, SSE support
 *	Gareth Hughes <gareth@valinux.com>, May 2000
6
 *
L
Linus Torvalds 已提交
7 8
 *  X86-64 port
 *	Andi Kleen.
A
Ashok Raj 已提交
9 10
 *
 *	CPU hotplug support - ashok.raj@intel.com
L
Linus Torvalds 已提交
11 12 13 14 15 16
 */

/*
 * This file handles the architecture-dependent parts of process handling..
 */

17
#include <linux/stackprotector.h>
A
Ashok Raj 已提交
18
#include <linux/cpu.h>
L
Linus Torvalds 已提交
19 20
#include <linux/errno.h>
#include <linux/sched.h>
21
#include <linux/fs.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27 28 29
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/elfcore.h>
#include <linux/smp.h>
#include <linux/slab.h>
#include <linux/user.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
30
#include <linux/module.h>
L
Linus Torvalds 已提交
31
#include <linux/ptrace.h>
A
Andi Kleen 已提交
32
#include <linux/notifier.h>
33
#include <linux/kprobes.h>
34
#include <linux/kdebug.h>
35
#include <linux/tick.h>
36
#include <linux/prctl.h>
37 38
#include <linux/uaccess.h>
#include <linux/io.h>
39
#include <linux/ftrace.h>
L
Len Brown 已提交
40
#include <linux/cpuidle.h>
L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48 49 50

#include <asm/pgtable.h>
#include <asm/system.h>
#include <asm/processor.h>
#include <asm/i387.h>
#include <asm/mmu_context.h>
#include <asm/prctl.h>
#include <asm/desc.h>
#include <asm/proto.h>
#include <asm/ia32.h>
A
Andi Kleen 已提交
51
#include <asm/idle.h>
52
#include <asm/syscalls.h>
53
#include <asm/debugreg.h>
54
#include <asm/nmi.h>
L
Linus Torvalds 已提交
55 56 57

asmlinkage extern void ret_from_fork(void);

58
DEFINE_PER_CPU(unsigned long, old_rsp);
59
static DEFINE_PER_CPU(unsigned char, is_idle);
60

61
static ATOMIC_NOTIFIER_HEAD(idle_notifier);
A
Andi Kleen 已提交
62 63 64

void idle_notifier_register(struct notifier_block *n)
{
65
	atomic_notifier_chain_register(&idle_notifier, n);
A
Andi Kleen 已提交
66
}
67 68 69 70 71 72 73
EXPORT_SYMBOL_GPL(idle_notifier_register);

void idle_notifier_unregister(struct notifier_block *n)
{
	atomic_notifier_chain_unregister(&idle_notifier, n);
}
EXPORT_SYMBOL_GPL(idle_notifier_unregister);
A
Andi Kleen 已提交
74 75 76

void enter_idle(void)
{
77
	percpu_write(is_idle, 1);
78
	atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
A
Andi Kleen 已提交
79 80 81 82
}

static void __exit_idle(void)
{
83
	if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
A
Andi Kleen 已提交
84
		return;
85
	atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
A
Andi Kleen 已提交
86 87 88 89 90
}

/* Called from interrupts to signify idle end */
void exit_idle(void)
{
A
Andi Kleen 已提交
91 92
	/* idle loop has pid 0 */
	if (current->pid)
A
Andi Kleen 已提交
93 94 95 96
		return;
	__exit_idle();
}

A
Alex Nixon 已提交
97
#ifndef CONFIG_SMP
A
Ashok Raj 已提交
98 99 100 101
static inline void play_dead(void)
{
	BUG();
}
A
Alex Nixon 已提交
102
#endif
A
Ashok Raj 已提交
103

L
Linus Torvalds 已提交
104 105 106 107 108 109
/*
 * The idle thread. There's no useful work to be
 * done, so just try to conserve power and have a
 * low exit latency (ie sit in a loop waiting for
 * somebody to say that they'd like to reschedule)
 */
P
Pavel Machek 已提交
110
void cpu_idle(void)
L
Linus Torvalds 已提交
111
{
112
	current_thread_info()->status |= TS_POLLING;
113 114

	/*
T
Tejun Heo 已提交
115 116 117 118 119
	 * If we're the non-boot CPU, nothing set the stack canary up
	 * for us.  CPU0 already has it initialized but no harm in
	 * doing it again.  This is a good place for updating it, as
	 * we wont ever return from this function (so the invalid
	 * canaries already on the stack wont ever trigger).
120
	 */
121 122
	boot_init_stack_canary();

L
Linus Torvalds 已提交
123 124
	/* endless idle loop with no priority at all */
	while (1) {
125
		tick_nohz_idle_enter();
L
Linus Torvalds 已提交
126 127 128
		while (!need_resched()) {

			rmb();
T
Thomas Gleixner 已提交
129

A
Ashok Raj 已提交
130 131
			if (cpu_is_offline(smp_processor_id()))
				play_dead();
132 133 134 135 136
			/*
			 * Idle routines should keep interrupts disabled
			 * from here on, until they go to idle.
			 * Otherwise, idle callbacks can misfire.
			 */
137
			local_touch_nmi();
138
			local_irq_disable();
A
Andi Kleen 已提交
139
			enter_idle();
140 141
			/* Don't trace irqs off for idle */
			stop_critical_timings();
142 143 144 145

			/* enter_idle() needs rcu for notifiers */
			rcu_idle_enter();

L
Len Brown 已提交
146 147
			if (cpuidle_idle_call())
				pm_idle();
148 149

			rcu_idle_exit();
150
			start_critical_timings();
151

A
Andi Kleen 已提交
152 153 154
			/* In many cases the interrupt that ended idle
			   has already called exit_idle. But some idle
			   loops can be woken up without interrupt. */
A
Andi Kleen 已提交
155
			__exit_idle();
L
Linus Torvalds 已提交
156 157
		}

158
		tick_nohz_idle_exit();
159
		preempt_enable_no_resched();
L
Linus Torvalds 已提交
160
		schedule();
161
		preempt_disable();
L
Linus Torvalds 已提交
162 163 164
	}
}

165
/* Prints also some state that isn't saved in the pt_regs */
166
void __show_regs(struct pt_regs *regs, int all)
L
Linus Torvalds 已提交
167 168
{
	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
169
	unsigned long d0, d1, d2, d3, d6, d7;
170 171
	unsigned int fsindex, gsindex;
	unsigned int ds, cs, es;
172 173

	show_regs_common();
174
	printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
175
	printk_address(regs->ip, 1);
176
	printk(KERN_DEFAULT "RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss,
177
			regs->sp, regs->flags);
178
	printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
179
	       regs->ax, regs->bx, regs->cx);
180
	printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
181
	       regs->dx, regs->si, regs->di);
182
	printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
183
	       regs->bp, regs->r8, regs->r9);
184
	printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
185
	       regs->r10, regs->r11, regs->r12);
186
	printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
187
	       regs->r13, regs->r14, regs->r15);
L
Linus Torvalds 已提交
188

189 190 191
	asm("movl %%ds,%0" : "=r" (ds));
	asm("movl %%cs,%0" : "=r" (cs));
	asm("movl %%es,%0" : "=r" (es));
L
Linus Torvalds 已提交
192 193 194 195
	asm("movl %%fs,%0" : "=r" (fsindex));
	asm("movl %%gs,%0" : "=r" (gsindex));

	rdmsrl(MSR_FS_BASE, fs);
196 197
	rdmsrl(MSR_GS_BASE, gs);
	rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
L
Linus Torvalds 已提交
198

199 200
	if (!all)
		return;
L
Linus Torvalds 已提交
201

202 203 204 205
	cr0 = read_cr0();
	cr2 = read_cr2();
	cr3 = read_cr3();
	cr4 = read_cr4();
L
Linus Torvalds 已提交
206

207
	printk(KERN_DEFAULT "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
208
	       fs, fsindex, gs, gsindex, shadowgs);
209
	printk(KERN_DEFAULT "CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
210
			es, cr0);
211
	printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
212
			cr4);
213 214 215 216

	get_debugreg(d0, 0);
	get_debugreg(d1, 1);
	get_debugreg(d2, 2);
217
	printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
218 219 220
	get_debugreg(d3, 3);
	get_debugreg(d6, 6);
	get_debugreg(d7, 7);
221
	printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
L
Linus Torvalds 已提交
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
}

void release_thread(struct task_struct *dead_task)
{
	if (dead_task->mm) {
		if (dead_task->mm->context.size) {
			printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
					dead_task->comm,
					dead_task->mm->context.ldt,
					dead_task->mm->context.size);
			BUG();
		}
	}
}

static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
{
239
	struct user_desc ud = {
L
Linus Torvalds 已提交
240 241 242 243 244 245
		.base_addr = addr,
		.limit = 0xfffff,
		.seg_32bit = 1,
		.limit_in_pages = 1,
		.useable = 1,
	};
J
Jan Engelhardt 已提交
246
	struct desc_struct *desc = t->thread.tls_array;
L
Linus Torvalds 已提交
247
	desc += tls;
248
	fill_ldt(desc, &ud);
L
Linus Torvalds 已提交
249 250 251 252
}

static inline u32 read_32bit_tls(struct task_struct *t, int tls)
{
R
Roland McGrath 已提交
253
	return get_desc_base(&t->thread.tls_array[tls]);
L
Linus Torvalds 已提交
254 255 256 257 258 259 260 261 262 263 264
}

/*
 * This gets called before we allocate a new thread and copy
 * the current task into it.
 */
void prepare_to_copy(struct task_struct *tsk)
{
	unlazy_fpu(tsk);
}

A
Alexey Dobriyan 已提交
265
int copy_thread(unsigned long clone_flags, unsigned long sp,
L
Linus Torvalds 已提交
266
		unsigned long unused,
267
	struct task_struct *p, struct pt_regs *regs)
L
Linus Torvalds 已提交
268 269
{
	int err;
270
	struct pt_regs *childregs;
L
Linus Torvalds 已提交
271 272
	struct task_struct *me = current;

273
	childregs = ((struct pt_regs *)
A
Al Viro 已提交
274
			(THREAD_SIZE + task_stack_page(p))) - 1;
L
Linus Torvalds 已提交
275 276
	*childregs = *regs;

277
	childregs->ax = 0;
278 279 280
	if (user_mode(regs))
		childregs->sp = sp;
	else
281
		childregs->sp = (unsigned long)childregs;
L
Linus Torvalds 已提交
282

283 284 285
	p->thread.sp = (unsigned long) childregs;
	p->thread.sp0 = (unsigned long) (childregs+1);
	p->thread.usersp = me->thread.usersp;
L
Linus Torvalds 已提交
286

A
Al Viro 已提交
287
	set_tsk_thread_flag(p, TIF_FORK);
L
Linus Torvalds 已提交
288

289
	p->thread.io_bitmap_ptr = NULL;
L
Linus Torvalds 已提交
290

291
	savesegment(gs, p->thread.gsindex);
292
	p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
293
	savesegment(fs, p->thread.fsindex);
294
	p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
295 296
	savesegment(es, p->thread.es);
	savesegment(ds, p->thread.ds);
L
Linus Torvalds 已提交
297

298
	err = -ENOMEM;
299
	memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
300

301
	if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
302 303
		p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
						  IO_BITMAP_BYTES, GFP_KERNEL);
L
Linus Torvalds 已提交
304 305 306 307
		if (!p->thread.io_bitmap_ptr) {
			p->thread.io_bitmap_max = 0;
			return -ENOMEM;
		}
308
		set_tsk_thread_flag(p, TIF_IO_BITMAP);
309
	}
L
Linus Torvalds 已提交
310 311 312 313 314 315 316

	/*
	 * Set a new TLS for the child thread?
	 */
	if (clone_flags & CLONE_SETTLS) {
#ifdef CONFIG_IA32_EMULATION
		if (test_thread_flag(TIF_IA32))
R
Roland McGrath 已提交
317
			err = do_set_thread_area(p, -1,
318
				(struct user_desc __user *)childregs->si, 0);
319 320 321 322
		else
#endif
			err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
		if (err)
L
Linus Torvalds 已提交
323 324 325 326 327 328 329 330
			goto out;
	}
	err = 0;
out:
	if (err && p->thread.io_bitmap_ptr) {
		kfree(p->thread.io_bitmap_ptr);
		p->thread.io_bitmap_max = 0;
	}
331

L
Linus Torvalds 已提交
332 333 334
	return err;
}

335 336 337 338
static void
start_thread_common(struct pt_regs *regs, unsigned long new_ip,
		    unsigned long new_sp,
		    unsigned int _cs, unsigned int _ss, unsigned int _ds)
I
Ingo Molnar 已提交
339
{
340
	loadsegment(fs, 0);
341 342
	loadsegment(es, _ds);
	loadsegment(ds, _ds);
I
Ingo Molnar 已提交
343 344 345
	load_gs_index(0);
	regs->ip		= new_ip;
	regs->sp		= new_sp;
346
	percpu_write(old_rsp, new_sp);
347 348
	regs->cs		= _cs;
	regs->ss		= _ss;
349
	regs->flags		= X86_EFLAGS_IF;
350 351 352 353
	/*
	 * Free the old FP and other extended state
	 */
	free_thread_xstate(current);
I
Ingo Molnar 已提交
354
}
355 356 357 358 359 360 361

void
start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
{
	start_thread_common(regs, new_ip, new_sp,
			    __USER_CS, __USER_DS, 0);
}
I
Ingo Molnar 已提交
362

363 364 365
#ifdef CONFIG_IA32_EMULATION
void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp)
{
366
	start_thread_common(regs, new_ip, new_sp,
H
H. Peter Anvin 已提交
367 368 369
			    test_thread_flag(TIF_X32)
			    ? __USER_CS : __USER32_CS,
			    __USER_DS, __USER_DS);
370 371
}
#endif
I
Ingo Molnar 已提交
372

L
Linus Torvalds 已提交
373 374 375
/*
 *	switch_to(x,y) should switch tasks from x to y.
 *
376
 * This could still be optimized:
L
Linus Torvalds 已提交
377 378
 * - fold all the options into a flag word and test it with a single test.
 * - could test fs/gs bitsliced
379 380
 *
 * Kprobes not supported here. Set the probe on schedule instead.
381
 * Function graph tracer not supported too.
L
Linus Torvalds 已提交
382
 */
383
__notrace_funcgraph struct task_struct *
384
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
L
Linus Torvalds 已提交
385
{
386 387
	struct thread_struct *prev = &prev_p->thread;
	struct thread_struct *next = &next_p->thread;
388
	int cpu = smp_processor_id();
L
Linus Torvalds 已提交
389
	struct tss_struct *tss = &per_cpu(init_tss, cpu);
390
	unsigned fsindex, gsindex;
391 392 393 394 395 396 397 398
	bool preload_fpu;

	/*
	 * If the task has used fpu the last 5 timeslices, just do a full
	 * restore of the math state immediately to avoid the trap; the
	 * chances of needing FPU soon are obviously high now
	 */
	preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
L
Linus Torvalds 已提交
399

400
	/* we're going to use this soon, after a few expensive things */
401
	if (preload_fpu)
402
		prefetch(next->fpu.state);
403

L
Linus Torvalds 已提交
404 405 406
	/*
	 * Reload esp0, LDT and the page table pointer:
	 */
407
	load_sp0(tss, next);
L
Linus Torvalds 已提交
408

409
	/*
L
Linus Torvalds 已提交
410 411 412
	 * Switch DS and ES.
	 * This won't pick up thread selector changes, but I guess that is ok.
	 */
413
	savesegment(es, prev->es);
L
Linus Torvalds 已提交
414
	if (unlikely(next->es | prev->es))
415
		loadsegment(es, next->es);
416 417

	savesegment(ds, prev->ds);
L
Linus Torvalds 已提交
418 419 420
	if (unlikely(next->ds | prev->ds))
		loadsegment(ds, next->ds);

421 422 423 424 425 426 427 428 429

	/* We must save %fs and %gs before load_TLS() because
	 * %fs and %gs may be cleared by load_TLS().
	 *
	 * (e.g. xen_load_tls())
	 */
	savesegment(fs, fsindex);
	savesegment(gs, gsindex);

L
Linus Torvalds 已提交
430 431
	load_TLS(next, cpu);

432
	/* Must be after DS reload */
433
	__unlazy_fpu(prev_p);
434

435 436 437 438
	/* Make sure cpu is ready for new context */
	if (preload_fpu)
		clts();

439 440 441 442 443 444 445
	/*
	 * Leave lazy mode, flushing any hypercalls made here.
	 * This must be done before restoring TLS segments so
	 * the GDT and LDT are properly updated, and must be
	 * done before math_state_restore, so the TS bit is up
	 * to date.
	 */
446
	arch_end_context_switch(next_p);
447

448
	/*
L
Linus Torvalds 已提交
449
	 * Switch FS and GS.
450 451 452 453
	 *
	 * Segment register != 0 always requires a reload.  Also
	 * reload when it has changed.  When prev process used 64bit
	 * base always reload to avoid an information leak.
L
Linus Torvalds 已提交
454
	 */
455 456
	if (unlikely(fsindex | next->fsindex | prev->fs)) {
		loadsegment(fs, next->fsindex);
457
		/*
458 459 460 461 462
		 * Check if the user used a selector != 0; if yes
		 *  clear 64bit base, since overloaded base is always
		 *  mapped to the Null selector
		 */
		if (fsindex)
463
			prev->fs = 0;
L
Linus Torvalds 已提交
464
	}
465 466 467 468 469 470 471 472
	/* when next process has a 64bit base use it */
	if (next->fs)
		wrmsrl(MSR_FS_BASE, next->fs);
	prev->fsindex = fsindex;

	if (unlikely(gsindex | next->gsindex | prev->gs)) {
		load_gs_index(next->gsindex);
		if (gsindex)
473
			prev->gs = 0;
L
Linus Torvalds 已提交
474
	}
475 476 477
	if (next->gs)
		wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
	prev->gsindex = gsindex;
L
Linus Torvalds 已提交
478

479
	/*
480
	 * Switch the PDA and FPU contexts.
L
Linus Torvalds 已提交
481
	 */
482 483
	prev->usersp = percpu_read(old_rsp);
	percpu_write(old_rsp, next->usersp);
484
	percpu_write(current_task, next_p);
485

486
	percpu_write(kernel_stack,
487
		  (unsigned long)task_stack_page(next_p) +
488
		  THREAD_SIZE - KERNEL_STACK_OFFSET);
L
Linus Torvalds 已提交
489 490

	/*
491
	 * Now maybe reload the debug registers and handle I/O bitmaps
L
Linus Torvalds 已提交
492
	 */
493 494
	if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
		     task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
495
		__switch_to_xtra(prev_p, next_p, tss);
L
Linus Torvalds 已提交
496

497 498 499
	/*
	 * Preload the FPU context, now that we've determined that the
	 * task is likely to be using it. 
500
	 */
501 502
	if (preload_fpu)
		__math_state_restore();
503

L
Linus Torvalds 已提交
504 505 506 507 508 509 510 511
	return prev_p;
}

void set_personality_64bit(void)
{
	/* inherit personality from parent */

	/* Make sure to be in 64bit mode */
512
	clear_thread_flag(TIF_IA32);
H
H. Peter Anvin 已提交
513
	clear_thread_flag(TIF_X32);
514
	clear_thread_flag(TIF_ADDR32);
515
	clear_thread_flag(TIF_X32);
L
Linus Torvalds 已提交
516

517 518 519 520
	/* Ensure the corresponding mm is not marked. */
	if (current->mm)
		current->mm->context.ia32_compat = 0;

L
Linus Torvalds 已提交
521 522 523
	/* TBD: overwrites user setup. Should have two bits.
	   But 64bit processes have always behaved this way,
	   so it's not too bad. The main problem is just that
524
	   32bit childs are affected again. */
L
Linus Torvalds 已提交
525 526 527
	current->personality &= ~READ_IMPLIES_EXEC;
}

H
H. Peter Anvin 已提交
528
void set_personality_ia32(bool x32)
529 530 531 532
{
	/* inherit personality from parent */

	/* Make sure to be in 32bit mode */
533
	set_thread_flag(TIF_ADDR32);
534

535 536 537 538
	/* Mark the associated mm as containing 32-bit tasks. */
	if (current->mm)
		current->mm->context.ia32_compat = 1;

H
H. Peter Anvin 已提交
539 540 541 542 543 544 545 546 547 548 549
	if (x32) {
		clear_thread_flag(TIF_IA32);
		set_thread_flag(TIF_X32);
		current->personality &= ~READ_IMPLIES_EXEC;
	} else {
		set_thread_flag(TIF_IA32);
		clear_thread_flag(TIF_X32);
		current->personality |= force_personality32;
		/* Prepare the first "return" to user space */
		current_thread_info()->status |= TS_COMPAT;
	}
550 551
}

L
Linus Torvalds 已提交
552 553 554
unsigned long get_wchan(struct task_struct *p)
{
	unsigned long stack;
555
	u64 fp, ip;
L
Linus Torvalds 已提交
556 557
	int count = 0;

558 559
	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;
A
Al Viro 已提交
560
	stack = (unsigned long)task_stack_page(p);
561
	if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
L
Linus Torvalds 已提交
562
		return 0;
563
	fp = *(u64 *)(p->thread.sp);
564
	do {
565
		if (fp < (unsigned long)stack ||
566
		    fp >= (unsigned long)stack+THREAD_SIZE)
567
			return 0;
568 569 570
		ip = *(u64 *)(fp+8);
		if (!in_sched_functions(ip))
			return ip;
571 572
		fp = *(u64 *)fp;
	} while (count++ < 16);
L
Linus Torvalds 已提交
573 574 575 576
	return 0;
}

long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
577 578
{
	int ret = 0;
L
Linus Torvalds 已提交
579 580 581
	int doit = task == current;
	int cpu;

582
	switch (code) {
L
Linus Torvalds 已提交
583
	case ARCH_SET_GS:
584
		if (addr >= TASK_SIZE_OF(task))
585
			return -EPERM;
L
Linus Torvalds 已提交
586
		cpu = get_cpu();
587
		/* handle small bases via the GDT because that's faster to
L
Linus Torvalds 已提交
588
		   switch. */
589 590 591
		if (addr <= 0xffffffff) {
			set_32bit_tls(task, GS_TLS, addr);
			if (doit) {
L
Linus Torvalds 已提交
592
				load_TLS(&task->thread, cpu);
593
				load_gs_index(GS_TLS_SEL);
L
Linus Torvalds 已提交
594
			}
595
			task->thread.gsindex = GS_TLS_SEL;
L
Linus Torvalds 已提交
596
			task->thread.gs = 0;
597
		} else {
L
Linus Torvalds 已提交
598 599 600
			task->thread.gsindex = 0;
			task->thread.gs = addr;
			if (doit) {
601 602
				load_gs_index(0);
				ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
603
			}
L
Linus Torvalds 已提交
604 605 606 607 608 609
		}
		put_cpu();
		break;
	case ARCH_SET_FS:
		/* Not strictly needed for fs, but do it for symmetry
		   with gs */
610
		if (addr >= TASK_SIZE_OF(task))
611
			return -EPERM;
L
Linus Torvalds 已提交
612
		cpu = get_cpu();
613
		/* handle small bases via the GDT because that's faster to
L
Linus Torvalds 已提交
614
		   switch. */
615
		if (addr <= 0xffffffff) {
L
Linus Torvalds 已提交
616
			set_32bit_tls(task, FS_TLS, addr);
617 618
			if (doit) {
				load_TLS(&task->thread, cpu);
619
				loadsegment(fs, FS_TLS_SEL);
L
Linus Torvalds 已提交
620 621 622
			}
			task->thread.fsindex = FS_TLS_SEL;
			task->thread.fs = 0;
623
		} else {
L
Linus Torvalds 已提交
624 625 626 627 628
			task->thread.fsindex = 0;
			task->thread.fs = addr;
			if (doit) {
				/* set the selector to 0 to not confuse
				   __switch_to */
629
				loadsegment(fs, 0);
630
				ret = checking_wrmsrl(MSR_FS_BASE, addr);
L
Linus Torvalds 已提交
631 632 633 634
			}
		}
		put_cpu();
		break;
635 636
	case ARCH_GET_FS: {
		unsigned long base;
L
Linus Torvalds 已提交
637 638
		if (task->thread.fsindex == FS_TLS_SEL)
			base = read_32bit_tls(task, FS_TLS);
639
		else if (doit)
L
Linus Torvalds 已提交
640
			rdmsrl(MSR_FS_BASE, base);
641
		else
L
Linus Torvalds 已提交
642
			base = task->thread.fs;
643 644
		ret = put_user(base, (unsigned long __user *)addr);
		break;
L
Linus Torvalds 已提交
645
	}
646
	case ARCH_GET_GS: {
L
Linus Torvalds 已提交
647
		unsigned long base;
648
		unsigned gsindex;
L
Linus Torvalds 已提交
649 650
		if (task->thread.gsindex == GS_TLS_SEL)
			base = read_32bit_tls(task, GS_TLS);
651
		else if (doit) {
652
			savesegment(gs, gsindex);
653 654 655 656
			if (gsindex)
				rdmsrl(MSR_KERNEL_GS_BASE, base);
			else
				base = task->thread.gs;
657
		} else
L
Linus Torvalds 已提交
658
			base = task->thread.gs;
659
		ret = put_user(base, (unsigned long __user *)addr);
L
Linus Torvalds 已提交
660 661 662 663 664 665
		break;
	}

	default:
		ret = -EINVAL;
		break;
666
	}
L
Linus Torvalds 已提交
667

668 669
	return ret;
}
L
Linus Torvalds 已提交
670 671 672 673 674 675

long sys_arch_prctl(int code, unsigned long addr)
{
	return do_arch_prctl(current, code, addr);
}

676 677 678 679 680
unsigned long KSTK_ESP(struct task_struct *task)
{
	return (test_tsk_thread_flag(task, TIF_IA32)) ?
			(task_pt_regs(task)->sp) : ((task)->thread.usersp);
}