process_64.c 16.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Pentium III FXSR, SSE support
 *	Gareth Hughes <gareth@valinux.com>, May 2000
6
 *
L
Linus Torvalds 已提交
7 8
 *  X86-64 port
 *	Andi Kleen.
A
Ashok Raj 已提交
9 10
 *
 *	CPU hotplug support - ashok.raj@intel.com
L
Linus Torvalds 已提交
11 12 13 14 15 16
 */

/*
 * This file handles the architecture-dependent parts of process handling..
 */

17
#include <linux/stackprotector.h>
A
Ashok Raj 已提交
18
#include <linux/cpu.h>
L
Linus Torvalds 已提交
19 20
#include <linux/errno.h>
#include <linux/sched.h>
21
#include <linux/fs.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27 28
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/elfcore.h>
#include <linux/smp.h>
#include <linux/slab.h>
#include <linux/user.h>
#include <linux/interrupt.h>
29
#include <linux/utsname.h>
L
Linus Torvalds 已提交
30
#include <linux/delay.h>
31
#include <linux/module.h>
L
Linus Torvalds 已提交
32
#include <linux/ptrace.h>
A
Andi Kleen 已提交
33
#include <linux/notifier.h>
34
#include <linux/kprobes.h>
35
#include <linux/kdebug.h>
36
#include <linux/tick.h>
37
#include <linux/prctl.h>
38 39
#include <linux/uaccess.h>
#include <linux/io.h>
40
#include <linux/ftrace.h>
41
#include <linux/dmi.h>
L
Linus Torvalds 已提交
42 43 44 45 46 47 48 49 50 51

#include <asm/pgtable.h>
#include <asm/system.h>
#include <asm/processor.h>
#include <asm/i387.h>
#include <asm/mmu_context.h>
#include <asm/prctl.h>
#include <asm/desc.h>
#include <asm/proto.h>
#include <asm/ia32.h>
A
Andi Kleen 已提交
52
#include <asm/idle.h>
53
#include <asm/syscalls.h>
54
#include <asm/ds.h>
55
#include <asm/debugreg.h>
L
Linus Torvalds 已提交
56 57 58

asmlinkage extern void ret_from_fork(void);

59
DEFINE_PER_CPU(unsigned long, old_rsp);
60
static DEFINE_PER_CPU(unsigned char, is_idle);
61

L
Linus Torvalds 已提交
62 63
unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;

64
static ATOMIC_NOTIFIER_HEAD(idle_notifier);
A
Andi Kleen 已提交
65 66 67

void idle_notifier_register(struct notifier_block *n)
{
68
	atomic_notifier_chain_register(&idle_notifier, n);
A
Andi Kleen 已提交
69
}
70 71 72 73 74 75 76
EXPORT_SYMBOL_GPL(idle_notifier_register);

void idle_notifier_unregister(struct notifier_block *n)
{
	atomic_notifier_chain_unregister(&idle_notifier, n);
}
EXPORT_SYMBOL_GPL(idle_notifier_unregister);
A
Andi Kleen 已提交
77 78 79

void enter_idle(void)
{
80
	percpu_write(is_idle, 1);
81
	atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
A
Andi Kleen 已提交
82 83 84 85
}

static void __exit_idle(void)
{
86
	if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
A
Andi Kleen 已提交
87
		return;
88
	atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
A
Andi Kleen 已提交
89 90 91 92 93
}

/* Called from interrupts to signify idle end */
void exit_idle(void)
{
A
Andi Kleen 已提交
94 95
	/* idle loop has pid 0 */
	if (current->pid)
A
Andi Kleen 已提交
96 97 98 99
		return;
	__exit_idle();
}

A
Alex Nixon 已提交
100
#ifndef CONFIG_SMP
A
Ashok Raj 已提交
101 102 103 104
static inline void play_dead(void)
{
	BUG();
}
A
Alex Nixon 已提交
105
#endif
A
Ashok Raj 已提交
106

L
Linus Torvalds 已提交
107 108 109 110 111 112
/*
 * The idle thread. There's no useful work to be
 * done, so just try to conserve power and have a
 * low exit latency (ie sit in a loop waiting for
 * somebody to say that they'd like to reschedule)
 */
P
Pavel Machek 已提交
113
void cpu_idle(void)
L
Linus Torvalds 已提交
114
{
115
	current_thread_info()->status |= TS_POLLING;
116 117

	/*
T
Tejun Heo 已提交
118 119 120 121 122
	 * If we're the non-boot CPU, nothing set the stack canary up
	 * for us.  CPU0 already has it initialized but no harm in
	 * doing it again.  This is a good place for updating it, as
	 * we wont ever return from this function (so the invalid
	 * canaries already on the stack wont ever trigger).
123
	 */
124 125
	boot_init_stack_canary();

L
Linus Torvalds 已提交
126 127
	/* endless idle loop with no priority at all */
	while (1) {
128
		tick_nohz_stop_sched_tick(1);
L
Linus Torvalds 已提交
129 130 131
		while (!need_resched()) {

			rmb();
T
Thomas Gleixner 已提交
132

A
Ashok Raj 已提交
133 134
			if (cpu_is_offline(smp_processor_id()))
				play_dead();
135 136 137 138 139 140
			/*
			 * Idle routines should keep interrupts disabled
			 * from here on, until they go to idle.
			 * Otherwise, idle callbacks can misfire.
			 */
			local_irq_disable();
A
Andi Kleen 已提交
141
			enter_idle();
142 143
			/* Don't trace irqs off for idle */
			stop_critical_timings();
T
Thomas Gleixner 已提交
144
			pm_idle();
145
			start_critical_timings();
A
Andi Kleen 已提交
146 147 148
			/* In many cases the interrupt that ended idle
			   has already called exit_idle. But some idle
			   loops can be woken up without interrupt. */
A
Andi Kleen 已提交
149
			__exit_idle();
L
Linus Torvalds 已提交
150 151
		}

152
		tick_nohz_restart_sched_tick();
153
		preempt_enable_no_resched();
L
Linus Torvalds 已提交
154
		schedule();
155
		preempt_disable();
L
Linus Torvalds 已提交
156 157 158
	}
}

159
/* Prints also some state that isn't saved in the pt_regs */
160
void __show_regs(struct pt_regs *regs, int all)
L
Linus Torvalds 已提交
161 162
{
	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
163
	unsigned long d0, d1, d2, d3, d6, d7;
164 165
	unsigned int fsindex, gsindex;
	unsigned int ds, cs, es;
166
	const char *board;
L
Linus Torvalds 已提交
167 168 169

	printk("\n");
	print_modules();
170 171 172 173
	board = dmi_get_system_info(DMI_PRODUCT_NAME);
	if (!board)
		board = "";
	printk(KERN_INFO "Pid: %d, comm: %.20s %s %s %.*s %s\n",
174
		current->pid, current->comm, print_tainted(),
175 176
		init_utsname()->release,
		(int)strcspn(init_utsname()->version, " "),
177
		init_utsname()->version, board);
178
	printk(KERN_INFO "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
179
	printk_address(regs->ip, 1);
180 181 182
	printk(KERN_INFO "RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss,
			regs->sp, regs->flags);
	printk(KERN_INFO "RAX: %016lx RBX: %016lx RCX: %016lx\n",
183
	       regs->ax, regs->bx, regs->cx);
184
	printk(KERN_INFO "RDX: %016lx RSI: %016lx RDI: %016lx\n",
185
	       regs->dx, regs->si, regs->di);
186
	printk(KERN_INFO "RBP: %016lx R08: %016lx R09: %016lx\n",
187
	       regs->bp, regs->r8, regs->r9);
188
	printk(KERN_INFO "R10: %016lx R11: %016lx R12: %016lx\n",
189
	       regs->r10, regs->r11, regs->r12);
190
	printk(KERN_INFO "R13: %016lx R14: %016lx R15: %016lx\n",
191
	       regs->r13, regs->r14, regs->r15);
L
Linus Torvalds 已提交
192

193 194 195
	asm("movl %%ds,%0" : "=r" (ds));
	asm("movl %%cs,%0" : "=r" (cs));
	asm("movl %%es,%0" : "=r" (es));
L
Linus Torvalds 已提交
196 197 198 199
	asm("movl %%fs,%0" : "=r" (fsindex));
	asm("movl %%gs,%0" : "=r" (gsindex));

	rdmsrl(MSR_FS_BASE, fs);
200 201
	rdmsrl(MSR_GS_BASE, gs);
	rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
L
Linus Torvalds 已提交
202

203 204
	if (!all)
		return;
L
Linus Torvalds 已提交
205

206 207 208 209
	cr0 = read_cr0();
	cr2 = read_cr2();
	cr3 = read_cr3();
	cr4 = read_cr4();
L
Linus Torvalds 已提交
210

211
	printk(KERN_INFO "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
212
	       fs, fsindex, gs, gsindex, shadowgs);
213 214 215 216
	printk(KERN_INFO "CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
			es, cr0);
	printk(KERN_INFO "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
			cr4);
217 218 219 220

	get_debugreg(d0, 0);
	get_debugreg(d1, 1);
	get_debugreg(d2, 2);
221
	printk(KERN_INFO "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
222 223 224
	get_debugreg(d3, 3);
	get_debugreg(d6, 6);
	get_debugreg(d7, 7);
225
	printk(KERN_INFO "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
L
Linus Torvalds 已提交
226 227 228 229
}

void show_regs(struct pt_regs *regs)
{
230
	show_registers(regs);
231
	show_trace(NULL, regs, (void *)(regs + 1), regs->bp);
L
Linus Torvalds 已提交
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
}

void release_thread(struct task_struct *dead_task)
{
	if (dead_task->mm) {
		if (dead_task->mm->context.size) {
			printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
					dead_task->comm,
					dead_task->mm->context.ldt,
					dead_task->mm->context.size);
			BUG();
		}
	}
}

static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
{
249
	struct user_desc ud = {
L
Linus Torvalds 已提交
250 251 252 253 254 255
		.base_addr = addr,
		.limit = 0xfffff,
		.seg_32bit = 1,
		.limit_in_pages = 1,
		.useable = 1,
	};
J
Jan Engelhardt 已提交
256
	struct desc_struct *desc = t->thread.tls_array;
L
Linus Torvalds 已提交
257
	desc += tls;
258
	fill_ldt(desc, &ud);
L
Linus Torvalds 已提交
259 260 261 262
}

static inline u32 read_32bit_tls(struct task_struct *t, int tls)
{
R
Roland McGrath 已提交
263
	return get_desc_base(&t->thread.tls_array[tls]);
L
Linus Torvalds 已提交
264 265 266 267 268 269 270 271 272 273 274
}

/*
 * This gets called before we allocate a new thread and copy
 * the current task into it.
 */
void prepare_to_copy(struct task_struct *tsk)
{
	unlazy_fpu(tsk);
}

A
Alexey Dobriyan 已提交
275
int copy_thread(unsigned long clone_flags, unsigned long sp,
L
Linus Torvalds 已提交
276
		unsigned long unused,
277
	struct task_struct *p, struct pt_regs *regs)
L
Linus Torvalds 已提交
278 279
{
	int err;
280
	struct pt_regs *childregs;
L
Linus Torvalds 已提交
281 282
	struct task_struct *me = current;

283
	childregs = ((struct pt_regs *)
A
Al Viro 已提交
284
			(THREAD_SIZE + task_stack_page(p))) - 1;
L
Linus Torvalds 已提交
285 286
	*childregs = *regs;

287 288 289 290
	childregs->ax = 0;
	childregs->sp = sp;
	if (sp == ~0UL)
		childregs->sp = (unsigned long)childregs;
L
Linus Torvalds 已提交
291

292 293 294
	p->thread.sp = (unsigned long) childregs;
	p->thread.sp0 = (unsigned long) (childregs+1);
	p->thread.usersp = me->thread.usersp;
L
Linus Torvalds 已提交
295

A
Al Viro 已提交
296
	set_tsk_thread_flag(p, TIF_FORK);
L
Linus Torvalds 已提交
297 298 299

	p->thread.fs = me->thread.fs;
	p->thread.gs = me->thread.gs;
300
	p->thread.io_bitmap_ptr = NULL;
L
Linus Torvalds 已提交
301

302 303 304 305
	savesegment(gs, p->thread.gsindex);
	savesegment(fs, p->thread.fsindex);
	savesegment(es, p->thread.es);
	savesegment(ds, p->thread.ds);
L
Linus Torvalds 已提交
306

307
	err = -ENOMEM;
308
	memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
309

310
	if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
L
Linus Torvalds 已提交
311 312 313 314 315
		p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
		if (!p->thread.io_bitmap_ptr) {
			p->thread.io_bitmap_max = 0;
			return -ENOMEM;
		}
316 317
		memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
				IO_BITMAP_BYTES);
318
		set_tsk_thread_flag(p, TIF_IO_BITMAP);
319
	}
L
Linus Torvalds 已提交
320 321 322 323 324 325 326

	/*
	 * Set a new TLS for the child thread?
	 */
	if (clone_flags & CLONE_SETTLS) {
#ifdef CONFIG_IA32_EMULATION
		if (test_thread_flag(TIF_IA32))
R
Roland McGrath 已提交
327
			err = do_set_thread_area(p, -1,
328
				(struct user_desc __user *)childregs->si, 0);
329 330 331 332
		else
#endif
			err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
		if (err)
L
Linus Torvalds 已提交
333 334
			goto out;
	}
335

M
Markus Metzger 已提交
336 337
	clear_tsk_thread_flag(p, TIF_DS_AREA_MSR);
	p->thread.ds_ctx = NULL;
338 339 340 341

	clear_tsk_thread_flag(p, TIF_DEBUGCTLMSR);
	p->thread.debugctlmsr = 0;

L
Linus Torvalds 已提交
342 343 344 345 346 347
	err = 0;
out:
	if (err && p->thread.io_bitmap_ptr) {
		kfree(p->thread.io_bitmap_ptr);
		p->thread.io_bitmap_max = 0;
	}
348

L
Linus Torvalds 已提交
349 350 351
	return err;
}

352 353 354 355
static void
start_thread_common(struct pt_regs *regs, unsigned long new_ip,
		    unsigned long new_sp,
		    unsigned int _cs, unsigned int _ss, unsigned int _ds)
I
Ingo Molnar 已提交
356
{
357
	loadsegment(fs, 0);
358 359
	loadsegment(es, _ds);
	loadsegment(ds, _ds);
I
Ingo Molnar 已提交
360 361 362
	load_gs_index(0);
	regs->ip		= new_ip;
	regs->sp		= new_sp;
363
	percpu_write(old_rsp, new_sp);
364 365
	regs->cs		= _cs;
	regs->ss		= _ss;
366
	regs->flags		= X86_EFLAGS_IF;
I
Ingo Molnar 已提交
367
	set_fs(USER_DS);
368 369 370 371
	/*
	 * Free the old FP and other extended state
	 */
	free_thread_xstate(current);
I
Ingo Molnar 已提交
372
}
373 374 375 376 377 378 379

void
start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
{
	start_thread_common(regs, new_ip, new_sp,
			    __USER_CS, __USER_DS, 0);
}
I
Ingo Molnar 已提交
380

381 382 383
#ifdef CONFIG_IA32_EMULATION
void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp)
{
384 385
	start_thread_common(regs, new_ip, new_sp,
			    __USER32_CS, __USER32_DS, __USER32_DS);
386 387
}
#endif
I
Ingo Molnar 已提交
388

L
Linus Torvalds 已提交
389 390 391
/*
 *	switch_to(x,y) should switch tasks from x to y.
 *
392
 * This could still be optimized:
L
Linus Torvalds 已提交
393 394
 * - fold all the options into a flag word and test it with a single test.
 * - could test fs/gs bitsliced
395 396
 *
 * Kprobes not supported here. Set the probe on schedule instead.
397
 * Function graph tracer not supported too.
L
Linus Torvalds 已提交
398
 */
399
__notrace_funcgraph struct task_struct *
400
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
L
Linus Torvalds 已提交
401
{
402 403
	struct thread_struct *prev = &prev_p->thread;
	struct thread_struct *next = &next_p->thread;
404
	int cpu = smp_processor_id();
L
Linus Torvalds 已提交
405
	struct tss_struct *tss = &per_cpu(init_tss, cpu);
406
	unsigned fsindex, gsindex;
407 408 409 410 411 412 413 414
	bool preload_fpu;

	/*
	 * If the task has used fpu the last 5 timeslices, just do a full
	 * restore of the math state immediately to avoid the trap; the
	 * chances of needing FPU soon are obviously high now
	 */
	preload_fpu = tsk_used_math(next_p) && next_p->fpu_counter > 5;
L
Linus Torvalds 已提交
415

416
	/* we're going to use this soon, after a few expensive things */
417
	if (preload_fpu)
418
		prefetch(next->xstate);
419

L
Linus Torvalds 已提交
420 421 422
	/*
	 * Reload esp0, LDT and the page table pointer:
	 */
423
	load_sp0(tss, next);
L
Linus Torvalds 已提交
424

425
	/*
L
Linus Torvalds 已提交
426 427 428
	 * Switch DS and ES.
	 * This won't pick up thread selector changes, but I guess that is ok.
	 */
429
	savesegment(es, prev->es);
L
Linus Torvalds 已提交
430
	if (unlikely(next->es | prev->es))
431
		loadsegment(es, next->es);
432 433

	savesegment(ds, prev->ds);
L
Linus Torvalds 已提交
434 435 436
	if (unlikely(next->ds | prev->ds))
		loadsegment(ds, next->ds);

437 438 439 440 441 442 443 444 445

	/* We must save %fs and %gs before load_TLS() because
	 * %fs and %gs may be cleared by load_TLS().
	 *
	 * (e.g. xen_load_tls())
	 */
	savesegment(fs, fsindex);
	savesegment(gs, gsindex);

L
Linus Torvalds 已提交
446 447
	load_TLS(next, cpu);

448 449 450
	/* Must be after DS reload */
	unlazy_fpu(prev_p);

451 452 453 454
	/* Make sure cpu is ready for new context */
	if (preload_fpu)
		clts();

455 456 457 458 459 460 461
	/*
	 * Leave lazy mode, flushing any hypercalls made here.
	 * This must be done before restoring TLS segments so
	 * the GDT and LDT are properly updated, and must be
	 * done before math_state_restore, so the TS bit is up
	 * to date.
	 */
462
	arch_end_context_switch(next_p);
463

464
	/*
L
Linus Torvalds 已提交
465
	 * Switch FS and GS.
466 467 468 469
	 *
	 * Segment register != 0 always requires a reload.  Also
	 * reload when it has changed.  When prev process used 64bit
	 * base always reload to avoid an information leak.
L
Linus Torvalds 已提交
470
	 */
471 472
	if (unlikely(fsindex | next->fsindex | prev->fs)) {
		loadsegment(fs, next->fsindex);
473
		/*
474 475 476 477 478
		 * Check if the user used a selector != 0; if yes
		 *  clear 64bit base, since overloaded base is always
		 *  mapped to the Null selector
		 */
		if (fsindex)
479
			prev->fs = 0;
L
Linus Torvalds 已提交
480
	}
481 482 483 484 485 486 487 488
	/* when next process has a 64bit base use it */
	if (next->fs)
		wrmsrl(MSR_FS_BASE, next->fs);
	prev->fsindex = fsindex;

	if (unlikely(gsindex | next->gsindex | prev->gs)) {
		load_gs_index(next->gsindex);
		if (gsindex)
489
			prev->gs = 0;
L
Linus Torvalds 已提交
490
	}
491 492 493
	if (next->gs)
		wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
	prev->gsindex = gsindex;
L
Linus Torvalds 已提交
494

495
	/*
496
	 * Switch the PDA and FPU contexts.
L
Linus Torvalds 已提交
497
	 */
498 499
	prev->usersp = percpu_read(old_rsp);
	percpu_write(old_rsp, next->usersp);
500
	percpu_write(current_task, next_p);
501

502
	percpu_write(kernel_stack,
503
		  (unsigned long)task_stack_page(next_p) +
504
		  THREAD_SIZE - KERNEL_STACK_OFFSET);
L
Linus Torvalds 已提交
505 506

	/*
507
	 * Now maybe reload the debug registers and handle I/O bitmaps
L
Linus Torvalds 已提交
508
	 */
509 510
	if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
		     task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
511
		__switch_to_xtra(prev_p, next_p, tss);
L
Linus Torvalds 已提交
512

513 514 515
	/*
	 * Preload the FPU context, now that we've determined that the
	 * task is likely to be using it. 
516
	 */
517 518
	if (preload_fpu)
		__math_state_restore();
519

L
Linus Torvalds 已提交
520 521 522 523 524 525 526 527
	return prev_p;
}

void set_personality_64bit(void)
{
	/* inherit personality from parent */

	/* Make sure to be in 64bit mode */
528
	clear_thread_flag(TIF_IA32);
L
Linus Torvalds 已提交
529 530 531 532

	/* TBD: overwrites user setup. Should have two bits.
	   But 64bit processes have always behaved this way,
	   so it's not too bad. The main problem is just that
533
	   32bit childs are affected again. */
L
Linus Torvalds 已提交
534 535 536
	current->personality &= ~READ_IMPLIES_EXEC;
}

537 538 539
asmlinkage long
sys_clone(unsigned long clone_flags, unsigned long newsp,
	  void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
L
Linus Torvalds 已提交
540 541
{
	if (!newsp)
542
		newsp = regs->sp;
L
Linus Torvalds 已提交
543 544 545 546 547 548
	return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
}

unsigned long get_wchan(struct task_struct *p)
{
	unsigned long stack;
549
	u64 fp, ip;
L
Linus Torvalds 已提交
550 551
	int count = 0;

552 553
	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;
A
Al Viro 已提交
554
	stack = (unsigned long)task_stack_page(p);
555
	if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
L
Linus Torvalds 已提交
556
		return 0;
557
	fp = *(u64 *)(p->thread.sp);
558
	do {
559
		if (fp < (unsigned long)stack ||
560
		    fp >= (unsigned long)stack+THREAD_SIZE)
561
			return 0;
562 563 564
		ip = *(u64 *)(fp+8);
		if (!in_sched_functions(ip))
			return ip;
565 566
		fp = *(u64 *)fp;
	} while (count++ < 16);
L
Linus Torvalds 已提交
567 568 569 570
	return 0;
}

long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
571 572
{
	int ret = 0;
L
Linus Torvalds 已提交
573 574 575
	int doit = task == current;
	int cpu;

576
	switch (code) {
L
Linus Torvalds 已提交
577
	case ARCH_SET_GS:
578
		if (addr >= TASK_SIZE_OF(task))
579
			return -EPERM;
L
Linus Torvalds 已提交
580
		cpu = get_cpu();
581
		/* handle small bases via the GDT because that's faster to
L
Linus Torvalds 已提交
582
		   switch. */
583 584 585
		if (addr <= 0xffffffff) {
			set_32bit_tls(task, GS_TLS, addr);
			if (doit) {
L
Linus Torvalds 已提交
586
				load_TLS(&task->thread, cpu);
587
				load_gs_index(GS_TLS_SEL);
L
Linus Torvalds 已提交
588
			}
589
			task->thread.gsindex = GS_TLS_SEL;
L
Linus Torvalds 已提交
590
			task->thread.gs = 0;
591
		} else {
L
Linus Torvalds 已提交
592 593 594
			task->thread.gsindex = 0;
			task->thread.gs = addr;
			if (doit) {
595 596
				load_gs_index(0);
				ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
597
			}
L
Linus Torvalds 已提交
598 599 600 601 602 603
		}
		put_cpu();
		break;
	case ARCH_SET_FS:
		/* Not strictly needed for fs, but do it for symmetry
		   with gs */
604
		if (addr >= TASK_SIZE_OF(task))
605
			return -EPERM;
L
Linus Torvalds 已提交
606
		cpu = get_cpu();
607
		/* handle small bases via the GDT because that's faster to
L
Linus Torvalds 已提交
608
		   switch. */
609
		if (addr <= 0xffffffff) {
L
Linus Torvalds 已提交
610
			set_32bit_tls(task, FS_TLS, addr);
611 612
			if (doit) {
				load_TLS(&task->thread, cpu);
613
				loadsegment(fs, FS_TLS_SEL);
L
Linus Torvalds 已提交
614 615 616
			}
			task->thread.fsindex = FS_TLS_SEL;
			task->thread.fs = 0;
617
		} else {
L
Linus Torvalds 已提交
618 619 620 621 622
			task->thread.fsindex = 0;
			task->thread.fs = addr;
			if (doit) {
				/* set the selector to 0 to not confuse
				   __switch_to */
623
				loadsegment(fs, 0);
624
				ret = checking_wrmsrl(MSR_FS_BASE, addr);
L
Linus Torvalds 已提交
625 626 627 628
			}
		}
		put_cpu();
		break;
629 630
	case ARCH_GET_FS: {
		unsigned long base;
L
Linus Torvalds 已提交
631 632
		if (task->thread.fsindex == FS_TLS_SEL)
			base = read_32bit_tls(task, FS_TLS);
633
		else if (doit)
L
Linus Torvalds 已提交
634
			rdmsrl(MSR_FS_BASE, base);
635
		else
L
Linus Torvalds 已提交
636
			base = task->thread.fs;
637 638
		ret = put_user(base, (unsigned long __user *)addr);
		break;
L
Linus Torvalds 已提交
639
	}
640
	case ARCH_GET_GS: {
L
Linus Torvalds 已提交
641
		unsigned long base;
642
		unsigned gsindex;
L
Linus Torvalds 已提交
643 644
		if (task->thread.gsindex == GS_TLS_SEL)
			base = read_32bit_tls(task, GS_TLS);
645
		else if (doit) {
646
			savesegment(gs, gsindex);
647 648 649 650
			if (gsindex)
				rdmsrl(MSR_KERNEL_GS_BASE, base);
			else
				base = task->thread.gs;
651
		} else
L
Linus Torvalds 已提交
652
			base = task->thread.gs;
653
		ret = put_user(base, (unsigned long __user *)addr);
L
Linus Torvalds 已提交
654 655 656 657 658 659
		break;
	}

	default:
		ret = -EINVAL;
		break;
660
	}
L
Linus Torvalds 已提交
661

662 663
	return ret;
}
L
Linus Torvalds 已提交
664 665 666 667 668 669

long sys_arch_prctl(int code, unsigned long addr)
{
	return do_arch_prctl(current, code, addr);
}

670 671 672 673 674
unsigned long KSTK_ESP(struct task_struct *task)
{
	return (test_tsk_thread_flag(task, TIF_IA32)) ?
			(task_pt_regs(task)->sp) : ((task)->thread.usersp);
}