process_64.c 17.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Pentium III FXSR, SSE support
 *	Gareth Hughes <gareth@valinux.com>, May 2000
6
 *
L
Linus Torvalds 已提交
7 8
 *  X86-64 port
 *	Andi Kleen.
A
Ashok Raj 已提交
9 10
 *
 *	CPU hotplug support - ashok.raj@intel.com
L
Linus Torvalds 已提交
11 12 13 14 15 16
 */

/*
 * This file handles the architecture-dependent parts of process handling..
 */

A
Ashok Raj 已提交
17
#include <linux/cpu.h>
L
Linus Torvalds 已提交
18 19
#include <linux/errno.h>
#include <linux/sched.h>
20
#include <linux/fs.h>
L
Linus Torvalds 已提交
21 22 23 24 25 26 27 28
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/elfcore.h>
#include <linux/smp.h>
#include <linux/slab.h>
#include <linux/user.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
29
#include <linux/module.h>
L
Linus Torvalds 已提交
30
#include <linux/ptrace.h>
A
Andi Kleen 已提交
31
#include <linux/notifier.h>
32
#include <linux/kprobes.h>
33
#include <linux/kdebug.h>
34
#include <linux/prctl.h>
35 36
#include <linux/uaccess.h>
#include <linux/io.h>
37
#include <linux/ftrace.h>
L
Linus Torvalds 已提交
38 39 40

#include <asm/pgtable.h>
#include <asm/processor.h>
41
#include <asm/fpu/internal.h>
L
Linus Torvalds 已提交
42 43 44 45 46
#include <asm/mmu_context.h>
#include <asm/prctl.h>
#include <asm/desc.h>
#include <asm/proto.h>
#include <asm/ia32.h>
A
Andi Kleen 已提交
47
#include <asm/idle.h>
48
#include <asm/syscalls.h>
49
#include <asm/debugreg.h>
50
#include <asm/switch_to.h>
L
Linus Torvalds 已提交
51 52 53

asmlinkage extern void ret_from_fork(void);

54
__visible DEFINE_PER_CPU(unsigned long, rsp_scratch);
L
Linus Torvalds 已提交
55

56
/* Prints also some state that isn't saved in the pt_regs */
57
void __show_regs(struct pt_regs *regs, int all)
L
Linus Torvalds 已提交
58 59
{
	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
60
	unsigned long d0, d1, d2, d3, d6, d7;
61 62
	unsigned int fsindex, gsindex;
	unsigned int ds, cs, es;
63

64
	printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
65
	printk_address(regs->ip);
66
	printk(KERN_DEFAULT "RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss,
67
			regs->sp, regs->flags);
68
	printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
69
	       regs->ax, regs->bx, regs->cx);
70
	printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
71
	       regs->dx, regs->si, regs->di);
72
	printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
73
	       regs->bp, regs->r8, regs->r9);
74
	printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
75
	       regs->r10, regs->r11, regs->r12);
76
	printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
77
	       regs->r13, regs->r14, regs->r15);
L
Linus Torvalds 已提交
78

79 80 81
	asm("movl %%ds,%0" : "=r" (ds));
	asm("movl %%cs,%0" : "=r" (cs));
	asm("movl %%es,%0" : "=r" (es));
L
Linus Torvalds 已提交
82 83 84 85
	asm("movl %%fs,%0" : "=r" (fsindex));
	asm("movl %%gs,%0" : "=r" (gsindex));

	rdmsrl(MSR_FS_BASE, fs);
86 87
	rdmsrl(MSR_GS_BASE, gs);
	rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
L
Linus Torvalds 已提交
88

89 90
	if (!all)
		return;
L
Linus Torvalds 已提交
91

92 93 94
	cr0 = read_cr0();
	cr2 = read_cr2();
	cr3 = read_cr3();
95
	cr4 = __read_cr4();
L
Linus Torvalds 已提交
96

97
	printk(KERN_DEFAULT "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
98
	       fs, fsindex, gs, gsindex, shadowgs);
99
	printk(KERN_DEFAULT "CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
100
			es, cr0);
101
	printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
102
			cr4);
103 104 105 106 107 108 109

	get_debugreg(d0, 0);
	get_debugreg(d1, 1);
	get_debugreg(d2, 2);
	get_debugreg(d3, 3);
	get_debugreg(d6, 6);
	get_debugreg(d7, 7);
110 111 112 113 114 115 116

	/* Only print out debug registers if they are in their non-default state. */
	if ((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
	    (d6 == DR6_RESERVED) && (d7 == 0x400))
		return;

	printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
117
	printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
118

L
Linus Torvalds 已提交
119 120 121 122 123 124
}

void release_thread(struct task_struct *dead_task)
{
	if (dead_task->mm) {
		if (dead_task->mm->context.size) {
125
			pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
126 127 128
				dead_task->comm,
				dead_task->mm->context.ldt,
				dead_task->mm->context.size);
L
Linus Torvalds 已提交
129 130 131 132 133 134 135
			BUG();
		}
	}
}

static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
{
136
	struct user_desc ud = {
L
Linus Torvalds 已提交
137 138 139 140 141 142
		.base_addr = addr,
		.limit = 0xfffff,
		.seg_32bit = 1,
		.limit_in_pages = 1,
		.useable = 1,
	};
J
Jan Engelhardt 已提交
143
	struct desc_struct *desc = t->thread.tls_array;
L
Linus Torvalds 已提交
144
	desc += tls;
145
	fill_ldt(desc, &ud);
L
Linus Torvalds 已提交
146 147 148 149
}

static inline u32 read_32bit_tls(struct task_struct *t, int tls)
{
R
Roland McGrath 已提交
150
	return get_desc_base(&t->thread.tls_array[tls]);
L
Linus Torvalds 已提交
151 152
}

A
Alexey Dobriyan 已提交
153
int copy_thread(unsigned long clone_flags, unsigned long sp,
154
		unsigned long arg, struct task_struct *p)
L
Linus Torvalds 已提交
155 156
{
	int err;
157
	struct pt_regs *childregs;
L
Linus Torvalds 已提交
158 159
	struct task_struct *me = current;

A
Al Viro 已提交
160 161
	p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
	childregs = task_pt_regs(p);
162
	p->thread.sp = (unsigned long) childregs;
A
Al Viro 已提交
163
	set_tsk_thread_flag(p, TIF_FORK);
164
	p->thread.io_bitmap_ptr = NULL;
L
Linus Torvalds 已提交
165

166
	savesegment(gs, p->thread.gsindex);
167
	p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
168
	savesegment(fs, p->thread.fsindex);
169
	p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
170 171
	savesegment(es, p->thread.es);
	savesegment(ds, p->thread.ds);
A
Al Viro 已提交
172 173
	memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));

174
	if (unlikely(p->flags & PF_KTHREAD)) {
A
Al Viro 已提交
175 176 177 178 179 180 181 182
		/* kernel thread */
		memset(childregs, 0, sizeof(struct pt_regs));
		childregs->sp = (unsigned long)childregs;
		childregs->ss = __KERNEL_DS;
		childregs->bx = sp; /* function */
		childregs->bp = arg;
		childregs->orig_ax = -1;
		childregs->cs = __KERNEL_CS | get_kernel_rpl();
183
		childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_FIXED;
A
Al Viro 已提交
184 185
		return 0;
	}
186
	*childregs = *current_pt_regs();
A
Al Viro 已提交
187 188

	childregs->ax = 0;
189 190
	if (sp)
		childregs->sp = sp;
L
Linus Torvalds 已提交
191

192
	err = -ENOMEM;
193
	if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
194 195
		p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
						  IO_BITMAP_BYTES, GFP_KERNEL);
L
Linus Torvalds 已提交
196 197 198 199
		if (!p->thread.io_bitmap_ptr) {
			p->thread.io_bitmap_max = 0;
			return -ENOMEM;
		}
200
		set_tsk_thread_flag(p, TIF_IO_BITMAP);
201
	}
L
Linus Torvalds 已提交
202 203 204 205 206 207

	/*
	 * Set a new TLS for the child thread?
	 */
	if (clone_flags & CLONE_SETTLS) {
#ifdef CONFIG_IA32_EMULATION
208
		if (is_ia32_task())
R
Roland McGrath 已提交
209
			err = do_set_thread_area(p, -1,
210
				(struct user_desc __user *)childregs->si, 0);
211 212 213 214
		else
#endif
			err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
		if (err)
L
Linus Torvalds 已提交
215 216 217 218 219 220 221 222
			goto out;
	}
	err = 0;
out:
	if (err && p->thread.io_bitmap_ptr) {
		kfree(p->thread.io_bitmap_ptr);
		p->thread.io_bitmap_max = 0;
	}
223

L
Linus Torvalds 已提交
224 225 226
	return err;
}

227 228 229 230
static void
start_thread_common(struct pt_regs *regs, unsigned long new_ip,
		    unsigned long new_sp,
		    unsigned int _cs, unsigned int _ss, unsigned int _ds)
I
Ingo Molnar 已提交
231
{
232
	loadsegment(fs, 0);
233 234
	loadsegment(es, _ds);
	loadsegment(ds, _ds);
I
Ingo Molnar 已提交
235 236 237
	load_gs_index(0);
	regs->ip		= new_ip;
	regs->sp		= new_sp;
238 239
	regs->cs		= _cs;
	regs->ss		= _ss;
240
	regs->flags		= X86_EFLAGS_IF;
241
	force_iret();
I
Ingo Molnar 已提交
242
}
243 244 245 246 247 248 249

void
start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
{
	start_thread_common(regs, new_ip, new_sp,
			    __USER_CS, __USER_DS, 0);
}
I
Ingo Molnar 已提交
250

251 252 253
#ifdef CONFIG_IA32_EMULATION
void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp)
{
254
	start_thread_common(regs, new_ip, new_sp,
H
H. Peter Anvin 已提交
255 256 257
			    test_thread_flag(TIF_X32)
			    ? __USER_CS : __USER32_CS,
			    __USER_DS, __USER_DS);
258 259
}
#endif
I
Ingo Molnar 已提交
260

L
Linus Torvalds 已提交
261 262 263
/*
 *	switch_to(x,y) should switch tasks from x to y.
 *
264
 * This could still be optimized:
L
Linus Torvalds 已提交
265 266
 * - fold all the options into a flag word and test it with a single test.
 * - could test fs/gs bitsliced
267 268
 *
 * Kprobes not supported here. Set the probe on schedule instead.
269
 * Function graph tracer not supported too.
L
Linus Torvalds 已提交
270
 */
271
__visible __notrace_funcgraph struct task_struct *
272
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
L
Linus Torvalds 已提交
273
{
274 275
	struct thread_struct *prev = &prev_p->thread;
	struct thread_struct *next = &next_p->thread;
276 277
	struct fpu *prev_fpu = &prev->fpu;
	struct fpu *next_fpu = &next->fpu;
278
	int cpu = smp_processor_id();
279
	struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
280
	unsigned fsindex, gsindex;
281
	fpu_switch_t fpu_switch;
282

283
	fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
284

285 286 287 288 289 290 291 292
	/* We must save %fs and %gs before load_TLS() because
	 * %fs and %gs may be cleared by load_TLS().
	 *
	 * (e.g. xen_load_tls())
	 */
	savesegment(fs, fsindex);
	savesegment(gs, gsindex);

293 294 295 296
	/*
	 * Load TLS before restoring any segments so that segment loads
	 * reference the correct GDT entries.
	 */
L
Linus Torvalds 已提交
297 298
	load_TLS(next, cpu);

299
	/*
300 301 302
	 * Leave lazy mode, flushing any hypercalls made here.  This
	 * must be done after loading TLS entries in the GDT but before
	 * loading segments that might reference them, and and it must
303
	 * be done before fpu__restore(), so the TS bit is up to
304
	 * date.
305
	 */
306
	arch_end_context_switch(next_p);
307

308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
	/* Switch DS and ES.
	 *
	 * Reading them only returns the selectors, but writing them (if
	 * nonzero) loads the full descriptor from the GDT or LDT.  The
	 * LDT for next is loaded in switch_mm, and the GDT is loaded
	 * above.
	 *
	 * We therefore need to write new values to the segment
	 * registers on every context switch unless both the new and old
	 * values are zero.
	 *
	 * Note that we don't need to do anything for CS and SS, as
	 * those are saved and restored as part of pt_regs.
	 */
	savesegment(es, prev->es);
	if (unlikely(next->es | prev->es))
		loadsegment(es, next->es);

	savesegment(ds, prev->ds);
	if (unlikely(next->ds | prev->ds))
		loadsegment(ds, next->ds);

330
	/*
L
Linus Torvalds 已提交
331
	 * Switch FS and GS.
332
	 *
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
	 * These are even more complicated than FS and GS: they have
	 * 64-bit bases are that controlled by arch_prctl.  Those bases
	 * only differ from the values in the GDT or LDT if the selector
	 * is 0.
	 *
	 * Loading the segment register resets the hidden base part of
	 * the register to 0 or the value from the GDT / LDT.  If the
	 * next base address zero, writing 0 to the segment register is
	 * much faster than using wrmsr to explicitly zero the base.
	 *
	 * The thread_struct.fs and thread_struct.gs values are 0
	 * if the fs and gs bases respectively are not overridden
	 * from the values implied by fsindex and gsindex.  They
	 * are nonzero, and store the nonzero base addresses, if
	 * the bases are overridden.
	 *
	 * (fs != 0 && fsindex != 0) || (gs != 0 && gsindex != 0) should
	 * be impossible.
	 *
	 * Therefore we need to reload the segment registers if either
	 * the old or new selector is nonzero, and we need to override
	 * the base address if next thread expects it to be overridden.
	 *
	 * This code is unnecessarily slow in the case where the old and
	 * new indexes are zero and the new base is nonzero -- it will
	 * unnecessarily write 0 to the selector before writing the new
	 * base address.
	 *
	 * Note: This all depends on arch_prctl being the only way that
	 * user code can override the segment base.  Once wrfsbase and
	 * wrgsbase are enabled, most of this code will need to change.
L
Linus Torvalds 已提交
364
	 */
365 366
	if (unlikely(fsindex | next->fsindex | prev->fs)) {
		loadsegment(fs, next->fsindex);
367

368
		/*
369 370 371 372 373 374 375
		 * If user code wrote a nonzero value to FS, then it also
		 * cleared the overridden base address.
		 *
		 * XXX: if user code wrote 0 to FS and cleared the base
		 * address itself, we won't notice and we'll incorrectly
		 * restore the prior base address next time we reschdule
		 * the process.
376 377
		 */
		if (fsindex)
378
			prev->fs = 0;
L
Linus Torvalds 已提交
379
	}
380 381 382 383 384 385
	if (next->fs)
		wrmsrl(MSR_FS_BASE, next->fs);
	prev->fsindex = fsindex;

	if (unlikely(gsindex | next->gsindex | prev->gs)) {
		load_gs_index(next->gsindex);
386 387

		/* This works (and fails) the same way as fsindex above. */
388
		if (gsindex)
389
			prev->gs = 0;
L
Linus Torvalds 已提交
390
	}
391 392 393
	if (next->gs)
		wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
	prev->gsindex = gsindex;
L
Linus Torvalds 已提交
394

395
	switch_fpu_finish(next_fpu, fpu_switch);
396

397
	/*
398
	 * Switch the PDA and FPU contexts.
L
Linus Torvalds 已提交
399
	 */
400
	this_cpu_write(current_task, next_p);
401

402 403 404 405 406 407 408 409
	/*
	 * If it were not for PREEMPT_ACTIVE we could guarantee that the
	 * preempt_count of all tasks was equal here and this would not be
	 * needed.
	 */
	task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
	this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);

410 411 412
	/* Reload esp0 and ss1.  This changes current_thread_info(). */
	load_sp0(tss, next);

413
	this_cpu_write(kernel_stack,
414
		(unsigned long)task_stack_page(next_p) + THREAD_SIZE);
L
Linus Torvalds 已提交
415 416

	/*
417
	 * Now maybe reload the debug registers and handle I/O bitmaps
L
Linus Torvalds 已提交
418
	 */
419 420
	if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
		     task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
421
		__switch_to_xtra(prev_p, next_p, tss);
L
Linus Torvalds 已提交
422

423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
	if (static_cpu_has_bug(X86_BUG_SYSRET_SS_ATTRS)) {
		/*
		 * AMD CPUs have a misfeature: SYSRET sets the SS selector but
		 * does not update the cached descriptor.  As a result, if we
		 * do SYSRET while SS is NULL, we'll end up in user mode with
		 * SS apparently equal to __USER_DS but actually unusable.
		 *
		 * The straightforward workaround would be to fix it up just
		 * before SYSRET, but that would slow down the system call
		 * fast paths.  Instead, we ensure that SS is never NULL in
		 * system call context.  We do this by replacing NULL SS
		 * selectors at every context switch.  SYSCALL sets up a valid
		 * SS, so the only way to get NULL is to re-enter the kernel
		 * from CPL 3 through an interrupt.  Since that can't happen
		 * in the same task as a running syscall, we are guaranteed to
		 * context switch between every interrupt vector entry and a
		 * subsequent SYSRET.
		 *
		 * We read SS first because SS reads are much faster than
		 * writes.  Out of caution, we force SS to __KERNEL_DS even if
		 * it previously had a different non-NULL value.
		 */
		unsigned short ss_sel;
		savesegment(ss, ss_sel);
		if (ss_sel != __KERNEL_DS)
			loadsegment(ss, __KERNEL_DS);
	}

L
Linus Torvalds 已提交
451 452 453 454 455 456 457 458
	return prev_p;
}

void set_personality_64bit(void)
{
	/* inherit personality from parent */

	/* Make sure to be in 64bit mode */
459
	clear_thread_flag(TIF_IA32);
460
	clear_thread_flag(TIF_ADDR32);
461
	clear_thread_flag(TIF_X32);
L
Linus Torvalds 已提交
462

463 464 465 466
	/* Ensure the corresponding mm is not marked. */
	if (current->mm)
		current->mm->context.ia32_compat = 0;

L
Linus Torvalds 已提交
467 468 469
	/* TBD: overwrites user setup. Should have two bits.
	   But 64bit processes have always behaved this way,
	   so it's not too bad. The main problem is just that
470
	   32bit childs are affected again. */
L
Linus Torvalds 已提交
471 472 473
	current->personality &= ~READ_IMPLIES_EXEC;
}

H
H. Peter Anvin 已提交
474
void set_personality_ia32(bool x32)
475 476 477 478
{
	/* inherit personality from parent */

	/* Make sure to be in 32bit mode */
479
	set_thread_flag(TIF_ADDR32);
480

481
	/* Mark the associated mm as containing 32-bit tasks. */
H
H. Peter Anvin 已提交
482 483 484
	if (x32) {
		clear_thread_flag(TIF_IA32);
		set_thread_flag(TIF_X32);
485 486
		if (current->mm)
			current->mm->context.ia32_compat = TIF_X32;
H
H. Peter Anvin 已提交
487
		current->personality &= ~READ_IMPLIES_EXEC;
488 489 490
		/* is_compat_task() uses the presence of the x32
		   syscall bit flag to determine compat status */
		current_thread_info()->status &= ~TS_COMPAT;
H
H. Peter Anvin 已提交
491 492 493
	} else {
		set_thread_flag(TIF_IA32);
		clear_thread_flag(TIF_X32);
494 495
		if (current->mm)
			current->mm->context.ia32_compat = TIF_IA32;
H
H. Peter Anvin 已提交
496 497 498 499
		current->personality |= force_personality32;
		/* Prepare the first "return" to user space */
		current_thread_info()->status |= TS_COMPAT;
	}
500
}
501
EXPORT_SYMBOL_GPL(set_personality_ia32);
502

L
Linus Torvalds 已提交
503 504 505
unsigned long get_wchan(struct task_struct *p)
{
	unsigned long stack;
506
	u64 fp, ip;
L
Linus Torvalds 已提交
507 508
	int count = 0;

509 510
	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;
A
Al Viro 已提交
511
	stack = (unsigned long)task_stack_page(p);
512
	if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
L
Linus Torvalds 已提交
513
		return 0;
514
	fp = *(u64 *)(p->thread.sp);
515
	do {
516
		if (fp < (unsigned long)stack ||
517
		    fp >= (unsigned long)stack+THREAD_SIZE)
518
			return 0;
519 520 521
		ip = *(u64 *)(fp+8);
		if (!in_sched_functions(ip))
			return ip;
522 523
		fp = *(u64 *)fp;
	} while (count++ < 16);
L
Linus Torvalds 已提交
524 525 526 527
	return 0;
}

long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
528 529
{
	int ret = 0;
L
Linus Torvalds 已提交
530 531 532
	int doit = task == current;
	int cpu;

533
	switch (code) {
L
Linus Torvalds 已提交
534
	case ARCH_SET_GS:
535
		if (addr >= TASK_SIZE_OF(task))
536
			return -EPERM;
L
Linus Torvalds 已提交
537
		cpu = get_cpu();
538
		/* handle small bases via the GDT because that's faster to
L
Linus Torvalds 已提交
539
		   switch. */
540 541 542
		if (addr <= 0xffffffff) {
			set_32bit_tls(task, GS_TLS, addr);
			if (doit) {
L
Linus Torvalds 已提交
543
				load_TLS(&task->thread, cpu);
544
				load_gs_index(GS_TLS_SEL);
L
Linus Torvalds 已提交
545
			}
546
			task->thread.gsindex = GS_TLS_SEL;
L
Linus Torvalds 已提交
547
			task->thread.gs = 0;
548
		} else {
L
Linus Torvalds 已提交
549 550 551
			task->thread.gsindex = 0;
			task->thread.gs = addr;
			if (doit) {
552
				load_gs_index(0);
553
				ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr);
554
			}
L
Linus Torvalds 已提交
555 556 557 558 559 560
		}
		put_cpu();
		break;
	case ARCH_SET_FS:
		/* Not strictly needed for fs, but do it for symmetry
		   with gs */
561
		if (addr >= TASK_SIZE_OF(task))
562
			return -EPERM;
L
Linus Torvalds 已提交
563
		cpu = get_cpu();
564
		/* handle small bases via the GDT because that's faster to
L
Linus Torvalds 已提交
565
		   switch. */
566
		if (addr <= 0xffffffff) {
L
Linus Torvalds 已提交
567
			set_32bit_tls(task, FS_TLS, addr);
568 569
			if (doit) {
				load_TLS(&task->thread, cpu);
570
				loadsegment(fs, FS_TLS_SEL);
L
Linus Torvalds 已提交
571 572 573
			}
			task->thread.fsindex = FS_TLS_SEL;
			task->thread.fs = 0;
574
		} else {
L
Linus Torvalds 已提交
575 576 577 578 579
			task->thread.fsindex = 0;
			task->thread.fs = addr;
			if (doit) {
				/* set the selector to 0 to not confuse
				   __switch_to */
580
				loadsegment(fs, 0);
581
				ret = wrmsrl_safe(MSR_FS_BASE, addr);
L
Linus Torvalds 已提交
582 583 584 585
			}
		}
		put_cpu();
		break;
586 587
	case ARCH_GET_FS: {
		unsigned long base;
L
Linus Torvalds 已提交
588 589
		if (task->thread.fsindex == FS_TLS_SEL)
			base = read_32bit_tls(task, FS_TLS);
590
		else if (doit)
L
Linus Torvalds 已提交
591
			rdmsrl(MSR_FS_BASE, base);
592
		else
L
Linus Torvalds 已提交
593
			base = task->thread.fs;
594 595
		ret = put_user(base, (unsigned long __user *)addr);
		break;
L
Linus Torvalds 已提交
596
	}
597
	case ARCH_GET_GS: {
L
Linus Torvalds 已提交
598
		unsigned long base;
599
		unsigned gsindex;
L
Linus Torvalds 已提交
600 601
		if (task->thread.gsindex == GS_TLS_SEL)
			base = read_32bit_tls(task, GS_TLS);
602
		else if (doit) {
603
			savesegment(gs, gsindex);
604 605 606 607
			if (gsindex)
				rdmsrl(MSR_KERNEL_GS_BASE, base);
			else
				base = task->thread.gs;
608
		} else
L
Linus Torvalds 已提交
609
			base = task->thread.gs;
610
		ret = put_user(base, (unsigned long __user *)addr);
L
Linus Torvalds 已提交
611 612 613 614 615 616
		break;
	}

	default:
		ret = -EINVAL;
		break;
617
	}
L
Linus Torvalds 已提交
618

619 620
	return ret;
}
L
Linus Torvalds 已提交
621 622 623 624 625 626

long sys_arch_prctl(int code, unsigned long addr)
{
	return do_arch_prctl(current, code, addr);
}

627 628
unsigned long KSTK_ESP(struct task_struct *task)
{
629
	return task_pt_regs(task)->sp;
630
}