process.c 15.6 KB
Newer Older
1 2 3 4
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
5
#include <linux/prctl.h>
6 7
#include <linux/slab.h>
#include <linux/sched.h>
8 9
#include <linux/module.h>
#include <linux/pm.h>
10
#include <linux/clockchips.h>
A
Amerigo Wang 已提交
11
#include <linux/random.h>
A
Avi Kivity 已提交
12
#include <linux/user-return-notifier.h>
13 14
#include <linux/dmi.h>
#include <linux/utsname.h>
15
#include <trace/events/power.h>
16
#include <linux/hw_breakpoint.h>
17
#include <asm/cpu.h>
Z
Zhao Yakui 已提交
18
#include <asm/system.h>
19
#include <asm/apic.h>
20
#include <asm/syscalls.h>
21 22 23
#include <asm/idle.h>
#include <asm/uaccess.h>
#include <asm/i387.h>
24
#include <asm/debugreg.h>
Z
Zhao Yakui 已提交
25

26
struct kmem_cache *task_xstate_cachep;
S
Sheng Yang 已提交
27
EXPORT_SYMBOL_GPL(task_xstate_cachep);
28 29 30

int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
31 32
	int ret;

33
	*dst = *src;
34 35 36 37 38 39
	if (fpu_allocated(&src->thread.fpu)) {
		memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu));
		ret = fpu_alloc(&dst->thread.fpu);
		if (ret)
			return ret;
		fpu_copy(&dst->thread.fpu, &src->thread.fpu);
40
	}
41 42 43
	return 0;
}

44
void free_thread_xstate(struct task_struct *tsk)
45
{
46
	fpu_free(&tsk->thread.fpu);
47 48 49 50 51
}

void free_thread_info(struct thread_info *ti)
{
	free_thread_xstate(ti->task);
S
Suresh Siddha 已提交
52
	free_pages((unsigned long)ti, get_order(THREAD_SIZE));
53 54 55 56 57 58 59
}

void arch_task_cache_init(void)
{
        task_xstate_cachep =
        	kmem_cache_create("task_xstate", xstate_size,
				  __alignof__(union thread_xstate),
V
Vegard Nossum 已提交
60
				  SLAB_PANIC | SLAB_NOTRACK, NULL);
61
}
62

63 64 65 66 67 68 69
/*
 * Free current thread data structures etc..
 */
void exit_thread(void)
{
	struct task_struct *me = current;
	struct thread_struct *t = &me->thread;
70
	unsigned long *bp = t->io_bitmap_ptr;
71

72
	if (bp) {
73 74 75 76 77 78 79 80 81 82
		struct tss_struct *tss = &per_cpu(init_tss, get_cpu());

		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
		t->io_bitmap_max = 0;
		put_cpu();
83
		kfree(bp);
84 85 86
	}
}

B
Brian Gerst 已提交
87 88 89
void show_regs(struct pt_regs *regs)
{
	show_registers(regs);
90
	show_trace(NULL, regs, (unsigned long *)kernel_stack_pointer(regs));
B
Brian Gerst 已提交
91 92
}

93 94
void show_regs_common(void)
{
95
	const char *board, *product;
96

97
	board = dmi_get_system_info(DMI_BOARD_NAME);
98 99
	if (!board)
		board = "";
100 101 102
	product = dmi_get_system_info(DMI_PRODUCT_NAME);
	if (!product)
		product = "";
103

104 105
	printk(KERN_CONT "\n");
	printk(KERN_DEFAULT "Pid: %d, comm: %.20s %s %s %.*s %s/%s\n",
106 107 108
		current->pid, current->comm, print_tainted(),
		init_utsname()->release,
		(int)strcspn(init_utsname()->version, " "),
109
		init_utsname()->version, board, product);
110 111
}

112 113 114 115
void flush_thread(void)
{
	struct task_struct *tsk = current;

116
	flush_ptrace_hw_breakpoint(tsk);
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
	/*
	 * Forget coprocessor state..
	 */
	tsk->fpu_counter = 0;
	clear_fpu(tsk);
	clear_used_math();
}

static void hard_disable_TSC(void)
{
	write_cr4(read_cr4() | X86_CR4_TSD);
}

void disable_TSC(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		hard_disable_TSC();
	preempt_enable();
}

static void hard_enable_TSC(void)
{
	write_cr4(read_cr4() & ~X86_CR4_TSD);
}

static void enable_TSC(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		hard_enable_TSC();
	preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
	unsigned int val;

	if (test_thread_flag(TIF_NOTSC))
		val = PR_TSC_SIGSEGV;
	else
		val = PR_TSC_ENABLE;

	return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
	if (val == PR_TSC_SIGSEGV)
		disable_TSC();
	else if (val == PR_TSC_ENABLE)
		enable_TSC();
	else
		return -EINVAL;

	return 0;
}

void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss)
{
	struct thread_struct *prev, *next;

	prev = &prev_p->thread;
	next = &next_p->thread;

P
Peter Zijlstra 已提交
192 193 194 195 196 197 198 199 200 201
	if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
	    test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl &= ~DEBUGCTLMSR_BTF;
		if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
			debugctl |= DEBUGCTLMSR_BTF;

		update_debugctlmsr(debugctl);
	}
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224

	if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
	    test_tsk_thread_flag(next_p, TIF_NOTSC)) {
		/* prev and next are different */
		if (test_tsk_thread_flag(next_p, TIF_NOTSC))
			hard_disable_TSC();
		else
			hard_enable_TSC();
	}

	if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
		/*
		 * Copy the relevant range of the IO bitmap.
		 * Normally this is 128 bytes or less:
		 */
		memcpy(tss->io_bitmap, next->io_bitmap_ptr,
		       max(prev->io_bitmap_max, next->io_bitmap_max));
	} else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
		/*
		 * Clear any possible leftover bits:
		 */
		memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	}
A
Avi Kivity 已提交
225
	propagate_user_return_notify(prev_p, next_p);
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
}

int sys_fork(struct pt_regs *regs)
{
	return do_fork(SIGCHLD, regs->sp, regs, 0, NULL, NULL);
}

/*
 * This is trivial, and on the face of it looks like it
 * could equally well be done in user mode.
 *
 * Not so, for quite unobvious reasons - register pressure.
 * In user mode vfork() cannot have a stack frame, and if
 * done by calling the "clone()" system call directly, you
 * do not have enough call-clobbered registers to hold all
 * the information you need.
 */
int sys_vfork(struct pt_regs *regs)
{
	return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->sp, regs, 0,
		       NULL, NULL);
}

B
Brian Gerst 已提交
249 250 251 252 253 254 255 256 257
long
sys_clone(unsigned long clone_flags, unsigned long newsp,
	  void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
{
	if (!newsp)
		newsp = regs->sp;
	return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
}

B
Brian Gerst 已提交
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
/*
 * This gets run with %si containing the
 * function to call, and %di containing
 * the "args".
 */
extern void kernel_thread_helper(void);

/*
 * Create a kernel thread
 */
int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
{
	struct pt_regs regs;

	memset(&regs, 0, sizeof(regs));

	regs.si = (unsigned long) fn;
	regs.di = (unsigned long) arg;

#ifdef CONFIG_X86_32
	regs.ds = __USER_DS;
	regs.es = __USER_DS;
	regs.fs = __KERNEL_PERCPU;
	regs.gs = __KERNEL_STACK_CANARY;
282 283
#else
	regs.ss = __KERNEL_DS;
B
Brian Gerst 已提交
284 285 286 287 288 289 290 291 292 293 294
#endif

	regs.orig_ax = -1;
	regs.ip = (unsigned long) kernel_thread_helper;
	regs.cs = __KERNEL_CS | get_kernel_rpl();
	regs.flags = X86_EFLAGS_IF | 0x2;

	/* Ok, create the new process.. */
	return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, &regs, 0, NULL, NULL);
}
EXPORT_SYMBOL(kernel_thread);
295

B
Brian Gerst 已提交
296 297 298
/*
 * sys_execve() executes a new program.
 */
299 300 301
long sys_execve(const char __user *name,
		const char __user *const __user *argv,
		const char __user *const __user *envp, struct pt_regs *regs)
B
Brian Gerst 已提交
302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
{
	long error;
	char *filename;

	filename = getname(name);
	error = PTR_ERR(filename);
	if (IS_ERR(filename))
		return error;
	error = do_execve(filename, argv, envp, regs);

#ifdef CONFIG_X86_32
	if (error == 0) {
		/* Make sure we don't return using sysenter.. */
                set_thread_flag(TIF_IRET);
        }
#endif

	putname(filename);
	return error;
}
322

323 324 325
/*
 * Idle related variables and functions
 */
326
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
EXPORT_SYMBOL(boot_option_idle_override);

/*
 * Powermanagement idle function, if any..
 */
void (*pm_idle)(void);
EXPORT_SYMBOL(pm_idle);

#ifdef CONFIG_X86_32
/*
 * This halt magic was a workaround for ancient floppy DMA
 * wreckage. It should be safe to remove.
 */
static int hlt_counter;
void disable_hlt(void)
{
	hlt_counter++;
}
EXPORT_SYMBOL(disable_hlt);

void enable_hlt(void)
{
	hlt_counter--;
}
EXPORT_SYMBOL(enable_hlt);

static inline int hlt_use_halt(void)
{
	return (!hlt_counter && boot_cpu_data.hlt_works_ok);
}
#else
static inline int hlt_use_halt(void)
{
	return 1;
}
#endif

/*
 * We use this if we don't have any better
 * idle routine..
 */
void default_idle(void)
{
	if (hlt_use_halt()) {
371
		trace_power_start(POWER_CSTATE, 1, smp_processor_id());
372
		trace_cpu_idle(1, smp_processor_id());
373 374 375 376 377 378 379 380 381 382 383 384
		current_thread_info()->status &= ~TS_POLLING;
		/*
		 * TS_POLLING-cleared state must be visible before we
		 * test NEED_RESCHED:
		 */
		smp_mb();

		if (!need_resched())
			safe_halt();	/* enables interrupts racelessly */
		else
			local_irq_enable();
		current_thread_info()->status |= TS_POLLING;
385 386
		trace_power_end(smp_processor_id());
		trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
387 388 389 390 391 392 393 394 395 396
	} else {
		local_irq_enable();
		/* loop is done by the caller */
		cpu_relax();
	}
}
#ifdef CONFIG_APM_MODULE
EXPORT_SYMBOL(default_idle);
#endif

397 398 399 400 401 402
void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
403
	set_cpu_online(smp_processor_id(), false);
404 405 406 407 408 409 410 411
	disable_local_APIC();

	for (;;) {
		if (hlt_works(smp_processor_id()))
			halt();
	}
}

412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
static void do_nothing(void *unused)
{
}

/*
 * cpu_idle_wait - Used to ensure that all the CPUs discard old value of
 * pm_idle and update to new pm_idle value. Required while changing pm_idle
 * handler on SMP systems.
 *
 * Caller must have changed pm_idle to the new value before the call. Old
 * pm_idle value will not be used by any CPU after the return of this function.
 */
void cpu_idle_wait(void)
{
	smp_mb();
	/* kick all the CPUs so that they exit out of pm_idle */
428
	smp_call_function(do_nothing, NULL, 1);
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
}
EXPORT_SYMBOL_GPL(cpu_idle_wait);

/*
 * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
 * which can obviate IPI to trigger checking of need_resched.
 * We execute MONITOR against need_resched and enter optimized wait state
 * through MWAIT. Whenever someone changes need_resched, we would be woken
 * up from MWAIT (without an IPI).
 *
 * New with Core Duo processors, MWAIT can take some hints based on CPU
 * capability.
 */
void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
{
	if (!need_resched()) {
445
		if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_CLFLUSH_MONITOR))
446 447
			clflush((void *)&current_thread_info()->flags);

448 449 450 451 452 453 454 455 456 457 458
		__monitor((void *)&current_thread_info()->flags, 0, 0);
		smp_mb();
		if (!need_resched())
			__mwait(ax, cx);
	}
}

/* Default MONITOR/MWAIT with no hints, used for default C1 state */
static void mwait_idle(void)
{
	if (!need_resched()) {
459
		trace_power_start(POWER_CSTATE, 1, smp_processor_id());
460
		trace_cpu_idle(1, smp_processor_id());
461
		if (cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_CLFLUSH_MONITOR))
462 463
			clflush((void *)&current_thread_info()->flags);

464 465 466 467 468 469
		__monitor((void *)&current_thread_info()->flags, 0, 0);
		smp_mb();
		if (!need_resched())
			__sti_mwait(0, 0);
		else
			local_irq_enable();
470 471
		trace_power_end(smp_processor_id());
		trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
472 473 474 475 476 477 478 479 480 481 482
	} else
		local_irq_enable();
}

/*
 * On SMP it's slightly faster (but much more power-consuming!)
 * to poll the ->work.need_resched flag instead of waiting for the
 * cross-CPU IPI to arrive. Use this option with caution.
 */
static void poll_idle(void)
{
483
	trace_power_start(POWER_CSTATE, 0, smp_processor_id());
484
	trace_cpu_idle(0, smp_processor_id());
485
	local_irq_enable();
486 487
	while (!need_resched())
		cpu_relax();
488 489
	trace_power_end(smp_processor_id());
	trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
490 491
}

492 493 494 495 496 497 498 499 500 501 502 503
/*
 * mwait selection logic:
 *
 * It depends on the CPU. For AMD CPUs that support MWAIT this is
 * wrong. Family 0x10 and 0x11 CPUs will enter C1 on HLT. Powersavings
 * then depend on a clock divisor and current Pstate of the core. If
 * all cores of a processor are in halt state (C1) the processor can
 * enter the C1E (C1 enhanced) state. If mwait is used this will never
 * happen.
 *
 * idle=mwait overrides this decision and forces the usage of mwait.
 */
504 505 506 507 508

#define MWAIT_INFO			0x05
#define MWAIT_ECX_EXTENDED_INFO		0x01
#define MWAIT_EDX_C1			0xf0

509
int __cpuinit mwait_usable(const struct cpuinfo_x86 *c)
510
{
511 512
	u32 eax, ebx, ecx, edx;

513
	if (boot_option_idle_override == IDLE_FORCE_MWAIT)
514 515
		return 1;

516 517 518 519 520 521 522 523 524 525 526 527 528
	if (c->cpuid_level < MWAIT_INFO)
		return 0;

	cpuid(MWAIT_INFO, &eax, &ebx, &ecx, &edx);
	/* Check, whether EDX has extended info about MWAIT */
	if (!(ecx & MWAIT_ECX_EXTENDED_INFO))
		return 1;

	/*
	 * edx enumeratios MONITOR/MWAIT extensions. Check, whether
	 * C1  supports MWAIT
	 */
	return (edx & MWAIT_EDX_C1);
529 530
}

531 532
bool c1e_detected;
EXPORT_SYMBOL(c1e_detected);
533

534
static cpumask_var_t c1e_mask;
535 536 537

void c1e_remove_cpu(int cpu)
{
538 539
	if (c1e_mask != NULL)
		cpumask_clear_cpu(cpu, c1e_mask);
540 541
}

542 543 544 545 546 547 548 549 550 551 552 553 554 555
/*
 * C1E aware idle routine. We check for C1E active in the interrupt
 * pending message MSR. If we detect C1E, then we handle it the same
 * way as C3 power states (local apic timer and TSC stop)
 */
static void c1e_idle(void)
{
	if (need_resched())
		return;

	if (!c1e_detected) {
		u32 lo, hi;

		rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
556

557
		if (lo & K8_INTP_C1E_ACTIVE_MASK) {
558
			c1e_detected = true;
559
			if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
560 561
				mark_tsc_unstable("TSC halt in AMD C1E");
			printk(KERN_INFO "System has AMD C1E enabled\n");
562 563 564 565 566 567
		}
	}

	if (c1e_detected) {
		int cpu = smp_processor_id();

568 569
		if (!cpumask_test_cpu(cpu, c1e_mask)) {
			cpumask_set_cpu(cpu, c1e_mask);
570
			/*
571
			 * Force broadcast so ACPI can not interfere.
572
			 */
573 574 575 576 577 578
			clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE,
					   &cpu);
			printk(KERN_INFO "Switch to broadcast mode on CPU%d\n",
			       cpu);
		}
		clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
579

580
		default_idle();
581 582 583 584 585 586 587 588

		/*
		 * The switch back from broadcast mode needs to be
		 * called with interrupts disabled.
		 */
		 local_irq_disable();
		 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
		 local_irq_enable();
589 590 591 592
	} else
		default_idle();
}

593 594
void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
{
595
#ifdef CONFIG_SMP
596
	if (pm_idle == poll_idle && smp_num_siblings > 1) {
597
		printk_once(KERN_WARNING "WARNING: polling idle and HT enabled,"
598 599 600
			" performance may degrade.\n");
	}
#endif
T
Thomas Gleixner 已提交
601 602 603
	if (pm_idle)
		return;

604
	if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) {
605 606 607
		/*
		 * One CPU supports mwait => All CPUs supports mwait
		 */
T
Thomas Gleixner 已提交
608 609
		printk(KERN_INFO "using mwait in idle threads.\n");
		pm_idle = mwait_idle;
610 611
	} else if (cpu_has_amd_erratum(amd_erratum_400)) {
		/* E400: APIC timer interrupt does not wake up CPU from C1e */
612 613
		printk(KERN_INFO "using C1E aware idle routine\n");
		pm_idle = c1e_idle;
T
Thomas Gleixner 已提交
614 615
	} else
		pm_idle = default_idle;
616 617
}

618 619 620
void __init init_c1e_mask(void)
{
	/* If we're using c1e_idle, we need to allocate c1e_mask. */
621 622
	if (pm_idle == c1e_idle)
		zalloc_cpumask_var(&c1e_mask, GFP_KERNEL);
623 624
}

625 626
static int __init idle_setup(char *str)
{
627 628 629
	if (!str)
		return -EINVAL;

630 631 632
	if (!strcmp(str, "poll")) {
		printk("using polling idle threads.\n");
		pm_idle = poll_idle;
633 634 635 636
		boot_option_idle_override = IDLE_POLL;
	} else if (!strcmp(str, "mwait")) {
		boot_option_idle_override = IDLE_FORCE_MWAIT;
	} else if (!strcmp(str, "halt")) {
Z
Zhao Yakui 已提交
637 638 639 640 641 642 643 644
		/*
		 * When the boot option of idle=halt is added, halt is
		 * forced to be used for CPU idle. In such case CPU C2/C3
		 * won't be used again.
		 * To continue to load the CPU idle driver, don't touch
		 * the boot_option_idle_override.
		 */
		pm_idle = default_idle;
645
		boot_option_idle_override = IDLE_HALT;
646 647 648 649 650 651 652
	} else if (!strcmp(str, "nomwait")) {
		/*
		 * If the boot option of "idle=nomwait" is added,
		 * it means that mwait will be disabled for CPU C2/C3
		 * states. In such case it won't touch the variable
		 * of boot_option_idle_override.
		 */
653
		boot_option_idle_override = IDLE_NOMWAIT;
Z
Zhao Yakui 已提交
654
	} else
655 656 657 658 659 660
		return -1;

	return 0;
}
early_param("idle", idle_setup);

A
Amerigo Wang 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673
unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() % 8192;
	return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
	unsigned long range_end = mm->brk + 0x02000000;
	return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
}