process.c 13.1 KB
Newer Older
1 2
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

3 4 5 6
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
7
#include <linux/prctl.h>
8 9
#include <linux/slab.h>
#include <linux/sched.h>
10 11
#include <linux/module.h>
#include <linux/pm.h>
12
#include <linux/tick.h>
A
Amerigo Wang 已提交
13
#include <linux/random.h>
A
Avi Kivity 已提交
14
#include <linux/user-return-notifier.h>
15 16
#include <linux/dmi.h>
#include <linux/utsname.h>
17 18 19
#include <linux/stackprotector.h>
#include <linux/tick.h>
#include <linux/cpuidle.h>
20
#include <trace/events/power.h>
21
#include <linux/hw_breakpoint.h>
22
#include <asm/cpu.h>
23
#include <asm/apic.h>
24
#include <asm/syscalls.h>
25 26
#include <asm/idle.h>
#include <asm/uaccess.h>
27
#include <asm/mwait.h>
28
#include <asm/fpu/internal.h>
29
#include <asm/debugreg.h>
30
#include <asm/nmi.h>
A
Andy Lutomirski 已提交
31
#include <asm/tlbflush.h>
32
#include <asm/mce.h>
33
#include <asm/vm86.h>
34

T
Thomas Gleixner 已提交
35 36 37 38 39 40 41
/*
 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 * no more per-task TSS's. The TSS size is kept cacheline-aligned
 * so they are allowed to end up in the .data..cacheline_aligned
 * section. Since TSS's are completely CPU-local, we want them
 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 */
42 43
__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
	.x86_tss = {
44
		.sp0 = TOP_OF_INIT_STACK,
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
#ifdef CONFIG_X86_32
		.ss0 = __KERNEL_DS,
		.ss1 = __KERNEL_CS,
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
#endif
	 },
#ifdef CONFIG_X86_32
	 /*
	  * Note that the .io_bitmap member must be extra-big. This is because
	  * the CPU will access an additional byte beyond the end of the IO
	  * permission bitmap. The extra byte must be all 1 bits, and must
	  * be within the limit.
	  */
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },
#endif
60 61 62
#ifdef CONFIG_X86_32
	.SYSENTER_stack_canary	= STACK_END_MAGIC,
#endif
63
};
64
EXPORT_PER_CPU_SYMBOL(cpu_tss);
T
Thomas Gleixner 已提交
65

66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
#ifdef CONFIG_X86_64
static DEFINE_PER_CPU(unsigned char, is_idle);
static ATOMIC_NOTIFIER_HEAD(idle_notifier);

void idle_notifier_register(struct notifier_block *n)
{
	atomic_notifier_chain_register(&idle_notifier, n);
}
EXPORT_SYMBOL_GPL(idle_notifier_register);

void idle_notifier_unregister(struct notifier_block *n)
{
	atomic_notifier_chain_unregister(&idle_notifier, n);
}
EXPORT_SYMBOL_GPL(idle_notifier_unregister);
#endif
Z
Zhao Yakui 已提交
82

83 84 85 86
/*
 * this gets called so that we can store lazy state into memory and copy the
 * current task into the new thread.
 */
87 88
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
89
	memcpy(dst, src, arch_task_struct_size);
90 91 92
#ifdef CONFIG_VM86
	dst->thread.vm86 = NULL;
#endif
93

94
	return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
95
}
96

97 98 99
/*
 * Free current thread data structures etc..
 */
100
void exit_thread(struct task_struct *tsk)
101
{
102
	struct thread_struct *t = &tsk->thread;
103
	unsigned long *bp = t->io_bitmap_ptr;
104
	struct fpu *fpu = &t->fpu;
105

106
	if (bp) {
107
		struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
108 109 110 111 112 113 114 115 116

		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
		t->io_bitmap_max = 0;
		put_cpu();
117
		kfree(bp);
118
	}
119

120 121
	free_vm86(t);

122
	fpu__drop(fpu);
123 124 125 126 127 128
}

void flush_thread(void)
{
	struct task_struct *tsk = current;

129
	flush_ptrace_hw_breakpoint(tsk);
130
	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
131

132
	fpu__clear(&tsk->thread.fpu);
133 134 135 136
}

static void hard_disable_TSC(void)
{
A
Andy Lutomirski 已提交
137
	cr4_set_bits(X86_CR4_TSD);
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
}

void disable_TSC(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		hard_disable_TSC();
	preempt_enable();
}

static void hard_enable_TSC(void)
{
A
Andy Lutomirski 已提交
154
	cr4_clear_bits(X86_CR4_TSD);
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
}

static void enable_TSC(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		hard_enable_TSC();
	preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
	unsigned int val;

	if (test_thread_flag(TIF_NOTSC))
		val = PR_TSC_SIGSEGV;
	else
		val = PR_TSC_ENABLE;

	return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
	if (val == PR_TSC_SIGSEGV)
		disable_TSC();
	else if (val == PR_TSC_ENABLE)
		enable_TSC();
	else
		return -EINVAL;

	return 0;
}

void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss)
{
	struct thread_struct *prev, *next;

	prev = &prev_p->thread;
	next = &next_p->thread;

P
Peter Zijlstra 已提交
201 202 203 204 205 206 207 208 209 210
	if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
	    test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl &= ~DEBUGCTLMSR_BTF;
		if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
			debugctl |= DEBUGCTLMSR_BTF;

		update_debugctlmsr(debugctl);
	}
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233

	if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
	    test_tsk_thread_flag(next_p, TIF_NOTSC)) {
		/* prev and next are different */
		if (test_tsk_thread_flag(next_p, TIF_NOTSC))
			hard_disable_TSC();
		else
			hard_enable_TSC();
	}

	if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
		/*
		 * Copy the relevant range of the IO bitmap.
		 * Normally this is 128 bytes or less:
		 */
		memcpy(tss->io_bitmap, next->io_bitmap_ptr,
		       max(prev->io_bitmap_max, next->io_bitmap_max));
	} else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
		/*
		 * Clear any possible leftover bits:
		 */
		memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	}
A
Avi Kivity 已提交
234
	propagate_user_return_notify(prev_p, next_p);
235 236
}

237 238 239
/*
 * Idle related variables and functions
 */
240
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
241 242
EXPORT_SYMBOL(boot_option_idle_override);

243
static void (*x86_idle)(void);
244

245 246 247 248 249 250 251 252 253 254
#ifndef CONFIG_SMP
static inline void play_dead(void)
{
	BUG();
}
#endif

#ifdef CONFIG_X86_64
void enter_idle(void)
{
255
	this_cpu_write(is_idle, 1);
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
	atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}

static void __exit_idle(void)
{
	if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
		return;
	atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
}

/* Called from interrupts to signify idle end */
void exit_idle(void)
{
	/* idle loop has pid 0 */
	if (current->pid)
		return;
	__exit_idle();
}
#endif

T
Thomas Gleixner 已提交
276 277 278 279 280
void arch_cpu_idle_enter(void)
{
	local_touch_nmi();
	enter_idle();
}
281

T
Thomas Gleixner 已提交
282 283 284 285
void arch_cpu_idle_exit(void)
{
	__exit_idle();
}
286

T
Thomas Gleixner 已提交
287 288 289 290
void arch_cpu_idle_dead(void)
{
	play_dead();
}
291

T
Thomas Gleixner 已提交
292 293 294 295 296
/*
 * Called from the generic idle code.
 */
void arch_cpu_idle(void)
{
297
	x86_idle();
298 299
}

300
/*
T
Thomas Gleixner 已提交
301
 * We use this if we don't have any better idle routine..
302 303 304
 */
void default_idle(void)
{
305
	trace_cpu_idle_rcuidle(1, smp_processor_id());
T
Thomas Gleixner 已提交
306
	safe_halt();
307
	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
308
}
309
#ifdef CONFIG_APM_MODULE
310 311 312
EXPORT_SYMBOL(default_idle);
#endif

313 314
#ifdef CONFIG_XEN
bool xen_set_default_idle(void)
315
{
316
	bool ret = !!x86_idle;
317

318
	x86_idle = default_idle;
319 320 321

	return ret;
}
322
#endif
323 324 325 326 327 328
void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
329
	set_cpu_online(smp_processor_id(), false);
330
	disable_local_APIC();
331
	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
332

333 334
	for (;;)
		halt();
335 336
}

337 338
bool amd_e400_c1e_detected;
EXPORT_SYMBOL(amd_e400_c1e_detected);
339

340
static cpumask_var_t amd_e400_c1e_mask;
341

342
void amd_e400_remove_cpu(int cpu)
343
{
344 345
	if (amd_e400_c1e_mask != NULL)
		cpumask_clear_cpu(cpu, amd_e400_c1e_mask);
346 347
}

348
/*
349
 * AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt
350 351 352
 * pending message MSR. If we detect C1E, then we handle it the same
 * way as C3 power states (local apic timer and TSC stop)
 */
353
static void amd_e400_idle(void)
354
{
355
	if (!amd_e400_c1e_detected) {
356 357 358
		u32 lo, hi;

		rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
359

360
		if (lo & K8_INTP_C1E_ACTIVE_MASK) {
361
			amd_e400_c1e_detected = true;
362
			if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
363
				mark_tsc_unstable("TSC halt in AMD C1E");
364
			pr_info("System has AMD C1E enabled\n");
365 366 367
		}
	}

368
	if (amd_e400_c1e_detected) {
369 370
		int cpu = smp_processor_id();

371 372
		if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
			cpumask_set_cpu(cpu, amd_e400_c1e_mask);
373 374
			/* Force broadcast so ACPI can not interfere. */
			tick_broadcast_force();
375
			pr_info("Switch to broadcast mode on CPU%d\n", cpu);
376
		}
377
		tick_broadcast_enter();
378

379
		default_idle();
380 381 382 383 384

		/*
		 * The switch back from broadcast mode needs to be
		 * called with interrupts disabled.
		 */
385
		local_irq_disable();
386
		tick_broadcast_exit();
387
		local_irq_enable();
388 389 390 391
	} else
		default_idle();
}

392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
/*
 * Intel Core2 and older machines prefer MWAIT over HALT for C1.
 * We can't rely on cpuidle installing MWAIT, because it will not load
 * on systems that support only C1 -- so the boot default must be MWAIT.
 *
 * Some AMD machines are the opposite, they depend on using HALT.
 *
 * So for default C1, which is used during boot until cpuidle loads,
 * use MWAIT-C1 on Intel HW that has it, else use HALT.
 */
static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
{
	if (c->x86_vendor != X86_VENDOR_INTEL)
		return 0;

	if (!cpu_has(c, X86_FEATURE_MWAIT))
		return 0;

	return 1;
}

/*
414 415 416
 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 * with interrupts enabled and no flags, which is backwards compatible with the
 * original MWAIT implementation.
417 418 419
 */
static void mwait_idle(void)
{
420
	if (!current_set_polling_and_test()) {
421
		trace_cpu_idle_rcuidle(1, smp_processor_id());
422
		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
423
			mb(); /* quirk */
424
			clflush((void *)&current_thread_info()->flags);
425
			mb(); /* quirk */
426
		}
427 428 429 430 431 432

		__monitor((void *)&current_thread_info()->flags, 0, 0);
		if (!need_resched())
			__sti_mwait(0, 0);
		else
			local_irq_enable();
433
		trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
434
	} else {
435
		local_irq_enable();
436 437
	}
	__current_clr_polling();
438 439
}

440
void select_idle_routine(const struct cpuinfo_x86 *c)
441
{
442
#ifdef CONFIG_SMP
T
Thomas Gleixner 已提交
443
	if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
444
		pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
445
#endif
T
Thomas Gleixner 已提交
446
	if (x86_idle || boot_option_idle_override == IDLE_POLL)
T
Thomas Gleixner 已提交
447 448
		return;

449
	if (cpu_has_bug(c, X86_BUG_AMD_APIC_C1E)) {
450
		/* E400: APIC timer interrupt does not wake up CPU from C1e */
451
		pr_info("using AMD E400 aware idle routine\n");
452
		x86_idle = amd_e400_idle;
453 454 455
	} else if (prefer_mwait_c1_over_halt(c)) {
		pr_info("using mwait in idle threads\n");
		x86_idle = mwait_idle;
T
Thomas Gleixner 已提交
456
	} else
457
		x86_idle = default_idle;
458 459
}

460
void __init init_amd_e400_c1e_mask(void)
461
{
462
	/* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
463
	if (x86_idle == amd_e400_idle)
464
		zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
465 466
}

467 468
static int __init idle_setup(char *str)
{
469 470 471
	if (!str)
		return -EINVAL;

472
	if (!strcmp(str, "poll")) {
473
		pr_info("using polling idle threads\n");
474
		boot_option_idle_override = IDLE_POLL;
T
Thomas Gleixner 已提交
475
		cpu_idle_poll_ctrl(true);
476
	} else if (!strcmp(str, "halt")) {
Z
Zhao Yakui 已提交
477 478 479 480 481 482 483
		/*
		 * When the boot option of idle=halt is added, halt is
		 * forced to be used for CPU idle. In such case CPU C2/C3
		 * won't be used again.
		 * To continue to load the CPU idle driver, don't touch
		 * the boot_option_idle_override.
		 */
484
		x86_idle = default_idle;
485
		boot_option_idle_override = IDLE_HALT;
486 487 488 489 490 491 492
	} else if (!strcmp(str, "nomwait")) {
		/*
		 * If the boot option of "idle=nomwait" is added,
		 * it means that mwait will be disabled for CPU C2/C3
		 * states. In such case it won't touch the variable
		 * of boot_option_idle_override.
		 */
493
		boot_option_idle_override = IDLE_NOMWAIT;
Z
Zhao Yakui 已提交
494
	} else
495 496 497 498 499 500
		return -1;

	return 0;
}
early_param("idle", idle_setup);

A
Amerigo Wang 已提交
501 502 503 504 505 506 507 508 509 510 511 512 513
unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() % 8192;
	return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
	unsigned long range_end = mm->brk + 0x02000000;
	return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
}

514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
/*
 * Called from fs/proc with a reference on @p to find the function
 * which called into schedule(). This needs to be done carefully
 * because the task might wake up and we might look at a stack
 * changing under us.
 */
unsigned long get_wchan(struct task_struct *p)
{
	unsigned long start, bottom, top, sp, fp, ip;
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;

	start = (unsigned long)task_stack_page(p);
	if (!start)
		return 0;

	/*
	 * Layout of the stack page:
	 *
	 * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
	 * PADDING
	 * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
	 * stack
	 * ----------- bottom = start + sizeof(thread_info)
	 * thread_info
	 * ----------- start
	 *
	 * The tasks stack pointer points at the location where the
	 * framepointer is stored. The data on the stack is:
	 * ... IP FP ... IP FP
	 *
	 * We need to read FP and IP, so we need to adjust the upper
	 * bound by another unsigned long.
	 */
	top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
	top -= 2 * sizeof(unsigned long);
	bottom = start + sizeof(struct thread_info);

	sp = READ_ONCE(p->thread.sp);
	if (sp < bottom || sp > top)
		return 0;

558
	fp = READ_ONCE_NOCHECK(*(unsigned long *)sp);
559 560 561
	do {
		if (fp < bottom || fp > top)
			return 0;
562
		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
563 564
		if (!in_sched_functions(ip))
			return ip;
565
		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
566 567 568
	} while (count++ < 16 && p->state != TASK_RUNNING);
	return 0;
}