process.c 12.7 KB
Newer Older
1 2
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

3 4 5 6
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
7
#include <linux/prctl.h>
8 9
#include <linux/slab.h>
#include <linux/sched.h>
10
#include <linux/sched/idle.h>
11
#include <linux/sched/debug.h>
12 13
#include <linux/init.h>
#include <linux/export.h>
14
#include <linux/pm.h>
15
#include <linux/tick.h>
A
Amerigo Wang 已提交
16
#include <linux/random.h>
A
Avi Kivity 已提交
17
#include <linux/user-return-notifier.h>
18 19
#include <linux/dmi.h>
#include <linux/utsname.h>
20 21 22
#include <linux/stackprotector.h>
#include <linux/tick.h>
#include <linux/cpuidle.h>
23
#include <trace/events/power.h>
24
#include <linux/hw_breakpoint.h>
25
#include <asm/cpu.h>
26
#include <asm/apic.h>
27
#include <asm/syscalls.h>
28
#include <linux/uaccess.h>
29
#include <asm/mwait.h>
30
#include <asm/fpu/internal.h>
31
#include <asm/debugreg.h>
32
#include <asm/nmi.h>
A
Andy Lutomirski 已提交
33
#include <asm/tlbflush.h>
34
#include <asm/mce.h>
35
#include <asm/vm86.h>
36
#include <asm/switch_to.h>
37
#include <asm/desc.h>
38

T
Thomas Gleixner 已提交
39 40 41 42 43 44 45
/*
 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 * no more per-task TSS's. The TSS size is kept cacheline-aligned
 * so they are allowed to end up in the .data..cacheline_aligned
 * section. Since TSS's are completely CPU-local, we want them
 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 */
46 47
__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
	.x86_tss = {
48
		.sp0 = TOP_OF_INIT_STACK,
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
#ifdef CONFIG_X86_32
		.ss0 = __KERNEL_DS,
		.ss1 = __KERNEL_CS,
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
#endif
	 },
#ifdef CONFIG_X86_32
	 /*
	  * Note that the .io_bitmap member must be extra-big. This is because
	  * the CPU will access an additional byte beyond the end of the IO
	  * permission bitmap. The extra byte must be all 1 bits, and must
	  * be within the limit.
	  */
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },
#endif
64 65 66
#ifdef CONFIG_X86_32
	.SYSENTER_stack_canary	= STACK_END_MAGIC,
#endif
67
};
68
EXPORT_PER_CPU_SYMBOL(cpu_tss);
T
Thomas Gleixner 已提交
69

70 71 72
DEFINE_PER_CPU(bool, need_tr_refresh);
EXPORT_PER_CPU_SYMBOL_GPL(need_tr_refresh);

73 74 75 76
/*
 * this gets called so that we can store lazy state into memory and copy the
 * current task into the new thread.
 */
77 78
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
79
	memcpy(dst, src, arch_task_struct_size);
80 81 82
#ifdef CONFIG_VM86
	dst->thread.vm86 = NULL;
#endif
83

84
	return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
85
}
86

87 88 89
/*
 * Free current thread data structures etc..
 */
90
void exit_thread(struct task_struct *tsk)
91
{
92
	struct thread_struct *t = &tsk->thread;
93
	unsigned long *bp = t->io_bitmap_ptr;
94
	struct fpu *fpu = &t->fpu;
95

96
	if (bp) {
97
		struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
98 99 100 101 102 103 104 105 106

		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
		t->io_bitmap_max = 0;
		put_cpu();
107
		kfree(bp);
108
	}
109

110 111
	free_vm86(t);

112
	fpu__drop(fpu);
113 114 115 116 117 118
}

void flush_thread(void)
{
	struct task_struct *tsk = current;

119
	flush_ptrace_hw_breakpoint(tsk);
120
	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
121

122
	fpu__clear(&tsk->thread.fpu);
123 124 125 126
}

static void hard_disable_TSC(void)
{
A
Andy Lutomirski 已提交
127
	cr4_set_bits(X86_CR4_TSD);
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
}

void disable_TSC(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		hard_disable_TSC();
	preempt_enable();
}

static void hard_enable_TSC(void)
{
A
Andy Lutomirski 已提交
144
	cr4_clear_bits(X86_CR4_TSD);
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
}

static void enable_TSC(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		hard_enable_TSC();
	preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
	unsigned int val;

	if (test_thread_flag(TIF_NOTSC))
		val = PR_TSC_SIGSEGV;
	else
		val = PR_TSC_ENABLE;

	return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
	if (val == PR_TSC_SIGSEGV)
		disable_TSC();
	else if (val == PR_TSC_ENABLE)
		enable_TSC();
	else
		return -EINVAL;

	return 0;
}

void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss)
{
	struct thread_struct *prev, *next;

	prev = &prev_p->thread;
	next = &next_p->thread;

P
Peter Zijlstra 已提交
191 192 193 194 195 196 197 198 199 200
	if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
	    test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl &= ~DEBUGCTLMSR_BTF;
		if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
			debugctl |= DEBUGCTLMSR_BTF;

		update_debugctlmsr(debugctl);
	}
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217

	if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
	    test_tsk_thread_flag(next_p, TIF_NOTSC)) {
		/* prev and next are different */
		if (test_tsk_thread_flag(next_p, TIF_NOTSC))
			hard_disable_TSC();
		else
			hard_enable_TSC();
	}

	if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
		/*
		 * Copy the relevant range of the IO bitmap.
		 * Normally this is 128 bytes or less:
		 */
		memcpy(tss->io_bitmap, next->io_bitmap_ptr,
		       max(prev->io_bitmap_max, next->io_bitmap_max));
218 219 220 221 222 223

		/*
		 * Make sure that the TSS limit is correct for the CPU
		 * to notice the IO bitmap.
		 */
		refresh_TR();
224 225 226 227 228 229
	} else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
		/*
		 * Clear any possible leftover bits:
		 */
		memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	}
A
Avi Kivity 已提交
230
	propagate_user_return_notify(prev_p, next_p);
231 232
}

233 234 235
/*
 * Idle related variables and functions
 */
236
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
237 238
EXPORT_SYMBOL(boot_option_idle_override);

239
static void (*x86_idle)(void);
240

241 242 243 244 245 246 247
#ifndef CONFIG_SMP
static inline void play_dead(void)
{
	BUG();
}
#endif

T
Thomas Gleixner 已提交
248 249
void arch_cpu_idle_enter(void)
{
250
	tsc_verify_tsc_adjust(false);
T
Thomas Gleixner 已提交
251 252
	local_touch_nmi();
}
253

T
Thomas Gleixner 已提交
254 255 256 257
void arch_cpu_idle_dead(void)
{
	play_dead();
}
258

T
Thomas Gleixner 已提交
259 260 261 262 263
/*
 * Called from the generic idle code.
 */
void arch_cpu_idle(void)
{
264
	x86_idle();
265 266
}

267
/*
T
Thomas Gleixner 已提交
268
 * We use this if we don't have any better idle routine..
269
 */
270
void __cpuidle default_idle(void)
271
{
272
	trace_cpu_idle_rcuidle(1, smp_processor_id());
T
Thomas Gleixner 已提交
273
	safe_halt();
274
	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
275
}
276
#ifdef CONFIG_APM_MODULE
277 278 279
EXPORT_SYMBOL(default_idle);
#endif

280 281
#ifdef CONFIG_XEN
bool xen_set_default_idle(void)
282
{
283
	bool ret = !!x86_idle;
284

285
	x86_idle = default_idle;
286 287 288

	return ret;
}
289
#endif
290 291 292 293 294 295
void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
296
	set_cpu_online(smp_processor_id(), false);
297
	disable_local_APIC();
298
	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
299

300 301
	for (;;)
		halt();
302 303
}

304
/*
305 306
 * AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
 * states (local apic timer and TSC stop).
307
 */
308
static void amd_e400_idle(void)
309
{
310 311 312 313 314 315 316 317
	/*
	 * We cannot use static_cpu_has_bug() here because X86_BUG_AMD_APIC_C1E
	 * gets set after static_cpu_has() places have been converted via
	 * alternatives.
	 */
	if (!boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		default_idle();
		return;
318 319
	}

320
	tick_broadcast_enter();
321

322
	default_idle();
323

324 325 326 327 328 329 330
	/*
	 * The switch back from broadcast mode needs to be called with
	 * interrupts disabled.
	 */
	local_irq_disable();
	tick_broadcast_exit();
	local_irq_enable();
331 332
}

333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
/*
 * Intel Core2 and older machines prefer MWAIT over HALT for C1.
 * We can't rely on cpuidle installing MWAIT, because it will not load
 * on systems that support only C1 -- so the boot default must be MWAIT.
 *
 * Some AMD machines are the opposite, they depend on using HALT.
 *
 * So for default C1, which is used during boot until cpuidle loads,
 * use MWAIT-C1 on Intel HW that has it, else use HALT.
 */
static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
{
	if (c->x86_vendor != X86_VENDOR_INTEL)
		return 0;

348
	if (!cpu_has(c, X86_FEATURE_MWAIT) || static_cpu_has_bug(X86_BUG_MONITOR))
349 350 351 352 353 354
		return 0;

	return 1;
}

/*
355 356 357
 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 * with interrupts enabled and no flags, which is backwards compatible with the
 * original MWAIT implementation.
358
 */
359
static __cpuidle void mwait_idle(void)
360
{
361
	if (!current_set_polling_and_test()) {
362
		trace_cpu_idle_rcuidle(1, smp_processor_id());
363
		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
364
			mb(); /* quirk */
365
			clflush((void *)&current_thread_info()->flags);
366
			mb(); /* quirk */
367
		}
368 369 370 371 372 373

		__monitor((void *)&current_thread_info()->flags, 0, 0);
		if (!need_resched())
			__sti_mwait(0, 0);
		else
			local_irq_enable();
374
		trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
375
	} else {
376
		local_irq_enable();
377 378
	}
	__current_clr_polling();
379 380
}

381
void select_idle_routine(const struct cpuinfo_x86 *c)
382
{
383
#ifdef CONFIG_SMP
T
Thomas Gleixner 已提交
384
	if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
385
		pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
386
#endif
T
Thomas Gleixner 已提交
387
	if (x86_idle || boot_option_idle_override == IDLE_POLL)
T
Thomas Gleixner 已提交
388 389
		return;

390
	if (boot_cpu_has_bug(X86_BUG_AMD_E400)) {
391
		pr_info("using AMD E400 aware idle routine\n");
392
		x86_idle = amd_e400_idle;
393 394 395
	} else if (prefer_mwait_c1_over_halt(c)) {
		pr_info("using mwait in idle threads\n");
		x86_idle = mwait_idle;
T
Thomas Gleixner 已提交
396
	} else
397
		x86_idle = default_idle;
398 399
}

400
void amd_e400_c1e_apic_setup(void)
401
{
402 403 404 405 406 407
	if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
		local_irq_disable();
		tick_broadcast_force();
		local_irq_enable();
	}
408 409
}

410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
void __init arch_post_acpi_subsys_init(void)
{
	u32 lo, hi;

	if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
		return;

	/*
	 * AMD E400 detection needs to happen after ACPI has been enabled. If
	 * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
	 * MSR_K8_INT_PENDING_MSG.
	 */
	rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
	if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
		return;

	boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);

	if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
		mark_tsc_unstable("TSC halt in AMD C1E");
	pr_info("System has AMD C1E enabled\n");
}

433 434
static int __init idle_setup(char *str)
{
435 436 437
	if (!str)
		return -EINVAL;

438
	if (!strcmp(str, "poll")) {
439
		pr_info("using polling idle threads\n");
440
		boot_option_idle_override = IDLE_POLL;
T
Thomas Gleixner 已提交
441
		cpu_idle_poll_ctrl(true);
442
	} else if (!strcmp(str, "halt")) {
Z
Zhao Yakui 已提交
443 444 445 446 447 448 449
		/*
		 * When the boot option of idle=halt is added, halt is
		 * forced to be used for CPU idle. In such case CPU C2/C3
		 * won't be used again.
		 * To continue to load the CPU idle driver, don't touch
		 * the boot_option_idle_override.
		 */
450
		x86_idle = default_idle;
451
		boot_option_idle_override = IDLE_HALT;
452 453 454 455 456 457 458
	} else if (!strcmp(str, "nomwait")) {
		/*
		 * If the boot option of "idle=nomwait" is added,
		 * it means that mwait will be disabled for CPU C2/C3
		 * states. In such case it won't touch the variable
		 * of boot_option_idle_override.
		 */
459
		boot_option_idle_override = IDLE_NOMWAIT;
Z
Zhao Yakui 已提交
460
	} else
461 462 463 464 465 466
		return -1;

	return 0;
}
early_param("idle", idle_setup);

A
Amerigo Wang 已提交
467 468 469 470 471 472 473 474 475
unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() % 8192;
	return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
476
	return randomize_page(mm->brk, 0x02000000);
A
Amerigo Wang 已提交
477 478
}

B
Brian Gerst 已提交
479 480 481 482 483 484 485 486 487 488 489
/*
 * Return saved PC of a blocked thread.
 * What is this good for? it will be always the scheduler or ret_from_fork.
 */
unsigned long thread_saved_pc(struct task_struct *tsk)
{
	struct inactive_task_frame *frame =
		(struct inactive_task_frame *) READ_ONCE(tsk->thread.sp);
	return READ_ONCE_NOCHECK(frame->ret_addr);
}

490 491 492 493 494 495 496 497
/*
 * Called from fs/proc with a reference on @p to find the function
 * which called into schedule(). This needs to be done carefully
 * because the task might wake up and we might look at a stack
 * changing under us.
 */
unsigned long get_wchan(struct task_struct *p)
{
498
	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
499 500 501 502 503
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;

504 505 506
	if (!try_get_task_stack(p))
		return 0;

507 508
	start = (unsigned long)task_stack_page(p);
	if (!start)
509
		goto out;
510 511 512 513 514 515 516 517

	/*
	 * Layout of the stack page:
	 *
	 * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
	 * PADDING
	 * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
	 * stack
518
	 * ----------- bottom = start
519 520 521 522 523 524 525 526 527 528
	 *
	 * The tasks stack pointer points at the location where the
	 * framepointer is stored. The data on the stack is:
	 * ... IP FP ... IP FP
	 *
	 * We need to read FP and IP, so we need to adjust the upper
	 * bound by another unsigned long.
	 */
	top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
	top -= 2 * sizeof(unsigned long);
529
	bottom = start;
530 531 532

	sp = READ_ONCE(p->thread.sp);
	if (sp < bottom || sp > top)
533
		goto out;
534

535
	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
536 537
	do {
		if (fp < bottom || fp > top)
538
			goto out;
539
		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
540 541 542 543
		if (!in_sched_functions(ip)) {
			ret = ip;
			goto out;
		}
544
		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
545
	} while (count++ < 16 && p->state != TASK_RUNNING);
546 547 548 549

out:
	put_task_stack(p);
	return ret;
550
}