process.c 14.8 KB
Newer Older
1 2
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

3 4 5 6
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
7
#include <linux/prctl.h>
8 9
#include <linux/slab.h>
#include <linux/sched.h>
10
#include <linux/sched/idle.h>
11
#include <linux/sched/debug.h>
12
#include <linux/sched/task.h>
13
#include <linux/sched/task_stack.h>
14 15
#include <linux/init.h>
#include <linux/export.h>
16
#include <linux/pm.h>
17
#include <linux/tick.h>
A
Amerigo Wang 已提交
18
#include <linux/random.h>
A
Avi Kivity 已提交
19
#include <linux/user-return-notifier.h>
20 21
#include <linux/dmi.h>
#include <linux/utsname.h>
22 23 24
#include <linux/stackprotector.h>
#include <linux/tick.h>
#include <linux/cpuidle.h>
25
#include <trace/events/power.h>
26
#include <linux/hw_breakpoint.h>
27
#include <asm/cpu.h>
28
#include <asm/apic.h>
29
#include <asm/syscalls.h>
30
#include <linux/uaccess.h>
31
#include <asm/mwait.h>
32
#include <asm/fpu/internal.h>
33
#include <asm/debugreg.h>
34
#include <asm/nmi.h>
A
Andy Lutomirski 已提交
35
#include <asm/tlbflush.h>
36
#include <asm/mce.h>
37
#include <asm/vm86.h>
38
#include <asm/switch_to.h>
39
#include <asm/desc.h>
40
#include <asm/prctl.h>
41

T
Thomas Gleixner 已提交
42 43 44 45 46 47 48
/*
 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 * no more per-task TSS's. The TSS size is kept cacheline-aligned
 * so they are allowed to end up in the .data..cacheline_aligned
 * section. Since TSS's are completely CPU-local, we want them
 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 */
49 50
__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
	.x86_tss = {
51
		.sp0 = TOP_OF_INIT_STACK,
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
#ifdef CONFIG_X86_32
		.ss0 = __KERNEL_DS,
		.ss1 = __KERNEL_CS,
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
#endif
	 },
#ifdef CONFIG_X86_32
	 /*
	  * Note that the .io_bitmap member must be extra-big. This is because
	  * the CPU will access an additional byte beyond the end of the IO
	  * permission bitmap. The extra byte must be all 1 bits, and must
	  * be within the limit.
	  */
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },
#endif
67 68 69
#ifdef CONFIG_X86_32
	.SYSENTER_stack_canary	= STACK_END_MAGIC,
#endif
70
};
71
EXPORT_PER_CPU_SYMBOL(cpu_tss);
T
Thomas Gleixner 已提交
72

73 74
DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
75

76 77 78 79
/*
 * this gets called so that we can store lazy state into memory and copy the
 * current task into the new thread.
 */
80 81
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
82
	memcpy(dst, src, arch_task_struct_size);
83 84 85
#ifdef CONFIG_VM86
	dst->thread.vm86 = NULL;
#endif
86

87
	return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
88
}
89

90 91 92
/*
 * Free current thread data structures etc..
 */
93
void exit_thread(struct task_struct *tsk)
94
{
95
	struct thread_struct *t = &tsk->thread;
96
	unsigned long *bp = t->io_bitmap_ptr;
97
	struct fpu *fpu = &t->fpu;
98

99
	if (bp) {
100
		struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
101 102 103 104 105 106 107 108 109

		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
		t->io_bitmap_max = 0;
		put_cpu();
110
		kfree(bp);
111
	}
112

113 114
	free_vm86(t);

115
	fpu__drop(fpu);
116 117 118 119 120 121
}

void flush_thread(void)
{
	struct task_struct *tsk = current;

122
	flush_ptrace_hw_breakpoint(tsk);
123
	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
124

125
	fpu__clear(&tsk->thread.fpu);
126 127 128 129 130 131 132 133 134 135
}

void disable_TSC(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
136
		cr4_set_bits(X86_CR4_TSD);
137 138 139 140 141 142 143 144 145 146 147
	preempt_enable();
}

static void enable_TSC(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
148
		cr4_clear_bits(X86_CR4_TSD);
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
	preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
	unsigned int val;

	if (test_thread_flag(TIF_NOTSC))
		val = PR_TSC_SIGSEGV;
	else
		val = PR_TSC_ENABLE;

	return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
	if (val == PR_TSC_SIGSEGV)
		disable_TSC();
	else if (val == PR_TSC_ENABLE)
		enable_TSC();
	else
		return -EINVAL;

	return 0;
}

176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
DEFINE_PER_CPU(u64, msr_misc_features_shadow);

static void set_cpuid_faulting(bool on)
{
	u64 msrval;

	msrval = this_cpu_read(msr_misc_features_shadow);
	msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;
	msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);
	this_cpu_write(msr_misc_features_shadow, msrval);
	wrmsrl(MSR_MISC_FEATURES_ENABLES, msrval);
}

static void disable_cpuid(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(true);
	}
	preempt_enable();
}

static void enable_cpuid(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(false);
	}
	preempt_enable();
}

static int get_cpuid_mode(void)
{
	return !test_thread_flag(TIF_NOCPUID);
}

static int set_cpuid_mode(struct task_struct *task, unsigned long cpuid_enabled)
{
	if (!static_cpu_has(X86_FEATURE_CPUID_FAULT))
		return -ENODEV;

	if (cpuid_enabled)
		enable_cpuid();
	else
		disable_cpuid();

	return 0;
}

/*
 * Called immediately after a successful exec.
 */
void arch_setup_new_exec(void)
{
	/* If cpuid was previously disabled for this task, re-enable it. */
	if (test_thread_flag(TIF_NOCPUID))
		enable_cpuid();
}

243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
static inline void switch_to_bitmap(struct tss_struct *tss,
				    struct thread_struct *prev,
				    struct thread_struct *next,
				    unsigned long tifp, unsigned long tifn)
{
	if (tifn & _TIF_IO_BITMAP) {
		/*
		 * Copy the relevant range of the IO bitmap.
		 * Normally this is 128 bytes or less:
		 */
		memcpy(tss->io_bitmap, next->io_bitmap_ptr,
		       max(prev->io_bitmap_max, next->io_bitmap_max));
		/*
		 * Make sure that the TSS limit is correct for the CPU
		 * to notice the IO bitmap.
		 */
		refresh_tss_limit();
	} else if (tifp & _TIF_IO_BITMAP) {
		/*
		 * Clear any possible leftover bits:
		 */
		memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	}
}

268 269 270 271
void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss)
{
	struct thread_struct *prev, *next;
272
	unsigned long tifp, tifn;
273 274 275 276

	prev = &prev_p->thread;
	next = &next_p->thread;

277 278 279 280 281 282
	tifn = READ_ONCE(task_thread_info(next_p)->flags);
	tifp = READ_ONCE(task_thread_info(prev_p)->flags);
	switch_to_bitmap(tss, prev, next, tifp, tifn);

	propagate_user_return_notify(prev_p, next_p);

283 284 285
	if ((tifp & _TIF_BLOCKSTEP || tifn & _TIF_BLOCKSTEP) &&
	    arch_has_block_step()) {
		unsigned long debugctl, msk;
P
Peter Zijlstra 已提交
286

287
		rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
P
Peter Zijlstra 已提交
288
		debugctl &= ~DEBUGCTLMSR_BTF;
289 290 291
		msk = tifn & _TIF_BLOCKSTEP;
		debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT;
		wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
P
Peter Zijlstra 已提交
292
	}
293

294 295
	if ((tifp ^ tifn) & _TIF_NOTSC)
		cr4_toggle_bits(X86_CR4_TSD);
296 297 298

	if ((tifp ^ tifn) & _TIF_NOCPUID)
		set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
299 300
}

301 302 303
/*
 * Idle related variables and functions
 */
304
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
305 306
EXPORT_SYMBOL(boot_option_idle_override);

307
static void (*x86_idle)(void);
308

309 310 311 312 313 314 315
#ifndef CONFIG_SMP
static inline void play_dead(void)
{
	BUG();
}
#endif

T
Thomas Gleixner 已提交
316 317
void arch_cpu_idle_enter(void)
{
318
	tsc_verify_tsc_adjust(false);
T
Thomas Gleixner 已提交
319 320
	local_touch_nmi();
}
321

T
Thomas Gleixner 已提交
322 323 324 325
void arch_cpu_idle_dead(void)
{
	play_dead();
}
326

T
Thomas Gleixner 已提交
327 328 329 330 331
/*
 * Called from the generic idle code.
 */
void arch_cpu_idle(void)
{
332
	x86_idle();
333 334
}

335
/*
T
Thomas Gleixner 已提交
336
 * We use this if we don't have any better idle routine..
337
 */
338
void __cpuidle default_idle(void)
339
{
340
	trace_cpu_idle_rcuidle(1, smp_processor_id());
T
Thomas Gleixner 已提交
341
	safe_halt();
342
	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
343
}
344
#ifdef CONFIG_APM_MODULE
345 346 347
EXPORT_SYMBOL(default_idle);
#endif

348 349
#ifdef CONFIG_XEN
bool xen_set_default_idle(void)
350
{
351
	bool ret = !!x86_idle;
352

353
	x86_idle = default_idle;
354 355 356

	return ret;
}
357
#endif
358

359 360 361 362 363 364
void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
365
	set_cpu_online(smp_processor_id(), false);
366
	disable_local_APIC();
367
	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
368

369 370 371 372 373 374 375 376 377 378 379 380 381 382
	for (;;) {
		/*
		 * Use wbinvd followed by hlt to stop the processor. This
		 * provides support for kexec on a processor that supports
		 * SME. With kexec, going from SME inactive to SME active
		 * requires clearing cache entries so that addresses without
		 * the encryption bit set don't corrupt the same physical
		 * address that has the encryption bit set when caches are
		 * flushed. To achieve this a wbinvd is performed followed by
		 * a hlt. Even if the processor is not in the kexec/SME
		 * scenario this only adds a wbinvd to a halting processor.
		 */
		asm volatile("wbinvd; hlt" : : : "memory");
	}
383 384
}

385
/*
386 387
 * AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
 * states (local apic timer and TSC stop).
388
 */
389
static void amd_e400_idle(void)
390
{
391 392 393 394 395 396 397 398
	/*
	 * We cannot use static_cpu_has_bug() here because X86_BUG_AMD_APIC_C1E
	 * gets set after static_cpu_has() places have been converted via
	 * alternatives.
	 */
	if (!boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		default_idle();
		return;
399 400
	}

401
	tick_broadcast_enter();
402

403
	default_idle();
404

405 406 407 408 409 410 411
	/*
	 * The switch back from broadcast mode needs to be called with
	 * interrupts disabled.
	 */
	local_irq_disable();
	tick_broadcast_exit();
	local_irq_enable();
412 413
}

414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
/*
 * Intel Core2 and older machines prefer MWAIT over HALT for C1.
 * We can't rely on cpuidle installing MWAIT, because it will not load
 * on systems that support only C1 -- so the boot default must be MWAIT.
 *
 * Some AMD machines are the opposite, they depend on using HALT.
 *
 * So for default C1, which is used during boot until cpuidle loads,
 * use MWAIT-C1 on Intel HW that has it, else use HALT.
 */
static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
{
	if (c->x86_vendor != X86_VENDOR_INTEL)
		return 0;

429
	if (!cpu_has(c, X86_FEATURE_MWAIT) || static_cpu_has_bug(X86_BUG_MONITOR))
430 431 432 433 434 435
		return 0;

	return 1;
}

/*
436 437 438
 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 * with interrupts enabled and no flags, which is backwards compatible with the
 * original MWAIT implementation.
439
 */
440
static __cpuidle void mwait_idle(void)
441
{
442
	if (!current_set_polling_and_test()) {
443
		trace_cpu_idle_rcuidle(1, smp_processor_id());
444
		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
445
			mb(); /* quirk */
446
			clflush((void *)&current_thread_info()->flags);
447
			mb(); /* quirk */
448
		}
449 450 451 452 453 454

		__monitor((void *)&current_thread_info()->flags, 0, 0);
		if (!need_resched())
			__sti_mwait(0, 0);
		else
			local_irq_enable();
455
		trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
456
	} else {
457
		local_irq_enable();
458 459
	}
	__current_clr_polling();
460 461
}

462
void select_idle_routine(const struct cpuinfo_x86 *c)
463
{
464
#ifdef CONFIG_SMP
T
Thomas Gleixner 已提交
465
	if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
466
		pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
467
#endif
T
Thomas Gleixner 已提交
468
	if (x86_idle || boot_option_idle_override == IDLE_POLL)
T
Thomas Gleixner 已提交
469 470
		return;

471
	if (boot_cpu_has_bug(X86_BUG_AMD_E400)) {
472
		pr_info("using AMD E400 aware idle routine\n");
473
		x86_idle = amd_e400_idle;
474 475 476
	} else if (prefer_mwait_c1_over_halt(c)) {
		pr_info("using mwait in idle threads\n");
		x86_idle = mwait_idle;
T
Thomas Gleixner 已提交
477
	} else
478
		x86_idle = default_idle;
479 480
}

481
void amd_e400_c1e_apic_setup(void)
482
{
483 484 485 486 487 488
	if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
		local_irq_disable();
		tick_broadcast_force();
		local_irq_enable();
	}
489 490
}

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
void __init arch_post_acpi_subsys_init(void)
{
	u32 lo, hi;

	if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
		return;

	/*
	 * AMD E400 detection needs to happen after ACPI has been enabled. If
	 * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
	 * MSR_K8_INT_PENDING_MSG.
	 */
	rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
	if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
		return;

	boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);

	if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
		mark_tsc_unstable("TSC halt in AMD C1E");
	pr_info("System has AMD C1E enabled\n");
}

514 515
static int __init idle_setup(char *str)
{
516 517 518
	if (!str)
		return -EINVAL;

519
	if (!strcmp(str, "poll")) {
520
		pr_info("using polling idle threads\n");
521
		boot_option_idle_override = IDLE_POLL;
T
Thomas Gleixner 已提交
522
		cpu_idle_poll_ctrl(true);
523
	} else if (!strcmp(str, "halt")) {
Z
Zhao Yakui 已提交
524 525 526 527 528 529 530
		/*
		 * When the boot option of idle=halt is added, halt is
		 * forced to be used for CPU idle. In such case CPU C2/C3
		 * won't be used again.
		 * To continue to load the CPU idle driver, don't touch
		 * the boot_option_idle_override.
		 */
531
		x86_idle = default_idle;
532
		boot_option_idle_override = IDLE_HALT;
533 534 535 536 537 538 539
	} else if (!strcmp(str, "nomwait")) {
		/*
		 * If the boot option of "idle=nomwait" is added,
		 * it means that mwait will be disabled for CPU C2/C3
		 * states. In such case it won't touch the variable
		 * of boot_option_idle_override.
		 */
540
		boot_option_idle_override = IDLE_NOMWAIT;
Z
Zhao Yakui 已提交
541
	} else
542 543 544 545 546 547
		return -1;

	return 0;
}
early_param("idle", idle_setup);

A
Amerigo Wang 已提交
548 549 550 551 552 553 554 555 556
unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() % 8192;
	return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
557
	return randomize_page(mm->brk, 0x02000000);
A
Amerigo Wang 已提交
558 559
}

560 561 562 563 564 565 566 567
/*
 * Called from fs/proc with a reference on @p to find the function
 * which called into schedule(). This needs to be done carefully
 * because the task might wake up and we might look at a stack
 * changing under us.
 */
unsigned long get_wchan(struct task_struct *p)
{
568
	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
569 570 571 572 573
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;

574 575 576
	if (!try_get_task_stack(p))
		return 0;

577 578
	start = (unsigned long)task_stack_page(p);
	if (!start)
579
		goto out;
580 581 582 583 584 585 586 587

	/*
	 * Layout of the stack page:
	 *
	 * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
	 * PADDING
	 * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
	 * stack
588
	 * ----------- bottom = start
589 590 591 592 593 594 595 596 597 598
	 *
	 * The tasks stack pointer points at the location where the
	 * framepointer is stored. The data on the stack is:
	 * ... IP FP ... IP FP
	 *
	 * We need to read FP and IP, so we need to adjust the upper
	 * bound by another unsigned long.
	 */
	top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
	top -= 2 * sizeof(unsigned long);
599
	bottom = start;
600 601 602

	sp = READ_ONCE(p->thread.sp);
	if (sp < bottom || sp > top)
603
		goto out;
604

605
	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
606 607
	do {
		if (fp < bottom || fp > top)
608
			goto out;
609
		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
610 611 612 613
		if (!in_sched_functions(ip)) {
			ret = ip;
			goto out;
		}
614
		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
615
	} while (count++ < 16 && p->state != TASK_RUNNING);
616 617 618 619

out:
	put_task_stack(p);
	return ret;
620
}
621 622 623 624

long do_arch_prctl_common(struct task_struct *task, int option,
			  unsigned long cpuid_enabled)
{
625 626 627 628 629 630 631
	switch (option) {
	case ARCH_GET_CPUID:
		return get_cpuid_mode();
	case ARCH_SET_CPUID:
		return set_cpuid_mode(task, cpuid_enabled);
	}

632 633
	return -EINVAL;
}