process.c 15.1 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

4 5 6 7
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
8
#include <linux/prctl.h>
9 10
#include <linux/slab.h>
#include <linux/sched.h>
11
#include <linux/sched/idle.h>
12
#include <linux/sched/debug.h>
13
#include <linux/sched/task.h>
14
#include <linux/sched/task_stack.h>
15 16
#include <linux/init.h>
#include <linux/export.h>
17
#include <linux/pm.h>
18
#include <linux/tick.h>
A
Amerigo Wang 已提交
19
#include <linux/random.h>
A
Avi Kivity 已提交
20
#include <linux/user-return-notifier.h>
21 22
#include <linux/dmi.h>
#include <linux/utsname.h>
23 24 25
#include <linux/stackprotector.h>
#include <linux/tick.h>
#include <linux/cpuidle.h>
26
#include <trace/events/power.h>
27
#include <linux/hw_breakpoint.h>
28
#include <asm/cpu.h>
29
#include <asm/apic.h>
30
#include <asm/syscalls.h>
31
#include <linux/uaccess.h>
32
#include <asm/mwait.h>
33
#include <asm/fpu/internal.h>
34
#include <asm/debugreg.h>
35
#include <asm/nmi.h>
A
Andy Lutomirski 已提交
36
#include <asm/tlbflush.h>
37
#include <asm/mce.h>
38
#include <asm/vm86.h>
39
#include <asm/switch_to.h>
40
#include <asm/desc.h>
41
#include <asm/prctl.h>
42

T
Thomas Gleixner 已提交
43 44 45 46 47 48 49
/*
 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 * no more per-task TSS's. The TSS size is kept cacheline-aligned
 * so they are allowed to end up in the .data..cacheline_aligned
 * section. Since TSS's are completely CPU-local, we want them
 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 */
50 51
__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
	.x86_tss = {
52 53 54 55 56 57 58
		/*
		 * .sp0 is only used when entering ring 0 from a lower
		 * privilege level.  Since the init task never runs anything
		 * but ring 0 code, there is no need for a valid value here.
		 * Poison it.
		 */
		.sp0 = (1UL << (BITS_PER_LONG-1)) + 1,
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
#ifdef CONFIG_X86_32
		.ss0 = __KERNEL_DS,
		.ss1 = __KERNEL_CS,
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
#endif
	 },
#ifdef CONFIG_X86_32
	 /*
	  * Note that the .io_bitmap member must be extra-big. This is because
	  * the CPU will access an additional byte beyond the end of the IO
	  * permission bitmap. The extra byte must be all 1 bits, and must
	  * be within the limit.
	  */
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },
#endif
74
	.SYSENTER_stack_canary	= STACK_END_MAGIC,
75
};
76
EXPORT_PER_CPU_SYMBOL(cpu_tss);
T
Thomas Gleixner 已提交
77

78 79
DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
80

81 82 83 84
/*
 * this gets called so that we can store lazy state into memory and copy the
 * current task into the new thread.
 */
85 86
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
87
	memcpy(dst, src, arch_task_struct_size);
88 89 90
#ifdef CONFIG_VM86
	dst->thread.vm86 = NULL;
#endif
91

92
	return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
93
}
94

95 96 97
/*
 * Free current thread data structures etc..
 */
98
void exit_thread(struct task_struct *tsk)
99
{
100
	struct thread_struct *t = &tsk->thread;
101
	unsigned long *bp = t->io_bitmap_ptr;
102
	struct fpu *fpu = &t->fpu;
103

104
	if (bp) {
105
		struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
106 107 108 109 110 111 112 113 114

		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
		t->io_bitmap_max = 0;
		put_cpu();
115
		kfree(bp);
116
	}
117

118 119
	free_vm86(t);

120
	fpu__drop(fpu);
121 122 123 124 125 126
}

void flush_thread(void)
{
	struct task_struct *tsk = current;

127
	flush_ptrace_hw_breakpoint(tsk);
128
	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
129

130
	fpu__clear(&tsk->thread.fpu);
131 132 133 134 135 136 137 138 139 140
}

void disable_TSC(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
141
		cr4_set_bits(X86_CR4_TSD);
142 143 144 145 146 147 148 149 150 151 152
	preempt_enable();
}

static void enable_TSC(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
153
		cr4_clear_bits(X86_CR4_TSD);
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
	preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
	unsigned int val;

	if (test_thread_flag(TIF_NOTSC))
		val = PR_TSC_SIGSEGV;
	else
		val = PR_TSC_ENABLE;

	return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
	if (val == PR_TSC_SIGSEGV)
		disable_TSC();
	else if (val == PR_TSC_ENABLE)
		enable_TSC();
	else
		return -EINVAL;

	return 0;
}

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
DEFINE_PER_CPU(u64, msr_misc_features_shadow);

static void set_cpuid_faulting(bool on)
{
	u64 msrval;

	msrval = this_cpu_read(msr_misc_features_shadow);
	msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;
	msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);
	this_cpu_write(msr_misc_features_shadow, msrval);
	wrmsrl(MSR_MISC_FEATURES_ENABLES, msrval);
}

static void disable_cpuid(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(true);
	}
	preempt_enable();
}

static void enable_cpuid(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(false);
	}
	preempt_enable();
}

static int get_cpuid_mode(void)
{
	return !test_thread_flag(TIF_NOCPUID);
}

static int set_cpuid_mode(struct task_struct *task, unsigned long cpuid_enabled)
{
	if (!static_cpu_has(X86_FEATURE_CPUID_FAULT))
		return -ENODEV;

	if (cpuid_enabled)
		enable_cpuid();
	else
		disable_cpuid();

	return 0;
}

/*
 * Called immediately after a successful exec.
 */
void arch_setup_new_exec(void)
{
	/* If cpuid was previously disabled for this task, re-enable it. */
	if (test_thread_flag(TIF_NOCPUID))
		enable_cpuid();
}

248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
static inline void switch_to_bitmap(struct tss_struct *tss,
				    struct thread_struct *prev,
				    struct thread_struct *next,
				    unsigned long tifp, unsigned long tifn)
{
	if (tifn & _TIF_IO_BITMAP) {
		/*
		 * Copy the relevant range of the IO bitmap.
		 * Normally this is 128 bytes or less:
		 */
		memcpy(tss->io_bitmap, next->io_bitmap_ptr,
		       max(prev->io_bitmap_max, next->io_bitmap_max));
		/*
		 * Make sure that the TSS limit is correct for the CPU
		 * to notice the IO bitmap.
		 */
		refresh_tss_limit();
	} else if (tifp & _TIF_IO_BITMAP) {
		/*
		 * Clear any possible leftover bits:
		 */
		memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	}
}

273 274 275 276
void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss)
{
	struct thread_struct *prev, *next;
277
	unsigned long tifp, tifn;
278 279 280 281

	prev = &prev_p->thread;
	next = &next_p->thread;

282 283 284 285 286 287
	tifn = READ_ONCE(task_thread_info(next_p)->flags);
	tifp = READ_ONCE(task_thread_info(prev_p)->flags);
	switch_to_bitmap(tss, prev, next, tifp, tifn);

	propagate_user_return_notify(prev_p, next_p);

288 289 290
	if ((tifp & _TIF_BLOCKSTEP || tifn & _TIF_BLOCKSTEP) &&
	    arch_has_block_step()) {
		unsigned long debugctl, msk;
P
Peter Zijlstra 已提交
291

292
		rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
P
Peter Zijlstra 已提交
293
		debugctl &= ~DEBUGCTLMSR_BTF;
294 295 296
		msk = tifn & _TIF_BLOCKSTEP;
		debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT;
		wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
P
Peter Zijlstra 已提交
297
	}
298

299 300
	if ((tifp ^ tifn) & _TIF_NOTSC)
		cr4_toggle_bits(X86_CR4_TSD);
301 302 303

	if ((tifp ^ tifn) & _TIF_NOCPUID)
		set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
304 305
}

306 307 308
/*
 * Idle related variables and functions
 */
309
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
310 311
EXPORT_SYMBOL(boot_option_idle_override);

312
static void (*x86_idle)(void);
313

314 315 316 317 318 319 320
#ifndef CONFIG_SMP
static inline void play_dead(void)
{
	BUG();
}
#endif

T
Thomas Gleixner 已提交
321 322
void arch_cpu_idle_enter(void)
{
323
	tsc_verify_tsc_adjust(false);
T
Thomas Gleixner 已提交
324 325
	local_touch_nmi();
}
326

T
Thomas Gleixner 已提交
327 328 329 330
void arch_cpu_idle_dead(void)
{
	play_dead();
}
331

T
Thomas Gleixner 已提交
332 333 334 335 336
/*
 * Called from the generic idle code.
 */
void arch_cpu_idle(void)
{
337
	x86_idle();
338 339
}

340
/*
T
Thomas Gleixner 已提交
341
 * We use this if we don't have any better idle routine..
342
 */
343
void __cpuidle default_idle(void)
344
{
345
	trace_cpu_idle_rcuidle(1, smp_processor_id());
T
Thomas Gleixner 已提交
346
	safe_halt();
347
	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
348
}
349
#ifdef CONFIG_APM_MODULE
350 351 352
EXPORT_SYMBOL(default_idle);
#endif

353 354
#ifdef CONFIG_XEN
bool xen_set_default_idle(void)
355
{
356
	bool ret = !!x86_idle;
357

358
	x86_idle = default_idle;
359 360 361

	return ret;
}
362
#endif
363

364 365 366 367 368 369
void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
370
	set_cpu_online(smp_processor_id(), false);
371
	disable_local_APIC();
372
	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
373

374 375 376 377 378 379 380 381 382 383 384 385 386 387
	for (;;) {
		/*
		 * Use wbinvd followed by hlt to stop the processor. This
		 * provides support for kexec on a processor that supports
		 * SME. With kexec, going from SME inactive to SME active
		 * requires clearing cache entries so that addresses without
		 * the encryption bit set don't corrupt the same physical
		 * address that has the encryption bit set when caches are
		 * flushed. To achieve this a wbinvd is performed followed by
		 * a hlt. Even if the processor is not in the kexec/SME
		 * scenario this only adds a wbinvd to a halting processor.
		 */
		asm volatile("wbinvd; hlt" : : : "memory");
	}
388 389
}

390
/*
391 392
 * AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
 * states (local apic timer and TSC stop).
393
 */
394
static void amd_e400_idle(void)
395
{
396 397 398 399 400 401 402 403
	/*
	 * We cannot use static_cpu_has_bug() here because X86_BUG_AMD_APIC_C1E
	 * gets set after static_cpu_has() places have been converted via
	 * alternatives.
	 */
	if (!boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		default_idle();
		return;
404 405
	}

406
	tick_broadcast_enter();
407

408
	default_idle();
409

410 411 412 413 414 415 416
	/*
	 * The switch back from broadcast mode needs to be called with
	 * interrupts disabled.
	 */
	local_irq_disable();
	tick_broadcast_exit();
	local_irq_enable();
417 418
}

419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
/*
 * Intel Core2 and older machines prefer MWAIT over HALT for C1.
 * We can't rely on cpuidle installing MWAIT, because it will not load
 * on systems that support only C1 -- so the boot default must be MWAIT.
 *
 * Some AMD machines are the opposite, they depend on using HALT.
 *
 * So for default C1, which is used during boot until cpuidle loads,
 * use MWAIT-C1 on Intel HW that has it, else use HALT.
 */
static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
{
	if (c->x86_vendor != X86_VENDOR_INTEL)
		return 0;

434
	if (!cpu_has(c, X86_FEATURE_MWAIT) || static_cpu_has_bug(X86_BUG_MONITOR))
435 436 437 438 439 440
		return 0;

	return 1;
}

/*
441 442 443
 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 * with interrupts enabled and no flags, which is backwards compatible with the
 * original MWAIT implementation.
444
 */
445
static __cpuidle void mwait_idle(void)
446
{
447
	if (!current_set_polling_and_test()) {
448
		trace_cpu_idle_rcuidle(1, smp_processor_id());
449
		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
450
			mb(); /* quirk */
451
			clflush((void *)&current_thread_info()->flags);
452
			mb(); /* quirk */
453
		}
454 455 456 457 458 459

		__monitor((void *)&current_thread_info()->flags, 0, 0);
		if (!need_resched())
			__sti_mwait(0, 0);
		else
			local_irq_enable();
460
		trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
461
	} else {
462
		local_irq_enable();
463 464
	}
	__current_clr_polling();
465 466
}

467
void select_idle_routine(const struct cpuinfo_x86 *c)
468
{
469
#ifdef CONFIG_SMP
T
Thomas Gleixner 已提交
470
	if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
471
		pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
472
#endif
T
Thomas Gleixner 已提交
473
	if (x86_idle || boot_option_idle_override == IDLE_POLL)
T
Thomas Gleixner 已提交
474 475
		return;

476
	if (boot_cpu_has_bug(X86_BUG_AMD_E400)) {
477
		pr_info("using AMD E400 aware idle routine\n");
478
		x86_idle = amd_e400_idle;
479 480 481
	} else if (prefer_mwait_c1_over_halt(c)) {
		pr_info("using mwait in idle threads\n");
		x86_idle = mwait_idle;
T
Thomas Gleixner 已提交
482
	} else
483
		x86_idle = default_idle;
484 485
}

486
void amd_e400_c1e_apic_setup(void)
487
{
488 489 490 491 492 493
	if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
		local_irq_disable();
		tick_broadcast_force();
		local_irq_enable();
	}
494 495
}

496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
void __init arch_post_acpi_subsys_init(void)
{
	u32 lo, hi;

	if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
		return;

	/*
	 * AMD E400 detection needs to happen after ACPI has been enabled. If
	 * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
	 * MSR_K8_INT_PENDING_MSG.
	 */
	rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
	if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
		return;

	boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);

	if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
		mark_tsc_unstable("TSC halt in AMD C1E");
	pr_info("System has AMD C1E enabled\n");
}

519 520
static int __init idle_setup(char *str)
{
521 522 523
	if (!str)
		return -EINVAL;

524
	if (!strcmp(str, "poll")) {
525
		pr_info("using polling idle threads\n");
526
		boot_option_idle_override = IDLE_POLL;
T
Thomas Gleixner 已提交
527
		cpu_idle_poll_ctrl(true);
528
	} else if (!strcmp(str, "halt")) {
Z
Zhao Yakui 已提交
529 530 531 532 533 534 535
		/*
		 * When the boot option of idle=halt is added, halt is
		 * forced to be used for CPU idle. In such case CPU C2/C3
		 * won't be used again.
		 * To continue to load the CPU idle driver, don't touch
		 * the boot_option_idle_override.
		 */
536
		x86_idle = default_idle;
537
		boot_option_idle_override = IDLE_HALT;
538 539 540 541 542 543 544
	} else if (!strcmp(str, "nomwait")) {
		/*
		 * If the boot option of "idle=nomwait" is added,
		 * it means that mwait will be disabled for CPU C2/C3
		 * states. In such case it won't touch the variable
		 * of boot_option_idle_override.
		 */
545
		boot_option_idle_override = IDLE_NOMWAIT;
Z
Zhao Yakui 已提交
546
	} else
547 548 549 550 551 552
		return -1;

	return 0;
}
early_param("idle", idle_setup);

A
Amerigo Wang 已提交
553 554 555 556 557 558 559 560 561
unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() % 8192;
	return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
562
	return randomize_page(mm->brk, 0x02000000);
A
Amerigo Wang 已提交
563 564
}

565 566 567 568 569 570 571 572
/*
 * Called from fs/proc with a reference on @p to find the function
 * which called into schedule(). This needs to be done carefully
 * because the task might wake up and we might look at a stack
 * changing under us.
 */
unsigned long get_wchan(struct task_struct *p)
{
573
	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
574 575 576 577 578
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;

579 580 581
	if (!try_get_task_stack(p))
		return 0;

582 583
	start = (unsigned long)task_stack_page(p);
	if (!start)
584
		goto out;
585 586 587 588 589 590 591 592

	/*
	 * Layout of the stack page:
	 *
	 * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
	 * PADDING
	 * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
	 * stack
593
	 * ----------- bottom = start
594 595 596 597 598 599 600 601 602 603
	 *
	 * The tasks stack pointer points at the location where the
	 * framepointer is stored. The data on the stack is:
	 * ... IP FP ... IP FP
	 *
	 * We need to read FP and IP, so we need to adjust the upper
	 * bound by another unsigned long.
	 */
	top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
	top -= 2 * sizeof(unsigned long);
604
	bottom = start;
605 606 607

	sp = READ_ONCE(p->thread.sp);
	if (sp < bottom || sp > top)
608
		goto out;
609

610
	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
611 612
	do {
		if (fp < bottom || fp > top)
613
			goto out;
614
		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
615 616 617 618
		if (!in_sched_functions(ip)) {
			ret = ip;
			goto out;
		}
619
		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
620
	} while (count++ < 16 && p->state != TASK_RUNNING);
621 622 623 624

out:
	put_task_stack(p);
	return ret;
625
}
626 627 628 629

long do_arch_prctl_common(struct task_struct *task, int option,
			  unsigned long cpuid_enabled)
{
630 631 632 633 634 635 636
	switch (option) {
	case ARCH_GET_CPUID:
		return get_cpuid_mode();
	case ARCH_SET_CPUID:
		return set_cpuid_mode(task, cpuid_enabled);
	}

637 638
	return -EINVAL;
}