process.c 15.1 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

4 5 6 7
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
8
#include <linux/prctl.h>
9 10
#include <linux/slab.h>
#include <linux/sched.h>
11
#include <linux/sched/idle.h>
12
#include <linux/sched/debug.h>
13
#include <linux/sched/task.h>
14
#include <linux/sched/task_stack.h>
15 16
#include <linux/init.h>
#include <linux/export.h>
17
#include <linux/pm.h>
18
#include <linux/tick.h>
A
Amerigo Wang 已提交
19
#include <linux/random.h>
A
Avi Kivity 已提交
20
#include <linux/user-return-notifier.h>
21 22
#include <linux/dmi.h>
#include <linux/utsname.h>
23 24 25
#include <linux/stackprotector.h>
#include <linux/tick.h>
#include <linux/cpuidle.h>
26
#include <trace/events/power.h>
27
#include <linux/hw_breakpoint.h>
28
#include <asm/cpu.h>
29
#include <asm/apic.h>
30
#include <asm/syscalls.h>
31
#include <linux/uaccess.h>
32
#include <asm/mwait.h>
33
#include <asm/fpu/internal.h>
34
#include <asm/debugreg.h>
35
#include <asm/nmi.h>
A
Andy Lutomirski 已提交
36
#include <asm/tlbflush.h>
37
#include <asm/mce.h>
38
#include <asm/vm86.h>
39
#include <asm/switch_to.h>
40
#include <asm/desc.h>
41
#include <asm/prctl.h>
42

T
Thomas Gleixner 已提交
43 44 45 46 47 48 49
/*
 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 * no more per-task TSS's. The TSS size is kept cacheline-aligned
 * so they are allowed to end up in the .data..cacheline_aligned
 * section. Since TSS's are completely CPU-local, we want them
 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 */
50 51
__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
	.x86_tss = {
52 53 54 55 56 57 58
		/*
		 * .sp0 is only used when entering ring 0 from a lower
		 * privilege level.  Since the init task never runs anything
		 * but ring 0 code, there is no need for a valid value here.
		 * Poison it.
		 */
		.sp0 = (1UL << (BITS_PER_LONG-1)) + 1,
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
#ifdef CONFIG_X86_32
		.ss0 = __KERNEL_DS,
		.ss1 = __KERNEL_CS,
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
#endif
	 },
#ifdef CONFIG_X86_32
	 /*
	  * Note that the .io_bitmap member must be extra-big. This is because
	  * the CPU will access an additional byte beyond the end of the IO
	  * permission bitmap. The extra byte must be all 1 bits, and must
	  * be within the limit.
	  */
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },
#endif
74 75 76
#ifdef CONFIG_X86_32
	.SYSENTER_stack_canary	= STACK_END_MAGIC,
#endif
77
};
78
EXPORT_PER_CPU_SYMBOL(cpu_tss);
T
Thomas Gleixner 已提交
79

80 81
DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
82

83 84 85 86
/*
 * this gets called so that we can store lazy state into memory and copy the
 * current task into the new thread.
 */
87 88
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
89
	memcpy(dst, src, arch_task_struct_size);
90 91 92
#ifdef CONFIG_VM86
	dst->thread.vm86 = NULL;
#endif
93

94
	return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
95
}
96

97 98 99
/*
 * Free current thread data structures etc..
 */
100
void exit_thread(struct task_struct *tsk)
101
{
102
	struct thread_struct *t = &tsk->thread;
103
	unsigned long *bp = t->io_bitmap_ptr;
104
	struct fpu *fpu = &t->fpu;
105

106
	if (bp) {
107
		struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
108 109 110 111 112 113 114 115 116

		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
		t->io_bitmap_max = 0;
		put_cpu();
117
		kfree(bp);
118
	}
119

120 121
	free_vm86(t);

122
	fpu__drop(fpu);
123 124 125 126 127 128
}

void flush_thread(void)
{
	struct task_struct *tsk = current;

129
	flush_ptrace_hw_breakpoint(tsk);
130
	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
131

132
	fpu__clear(&tsk->thread.fpu);
133 134 135 136 137 138 139 140 141 142
}

void disable_TSC(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
143
		cr4_set_bits(X86_CR4_TSD);
144 145 146 147 148 149 150 151 152 153 154
	preempt_enable();
}

static void enable_TSC(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
155
		cr4_clear_bits(X86_CR4_TSD);
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
	preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
	unsigned int val;

	if (test_thread_flag(TIF_NOTSC))
		val = PR_TSC_SIGSEGV;
	else
		val = PR_TSC_ENABLE;

	return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
	if (val == PR_TSC_SIGSEGV)
		disable_TSC();
	else if (val == PR_TSC_ENABLE)
		enable_TSC();
	else
		return -EINVAL;

	return 0;
}

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
DEFINE_PER_CPU(u64, msr_misc_features_shadow);

static void set_cpuid_faulting(bool on)
{
	u64 msrval;

	msrval = this_cpu_read(msr_misc_features_shadow);
	msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;
	msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);
	this_cpu_write(msr_misc_features_shadow, msrval);
	wrmsrl(MSR_MISC_FEATURES_ENABLES, msrval);
}

static void disable_cpuid(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(true);
	}
	preempt_enable();
}

static void enable_cpuid(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(false);
	}
	preempt_enable();
}

static int get_cpuid_mode(void)
{
	return !test_thread_flag(TIF_NOCPUID);
}

static int set_cpuid_mode(struct task_struct *task, unsigned long cpuid_enabled)
{
	if (!static_cpu_has(X86_FEATURE_CPUID_FAULT))
		return -ENODEV;

	if (cpuid_enabled)
		enable_cpuid();
	else
		disable_cpuid();

	return 0;
}

/*
 * Called immediately after a successful exec.
 */
void arch_setup_new_exec(void)
{
	/* If cpuid was previously disabled for this task, re-enable it. */
	if (test_thread_flag(TIF_NOCPUID))
		enable_cpuid();
}

250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
static inline void switch_to_bitmap(struct tss_struct *tss,
				    struct thread_struct *prev,
				    struct thread_struct *next,
				    unsigned long tifp, unsigned long tifn)
{
	if (tifn & _TIF_IO_BITMAP) {
		/*
		 * Copy the relevant range of the IO bitmap.
		 * Normally this is 128 bytes or less:
		 */
		memcpy(tss->io_bitmap, next->io_bitmap_ptr,
		       max(prev->io_bitmap_max, next->io_bitmap_max));
		/*
		 * Make sure that the TSS limit is correct for the CPU
		 * to notice the IO bitmap.
		 */
		refresh_tss_limit();
	} else if (tifp & _TIF_IO_BITMAP) {
		/*
		 * Clear any possible leftover bits:
		 */
		memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	}
}

275 276 277 278
void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss)
{
	struct thread_struct *prev, *next;
279
	unsigned long tifp, tifn;
280 281 282 283

	prev = &prev_p->thread;
	next = &next_p->thread;

284 285 286 287 288 289
	tifn = READ_ONCE(task_thread_info(next_p)->flags);
	tifp = READ_ONCE(task_thread_info(prev_p)->flags);
	switch_to_bitmap(tss, prev, next, tifp, tifn);

	propagate_user_return_notify(prev_p, next_p);

290 291 292
	if ((tifp & _TIF_BLOCKSTEP || tifn & _TIF_BLOCKSTEP) &&
	    arch_has_block_step()) {
		unsigned long debugctl, msk;
P
Peter Zijlstra 已提交
293

294
		rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
P
Peter Zijlstra 已提交
295
		debugctl &= ~DEBUGCTLMSR_BTF;
296 297 298
		msk = tifn & _TIF_BLOCKSTEP;
		debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT;
		wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
P
Peter Zijlstra 已提交
299
	}
300

301 302
	if ((tifp ^ tifn) & _TIF_NOTSC)
		cr4_toggle_bits(X86_CR4_TSD);
303 304 305

	if ((tifp ^ tifn) & _TIF_NOCPUID)
		set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
306 307
}

308 309 310
/*
 * Idle related variables and functions
 */
311
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
312 313
EXPORT_SYMBOL(boot_option_idle_override);

314
static void (*x86_idle)(void);
315

316 317 318 319 320 321 322
#ifndef CONFIG_SMP
static inline void play_dead(void)
{
	BUG();
}
#endif

T
Thomas Gleixner 已提交
323 324
void arch_cpu_idle_enter(void)
{
325
	tsc_verify_tsc_adjust(false);
T
Thomas Gleixner 已提交
326 327
	local_touch_nmi();
}
328

T
Thomas Gleixner 已提交
329 330 331 332
void arch_cpu_idle_dead(void)
{
	play_dead();
}
333

T
Thomas Gleixner 已提交
334 335 336 337 338
/*
 * Called from the generic idle code.
 */
void arch_cpu_idle(void)
{
339
	x86_idle();
340 341
}

342
/*
T
Thomas Gleixner 已提交
343
 * We use this if we don't have any better idle routine..
344
 */
345
void __cpuidle default_idle(void)
346
{
347
	trace_cpu_idle_rcuidle(1, smp_processor_id());
T
Thomas Gleixner 已提交
348
	safe_halt();
349
	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
350
}
351
#ifdef CONFIG_APM_MODULE
352 353 354
EXPORT_SYMBOL(default_idle);
#endif

355 356
#ifdef CONFIG_XEN
bool xen_set_default_idle(void)
357
{
358
	bool ret = !!x86_idle;
359

360
	x86_idle = default_idle;
361 362 363

	return ret;
}
364
#endif
365

366 367 368 369 370 371
void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
372
	set_cpu_online(smp_processor_id(), false);
373
	disable_local_APIC();
374
	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
375

376 377 378 379 380 381 382 383 384 385 386 387 388 389
	for (;;) {
		/*
		 * Use wbinvd followed by hlt to stop the processor. This
		 * provides support for kexec on a processor that supports
		 * SME. With kexec, going from SME inactive to SME active
		 * requires clearing cache entries so that addresses without
		 * the encryption bit set don't corrupt the same physical
		 * address that has the encryption bit set when caches are
		 * flushed. To achieve this a wbinvd is performed followed by
		 * a hlt. Even if the processor is not in the kexec/SME
		 * scenario this only adds a wbinvd to a halting processor.
		 */
		asm volatile("wbinvd; hlt" : : : "memory");
	}
390 391
}

392
/*
393 394
 * AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
 * states (local apic timer and TSC stop).
395
 */
396
static void amd_e400_idle(void)
397
{
398 399 400 401 402 403 404 405
	/*
	 * We cannot use static_cpu_has_bug() here because X86_BUG_AMD_APIC_C1E
	 * gets set after static_cpu_has() places have been converted via
	 * alternatives.
	 */
	if (!boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		default_idle();
		return;
406 407
	}

408
	tick_broadcast_enter();
409

410
	default_idle();
411

412 413 414 415 416 417 418
	/*
	 * The switch back from broadcast mode needs to be called with
	 * interrupts disabled.
	 */
	local_irq_disable();
	tick_broadcast_exit();
	local_irq_enable();
419 420
}

421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
/*
 * Intel Core2 and older machines prefer MWAIT over HALT for C1.
 * We can't rely on cpuidle installing MWAIT, because it will not load
 * on systems that support only C1 -- so the boot default must be MWAIT.
 *
 * Some AMD machines are the opposite, they depend on using HALT.
 *
 * So for default C1, which is used during boot until cpuidle loads,
 * use MWAIT-C1 on Intel HW that has it, else use HALT.
 */
static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
{
	if (c->x86_vendor != X86_VENDOR_INTEL)
		return 0;

436
	if (!cpu_has(c, X86_FEATURE_MWAIT) || static_cpu_has_bug(X86_BUG_MONITOR))
437 438 439 440 441 442
		return 0;

	return 1;
}

/*
443 444 445
 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 * with interrupts enabled and no flags, which is backwards compatible with the
 * original MWAIT implementation.
446
 */
447
static __cpuidle void mwait_idle(void)
448
{
449
	if (!current_set_polling_and_test()) {
450
		trace_cpu_idle_rcuidle(1, smp_processor_id());
451
		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
452
			mb(); /* quirk */
453
			clflush((void *)&current_thread_info()->flags);
454
			mb(); /* quirk */
455
		}
456 457 458 459 460 461

		__monitor((void *)&current_thread_info()->flags, 0, 0);
		if (!need_resched())
			__sti_mwait(0, 0);
		else
			local_irq_enable();
462
		trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
463
	} else {
464
		local_irq_enable();
465 466
	}
	__current_clr_polling();
467 468
}

469
void select_idle_routine(const struct cpuinfo_x86 *c)
470
{
471
#ifdef CONFIG_SMP
T
Thomas Gleixner 已提交
472
	if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
473
		pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
474
#endif
T
Thomas Gleixner 已提交
475
	if (x86_idle || boot_option_idle_override == IDLE_POLL)
T
Thomas Gleixner 已提交
476 477
		return;

478
	if (boot_cpu_has_bug(X86_BUG_AMD_E400)) {
479
		pr_info("using AMD E400 aware idle routine\n");
480
		x86_idle = amd_e400_idle;
481 482 483
	} else if (prefer_mwait_c1_over_halt(c)) {
		pr_info("using mwait in idle threads\n");
		x86_idle = mwait_idle;
T
Thomas Gleixner 已提交
484
	} else
485
		x86_idle = default_idle;
486 487
}

488
void amd_e400_c1e_apic_setup(void)
489
{
490 491 492 493 494 495
	if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
		local_irq_disable();
		tick_broadcast_force();
		local_irq_enable();
	}
496 497
}

498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520
void __init arch_post_acpi_subsys_init(void)
{
	u32 lo, hi;

	if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
		return;

	/*
	 * AMD E400 detection needs to happen after ACPI has been enabled. If
	 * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
	 * MSR_K8_INT_PENDING_MSG.
	 */
	rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
	if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
		return;

	boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);

	if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
		mark_tsc_unstable("TSC halt in AMD C1E");
	pr_info("System has AMD C1E enabled\n");
}

521 522
static int __init idle_setup(char *str)
{
523 524 525
	if (!str)
		return -EINVAL;

526
	if (!strcmp(str, "poll")) {
527
		pr_info("using polling idle threads\n");
528
		boot_option_idle_override = IDLE_POLL;
T
Thomas Gleixner 已提交
529
		cpu_idle_poll_ctrl(true);
530
	} else if (!strcmp(str, "halt")) {
Z
Zhao Yakui 已提交
531 532 533 534 535 536 537
		/*
		 * When the boot option of idle=halt is added, halt is
		 * forced to be used for CPU idle. In such case CPU C2/C3
		 * won't be used again.
		 * To continue to load the CPU idle driver, don't touch
		 * the boot_option_idle_override.
		 */
538
		x86_idle = default_idle;
539
		boot_option_idle_override = IDLE_HALT;
540 541 542 543 544 545 546
	} else if (!strcmp(str, "nomwait")) {
		/*
		 * If the boot option of "idle=nomwait" is added,
		 * it means that mwait will be disabled for CPU C2/C3
		 * states. In such case it won't touch the variable
		 * of boot_option_idle_override.
		 */
547
		boot_option_idle_override = IDLE_NOMWAIT;
Z
Zhao Yakui 已提交
548
	} else
549 550 551 552 553 554
		return -1;

	return 0;
}
early_param("idle", idle_setup);

A
Amerigo Wang 已提交
555 556 557 558 559 560 561 562 563
unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() % 8192;
	return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
564
	return randomize_page(mm->brk, 0x02000000);
A
Amerigo Wang 已提交
565 566
}

567 568 569 570 571 572 573 574
/*
 * Called from fs/proc with a reference on @p to find the function
 * which called into schedule(). This needs to be done carefully
 * because the task might wake up and we might look at a stack
 * changing under us.
 */
unsigned long get_wchan(struct task_struct *p)
{
575
	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
576 577 578 579 580
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;

581 582 583
	if (!try_get_task_stack(p))
		return 0;

584 585
	start = (unsigned long)task_stack_page(p);
	if (!start)
586
		goto out;
587 588 589 590 591 592 593 594

	/*
	 * Layout of the stack page:
	 *
	 * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
	 * PADDING
	 * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
	 * stack
595
	 * ----------- bottom = start
596 597 598 599 600 601 602 603 604 605
	 *
	 * The tasks stack pointer points at the location where the
	 * framepointer is stored. The data on the stack is:
	 * ... IP FP ... IP FP
	 *
	 * We need to read FP and IP, so we need to adjust the upper
	 * bound by another unsigned long.
	 */
	top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
	top -= 2 * sizeof(unsigned long);
606
	bottom = start;
607 608 609

	sp = READ_ONCE(p->thread.sp);
	if (sp < bottom || sp > top)
610
		goto out;
611

612
	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
613 614
	do {
		if (fp < bottom || fp > top)
615
			goto out;
616
		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
617 618 619 620
		if (!in_sched_functions(ip)) {
			ret = ip;
			goto out;
		}
621
		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
622
	} while (count++ < 16 && p->state != TASK_RUNNING);
623 624 625 626

out:
	put_task_stack(p);
	return ret;
627
}
628 629 630 631

long do_arch_prctl_common(struct task_struct *task, int option,
			  unsigned long cpuid_enabled)
{
632 633 634 635 636 637 638
	switch (option) {
	case ARCH_GET_CPUID:
		return get_cpuid_mode();
	case ARCH_SET_CPUID:
		return set_cpuid_mode(task, cpuid_enabled);
	}

639 640
	return -EINVAL;
}