sched.h 56.3 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
L
Linus Torvalds 已提交
2 3 4
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H

5 6 7 8
/*
 * Define 'struct task_struct' and provide the main scheduler
 * APIs (schedule(), wakeup variants, etc.)
 */
9

10
#include <uapi/linux/sched.h>
11

12
#include <asm/current.h>
L
Linus Torvalds 已提交
13

14
#include <linux/pid.h>
L
Linus Torvalds 已提交
15
#include <linux/sem.h>
16
#include <linux/shm.h>
17 18 19 20
#include <linux/kcov.h>
#include <linux/mutex.h>
#include <linux/plist.h>
#include <linux/hrtimer.h>
21
#include <linux/irqflags.h>
L
Linus Torvalds 已提交
22
#include <linux/seccomp.h>
23
#include <linux/nodemask.h>
24
#include <linux/rcupdate.h>
25
#include <linux/refcount.h>
26
#include <linux/resource.h>
A
Arjan van de Ven 已提交
27
#include <linux/latencytop.h>
28
#include <linux/sched/prio.h>
29
#include <linux/sched/types.h>
30 31 32
#include <linux/signal_types.h>
#include <linux/mm_types_task.h>
#include <linux/task_io_accounting.h>
33
#include <linux/posix-timers.h>
34
#include <linux/rseq.h>
35
#include <linux/kcsan.h>
36

37
/* task_struct member predeclarations (sorted alphabetically): */
38 39
struct audit_context;
struct backing_dev_info;
40
struct bio_list;
41
struct blk_plug;
42
struct capture_control;
43 44 45 46 47
struct cfs_rq;
struct fs_struct;
struct futex_pi_state;
struct io_context;
struct mempolicy;
48
struct nameidata;
49 50 51 52 53 54 55
struct nsproxy;
struct perf_event_context;
struct pid_namespace;
struct pipe_inode_info;
struct rcu_node;
struct reclaim_state;
struct robust_list_head;
56 57
struct root_domain;
struct rq;
58 59
struct sched_attr;
struct sched_param;
I
Ingo Molnar 已提交
60
struct seq_file;
61 62 63
struct sighand_struct;
struct signal_struct;
struct task_delay_info;
64
struct task_group;
L
Linus Torvalds 已提交
65

66 67 68 69 70 71 72 73 74 75
/*
 * Task state bitmask. NOTE! These bits are also
 * encoded in fs/proc/array.c: get_task_state().
 *
 * We have two separate sets of flags: task->state
 * is about runnability, while task->exit_state are
 * about the task exiting. Confusing, but this way
 * modifying one set can't modify the other one by
 * mistake.
 */
76 77

/* Used in tsk->state: */
78 79 80 81 82
#define TASK_RUNNING			0x0000
#define TASK_INTERRUPTIBLE		0x0001
#define TASK_UNINTERRUPTIBLE		0x0002
#define __TASK_STOPPED			0x0004
#define __TASK_TRACED			0x0008
83
/* Used in tsk->exit_state: */
84 85
#define EXIT_DEAD			0x0010
#define EXIT_ZOMBIE			0x0020
86 87
#define EXIT_TRACE			(EXIT_ZOMBIE | EXIT_DEAD)
/* Used in tsk->state again: */
88 89 90 91
#define TASK_PARKED			0x0040
#define TASK_DEAD			0x0080
#define TASK_WAKEKILL			0x0100
#define TASK_WAKING			0x0200
92 93 94
#define TASK_NOLOAD			0x0400
#define TASK_NEW			0x0800
#define TASK_STATE_MAX			0x1000
95 96 97 98 99 100 101 102 103 104 105 106 107 108

/* Convenience macros for the sake of set_current_state: */
#define TASK_KILLABLE			(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
#define TASK_STOPPED			(TASK_WAKEKILL | __TASK_STOPPED)
#define TASK_TRACED			(TASK_WAKEKILL | __TASK_TRACED)

#define TASK_IDLE			(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)

/* Convenience macros for the sake of wake_up(): */
#define TASK_NORMAL			(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)

/* get_task_state(): */
#define TASK_REPORT			(TASK_RUNNING | TASK_INTERRUPTIBLE | \
					 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
109 110
					 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
					 TASK_PARKED)
111 112 113 114 115 116 117

#define task_is_traced(task)		((task->state & __TASK_TRACED) != 0)

#define task_is_stopped(task)		((task->state & __TASK_STOPPED) != 0)

#define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)

P
Peter Zijlstra 已提交
118 119
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP

120 121 122 123 124
/*
 * Special states are those that do not use the normal wait-loop pattern. See
 * the comment with set_special_state().
 */
#define is_special_task_state(state)				\
125
	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
126

P
Peter Zijlstra 已提交
127 128
#define __set_current_state(state_value)			\
	do {							\
129
		WARN_ON_ONCE(is_special_task_state(state_value));\
P
Peter Zijlstra 已提交
130 131 132
		current->task_state_change = _THIS_IP_;		\
		current->state = (state_value);			\
	} while (0)
133

P
Peter Zijlstra 已提交
134 135
#define set_current_state(state_value)				\
	do {							\
136
		WARN_ON_ONCE(is_special_task_state(state_value));\
P
Peter Zijlstra 已提交
137
		current->task_state_change = _THIS_IP_;		\
138
		smp_store_mb(current->state, (state_value));	\
P
Peter Zijlstra 已提交
139 140
	} while (0)

141 142 143 144 145 146 147 148 149
#define set_special_state(state_value)					\
	do {								\
		unsigned long flags; /* may shadow */			\
		WARN_ON_ONCE(!is_special_task_state(state_value));	\
		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
		current->task_state_change = _THIS_IP_;			\
		current->state = (state_value);				\
		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
	} while (0)
P
Peter Zijlstra 已提交
150
#else
151 152 153 154 155
/*
 * set_current_state() includes a barrier so that the write of current->state
 * is correctly serialised wrt the caller's subsequent test of whether to
 * actually sleep:
 *
156
 *   for (;;) {
157
 *	set_current_state(TASK_UNINTERRUPTIBLE);
P
Peter Zijlstra 已提交
158 159
 *	if (CONDITION)
 *	   break;
160 161 162 163 164 165
 *
 *	schedule();
 *   }
 *   __set_current_state(TASK_RUNNING);
 *
 * If the caller does not need such serialisation (because, for instance, the
P
Peter Zijlstra 已提交
166
 * CONDITION test and condition change and wakeup are under the same lock) then
167 168 169 170
 * use __set_current_state().
 *
 * The above is typically ordered against the wakeup, which does:
 *
P
Peter Zijlstra 已提交
171
 *   CONDITION = 1;
172
 *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
173
 *
P
Peter Zijlstra 已提交
174 175
 * where wake_up_state()/try_to_wake_up() executes a full memory barrier before
 * accessing p->state.
176 177 178 179
 *
 * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
 * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
 * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
180
 *
181
 * However, with slightly different timing the wakeup TASK_RUNNING store can
I
Ingo Molnar 已提交
182
 * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
183 184
 * a problem either because that will result in one extra go around the loop
 * and our @cond test will save the day.
185
 *
186
 * Also see the comments of try_to_wake_up().
187
 */
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
#define __set_current_state(state_value)				\
	current->state = (state_value)

#define set_current_state(state_value)					\
	smp_store_mb(current->state, (state_value))

/*
 * set_special_state() should be used for those states when the blocking task
 * can not use the regular condition based wait-loop. In that case we must
 * serialize against wakeups such that any possible in-flight TASK_RUNNING stores
 * will not collide with our state change.
 */
#define set_special_state(state_value)					\
	do {								\
		unsigned long flags; /* may shadow */			\
		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
		current->state = (state_value);				\
		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
	} while (0)

P
Peter Zijlstra 已提交
208 209
#endif

210 211
/* Task command name length: */
#define TASK_COMM_LEN			16
L
Linus Torvalds 已提交
212 213 214

extern void scheduler_tick(void);

215 216 217 218 219 220 221
#define	MAX_SCHEDULE_TIMEOUT		LONG_MAX

extern long schedule_timeout(long timeout);
extern long schedule_timeout_interruptible(long timeout);
extern long schedule_timeout_killable(long timeout);
extern long schedule_timeout_uninterruptible(long timeout);
extern long schedule_timeout_idle(long timeout);
L
Linus Torvalds 已提交
222
asmlinkage void schedule(void);
223
extern void schedule_preempt_disabled(void);
224
asmlinkage void preempt_schedule_irq(void);
L
Linus Torvalds 已提交
225

226 227
extern int __must_check io_schedule_prepare(void);
extern void io_schedule_finish(int token);
228
extern long io_schedule_timeout(long timeout);
229
extern void io_schedule(void);
230

231
/**
232
 * struct prev_cputime - snapshot of system and user cputime
233 234
 * @utime: time spent in user mode
 * @stime: time spent in system mode
235
 * @lock: protects the above two fields
236
 *
237 238
 * Stores previous user/system time values such that we can guarantee
 * monotonicity.
239
 */
240 241
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
242 243 244
	u64				utime;
	u64				stime;
	raw_spinlock_t			lock;
245
#endif
246 247
};

248 249 250
enum vtime_state {
	/* Task is sleeping or running in a CPU with VTIME inactive: */
	VTIME_INACTIVE = 0,
251 252
	/* Task is idle */
	VTIME_IDLE,
253 254
	/* Task runs in kernelspace in a CPU with VTIME active: */
	VTIME_SYS,
255 256
	/* Task runs in userspace in a CPU with VTIME active: */
	VTIME_USER,
257 258
	/* Task runs as guests in a CPU with VTIME active: */
	VTIME_GUEST,
259 260 261 262 263 264
};

struct vtime {
	seqcount_t		seqcount;
	unsigned long long	starttime;
	enum vtime_state	state;
265
	unsigned int		cpu;
266 267 268
	u64			utime;
	u64			stime;
	u64			gtime;
269 270
};

271 272 273 274 275 276 277 278 279 280 281 282
/*
 * Utilization clamp constraints.
 * @UCLAMP_MIN:	Minimum utilization
 * @UCLAMP_MAX:	Maximum utilization
 * @UCLAMP_CNT:	Utilization clamp constraints count
 */
enum uclamp_id {
	UCLAMP_MIN = 0,
	UCLAMP_MAX,
	UCLAMP_CNT
};

283 284 285 286 287
#ifdef CONFIG_SMP
extern struct root_domain def_root_domain;
extern struct mutex sched_domains_mutex;
#endif

L
Linus Torvalds 已提交
288
struct sched_info {
289
#ifdef CONFIG_SCHED_INFO
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
	/* Cumulative counters: */

	/* # of times we have run on this CPU: */
	unsigned long			pcount;

	/* Time spent waiting on a runqueue: */
	unsigned long long		run_delay;

	/* Timestamps: */

	/* When did we last run on a CPU? */
	unsigned long long		last_arrival;

	/* When were we last queued to run? */
	unsigned long long		last_queued;
L
Linus Torvalds 已提交
305

306
#endif /* CONFIG_SCHED_INFO */
307
};
L
Linus Torvalds 已提交
308

309 310 311 312 313 314 315
/*
 * Integer metrics need fixed point arithmetic, e.g., sched/fair
 * has a few: load, load_avg, util_avg, freq, and capacity.
 *
 * We define a basic fixed point arithmetic range, and then formalize
 * all these metrics based on that basic range.
 */
316 317
# define SCHED_FIXEDPOINT_SHIFT		10
# define SCHED_FIXEDPOINT_SCALE		(1L << SCHED_FIXEDPOINT_SHIFT)
318

319 320 321 322
/* Increase resolution of cpu_capacity calculations */
# define SCHED_CAPACITY_SHIFT		SCHED_FIXEDPOINT_SHIFT
# define SCHED_CAPACITY_SCALE		(1L << SCHED_CAPACITY_SHIFT)

I
Ingo Molnar 已提交
323
struct load_weight {
324 325
	unsigned long			weight;
	u32				inv_weight;
I
Ingo Molnar 已提交
326 327
};

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
/**
 * struct util_est - Estimation utilization of FAIR tasks
 * @enqueued: instantaneous estimated utilization of a task/cpu
 * @ewma:     the Exponential Weighted Moving Average (EWMA)
 *            utilization of a task
 *
 * Support data structure to track an Exponential Weighted Moving Average
 * (EWMA) of a FAIR task's utilization. New samples are added to the moving
 * average each time a task completes an activation. Sample's weight is chosen
 * so that the EWMA will be relatively insensitive to transient changes to the
 * task's workload.
 *
 * The enqueued attribute has a slightly different meaning for tasks and cpus:
 * - task:   the task's util_avg at last task dequeue time
 * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
 * Thus, the util_est.enqueued of a task represents the contribution on the
 * estimated utilization of the CPU where that task is currently enqueued.
 *
 * Only for tasks we track a moving average of the past instantaneous
 * estimated utilization. This allows to absorb sporadic drops in utilization
 * of an otherwise almost periodic task.
 */
struct util_est {
	unsigned int			enqueued;
	unsigned int			ewma;
#define UTIL_EST_WEIGHT_SHIFT		2
354
} __attribute__((__aligned__(sizeof(u64))));
355

356
/*
357
 * The load/runnable/util_avg accumulates an infinite geometric series
358
 * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
359 360 361 362 363
 *
 * [load_avg definition]
 *
 *   load_avg = runnable% * scale_load_down(load)
 *
364 365 366
 * [runnable_avg definition]
 *
 *   runnable_avg = runnable% * SCHED_CAPACITY_SCALE
367 368 369 370 371
 *
 * [util_avg definition]
 *
 *   util_avg = running% * SCHED_CAPACITY_SCALE
 *
372 373 374 375 376
 * where runnable% is the time ratio that a sched_entity is runnable and
 * running% the time ratio that a sched_entity is running.
 *
 * For cfs_rq, they are the aggregated values of all runnable and blocked
 * sched_entities.
377
 *
王文虎 已提交
378
 * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
379 380
 * capacity scaling. The scaling is done through the rq_clock_pelt that is used
 * for computing those signals (see update_rq_clock_pelt())
381
 *
382 383 384 385
 * N.B., the above ratios (runnable% and running%) themselves are in the
 * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
 * to as large a range as necessary. This is for example reflected by
 * util_avg's SCHED_CAPACITY_SCALE.
386 387 388 389 390 391 392 393 394 395 396 397 398 399
 *
 * [Overflow issue]
 *
 * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
 * with the highest load (=88761), always runnable on a single cfs_rq,
 * and should not overflow as the number already hits PID_MAX_LIMIT.
 *
 * For all other cases (including 32-bit kernels), struct load_weight's
 * weight will overflow first before we do, because:
 *
 *    Max(load_avg) <= Max(load.weight)
 *
 * Then it is the load_weight's responsibility to consider overflow
 * issues.
400
 */
401
struct sched_avg {
402 403
	u64				last_update_time;
	u64				load_sum;
404
	u64				runnable_sum;
405 406 407
	u32				util_sum;
	u32				period_contrib;
	unsigned long			load_avg;
408
	unsigned long			runnable_avg;
409
	unsigned long			util_avg;
410
	struct util_est			util_est;
411
} ____cacheline_aligned;
412

413
struct sched_statistics {
414
#ifdef CONFIG_SCHEDSTATS
415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
	u64				wait_start;
	u64				wait_max;
	u64				wait_count;
	u64				wait_sum;
	u64				iowait_count;
	u64				iowait_sum;

	u64				sleep_start;
	u64				sleep_max;
	s64				sum_sleep_runtime;

	u64				block_start;
	u64				block_max;
	u64				exec_max;
	u64				slice_max;

	u64				nr_migrations_cold;
	u64				nr_failed_migrations_affine;
	u64				nr_failed_migrations_running;
	u64				nr_failed_migrations_hot;
	u64				nr_forced_migrations;

	u64				nr_wakeups;
	u64				nr_wakeups_sync;
	u64				nr_wakeups_migrate;
	u64				nr_wakeups_local;
	u64				nr_wakeups_remote;
	u64				nr_wakeups_affine;
	u64				nr_wakeups_affine_attempts;
	u64				nr_wakeups_passive;
	u64				nr_wakeups_idle;
446
#endif
447
};
448 449

struct sched_entity {
450 451 452 453 454
	/* For load-balancing: */
	struct load_weight		load;
	struct rb_node			run_node;
	struct list_head		group_node;
	unsigned int			on_rq;
455

456 457 458 459
	u64				exec_start;
	u64				sum_exec_runtime;
	u64				vruntime;
	u64				prev_sum_exec_runtime;
460

461
	u64				nr_migrations;
462

463
	struct sched_statistics		statistics;
464

I
Ingo Molnar 已提交
465
#ifdef CONFIG_FAIR_GROUP_SCHED
466 467
	int				depth;
	struct sched_entity		*parent;
I
Ingo Molnar 已提交
468
	/* rq on which this entity is (to be) queued: */
469
	struct cfs_rq			*cfs_rq;
I
Ingo Molnar 已提交
470
	/* rq "owned" by this entity/group: */
471
	struct cfs_rq			*my_q;
472 473
	/* cached value of my_q->h_nr_running */
	unsigned long			runnable_weight;
I
Ingo Molnar 已提交
474
#endif
475

476
#ifdef CONFIG_SMP
477 478 479 480 481 482
	/*
	 * Per entity load average tracking.
	 *
	 * Put into separate cache line so it does not
	 * collide with read-mostly values above.
	 */
483
	struct sched_avg		avg;
484
#endif
I
Ingo Molnar 已提交
485
};
486

P
Peter Zijlstra 已提交
487
struct sched_rt_entity {
488 489 490 491 492 493 494 495
	struct list_head		run_list;
	unsigned long			timeout;
	unsigned long			watchdog_stamp;
	unsigned int			time_slice;
	unsigned short			on_rq;
	unsigned short			on_list;

	struct sched_rt_entity		*back;
496
#ifdef CONFIG_RT_GROUP_SCHED
497
	struct sched_rt_entity		*parent;
P
Peter Zijlstra 已提交
498
	/* rq on which this entity is (to be) queued: */
499
	struct rt_rq			*rt_rq;
P
Peter Zijlstra 已提交
500
	/* rq "owned" by this entity/group: */
501
	struct rt_rq			*my_q;
P
Peter Zijlstra 已提交
502
#endif
503
} __randomize_layout;
P
Peter Zijlstra 已提交
504

505
struct sched_dl_entity {
506
	struct rb_node			rb_node;
507 508 509

	/*
	 * Original scheduling parameters. Copied here from sched_attr
510 511
	 * during sched_setattr(), they will remain the same until
	 * the next sched_setattr().
512
	 */
513 514 515
	u64				dl_runtime;	/* Maximum runtime for each instance	*/
	u64				dl_deadline;	/* Relative deadline of each instance	*/
	u64				dl_period;	/* Separation of two instances (period) */
516
	u64				dl_bw;		/* dl_runtime / dl_period		*/
517
	u64				dl_density;	/* dl_runtime / dl_deadline		*/
518 519 520

	/*
	 * Actual scheduling parameters. Initialized with the values above,
I
Ingo Molnar 已提交
521
	 * they are continuously updated during task execution. Note that
522 523
	 * the remaining runtime could be < 0 in case we are in overrun.
	 */
524 525 526
	s64				runtime;	/* Remaining runtime for this instance	*/
	u64				deadline;	/* Absolute deadline for this instance	*/
	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
527 528 529 530 531 532 533 534

	/*
	 * Some bool flags:
	 *
	 * @dl_throttled tells if we exhausted the runtime. If so, the
	 * task has to wait for a replenishment to be performed at the
	 * next firing of dl_timer.
	 *
535 536
	 * @dl_boosted tells if we are boosted due to DI. If so we are
	 * outside bandwidth enforcement mechanism (but only until we
537 538
	 * exit the critical section);
	 *
539
	 * @dl_yielded tells if task gave up the CPU before consuming
540
	 * all its available runtime during the last job.
541 542 543 544 545 546 547
	 *
	 * @dl_non_contending tells if the task is inactive while still
	 * contributing to the active utilization. In other words, it
	 * indicates if the inactive timer has been armed and its handler
	 * has not been executed yet. This flag is useful to avoid race
	 * conditions between the inactive timer handler and the wakeup
	 * code.
548 549 550
	 *
	 * @dl_overrun tells if the task asked to be informed about runtime
	 * overruns.
551
	 */
552 553 554 555
	unsigned int			dl_throttled      : 1;
	unsigned int			dl_boosted        : 1;
	unsigned int			dl_yielded        : 1;
	unsigned int			dl_non_contending : 1;
556
	unsigned int			dl_overrun	  : 1;
557 558 559 560 561

	/*
	 * Bandwidth enforcement timer. Each -deadline task has its
	 * own bandwidth to be enforced, thus we need one timer per task.
	 */
562
	struct hrtimer			dl_timer;
563 564 565 566 567 568 569 570 571

	/*
	 * Inactive timer, responsible for decreasing the active utilization
	 * at the "0-lag time". When a -deadline task blocks, it contributes
	 * to GRUB's active utilization until the "0-lag time", hence a
	 * timer is needed to decrease the active utilization at the correct
	 * time.
	 */
	struct hrtimer inactive_timer;
572
};
573

574 575 576 577 578 579 580 581
#ifdef CONFIG_UCLAMP_TASK
/* Number of utilization clamp buckets (shorter alias) */
#define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT

/*
 * Utilization clamp for a scheduling entity
 * @value:		clamp value "assigned" to a se
 * @bucket_id:		bucket index corresponding to the "assigned" value
582
 * @active:		the se is currently refcounted in a rq's bucket
583
 * @user_defined:	the requested clamp value comes from user-space
584 585 586 587
 *
 * The bucket_id is the index of the clamp bucket matching the clamp value
 * which is pre-computed and stored to avoid expensive integer divisions from
 * the fast path.
588 589 590 591 592
 *
 * The active bit is set whenever a task has got an "effective" value assigned,
 * which can be different from the clamp value "requested" from user-space.
 * This allows to know a task is refcounted in the rq's bucket corresponding
 * to the "effective" bucket_id.
593 594 595 596 597 598 599
 *
 * The user_defined bit is set whenever a task has got a task-specific clamp
 * value requested from userspace, i.e. the system defaults apply to this task
 * just as a restriction. This allows to relax default clamps when a less
 * restrictive task-specific value has been requested, thus allowing to
 * implement a "nice" semantic. For example, a task running with a 20%
 * default boost can still drop its own boosting to 0%.
600 601 602 603
 */
struct uclamp_se {
	unsigned int value		: bits_per(SCHED_CAPACITY_SCALE);
	unsigned int bucket_id		: bits_per(UCLAMP_BUCKETS);
604
	unsigned int active		: 1;
605
	unsigned int user_defined	: 1;
606 607 608
};
#endif /* CONFIG_UCLAMP_TASK */

609 610
union rcu_special {
	struct {
611 612
		u8			blocked;
		u8			need_qs;
613
		u8			exp_hint; /* Hint for performance. */
614
		u8			need_mb; /* Readers need smp_mb(). */
615
	} b; /* Bits. */
616
	u32 s; /* Set of bits. */
617
};
618

P
Peter Zijlstra 已提交
619 620 621
enum perf_event_task_context {
	perf_invalid_context = -1,
	perf_hw_context = 0,
622
	perf_sw_context,
P
Peter Zijlstra 已提交
623 624 625
	perf_nr_task_contexts,
};

626 627 628 629
struct wake_q_node {
	struct wake_q_node *next;
};

L
Linus Torvalds 已提交
630
struct task_struct {
631 632 633 634 635
#ifdef CONFIG_THREAD_INFO_IN_TASK
	/*
	 * For reasons of header soup (see current_thread_info()), this
	 * must be the first element of task_struct.
	 */
636
	struct thread_info		thread_info;
637
#endif
638 639
	/* -1 unrunnable, 0 runnable, >0 stopped: */
	volatile long			state;
K
Kees Cook 已提交
640 641 642 643 644 645 646

	/*
	 * This begins the randomizable portion of task_struct. Only
	 * scheduling-critical items should be added above here.
	 */
	randomized_struct_fields_start

647
	void				*stack;
648
	refcount_t			usage;
649 650 651
	/* Per task flags (PF_*), defined further below: */
	unsigned int			flags;
	unsigned int			ptrace;
L
Linus Torvalds 已提交
652

653
#ifdef CONFIG_SMP
654
	int				on_cpu;
655
	struct __call_single_node	wake_entry;
656
#ifdef CONFIG_THREAD_INFO_IN_TASK
657 658
	/* Current CPU: */
	unsigned int			cpu;
659
#endif
660 661 662
	unsigned int			wakee_flips;
	unsigned long			wakee_flip_decay_ts;
	struct task_struct		*last_wakee;
663

664 665 666 667 668 669 670 671
	/*
	 * recent_used_cpu is initially set as the last CPU used by a task
	 * that wakes affine another task. Waker/wakee relationships can
	 * push tasks around a CPU where each wakeup moves to the next one.
	 * Tracking a recently used CPU allows a quick search for a recently
	 * used CPU that may be idle.
	 */
	int				recent_used_cpu;
672
	int				wake_cpu;
673
#endif
674 675 676 677 678 679
	int				on_rq;

	int				prio;
	int				static_prio;
	int				normal_prio;
	unsigned int			rt_priority;
680

681 682 683
	const struct sched_class	*sched_class;
	struct sched_entity		se;
	struct sched_rt_entity		rt;
P
Peter Zijlstra 已提交
684
#ifdef CONFIG_CGROUP_SCHED
685
	struct task_group		*sched_task_group;
P
Peter Zijlstra 已提交
686
#endif
687
	struct sched_dl_entity		dl;
L
Linus Torvalds 已提交
688

689
#ifdef CONFIG_UCLAMP_TASK
690 691 692 693
	/*
	 * Clamp values requested for a scheduling entity.
	 * Must be updated with task_rq_lock() held.
	 */
694
	struct uclamp_se		uclamp_req[UCLAMP_CNT];
695 696 697 698
	/*
	 * Effective clamp values used for a scheduling entity.
	 * Must be updated with task_rq_lock() held.
	 */
699 700 701
	struct uclamp_se		uclamp[UCLAMP_CNT];
#endif

702
#ifdef CONFIG_PREEMPT_NOTIFIERS
703 704
	/* List of struct preempt_notifier: */
	struct hlist_head		preempt_notifiers;
705 706
#endif

707
#ifdef CONFIG_BLK_DEV_IO_TRACE
708
	unsigned int			btrace_seq;
709
#endif
L
Linus Torvalds 已提交
710

711 712
	unsigned int			policy;
	int				nr_cpus_allowed;
713 714
	const cpumask_t			*cpus_ptr;
	cpumask_t			cpus_mask;
L
Linus Torvalds 已提交
715

P
Paul E. McKenney 已提交
716
#ifdef CONFIG_PREEMPT_RCU
717 718 719 720
	int				rcu_read_lock_nesting;
	union rcu_special		rcu_read_unlock_special;
	struct list_head		rcu_node_entry;
	struct rcu_node			*rcu_blocked_node;
721
#endif /* #ifdef CONFIG_PREEMPT_RCU */
722

P
Paul E. McKenney 已提交
723
#ifdef CONFIG_TASKS_RCU
724
	unsigned long			rcu_tasks_nvcsw;
725 726
	u8				rcu_tasks_holdout;
	u8				rcu_tasks_idx;
727
	int				rcu_tasks_idle_cpu;
728
	struct list_head		rcu_tasks_holdout_list;
P
Paul E. McKenney 已提交
729
#endif /* #ifdef CONFIG_TASKS_RCU */
P
Paul E. McKenney 已提交
730

731 732 733
#ifdef CONFIG_TASKS_TRACE_RCU
	int				trc_reader_nesting;
	int				trc_ipi_to_cpu;
734
	union rcu_special		trc_reader_special;
735 736 737 738
	bool				trc_reader_checked;
	struct list_head		trc_holdout_list;
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */

739
	struct sched_info		sched_info;
L
Linus Torvalds 已提交
740

741
	struct list_head		tasks;
742
#ifdef CONFIG_SMP
743 744
	struct plist_node		pushable_tasks;
	struct rb_node			pushable_dl_tasks;
745
#endif
L
Linus Torvalds 已提交
746

747 748
	struct mm_struct		*mm;
	struct mm_struct		*active_mm;
749 750

	/* Per-thread vma caching: */
751
	struct vmacache			vmacache;
752

753 754
#ifdef SPLIT_RSS_COUNTING
	struct task_rss_stat		rss_stat;
755
#endif
756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771
	int				exit_state;
	int				exit_code;
	int				exit_signal;
	/* The signal sent when the parent dies: */
	int				pdeath_signal;
	/* JOBCTL_*, siglock protected: */
	unsigned long			jobctl;

	/* Used for emulating ABI behavior of previous Linux versions: */
	unsigned int			personality;

	/* Scheduler bits, serialized by scheduler locks: */
	unsigned			sched_reset_on_fork:1;
	unsigned			sched_contributes_to_load:1;
	unsigned			sched_migrated:1;
	unsigned			sched_remote_wakeup:1;
772 773 774 775
#ifdef CONFIG_PSI
	unsigned			sched_psi_wake_requeue:1;
#endif

776 777 778 779 780 781 782 783 784 785
	/* Force alignment to the next boundary: */
	unsigned			:0;

	/* Unserialized, strictly 'current' */

	/* Bit to tell LSMs we're in execve(): */
	unsigned			in_execve:1;
	unsigned			in_iowait:1;
#ifndef TIF_RESTORE_SIGMASK
	unsigned			restore_sigmask:1;
786
#endif
T
Tejun Heo 已提交
787
#ifdef CONFIG_MEMCG
788
	unsigned			in_user_fault:1;
789
#endif
790
#ifdef CONFIG_COMPAT_BRK
791
	unsigned			brk_randomized:1;
792
#endif
793 794 795
#ifdef CONFIG_CGROUPS
	/* disallow userland-initiated cgroup migration */
	unsigned			no_cgroup_migration:1;
R
Roman Gushchin 已提交
796 797
	/* task is frozen/stopped (used by the cgroup freezer) */
	unsigned			frozen:1;
798
#endif
799 800 801
#ifdef CONFIG_BLK_CGROUP
	unsigned			use_memdelay:1;
#endif
802 803 804 805
#ifdef CONFIG_PSI
	/* Stalled due to lack of memory */
	unsigned			in_memstall:1;
#endif
806

807
	unsigned long			atomic_flags; /* Flags requiring atomic access. */
808

809
	struct restart_block		restart_block;
810

811 812
	pid_t				pid;
	pid_t				tgid;
813

814
#ifdef CONFIG_STACKPROTECTOR
815 816
	/* Canary value for the -fstack-protector GCC feature: */
	unsigned long			stack_canary;
817
#endif
818
	/*
819
	 * Pointers to the (original) parent process, youngest child, younger sibling,
820
	 * older sibling, respectively.  (p->father can be replaced with
R
Roland McGrath 已提交
821
	 * p->real_parent->pid)
L
Linus Torvalds 已提交
822
	 */
823 824 825 826 827 828 829

	/* Real parent process: */
	struct task_struct __rcu	*real_parent;

	/* Recipient of SIGCHLD, wait4() reports: */
	struct task_struct __rcu	*parent;

L
Linus Torvalds 已提交
830
	/*
831
	 * Children/sibling form the list of natural children:
L
Linus Torvalds 已提交
832
	 */
833 834 835
	struct list_head		children;
	struct list_head		sibling;
	struct task_struct		*group_leader;
L
Linus Torvalds 已提交
836

R
Roland McGrath 已提交
837
	/*
838 839
	 * 'ptraced' is the list of tasks this task is using ptrace() on.
	 *
R
Roland McGrath 已提交
840
	 * This includes both natural children and PTRACE_ATTACH targets.
841
	 * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
R
Roland McGrath 已提交
842
	 */
843 844
	struct list_head		ptraced;
	struct list_head		ptrace_entry;
R
Roland McGrath 已提交
845

L
Linus Torvalds 已提交
846
	/* PID/PID hash table linkage. */
847 848
	struct pid			*thread_pid;
	struct hlist_node		pid_links[PIDTYPE_MAX];
849 850 851 852
	struct list_head		thread_group;
	struct list_head		thread_node;

	struct completion		*vfork_done;
L
Linus Torvalds 已提交
853

854 855
	/* CLONE_CHILD_SETTID: */
	int __user			*set_child_tid;
L
Linus Torvalds 已提交
856

857 858 859 860 861
	/* CLONE_CHILD_CLEARTID: */
	int __user			*clear_child_tid;

	u64				utime;
	u64				stime;
862
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
863 864
	u64				utimescaled;
	u64				stimescaled;
865
#endif
866 867
	u64				gtime;
	struct prev_cputime		prev_cputime;
868
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
869
	struct vtime			vtime;
870
#endif
871 872

#ifdef CONFIG_NO_HZ_FULL
873
	atomic_t			tick_dep_mask;
874
#endif
875 876 877 878 879 880 881 882
	/* Context switch counts: */
	unsigned long			nvcsw;
	unsigned long			nivcsw;

	/* Monotonic time in nsecs: */
	u64				start_time;

	/* Boot based time in nsecs: */
883
	u64				start_boottime;
884 885 886 887

	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
	unsigned long			min_flt;
	unsigned long			maj_flt;
L
Linus Torvalds 已提交
888

889 890
	/* Empty if CONFIG_POSIX_CPUTIMERS=n */
	struct posix_cputimers		posix_cputimers;
L
Linus Torvalds 已提交
891

892 893 894 895 896 897 898 899 900 901 902
	/* Process credentials: */

	/* Tracer's credentials at attach: */
	const struct cred __rcu		*ptracer_cred;

	/* Objective and real subjective task credentials (COW): */
	const struct cred __rcu		*real_cred;

	/* Effective (overridable) subjective task credentials (COW): */
	const struct cred __rcu		*cred;

903 904 905 906 907
#ifdef CONFIG_KEYS
	/* Cached requested key. */
	struct key			*cached_requested_key;
#endif

908 909 910 911 912 913 914 915 916 917 918
	/*
	 * executable name, excluding path.
	 *
	 * - normally initialized setup_new_exec()
	 * - access it with [gs]et_task_comm()
	 * - lock it with task_lock()
	 */
	char				comm[TASK_COMM_LEN];

	struct nameidata		*nameidata;

919
#ifdef CONFIG_SYSVIPC
920 921
	struct sysv_sem			sysvsem;
	struct sysv_shm			sysvshm;
922
#endif
923
#ifdef CONFIG_DETECT_HUNG_TASK
924
	unsigned long			last_switch_count;
925
	unsigned long			last_switch_time;
926
#endif
927 928 929 930 931 932 933 934 935 936 937
	/* Filesystem information: */
	struct fs_struct		*fs;

	/* Open file information: */
	struct files_struct		*files;

	/* Namespaces: */
	struct nsproxy			*nsproxy;

	/* Signal handlers: */
	struct signal_struct		*signal;
938
	struct sighand_struct __rcu		*sighand;
939 940 941 942 943 944 945 946 947 948 949
	sigset_t			blocked;
	sigset_t			real_blocked;
	/* Restored if set_restore_sigmask() was used: */
	sigset_t			saved_sigmask;
	struct sigpending		pending;
	unsigned long			sas_ss_sp;
	size_t				sas_ss_size;
	unsigned int			sas_ss_flags;

	struct callback_head		*task_works;

950
#ifdef CONFIG_AUDIT
A
Al Viro 已提交
951
#ifdef CONFIG_AUDITSYSCALL
952 953
	struct audit_context		*audit_context;
#endif
954 955
	kuid_t				loginuid;
	unsigned int			sessionid;
A
Al Viro 已提交
956
#endif
957 958 959
	struct seccomp			seccomp;

	/* Thread group tracking: */
960 961
	u64				parent_exec_id;
	u64				self_exec_id;
L
Linus Torvalds 已提交
962

963 964
	/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
	spinlock_t			alloc_lock;
L
Linus Torvalds 已提交
965

966
	/* Protection of the PI data structures: */
967
	raw_spinlock_t			pi_lock;
968

969
	struct wake_q_node		wake_q;
970

I
Ingo Molnar 已提交
971
#ifdef CONFIG_RT_MUTEXES
972
	/* PI waiters blocked on a rt_mutex held by this task: */
973
	struct rb_root_cached		pi_waiters;
974 975
	/* Updated under owner's pi_lock and rq lock */
	struct task_struct		*pi_top_task;
976 977
	/* Deadlock detection and priority inheritance handling: */
	struct rt_mutex_waiter		*pi_blocked_on;
I
Ingo Molnar 已提交
978 979
#endif

980
#ifdef CONFIG_DEBUG_MUTEXES
981 982
	/* Mutex deadlock detection: */
	struct mutex_waiter		*blocked_on;
983
#endif
984

985 986 987 988
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
	int				non_block_count;
#endif

989
#ifdef CONFIG_TRACE_IRQFLAGS
990
	struct irqtrace_events		irqtrace;
991
	unsigned int			hardirq_threaded;
992
	u64				hardirq_chain_key;
993 994
	int				softirqs_enabled;
	int				softirq_context;
995
	int				irq_config;
996
#endif
997

I
Ingo Molnar 已提交
998
#ifdef CONFIG_LOCKDEP
999 1000 1001 1002 1003
# define MAX_LOCK_DEPTH			48UL
	u64				curr_chain_key;
	int				lockdep_depth;
	unsigned int			lockdep_recursion;
	struct held_lock		held_locks[MAX_LOCK_DEPTH];
I
Ingo Molnar 已提交
1004
#endif
1005

1006
#ifdef CONFIG_UBSAN
1007
	unsigned int			in_ubsan;
1008
#endif
1009

1010 1011
	/* Journalling filesystem info: */
	void				*journal_info;
L
Linus Torvalds 已提交
1012

1013 1014
	/* Stacked block device info: */
	struct bio_list			*bio_list;
1015

1016
#ifdef CONFIG_BLOCK
1017 1018
	/* Stack plugging: */
	struct blk_plug			*plug;
1019 1020
#endif

1021 1022 1023 1024
	/* VM state: */
	struct reclaim_state		*reclaim_state;

	struct backing_dev_info		*backing_dev_info;
L
Linus Torvalds 已提交
1025

1026
	struct io_context		*io_context;
L
Linus Torvalds 已提交
1027

1028 1029 1030
#ifdef CONFIG_COMPACTION
	struct capture_control		*capture_control;
#endif
1031 1032
	/* Ptrace state: */
	unsigned long			ptrace_message;
1033
	kernel_siginfo_t		*last_siginfo;
L
Linus Torvalds 已提交
1034

1035
	struct task_io_accounting	ioac;
1036 1037 1038 1039
#ifdef CONFIG_PSI
	/* Pressure stall state */
	unsigned int			psi_flags;
#endif
1040 1041 1042 1043 1044 1045 1046
#ifdef CONFIG_TASK_XACCT
	/* Accumulated RSS usage: */
	u64				acct_rss_mem1;
	/* Accumulated virtual memory usage: */
	u64				acct_vm_mem1;
	/* stime + utime since last update: */
	u64				acct_timexpd;
L
Linus Torvalds 已提交
1047 1048
#endif
#ifdef CONFIG_CPUSETS
1049 1050 1051 1052 1053 1054
	/* Protected by ->alloc_lock: */
	nodemask_t			mems_allowed;
	/* Seqence number to catch updates: */
	seqcount_t			mems_allowed_seq;
	int				cpuset_mem_spread_rotor;
	int				cpuset_slab_spread_rotor;
L
Linus Torvalds 已提交
1055
#endif
1056
#ifdef CONFIG_CGROUPS
1057 1058 1059 1060
	/* Control Group info protected by css_set_lock: */
	struct css_set __rcu		*cgroups;
	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
	struct list_head		cg_list;
1061
#endif
1062
#ifdef CONFIG_X86_CPU_RESCTRL
1063
	u32				closid;
1064
	u32				rmid;
F
Fenghua Yu 已提交
1065
#endif
1066
#ifdef CONFIG_FUTEX
1067
	struct robust_list_head __user	*robust_list;
1068 1069 1070
#ifdef CONFIG_COMPAT
	struct compat_robust_list_head __user *compat_robust_list;
#endif
1071 1072
	struct list_head		pi_state_list;
	struct futex_pi_state		*pi_state_cache;
1073
	struct mutex			futex_exit_mutex;
1074
	unsigned int			futex_state;
1075
#endif
1076
#ifdef CONFIG_PERF_EVENTS
1077 1078 1079
	struct perf_event_context	*perf_event_ctxp[perf_nr_task_contexts];
	struct mutex			perf_event_mutex;
	struct list_head		perf_event_list;
1080
#endif
1081
#ifdef CONFIG_DEBUG_PREEMPT
1082
	unsigned long			preempt_disable_ip;
1083
#endif
1084
#ifdef CONFIG_NUMA
1085 1086
	/* Protected by alloc_lock: */
	struct mempolicy		*mempolicy;
1087
	short				il_prev;
1088
	short				pref_node_fork;
1089
#endif
1090
#ifdef CONFIG_NUMA_BALANCING
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
	int				numa_scan_seq;
	unsigned int			numa_scan_period;
	unsigned int			numa_scan_period_max;
	int				numa_preferred_nid;
	unsigned long			numa_migrate_retry;
	/* Migration stamp: */
	u64				node_stamp;
	u64				last_task_numa_placement;
	u64				last_sum_exec_runtime;
	struct callback_head		numa_work;

1102 1103 1104 1105 1106 1107 1108 1109 1110
	/*
	 * This pointer is only modified for current in syscall and
	 * pagefault context (and for tasks being destroyed), so it can be read
	 * from any of the following contexts:
	 *  - RCU read-side critical section
	 *  - current->numa_group from everywhere
	 *  - task's runqueue locked, task not running
	 */
	struct numa_group __rcu		*numa_group;
1111

1112
	/*
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
	 * numa_faults is an array split into four regions:
	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
	 * in this precise order.
	 *
	 * faults_memory: Exponential decaying average of faults on a per-node
	 * basis. Scheduling placement decisions are made based on these
	 * counts. The values remain static for the duration of a PTE scan.
	 * faults_cpu: Track the nodes the process was running on when a NUMA
	 * hinting fault was incurred.
	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
	 * during the current scan window. When the scan completes, the counts
	 * in faults_memory and faults_cpu decay and these values are copied.
1125
	 */
1126 1127
	unsigned long			*numa_faults;
	unsigned long			total_numa_faults;
1128

1129 1130
	/*
	 * numa_faults_locality tracks if faults recorded during the last
1131 1132 1133
	 * scan window were remote/local or failed to migrate. The task scan
	 * period is adapted based on the locality of the faults with different
	 * weights depending on whether they were shared or private faults
1134
	 */
1135
	unsigned long			numa_faults_locality[3];
1136

1137
	unsigned long			numa_pages_migrated;
1138 1139
#endif /* CONFIG_NUMA_BALANCING */

1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
#ifdef CONFIG_RSEQ
	struct rseq __user *rseq;
	u32 rseq_sig;
	/*
	 * RmW on rseq_event_mask must be performed atomically
	 * with respect to preemption.
	 */
	unsigned long rseq_event_mask;
#endif

1150
	struct tlbflush_unmap_batch	tlb_ubc;
1151

1152 1153 1154 1155
	union {
		refcount_t		rcu_users;
		struct rcu_head		rcu;
	};
1156

1157 1158
	/* Cache last used pipe for splice(): */
	struct pipe_inode_info		*splice_pipe;
1159

1160
	struct page_frag		task_frag;
1161

1162 1163
#ifdef CONFIG_TASK_DELAY_ACCT
	struct task_delay_info		*delays;
1164
#endif
1165

1166
#ifdef CONFIG_FAULT_INJECTION
1167
	int				make_it_fail;
1168
	unsigned int			fail_nth;
1169
#endif
1170
	/*
1171 1172
	 * When (nr_dirtied >= nr_dirtied_pause), it's time to call
	 * balance_dirty_pages() for a dirty throttling pause:
1173
	 */
1174 1175 1176 1177
	int				nr_dirtied;
	int				nr_dirtied_pause;
	/* Start of a write-and-pause period: */
	unsigned long			dirty_paused_when;
1178

A
Arjan van de Ven 已提交
1179
#ifdef CONFIG_LATENCYTOP
1180 1181
	int				latency_record_count;
	struct latency_record		latency_record[LT_SAVECOUNT];
A
Arjan van de Ven 已提交
1182
#endif
1183
	/*
1184
	 * Time slack values; these are used to round up poll() and
1185 1186
	 * select() etc timeout values. These are in nanoseconds.
	 */
1187 1188
	u64				timer_slack_ns;
	u64				default_timer_slack_ns;
1189

1190
#ifdef CONFIG_KASAN
1191
	unsigned int			kasan_depth;
1192
#endif
1193

1194 1195
#ifdef CONFIG_KCSAN
	struct kcsan_ctx		kcsan_ctx;
1196 1197 1198
#ifdef CONFIG_TRACE_IRQFLAGS
	struct irqtrace_events		kcsan_save_irqtrace;
#endif
1199
#endif
1200

1201
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1202 1203
	/* Index of current stored address in ret_stack: */
	int				curr_ret_stack;
1204
	int				curr_ret_depth;
1205 1206 1207 1208 1209 1210 1211

	/* Stack of return addresses for return function tracing: */
	struct ftrace_ret_stack		*ret_stack;

	/* Timestamp for last schedule: */
	unsigned long long		ftrace_timestamp;

1212 1213
	/*
	 * Number of functions that haven't been traced
1214
	 * because of depth overrun:
1215
	 */
1216 1217 1218 1219
	atomic_t			trace_overrun;

	/* Pause tracing: */
	atomic_t			tracing_graph_pause;
1220
#endif
1221

1222
#ifdef CONFIG_TRACING
1223 1224 1225 1226 1227
	/* State flags for use by tracers: */
	unsigned long			trace;

	/* Bitmask and counter of trace recursion: */
	unsigned long			trace_recursion;
1228
#endif /* CONFIG_TRACING */
1229

D
Dmitry Vyukov 已提交
1230
#ifdef CONFIG_KCOV
A
Andrey Konovalov 已提交
1231 1232
	/* See kernel/kcov.c for more details. */

1233
	/* Coverage collection mode enabled for this task (0 if disabled): */
1234
	unsigned int			kcov_mode;
1235 1236 1237 1238 1239 1240 1241 1242 1243

	/* Size of the kcov_area: */
	unsigned int			kcov_size;

	/* Buffer for coverage collection: */
	void				*kcov_area;

	/* KCOV descriptor wired with this task or NULL: */
	struct kcov			*kcov;
A
Andrey Konovalov 已提交
1244 1245 1246 1247 1248 1249

	/* KCOV common handle for remote coverage collection: */
	u64				kcov_handle;

	/* KCOV sequence number: */
	int				kcov_sequence;
1250 1251 1252

	/* Collect coverage from softirq context: */
	unsigned int			kcov_softirq;
D
Dmitry Vyukov 已提交
1253
#endif
1254

1255
#ifdef CONFIG_MEMCG
1256 1257 1258
	struct mem_cgroup		*memcg_in_oom;
	gfp_t				memcg_oom_gfp_mask;
	int				memcg_oom_order;
1259

1260 1261
	/* Number of pages to reclaim on returning to userland: */
	unsigned int			memcg_nr_pages_over_high;
1262 1263 1264

	/* Used by memcontrol for targeted memcg charge: */
	struct mem_cgroup		*active_memcg;
1265
#endif
1266

1267 1268 1269 1270
#ifdef CONFIG_BLK_CGROUP
	struct request_queue		*throttle_queue;
#endif

1271
#ifdef CONFIG_UPROBES
1272
	struct uprobe_task		*utask;
1273
#endif
K
Kent Overstreet 已提交
1274
#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1275 1276
	unsigned int			sequential_io;
	unsigned int			sequential_io_avg;
K
Kent Overstreet 已提交
1277
#endif
P
Peter Zijlstra 已提交
1278
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1279
	unsigned long			task_state_change;
P
Peter Zijlstra 已提交
1280
#endif
1281
	int				pagefault_disabled;
1282
#ifdef CONFIG_MMU
1283
	struct task_struct		*oom_reaper_list;
1284
#endif
1285
#ifdef CONFIG_VMAP_STACK
1286
	struct vm_struct		*stack_vm_area;
1287
#endif
1288
#ifdef CONFIG_THREAD_INFO_IN_TASK
1289
	/* A live task holds one reference: */
1290
	refcount_t			stack_refcount;
1291 1292 1293
#endif
#ifdef CONFIG_LIVEPATCH
	int patch_state;
1294
#endif
1295 1296 1297
#ifdef CONFIG_SECURITY
	/* Used by LSM modules for access restriction: */
	void				*security;
1298
#endif
K
Kees Cook 已提交
1299

1300 1301
#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
	unsigned long			lowest_stack;
1302
	unsigned long			prev_lowest_stack;
1303 1304
#endif

1305 1306
#ifdef CONFIG_X86_MCE
	u64				mce_addr;
1307 1308 1309
	__u64				mce_ripv : 1,
					mce_whole_page : 1,
					__mce_reserved : 62;
1310 1311 1312
	struct callback_head		mce_kill_me;
#endif

K
Kees Cook 已提交
1313 1314 1315 1316 1317 1318
	/*
	 * New fields for task_struct should be added above here, so that
	 * they are included in the randomized portion of task_struct.
	 */
	randomized_struct_fields_end

1319 1320 1321 1322 1323 1324 1325 1326 1327
	/* CPU-specific state of this task: */
	struct thread_struct		thread;

	/*
	 * WARNING: on x86, 'thread_struct' contains a variable-sized
	 * structure.  It *MUST* be at the end of 'task_struct'.
	 *
	 * Do not put anything below here!
	 */
L
Linus Torvalds 已提交
1328 1329
};

A
Alexey Dobriyan 已提交
1330
static inline struct pid *task_pid(struct task_struct *task)
1331
{
1332
	return task->thread_pid;
1333 1334
}

1335 1336 1337 1338 1339
/*
 * the helpers to get the task's different pids as they are seen
 * from various namespaces
 *
 * task_xid_nr()     : global id, i.e. the id seen from the init namespace;
E
Eric W. Biederman 已提交
1340 1341
 * task_xid_vnr()    : virtual id, i.e. the id seen from the pid namespace of
 *                     current.
1342 1343 1344 1345
 * task_xid_nr_ns()  : id seen from the ns specified;
 *
 * see also pid_nr() etc in include/linux/pid.h
 */
1346
pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
1347

A
Alexey Dobriyan 已提交
1348
static inline pid_t task_pid_nr(struct task_struct *tsk)
1349 1350 1351 1352
{
	return tsk->pid;
}

1353
static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1354 1355 1356
{
	return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
}
1357 1358 1359

static inline pid_t task_pid_vnr(struct task_struct *tsk)
{
1360
	return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
1361 1362 1363
}


A
Alexey Dobriyan 已提交
1364
static inline pid_t task_tgid_nr(struct task_struct *tsk)
1365 1366 1367 1368
{
	return tsk->tgid;
}

1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
/**
 * pid_alive - check that a task structure is not stale
 * @p: Task structure to be checked.
 *
 * Test if a process is not yet dead (at most zombie state)
 * If pid_alive fails, then pointers within the task structure
 * can be stale and must not be dereferenced.
 *
 * Return: 1 if the process is alive. 0 otherwise.
 */
static inline int pid_alive(const struct task_struct *p)
{
1381
	return p->thread_pid != NULL;
1382
}
1383

1384
static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1385
{
1386
	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
1387 1388 1389 1390
}

static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
{
1391
	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
1392 1393 1394
}


1395
static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1396
{
1397
	return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
1398 1399 1400 1401
}

static inline pid_t task_session_vnr(struct task_struct *tsk)
{
1402
	return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
1403 1404
}

1405 1406
static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
E
Eric W. Biederman 已提交
1407
	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
1408 1409 1410 1411
}

static inline pid_t task_tgid_vnr(struct task_struct *tsk)
{
E
Eric W. Biederman 已提交
1412
	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
}

static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
{
	pid_t pid = 0;

	rcu_read_lock();
	if (pid_alive(tsk))
		pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
	rcu_read_unlock();

	return pid;
}

static inline pid_t task_ppid_nr(const struct task_struct *tsk)
{
	return task_ppid_nr_ns(tsk, &init_pid_ns);
}

1432
/* Obsolete, do not use: */
1433 1434 1435 1436
static inline pid_t task_pgrp_nr(struct task_struct *tsk)
{
	return task_pgrp_nr_ns(tsk, &init_pid_ns);
}
1437

1438 1439 1440
#define TASK_REPORT_IDLE	(TASK_REPORT + 1)
#define TASK_REPORT_MAX		(TASK_REPORT_IDLE << 1)

1441
static inline unsigned int task_state_index(struct task_struct *tsk)
1442
{
1443 1444
	unsigned int tsk_state = READ_ONCE(tsk->state);
	unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT;
1445

1446 1447 1448 1449 1450
	BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);

	if (tsk_state == TASK_IDLE)
		state = TASK_REPORT_IDLE;

1451 1452 1453
	return fls(state);
}

1454
static inline char task_index_to_char(unsigned int state)
1455
{
1456
	static const char state_char[] = "RSDTtXZPI";
1457

1458
	BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
1459

1460 1461 1462 1463 1464
	return state_char[state];
}

static inline char task_state_to_char(struct task_struct *tsk)
{
1465
	return task_index_to_char(task_state_index(tsk));
1466 1467
}

1468
/**
1469 1470
 * is_global_init - check if a task structure is init. Since init
 * is free to have sub-threads we need to check tgid.
1471 1472 1473
 * @tsk: Task structure to be checked.
 *
 * Check if a task structure is the first user space task the kernel created.
1474 1475
 *
 * Return: 1 if the task structure is init. 0 otherwise.
1476
 */
A
Alexey Dobriyan 已提交
1477
static inline int is_global_init(struct task_struct *tsk)
1478
{
1479
	return task_tgid_nr(tsk) == 1;
1480
}
1481

1482 1483
extern struct pid *cad_pid;

L
Linus Torvalds 已提交
1484 1485 1486
/*
 * Per process flags
 */
1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
#define PF_IDLE			0x00000002	/* I am an IDLE thread */
#define PF_EXITING		0x00000004	/* Getting shut down */
#define PF_VCPU			0x00000010	/* I'm a virtual CPU */
#define PF_WQ_WORKER		0x00000020	/* I'm a workqueue worker */
#define PF_FORKNOEXEC		0x00000040	/* Forked but didn't exec */
#define PF_MCE_PROCESS		0x00000080      /* Process policy on mce errors */
#define PF_SUPERPRIV		0x00000100	/* Used super-user privileges */
#define PF_DUMPCORE		0x00000200	/* Dumped core */
#define PF_SIGNALED		0x00000400	/* Killed by a signal */
#define PF_MEMALLOC		0x00000800	/* Allocating memory */
#define PF_NPROC_EXCEEDED	0x00001000	/* set_user() noticed that RLIMIT_NPROC was exceeded */
#define PF_USED_MATH		0x00002000	/* If unset the fpu must be initialized before use */
#define PF_USED_ASYNC		0x00004000	/* Used async_schedule*(), used by module init */
#define PF_NOFREEZE		0x00008000	/* This thread should not be frozen */
#define PF_FROZEN		0x00010000	/* Frozen for system suspend */
1502 1503 1504
#define PF_KSWAPD		0x00020000	/* I am kswapd */
#define PF_MEMALLOC_NOFS	0x00040000	/* All allocation requests will inherit GFP_NOFS */
#define PF_MEMALLOC_NOIO	0x00080000	/* All allocation requests will inherit GFP_NOIO */
1505 1506
#define PF_LOCAL_THROTTLE	0x00100000	/* Throttle writes only against the bdi I write to,
						 * I am cleaning dirty pages from some other bdi. */
1507 1508 1509
#define PF_KTHREAD		0x00200000	/* I am a kernel thread */
#define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */
#define PF_SWAPWRITE		0x00800000	/* Allowed to write to swap */
1510
#define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_mask */
1511
#define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */
1512
#define PF_MEMALLOC_NOCMA	0x10000000	/* All allocation request will have _GFP_MOVABLE cleared */
1513
#define PF_IO_WORKER		0x20000000	/* Task is an IO worker */
1514 1515
#define PF_FREEZER_SKIP		0x40000000	/* Freezer should not count it as freezable */
#define PF_SUSPEND_TASK		0x80000000      /* This thread called freeze_processes() and should not be frozen */
L
Linus Torvalds 已提交
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527

/*
 * Only the _current_ task can read/write to tsk->flags, but other
 * tasks can access tsk->flags in readonly mode for example
 * with tsk_used_math (like during threaded core dumping).
 * There is however an exception to this rule during ptrace
 * or during fork: the ptracer task is allowed to write to the
 * child->flags of its traced child (same goes for fork, the parent
 * can write to the child->flags), because we're guaranteed the
 * child is not running and in turn not changing child->flags
 * at the same time the parent does it.
 */
1528 1529 1530 1531 1532
#define clear_stopped_child_used_math(child)	do { (child)->flags &= ~PF_USED_MATH; } while (0)
#define set_stopped_child_used_math(child)	do { (child)->flags |= PF_USED_MATH; } while (0)
#define clear_used_math()			clear_stopped_child_used_math(current)
#define set_used_math()				set_stopped_child_used_math(current)

L
Linus Torvalds 已提交
1533 1534
#define conditional_stopped_child_used_math(condition, child) \
	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
1535 1536 1537

#define conditional_used_math(condition)	conditional_stopped_child_used_math(condition, current)

L
Linus Torvalds 已提交
1538 1539
#define copy_to_stopped_child_used_math(child) \
	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
1540

L
Linus Torvalds 已提交
1541
/* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
1542 1543
#define tsk_used_math(p)			((p)->flags & PF_USED_MATH)
#define used_math()				tsk_used_math(current)
L
Linus Torvalds 已提交
1544

1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
static inline bool is_percpu_thread(void)
{
#ifdef CONFIG_SMP
	return (current->flags & PF_NO_SETAFFINITY) &&
		(current->nr_cpus_allowed  == 1);
#else
	return true;
#endif
}

1555
/* Per-process atomic flags. */
1556 1557 1558
#define PFA_NO_NEW_PRIVS		0	/* May not gain new privileges. */
#define PFA_SPREAD_PAGE			1	/* Spread page cache over cpuset */
#define PFA_SPREAD_SLAB			2	/* Spread some slab caches over cpuset */
1559 1560
#define PFA_SPEC_SSB_DISABLE		3	/* Speculative Store Bypass disabled */
#define PFA_SPEC_SSB_FORCE_DISABLE	4	/* Speculative Store Bypass force disabled*/
1561 1562
#define PFA_SPEC_IB_DISABLE		5	/* Indirect branch speculation restricted */
#define PFA_SPEC_IB_FORCE_DISABLE	6	/* Indirect branch speculation permanently restricted */
1563
#define PFA_SPEC_SSB_NOEXEC		7	/* Speculative Store Bypass clear on execve() */
1564

1565 1566 1567
#define TASK_PFA_TEST(name, func)					\
	static inline bool task_##func(struct task_struct *p)		\
	{ return test_bit(PFA_##name, &p->atomic_flags); }
1568

1569 1570 1571
#define TASK_PFA_SET(name, func)					\
	static inline void task_set_##func(struct task_struct *p)	\
	{ set_bit(PFA_##name, &p->atomic_flags); }
1572

1573 1574 1575 1576 1577 1578
#define TASK_PFA_CLEAR(name, func)					\
	static inline void task_clear_##func(struct task_struct *p)	\
	{ clear_bit(PFA_##name, &p->atomic_flags); }

TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
1579

1580 1581 1582 1583 1584 1585 1586
TASK_PFA_TEST(SPREAD_PAGE, spread_page)
TASK_PFA_SET(SPREAD_PAGE, spread_page)
TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)

TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
TASK_PFA_SET(SPREAD_SLAB, spread_slab)
TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1587

1588 1589 1590 1591
TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)

1592 1593 1594 1595
TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)

1596 1597 1598
TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)

1599 1600 1601 1602 1603 1604 1605
TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)

TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)

1606
static inline void
1607
current_restore_flags(unsigned long orig_flags, unsigned long flags)
1608
{
1609 1610
	current->flags &= ~flags;
	current->flags |= orig_flags & flags;
1611 1612
}

1613 1614
extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
L
Linus Torvalds 已提交
1615
#ifdef CONFIG_SMP
1616 1617
extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
L
Linus Torvalds 已提交
1618
#else
1619
static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1620 1621
{
}
1622
static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
L
Linus Torvalds 已提交
1623
{
1624
	if (!cpumask_test_cpu(0, new_mask))
L
Linus Torvalds 已提交
1625 1626 1627 1628
		return -EINVAL;
	return 0;
}
#endif
1629

1630
extern int yield_to(struct task_struct *p, bool preempt);
1631 1632
extern void set_user_nice(struct task_struct *p, long nice);
extern int task_prio(const struct task_struct *p);
1633

1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 *
 * Return: The nice value [ -20 ... 0 ... 19 ].
 */
static inline int task_nice(const struct task_struct *p)
{
	return PRIO_TO_NICE((p)->static_prio);
}
1644

1645 1646
extern int can_nice(const struct task_struct *p, const int nice);
extern int task_curr(const struct task_struct *p);
L
Linus Torvalds 已提交
1647
extern int idle_cpu(int cpu);
1648
extern int available_idle_cpu(int cpu);
1649 1650 1651
extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1652
extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
1653
extern struct task_struct *idle_task(int cpu);
1654

1655 1656
/**
 * is_idle_task - is the specified task an idle task?
1657
 * @p: the task in question.
1658 1659
 *
 * Return: 1 if @p is an idle task. 0 otherwise.
1660
 */
1661
static inline bool is_idle_task(const struct task_struct *p)
1662
{
1663
	return !!(p->flags & PF_IDLE);
1664
}
1665

1666
extern struct task_struct *curr_task(int cpu);
1667
extern void ia64_set_curr_task(int cpu, struct task_struct *p);
L
Linus Torvalds 已提交
1668 1669 1670 1671

void yield(void);

union thread_union {
1672 1673 1674
#ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
	struct task_struct task;
#endif
1675
#ifndef CONFIG_THREAD_INFO_IN_TASK
L
Linus Torvalds 已提交
1676
	struct thread_info thread_info;
1677
#endif
L
Linus Torvalds 已提交
1678 1679 1680
	unsigned long stack[THREAD_SIZE/sizeof(long)];
};

1681 1682 1683 1684 1685 1686
#ifndef CONFIG_THREAD_INFO_IN_TASK
extern struct thread_info init_thread_info;
#endif

extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];

1687 1688 1689 1690 1691 1692 1693 1694 1695
#ifdef CONFIG_THREAD_INFO_IN_TASK
static inline struct thread_info *task_thread_info(struct task_struct *task)
{
	return &task->thread_info;
}
#elif !defined(__HAVE_THREAD_FUNCTIONS)
# define task_thread_info(task)	((struct thread_info *)(task)->stack)
#endif

1696 1697 1698 1699 1700
/*
 * find a task by one of its numerical ids
 *
 * find_task_by_pid_ns():
 *      finds a task by its pid in the specified namespace
1701 1702
 * find_task_by_vpid():
 *      finds a task by its virtual pid
1703
 *
1704
 * see also find_vpid() etc in include/linux/pid.h
1705 1706
 */

1707
extern struct task_struct *find_task_by_vpid(pid_t nr);
1708
extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1709

1710 1711 1712 1713 1714
/*
 * find a task by its virtual pid and get the task struct
 */
extern struct task_struct *find_get_task_by_vpid(pid_t nr);

1715 1716
extern int wake_up_state(struct task_struct *tsk, unsigned int state);
extern int wake_up_process(struct task_struct *tsk);
1717
extern void wake_up_new_task(struct task_struct *tsk);
1718

L
Linus Torvalds 已提交
1719
#ifdef CONFIG_SMP
1720
extern void kick_process(struct task_struct *tsk);
L
Linus Torvalds 已提交
1721
#else
1722
static inline void kick_process(struct task_struct *tsk) { }
L
Linus Torvalds 已提交
1723 1724
#endif

1725
extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
1726

1727 1728 1729 1730
static inline void set_task_comm(struct task_struct *tsk, const char *from)
{
	__set_task_comm(tsk, from, false);
}
1731

1732 1733 1734 1735 1736
extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
#define get_task_comm(buf, tsk) ({			\
	BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN);	\
	__get_task_comm(buf, sizeof(buf), tsk);		\
})
L
Linus Torvalds 已提交
1737 1738

#ifdef CONFIG_SMP
1739 1740 1741 1742 1743 1744 1745 1746 1747
static __always_inline void scheduler_ipi(void)
{
	/*
	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
	 * TIF_NEED_RESCHED remotely (for the first time) will also send
	 * this IPI.
	 */
	preempt_fold_need_resched();
}
R
Roland McGrath 已提交
1748
extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
L
Linus Torvalds 已提交
1749
#else
1750
static inline void scheduler_ipi(void) { }
1751
static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state)
R
Roland McGrath 已提交
1752 1753 1754
{
	return 1;
}
L
Linus Torvalds 已提交
1755 1756
#endif

1757 1758 1759
/*
 * Set thread flags in other task's structures.
 * See asm/thread_info.h for TIF_xxxx flags available:
L
Linus Torvalds 已提交
1760 1761 1762
 */
static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
A
Al Viro 已提交
1763
	set_ti_thread_flag(task_thread_info(tsk), flag);
L
Linus Torvalds 已提交
1764 1765 1766 1767
}

static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
A
Al Viro 已提交
1768
	clear_ti_thread_flag(task_thread_info(tsk), flag);
L
Linus Torvalds 已提交
1769 1770
}

1771 1772 1773 1774 1775 1776
static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
					  bool value)
{
	update_ti_thread_flag(task_thread_info(tsk), flag, value);
}

L
Linus Torvalds 已提交
1777 1778
static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
A
Al Viro 已提交
1779
	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
L
Linus Torvalds 已提交
1780 1781 1782 1783
}

static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
A
Al Viro 已提交
1784
	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
L
Linus Torvalds 已提交
1785 1786 1787 1788
}

static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
{
A
Al Viro 已提交
1789
	return test_ti_thread_flag(task_thread_info(tsk), flag);
L
Linus Torvalds 已提交
1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801
}

static inline void set_tsk_need_resched(struct task_struct *tsk)
{
	set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
}

static inline void clear_tsk_need_resched(struct task_struct *tsk)
{
	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
}

1802 1803 1804 1805 1806
static inline int test_tsk_need_resched(struct task_struct *tsk)
{
	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
}

L
Linus Torvalds 已提交
1807 1808 1809 1810 1811 1812
/*
 * cond_resched() and cond_resched_lock(): latency reduction via
 * explicit rescheduling in places that are safe. The return
 * value indicates whether a reschedule was done in fact.
 * cond_resched_lock() will drop the spinlock before scheduling,
 */
1813
#ifndef CONFIG_PREEMPTION
1814
extern int _cond_resched(void);
1815 1816 1817
#else
static inline int _cond_resched(void) { return 0; }
#endif
1818

1819
#define cond_resched() ({			\
1820
	___might_sleep(__FILE__, __LINE__, 0);	\
1821 1822
	_cond_resched();			\
})
1823

1824 1825 1826
extern int __cond_resched_lock(spinlock_t *lock);

#define cond_resched_lock(lock) ({				\
1827
	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1828 1829 1830
	__cond_resched_lock(lock);				\
})

1831 1832 1833 1834 1835 1836 1837 1838 1839
static inline void cond_resched_rcu(void)
{
#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
	rcu_read_unlock();
	cond_resched();
	rcu_read_lock();
#endif
}

L
Linus Torvalds 已提交
1840 1841
/*
 * Does a critical section need to be broken due to another
1842
 * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
N
Nick Piggin 已提交
1843
 * but a general need for low latency)
L
Linus Torvalds 已提交
1844
 */
N
Nick Piggin 已提交
1845
static inline int spin_needbreak(spinlock_t *lock)
L
Linus Torvalds 已提交
1846
{
1847
#ifdef CONFIG_PREEMPTION
N
Nick Piggin 已提交
1848 1849
	return spin_is_contended(lock);
#else
L
Linus Torvalds 已提交
1850
	return 0;
N
Nick Piggin 已提交
1851
#endif
L
Linus Torvalds 已提交
1852 1853
}

1854 1855 1856 1857 1858
static __always_inline bool need_resched(void)
{
	return unlikely(tif_need_resched());
}

L
Linus Torvalds 已提交
1859 1860 1861 1862 1863 1864 1865
/*
 * Wrappers for p->thread_info->cpu access. No-op on UP.
 */
#ifdef CONFIG_SMP

static inline unsigned int task_cpu(const struct task_struct *p)
{
1866
#ifdef CONFIG_THREAD_INFO_IN_TASK
1867
	return READ_ONCE(p->cpu);
1868
#else
1869
	return READ_ONCE(task_thread_info(p)->cpu);
1870
#endif
L
Linus Torvalds 已提交
1871 1872
}

I
Ingo Molnar 已提交
1873
extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
L
Linus Torvalds 已提交
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887

#else

static inline unsigned int task_cpu(const struct task_struct *p)
{
	return 0;
}

static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
{
}

#endif /* CONFIG_SMP */

1888 1889 1890 1891 1892 1893 1894 1895 1896
/*
 * In order to reduce various lock holder preemption latencies provide an
 * interface to see if a vCPU is currently running or not.
 *
 * This allows us to terminate optimistic spin loops and block, analogous to
 * the native optimistic spin heuristic of testing if the lock owner task is
 * running or not.
 */
#ifndef vcpu_is_preempted
1897 1898 1899 1900
static inline bool vcpu_is_preempted(int cpu)
{
	return false;
}
1901 1902
#endif

1903 1904
extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
1905

D
Dave Hansen 已提交
1906 1907 1908 1909
#ifndef TASK_SIZE_OF
#define TASK_SIZE_OF(tsk)	TASK_SIZE
#endif

1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
#ifdef CONFIG_RSEQ

/*
 * Map the event mask on the user-space ABI enum rseq_cs_flags
 * for direct mask checks.
 */
enum rseq_event_mask_bits {
	RSEQ_EVENT_PREEMPT_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
	RSEQ_EVENT_SIGNAL_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
	RSEQ_EVENT_MIGRATE_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
};

enum rseq_event_mask {
	RSEQ_EVENT_PREEMPT	= (1U << RSEQ_EVENT_PREEMPT_BIT),
	RSEQ_EVENT_SIGNAL	= (1U << RSEQ_EVENT_SIGNAL_BIT),
	RSEQ_EVENT_MIGRATE	= (1U << RSEQ_EVENT_MIGRATE_BIT),
};

static inline void rseq_set_notify_resume(struct task_struct *t)
{
	if (t->rseq)
		set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
}

1934
void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
1935

1936 1937
static inline void rseq_handle_notify_resume(struct ksignal *ksig,
					     struct pt_regs *regs)
1938 1939
{
	if (current->rseq)
1940
		__rseq_handle_notify_resume(ksig, regs);
1941 1942
}

1943 1944
static inline void rseq_signal_deliver(struct ksignal *ksig,
				       struct pt_regs *regs)
1945 1946 1947 1948
{
	preempt_disable();
	__set_bit(RSEQ_EVENT_SIGNAL_BIT, &current->rseq_event_mask);
	preempt_enable();
1949
	rseq_handle_notify_resume(ksig, regs);
1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
}

/* rseq_preempt() requires preemption to be disabled. */
static inline void rseq_preempt(struct task_struct *t)
{
	__set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
	rseq_set_notify_resume(t);
}

/* rseq_migrate() requires preemption to be disabled. */
static inline void rseq_migrate(struct task_struct *t)
{
	__set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
	rseq_set_notify_resume(t);
}

/*
 * If parent process has a registered restartable sequences area, the
1968
 * child inherits. Unregister rseq for a clone with CLONE_VM set.
1969 1970 1971
 */
static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
{
1972
	if (clone_flags & CLONE_VM) {
1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994
		t->rseq = NULL;
		t->rseq_sig = 0;
		t->rseq_event_mask = 0;
	} else {
		t->rseq = current->rseq;
		t->rseq_sig = current->rseq_sig;
		t->rseq_event_mask = current->rseq_event_mask;
	}
}

static inline void rseq_execve(struct task_struct *t)
{
	t->rseq = NULL;
	t->rseq_sig = 0;
	t->rseq_event_mask = 0;
}

#else

static inline void rseq_set_notify_resume(struct task_struct *t)
{
}
1995 1996
static inline void rseq_handle_notify_resume(struct ksignal *ksig,
					     struct pt_regs *regs)
1997 1998
{
}
1999 2000
static inline void rseq_signal_deliver(struct ksignal *ksig,
				       struct pt_regs *regs)
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
{
}
static inline void rseq_preempt(struct task_struct *t)
{
}
static inline void rseq_migrate(struct task_struct *t)
{
}
static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
{
}
static inline void rseq_execve(struct task_struct *t)
{
}

#endif

#ifdef CONFIG_DEBUG_RSEQ

void rseq_syscall(struct pt_regs *regs);

#else

static inline void rseq_syscall(struct pt_regs *regs)
{
}

#endif

2030 2031 2032 2033 2034 2035 2036 2037 2038
const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq);
char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len);
int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);

const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq);
const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq);
const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);

int sched_trace_rq_cpu(struct rq *rq);
2039
int sched_trace_rq_nr_running(struct rq *rq);
2040 2041 2042

const struct cpumask *sched_trace_rd_span(struct root_domain *rd);

L
Linus Torvalds 已提交
2043
#endif