sched.c 218.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
25 26
 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 *              Thomas Gleixner, Mike Kravetz
L
Linus Torvalds 已提交
27 28 29 30 31 32
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
33
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
34 35 36
#include <linux/highmem.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
37
#include <linux/capability.h>
L
Linus Torvalds 已提交
38 39
#include <linux/completion.h>
#include <linux/kernel_stat.h>
40
#include <linux/debug_locks.h>
41
#include <linux/perf_event.h>
L
Linus Torvalds 已提交
42 43 44
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
45
#include <linux/freezer.h>
46
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
47 48
#include <linux/blkdev.h>
#include <linux/delay.h>
49
#include <linux/pid_namespace.h>
L
Linus Torvalds 已提交
50 51 52 53 54 55 56
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
57
#include <linux/proc_fs.h>
L
Linus Torvalds 已提交
58
#include <linux/seq_file.h>
59
#include <linux/stop_machine.h>
60
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
61 62
#include <linux/syscalls.h>
#include <linux/times.h>
63
#include <linux/tsacct_kern.h>
64
#include <linux/kprobes.h>
65
#include <linux/delayacct.h>
66
#include <linux/unistd.h>
J
Jens Axboe 已提交
67
#include <linux/pagemap.h>
P
Peter Zijlstra 已提交
68
#include <linux/hrtimer.h>
R
Reynes Philippe 已提交
69
#include <linux/tick.h>
P
Peter Zijlstra 已提交
70 71
#include <linux/debugfs.h>
#include <linux/ctype.h>
72
#include <linux/ftrace.h>
73
#include <linux/slab.h>
L
Linus Torvalds 已提交
74

75
#include <asm/tlb.h>
76
#include <asm/irq_regs.h>
77
#include <asm/mutex.h>
L
Linus Torvalds 已提交
78

79
#include "sched_cpupri.h"
T
Tejun Heo 已提交
80
#include "workqueue_sched.h"
81
#include "sched_autogroup.h"
82

83
#define CREATE_TRACE_POINTS
84
#include <trace/events/sched.h>
85

L
Linus Torvalds 已提交
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
105
 * Helpers for converting nanosecond timing to jiffy resolution
L
Linus Torvalds 已提交
106
 */
107
#define NS_TO_JIFFIES(TIME)	((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
L
Linus Torvalds 已提交
108

I
Ingo Molnar 已提交
109 110 111
#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

L
Linus Torvalds 已提交
112 113 114
/*
 * These are the 'tuning knobs' of the scheduler:
 *
D
Dmitry Adamushko 已提交
115
 * default timeslice is 100 msecs (used only for SCHED_RR tasks).
L
Linus Torvalds 已提交
116 117 118
 * Timeslices get refilled after they expire.
 */
#define DEF_TIMESLICE		(100 * HZ / 1000)
119

120 121 122 123 124
/*
 * single value that denotes runtime == period, ie unlimited time.
 */
#define RUNTIME_INF	((u64)~0ULL)

125 126
static inline int rt_policy(int policy)
{
127
	if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
128 129 130 131 132 133 134 135 136
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

L
Linus Torvalds 已提交
137
/*
I
Ingo Molnar 已提交
138
 * This is the priority-queue data structure of the RT scheduling class:
L
Linus Torvalds 已提交
139
 */
I
Ingo Molnar 已提交
140 141 142 143 144
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

145
struct rt_bandwidth {
I
Ingo Molnar 已提交
146
	/* nests inside the rq lock: */
147
	raw_spinlock_t		rt_runtime_lock;
I
Ingo Molnar 已提交
148 149 150
	ktime_t			rt_period;
	u64			rt_runtime;
	struct hrtimer		rt_period_timer;
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
};

static struct rt_bandwidth def_rt_bandwidth;

static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);

static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
{
	struct rt_bandwidth *rt_b =
		container_of(timer, struct rt_bandwidth, rt_period_timer);
	ktime_t now;
	int overrun;
	int idle = 0;

	for (;;) {
		now = hrtimer_cb_get_time(timer);
		overrun = hrtimer_forward(timer, now, rt_b->rt_period);

		if (!overrun)
			break;

		idle = do_sched_rt_period_timer(rt_b, overrun);
	}

	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
}

static
void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
{
	rt_b->rt_period = ns_to_ktime(period);
	rt_b->rt_runtime = runtime;

184
	raw_spin_lock_init(&rt_b->rt_runtime_lock);
P
Peter Zijlstra 已提交
185

186 187 188 189 190
	hrtimer_init(&rt_b->rt_period_timer,
			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rt_b->rt_period_timer.function = sched_rt_period_timer;
}

191 192 193
static inline int rt_bandwidth_enabled(void)
{
	return sysctl_sched_rt_runtime >= 0;
194 195 196 197 198 199
}

static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	ktime_t now;

200
	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
201 202 203 204 205
		return;

	if (hrtimer_active(&rt_b->rt_period_timer))
		return;

206
	raw_spin_lock(&rt_b->rt_runtime_lock);
207
	for (;;) {
208 209 210
		unsigned long delta;
		ktime_t soft, hard;

211 212 213 214 215
		if (hrtimer_active(&rt_b->rt_period_timer))
			break;

		now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
		hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
216 217 218 219 220

		soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
		hard = hrtimer_get_expires(&rt_b->rt_period_timer);
		delta = ktime_to_ns(ktime_sub(hard, soft));
		__hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
221
				HRTIMER_MODE_ABS_PINNED, 0);
222
	}
223
	raw_spin_unlock(&rt_b->rt_runtime_lock);
224 225 226 227 228 229 230 231 232
}

#ifdef CONFIG_RT_GROUP_SCHED
static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	hrtimer_cancel(&rt_b->rt_period_timer);
}
#endif

233
/*
234
 * sched_domains_mutex serializes calls to init_sched_domains,
235 236 237 238
 * detach_destroy_domains and partition_sched_domains.
 */
static DEFINE_MUTEX(sched_domains_mutex);

D
Dhaval Giani 已提交
239
#ifdef CONFIG_CGROUP_SCHED
S
Srivatsa Vaddagiri 已提交
240

241 242
#include <linux/cgroup.h>

S
Srivatsa Vaddagiri 已提交
243 244
struct cfs_rq;

P
Peter Zijlstra 已提交
245 246
static LIST_HEAD(task_groups);

S
Srivatsa Vaddagiri 已提交
247
/* task group related information */
248
struct task_group {
249
	struct cgroup_subsys_state css;
250

251
#ifdef CONFIG_FAIR_GROUP_SCHED
S
Srivatsa Vaddagiri 已提交
252 253 254 255 256
	/* schedulable entities of this group on each cpu */
	struct sched_entity **se;
	/* runqueue "owned" by this group on each cpu */
	struct cfs_rq **cfs_rq;
	unsigned long shares;
P
Peter Zijlstra 已提交
257 258

	atomic_t load_weight;
259 260 261 262 263 264
#endif

#ifdef CONFIG_RT_GROUP_SCHED
	struct sched_rt_entity **rt_se;
	struct rt_rq **rt_rq;

265
	struct rt_bandwidth rt_bandwidth;
266
#endif
267

268
	struct rcu_head rcu;
P
Peter Zijlstra 已提交
269
	struct list_head list;
P
Peter Zijlstra 已提交
270 271 272 273

	struct task_group *parent;
	struct list_head siblings;
	struct list_head children;
274 275 276 277

#ifdef CONFIG_SCHED_AUTOGROUP
	struct autogroup *autogroup;
#endif
S
Srivatsa Vaddagiri 已提交
278 279
};

280
/* task_group_lock serializes the addition/removal of task groups */
281
static DEFINE_SPINLOCK(task_group_lock);
282

283 284
#ifdef CONFIG_FAIR_GROUP_SCHED

285
# define ROOT_TASK_GROUP_LOAD	NICE_0_LOAD
286

287
/*
288 289 290 291
 * A weight of 0 or 1 can cause arithmetics problems.
 * A weight of a cfs_rq is the sum of weights of which entities
 * are queued on this cfs_rq, so a weight of a entity should not be
 * too large, so as the shares value of a task group.
292 293 294
 * (The default weight is 1024 - so there's no practical
 *  limitation from this.)
 */
295
#define MIN_SHARES	2
296
#define MAX_SHARES	(1UL << 18)
297

298
static int root_task_group_load = ROOT_TASK_GROUP_LOAD;
299 300
#endif

S
Srivatsa Vaddagiri 已提交
301
/* Default task group.
I
Ingo Molnar 已提交
302
 *	Every task in system belong to this group at bootup.
S
Srivatsa Vaddagiri 已提交
303
 */
304
struct task_group root_task_group;
S
Srivatsa Vaddagiri 已提交
305

D
Dhaval Giani 已提交
306
#endif	/* CONFIG_CGROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
307

I
Ingo Molnar 已提交
308 309 310 311 312 313
/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	u64 exec_clock;
I
Ingo Molnar 已提交
314
	u64 min_vruntime;
315 316 317
#ifndef CONFIG_64BIT
	u64 min_vruntime_copy;
#endif
I
Ingo Molnar 已提交
318 319 320

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;
321 322 323 324 325 326

	struct list_head tasks;
	struct list_head *balance_iterator;

	/*
	 * 'curr' points to currently running entity on this cfs_rq.
I
Ingo Molnar 已提交
327 328
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
329
	struct sched_entity *curr, *next, *last, *skip;
P
Peter Zijlstra 已提交
330

331
#ifdef	CONFIG_SCHED_DEBUG
P
Peter Zijlstra 已提交
332
	unsigned int nr_spread_over;
333
#endif
P
Peter Zijlstra 已提交
334

335
#ifdef CONFIG_FAIR_GROUP_SCHED
I
Ingo Molnar 已提交
336 337
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

I
Ingo Molnar 已提交
338 339
	/*
	 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
I
Ingo Molnar 已提交
340 341 342 343 344 345
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
346
	int on_list;
I
Ingo Molnar 已提交
347 348
	struct list_head leaf_cfs_rq_list;
	struct task_group *tg;	/* group that "owns" this runqueue */
349 350 351

#ifdef CONFIG_SMP
	/*
352
	 * the part of load.weight contributed by tasks
353
	 */
354
	unsigned long task_weight;
355

356 357 358 359 360 361 362
	/*
	 *   h_load = weight * f(tg)
	 *
	 * Where f(tg) is the recursive weight fraction assigned to
	 * this group.
	 */
	unsigned long h_load;
363

364
	/*
365 366 367 368 369
	 * Maintaining per-cpu shares distribution for group scheduling
	 *
	 * load_stamp is the last time we updated the load average
	 * load_last is the last time we updated the load average and saw load
	 * load_unacc_exec_time is currently unaccounted execution time
370
	 */
P
Peter Zijlstra 已提交
371 372
	u64 load_avg;
	u64 load_period;
373
	u64 load_stamp, load_last, load_unacc_exec_time;
374

P
Peter Zijlstra 已提交
375
	unsigned long load_contribution;
376
#endif
I
Ingo Molnar 已提交
377 378
#endif
};
L
Linus Torvalds 已提交
379

I
Ingo Molnar 已提交
380 381 382
/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
383
	unsigned long rt_nr_running;
384
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
385 386
	struct {
		int curr; /* highest queued rt task prio */
387
#ifdef CONFIG_SMP
388
		int next; /* next highest */
389
#endif
390
	} highest_prio;
P
Peter Zijlstra 已提交
391
#endif
P
Peter Zijlstra 已提交
392
#ifdef CONFIG_SMP
393
	unsigned long rt_nr_migratory;
394
	unsigned long rt_nr_total;
G
Gregory Haskins 已提交
395
	int overloaded;
396
	struct plist_head pushable_tasks;
P
Peter Zijlstra 已提交
397
#endif
P
Peter Zijlstra 已提交
398
	int rt_throttled;
P
Peter Zijlstra 已提交
399
	u64 rt_time;
P
Peter Zijlstra 已提交
400
	u64 rt_runtime;
I
Ingo Molnar 已提交
401
	/* Nests inside the rq lock: */
402
	raw_spinlock_t rt_runtime_lock;
P
Peter Zijlstra 已提交
403

404
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
405 406
	unsigned long rt_nr_boosted;

P
Peter Zijlstra 已提交
407 408 409 410
	struct rq *rq;
	struct list_head leaf_rt_rq_list;
	struct task_group *tg;
#endif
I
Ingo Molnar 已提交
411 412
};

G
Gregory Haskins 已提交
413 414 415 416
#ifdef CONFIG_SMP

/*
 * We add the notion of a root-domain which will be used to define per-domain
I
Ingo Molnar 已提交
417 418
 * variables. Each exclusive cpuset essentially defines an island domain by
 * fully partitioning the member cpus from any other cpuset. Whenever a new
G
Gregory Haskins 已提交
419 420 421 422 423 424
 * exclusive cpuset is created, we also create and attach a new root-domain
 * object.
 *
 */
struct root_domain {
	atomic_t refcount;
425
	struct rcu_head rcu;
426 427
	cpumask_var_t span;
	cpumask_var_t online;
428

I
Ingo Molnar 已提交
429
	/*
430 431 432
	 * The "RT overload" flag: it gets set if a CPU has more than
	 * one runnable RT task.
	 */
433
	cpumask_var_t rto_mask;
I
Ingo Molnar 已提交
434
	atomic_t rto_count;
435
	struct cpupri cpupri;
G
Gregory Haskins 已提交
436 437
};

438 439 440 441
/*
 * By default the system creates a single root-domain with all cpus as
 * members (mimicking the global state we have today).
 */
G
Gregory Haskins 已提交
442 443
static struct root_domain def_root_domain;

444
#endif /* CONFIG_SMP */
G
Gregory Haskins 已提交
445

L
Linus Torvalds 已提交
446 447 448 449 450 451 452
/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
453
struct rq {
454
	/* runqueue lock: */
455
	raw_spinlock_t lock;
L
Linus Torvalds 已提交
456 457 458 459 460 461

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
I
Ingo Molnar 已提交
462 463
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
464
	unsigned long last_load_update_tick;
465
#ifdef CONFIG_NO_HZ
M
Mike Galbraith 已提交
466
	u64 nohz_stamp;
467
	unsigned char nohz_balance_kick;
468
#endif
469
	int skip_clock_update;
470

471 472
	/* capture load from *all* tasks on this cpu: */
	struct load_weight load;
I
Ingo Molnar 已提交
473 474 475 476
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
P
Peter Zijlstra 已提交
477 478
	struct rt_rq rt;

I
Ingo Molnar 已提交
479
#ifdef CONFIG_FAIR_GROUP_SCHED
480 481
	/* list of leaf cfs_rq on this cpu: */
	struct list_head leaf_cfs_rq_list;
482 483
#endif
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
484
	struct list_head leaf_rt_rq_list;
L
Linus Torvalds 已提交
485 486 487 488 489 490 491 492 493 494
#endif

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

495
	struct task_struct *curr, *idle, *stop;
496
	unsigned long next_balance;
L
Linus Torvalds 已提交
497
	struct mm_struct *prev_mm;
I
Ingo Molnar 已提交
498

499
	u64 clock;
500
	u64 clock_task;
I
Ingo Molnar 已提交
501

L
Linus Torvalds 已提交
502 503 504
	atomic_t nr_iowait;

#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
505
	struct root_domain *rd;
L
Linus Torvalds 已提交
506 507
	struct sched_domain *sd;

508 509
	unsigned long cpu_power;

510
	unsigned char idle_at_tick;
L
Linus Torvalds 已提交
511
	/* For active balancing */
512
	int post_schedule;
L
Linus Torvalds 已提交
513 514
	int active_balance;
	int push_cpu;
515
	struct cpu_stop_work active_balance_work;
516 517
	/* cpu of this runqueue: */
	int cpu;
518
	int online;
L
Linus Torvalds 已提交
519

520
	unsigned long avg_load_per_task;
L
Linus Torvalds 已提交
521

522 523
	u64 rt_avg;
	u64 age_stamp;
M
Mike Galbraith 已提交
524 525
	u64 idle_stamp;
	u64 avg_idle;
L
Linus Torvalds 已提交
526 527
#endif

528 529 530 531
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	u64 prev_irq_time;
#endif

532 533 534 535
	/* calc_load related fields */
	unsigned long calc_load_update;
	long calc_load_active;

P
Peter Zijlstra 已提交
536
#ifdef CONFIG_SCHED_HRTICK
537 538 539 540
#ifdef CONFIG_SMP
	int hrtick_csd_pending;
	struct call_single_data hrtick_csd;
#endif
P
Peter Zijlstra 已提交
541 542 543
	struct hrtimer hrtick_timer;
#endif

L
Linus Torvalds 已提交
544 545 546
#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;
547 548
	unsigned long long rq_cpu_time;
	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
L
Linus Torvalds 已提交
549 550

	/* sys_sched_yield() stats */
551
	unsigned int yld_count;
L
Linus Torvalds 已提交
552 553

	/* schedule() stats */
554 555 556
	unsigned int sched_switch;
	unsigned int sched_count;
	unsigned int sched_goidle;
L
Linus Torvalds 已提交
557 558

	/* try_to_wake_up() stats */
559 560
	unsigned int ttwu_count;
	unsigned int ttwu_local;
L
Linus Torvalds 已提交
561
#endif
562 563 564 565

#ifdef CONFIG_SMP
	struct task_struct *wake_list;
#endif
L
Linus Torvalds 已提交
566 567
};

568
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
L
Linus Torvalds 已提交
569

570

571
static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
I
Ingo Molnar 已提交
572

573 574 575 576 577 578 579 580 581
static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

582
#define rcu_dereference_check_sched_domain(p) \
583
	rcu_dereference_check((p), \
584
			      rcu_read_lock_held() || \
585 586
			      lockdep_is_held(&sched_domains_mutex))

N
Nick Piggin 已提交
587 588
/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
589
 * See detach_destroy_domains: synchronize_sched for details.
N
Nick Piggin 已提交
590 591 592 593
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
594
#define for_each_domain(cpu, __sd) \
595
	for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
L
Linus Torvalds 已提交
596 597 598 599 600

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)
601
#define raw_rq()		(&__raw_get_cpu_var(runqueues))
L
Linus Torvalds 已提交
602

603 604 605 606 607 608
#ifdef CONFIG_CGROUP_SCHED

/*
 * Return the group to which this tasks belongs.
 *
 * We use task_subsys_state_check() and extend the RCU verification
609
 * with lockdep_is_held(&p->pi_lock) because cpu_cgroup_attach()
610 611 612 613 614
 * holds that lock for each task it moves into the cgroup. Therefore
 * by holding that lock, we pin the task to the current cgroup.
 */
static inline struct task_group *task_group(struct task_struct *p)
{
615
	struct task_group *tg;
616 617 618
	struct cgroup_subsys_state *css;

	css = task_subsys_state_check(p, cpu_cgroup_subsys_id,
619
			lockdep_is_held(&p->pi_lock));
620 621 622
	tg = container_of(css, struct task_group, css);

	return autogroup_task_group(p, tg);
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
}

/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
{
#ifdef CONFIG_FAIR_GROUP_SCHED
	p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
	p->se.parent = task_group(p)->se[cpu];
#endif

#ifdef CONFIG_RT_GROUP_SCHED
	p->rt.rt_rq  = task_group(p)->rt_rq[cpu];
	p->rt.parent = task_group(p)->rt_se[cpu];
#endif
}

#else /* CONFIG_CGROUP_SCHED */

static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
static inline struct task_group *task_group(struct task_struct *p)
{
	return NULL;
}

#endif /* CONFIG_CGROUP_SCHED */

649
static void update_rq_clock_task(struct rq *rq, s64 delta);
650

651
static void update_rq_clock(struct rq *rq)
652
{
653
	s64 delta;
654

655
	if (rq->skip_clock_update > 0)
656
		return;
657

658 659 660
	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
	rq->clock += delta;
	update_rq_clock_task(rq, delta);
661 662
}

I
Ingo Molnar 已提交
663 664 665 666 667 668 669 670 671
/*
 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
 */
#ifdef CONFIG_SCHED_DEBUG
# define const_debug __read_mostly
#else
# define const_debug static const
#endif

I
Ingo Molnar 已提交
672
/**
673
 * runqueue_is_locked - Returns true if the current cpu runqueue is locked
674
 * @cpu: the processor in question.
I
Ingo Molnar 已提交
675 676 677 678
 *
 * This interface allows printk to be called with the runqueue lock
 * held and know whether or not it is OK to wake up the klogd.
 */
679
int runqueue_is_locked(int cpu)
I
Ingo Molnar 已提交
680
{
681
	return raw_spin_is_locked(&cpu_rq(cpu)->lock);
I
Ingo Molnar 已提交
682 683
}

I
Ingo Molnar 已提交
684 685 686
/*
 * Debugging: various feature bits
 */
P
Peter Zijlstra 已提交
687 688 689 690

#define SCHED_FEAT(name, enabled)	\
	__SCHED_FEAT_##name ,

I
Ingo Molnar 已提交
691
enum {
P
Peter Zijlstra 已提交
692
#include "sched_features.h"
I
Ingo Molnar 已提交
693 694
};

P
Peter Zijlstra 已提交
695 696 697 698 699
#undef SCHED_FEAT

#define SCHED_FEAT(name, enabled)	\
	(1UL << __SCHED_FEAT_##name) * enabled |

I
Ingo Molnar 已提交
700
const_debug unsigned int sysctl_sched_features =
P
Peter Zijlstra 已提交
701 702 703 704 705 706 707 708 709
#include "sched_features.h"
	0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG
#define SCHED_FEAT(name, enabled)	\
	#name ,

710
static __read_mostly char *sched_feat_names[] = {
P
Peter Zijlstra 已提交
711 712 713 714 715 716
#include "sched_features.h"
	NULL
};

#undef SCHED_FEAT

L
Li Zefan 已提交
717
static int sched_feat_show(struct seq_file *m, void *v)
P
Peter Zijlstra 已提交
718 719 720 721
{
	int i;

	for (i = 0; sched_feat_names[i]; i++) {
L
Li Zefan 已提交
722 723 724
		if (!(sysctl_sched_features & (1UL << i)))
			seq_puts(m, "NO_");
		seq_printf(m, "%s ", sched_feat_names[i]);
P
Peter Zijlstra 已提交
725
	}
L
Li Zefan 已提交
726
	seq_puts(m, "\n");
P
Peter Zijlstra 已提交
727

L
Li Zefan 已提交
728
	return 0;
P
Peter Zijlstra 已提交
729 730 731 732 733 734 735
}

static ssize_t
sched_feat_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
	char buf[64];
736
	char *cmp;
P
Peter Zijlstra 已提交
737 738 739 740 741 742 743 744 745 746
	int neg = 0;
	int i;

	if (cnt > 63)
		cnt = 63;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;
747
	cmp = strstrip(buf);
P
Peter Zijlstra 已提交
748

H
Hillf Danton 已提交
749
	if (strncmp(cmp, "NO_", 3) == 0) {
P
Peter Zijlstra 已提交
750 751 752 753 754
		neg = 1;
		cmp += 3;
	}

	for (i = 0; sched_feat_names[i]; i++) {
755
		if (strcmp(cmp, sched_feat_names[i]) == 0) {
P
Peter Zijlstra 已提交
756 757 758 759 760 761 762 763 764 765 766
			if (neg)
				sysctl_sched_features &= ~(1UL << i);
			else
				sysctl_sched_features |= (1UL << i);
			break;
		}
	}

	if (!sched_feat_names[i])
		return -EINVAL;

767
	*ppos += cnt;
P
Peter Zijlstra 已提交
768 769 770 771

	return cnt;
}

L
Li Zefan 已提交
772 773 774 775 776
static int sched_feat_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_feat_show, NULL);
}

777
static const struct file_operations sched_feat_fops = {
L
Li Zefan 已提交
778 779 780 781 782
	.open		= sched_feat_open,
	.write		= sched_feat_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
P
Peter Zijlstra 已提交
783 784 785 786 787 788 789 790 791 792 793 794 795 796
};

static __init int sched_init_debug(void)
{
	debugfs_create_file("sched_features", 0644, NULL, NULL,
			&sched_feat_fops);

	return 0;
}
late_initcall(sched_init_debug);

#endif

#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
I
Ingo Molnar 已提交
797

798 799 800 801 802 803
/*
 * Number of tasks to iterate in a single balance run.
 * Limited because this is done with IRQs disabled.
 */
const_debug unsigned int sysctl_sched_nr_migrate = 32;

804 805 806 807 808 809 810 811
/*
 * period over which we average the RT time consumption, measured
 * in ms.
 *
 * default: 1s
 */
const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;

P
Peter Zijlstra 已提交
812
/*
P
Peter Zijlstra 已提交
813
 * period over which we measure -rt task cpu usage in us.
P
Peter Zijlstra 已提交
814 815
 * default: 1s
 */
P
Peter Zijlstra 已提交
816
unsigned int sysctl_sched_rt_period = 1000000;
P
Peter Zijlstra 已提交
817

818 819
static __read_mostly int scheduler_running;

P
Peter Zijlstra 已提交
820 821 822 823 824
/*
 * part of the period that we allow rt tasks to run in us.
 * default: 0.95s
 */
int sysctl_sched_rt_runtime = 950000;
P
Peter Zijlstra 已提交
825

826 827 828 829 830 831 832
static inline u64 global_rt_period(void)
{
	return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
}

static inline u64 global_rt_runtime(void)
{
833
	if (sysctl_sched_rt_runtime < 0)
834 835 836 837
		return RUNTIME_INF;

	return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
}
P
Peter Zijlstra 已提交
838

L
Linus Torvalds 已提交
839
#ifndef prepare_arch_switch
840 841 842 843 844 845
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

846 847 848 849 850
static inline int task_current(struct rq *rq, struct task_struct *p)
{
	return rq->curr == p;
}

851
static inline int task_running(struct rq *rq, struct task_struct *p)
852
{
P
Peter Zijlstra 已提交
853 854 855
#ifdef CONFIG_SMP
	return p->on_cpu;
#else
856
	return task_current(rq, p);
P
Peter Zijlstra 已提交
857
#endif
858 859
}

P
Peter Zijlstra 已提交
860
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
861
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
862
{
P
Peter Zijlstra 已提交
863 864 865 866 867 868 869 870
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->on_cpu = 1;
#endif
871 872
}

873
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
874
{
P
Peter Zijlstra 已提交
875 876 877 878 879 880 881 882 883
#ifdef CONFIG_SMP
	/*
	 * After ->on_cpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->on_cpu = 0;
#endif
884 885 886 887
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
888 889 890 891 892 893 894
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

895
	raw_spin_unlock_irq(&rq->lock);
896 897 898
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
899
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
900 901 902 903 904 905 906
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
P
Peter Zijlstra 已提交
907
	next->on_cpu = 1;
908 909
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
910
	raw_spin_unlock_irq(&rq->lock);
911
#else
912
	raw_spin_unlock(&rq->lock);
913 914 915
#endif
}

916
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
917 918 919
{
#ifdef CONFIG_SMP
	/*
P
Peter Zijlstra 已提交
920
	 * After ->on_cpu is cleared, the task can be moved to a different CPU.
921 922 923 924
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
P
Peter Zijlstra 已提交
925
	prev->on_cpu = 0;
926 927 928
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
L
Linus Torvalds 已提交
929
#endif
930 931
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
L
Linus Torvalds 已提交
932

933
/*
934
 * __task_rq_lock - lock the rq @p resides on.
935
 */
936
static inline struct rq *__task_rq_lock(struct task_struct *p)
937 938
	__acquires(rq->lock)
{
939 940
	struct rq *rq;

941 942
	lockdep_assert_held(&p->pi_lock);

943
	for (;;) {
944
		rq = task_rq(p);
945
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
946
		if (likely(rq == task_rq(p)))
947
			return rq;
948
		raw_spin_unlock(&rq->lock);
949 950 951
	}
}

L
Linus Torvalds 已提交
952
/*
953
 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
L
Linus Torvalds 已提交
954
 */
955
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
956
	__acquires(p->pi_lock)
L
Linus Torvalds 已提交
957 958
	__acquires(rq->lock)
{
959
	struct rq *rq;
L
Linus Torvalds 已提交
960

961
	for (;;) {
962
		raw_spin_lock_irqsave(&p->pi_lock, *flags);
963
		rq = task_rq(p);
964
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
965
		if (likely(rq == task_rq(p)))
966
			return rq;
967 968
		raw_spin_unlock(&rq->lock);
		raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
969 970 971
	}
}

A
Alexey Dobriyan 已提交
972
static void __task_rq_unlock(struct rq *rq)
973 974
	__releases(rq->lock)
{
975
	raw_spin_unlock(&rq->lock);
976 977
}

978 979
static inline void
task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
980
	__releases(rq->lock)
981
	__releases(p->pi_lock)
L
Linus Torvalds 已提交
982
{
983 984
	raw_spin_unlock(&rq->lock);
	raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
985 986 987
}

/*
988
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
989
 */
A
Alexey Dobriyan 已提交
990
static struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
991 992
	__acquires(rq->lock)
{
993
	struct rq *rq;
L
Linus Torvalds 已提交
994 995 996

	local_irq_disable();
	rq = this_rq();
997
	raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
998 999 1000 1001

	return rq;
}

P
Peter Zijlstra 已提交
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
#ifdef CONFIG_SCHED_HRTICK
/*
 * Use HR-timers to deliver accurate preemption points.
 *
 * Its all a bit involved since we cannot program an hrt while holding the
 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
 * reschedule event.
 *
 * When we get rescheduled we reprogram the hrtick_timer outside of the
 * rq->lock.
 */

/*
 * Use hrtick when:
 *  - enabled by features
 *  - hrtimer is actually high res
 */
static inline int hrtick_enabled(struct rq *rq)
{
	if (!sched_feat(HRTICK))
		return 0;
1023
	if (!cpu_active(cpu_of(rq)))
1024
		return 0;
P
Peter Zijlstra 已提交
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
	return hrtimer_is_hres_active(&rq->hrtick_timer);
}

static void hrtick_clear(struct rq *rq)
{
	if (hrtimer_active(&rq->hrtick_timer))
		hrtimer_cancel(&rq->hrtick_timer);
}

/*
 * High-resolution timer tick.
 * Runs from hardirq context with interrupts disabled.
 */
static enum hrtimer_restart hrtick(struct hrtimer *timer)
{
	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

1044
	raw_spin_lock(&rq->lock);
1045
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
1046
	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
1047
	raw_spin_unlock(&rq->lock);
P
Peter Zijlstra 已提交
1048 1049 1050 1051

	return HRTIMER_NORESTART;
}

1052
#ifdef CONFIG_SMP
1053 1054 1055 1056
/*
 * called from hardirq (IPI) context
 */
static void __hrtick_start(void *arg)
1057
{
1058
	struct rq *rq = arg;
1059

1060
	raw_spin_lock(&rq->lock);
1061 1062
	hrtimer_restart(&rq->hrtick_timer);
	rq->hrtick_csd_pending = 0;
1063
	raw_spin_unlock(&rq->lock);
1064 1065
}

1066 1067 1068 1069 1070 1071
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
1072
{
1073 1074
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
1075

1076
	hrtimer_set_expires(timer, time);
1077 1078 1079 1080

	if (rq == this_rq()) {
		hrtimer_restart(timer);
	} else if (!rq->hrtick_csd_pending) {
1081
		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
1082 1083
		rq->hrtick_csd_pending = 1;
	}
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
}

static int
hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
	int cpu = (int)(long)hcpu;

	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
1098
		hrtick_clear(cpu_rq(cpu));
1099 1100 1101 1102 1103 1104
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

1105
static __init void init_hrtick(void)
1106 1107 1108
{
	hotcpu_notifier(hotplug_hrtick, 0);
}
1109 1110 1111 1112 1113 1114 1115 1116
#else
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
{
1117
	__hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
1118
			HRTIMER_MODE_REL_PINNED, 0);
1119
}
1120

A
Andrew Morton 已提交
1121
static inline void init_hrtick(void)
P
Peter Zijlstra 已提交
1122 1123
{
}
1124
#endif /* CONFIG_SMP */
P
Peter Zijlstra 已提交
1125

1126
static void init_rq_hrtick(struct rq *rq)
P
Peter Zijlstra 已提交
1127
{
1128 1129
#ifdef CONFIG_SMP
	rq->hrtick_csd_pending = 0;
P
Peter Zijlstra 已提交
1130

1131 1132 1133 1134
	rq->hrtick_csd.flags = 0;
	rq->hrtick_csd.func = __hrtick_start;
	rq->hrtick_csd.info = rq;
#endif
P
Peter Zijlstra 已提交
1135

1136 1137
	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rq->hrtick_timer.function = hrtick;
P
Peter Zijlstra 已提交
1138
}
A
Andrew Morton 已提交
1139
#else	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
1140 1141 1142 1143 1144 1145 1146 1147
static inline void hrtick_clear(struct rq *rq)
{
}

static inline void init_rq_hrtick(struct rq *rq)
{
}

1148 1149 1150
static inline void init_hrtick(void)
{
}
A
Andrew Morton 已提交
1151
#endif	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
1152

I
Ingo Molnar 已提交
1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

1166
static void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
1167 1168 1169
{
	int cpu;

1170
	assert_raw_spin_locked(&task_rq(p)->lock);
I
Ingo Molnar 已提交
1171

1172
	if (test_tsk_need_resched(p))
I
Ingo Molnar 已提交
1173 1174
		return;

1175
	set_tsk_need_resched(p);
I
Ingo Molnar 已提交
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

1192
	if (!raw_spin_trylock_irqsave(&rq->lock, flags))
I
Ingo Molnar 已提交
1193 1194
		return;
	resched_task(cpu_curr(cpu));
1195
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
1196
}
1197 1198

#ifdef CONFIG_NO_HZ
1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
/*
 * In the semi idle case, use the nearest busy cpu for migrating timers
 * from an idle cpu.  This is good for power-savings.
 *
 * We don't do similar optimization for completely idle system, as
 * selecting an idle cpu will add more delays to the timers than intended
 * (as that cpu's timer base may not be uptodate wrt jiffies etc).
 */
int get_nohz_timer_target(void)
{
	int cpu = smp_processor_id();
	int i;
	struct sched_domain *sd;

1213
	rcu_read_lock();
1214
	for_each_domain(cpu, sd) {
1215 1216 1217 1218 1219 1220
		for_each_cpu(i, sched_domain_span(sd)) {
			if (!idle_cpu(i)) {
				cpu = i;
				goto unlock;
			}
		}
1221
	}
1222 1223
unlock:
	rcu_read_unlock();
1224 1225
	return cpu;
}
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257
/*
 * When add_timer_on() enqueues a timer into the timer wheel of an
 * idle CPU then this timer might expire before the next timer event
 * which is scheduled to wake up that CPU. In case of a completely
 * idle system the next event might even be infinite time into the
 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 * leaves the inner idle loop so the newly added timer is taken into
 * account when the CPU goes back to idle and evaluates the timer
 * wheel for the next timer event.
 */
void wake_up_idle_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);

	if (cpu == smp_processor_id())
		return;

	/*
	 * This is safe, as this function is called with the timer
	 * wheel base lock of (cpu) held. When the CPU is on the way
	 * to idle and has not yet set rq->curr to idle then it will
	 * be serialized on the timer wheel base lock and take the new
	 * timer into account automatically.
	 */
	if (rq->curr != rq->idle)
		return;

	/*
	 * We can set TIF_RESCHED on the idle task of the other CPU
	 * lockless. The worst case is that the other CPU runs the
	 * idle task through an additional NOOP schedule()
	 */
1258
	set_tsk_need_resched(rq->idle);
1259 1260 1261 1262 1263 1264

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(rq->idle))
		smp_send_reschedule(cpu);
}
M
Mike Galbraith 已提交
1265

1266
#endif /* CONFIG_NO_HZ */
1267

1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
static u64 sched_avg_period(void)
{
	return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
}

static void sched_avg_update(struct rq *rq)
{
	s64 period = sched_avg_period();

	while ((s64)(rq->clock - rq->age_stamp) > period) {
1278 1279 1280 1281 1282 1283
		/*
		 * Inline assembly required to prevent the compiler
		 * optimising this loop into a divmod call.
		 * See __iter_div_u64_rem() for another example of this.
		 */
		asm("" : "+rm" (rq->age_stamp));
1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
		rq->age_stamp += period;
		rq->rt_avg /= 2;
	}
}

static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
{
	rq->rt_avg += rt_delta;
	sched_avg_update(rq);
}

1295
#else /* !CONFIG_SMP */
1296
static void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
1297
{
1298
	assert_raw_spin_locked(&task_rq(p)->lock);
1299
	set_tsk_need_resched(p);
I
Ingo Molnar 已提交
1300
}
1301 1302 1303 1304

static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
{
}
1305 1306 1307 1308

static void sched_avg_update(struct rq *rq)
{
}
1309
#endif /* CONFIG_SMP */
I
Ingo Molnar 已提交
1310

1311 1312 1313 1314 1315 1316 1317 1318
#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

I
Ingo Molnar 已提交
1319 1320 1321
/*
 * Shift right and round:
 */
I
Ingo Molnar 已提交
1322
#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
I
Ingo Molnar 已提交
1323

1324 1325 1326
/*
 * delta *= weight / lw
 */
1327
static unsigned long
1328 1329 1330 1331 1332
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

1333 1334 1335 1336 1337 1338 1339
	if (!lw->inv_weight) {
		if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
			lw->inv_weight = 1;
		else
			lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
				/ (lw->weight+1);
	}
1340 1341 1342 1343 1344

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
I
Ingo Molnar 已提交
1345
	if (unlikely(tmp > WMULT_CONST))
I
Ingo Molnar 已提交
1346
		tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
I
Ingo Molnar 已提交
1347 1348
			WMULT_SHIFT/2);
	else
I
Ingo Molnar 已提交
1349
		tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
1350

1351
	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
1352 1353
}

1354
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
1355 1356
{
	lw->weight += inc;
I
Ingo Molnar 已提交
1357
	lw->inv_weight = 0;
1358 1359
}

1360
static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
1361 1362
{
	lw->weight -= dec;
I
Ingo Molnar 已提交
1363
	lw->inv_weight = 0;
1364 1365
}

P
Peter Zijlstra 已提交
1366 1367 1368 1369 1370 1371
static inline void update_load_set(struct load_weight *lw, unsigned long w)
{
	lw->weight = w;
	lw->inv_weight = 0;
}

1372 1373 1374 1375
/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
I
Ingo Molnar 已提交
1376
 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
1377 1378 1379 1380
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

P
Peter Zijlstra 已提交
1381 1382
#define WEIGHT_IDLEPRIO                3
#define WMULT_IDLEPRIO         1431655765
I
Ingo Molnar 已提交
1383 1384 1385 1386 1387 1388 1389 1390 1391

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
1392 1393 1394
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
I
Ingo Molnar 已提交
1395 1396
 */
static const int prio_to_weight[40] = {
1397 1398 1399 1400 1401 1402 1403 1404
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,
 /*  10 */       110,        87,        70,        56,        45,
 /*  15 */        36,        29,        23,        18,        15,
I
Ingo Molnar 已提交
1405 1406
};

1407 1408 1409 1410 1411 1412 1413
/*
 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
 *
 * In cases where the weight does not change often, we can use the
 * precalculated inverse to speed up arithmetics by turning divisions
 * into multiplications:
 */
I
Ingo Molnar 已提交
1414
static const u32 prio_to_wmult[40] = {
1415 1416 1417 1418 1419 1420 1421 1422
 /* -20 */     48388,     59856,     76040,     92818,    118348,
 /* -15 */    147320,    184698,    229616,    287308,    360437,
 /* -10 */    449829,    563644,    704093,    875809,   1099582,
 /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
 /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
 /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
 /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
 /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
I
Ingo Molnar 已提交
1423
};
1424

1425 1426 1427 1428 1429 1430 1431 1432
/* Time spent by the tasks of the cpu accounting group executing in ... */
enum cpuacct_stat_index {
	CPUACCT_STAT_USER,	/* ... user mode */
	CPUACCT_STAT_SYSTEM,	/* ... kernel mode */

	CPUACCT_STAT_NSTATS,
};

1433 1434
#ifdef CONFIG_CGROUP_CPUACCT
static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
1435 1436
static void cpuacct_update_stats(struct task_struct *tsk,
		enum cpuacct_stat_index idx, cputime_t val);
1437 1438
#else
static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
1439 1440
static inline void cpuacct_update_stats(struct task_struct *tsk,
		enum cpuacct_stat_index idx, cputime_t val) {}
1441 1442
#endif

1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
static inline void inc_cpu_load(struct rq *rq, unsigned long load)
{
	update_load_add(&rq->load, load);
}

static inline void dec_cpu_load(struct rq *rq, unsigned long load)
{
	update_load_sub(&rq->load, load);
}

I
Ingo Molnar 已提交
1453
#if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
P
Peter Zijlstra 已提交
1454
typedef int (*tg_visitor)(struct task_group *, void *);
1455 1456 1457 1458 1459

/*
 * Iterate the full tree, calling @down when first entering a node and @up when
 * leaving it for the final time.
 */
P
Peter Zijlstra 已提交
1460
static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
1461 1462
{
	struct task_group *parent, *child;
P
Peter Zijlstra 已提交
1463
	int ret;
1464 1465 1466 1467

	rcu_read_lock();
	parent = &root_task_group;
down:
P
Peter Zijlstra 已提交
1468 1469 1470
	ret = (*down)(parent, data);
	if (ret)
		goto out_unlock;
1471 1472 1473 1474 1475 1476 1477
	list_for_each_entry_rcu(child, &parent->children, siblings) {
		parent = child;
		goto down;

up:
		continue;
	}
P
Peter Zijlstra 已提交
1478 1479 1480
	ret = (*up)(parent, data);
	if (ret)
		goto out_unlock;
1481 1482 1483 1484 1485

	child = parent;
	parent = parent->parent;
	if (parent)
		goto up;
P
Peter Zijlstra 已提交
1486
out_unlock:
1487
	rcu_read_unlock();
P
Peter Zijlstra 已提交
1488 1489

	return ret;
1490 1491
}

P
Peter Zijlstra 已提交
1492 1493 1494
static int tg_nop(struct task_group *tg, void *data)
{
	return 0;
1495
}
P
Peter Zijlstra 已提交
1496 1497 1498
#endif

#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
/* Used instead of source_load when we know the type == 0 */
static unsigned long weighted_cpuload(const int cpu)
{
	return cpu_rq(cpu)->load.weight;
}

/*
 * Return a low guess at the load of a migration-source cpu weighted
 * according to the scheduling class and "nice" value.
 *
 * We want to under-estimate the load of migration sources, to
 * balance conservatively.
 */
static unsigned long source_load(int cpu, int type)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long total = weighted_cpuload(cpu);

	if (type == 0 || !sched_feat(LB_BIAS))
		return total;

	return min(rq->cpu_load[type-1], total);
}

/*
 * Return a high guess at the load of a migration-target cpu weighted
 * according to the scheduling class and "nice" value.
 */
static unsigned long target_load(int cpu, int type)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long total = weighted_cpuload(cpu);

	if (type == 0 || !sched_feat(LB_BIAS))
		return total;

	return max(rq->cpu_load[type-1], total);
}

1538 1539
static unsigned long power_of(int cpu)
{
1540
	return cpu_rq(cpu)->cpu_power;
1541 1542
}

P
Peter Zijlstra 已提交
1543 1544 1545 1546 1547
static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);

static unsigned long cpu_avg_load_per_task(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
1548
	unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
P
Peter Zijlstra 已提交
1549

1550 1551
	if (nr_running)
		rq->avg_load_per_task = rq->load.weight / nr_running;
1552 1553
	else
		rq->avg_load_per_task = 0;
P
Peter Zijlstra 已提交
1554 1555 1556 1557 1558

	return rq->avg_load_per_task;
}

#ifdef CONFIG_FAIR_GROUP_SCHED
1559 1560

/*
1561 1562 1563
 * Compute the cpu's hierarchical load factor for each task group.
 * This needs to be done in a top-down fashion because the load of a child
 * group is a fraction of its parents load.
1564
 */
P
Peter Zijlstra 已提交
1565
static int tg_load_down(struct task_group *tg, void *data)
1566
{
1567
	unsigned long load;
P
Peter Zijlstra 已提交
1568
	long cpu = (long)data;
1569

1570 1571 1572 1573
	if (!tg->parent) {
		load = cpu_rq(cpu)->load.weight;
	} else {
		load = tg->parent->cfs_rq[cpu]->h_load;
P
Peter Zijlstra 已提交
1574
		load *= tg->se[cpu]->load.weight;
1575 1576
		load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
	}
1577

1578
	tg->cfs_rq[cpu]->h_load = load;
1579

P
Peter Zijlstra 已提交
1580
	return 0;
1581 1582
}

P
Peter Zijlstra 已提交
1583
static void update_h_load(long cpu)
1584
{
P
Peter Zijlstra 已提交
1585
	walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
1586 1587
}

1588 1589
#endif

1590 1591
#ifdef CONFIG_PREEMPT

1592 1593
static void double_rq_lock(struct rq *rq1, struct rq *rq2);

1594
/*
1595 1596 1597 1598 1599 1600
 * fair double_lock_balance: Safely acquires both rq->locks in a fair
 * way at the expense of forcing extra atomic operations in all
 * invocations.  This assures that the double_lock is acquired using the
 * same underlying policy as the spinlock_t on this architecture, which
 * reduces latency compared to the unfair variant below.  However, it
 * also adds more overhead and therefore may reduce throughput.
1601
 */
1602 1603 1604 1605 1606
static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
1607
	raw_spin_unlock(&this_rq->lock);
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
	double_rq_lock(this_rq, busiest);

	return 1;
}

#else
/*
 * Unfair double_lock_balance: Optimizes throughput at the expense of
 * latency by eliminating extra atomic operations when the locks are
 * already in proper order on entry.  This favors lower cpu-ids and will
 * grant the double lock to lower cpus over higher ids under contention,
 * regardless of entry order into the function.
 */
static int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
1622 1623 1624 1625 1626 1627
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
	int ret = 0;

1628
	if (unlikely(!raw_spin_trylock(&busiest->lock))) {
1629
		if (busiest < this_rq) {
1630 1631 1632 1633
			raw_spin_unlock(&this_rq->lock);
			raw_spin_lock(&busiest->lock);
			raw_spin_lock_nested(&this_rq->lock,
					      SINGLE_DEPTH_NESTING);
1634 1635
			ret = 1;
		} else
1636 1637
			raw_spin_lock_nested(&busiest->lock,
					      SINGLE_DEPTH_NESTING);
1638 1639 1640 1641
	}
	return ret;
}

1642 1643 1644 1645 1646 1647 1648 1649 1650
#endif /* CONFIG_PREEMPT */

/*
 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
 */
static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
{
	if (unlikely(!irqs_disabled())) {
		/* printk() doesn't work good under rq->lock */
1651
		raw_spin_unlock(&this_rq->lock);
1652 1653 1654 1655 1656 1657
		BUG_ON(1);
	}

	return _double_lock_balance(this_rq, busiest);
}

1658 1659 1660
static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
	__releases(busiest->lock)
{
1661
	raw_spin_unlock(&busiest->lock);
1662 1663
	lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
}
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706

/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
	BUG_ON(!irqs_disabled());
	if (rq1 == rq2) {
		raw_spin_lock(&rq1->lock);
		__acquire(rq2->lock);	/* Fake it out ;) */
	} else {
		if (rq1 < rq2) {
			raw_spin_lock(&rq1->lock);
			raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
		} else {
			raw_spin_lock(&rq2->lock);
			raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
		}
	}
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	raw_spin_unlock(&rq1->lock);
	if (rq1 != rq2)
		raw_spin_unlock(&rq2->lock);
	else
		__release(rq2->lock);
}

1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739
#else /* CONFIG_SMP */

/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
	BUG_ON(!irqs_disabled());
	BUG_ON(rq1 != rq2);
	raw_spin_lock(&rq1->lock);
	__acquire(rq2->lock);	/* Fake it out ;) */
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	BUG_ON(rq1 != rq2);
	raw_spin_unlock(&rq1->lock);
	__release(rq2->lock);
}

1740 1741
#endif

1742
static void calc_load_account_idle(struct rq *this_rq);
1743
static void update_sysctl(void);
1744
static int get_update_sysctl_factor(void);
1745
static void update_cpu_load(struct rq *this_rq);
1746

P
Peter Zijlstra 已提交
1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
	set_task_rq(p, cpu);
#ifdef CONFIG_SMP
	/*
	 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
	 * successfuly executed on another CPU. We must ensure that updates of
	 * per-task data have been completed by this moment.
	 */
	smp_wmb();
	task_thread_info(p)->cpu = cpu;
#endif
}
1760

1761
static const struct sched_class rt_sched_class;
I
Ingo Molnar 已提交
1762

1763
#define sched_class_highest (&stop_sched_class)
1764 1765
#define for_each_class(class) \
   for (class = sched_class_highest; class; class = class->next)
I
Ingo Molnar 已提交
1766

1767 1768
#include "sched_stats.h"

1769
static void inc_nr_running(struct rq *rq)
1770 1771 1772 1773
{
	rq->nr_running++;
}

1774
static void dec_nr_running(struct rq *rq)
1775 1776 1777 1778
{
	rq->nr_running--;
}

1779 1780
static void set_load_weight(struct task_struct *p)
{
I
Ingo Molnar 已提交
1781 1782 1783 1784 1785 1786 1787 1788
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
		p->se.load.weight = WEIGHT_IDLEPRIO;
		p->se.load.inv_weight = WMULT_IDLEPRIO;
		return;
	}
1789

I
Ingo Molnar 已提交
1790 1791
	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
1792 1793
}

1794
static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
1795
{
1796
	update_rq_clock(rq);
I
Ingo Molnar 已提交
1797
	sched_info_queued(p);
1798
	p->sched_class->enqueue_task(rq, p, flags);
1799 1800
}

1801
static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
1802
{
1803
	update_rq_clock(rq);
1804
	sched_info_dequeued(p);
1805
	p->sched_class->dequeue_task(rq, p, flags);
1806 1807
}

1808 1809 1810
/*
 * activate_task - move a task to the runqueue.
 */
1811
static void activate_task(struct rq *rq, struct task_struct *p, int flags)
1812 1813 1814 1815
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible--;

1816
	enqueue_task(rq, p, flags);
1817 1818 1819 1820 1821 1822
	inc_nr_running(rq);
}

/*
 * deactivate_task - remove a task from the runqueue.
 */
1823
static void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
1824 1825 1826 1827
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible++;

1828
	dequeue_task(rq, p, flags);
1829 1830 1831
	dec_nr_running(rq);
}

1832 1833
#ifdef CONFIG_IRQ_TIME_ACCOUNTING

1834 1835 1836 1837 1838 1839 1840
/*
 * There are no locks covering percpu hardirq/softirq time.
 * They are only modified in account_system_vtime, on corresponding CPU
 * with interrupts disabled. So, writes are safe.
 * They are read and saved off onto struct rq in update_rq_clock().
 * This may result in other CPU reading this CPU's irq time and can
 * race with irq/account_system_vtime on this CPU. We would either get old
1841 1842 1843
 * or new value with a side effect of accounting a slice of irq time to wrong
 * task when irq is in progress while we read rq->clock. That is a worthy
 * compromise in place of having locks on each irq in account_system_time.
1844
 */
1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
static DEFINE_PER_CPU(u64, cpu_hardirq_time);
static DEFINE_PER_CPU(u64, cpu_softirq_time);

static DEFINE_PER_CPU(u64, irq_start_time);
static int sched_clock_irqtime;

void enable_sched_clock_irqtime(void)
{
	sched_clock_irqtime = 1;
}

void disable_sched_clock_irqtime(void)
{
	sched_clock_irqtime = 0;
}

1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898
#ifndef CONFIG_64BIT
static DEFINE_PER_CPU(seqcount_t, irq_time_seq);

static inline void irq_time_write_begin(void)
{
	__this_cpu_inc(irq_time_seq.sequence);
	smp_wmb();
}

static inline void irq_time_write_end(void)
{
	smp_wmb();
	__this_cpu_inc(irq_time_seq.sequence);
}

static inline u64 irq_time_read(int cpu)
{
	u64 irq_time;
	unsigned seq;

	do {
		seq = read_seqcount_begin(&per_cpu(irq_time_seq, cpu));
		irq_time = per_cpu(cpu_softirq_time, cpu) +
			   per_cpu(cpu_hardirq_time, cpu);
	} while (read_seqcount_retry(&per_cpu(irq_time_seq, cpu), seq));

	return irq_time;
}
#else /* CONFIG_64BIT */
static inline void irq_time_write_begin(void)
{
}

static inline void irq_time_write_end(void)
{
}

static inline u64 irq_time_read(int cpu)
1899 1900 1901
{
	return per_cpu(cpu_softirq_time, cpu) + per_cpu(cpu_hardirq_time, cpu);
}
1902
#endif /* CONFIG_64BIT */
1903

1904 1905 1906 1907
/*
 * Called before incrementing preempt_count on {soft,}irq_enter
 * and before decrementing preempt_count on {soft,}irq_exit.
 */
1908 1909 1910
void account_system_vtime(struct task_struct *curr)
{
	unsigned long flags;
1911
	s64 delta;
1912 1913 1914 1915 1916 1917 1918 1919
	int cpu;

	if (!sched_clock_irqtime)
		return;

	local_irq_save(flags);

	cpu = smp_processor_id();
1920 1921 1922
	delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
	__this_cpu_add(irq_start_time, delta);

1923
	irq_time_write_begin();
1924 1925 1926 1927 1928 1929 1930
	/*
	 * We do not account for softirq time from ksoftirqd here.
	 * We want to continue accounting softirq time to ksoftirqd thread
	 * in that case, so as not to confuse scheduler with a special task
	 * that do not consume any time, but still wants to run.
	 */
	if (hardirq_count())
1931
		__this_cpu_add(cpu_hardirq_time, delta);
1932
	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
1933
		__this_cpu_add(cpu_softirq_time, delta);
1934

1935
	irq_time_write_end();
1936 1937
	local_irq_restore(flags);
}
I
Ingo Molnar 已提交
1938
EXPORT_SYMBOL_GPL(account_system_vtime);
1939

1940
static void update_rq_clock_task(struct rq *rq, s64 delta)
1941
{
1942 1943
	s64 irq_delta;

1944
	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969

	/*
	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
	 * this case when a previous update_rq_clock() happened inside a
	 * {soft,}irq region.
	 *
	 * When this happens, we stop ->clock_task and only update the
	 * prev_irq_time stamp to account for the part that fit, so that a next
	 * update will consume the rest. This ensures ->clock_task is
	 * monotonic.
	 *
	 * It does however cause some slight miss-attribution of {soft,}irq
	 * time, a more accurate solution would be to update the irq_time using
	 * the current rq->clock timestamp, except that would require using
	 * atomic ops.
	 */
	if (irq_delta > delta)
		irq_delta = delta;

	rq->prev_irq_time += irq_delta;
	delta -= irq_delta;
	rq->clock_task += delta;

	if (irq_delta && sched_feat(NONIRQ_POWER))
		sched_rt_avg_update(rq, irq_delta);
1970 1971
}

1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
static int irqtime_account_hi_update(void)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	unsigned long flags;
	u64 latest_ns;
	int ret = 0;

	local_irq_save(flags);
	latest_ns = this_cpu_read(cpu_hardirq_time);
	if (cputime64_gt(nsecs_to_cputime64(latest_ns), cpustat->irq))
		ret = 1;
	local_irq_restore(flags);
	return ret;
}

static int irqtime_account_si_update(void)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	unsigned long flags;
	u64 latest_ns;
	int ret = 0;

	local_irq_save(flags);
	latest_ns = this_cpu_read(cpu_softirq_time);
	if (cputime64_gt(nsecs_to_cputime64(latest_ns), cpustat->softirq))
		ret = 1;
	local_irq_restore(flags);
	return ret;
}

2002
#else /* CONFIG_IRQ_TIME_ACCOUNTING */
2003

2004 2005
#define sched_clock_irqtime	(0)

2006
static void update_rq_clock_task(struct rq *rq, s64 delta)
2007
{
2008
	rq->clock_task += delta;
2009 2010
}

2011
#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
2012

2013 2014 2015
#include "sched_idletask.c"
#include "sched_fair.c"
#include "sched_rt.c"
2016
#include "sched_autogroup.c"
2017
#include "sched_stoptask.c"
2018 2019 2020 2021
#ifdef CONFIG_SCHED_DEBUG
# include "sched_debug.c"
#endif

2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051
void sched_set_stop_task(int cpu, struct task_struct *stop)
{
	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	struct task_struct *old_stop = cpu_rq(cpu)->stop;

	if (stop) {
		/*
		 * Make it appear like a SCHED_FIFO task, its something
		 * userspace knows about and won't get confused about.
		 *
		 * Also, it will make PI more or less work without too
		 * much confusion -- but then, stop work should not
		 * rely on PI working anyway.
		 */
		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);

		stop->sched_class = &stop_sched_class;
	}

	cpu_rq(cpu)->stop = stop;

	if (old_stop) {
		/*
		 * Reset it back to a normal scheduling class so that
		 * it can die in pieces.
		 */
		old_stop->sched_class = &rt_sched_class;
	}
}

2052
/*
I
Ingo Molnar 已提交
2053
 * __normal_prio - return the priority that is based on the static prio
2054 2055 2056
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
2057
	return p->static_prio;
2058 2059
}

2060 2061 2062 2063 2064 2065 2066
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
2067
static inline int normal_prio(struct task_struct *p)
2068 2069 2070
{
	int prio;

2071
	if (task_has_rt_policy(p))
2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
2085
static int effective_prio(struct task_struct *p)
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
2098 2099 2100 2101
/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
2102
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
2103 2104 2105 2106
{
	return cpu_curr(task_cpu(p)) == p;
}

2107 2108
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
				       const struct sched_class *prev_class,
P
Peter Zijlstra 已提交
2109
				       int oldprio)
2110 2111 2112
{
	if (prev_class != p->sched_class) {
		if (prev_class->switched_from)
P
Peter Zijlstra 已提交
2113 2114 2115 2116
			prev_class->switched_from(rq, p);
		p->sched_class->switched_to(rq, p);
	} else if (oldprio != p->prio)
		p->sched_class->prio_changed(rq, p, oldprio);
2117 2118
}

2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
{
	const struct sched_class *class;

	if (p->sched_class == rq->curr->sched_class) {
		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
	} else {
		for_each_class(class) {
			if (class == rq->curr->sched_class)
				break;
			if (class == p->sched_class) {
				resched_task(rq->curr);
				break;
			}
		}
	}

	/*
	 * A queue event has occurred, and we're going to schedule.  In
	 * this case, we can save a useless back to back clock update.
	 */
P
Peter Zijlstra 已提交
2140
	if (rq->curr->on_rq && test_tsk_need_resched(rq->curr))
2141 2142 2143
		rq->skip_clock_update = 1;
}

L
Linus Torvalds 已提交
2144
#ifdef CONFIG_SMP
2145 2146 2147
/*
 * Is this task likely cache-hot:
 */
2148
static int
2149 2150 2151 2152
task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
{
	s64 delta;

P
Peter Zijlstra 已提交
2153 2154 2155
	if (p->sched_class != &fair_sched_class)
		return 0;

2156 2157 2158
	if (unlikely(p->policy == SCHED_IDLE))
		return 0;

2159 2160 2161
	/*
	 * Buddy candidates are cache hot:
	 */
2162
	if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
P
Peter Zijlstra 已提交
2163 2164
			(&p->se == cfs_rq_of(&p->se)->next ||
			 &p->se == cfs_rq_of(&p->se)->last))
2165 2166
		return 1;

2167 2168 2169 2170 2171
	if (sysctl_sched_migration_cost == -1)
		return 1;
	if (sysctl_sched_migration_cost == 0)
		return 0;

2172 2173 2174 2175 2176
	delta = now - p->se.exec_start;

	return delta < (s64)sysctl_sched_migration_cost;
}

I
Ingo Molnar 已提交
2177
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
2178
{
2179 2180 2181 2182 2183
#ifdef CONFIG_SCHED_DEBUG
	/*
	 * We should never call set_task_cpu() on a blocked task,
	 * ttwu() will sort out the placement.
	 */
P
Peter Zijlstra 已提交
2184 2185
	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
			!(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
2186 2187 2188 2189 2190

#ifdef CONFIG_LOCKDEP
	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
				      lockdep_is_held(&task_rq(p)->lock)));
#endif
2191 2192
#endif

2193
	trace_sched_migrate_task(p, new_cpu);
2194

2195 2196 2197 2198
	if (task_cpu(p) != new_cpu) {
		p->se.nr_migrations++;
		perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
	}
I
Ingo Molnar 已提交
2199 2200

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
2201 2202
}

2203
struct migration_arg {
2204
	struct task_struct *task;
L
Linus Torvalds 已提交
2205
	int dest_cpu;
2206
};
L
Linus Torvalds 已提交
2207

2208 2209
static int migration_cpu_stop(void *data);

L
Linus Torvalds 已提交
2210 2211 2212
/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
R
Roland McGrath 已提交
2213 2214 2215 2216 2217 2218 2219
 * If @match_state is nonzero, it's the @p->state value just checked and
 * not expected to change.  If it changes, i.e. @p might have woken up,
 * then return zero.  When we succeed in waiting for @p to be off its CPU,
 * we return a positive number (its total switch count).  If a second call
 * a short while later returns the same number, the caller can be sure that
 * @p has remained unscheduled the whole time.
 *
L
Linus Torvalds 已提交
2220 2221 2222 2223 2224 2225
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
R
Roland McGrath 已提交
2226
unsigned long wait_task_inactive(struct task_struct *p, long match_state)
L
Linus Torvalds 已提交
2227 2228
{
	unsigned long flags;
I
Ingo Molnar 已提交
2229
	int running, on_rq;
R
Roland McGrath 已提交
2230
	unsigned long ncsw;
2231
	struct rq *rq;
L
Linus Torvalds 已提交
2232

2233 2234 2235 2236 2237 2238 2239 2240
	for (;;) {
		/*
		 * We do the initial early heuristics without holding
		 * any task-queue locks at all. We'll only try to get
		 * the runqueue lock when things look like they will
		 * work out!
		 */
		rq = task_rq(p);
2241

2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252
		/*
		 * If the task is actively running on another CPU
		 * still, just relax and busy-wait without holding
		 * any locks.
		 *
		 * NOTE! Since we don't hold any locks, it's not
		 * even sure that "rq" stays as the right runqueue!
		 * But we don't care, since "task_running()" will
		 * return false if the runqueue has changed and p
		 * is actually now running somewhere else!
		 */
R
Roland McGrath 已提交
2253 2254 2255
		while (task_running(rq, p)) {
			if (match_state && unlikely(p->state != match_state))
				return 0;
2256
			cpu_relax();
R
Roland McGrath 已提交
2257
		}
2258

2259 2260 2261 2262 2263 2264
		/*
		 * Ok, time to look more closely! We need the rq
		 * lock now, to be *sure*. If we're wrong, we'll
		 * just go back and repeat.
		 */
		rq = task_rq_lock(p, &flags);
2265
		trace_sched_wait_task(p);
2266
		running = task_running(rq, p);
P
Peter Zijlstra 已提交
2267
		on_rq = p->on_rq;
R
Roland McGrath 已提交
2268
		ncsw = 0;
2269
		if (!match_state || p->state == match_state)
2270
			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
2271
		task_rq_unlock(rq, p, &flags);
2272

R
Roland McGrath 已提交
2273 2274 2275 2276 2277 2278
		/*
		 * If it changed from the expected state, bail out now.
		 */
		if (unlikely(!ncsw))
			break;

2279 2280 2281 2282 2283 2284 2285 2286 2287 2288
		/*
		 * Was it really running after all now that we
		 * checked with the proper locks actually held?
		 *
		 * Oops. Go back and try again..
		 */
		if (unlikely(running)) {
			cpu_relax();
			continue;
		}
2289

2290 2291 2292 2293 2294
		/*
		 * It's not enough that it's not actively running,
		 * it must be off the runqueue _entirely_, and not
		 * preempted!
		 *
2295
		 * So if it was still runnable (but just not actively
2296 2297 2298 2299
		 * running right now), it's preempted, and we should
		 * yield - it could be a while.
		 */
		if (unlikely(on_rq)) {
2300 2301 2302 2303
			ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);

			set_current_state(TASK_UNINTERRUPTIBLE);
			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
2304 2305
			continue;
		}
2306

2307 2308 2309 2310 2311 2312 2313
		/*
		 * Ahh, all good. It wasn't running, and it wasn't
		 * runnable, which means that it will never become
		 * running in the future either. We're all done!
		 */
		break;
	}
R
Roland McGrath 已提交
2314 2315

	return ncsw;
L
Linus Torvalds 已提交
2316 2317 2318 2319 2320 2321 2322 2323 2324
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
L
Lucas De Marchi 已提交
2325
 * NOTE: this function doesn't have to take the runqueue lock,
L
Linus Torvalds 已提交
2326 2327 2328 2329 2330
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
2331
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
2332 2333 2334 2335 2336 2337 2338 2339 2340
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}
R
Rusty Russell 已提交
2341
EXPORT_SYMBOL_GPL(kick_process);
N
Nick Piggin 已提交
2342
#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
2343

2344
#ifdef CONFIG_SMP
2345
/*
2346
 * ->cpus_allowed is protected by both rq->lock and p->pi_lock
2347
 */
2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363
static int select_fallback_rq(int cpu, struct task_struct *p)
{
	int dest_cpu;
	const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));

	/* Look for allowed, online CPU in same node. */
	for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
		if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
			return dest_cpu;

	/* Any allowed, online CPU? */
	dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
	if (dest_cpu < nr_cpu_ids)
		return dest_cpu;

	/* No more Mr. Nice Guy. */
2364 2365 2366 2367 2368 2369 2370 2371 2372
	dest_cpu = cpuset_cpus_allowed_fallback(p);
	/*
	 * Don't tell them about moving exiting tasks or
	 * kernel threads (both mm NULL), since they never
	 * leave kernel.
	 */
	if (p->mm && printk_ratelimit()) {
		printk(KERN_INFO "process %d (%s) no longer affine to cpu%d\n",
				task_pid_nr(p), p->comm, cpu);
2373 2374 2375 2376 2377
	}

	return dest_cpu;
}

2378
/*
2379
 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
2380
 */
2381
static inline
2382
int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
2383
{
2384
	int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396

	/*
	 * In order not to call set_task_cpu() on a blocking task we need
	 * to rely on ttwu() to place the task on a valid ->cpus_allowed
	 * cpu.
	 *
	 * Since this is common to all placement strategies, this lives here.
	 *
	 * [ this allows ->select_task() to simply return task_cpu(p) and
	 *   not worry about this generic constraint ]
	 */
	if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
P
Peter Zijlstra 已提交
2397
		     !cpu_online(cpu)))
2398
		cpu = select_fallback_rq(task_cpu(p), p);
2399 2400

	return cpu;
2401
}
2402 2403 2404 2405 2406 2407

static void update_avg(u64 *avg, u64 sample)
{
	s64 diff = sample - *avg;
	*avg += diff >> 3;
}
2408 2409
#endif

P
Peter Zijlstra 已提交
2410
static void
2411
ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
T
Tejun Heo 已提交
2412
{
P
Peter Zijlstra 已提交
2413
#ifdef CONFIG_SCHEDSTATS
2414 2415
	struct rq *rq = this_rq();

P
Peter Zijlstra 已提交
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425
#ifdef CONFIG_SMP
	int this_cpu = smp_processor_id();

	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
		schedstat_inc(p, se.statistics.nr_wakeups_local);
	} else {
		struct sched_domain *sd;

		schedstat_inc(p, se.statistics.nr_wakeups_remote);
2426
		rcu_read_lock();
P
Peter Zijlstra 已提交
2427 2428 2429 2430 2431 2432
		for_each_domain(this_cpu, sd) {
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
				schedstat_inc(sd, ttwu_wake_remote);
				break;
			}
		}
2433
		rcu_read_unlock();
P
Peter Zijlstra 已提交
2434 2435 2436 2437
	}
#endif /* CONFIG_SMP */

	schedstat_inc(rq, ttwu_count);
T
Tejun Heo 已提交
2438
	schedstat_inc(p, se.statistics.nr_wakeups);
P
Peter Zijlstra 已提交
2439 2440

	if (wake_flags & WF_SYNC)
T
Tejun Heo 已提交
2441
		schedstat_inc(p, se.statistics.nr_wakeups_sync);
P
Peter Zijlstra 已提交
2442 2443

	if (cpu != task_cpu(p))
T
Tejun Heo 已提交
2444 2445
		schedstat_inc(p, se.statistics.nr_wakeups_migrate);

P
Peter Zijlstra 已提交
2446 2447 2448 2449 2450
#endif /* CONFIG_SCHEDSTATS */
}

static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
{
T
Tejun Heo 已提交
2451
	activate_task(rq, p, en_flags);
P
Peter Zijlstra 已提交
2452
	p->on_rq = 1;
2453 2454 2455 2456

	/* if a worker is waking up, notify workqueue */
	if (p->flags & PF_WQ_WORKER)
		wq_worker_waking_up(p, cpu_of(rq));
T
Tejun Heo 已提交
2457 2458
}

2459 2460 2461
/*
 * Mark the task runnable and perform wakeup-preemption.
 */
2462
static void
2463
ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
T
Tejun Heo 已提交
2464
{
2465
	trace_sched_wakeup(p, true);
T
Tejun Heo 已提交
2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485
	check_preempt_curr(rq, p, wake_flags);

	p->state = TASK_RUNNING;
#ifdef CONFIG_SMP
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);

	if (unlikely(rq->idle_stamp)) {
		u64 delta = rq->clock - rq->idle_stamp;
		u64 max = 2*sysctl_sched_migration_cost;

		if (delta > max)
			rq->avg_idle = max;
		else
			update_avg(&rq->avg_idle, delta);
		rq->idle_stamp = 0;
	}
#endif
}

2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
static void
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
{
#ifdef CONFIG_SMP
	if (p->sched_contributes_to_load)
		rq->nr_uninterruptible--;
#endif

	ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
	ttwu_do_wakeup(rq, p, wake_flags);
}

/*
 * Called in case the task @p isn't fully descheduled from its runqueue,
 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
 * since all we need to do is flip p->state to TASK_RUNNING, since
 * the task is still ->on_rq.
 */
static int ttwu_remote(struct task_struct *p, int wake_flags)
{
	struct rq *rq;
	int ret = 0;

	rq = __task_rq_lock(p);
	if (p->on_rq) {
		ttwu_do_wakeup(rq, p, wake_flags);
		ret = 1;
	}
	__task_rq_unlock(rq);

	return ret;
}

2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562
#ifdef CONFIG_SMP
static void sched_ttwu_pending(void)
{
	struct rq *rq = this_rq();
	struct task_struct *list = xchg(&rq->wake_list, NULL);

	if (!list)
		return;

	raw_spin_lock(&rq->lock);

	while (list) {
		struct task_struct *p = list;
		list = list->wake_entry;
		ttwu_do_activate(rq, p, 0);
	}

	raw_spin_unlock(&rq->lock);
}

void scheduler_ipi(void)
{
	sched_ttwu_pending();
}

static void ttwu_queue_remote(struct task_struct *p, int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	struct task_struct *next = rq->wake_list;

	for (;;) {
		struct task_struct *old = next;

		p->wake_entry = next;
		next = cmpxchg(&rq->wake_list, old, p);
		if (next == old)
			break;
	}

	if (!next)
		smp_send_reschedule(cpu);
}
#endif

2563 2564 2565 2566
static void ttwu_queue(struct task_struct *p, int cpu)
{
	struct rq *rq = cpu_rq(cpu);

2567 2568 2569 2570 2571 2572 2573
#if defined(CONFIG_SMP) && defined(CONFIG_SCHED_TTWU_QUEUE)
	if (sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) {
		ttwu_queue_remote(p, cpu);
		return;
	}
#endif

2574 2575 2576
	raw_spin_lock(&rq->lock);
	ttwu_do_activate(rq, p, 0);
	raw_spin_unlock(&rq->lock);
T
Tejun Heo 已提交
2577 2578 2579
}

/**
L
Linus Torvalds 已提交
2580
 * try_to_wake_up - wake up a thread
T
Tejun Heo 已提交
2581
 * @p: the thread to be awakened
L
Linus Torvalds 已提交
2582
 * @state: the mask of task states that can be woken
T
Tejun Heo 已提交
2583
 * @wake_flags: wake modifier flags (WF_*)
L
Linus Torvalds 已提交
2584 2585 2586 2587 2588 2589 2590
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
T
Tejun Heo 已提交
2591 2592
 * Returns %true if @p was woken up, %false if it was already running
 * or @state didn't match @p's state.
L
Linus Torvalds 已提交
2593
 */
2594 2595
static int
try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
L
Linus Torvalds 已提交
2596 2597
{
	unsigned long flags;
2598
	int cpu, success = 0;
P
Peter Zijlstra 已提交
2599

2600
	smp_wmb();
2601
	raw_spin_lock_irqsave(&p->pi_lock, flags);
P
Peter Zijlstra 已提交
2602
	if (!(p->state & state))
L
Linus Torvalds 已提交
2603 2604
		goto out;

2605
	success = 1; /* we're going to change ->state */
L
Linus Torvalds 已提交
2606 2607
	cpu = task_cpu(p);

2608 2609
	if (p->on_rq && ttwu_remote(p, wake_flags))
		goto stat;
L
Linus Torvalds 已提交
2610 2611

#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
2612
	/*
2613 2614
	 * If the owning (remote) cpu is still in the middle of schedule() with
	 * this task as prev, wait until its done referencing the task.
P
Peter Zijlstra 已提交
2615
	 */
2616 2617 2618 2619 2620 2621 2622 2623
	while (p->on_cpu) {
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
		/*
		 * If called from interrupt context we could have landed in the
		 * middle of schedule(), in this case we should take care not
		 * to spin on ->on_cpu if p is current, since that would
		 * deadlock.
		 */
2624 2625 2626 2627
		if (p == current) {
			ttwu_queue(p, cpu);
			goto stat;
		}
2628 2629
#endif
		cpu_relax();
2630
	}
2631
	/*
2632
	 * Pairs with the smp_wmb() in finish_lock_switch().
2633
	 */
2634
	smp_rmb();
L
Linus Torvalds 已提交
2635

2636
	p->sched_contributes_to_load = !!task_contributes_to_load(p);
P
Peter Zijlstra 已提交
2637
	p->state = TASK_WAKING;
2638

2639
	if (p->sched_class->task_waking)
2640
		p->sched_class->task_waking(p);
2641

2642
	cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
2643
	if (task_cpu(p) != cpu)
2644
		set_task_cpu(p, cpu);
L
Linus Torvalds 已提交
2645 2646
#endif /* CONFIG_SMP */

2647 2648
	ttwu_queue(p, cpu);
stat:
2649
	ttwu_stat(p, cpu, wake_flags);
L
Linus Torvalds 已提交
2650
out:
2651
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
2652 2653 2654 2655

	return success;
}

T
Tejun Heo 已提交
2656 2657 2658 2659
/**
 * try_to_wake_up_local - try to wake up a local task with rq lock held
 * @p: the thread to be awakened
 *
2660
 * Put @p on the run-queue if it's not already there. The caller must
T
Tejun Heo 已提交
2661
 * ensure that this_rq() is locked, @p is bound to this_rq() and not
2662
 * the current task.
T
Tejun Heo 已提交
2663 2664 2665 2666 2667 2668 2669 2670 2671
 */
static void try_to_wake_up_local(struct task_struct *p)
{
	struct rq *rq = task_rq(p);

	BUG_ON(rq != this_rq());
	BUG_ON(p == current);
	lockdep_assert_held(&rq->lock);

2672 2673 2674 2675 2676 2677
	if (!raw_spin_trylock(&p->pi_lock)) {
		raw_spin_unlock(&rq->lock);
		raw_spin_lock(&p->pi_lock);
		raw_spin_lock(&rq->lock);
	}

T
Tejun Heo 已提交
2678
	if (!(p->state & TASK_NORMAL))
2679
		goto out;
T
Tejun Heo 已提交
2680

P
Peter Zijlstra 已提交
2681
	if (!p->on_rq)
P
Peter Zijlstra 已提交
2682 2683
		ttwu_activate(rq, p, ENQUEUE_WAKEUP);

2684
	ttwu_do_wakeup(rq, p, 0);
2685
	ttwu_stat(p, smp_processor_id(), 0);
2686 2687
out:
	raw_spin_unlock(&p->pi_lock);
T
Tejun Heo 已提交
2688 2689
}

2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700
/**
 * wake_up_process - Wake up a specific process
 * @p: The process to be woken up.
 *
 * Attempt to wake up the nominated process and move it to the set of runnable
 * processes.  Returns 1 if the process was woken up, 0 if it was already
 * running.
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
 */
2701
int wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
2702
{
2703
	return try_to_wake_up(p, TASK_ALL, 0);
L
Linus Torvalds 已提交
2704 2705 2706
}
EXPORT_SYMBOL(wake_up_process);

2707
int wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
2708 2709 2710 2711 2712 2713 2714
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
2715 2716 2717 2718 2719
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
P
Peter Zijlstra 已提交
2720 2721 2722
	p->on_rq			= 0;

	p->se.on_rq			= 0;
I
Ingo Molnar 已提交
2723 2724
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
2725
	p->se.prev_sum_exec_runtime	= 0;
2726
	p->se.nr_migrations		= 0;
P
Peter Zijlstra 已提交
2727
	p->se.vruntime			= 0;
P
Peter Zijlstra 已提交
2728
	INIT_LIST_HEAD(&p->se.group_node);
I
Ingo Molnar 已提交
2729 2730

#ifdef CONFIG_SCHEDSTATS
2731
	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
I
Ingo Molnar 已提交
2732
#endif
N
Nick Piggin 已提交
2733

P
Peter Zijlstra 已提交
2734
	INIT_LIST_HEAD(&p->rt.run_list);
N
Nick Piggin 已提交
2735

2736 2737 2738
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif
I
Ingo Molnar 已提交
2739 2740 2741 2742 2743
}

/*
 * fork()/clone()-time setup:
 */
2744
void sched_fork(struct task_struct *p)
I
Ingo Molnar 已提交
2745
{
2746
	unsigned long flags;
I
Ingo Molnar 已提交
2747 2748 2749
	int cpu = get_cpu();

	__sched_fork(p);
2750
	/*
2751
	 * We mark the process as running here. This guarantees that
2752 2753 2754
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
2755
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
2756

2757 2758 2759 2760
	/*
	 * Revert to default priority/policy on fork if requested.
	 */
	if (unlikely(p->sched_reset_on_fork)) {
2761
		if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) {
2762
			p->policy = SCHED_NORMAL;
2763 2764
			p->normal_prio = p->static_prio;
		}
2765

2766 2767
		if (PRIO_TO_NICE(p->static_prio) < 0) {
			p->static_prio = NICE_TO_PRIO(0);
2768
			p->normal_prio = p->static_prio;
2769 2770 2771
			set_load_weight(p);
		}

2772 2773 2774 2775 2776 2777
		/*
		 * We don't need the reset flag anymore after the fork. It has
		 * fulfilled its duty:
		 */
		p->sched_reset_on_fork = 0;
	}
2778

2779 2780 2781 2782 2783
	/*
	 * Make sure we do not leak PI boosting priority to the child.
	 */
	p->prio = current->normal_prio;

H
Hiroshi Shimamoto 已提交
2784 2785
	if (!rt_prio(p->prio))
		p->sched_class = &fair_sched_class;
2786

P
Peter Zijlstra 已提交
2787 2788 2789
	if (p->sched_class->task_fork)
		p->sched_class->task_fork(p);

2790 2791 2792 2793 2794 2795 2796
	/*
	 * The child is not yet in the pid-hash so no cgroup attach races,
	 * and the cgroup is pinned to this child due to cgroup_fork()
	 * is ran before sched_fork().
	 *
	 * Silence PROVE_RCU.
	 */
2797
	raw_spin_lock_irqsave(&p->pi_lock, flags);
2798
	set_task_cpu(p, cpu);
2799
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2800

2801
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
2802
	if (likely(sched_info_on()))
2803
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
2804
#endif
P
Peter Zijlstra 已提交
2805 2806
#if defined(CONFIG_SMP)
	p->on_cpu = 0;
2807
#endif
L
Linus Torvalds 已提交
2808
#ifdef CONFIG_PREEMPT
2809
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
2810
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
2811
#endif
2812
#ifdef CONFIG_SMP
2813
	plist_node_init(&p->pushable_tasks, MAX_PRIO);
2814
#endif
2815

N
Nick Piggin 已提交
2816
	put_cpu();
L
Linus Torvalds 已提交
2817 2818 2819 2820 2821 2822 2823 2824 2825
}

/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
2826
void wake_up_new_task(struct task_struct *p)
L
Linus Torvalds 已提交
2827 2828
{
	unsigned long flags;
I
Ingo Molnar 已提交
2829
	struct rq *rq;
2830

2831
	raw_spin_lock_irqsave(&p->pi_lock, flags);
2832 2833 2834 2835 2836 2837
#ifdef CONFIG_SMP
	/*
	 * Fork balancing, do it here and not earlier because:
	 *  - cpus_allowed can change in the fork path
	 *  - any previously selected cpu might disappear through hotplug
	 */
2838
	set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0));
2839 2840
#endif

2841
	rq = __task_rq_lock(p);
P
Peter Zijlstra 已提交
2842
	activate_task(rq, p, 0);
P
Peter Zijlstra 已提交
2843
	p->on_rq = 1;
2844
	trace_sched_wakeup_new(p, true);
P
Peter Zijlstra 已提交
2845
	check_preempt_curr(rq, p, WF_FORK);
2846
#ifdef CONFIG_SMP
2847 2848
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);
2849
#endif
2850
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
2851 2852
}

2853 2854 2855
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
2856
 * preempt_notifier_register - tell me when current is being preempted & rescheduled
R
Randy Dunlap 已提交
2857
 * @notifier: notifier struct to register
2858 2859 2860 2861 2862 2863 2864 2865 2866
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
2867
 * @notifier: notifier struct to unregister
2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_out(notifier, next);
}

2897
#else /* !CONFIG_PREEMPT_NOTIFIERS */
2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

2909
#endif /* CONFIG_PREEMPT_NOTIFIERS */
2910

2911 2912 2913
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
2914
 * @prev: the current task that is being switched out
2915 2916 2917 2918 2919 2920 2921 2922 2923
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
2924 2925 2926
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
2927
{
2928 2929
	sched_info_switch(prev, next);
	perf_event_task_sched_out(prev, next);
2930
	fire_sched_out_preempt_notifiers(prev, next);
2931 2932
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
2933
	trace_sched_switch(prev, next);
2934 2935
}

L
Linus Torvalds 已提交
2936 2937
/**
 * finish_task_switch - clean up after a task-switch
2938
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
2939 2940
 * @prev: the thread we just switched away from.
 *
2941 2942 2943 2944
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
2945 2946
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
I
Ingo Molnar 已提交
2947
 * so, we finish that here outside of the runqueue lock. (Doing it
L
Linus Torvalds 已提交
2948 2949 2950
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
A
Alexey Dobriyan 已提交
2951
static void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
2952 2953 2954
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
2955
	long prev_state;
L
Linus Torvalds 已提交
2956 2957 2958 2959 2960

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
2961
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
2962 2963
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
2964
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
2965 2966 2967 2968 2969
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
2970
	prev_state = prev->state;
2971
	finish_arch_switch(prev);
2972 2973 2974
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_disable();
#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
2975
	perf_event_task_sched_in(current);
2976 2977 2978
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
2979
	finish_lock_switch(rq, prev);
S
Steven Rostedt 已提交
2980

2981
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
2982 2983
	if (mm)
		mmdrop(mm);
2984
	if (unlikely(prev_state == TASK_DEAD)) {
2985 2986 2987
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
2988
		 */
2989
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
2990
		put_task_struct(prev);
2991
	}
L
Linus Torvalds 已提交
2992 2993
}

2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008
#ifdef CONFIG_SMP

/* assumes rq->lock is held */
static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
{
	if (prev->sched_class->pre_schedule)
		prev->sched_class->pre_schedule(rq, prev);
}

/* rq->lock is NOT held, but preemption is disabled */
static inline void post_schedule(struct rq *rq)
{
	if (rq->post_schedule) {
		unsigned long flags;

3009
		raw_spin_lock_irqsave(&rq->lock, flags);
3010 3011
		if (rq->curr->sched_class->post_schedule)
			rq->curr->sched_class->post_schedule(rq);
3012
		raw_spin_unlock_irqrestore(&rq->lock, flags);
3013 3014 3015 3016 3017 3018

		rq->post_schedule = 0;
	}
}

#else
3019

3020 3021 3022 3023 3024 3025
static inline void pre_schedule(struct rq *rq, struct task_struct *p)
{
}

static inline void post_schedule(struct rq *rq)
{
L
Linus Torvalds 已提交
3026 3027
}

3028 3029
#endif

L
Linus Torvalds 已提交
3030 3031 3032 3033
/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
3034
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
3035 3036
	__releases(rq->lock)
{
3037 3038
	struct rq *rq = this_rq();

3039
	finish_task_switch(rq, prev);
3040

3041 3042 3043 3044 3045
	/*
	 * FIXME: do we need to worry about rq being invalidated by the
	 * task_switch?
	 */
	post_schedule(rq);
3046

3047 3048 3049 3050
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
3051
	if (current->set_child_tid)
3052
		put_user(task_pid_vnr(current), current->set_child_tid);
L
Linus Torvalds 已提交
3053 3054 3055 3056 3057 3058
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
3059
static inline void
3060
context_switch(struct rq *rq, struct task_struct *prev,
3061
	       struct task_struct *next)
L
Linus Torvalds 已提交
3062
{
I
Ingo Molnar 已提交
3063
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
3064

3065
	prepare_task_switch(rq, prev, next);
3066

I
Ingo Molnar 已提交
3067 3068
	mm = next->mm;
	oldmm = prev->active_mm;
3069 3070 3071 3072 3073
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
3074
	arch_start_context_switch(prev);
3075

3076
	if (!mm) {
L
Linus Torvalds 已提交
3077 3078 3079 3080 3081 3082
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

3083
	if (!prev->mm) {
L
Linus Torvalds 已提交
3084 3085 3086
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
3087 3088 3089 3090 3091 3092 3093
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
3094
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
3095
#endif
L
Linus Torvalds 已提交
3096 3097 3098 3099

	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
3100 3101 3102 3103 3104 3105 3106
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
}

/*
 * nr_running, nr_uninterruptible and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, current number of uninterruptible-sleeping threads, total
 * number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
3124
}
L
Linus Torvalds 已提交
3125 3126

unsigned long nr_uninterruptible(void)
3127
{
L
Linus Torvalds 已提交
3128
	unsigned long i, sum = 0;
3129

3130
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
3131
		sum += cpu_rq(i)->nr_uninterruptible;
3132 3133

	/*
L
Linus Torvalds 已提交
3134 3135
	 * Since we read the counters lockless, it might be slightly
	 * inaccurate. Do not allow it to go below zero though:
3136
	 */
L
Linus Torvalds 已提交
3137 3138
	if (unlikely((long)sum < 0))
		sum = 0;
3139

L
Linus Torvalds 已提交
3140
	return sum;
3141 3142
}

L
Linus Torvalds 已提交
3143
unsigned long long nr_context_switches(void)
3144
{
3145 3146
	int i;
	unsigned long long sum = 0;
3147

3148
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
3149
		sum += cpu_rq(i)->nr_switches;
3150

L
Linus Torvalds 已提交
3151 3152
	return sum;
}
3153

L
Linus Torvalds 已提交
3154 3155 3156
unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;
3157

3158
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
3159
		sum += atomic_read(&cpu_rq(i)->nr_iowait);
3160

L
Linus Torvalds 已提交
3161 3162
	return sum;
}
3163

3164
unsigned long nr_iowait_cpu(int cpu)
3165
{
3166
	struct rq *this = cpu_rq(cpu);
3167 3168
	return atomic_read(&this->nr_iowait);
}
3169

3170 3171 3172 3173 3174
unsigned long this_cpu_load(void)
{
	struct rq *this = this_rq();
	return this->cpu_load[0];
}
3175

3176

3177 3178 3179 3180 3181
/* Variables and functions for calc_load */
static atomic_long_t calc_load_tasks;
static unsigned long calc_load_update;
unsigned long avenrun[3];
EXPORT_SYMBOL(avenrun);
3182

3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197
static long calc_load_fold_active(struct rq *this_rq)
{
	long nr_active, delta = 0;

	nr_active = this_rq->nr_running;
	nr_active += (long) this_rq->nr_uninterruptible;

	if (nr_active != this_rq->calc_load_active) {
		delta = nr_active - this_rq->calc_load_active;
		this_rq->calc_load_active = nr_active;
	}

	return delta;
}

3198 3199 3200 3201 3202 3203 3204 3205 3206
static unsigned long
calc_load(unsigned long load, unsigned long exp, unsigned long active)
{
	load *= exp;
	load += active * (FIXED_1 - exp);
	load += 1UL << (FSHIFT - 1);
	return load >> FSHIFT;
}

3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
#ifdef CONFIG_NO_HZ
/*
 * For NO_HZ we delay the active fold to the next LOAD_FREQ update.
 *
 * When making the ILB scale, we should try to pull this in as well.
 */
static atomic_long_t calc_load_tasks_idle;

static void calc_load_account_idle(struct rq *this_rq)
{
	long delta;

	delta = calc_load_fold_active(this_rq);
	if (delta)
		atomic_long_add(delta, &calc_load_tasks_idle);
}

static long calc_load_fold_idle(void)
{
	long delta = 0;

	/*
	 * Its got a race, we don't care...
	 */
	if (atomic_long_read(&calc_load_tasks_idle))
		delta = atomic_long_xchg(&calc_load_tasks_idle, 0);

	return delta;
}
3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357

/**
 * fixed_power_int - compute: x^n, in O(log n) time
 *
 * @x:         base of the power
 * @frac_bits: fractional bits of @x
 * @n:         power to raise @x to.
 *
 * By exploiting the relation between the definition of the natural power
 * function: x^n := x*x*...*x (x multiplied by itself for n times), and
 * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i,
 * (where: n_i \elem {0, 1}, the binary vector representing n),
 * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is
 * of course trivially computable in O(log_2 n), the length of our binary
 * vector.
 */
static unsigned long
fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n)
{
	unsigned long result = 1UL << frac_bits;

	if (n) for (;;) {
		if (n & 1) {
			result *= x;
			result += 1UL << (frac_bits - 1);
			result >>= frac_bits;
		}
		n >>= 1;
		if (!n)
			break;
		x *= x;
		x += 1UL << (frac_bits - 1);
		x >>= frac_bits;
	}

	return result;
}

/*
 * a1 = a0 * e + a * (1 - e)
 *
 * a2 = a1 * e + a * (1 - e)
 *    = (a0 * e + a * (1 - e)) * e + a * (1 - e)
 *    = a0 * e^2 + a * (1 - e) * (1 + e)
 *
 * a3 = a2 * e + a * (1 - e)
 *    = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e)
 *    = a0 * e^3 + a * (1 - e) * (1 + e + e^2)
 *
 *  ...
 *
 * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1]
 *    = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e)
 *    = a0 * e^n + a * (1 - e^n)
 *
 * [1] application of the geometric series:
 *
 *              n         1 - x^(n+1)
 *     S_n := \Sum x^i = -------------
 *             i=0          1 - x
 */
static unsigned long
calc_load_n(unsigned long load, unsigned long exp,
	    unsigned long active, unsigned int n)
{

	return calc_load(load, fixed_power_int(exp, FSHIFT, n), active);
}

/*
 * NO_HZ can leave us missing all per-cpu ticks calling
 * calc_load_account_active(), but since an idle CPU folds its delta into
 * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold
 * in the pending idle delta if our idle period crossed a load cycle boundary.
 *
 * Once we've updated the global active value, we need to apply the exponential
 * weights adjusted to the number of cycles missed.
 */
static void calc_global_nohz(unsigned long ticks)
{
	long delta, active, n;

	if (time_before(jiffies, calc_load_update))
		return;

	/*
	 * If we crossed a calc_load_update boundary, make sure to fold
	 * any pending idle changes, the respective CPUs might have
	 * missed the tick driven calc_load_account_active() update
	 * due to NO_HZ.
	 */
	delta = calc_load_fold_idle();
	if (delta)
		atomic_long_add(delta, &calc_load_tasks);

	/*
	 * If we were idle for multiple load cycles, apply them.
	 */
	if (ticks >= LOAD_FREQ) {
		n = ticks / LOAD_FREQ;

		active = atomic_long_read(&calc_load_tasks);
		active = active > 0 ? active * FIXED_1 : 0;

		avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n);
		avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n);
		avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n);

		calc_load_update += n * LOAD_FREQ;
	}

	/*
	 * Its possible the remainder of the above division also crosses
	 * a LOAD_FREQ period, the regular check in calc_global_load()
	 * which comes after this will take care of that.
	 *
	 * Consider us being 11 ticks before a cycle completion, and us
	 * sleeping for 4*LOAD_FREQ + 22 ticks, then the above code will
	 * age us 4 cycles, and the test in calc_global_load() will
	 * pick up the final one.
	 */
}
3358 3359 3360 3361 3362 3363 3364 3365 3366
#else
static void calc_load_account_idle(struct rq *this_rq)
{
}

static inline long calc_load_fold_idle(void)
{
	return 0;
}
3367 3368 3369 3370

static void calc_global_nohz(unsigned long ticks)
{
}
3371 3372
#endif

3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385
/**
 * get_avenrun - get the load average array
 * @loads:	pointer to dest load array
 * @offset:	offset to add
 * @shift:	shift count to shift the result left
 *
 * These values are estimates at best, so no need for locking.
 */
void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
{
	loads[0] = (avenrun[0] + offset) << shift;
	loads[1] = (avenrun[1] + offset) << shift;
	loads[2] = (avenrun[2] + offset) << shift;
3386 3387 3388
}

/*
3389 3390
 * calc_load - update the avenrun load estimates 10 ticks after the
 * CPUs have updated calc_load_tasks.
3391
 */
3392
void calc_global_load(unsigned long ticks)
3393
{
3394
	long active;
L
Linus Torvalds 已提交
3395

3396 3397 3398
	calc_global_nohz(ticks);

	if (time_before(jiffies, calc_load_update + 10))
3399
		return;
L
Linus Torvalds 已提交
3400

3401 3402
	active = atomic_long_read(&calc_load_tasks);
	active = active > 0 ? active * FIXED_1 : 0;
L
Linus Torvalds 已提交
3403

3404 3405 3406
	avenrun[0] = calc_load(avenrun[0], EXP_1, active);
	avenrun[1] = calc_load(avenrun[1], EXP_5, active);
	avenrun[2] = calc_load(avenrun[2], EXP_15, active);
I
Ingo Molnar 已提交
3407

3408 3409
	calc_load_update += LOAD_FREQ;
}
L
Linus Torvalds 已提交
3410

3411
/*
3412 3413
 * Called from update_cpu_load() to periodically update this CPU's
 * active count.
3414 3415 3416
 */
static void calc_load_account_active(struct rq *this_rq)
{
3417
	long delta;
3418

3419 3420
	if (time_before(jiffies, this_rq->calc_load_update))
		return;
3421

3422 3423 3424
	delta  = calc_load_fold_active(this_rq);
	delta += calc_load_fold_idle();
	if (delta)
3425
		atomic_long_add(delta, &calc_load_tasks);
3426 3427

	this_rq->calc_load_update += LOAD_FREQ;
3428 3429
}

3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496
/*
 * The exact cpuload at various idx values, calculated at every tick would be
 * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load
 *
 * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called
 * on nth tick when cpu may be busy, then we have:
 * load = ((2^idx - 1) / 2^idx)^(n-1) * load
 * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load
 *
 * decay_load_missed() below does efficient calculation of
 * load = ((2^idx - 1) / 2^idx)^(n-1) * load
 * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load
 *
 * The calculation is approximated on a 128 point scale.
 * degrade_zero_ticks is the number of ticks after which load at any
 * particular idx is approximated to be zero.
 * degrade_factor is a precomputed table, a row for each load idx.
 * Each column corresponds to degradation factor for a power of two ticks,
 * based on 128 point scale.
 * Example:
 * row 2, col 3 (=12) says that the degradation at load idx 2 after
 * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8).
 *
 * With this power of 2 load factors, we can degrade the load n times
 * by looking at 1 bits in n and doing as many mult/shift instead of
 * n mult/shifts needed by the exact degradation.
 */
#define DEGRADE_SHIFT		7
static const unsigned char
		degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
static const unsigned char
		degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
					{0, 0, 0, 0, 0, 0, 0, 0},
					{64, 32, 8, 0, 0, 0, 0, 0},
					{96, 72, 40, 12, 1, 0, 0},
					{112, 98, 75, 43, 15, 1, 0},
					{120, 112, 98, 76, 45, 16, 2} };

/*
 * Update cpu_load for any missed ticks, due to tickless idle. The backlog
 * would be when CPU is idle and so we just decay the old load without
 * adding any new load.
 */
static unsigned long
decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
{
	int j = 0;

	if (!missed_updates)
		return load;

	if (missed_updates >= degrade_zero_ticks[idx])
		return 0;

	if (idx == 1)
		return load >> missed_updates;

	while (missed_updates) {
		if (missed_updates % 2)
			load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;

		missed_updates >>= 1;
		j++;
	}
	return load;
}

3497
/*
I
Ingo Molnar 已提交
3498
 * Update rq->cpu_load[] statistics. This function is usually called every
3499 3500
 * scheduler tick (TICK_NSEC). With tickless idle this will not be called
 * every tick. We fix it up based on jiffies.
3501
 */
I
Ingo Molnar 已提交
3502
static void update_cpu_load(struct rq *this_rq)
3503
{
3504
	unsigned long this_load = this_rq->load.weight;
3505 3506
	unsigned long curr_jiffies = jiffies;
	unsigned long pending_updates;
I
Ingo Molnar 已提交
3507
	int i, scale;
3508

I
Ingo Molnar 已提交
3509
	this_rq->nr_load_updates++;
3510

3511 3512 3513 3514 3515 3516 3517
	/* Avoid repeated calls on same jiffy, when moving in and out of idle */
	if (curr_jiffies == this_rq->last_load_update_tick)
		return;

	pending_updates = curr_jiffies - this_rq->last_load_update_tick;
	this_rq->last_load_update_tick = curr_jiffies;

I
Ingo Molnar 已提交
3518
	/* Update our load: */
3519 3520
	this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
	for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
I
Ingo Molnar 已提交
3521
		unsigned long old_load, new_load;
3522

I
Ingo Molnar 已提交
3523
		/* scale is effectively 1 << i now, and >> i divides by scale */
3524

I
Ingo Molnar 已提交
3525
		old_load = this_rq->cpu_load[i];
3526
		old_load = decay_load_missed(old_load, pending_updates - 1, i);
I
Ingo Molnar 已提交
3527
		new_load = this_load;
I
Ingo Molnar 已提交
3528 3529 3530 3531 3532 3533
		/*
		 * Round up the averaging division if load is increasing. This
		 * prevents us from getting stuck on 9 if the load is 10, for
		 * example.
		 */
		if (new_load > old_load)
3534 3535 3536
			new_load += scale - 1;

		this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
I
Ingo Molnar 已提交
3537
	}
3538 3539

	sched_avg_update(this_rq);
3540 3541 3542 3543 3544
}

static void update_cpu_load_active(struct rq *this_rq)
{
	update_cpu_load(this_rq);
3545

3546
	calc_load_account_active(this_rq);
3547 3548
}

I
Ingo Molnar 已提交
3549
#ifdef CONFIG_SMP
3550

3551
/*
P
Peter Zijlstra 已提交
3552 3553
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
3554
 */
P
Peter Zijlstra 已提交
3555
void sched_exec(void)
3556
{
P
Peter Zijlstra 已提交
3557
	struct task_struct *p = current;
L
Linus Torvalds 已提交
3558
	unsigned long flags;
3559
	int dest_cpu;
3560

3561
	raw_spin_lock_irqsave(&p->pi_lock, flags);
3562
	dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0);
3563 3564
	if (dest_cpu == smp_processor_id())
		goto unlock;
P
Peter Zijlstra 已提交
3565

3566
	if (likely(cpu_active(dest_cpu))) {
3567
		struct migration_arg arg = { p, dest_cpu };
3568

3569 3570
		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
3571 3572
		return;
	}
3573
unlock:
3574
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
3575
}
I
Ingo Molnar 已提交
3576

L
Linus Torvalds 已提交
3577 3578 3579 3580 3581 3582 3583
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
3584
 * Return any ns on the sched_clock that have not yet been accounted in
3585
 * @p in case that task is currently running.
3586 3587
 *
 * Called with task_rq_lock() held on @rq.
L
Linus Torvalds 已提交
3588
 */
3589 3590 3591 3592 3593 3594
static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
{
	u64 ns = 0;

	if (task_current(rq, p)) {
		update_rq_clock(rq);
3595
		ns = rq->clock_task - p->se.exec_start;
3596 3597 3598 3599 3600 3601 3602
		if ((s64)ns < 0)
			ns = 0;
	}

	return ns;
}

3603
unsigned long long task_delta_exec(struct task_struct *p)
L
Linus Torvalds 已提交
3604 3605
{
	unsigned long flags;
3606
	struct rq *rq;
3607
	u64 ns = 0;
3608

3609
	rq = task_rq_lock(p, &flags);
3610
	ns = do_task_delta_exec(p, rq);
3611
	task_rq_unlock(rq, p, &flags);
3612

3613 3614
	return ns;
}
3615

3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628
/*
 * Return accounted runtime for the task.
 * In case the task is currently running, return the runtime plus current's
 * pending runtime that have not been accounted yet.
 */
unsigned long long task_sched_runtime(struct task_struct *p)
{
	unsigned long flags;
	struct rq *rq;
	u64 ns = 0;

	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
3629
	task_rq_unlock(rq, p, &flags);
3630 3631 3632

	return ns;
}
3633

3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652
/*
 * Return sum_exec_runtime for the thread group.
 * In case the task is currently running, return the sum plus current's
 * pending runtime that have not been accounted yet.
 *
 * Note that the thread group might have other running tasks as well,
 * so the return value not includes other pending runtime that other
 * running tasks might have.
 */
unsigned long long thread_group_sched_runtime(struct task_struct *p)
{
	struct task_cputime totals;
	unsigned long flags;
	struct rq *rq;
	u64 ns;

	rq = task_rq_lock(p, &flags);
	thread_group_cputime(p, &totals);
	ns = totals.sum_exec_runtime + do_task_delta_exec(p, rq);
3653
	task_rq_unlock(rq, p, &flags);
3654

L
Linus Torvalds 已提交
3655 3656 3657 3658 3659 3660 3661
	return ns;
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in user space since the last update
3662
 * @cputime_scaled: cputime scaled by cpu frequency
L
Linus Torvalds 已提交
3663
 */
3664 3665
void account_user_time(struct task_struct *p, cputime_t cputime,
		       cputime_t cputime_scaled)
L
Linus Torvalds 已提交
3666 3667 3668 3669
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

3670
	/* Add user time to process. */
L
Linus Torvalds 已提交
3671
	p->utime = cputime_add(p->utime, cputime);
3672
	p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
3673
	account_group_user_time(p, cputime);
L
Linus Torvalds 已提交
3674 3675 3676 3677 3678 3679 3680

	/* Add user time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (TASK_NICE(p) > 0)
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
	else
		cpustat->user = cputime64_add(cpustat->user, tmp);
3681 3682

	cpuacct_update_stats(p, CPUACCT_STAT_USER, cputime);
3683 3684
	/* Account for user time used */
	acct_update_integrals(p);
L
Linus Torvalds 已提交
3685 3686
}

3687 3688 3689 3690
/*
 * Account guest cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in virtual machine since the last update
3691
 * @cputime_scaled: cputime scaled by cpu frequency
3692
 */
3693 3694
static void account_guest_time(struct task_struct *p, cputime_t cputime,
			       cputime_t cputime_scaled)
3695 3696 3697 3698 3699 3700
{
	cputime64_t tmp;
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;

	tmp = cputime_to_cputime64(cputime);

3701
	/* Add guest time to process. */
3702
	p->utime = cputime_add(p->utime, cputime);
3703
	p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
3704
	account_group_user_time(p, cputime);
3705 3706
	p->gtime = cputime_add(p->gtime, cputime);

3707
	/* Add guest time to cpustat. */
3708 3709 3710 3711 3712 3713 3714
	if (TASK_NICE(p) > 0) {
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
		cpustat->guest_nice = cputime64_add(cpustat->guest_nice, tmp);
	} else {
		cpustat->user = cputime64_add(cpustat->user, tmp);
		cpustat->guest = cputime64_add(cpustat->guest, tmp);
	}
3715 3716
}

3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742
/*
 * Account system cpu time to a process and desired cpustat field
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in kernel space since the last update
 * @cputime_scaled: cputime scaled by cpu frequency
 * @target_cputime64: pointer to cpustat field that has to be updated
 */
static inline
void __account_system_time(struct task_struct *p, cputime_t cputime,
			cputime_t cputime_scaled, cputime64_t *target_cputime64)
{
	cputime64_t tmp = cputime_to_cputime64(cputime);

	/* Add system time to process. */
	p->stime = cputime_add(p->stime, cputime);
	p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
	account_group_system_time(p, cputime);

	/* Add system time to cpustat. */
	*target_cputime64 = cputime64_add(*target_cputime64, tmp);
	cpuacct_update_stats(p, CPUACCT_STAT_SYSTEM, cputime);

	/* Account for system time used */
	acct_update_integrals(p);
}

L
Linus Torvalds 已提交
3743 3744 3745 3746 3747
/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
3748
 * @cputime_scaled: cputime scaled by cpu frequency
L
Linus Torvalds 已提交
3749 3750
 */
void account_system_time(struct task_struct *p, int hardirq_offset,
3751
			 cputime_t cputime, cputime_t cputime_scaled)
L
Linus Torvalds 已提交
3752 3753
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3754
	cputime64_t *target_cputime64;
L
Linus Torvalds 已提交
3755

3756
	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
3757
		account_guest_time(p, cputime, cputime_scaled);
3758 3759
		return;
	}
3760

L
Linus Torvalds 已提交
3761
	if (hardirq_count() - hardirq_offset)
3762
		target_cputime64 = &cpustat->irq;
3763
	else if (in_serving_softirq())
3764
		target_cputime64 = &cpustat->softirq;
L
Linus Torvalds 已提交
3765
	else
3766
		target_cputime64 = &cpustat->system;
3767

3768
	__account_system_time(p, cputime, cputime_scaled, target_cputime64);
L
Linus Torvalds 已提交
3769 3770
}

3771
/*
L
Linus Torvalds 已提交
3772
 * Account for involuntary wait time.
3773
 * @cputime: the cpu time spent in involuntary wait
3774
 */
3775
void account_steal_time(cputime_t cputime)
3776
{
3777 3778 3779 3780
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t cputime64 = cputime_to_cputime64(cputime);

	cpustat->steal = cputime64_add(cpustat->steal, cputime64);
3781 3782
}

L
Linus Torvalds 已提交
3783
/*
3784 3785
 * Account for idle time.
 * @cputime: the cpu time spent in idle wait
L
Linus Torvalds 已提交
3786
 */
3787
void account_idle_time(cputime_t cputime)
L
Linus Torvalds 已提交
3788 3789
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3790
	cputime64_t cputime64 = cputime_to_cputime64(cputime);
3791
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3792

3793 3794 3795 3796
	if (atomic_read(&rq->nr_iowait) > 0)
		cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
	else
		cpustat->idle = cputime64_add(cpustat->idle, cputime64);
L
Linus Torvalds 已提交
3797 3798
}

3799 3800
#ifndef CONFIG_VIRT_CPU_ACCOUNTING

3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
/*
 * Account a tick to a process and cpustat
 * @p: the process that the cpu time gets accounted to
 * @user_tick: is the tick from userspace
 * @rq: the pointer to rq
 *
 * Tick demultiplexing follows the order
 * - pending hardirq update
 * - pending softirq update
 * - user_time
 * - idle_time
 * - system time
 *   - check for guest_time
 *   - else account as system_time
 *
 * Check for hardirq is done both for system and user time as there is
 * no timer going off while we are on hardirq and hence we may never get an
 * opportunity to update it solely in system time.
 * p->stime and friends are only updated on system time and not on irq
 * softirq as those do not count in task exec_runtime any more.
 */
static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
						struct rq *rq)
{
	cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
	cputime64_t tmp = cputime_to_cputime64(cputime_one_jiffy);
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;

	if (irqtime_account_hi_update()) {
		cpustat->irq = cputime64_add(cpustat->irq, tmp);
	} else if (irqtime_account_si_update()) {
		cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
3834 3835 3836 3837 3838 3839 3840 3841
	} else if (this_cpu_ksoftirqd() == p) {
		/*
		 * ksoftirqd time do not get accounted in cpu_softirq_time.
		 * So, we have to handle it separately here.
		 * Also, p->stime needs to be updated for ksoftirqd.
		 */
		__account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
					&cpustat->softirq);
3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861
	} else if (user_tick) {
		account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
	} else if (p == rq->idle) {
		account_idle_time(cputime_one_jiffy);
	} else if (p->flags & PF_VCPU) { /* System time or guest time */
		account_guest_time(p, cputime_one_jiffy, one_jiffy_scaled);
	} else {
		__account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
					&cpustat->system);
	}
}

static void irqtime_account_idle_ticks(int ticks)
{
	int i;
	struct rq *rq = this_rq();

	for (i = 0; i < ticks; i++)
		irqtime_account_process_tick(current, 0, rq);
}
3862
#else /* CONFIG_IRQ_TIME_ACCOUNTING */
3863 3864 3865
static void irqtime_account_idle_ticks(int ticks) {}
static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
						struct rq *rq) {}
3866
#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
3867 3868 3869 3870 3871 3872 3873 3874

/*
 * Account a single tick of cpu time.
 * @p: the process that the cpu time gets accounted to
 * @user_tick: indicates if the tick is a user or a system tick
 */
void account_process_tick(struct task_struct *p, int user_tick)
{
3875
	cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
3876 3877
	struct rq *rq = this_rq();

3878 3879 3880 3881 3882
	if (sched_clock_irqtime) {
		irqtime_account_process_tick(p, user_tick, rq);
		return;
	}

3883
	if (user_tick)
3884
		account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
3885
	else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
3886
		account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
3887 3888
				    one_jiffy_scaled);
	else
3889
		account_idle_time(cputime_one_jiffy);
3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
}

/*
 * Account multiple ticks of steal time.
 * @p: the process from which the cpu time has been stolen
 * @ticks: number of stolen ticks
 */
void account_steal_ticks(unsigned long ticks)
{
	account_steal_time(jiffies_to_cputime(ticks));
}

/*
 * Account multiple ticks of idle time.
 * @ticks: number of stolen ticks
 */
void account_idle_ticks(unsigned long ticks)
{
3908 3909 3910 3911 3912 3913

	if (sched_clock_irqtime) {
		irqtime_account_idle_ticks(ticks);
		return;
	}

3914
	account_idle_time(jiffies_to_cputime(ticks));
L
Linus Torvalds 已提交
3915 3916
}

3917 3918
#endif

3919 3920 3921 3922
/*
 * Use precise platform statistics if available:
 */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
3923
void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
3924
{
3925 3926
	*ut = p->utime;
	*st = p->stime;
3927 3928
}

3929
void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
3930
{
3931 3932 3933 3934 3935 3936
	struct task_cputime cputime;

	thread_group_cputime(p, &cputime);

	*ut = cputime.utime;
	*st = cputime.stime;
3937 3938
}
#else
3939 3940

#ifndef nsecs_to_cputime
3941
# define nsecs_to_cputime(__nsecs)	nsecs_to_jiffies(__nsecs)
3942 3943
#endif

3944
void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
3945
{
3946
	cputime_t rtime, utime = p->utime, total = cputime_add(utime, p->stime);
3947 3948 3949 3950

	/*
	 * Use CFS's precise accounting:
	 */
3951
	rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
3952 3953

	if (total) {
3954
		u64 temp = rtime;
3955

3956
		temp *= utime;
3957
		do_div(temp, total);
3958 3959 3960
		utime = (cputime_t)temp;
	} else
		utime = rtime;
3961

3962 3963 3964
	/*
	 * Compare with previous values, to keep monotonicity:
	 */
3965
	p->prev_utime = max(p->prev_utime, utime);
3966
	p->prev_stime = max(p->prev_stime, cputime_sub(rtime, p->prev_utime));
3967

3968 3969
	*ut = p->prev_utime;
	*st = p->prev_stime;
3970 3971
}

3972 3973 3974 3975
/*
 * Must be called with siglock held.
 */
void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
3976
{
3977 3978 3979
	struct signal_struct *sig = p->signal;
	struct task_cputime cputime;
	cputime_t rtime, utime, total;
3980

3981
	thread_group_cputime(p, &cputime);
3982

3983 3984
	total = cputime_add(cputime.utime, cputime.stime);
	rtime = nsecs_to_cputime(cputime.sum_exec_runtime);
3985

3986
	if (total) {
3987
		u64 temp = rtime;
3988

3989
		temp *= cputime.utime;
3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000
		do_div(temp, total);
		utime = (cputime_t)temp;
	} else
		utime = rtime;

	sig->prev_utime = max(sig->prev_utime, utime);
	sig->prev_stime = max(sig->prev_stime,
			      cputime_sub(rtime, sig->prev_utime));

	*ut = sig->prev_utime;
	*st = sig->prev_stime;
4001 4002 4003
}
#endif

4004 4005 4006 4007 4008 4009 4010 4011
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
4012
	struct task_struct *curr = rq->curr;
4013 4014

	sched_clock_tick();
I
Ingo Molnar 已提交
4015

4016
	raw_spin_lock(&rq->lock);
4017
	update_rq_clock(rq);
4018
	update_cpu_load_active(rq);
P
Peter Zijlstra 已提交
4019
	curr->sched_class->task_tick(rq, curr, 0);
4020
	raw_spin_unlock(&rq->lock);
4021

4022
	perf_event_task_tick();
4023

4024
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
4025 4026
	rq->idle_at_tick = idle_cpu(cpu);
	trigger_load_balance(rq, cpu);
4027
#endif
L
Linus Torvalds 已提交
4028 4029
}

4030
notrace unsigned long get_parent_ip(unsigned long addr)
4031 4032 4033 4034 4035 4036 4037 4038
{
	if (in_lock_functions(addr)) {
		addr = CALLER_ADDR2;
		if (in_lock_functions(addr))
			addr = CALLER_ADDR3;
	}
	return addr;
}
L
Linus Torvalds 已提交
4039

4040 4041 4042
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
				defined(CONFIG_PREEMPT_TRACER))

4043
void __kprobes add_preempt_count(int val)
L
Linus Torvalds 已提交
4044
{
4045
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4046 4047 4048
	/*
	 * Underflow?
	 */
4049 4050
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
4051
#endif
L
Linus Torvalds 已提交
4052
	preempt_count() += val;
4053
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4054 4055 4056
	/*
	 * Spinlock count overflowing soon?
	 */
4057 4058
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
4059 4060 4061
#endif
	if (preempt_count() == val)
		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
4062 4063 4064
}
EXPORT_SYMBOL(add_preempt_count);

4065
void __kprobes sub_preempt_count(int val)
L
Linus Torvalds 已提交
4066
{
4067
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4068 4069 4070
	/*
	 * Underflow?
	 */
4071
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
4072
		return;
L
Linus Torvalds 已提交
4073 4074 4075
	/*
	 * Is the spinlock portion underflowing?
	 */
4076 4077 4078
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;
4079
#endif
4080

4081 4082
	if (preempt_count() == val)
		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
4083 4084 4085 4086 4087 4088 4089
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
4090
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
4091
 */
I
Ingo Molnar 已提交
4092
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
4093
{
4094 4095
	struct pt_regs *regs = get_irq_regs();

P
Peter Zijlstra 已提交
4096 4097
	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
		prev->comm, prev->pid, preempt_count());
4098

I
Ingo Molnar 已提交
4099
	debug_show_held_locks(prev);
4100
	print_modules();
I
Ingo Molnar 已提交
4101 4102
	if (irqs_disabled())
		print_irqtrace_events(prev);
4103 4104 4105 4106 4107

	if (regs)
		show_regs(regs);
	else
		dump_stack();
I
Ingo Molnar 已提交
4108
}
L
Linus Torvalds 已提交
4109

I
Ingo Molnar 已提交
4110 4111 4112 4113 4114
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
4115
	/*
I
Ingo Molnar 已提交
4116
	 * Test if we are atomic. Since do_exit() needs to call into
L
Linus Torvalds 已提交
4117 4118 4119
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
4120
	if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
I
Ingo Molnar 已提交
4121 4122
		__schedule_bug(prev);

L
Linus Torvalds 已提交
4123 4124
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

4125
	schedstat_inc(this_rq(), sched_count);
I
Ingo Molnar 已提交
4126 4127
}

P
Peter Zijlstra 已提交
4128
static void put_prev_task(struct rq *rq, struct task_struct *prev)
M
Mike Galbraith 已提交
4129
{
4130
	if (prev->on_rq || rq->skip_clock_update < 0)
4131
		update_rq_clock(rq);
P
Peter Zijlstra 已提交
4132
	prev->sched_class->put_prev_task(rq, prev);
M
Mike Galbraith 已提交
4133 4134
}

I
Ingo Molnar 已提交
4135 4136 4137 4138
/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
4139
pick_next_task(struct rq *rq)
I
Ingo Molnar 已提交
4140
{
4141
	const struct sched_class *class;
I
Ingo Molnar 已提交
4142
	struct task_struct *p;
L
Linus Torvalds 已提交
4143 4144

	/*
I
Ingo Molnar 已提交
4145 4146
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
4147
	 */
I
Ingo Molnar 已提交
4148
	if (likely(rq->nr_running == rq->cfs.nr_running)) {
4149
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
4150 4151
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
4152 4153
	}

4154
	for_each_class(class) {
4155
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
4156 4157 4158
		if (p)
			return p;
	}
4159 4160

	BUG(); /* the idle class will always have a runnable task */
I
Ingo Molnar 已提交
4161
}
L
Linus Torvalds 已提交
4162

I
Ingo Molnar 已提交
4163 4164 4165
/*
 * schedule() is the main scheduler function.
 */
4166
asmlinkage void __sched schedule(void)
I
Ingo Molnar 已提交
4167 4168
{
	struct task_struct *prev, *next;
4169
	unsigned long *switch_count;
I
Ingo Molnar 已提交
4170
	struct rq *rq;
4171
	int cpu;
I
Ingo Molnar 已提交
4172

4173 4174
need_resched:
	preempt_disable();
I
Ingo Molnar 已提交
4175 4176
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
4177
	rcu_note_context_switch(cpu);
I
Ingo Molnar 已提交
4178 4179 4180
	prev = rq->curr;

	schedule_debug(prev);
L
Linus Torvalds 已提交
4181

4182
	if (sched_feat(HRTICK))
M
Mike Galbraith 已提交
4183
		hrtick_clear(rq);
P
Peter Zijlstra 已提交
4184

4185
	raw_spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
4186

4187
	switch_count = &prev->nivcsw;
L
Linus Torvalds 已提交
4188
	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
T
Tejun Heo 已提交
4189
		if (unlikely(signal_pending_state(prev->state, prev))) {
L
Linus Torvalds 已提交
4190
			prev->state = TASK_RUNNING;
T
Tejun Heo 已提交
4191
		} else {
4192 4193 4194
			deactivate_task(rq, prev, DEQUEUE_SLEEP);
			prev->on_rq = 0;

T
Tejun Heo 已提交
4195
			/*
4196 4197 4198
			 * If a worker went to sleep, notify and ask workqueue
			 * whether it wants to wake up a task to maintain
			 * concurrency.
T
Tejun Heo 已提交
4199 4200 4201 4202 4203 4204 4205 4206
			 */
			if (prev->flags & PF_WQ_WORKER) {
				struct task_struct *to_wakeup;

				to_wakeup = wq_worker_sleeping(prev, cpu);
				if (to_wakeup)
					try_to_wake_up_local(to_wakeup);
			}
P
Peter Zijlstra 已提交
4207

4208
			/*
4209 4210
			 * If we are going to sleep and we have plugged IO
			 * queued, make sure to submit it to avoid deadlocks.
4211 4212 4213
			 */
			if (blk_needs_flush_plug(prev)) {
				raw_spin_unlock(&rq->lock);
4214
				blk_schedule_flush_plug(prev);
4215 4216
				raw_spin_lock(&rq->lock);
			}
T
Tejun Heo 已提交
4217
		}
I
Ingo Molnar 已提交
4218
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
4219 4220
	}

4221
	pre_schedule(rq, prev);
4222

I
Ingo Molnar 已提交
4223
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
4224 4225
		idle_balance(cpu, rq);

M
Mike Galbraith 已提交
4226
	put_prev_task(rq, prev);
4227
	next = pick_next_task(rq);
4228 4229
	clear_tsk_need_resched(prev);
	rq->skip_clock_update = 0;
L
Linus Torvalds 已提交
4230 4231 4232 4233 4234 4235

	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
4236
		context_switch(rq, prev, next); /* unlocks the rq */
P
Peter Zijlstra 已提交
4237
		/*
4238 4239 4240 4241
		 * The context switch have flipped the stack from under us
		 * and restored the local variables which were saved when
		 * this task called schedule() in the past. prev == current
		 * is still correct, but it can be moved to another cpu/rq.
P
Peter Zijlstra 已提交
4242 4243 4244
		 */
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4245
	} else
4246
		raw_spin_unlock_irq(&rq->lock);
L
Linus Torvalds 已提交
4247

4248
	post_schedule(rq);
L
Linus Torvalds 已提交
4249 4250

	preempt_enable_no_resched();
4251
	if (need_resched())
L
Linus Torvalds 已提交
4252 4253 4254 4255
		goto need_resched;
}
EXPORT_SYMBOL(schedule);

4256
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
4257

4258 4259 4260
static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
{
	bool ret = false;
4261

4262 4263 4264
	rcu_read_lock();
	if (lock->owner != owner)
		goto fail;
4265 4266

	/*
4267 4268 4269 4270
	 * Ensure we emit the owner->on_cpu, dereference _after_ checking
	 * lock->owner still matches owner, if that fails, owner might
	 * point to free()d memory, if it still matches, the rcu_read_lock()
	 * ensures the memory stays valid.
4271
	 */
4272
	barrier();
4273

4274 4275 4276
	ret = owner->on_cpu;
fail:
	rcu_read_unlock();
4277

4278 4279
	return ret;
}
4280

4281 4282 4283 4284 4285 4286 4287 4288
/*
 * Look out! "owner" is an entirely speculative pointer
 * access and not reliable.
 */
int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
{
	if (!sched_feat(OWNER_SPIN))
		return 0;
4289

4290 4291
	while (owner_running(lock, owner)) {
		if (need_resched())
4292 4293
			return 0;

4294
		arch_mutex_cpu_relax();
4295
	}
4296

4297 4298 4299 4300 4301 4302 4303
	/*
	 * If the owner changed to another task there is likely
	 * heavy contention, stop spinning.
	 */
	if (lock->owner)
		return 0;

4304 4305 4306 4307
	return 1;
}
#endif

L
Linus Torvalds 已提交
4308 4309
#ifdef CONFIG_PREEMPT
/*
4310
 * this is the entry point to schedule() from in-kernel preemption
I
Ingo Molnar 已提交
4311
 * off of preempt_enable. Kernel preemptions off return from interrupt
L
Linus Torvalds 已提交
4312 4313
 * occur there and call schedule directly.
 */
4314
asmlinkage void __sched notrace preempt_schedule(void)
L
Linus Torvalds 已提交
4315 4316
{
	struct thread_info *ti = current_thread_info();
4317

L
Linus Torvalds 已提交
4318 4319
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
I
Ingo Molnar 已提交
4320
	 * we do not want to preempt the current task. Just return..
L
Linus Torvalds 已提交
4321
	 */
N
Nick Piggin 已提交
4322
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
4323 4324
		return;

4325
	do {
4326
		add_preempt_count_notrace(PREEMPT_ACTIVE);
4327
		schedule();
4328
		sub_preempt_count_notrace(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4329

4330 4331 4332 4333 4334
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
4335
	} while (need_resched());
L
Linus Torvalds 已提交
4336 4337 4338 4339
}
EXPORT_SYMBOL(preempt_schedule);

/*
4340
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
4341 4342 4343 4344 4345 4346 4347
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
4348

4349
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
4350 4351
	BUG_ON(ti->preempt_count || !irqs_disabled());

4352 4353 4354 4355 4356 4357
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		local_irq_enable();
		schedule();
		local_irq_disable();
		sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4358

4359 4360 4361 4362 4363
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
4364
	} while (need_resched());
L
Linus Torvalds 已提交
4365 4366 4367 4368
}

#endif /* CONFIG_PREEMPT */

P
Peter Zijlstra 已提交
4369
int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
I
Ingo Molnar 已提交
4370
			  void *key)
L
Linus Torvalds 已提交
4371
{
P
Peter Zijlstra 已提交
4372
	return try_to_wake_up(curr->private, mode, wake_flags);
L
Linus Torvalds 已提交
4373 4374 4375 4376
}
EXPORT_SYMBOL(default_wake_function);

/*
I
Ingo Molnar 已提交
4377 4378
 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
L
Linus Torvalds 已提交
4379 4380 4381
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
I
Ingo Molnar 已提交
4382
 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
L
Linus Torvalds 已提交
4383 4384
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
4385
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
P
Peter Zijlstra 已提交
4386
			int nr_exclusive, int wake_flags, void *key)
L
Linus Torvalds 已提交
4387
{
4388
	wait_queue_t *curr, *next;
L
Linus Torvalds 已提交
4389

4390
	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
4391 4392
		unsigned flags = curr->flags;

P
Peter Zijlstra 已提交
4393
		if (curr->func(curr, mode, wake_flags, key) &&
4394
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
4395 4396 4397 4398 4399 4400 4401 4402 4403
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
4404
 * @key: is directly passed to the wakeup function
4405 4406 4407
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
L
Linus Torvalds 已提交
4408
 */
4409
void __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
4410
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
4423
void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
L
Linus Torvalds 已提交
4424 4425 4426
{
	__wake_up_common(q, mode, 1, 0, NULL);
}
4427
EXPORT_SYMBOL_GPL(__wake_up_locked);
L
Linus Torvalds 已提交
4428

4429 4430 4431 4432
void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
{
	__wake_up_common(q, mode, 1, 0, key);
}
4433
EXPORT_SYMBOL_GPL(__wake_up_locked_key);
4434

L
Linus Torvalds 已提交
4435
/**
4436
 * __wake_up_sync_key - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
4437 4438 4439
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
4440
 * @key: opaque value to be passed to wakeup targets
L
Linus Torvalds 已提交
4441 4442 4443 4444 4445 4446 4447
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
4448 4449 4450
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
L
Linus Torvalds 已提交
4451
 */
4452 4453
void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
4454 4455
{
	unsigned long flags;
P
Peter Zijlstra 已提交
4456
	int wake_flags = WF_SYNC;
L
Linus Torvalds 已提交
4457 4458 4459 4460 4461

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
P
Peter Zijlstra 已提交
4462
		wake_flags = 0;
L
Linus Torvalds 已提交
4463 4464

	spin_lock_irqsave(&q->lock, flags);
P
Peter Zijlstra 已提交
4465
	__wake_up_common(q, mode, nr_exclusive, wake_flags, key);
L
Linus Torvalds 已提交
4466 4467
	spin_unlock_irqrestore(&q->lock, flags);
}
4468 4469 4470 4471 4472 4473 4474 4475 4476
EXPORT_SYMBOL_GPL(__wake_up_sync_key);

/*
 * __wake_up_sync - see __wake_up_sync_key()
 */
void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
{
	__wake_up_sync_key(q, mode, nr_exclusive, NULL);
}
L
Linus Torvalds 已提交
4477 4478
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

4479 4480 4481 4482 4483 4484 4485 4486
/**
 * complete: - signals a single thread waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up a single thread waiting on this completion. Threads will be
 * awakened in the same order in which they were queued.
 *
 * See also complete_all(), wait_for_completion() and related routines.
4487 4488 4489
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
4490
 */
4491
void complete(struct completion *x)
L
Linus Torvalds 已提交
4492 4493 4494 4495 4496
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
4497
	__wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
L
Linus Torvalds 已提交
4498 4499 4500 4501
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

4502 4503 4504 4505 4506
/**
 * complete_all: - signals all threads waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up all threads waiting on this particular completion event.
4507 4508 4509
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
4510
 */
4511
void complete_all(struct completion *x)
L
Linus Torvalds 已提交
4512 4513 4514 4515 4516
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
4517
	__wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
L
Linus Torvalds 已提交
4518 4519 4520 4521
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

4522 4523
static inline long __sched
do_wait_for_common(struct completion *x, long timeout, int state)
L
Linus Torvalds 已提交
4524 4525 4526 4527
{
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

C
Changli Gao 已提交
4528
		__add_wait_queue_tail_exclusive(&x->wait, &wait);
L
Linus Torvalds 已提交
4529
		do {
4530
			if (signal_pending_state(state, current)) {
4531 4532
				timeout = -ERESTARTSYS;
				break;
4533 4534
			}
			__set_current_state(state);
L
Linus Torvalds 已提交
4535 4536 4537
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
4538
		} while (!x->done && timeout);
L
Linus Torvalds 已提交
4539
		__remove_wait_queue(&x->wait, &wait);
4540 4541
		if (!x->done)
			return timeout;
L
Linus Torvalds 已提交
4542 4543
	}
	x->done--;
4544
	return timeout ?: 1;
L
Linus Torvalds 已提交
4545 4546
}

4547 4548
static long __sched
wait_for_common(struct completion *x, long timeout, int state)
L
Linus Torvalds 已提交
4549 4550 4551 4552
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
4553
	timeout = do_wait_for_common(x, timeout, state);
L
Linus Torvalds 已提交
4554
	spin_unlock_irq(&x->wait.lock);
4555 4556
	return timeout;
}
L
Linus Torvalds 已提交
4557

4558 4559 4560 4561 4562 4563 4564 4565 4566 4567
/**
 * wait_for_completion: - waits for completion of a task
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It is NOT
 * interruptible and there is no timeout.
 *
 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
 * and interrupt capability. Also see complete().
 */
4568
void __sched wait_for_completion(struct completion *x)
4569 4570
{
	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
4571
}
4572
EXPORT_SYMBOL(wait_for_completion);
L
Linus Torvalds 已提交
4573

4574 4575 4576 4577 4578 4579 4580 4581 4582
/**
 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. The timeout is in jiffies. It is not
 * interruptible.
 */
4583
unsigned long __sched
4584
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
L
Linus Torvalds 已提交
4585
{
4586
	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
4587
}
4588
EXPORT_SYMBOL(wait_for_completion_timeout);
L
Linus Torvalds 已提交
4589

4590 4591 4592 4593 4594 4595 4596
/**
 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
 * @x:  holds the state of this particular completion
 *
 * This waits for completion of a specific task to be signaled. It is
 * interruptible.
 */
4597
int __sched wait_for_completion_interruptible(struct completion *x)
I
Ingo Molnar 已提交
4598
{
4599 4600 4601 4602
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
I
Ingo Molnar 已提交
4603
}
4604
EXPORT_SYMBOL(wait_for_completion_interruptible);
L
Linus Torvalds 已提交
4605

4606 4607 4608 4609 4610 4611 4612 4613
/**
 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
 */
4614
long __sched
4615 4616
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
I
Ingo Molnar 已提交
4617
{
4618
	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
I
Ingo Molnar 已提交
4619
}
4620
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
L
Linus Torvalds 已提交
4621

4622 4623 4624 4625 4626 4627 4628
/**
 * wait_for_completion_killable: - waits for completion of a task (killable)
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It can be
 * interrupted by a kill signal.
 */
M
Matthew Wilcox 已提交
4629 4630 4631 4632 4633 4634 4635 4636 4637
int __sched wait_for_completion_killable(struct completion *x)
{
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
}
EXPORT_SYMBOL(wait_for_completion_killable);

4638 4639 4640 4641 4642 4643 4644 4645 4646
/**
 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be
 * signaled or for a specified timeout to expire. It can be
 * interrupted by a kill signal. The timeout is in jiffies.
 */
4647
long __sched
4648 4649 4650 4651 4652 4653 4654
wait_for_completion_killable_timeout(struct completion *x,
				     unsigned long timeout)
{
	return wait_for_common(x, timeout, TASK_KILLABLE);
}
EXPORT_SYMBOL(wait_for_completion_killable_timeout);

4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668
/**
 *	try_wait_for_completion - try to decrement a completion without blocking
 *	@x:	completion structure
 *
 *	Returns: 0 if a decrement cannot be done without blocking
 *		 1 if a decrement succeeded.
 *
 *	If a completion is being used as a counting completion,
 *	attempt to decrement the counter without blocking. This
 *	enables us to avoid waiting if the resource the completion
 *	is protecting is not available.
 */
bool try_wait_for_completion(struct completion *x)
{
4669
	unsigned long flags;
4670 4671
	int ret = 1;

4672
	spin_lock_irqsave(&x->wait.lock, flags);
4673 4674 4675 4676
	if (!x->done)
		ret = 0;
	else
		x->done--;
4677
	spin_unlock_irqrestore(&x->wait.lock, flags);
4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691
	return ret;
}
EXPORT_SYMBOL(try_wait_for_completion);

/**
 *	completion_done - Test to see if a completion has any waiters
 *	@x:	completion structure
 *
 *	Returns: 0 if there are waiters (wait_for_completion() in progress)
 *		 1 if there are no waiters.
 *
 */
bool completion_done(struct completion *x)
{
4692
	unsigned long flags;
4693 4694
	int ret = 1;

4695
	spin_lock_irqsave(&x->wait.lock, flags);
4696 4697
	if (!x->done)
		ret = 0;
4698
	spin_unlock_irqrestore(&x->wait.lock, flags);
4699 4700 4701 4702
	return ret;
}
EXPORT_SYMBOL(completion_done);

4703 4704
static long __sched
sleep_on_common(wait_queue_head_t *q, int state, long timeout)
L
Linus Torvalds 已提交
4705
{
I
Ingo Molnar 已提交
4706 4707 4708 4709
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
4710

4711
	__set_current_state(state);
L
Linus Torvalds 已提交
4712

4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726
	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, &wait);
	spin_unlock(&q->lock);
	timeout = schedule_timeout(timeout);
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, &wait);
	spin_unlock_irqrestore(&q->lock, flags);

	return timeout;
}

void __sched interruptible_sleep_on(wait_queue_head_t *q)
{
	sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
4727 4728 4729
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
4730
long __sched
I
Ingo Molnar 已提交
4731
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
4732
{
4733
	return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
4734 4735 4736
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
4737
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
4738
{
4739
	sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
4740 4741 4742
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
4743
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
4744
{
4745
	return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
4746 4747 4748
}
EXPORT_SYMBOL(sleep_on_timeout);

4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
4761
void rt_mutex_setprio(struct task_struct *p, int prio)
4762
{
4763
	int oldprio, on_rq, running;
4764
	struct rq *rq;
4765
	const struct sched_class *prev_class;
4766 4767 4768

	BUG_ON(prio < 0 || prio > MAX_PRIO);

4769
	rq = __task_rq_lock(p);
4770

4771
	trace_sched_pi_setprio(p, prio);
4772
	oldprio = p->prio;
4773
	prev_class = p->sched_class;
P
Peter Zijlstra 已提交
4774
	on_rq = p->on_rq;
4775
	running = task_current(rq, p);
4776
	if (on_rq)
4777
		dequeue_task(rq, p, 0);
4778 4779
	if (running)
		p->sched_class->put_prev_task(rq, p);
I
Ingo Molnar 已提交
4780 4781 4782 4783 4784 4785

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

4786 4787
	p->prio = prio;

4788 4789
	if (running)
		p->sched_class->set_curr_task(rq);
P
Peter Zijlstra 已提交
4790
	if (on_rq)
4791
		enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);
4792

P
Peter Zijlstra 已提交
4793
	check_class_changed(rq, p, prev_class, oldprio);
4794
	__task_rq_unlock(rq);
4795 4796 4797 4798
}

#endif

4799
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
4800
{
I
Ingo Molnar 已提交
4801
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
4802
	unsigned long flags;
4803
	struct rq *rq;
L
Linus Torvalds 已提交
4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
4816
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
4817
	 */
4818
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
4819 4820 4821
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
P
Peter Zijlstra 已提交
4822
	on_rq = p->on_rq;
4823
	if (on_rq)
4824
		dequeue_task(rq, p, 0);
L
Linus Torvalds 已提交
4825 4826

	p->static_prio = NICE_TO_PRIO(nice);
4827
	set_load_weight(p);
4828 4829 4830
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
4831

I
Ingo Molnar 已提交
4832
	if (on_rq) {
4833
		enqueue_task(rq, p, 0);
L
Linus Torvalds 已提交
4834
		/*
4835 4836
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
4837
		 */
4838
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
4839 4840 4841
			resched_task(rq->curr);
	}
out_unlock:
4842
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
4843 4844 4845
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
4846 4847 4848 4849 4850
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
4851
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
4852
{
4853 4854
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
4855

4856
	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
M
Matt Mackall 已提交
4857 4858 4859
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
4860 4861 4862 4863 4864 4865 4866 4867 4868
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
4869
SYSCALL_DEFINE1(nice, int, increment)
L
Linus Torvalds 已提交
4870
{
4871
	long nice, retval;
L
Linus Torvalds 已提交
4872 4873 4874 4875 4876 4877

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
4878 4879
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
4880 4881 4882
	if (increment > 40)
		increment = 40;

4883
	nice = TASK_NICE(current) + increment;
L
Linus Torvalds 已提交
4884 4885 4886 4887 4888
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
4889 4890 4891
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
4910
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
4911 4912 4913 4914 4915 4916 4917 4918
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
4919
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
4920 4921 4922
{
	return TASK_NICE(p);
}
P
Pavel Roskin 已提交
4923
EXPORT_SYMBOL(task_nice);
L
Linus Torvalds 已提交
4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
4938
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
4939 4940 4941 4942 4943 4944 4945 4946
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
A
Alexey Dobriyan 已提交
4947
static struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
4948
{
4949
	return pid ? find_task_by_vpid(pid) : current;
L
Linus Torvalds 已提交
4950 4951 4952
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
4953 4954
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
4955 4956 4957
{
	p->policy = policy;
	p->rt_priority = prio;
4958 4959 4960
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
4961 4962 4963 4964
	if (rt_prio(p->prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;
4965
	set_load_weight(p);
L
Linus Torvalds 已提交
4966 4967
}

4968 4969 4970 4971 4972 4973 4974 4975 4976 4977
/*
 * check the target process has a UID that matches the current process's
 */
static bool check_same_owner(struct task_struct *p)
{
	const struct cred *cred = current_cred(), *pcred;
	bool match;

	rcu_read_lock();
	pcred = __task_cred(p);
4978 4979 4980 4981 4982
	if (cred->user->user_ns == pcred->user->user_ns)
		match = (cred->euid == pcred->euid ||
			 cred->euid == pcred->uid);
	else
		match = false;
4983 4984 4985 4986
	rcu_read_unlock();
	return match;
}

4987
static int __sched_setscheduler(struct task_struct *p, int policy,
4988
				const struct sched_param *param, bool user)
L
Linus Torvalds 已提交
4989
{
4990
	int retval, oldprio, oldpolicy = -1, on_rq, running;
L
Linus Torvalds 已提交
4991
	unsigned long flags;
4992
	const struct sched_class *prev_class;
4993
	struct rq *rq;
4994
	int reset_on_fork;
L
Linus Torvalds 已提交
4995

4996 4997
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
4998 4999
recheck:
	/* double check policy once rq lock held */
5000 5001
	if (policy < 0) {
		reset_on_fork = p->sched_reset_on_fork;
L
Linus Torvalds 已提交
5002
		policy = oldpolicy = p->policy;
5003 5004 5005 5006 5007 5008 5009 5010 5011 5012
	} else {
		reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
		policy &= ~SCHED_RESET_ON_FORK;

		if (policy != SCHED_FIFO && policy != SCHED_RR &&
				policy != SCHED_NORMAL && policy != SCHED_BATCH &&
				policy != SCHED_IDLE)
			return -EINVAL;
	}

L
Linus Torvalds 已提交
5013 5014
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
5015 5016
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
5017 5018
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
5019
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
5020
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
5021
		return -EINVAL;
5022
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
5023 5024
		return -EINVAL;

5025 5026 5027
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
5028
	if (user && !capable(CAP_SYS_NICE)) {
5029
		if (rt_policy(policy)) {
5030 5031
			unsigned long rlim_rtprio =
					task_rlimit(p, RLIMIT_RTPRIO);
5032 5033 5034 5035 5036 5037 5038 5039 5040 5041

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
5042

I
Ingo Molnar 已提交
5043
		/*
5044 5045
		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
I
Ingo Molnar 已提交
5046
		 */
5047 5048 5049 5050
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
			if (!can_nice(p, TASK_NICE(p)))
				return -EPERM;
		}
5051

5052
		/* can't change other user's priorities */
5053
		if (!check_same_owner(p))
5054
			return -EPERM;
5055 5056 5057 5058

		/* Normal users shall not reset the sched_reset_on_fork flag */
		if (p->sched_reset_on_fork && !reset_on_fork)
			return -EPERM;
5059
	}
L
Linus Torvalds 已提交
5060

5061
	if (user) {
5062
		retval = security_task_setscheduler(p);
5063 5064 5065 5066
		if (retval)
			return retval;
	}

5067 5068 5069
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
5070
	 *
L
Lucas De Marchi 已提交
5071
	 * To be able to change p->policy safely, the appropriate
L
Linus Torvalds 已提交
5072 5073
	 * runqueue lock must be held.
	 */
5074
	rq = task_rq_lock(p, &flags);
5075

5076 5077 5078 5079
	/*
	 * Changing the policy of the stop threads its a very bad idea
	 */
	if (p == rq->stop) {
5080
		task_rq_unlock(rq, p, &flags);
5081 5082 5083
		return -EINVAL;
	}

5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094
	/*
	 * If not changing anything there's no need to proceed further:
	 */
	if (unlikely(policy == p->policy && (!rt_policy(policy) ||
			param->sched_priority == p->rt_priority))) {

		__task_rq_unlock(rq);
		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
		return 0;
	}

5095 5096 5097 5098 5099 5100 5101
#ifdef CONFIG_RT_GROUP_SCHED
	if (user) {
		/*
		 * Do not allow realtime tasks into groups that have no runtime
		 * assigned.
		 */
		if (rt_bandwidth_enabled() && rt_policy(policy) &&
5102 5103
				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
				!task_group_is_autogroup(task_group(p))) {
5104
			task_rq_unlock(rq, p, &flags);
5105 5106 5107 5108 5109
			return -EPERM;
		}
	}
#endif

L
Linus Torvalds 已提交
5110 5111 5112
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
5113
		task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
5114 5115
		goto recheck;
	}
P
Peter Zijlstra 已提交
5116
	on_rq = p->on_rq;
5117
	running = task_current(rq, p);
5118
	if (on_rq)
5119
		deactivate_task(rq, p, 0);
5120 5121
	if (running)
		p->sched_class->put_prev_task(rq, p);
5122

5123 5124
	p->sched_reset_on_fork = reset_on_fork;

L
Linus Torvalds 已提交
5125
	oldprio = p->prio;
5126
	prev_class = p->sched_class;
I
Ingo Molnar 已提交
5127
	__setscheduler(rq, p, policy, param->sched_priority);
5128

5129 5130
	if (running)
		p->sched_class->set_curr_task(rq);
P
Peter Zijlstra 已提交
5131
	if (on_rq)
I
Ingo Molnar 已提交
5132
		activate_task(rq, p, 0);
5133

P
Peter Zijlstra 已提交
5134
	check_class_changed(rq, p, prev_class, oldprio);
5135
	task_rq_unlock(rq, p, &flags);
5136

5137 5138
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
5139 5140
	return 0;
}
5141 5142 5143 5144 5145 5146 5147 5148 5149 5150

/**
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * NOTE that the task may be already dead.
 */
int sched_setscheduler(struct task_struct *p, int policy,
5151
		       const struct sched_param *param)
5152 5153 5154
{
	return __sched_setscheduler(p, policy, param, true);
}
L
Linus Torvalds 已提交
5155 5156
EXPORT_SYMBOL_GPL(sched_setscheduler);

5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168
/**
 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * Just like sched_setscheduler, only don't bother checking if the
 * current context has permission.  For example, this is needed in
 * stop_machine(): we create temporary high priority worker threads,
 * but our caller might not have that capability.
 */
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
5169
			       const struct sched_param *param)
5170 5171 5172 5173
{
	return __sched_setscheduler(p, policy, param, false);
}

I
Ingo Molnar 已提交
5174 5175
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
5176 5177 5178
{
	struct sched_param lparam;
	struct task_struct *p;
5179
	int retval;
L
Linus Torvalds 已提交
5180 5181 5182 5183 5184

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
5185 5186 5187

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
5188
	p = find_process_by_pid(pid);
5189 5190 5191
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
5192

L
Linus Torvalds 已提交
5193 5194 5195 5196 5197 5198 5199 5200 5201
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
5202 5203
SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
		struct sched_param __user *, param)
L
Linus Torvalds 已提交
5204
{
5205 5206 5207 5208
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
5209 5210 5211 5212 5213 5214 5215 5216
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
5217
SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
5218 5219 5220 5221 5222 5223 5224 5225
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
5226
SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
L
Linus Torvalds 已提交
5227
{
5228
	struct task_struct *p;
5229
	int retval;
L
Linus Torvalds 已提交
5230 5231

	if (pid < 0)
5232
		return -EINVAL;
L
Linus Torvalds 已提交
5233 5234

	retval = -ESRCH;
5235
	rcu_read_lock();
L
Linus Torvalds 已提交
5236 5237 5238 5239
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
5240 5241
			retval = p->policy
				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
L
Linus Torvalds 已提交
5242
	}
5243
	rcu_read_unlock();
L
Linus Torvalds 已提交
5244 5245 5246 5247
	return retval;
}

/**
5248
 * sys_sched_getparam - get the RT priority of a thread
L
Linus Torvalds 已提交
5249 5250 5251
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
5252
SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
5253 5254
{
	struct sched_param lp;
5255
	struct task_struct *p;
5256
	int retval;
L
Linus Torvalds 已提交
5257 5258

	if (!param || pid < 0)
5259
		return -EINVAL;
L
Linus Torvalds 已提交
5260

5261
	rcu_read_lock();
L
Linus Torvalds 已提交
5262 5263 5264 5265 5266 5267 5268 5269 5270 5271
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
5272
	rcu_read_unlock();
L
Linus Torvalds 已提交
5273 5274 5275 5276 5277 5278 5279 5280 5281

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

	return retval;

out_unlock:
5282
	rcu_read_unlock();
L
Linus Torvalds 已提交
5283 5284 5285
	return retval;
}

5286
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
L
Linus Torvalds 已提交
5287
{
5288
	cpumask_var_t cpus_allowed, new_mask;
5289 5290
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
5291

5292
	get_online_cpus();
5293
	rcu_read_lock();
L
Linus Torvalds 已提交
5294 5295 5296

	p = find_process_by_pid(pid);
	if (!p) {
5297
		rcu_read_unlock();
5298
		put_online_cpus();
L
Linus Torvalds 已提交
5299 5300 5301
		return -ESRCH;
	}

5302
	/* Prevent p going away */
L
Linus Torvalds 已提交
5303
	get_task_struct(p);
5304
	rcu_read_unlock();
L
Linus Torvalds 已提交
5305

5306 5307 5308 5309 5310 5311 5312 5313
	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_put_task;
	}
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_free_cpus_allowed;
	}
L
Linus Torvalds 已提交
5314
	retval = -EPERM;
5315
	if (!check_same_owner(p) && !task_ns_capable(p, CAP_SYS_NICE))
L
Linus Torvalds 已提交
5316 5317
		goto out_unlock;

5318
	retval = security_task_setscheduler(p);
5319 5320 5321
	if (retval)
		goto out_unlock;

5322 5323
	cpuset_cpus_allowed(p, cpus_allowed);
	cpumask_and(new_mask, in_mask, cpus_allowed);
P
Peter Zijlstra 已提交
5324
again:
5325
	retval = set_cpus_allowed_ptr(p, new_mask);
L
Linus Torvalds 已提交
5326

P
Paul Menage 已提交
5327
	if (!retval) {
5328 5329
		cpuset_cpus_allowed(p, cpus_allowed);
		if (!cpumask_subset(new_mask, cpus_allowed)) {
P
Paul Menage 已提交
5330 5331 5332 5333 5334
			/*
			 * We must have raced with a concurrent cpuset
			 * update. Just reset the cpus_allowed to the
			 * cpuset's cpus_allowed
			 */
5335
			cpumask_copy(new_mask, cpus_allowed);
P
Paul Menage 已提交
5336 5337 5338
			goto again;
		}
	}
L
Linus Torvalds 已提交
5339
out_unlock:
5340 5341 5342 5343
	free_cpumask_var(new_mask);
out_free_cpus_allowed:
	free_cpumask_var(cpus_allowed);
out_put_task:
L
Linus Torvalds 已提交
5344
	put_task_struct(p);
5345
	put_online_cpus();
L
Linus Torvalds 已提交
5346 5347 5348 5349
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5350
			     struct cpumask *new_mask)
L
Linus Torvalds 已提交
5351
{
5352 5353 5354 5355 5356
	if (len < cpumask_size())
		cpumask_clear(new_mask);
	else if (len > cpumask_size())
		len = cpumask_size();

L
Linus Torvalds 已提交
5357 5358 5359 5360 5361 5362 5363 5364 5365
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
5366 5367
SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
5368
{
5369
	cpumask_var_t new_mask;
L
Linus Torvalds 已提交
5370 5371
	int retval;

5372 5373
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
5374

5375 5376 5377 5378 5379
	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
	if (retval == 0)
		retval = sched_setaffinity(pid, new_mask);
	free_cpumask_var(new_mask);
	return retval;
L
Linus Torvalds 已提交
5380 5381
}

5382
long sched_getaffinity(pid_t pid, struct cpumask *mask)
L
Linus Torvalds 已提交
5383
{
5384
	struct task_struct *p;
5385
	unsigned long flags;
L
Linus Torvalds 已提交
5386 5387
	int retval;

5388
	get_online_cpus();
5389
	rcu_read_lock();
L
Linus Torvalds 已提交
5390 5391 5392 5393 5394 5395

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

5396 5397 5398 5399
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

5400
	raw_spin_lock_irqsave(&p->pi_lock, flags);
5401
	cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5402
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
5403 5404

out_unlock:
5405
	rcu_read_unlock();
5406
	put_online_cpus();
L
Linus Torvalds 已提交
5407

5408
	return retval;
L
Linus Torvalds 已提交
5409 5410 5411 5412 5413 5414 5415 5416
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
5417 5418
SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
5419 5420
{
	int ret;
5421
	cpumask_var_t mask;
L
Linus Torvalds 已提交
5422

A
Anton Blanchard 已提交
5423
	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
5424 5425
		return -EINVAL;
	if (len & (sizeof(unsigned long)-1))
L
Linus Torvalds 已提交
5426 5427
		return -EINVAL;

5428 5429
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
5430

5431 5432
	ret = sched_getaffinity(pid, mask);
	if (ret == 0) {
5433
		size_t retlen = min_t(size_t, len, cpumask_size());
5434 5435

		if (copy_to_user(user_mask_ptr, mask, retlen))
5436 5437
			ret = -EFAULT;
		else
5438
			ret = retlen;
5439 5440
	}
	free_cpumask_var(mask);
L
Linus Torvalds 已提交
5441

5442
	return ret;
L
Linus Torvalds 已提交
5443 5444 5445 5446 5447
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
5448 5449
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
5450
 */
5451
SYSCALL_DEFINE0(sched_yield)
L
Linus Torvalds 已提交
5452
{
5453
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
5454

5455
	schedstat_inc(rq, yld_count);
5456
	current->sched_class->yield_task(rq);
L
Linus Torvalds 已提交
5457 5458 5459 5460 5461 5462

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
5463
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
5464
	do_raw_spin_unlock(&rq->lock);
L
Linus Torvalds 已提交
5465 5466 5467 5468 5469 5470 5471
	preempt_enable_no_resched();

	schedule();

	return 0;
}

P
Peter Zijlstra 已提交
5472 5473 5474 5475 5476
static inline int should_resched(void)
{
	return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
}

A
Andrew Morton 已提交
5477
static void __cond_resched(void)
L
Linus Torvalds 已提交
5478
{
5479 5480 5481
	add_preempt_count(PREEMPT_ACTIVE);
	schedule();
	sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
5482 5483
}

5484
int __sched _cond_resched(void)
L
Linus Torvalds 已提交
5485
{
P
Peter Zijlstra 已提交
5486
	if (should_resched()) {
L
Linus Torvalds 已提交
5487 5488 5489 5490 5491
		__cond_resched();
		return 1;
	}
	return 0;
}
5492
EXPORT_SYMBOL(_cond_resched);
L
Linus Torvalds 已提交
5493 5494

/*
5495
 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
L
Linus Torvalds 已提交
5496 5497
 * call schedule, and on return reacquire the lock.
 *
I
Ingo Molnar 已提交
5498
 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
L
Linus Torvalds 已提交
5499 5500 5501
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
5502
int __cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
5503
{
P
Peter Zijlstra 已提交
5504
	int resched = should_resched();
J
Jan Kara 已提交
5505 5506
	int ret = 0;

5507 5508
	lockdep_assert_held(lock);

N
Nick Piggin 已提交
5509
	if (spin_needbreak(lock) || resched) {
L
Linus Torvalds 已提交
5510
		spin_unlock(lock);
P
Peter Zijlstra 已提交
5511
		if (resched)
N
Nick Piggin 已提交
5512 5513 5514
			__cond_resched();
		else
			cpu_relax();
J
Jan Kara 已提交
5515
		ret = 1;
L
Linus Torvalds 已提交
5516 5517
		spin_lock(lock);
	}
J
Jan Kara 已提交
5518
	return ret;
L
Linus Torvalds 已提交
5519
}
5520
EXPORT_SYMBOL(__cond_resched_lock);
L
Linus Torvalds 已提交
5521

5522
int __sched __cond_resched_softirq(void)
L
Linus Torvalds 已提交
5523 5524 5525
{
	BUG_ON(!in_softirq());

P
Peter Zijlstra 已提交
5526
	if (should_resched()) {
5527
		local_bh_enable();
L
Linus Torvalds 已提交
5528 5529 5530 5531 5532 5533
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
5534
EXPORT_SYMBOL(__cond_resched_softirq);
L
Linus Torvalds 已提交
5535 5536 5537 5538

/**
 * yield - yield the current processor to other threads.
 *
5539
 * This is a shortcut for kernel-space yielding - it marks the
L
Linus Torvalds 已提交
5540 5541 5542 5543 5544 5545 5546 5547 5548
 * thread runnable and calls sys_sched_yield().
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

5549 5550 5551 5552
/**
 * yield_to - yield the current processor to another thread in
 * your thread group, or accelerate that thread toward the
 * processor it's on.
R
Randy Dunlap 已提交
5553 5554
 * @p: target task
 * @preempt: whether task preemption is allowed or not
5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588
 *
 * It's the caller's job to ensure that the target task struct
 * can't go away on us before we can do any checks.
 *
 * Returns true if we indeed boosted the target task.
 */
bool __sched yield_to(struct task_struct *p, bool preempt)
{
	struct task_struct *curr = current;
	struct rq *rq, *p_rq;
	unsigned long flags;
	bool yielded = 0;

	local_irq_save(flags);
	rq = this_rq();

again:
	p_rq = task_rq(p);
	double_rq_lock(rq, p_rq);
	while (task_rq(p) != p_rq) {
		double_rq_unlock(rq, p_rq);
		goto again;
	}

	if (!curr->sched_class->yield_to_task)
		goto out;

	if (curr->sched_class != p->sched_class)
		goto out;

	if (task_running(p_rq, p) || p->state)
		goto out;

	yielded = curr->sched_class->yield_to_task(rq, p, preempt);
5589
	if (yielded) {
5590
		schedstat_inc(rq, yld_count);
5591 5592 5593 5594 5595 5596 5597
		/*
		 * Make p's CPU reschedule; pick_next_entity takes care of
		 * fairness.
		 */
		if (preempt && rq != p_rq)
			resched_task(p_rq->curr);
	}
5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609

out:
	double_rq_unlock(rq, p_rq);
	local_irq_restore(flags);

	if (yielded)
		schedule();

	return yielded;
}
EXPORT_SYMBOL_GPL(yield_to);

L
Linus Torvalds 已提交
5610
/*
I
Ingo Molnar 已提交
5611
 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
L
Linus Torvalds 已提交
5612 5613 5614 5615
 * that process accounting knows that this is a task in IO wait state.
 */
void __sched io_schedule(void)
{
5616
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
5617

5618
	delayacct_blkio_start();
L
Linus Torvalds 已提交
5619
	atomic_inc(&rq->nr_iowait);
5620
	blk_flush_plug(current);
5621
	current->in_iowait = 1;
L
Linus Torvalds 已提交
5622
	schedule();
5623
	current->in_iowait = 0;
L
Linus Torvalds 已提交
5624
	atomic_dec(&rq->nr_iowait);
5625
	delayacct_blkio_end();
L
Linus Torvalds 已提交
5626 5627 5628 5629 5630
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
5631
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
5632 5633
	long ret;

5634
	delayacct_blkio_start();
L
Linus Torvalds 已提交
5635
	atomic_inc(&rq->nr_iowait);
5636
	blk_flush_plug(current);
5637
	current->in_iowait = 1;
L
Linus Torvalds 已提交
5638
	ret = schedule_timeout(timeout);
5639
	current->in_iowait = 0;
L
Linus Torvalds 已提交
5640
	atomic_dec(&rq->nr_iowait);
5641
	delayacct_blkio_end();
L
Linus Torvalds 已提交
5642 5643 5644 5645 5646 5647 5648 5649 5650 5651
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
5652
SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
L
Linus Torvalds 已提交
5653 5654 5655 5656 5657 5658 5659 5660 5661
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
5662
	case SCHED_BATCH:
I
Ingo Molnar 已提交
5663
	case SCHED_IDLE:
L
Linus Torvalds 已提交
5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
5677
SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
L
Linus Torvalds 已提交
5678 5679 5680 5681 5682 5683 5684 5685 5686
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
5687
	case SCHED_BATCH:
I
Ingo Molnar 已提交
5688
	case SCHED_IDLE:
L
Linus Torvalds 已提交
5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
5702
SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
5703
		struct timespec __user *, interval)
L
Linus Torvalds 已提交
5704
{
5705
	struct task_struct *p;
D
Dmitry Adamushko 已提交
5706
	unsigned int time_slice;
5707 5708
	unsigned long flags;
	struct rq *rq;
5709
	int retval;
L
Linus Torvalds 已提交
5710 5711 5712
	struct timespec t;

	if (pid < 0)
5713
		return -EINVAL;
L
Linus Torvalds 已提交
5714 5715

	retval = -ESRCH;
5716
	rcu_read_lock();
L
Linus Torvalds 已提交
5717 5718 5719 5720 5721 5722 5723 5724
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

5725 5726
	rq = task_rq_lock(p, &flags);
	time_slice = p->sched_class->get_rr_interval(rq, p);
5727
	task_rq_unlock(rq, p, &flags);
D
Dmitry Adamushko 已提交
5728

5729
	rcu_read_unlock();
D
Dmitry Adamushko 已提交
5730
	jiffies_to_timespec(time_slice, &t);
L
Linus Torvalds 已提交
5731 5732
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
	return retval;
5733

L
Linus Torvalds 已提交
5734
out_unlock:
5735
	rcu_read_unlock();
L
Linus Torvalds 已提交
5736 5737 5738
	return retval;
}

5739
static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5740

5741
void sched_show_task(struct task_struct *p)
L
Linus Torvalds 已提交
5742 5743
{
	unsigned long free = 0;
5744
	unsigned state;
L
Linus Torvalds 已提交
5745 5746

	state = p->state ? __ffs(p->state) + 1 : 0;
5747
	printk(KERN_INFO "%-15.15s %c", p->comm,
5748
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5749
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
5750
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
5751
		printk(KERN_CONT " running  ");
L
Linus Torvalds 已提交
5752
	else
P
Peter Zijlstra 已提交
5753
		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
5754 5755
#else
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
5756
		printk(KERN_CONT "  running task    ");
L
Linus Torvalds 已提交
5757
	else
P
Peter Zijlstra 已提交
5758
		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
5759 5760
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
5761
	free = stack_not_used(p);
L
Linus Torvalds 已提交
5762
#endif
P
Peter Zijlstra 已提交
5763
	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
5764 5765
		task_pid_nr(p), task_pid_nr(p->real_parent),
		(unsigned long)task_thread_info(p)->flags);
L
Linus Torvalds 已提交
5766

5767
	show_stack(p, NULL);
L
Linus Torvalds 已提交
5768 5769
}

I
Ingo Molnar 已提交
5770
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
5771
{
5772
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
5773

5774
#if BITS_PER_LONG == 32
P
Peter Zijlstra 已提交
5775 5776
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
5777
#else
P
Peter Zijlstra 已提交
5778 5779
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
5780 5781 5782 5783 5784
#endif
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
L
Lucas De Marchi 已提交
5785
		 * console might take a lot of time:
L
Linus Torvalds 已提交
5786 5787
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
5788
		if (!state_filter || (p->state & state_filter))
5789
			sched_show_task(p);
L
Linus Torvalds 已提交
5790 5791
	} while_each_thread(g, p);

5792 5793
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
5794 5795 5796
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
L
Linus Torvalds 已提交
5797
	read_unlock(&tasklist_lock);
I
Ingo Molnar 已提交
5798 5799 5800
	/*
	 * Only show locks if all tasks are dumped:
	 */
5801
	if (!state_filter)
I
Ingo Molnar 已提交
5802
		debug_show_all_locks();
L
Linus Torvalds 已提交
5803 5804
}

I
Ingo Molnar 已提交
5805 5806
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
5807
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
5808 5809
}

5810 5811 5812 5813 5814 5815 5816 5817
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
5818
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
5819
{
5820
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
5821 5822
	unsigned long flags;

5823
	raw_spin_lock_irqsave(&rq->lock, flags);
5824

I
Ingo Molnar 已提交
5825
	__sched_fork(idle);
5826
	idle->state = TASK_RUNNING;
I
Ingo Molnar 已提交
5827 5828
	idle->se.exec_start = sched_clock();

5829
	cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840
	/*
	 * We're having a chicken and egg problem, even though we are
	 * holding rq->lock, the cpu isn't yet set to this cpu so the
	 * lockdep check in task_group() will fail.
	 *
	 * Similar case to sched_fork(). / Alternatively we could
	 * use task_rq_lock() here and obtain the other rq->lock.
	 *
	 * Silence PROVE_RCU
	 */
	rcu_read_lock();
I
Ingo Molnar 已提交
5841
	__set_task_cpu(idle, cpu);
5842
	rcu_read_unlock();
L
Linus Torvalds 已提交
5843 5844

	rq->curr = rq->idle = idle;
P
Peter Zijlstra 已提交
5845 5846
#if defined(CONFIG_SMP)
	idle->on_cpu = 1;
5847
#endif
5848
	raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
5849 5850

	/* Set the preempt count _outside_ the spinlocks! */
A
Al Viro 已提交
5851
	task_thread_info(idle)->preempt_count = 0;
J
Jonathan Corbet 已提交
5852

I
Ingo Molnar 已提交
5853 5854 5855 5856
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
5857
	ftrace_graph_init_idle_task(idle, cpu);
L
Linus Torvalds 已提交
5858 5859 5860 5861 5862 5863 5864
}

/*
 * In a system that switches off the HZ timer nohz_cpu_mask
 * indicates which cpus entered this state. This is used
 * in the rcu update to wait only for active cpus. For system
 * which do not switch off the HZ timer nohz_cpu_mask should
5865
 * always be CPU_BITS_NONE.
L
Linus Torvalds 已提交
5866
 */
5867
cpumask_var_t nohz_cpu_mask;
L
Linus Torvalds 已提交
5868

I
Ingo Molnar 已提交
5869 5870 5871 5872 5873 5874 5875 5876 5877
/*
 * Increase the granularity value when there are more CPUs,
 * because with more CPUs the 'effective latency' as visible
 * to users decreases. But the relationship is not linear,
 * so pick a second-best guess by going with the log2 of the
 * number of CPUs.
 *
 * This idea comes from the SD scheduler of Con Kolivas:
 */
5878
static int get_update_sysctl_factor(void)
I
Ingo Molnar 已提交
5879
{
5880
	unsigned int cpus = min_t(int, num_online_cpus(), 8);
5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894
	unsigned int factor;

	switch (sysctl_sched_tunable_scaling) {
	case SCHED_TUNABLESCALING_NONE:
		factor = 1;
		break;
	case SCHED_TUNABLESCALING_LINEAR:
		factor = cpus;
		break;
	case SCHED_TUNABLESCALING_LOG:
	default:
		factor = 1 + ilog2(cpus);
		break;
	}
I
Ingo Molnar 已提交
5895

5896 5897
	return factor;
}
I
Ingo Molnar 已提交
5898

5899 5900 5901
static void update_sysctl(void)
{
	unsigned int factor = get_update_sysctl_factor();
I
Ingo Molnar 已提交
5902

5903 5904 5905 5906 5907 5908 5909
#define SET_SYSCTL(name) \
	(sysctl_##name = (factor) * normalized_sysctl_##name)
	SET_SYSCTL(sched_min_granularity);
	SET_SYSCTL(sched_latency);
	SET_SYSCTL(sched_wakeup_granularity);
#undef SET_SYSCTL
}
5910

5911 5912 5913
static inline void sched_init_granularity(void)
{
	update_sysctl();
I
Ingo Molnar 已提交
5914 5915
}

L
Linus Torvalds 已提交
5916 5917 5918 5919
#ifdef CONFIG_SMP
/*
 * This is how migration works:
 *
5920 5921 5922 5923 5924 5925
 * 1) we invoke migration_cpu_stop() on the target CPU using
 *    stop_one_cpu().
 * 2) stopper starts to run (implicitly forcing the migrated thread
 *    off the CPU)
 * 3) it checks whether the migrated task is still in the wrong runqueue.
 * 4) if it's in the wrong runqueue then the migration thread removes
L
Linus Torvalds 已提交
5926
 *    it and puts it into the right queue.
5927 5928
 * 5) stopper completes and stop_one_cpu() returns and the migration
 *    is done.
L
Linus Torvalds 已提交
5929 5930 5931 5932 5933 5934 5935 5936
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
I
Ingo Molnar 已提交
5937
 * task must not exit() & deallocate itself prematurely. The
L
Linus Torvalds 已提交
5938 5939
 * call is not atomic; no spinlocks may be held.
 */
5940
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
L
Linus Torvalds 已提交
5941 5942
{
	unsigned long flags;
5943
	struct rq *rq;
5944
	unsigned int dest_cpu;
5945
	int ret = 0;
L
Linus Torvalds 已提交
5946 5947

	rq = task_rq_lock(p, &flags);
5948

5949
	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
L
Linus Torvalds 已提交
5950 5951 5952 5953
		ret = -EINVAL;
		goto out;
	}

5954
	if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5955
		     !cpumask_equal(&p->cpus_allowed, new_mask))) {
5956 5957 5958 5959
		ret = -EINVAL;
		goto out;
	}

5960
	if (p->sched_class->set_cpus_allowed)
5961
		p->sched_class->set_cpus_allowed(p, new_mask);
5962
	else {
5963 5964
		cpumask_copy(&p->cpus_allowed, new_mask);
		p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5965 5966
	}

L
Linus Torvalds 已提交
5967
	/* Can the task run on the task's current CPU? If so, we're done */
5968
	if (cpumask_test_cpu(task_cpu(p), new_mask))
L
Linus Torvalds 已提交
5969 5970
		goto out;

5971
	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
5972
	if (p->on_rq) {
5973
		struct migration_arg arg = { p, dest_cpu };
L
Linus Torvalds 已提交
5974
		/* Need help from migration thread: drop lock and wait. */
5975
		task_rq_unlock(rq, p, &flags);
5976
		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
5977 5978 5979 5980
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
5981
	task_rq_unlock(rq, p, &flags);
5982

L
Linus Torvalds 已提交
5983 5984
	return ret;
}
5985
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
L
Linus Torvalds 已提交
5986 5987

/*
I
Ingo Molnar 已提交
5988
 * Move (not current) task off this cpu, onto dest cpu. We're doing
L
Linus Torvalds 已提交
5989 5990 5991 5992 5993 5994
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
5995 5996
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
5997
 */
5998
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
5999
{
6000
	struct rq *rq_dest, *rq_src;
6001
	int ret = 0;
L
Linus Torvalds 已提交
6002

6003
	if (unlikely(!cpu_active(dest_cpu)))
6004
		return ret;
L
Linus Torvalds 已提交
6005 6006 6007 6008

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

6009
	raw_spin_lock(&p->pi_lock);
L
Linus Torvalds 已提交
6010 6011 6012
	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
L
Linus Torvalds 已提交
6013
		goto done;
L
Linus Torvalds 已提交
6014
	/* Affinity changed (again). */
6015
	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
L
Linus Torvalds 已提交
6016
		goto fail;
L
Linus Torvalds 已提交
6017

6018 6019 6020 6021
	/*
	 * If we're not on a rq, the next wake-up will ensure we're
	 * placed properly.
	 */
P
Peter Zijlstra 已提交
6022
	if (p->on_rq) {
6023
		deactivate_task(rq_src, p, 0);
6024
		set_task_cpu(p, dest_cpu);
I
Ingo Molnar 已提交
6025
		activate_task(rq_dest, p, 0);
6026
		check_preempt_curr(rq_dest, p, 0);
L
Linus Torvalds 已提交
6027
	}
L
Linus Torvalds 已提交
6028
done:
6029
	ret = 1;
L
Linus Torvalds 已提交
6030
fail:
L
Linus Torvalds 已提交
6031
	double_rq_unlock(rq_src, rq_dest);
6032
	raw_spin_unlock(&p->pi_lock);
6033
	return ret;
L
Linus Torvalds 已提交
6034 6035 6036
}

/*
6037 6038 6039
 * migration_cpu_stop - this will be executed by a highprio stopper thread
 * and performs thread migration by bumping thread off CPU then
 * 'pushing' onto another runqueue.
L
Linus Torvalds 已提交
6040
 */
6041
static int migration_cpu_stop(void *data)
L
Linus Torvalds 已提交
6042
{
6043
	struct migration_arg *arg = data;
6044

6045 6046 6047 6048
	/*
	 * The original target cpu might have gone down and we might
	 * be on another cpu but it doesn't matter.
	 */
6049
	local_irq_disable();
6050
	__migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
6051
	local_irq_enable();
L
Linus Torvalds 已提交
6052
	return 0;
6053 6054
}

L
Linus Torvalds 已提交
6055
#ifdef CONFIG_HOTPLUG_CPU
6056

6057
/*
6058 6059
 * Ensures that the idle task is using init_mm right before its cpu goes
 * offline.
6060
 */
6061
void idle_task_exit(void)
L
Linus Torvalds 已提交
6062
{
6063
	struct mm_struct *mm = current->active_mm;
6064

6065
	BUG_ON(cpu_online(smp_processor_id()));
6066

6067 6068 6069
	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
L
Linus Torvalds 已提交
6070 6071 6072 6073 6074 6075 6076 6077 6078
}

/*
 * While a dead CPU has no uninterruptible tasks queued at this point,
 * it might still have a nonzero ->nr_uninterruptible counter, because
 * for performance reasons the counter is not stricly tracking tasks to
 * their home CPUs. So we just add the counter to another CPU's counter,
 * to keep the global sum constant after CPU-down:
 */
6079
static void migrate_nr_uninterruptible(struct rq *rq_src)
L
Linus Torvalds 已提交
6080
{
6081
	struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
L
Linus Torvalds 已提交
6082 6083 6084 6085 6086

	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
	rq_src->nr_uninterruptible = 0;
}

I
Ingo Molnar 已提交
6087
/*
6088
 * remove the tasks which were accounted by rq from calc_load_tasks.
L
Linus Torvalds 已提交
6089
 */
6090
static void calc_global_load_remove(struct rq *rq)
L
Linus Torvalds 已提交
6091
{
6092 6093
	atomic_long_sub(rq->calc_load_active, &calc_load_tasks);
	rq->calc_load_active = 0;
L
Linus Torvalds 已提交
6094 6095
}

6096
/*
6097 6098 6099 6100 6101 6102
 * Migrate all tasks from the rq, sleeping tasks will be migrated by
 * try_to_wake_up()->select_task_rq().
 *
 * Called with rq->lock held even though we'er in stop_machine() and
 * there's no concurrency possible, we hold the required locks anyway
 * because of lock validation efforts.
L
Linus Torvalds 已提交
6103
 */
6104
static void migrate_tasks(unsigned int dead_cpu)
L
Linus Torvalds 已提交
6105
{
6106
	struct rq *rq = cpu_rq(dead_cpu);
6107 6108
	struct task_struct *next, *stop = rq->stop;
	int dest_cpu;
L
Linus Torvalds 已提交
6109 6110

	/*
6111 6112 6113 6114 6115 6116 6117
	 * Fudge the rq selection such that the below task selection loop
	 * doesn't get stuck on the currently eligible stop task.
	 *
	 * We're currently inside stop_machine() and the rq is either stuck
	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
	 * either way we should never end up calling schedule() until we're
	 * done here.
L
Linus Torvalds 已提交
6118
	 */
6119
	rq->stop = NULL;
6120

I
Ingo Molnar 已提交
6121
	for ( ; ; ) {
6122 6123 6124 6125 6126
		/*
		 * There's this thread running, bail when that's the only
		 * remaining thread.
		 */
		if (rq->nr_running == 1)
I
Ingo Molnar 已提交
6127
			break;
6128

6129
		next = pick_next_task(rq);
6130
		BUG_ON(!next);
D
Dmitry Adamushko 已提交
6131
		next->sched_class->put_prev_task(rq, next);
6132

6133 6134 6135 6136 6137 6138 6139
		/* Find suitable destination for @next, with force if needed. */
		dest_cpu = select_fallback_rq(dead_cpu, next);
		raw_spin_unlock(&rq->lock);

		__migrate_task(next, dead_cpu, dest_cpu);

		raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
6140
	}
6141

6142
	rq->stop = stop;
6143
}
6144

L
Linus Torvalds 已提交
6145 6146
#endif /* CONFIG_HOTPLUG_CPU */

6147 6148 6149
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
6150 6151
	{
		.procname	= "sched_domain",
6152
		.mode		= 0555,
6153
	},
6154
	{}
6155 6156 6157
};

static struct ctl_table sd_ctl_root[] = {
6158 6159
	{
		.procname	= "kernel",
6160
		.mode		= 0555,
6161 6162
		.child		= sd_ctl_dir,
	},
6163
	{}
6164 6165 6166 6167 6168
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
6169
		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
6170 6171 6172 6173

	return entry;
}

6174 6175
static void sd_free_ctl_entry(struct ctl_table **tablep)
{
6176
	struct ctl_table *entry;
6177

6178 6179 6180
	/*
	 * In the intermediate directories, both the child directory and
	 * procname are dynamically allocated and could fail but the mode
I
Ingo Molnar 已提交
6181
	 * will always be set. In the lowest directory the names are
6182 6183 6184
	 * static strings and all have proc handlers.
	 */
	for (entry = *tablep; entry->mode; entry++) {
6185 6186
		if (entry->child)
			sd_free_ctl_entry(&entry->child);
6187 6188 6189
		if (entry->proc_handler == NULL)
			kfree(entry->procname);
	}
6190 6191 6192 6193 6194

	kfree(*tablep);
	*tablep = NULL;
}

6195
static void
6196
set_table_entry(struct ctl_table *entry,
6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209
		const char *procname, void *data, int maxlen,
		mode_t mode, proc_handler *proc_handler)
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
6210
	struct ctl_table *table = sd_alloc_ctl_entry(13);
6211

6212 6213 6214
	if (table == NULL)
		return NULL;

6215
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
6216
		sizeof(long), 0644, proc_doulongvec_minmax);
6217
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
6218
		sizeof(long), 0644, proc_doulongvec_minmax);
6219
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
6220
		sizeof(int), 0644, proc_dointvec_minmax);
6221
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
6222
		sizeof(int), 0644, proc_dointvec_minmax);
6223
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
6224
		sizeof(int), 0644, proc_dointvec_minmax);
6225
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
6226
		sizeof(int), 0644, proc_dointvec_minmax);
6227
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
6228
		sizeof(int), 0644, proc_dointvec_minmax);
6229
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
6230
		sizeof(int), 0644, proc_dointvec_minmax);
6231
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
6232
		sizeof(int), 0644, proc_dointvec_minmax);
6233
	set_table_entry(&table[9], "cache_nice_tries",
6234 6235
		&sd->cache_nice_tries,
		sizeof(int), 0644, proc_dointvec_minmax);
6236
	set_table_entry(&table[10], "flags", &sd->flags,
6237
		sizeof(int), 0644, proc_dointvec_minmax);
6238 6239 6240
	set_table_entry(&table[11], "name", sd->name,
		CORENAME_MAX_SIZE, 0444, proc_dostring);
	/* &table[12] is terminator */
6241 6242 6243 6244

	return table;
}

6245
static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
6246 6247 6248 6249 6250 6251 6252 6253 6254
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);
6255 6256
	if (table == NULL)
		return NULL;
6257 6258 6259 6260 6261

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
6262
		entry->mode = 0555;
6263 6264 6265 6266 6267 6268 6269 6270
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
6271
static void register_sched_domain_sysctl(void)
6272
{
6273
	int i, cpu_num = num_possible_cpus();
6274 6275 6276
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

6277 6278 6279
	WARN_ON(sd_ctl_dir[0].child);
	sd_ctl_dir[0].child = entry;

6280 6281 6282
	if (entry == NULL)
		return;

6283
	for_each_possible_cpu(i) {
6284 6285
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
6286
		entry->mode = 0555;
6287
		entry->child = sd_alloc_ctl_cpu_table(i);
6288
		entry++;
6289
	}
6290 6291

	WARN_ON(sd_sysctl_header);
6292 6293
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
6294

6295
/* may be called multiple times per register */
6296 6297
static void unregister_sched_domain_sysctl(void)
{
6298 6299
	if (sd_sysctl_header)
		unregister_sysctl_table(sd_sysctl_header);
6300
	sd_sysctl_header = NULL;
6301 6302
	if (sd_ctl_dir[0].child)
		sd_free_ctl_entry(&sd_ctl_dir[0].child);
6303
}
6304
#else
6305 6306 6307 6308
static void register_sched_domain_sysctl(void)
{
}
static void unregister_sched_domain_sysctl(void)
6309 6310 6311 6312
{
}
#endif

6313 6314 6315 6316 6317
static void set_rq_online(struct rq *rq)
{
	if (!rq->online) {
		const struct sched_class *class;

6318
		cpumask_set_cpu(rq->cpu, rq->rd->online);
6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337
		rq->online = 1;

		for_each_class(class) {
			if (class->rq_online)
				class->rq_online(rq);
		}
	}
}

static void set_rq_offline(struct rq *rq)
{
	if (rq->online) {
		const struct sched_class *class;

		for_each_class(class) {
			if (class->rq_offline)
				class->rq_offline(rq);
		}

6338
		cpumask_clear_cpu(rq->cpu, rq->rd->online);
6339 6340 6341 6342
		rq->online = 0;
	}
}

L
Linus Torvalds 已提交
6343 6344 6345 6346
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
6347 6348
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
6349
{
6350
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
6351
	unsigned long flags;
6352
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
6353

6354
	switch (action & ~CPU_TASKS_FROZEN) {
6355

L
Linus Torvalds 已提交
6356
	case CPU_UP_PREPARE:
6357
		rq->calc_load_update = calc_load_update;
L
Linus Torvalds 已提交
6358
		break;
6359

L
Linus Torvalds 已提交
6360
	case CPU_ONLINE:
6361
		/* Update our root-domain */
6362
		raw_spin_lock_irqsave(&rq->lock, flags);
6363
		if (rq->rd) {
6364
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6365 6366

			set_rq_online(rq);
6367
		}
6368
		raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
6369
		break;
6370

L
Linus Torvalds 已提交
6371
#ifdef CONFIG_HOTPLUG_CPU
6372
	case CPU_DYING:
6373
		sched_ttwu_pending();
G
Gregory Haskins 已提交
6374
		/* Update our root-domain */
6375
		raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
6376
		if (rq->rd) {
6377
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6378
			set_rq_offline(rq);
G
Gregory Haskins 已提交
6379
		}
6380 6381
		migrate_tasks(cpu);
		BUG_ON(rq->nr_running != 1); /* the migration thread */
6382
		raw_spin_unlock_irqrestore(&rq->lock, flags);
6383 6384 6385

		migrate_nr_uninterruptible(rq);
		calc_global_load_remove(rq);
G
Gregory Haskins 已提交
6386
		break;
L
Linus Torvalds 已提交
6387 6388
#endif
	}
6389 6390 6391

	update_max_interval();

L
Linus Torvalds 已提交
6392 6393 6394
	return NOTIFY_OK;
}

6395 6396 6397
/*
 * Register at high priority so that task migration (migrate_all_tasks)
 * happens before everything else.  This has to be lower priority than
6398
 * the notifier in the perf_event subsystem, though.
L
Linus Torvalds 已提交
6399
 */
6400
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
6401
	.notifier_call = migration_call,
6402
	.priority = CPU_PRI_MIGRATION,
L
Linus Torvalds 已提交
6403 6404
};

6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429
static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
				      unsigned long action, void *hcpu)
{
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_ONLINE:
	case CPU_DOWN_FAILED:
		set_cpu_active((long)hcpu, true);
		return NOTIFY_OK;
	default:
		return NOTIFY_DONE;
	}
}

static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb,
					unsigned long action, void *hcpu)
{
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_DOWN_PREPARE:
		set_cpu_active((long)hcpu, false);
		return NOTIFY_OK;
	default:
		return NOTIFY_DONE;
	}
}

6430
static int __init migration_init(void)
L
Linus Torvalds 已提交
6431 6432
{
	void *cpu = (void *)(long)smp_processor_id();
6433
	int err;
6434

6435
	/* Initialize migration for the boot CPU */
6436 6437
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
6438 6439
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
6440

6441 6442 6443 6444
	/* Register cpu active notifiers */
	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);

6445
	return 0;
L
Linus Torvalds 已提交
6446
}
6447
early_initcall(migration_init);
L
Linus Torvalds 已提交
6448 6449 6450
#endif

#ifdef CONFIG_SMP
6451

6452 6453
static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */

6454
#ifdef CONFIG_SCHED_DEBUG
I
Ingo Molnar 已提交
6455

6456 6457 6458 6459 6460 6461 6462 6463 6464 6465
static __read_mostly int sched_domain_debug_enabled;

static int __init sched_domain_debug_setup(char *str)
{
	sched_domain_debug_enabled = 1;

	return 0;
}
early_param("sched_debug", sched_domain_debug_setup);

6466
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6467
				  struct cpumask *groupmask)
L
Linus Torvalds 已提交
6468
{
I
Ingo Molnar 已提交
6469
	struct sched_group *group = sd->groups;
6470
	char str[256];
L
Linus Torvalds 已提交
6471

R
Rusty Russell 已提交
6472
	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6473
	cpumask_clear(groupmask);
I
Ingo Molnar 已提交
6474 6475 6476 6477

	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);

	if (!(sd->flags & SD_LOAD_BALANCE)) {
P
Peter Zijlstra 已提交
6478
		printk("does not load-balance\n");
I
Ingo Molnar 已提交
6479
		if (sd->parent)
P
Peter Zijlstra 已提交
6480 6481
			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
					" has parent");
I
Ingo Molnar 已提交
6482
		return -1;
N
Nick Piggin 已提交
6483 6484
	}

P
Peter Zijlstra 已提交
6485
	printk(KERN_CONT "span %s level %s\n", str, sd->name);
I
Ingo Molnar 已提交
6486

6487
	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
P
Peter Zijlstra 已提交
6488 6489
		printk(KERN_ERR "ERROR: domain->span does not contain "
				"CPU%d\n", cpu);
I
Ingo Molnar 已提交
6490
	}
6491
	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
6492 6493
		printk(KERN_ERR "ERROR: domain->groups does not contain"
				" CPU%d\n", cpu);
I
Ingo Molnar 已提交
6494
	}
L
Linus Torvalds 已提交
6495

I
Ingo Molnar 已提交
6496
	printk(KERN_DEBUG "%*s groups:", level + 1, "");
L
Linus Torvalds 已提交
6497
	do {
I
Ingo Molnar 已提交
6498
		if (!group) {
P
Peter Zijlstra 已提交
6499 6500
			printk("\n");
			printk(KERN_ERR "ERROR: group is NULL\n");
L
Linus Torvalds 已提交
6501 6502 6503
			break;
		}

6504
		if (!group->cpu_power) {
P
Peter Zijlstra 已提交
6505 6506 6507
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: domain->cpu_power not "
					"set\n");
I
Ingo Molnar 已提交
6508 6509
			break;
		}
L
Linus Torvalds 已提交
6510

6511
		if (!cpumask_weight(sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
6512 6513
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: empty group\n");
I
Ingo Molnar 已提交
6514 6515
			break;
		}
L
Linus Torvalds 已提交
6516

6517
		if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
6518 6519
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: repeated CPUs\n");
I
Ingo Molnar 已提交
6520 6521
			break;
		}
L
Linus Torvalds 已提交
6522

6523
		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
L
Linus Torvalds 已提交
6524

R
Rusty Russell 已提交
6525
		cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
6526

P
Peter Zijlstra 已提交
6527
		printk(KERN_CONT " %s", str);
6528
		if (group->cpu_power != SCHED_LOAD_SCALE) {
P
Peter Zijlstra 已提交
6529 6530
			printk(KERN_CONT " (cpu_power = %d)",
				group->cpu_power);
6531
		}
L
Linus Torvalds 已提交
6532

I
Ingo Molnar 已提交
6533 6534
		group = group->next;
	} while (group != sd->groups);
P
Peter Zijlstra 已提交
6535
	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
6536

6537
	if (!cpumask_equal(sched_domain_span(sd), groupmask))
P
Peter Zijlstra 已提交
6538
		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
L
Linus Torvalds 已提交
6539

6540 6541
	if (sd->parent &&
	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
P
Peter Zijlstra 已提交
6542 6543
		printk(KERN_ERR "ERROR: parent span is not a superset "
			"of domain->span\n");
I
Ingo Molnar 已提交
6544 6545
	return 0;
}
L
Linus Torvalds 已提交
6546

I
Ingo Molnar 已提交
6547 6548 6549
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;
L
Linus Torvalds 已提交
6550

6551 6552 6553
	if (!sched_domain_debug_enabled)
		return;

I
Ingo Molnar 已提交
6554 6555 6556 6557
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}
L
Linus Torvalds 已提交
6558

I
Ingo Molnar 已提交
6559 6560 6561
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	for (;;) {
6562
		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
I
Ingo Molnar 已提交
6563
			break;
L
Linus Torvalds 已提交
6564 6565
		level++;
		sd = sd->parent;
6566
		if (!sd)
I
Ingo Molnar 已提交
6567 6568
			break;
	}
L
Linus Torvalds 已提交
6569
}
6570
#else /* !CONFIG_SCHED_DEBUG */
6571
# define sched_domain_debug(sd, cpu) do { } while (0)
6572
#endif /* CONFIG_SCHED_DEBUG */
L
Linus Torvalds 已提交
6573

6574
static int sd_degenerate(struct sched_domain *sd)
6575
{
6576
	if (cpumask_weight(sched_domain_span(sd)) == 1)
6577 6578 6579 6580 6581 6582
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
6583 6584 6585
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
6586 6587 6588 6589 6590
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
6591
	if (sd->flags & (SD_WAKE_AFFINE))
6592 6593 6594 6595 6596
		return 0;

	return 1;
}

6597 6598
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6599 6600 6601 6602 6603 6604
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

6605
	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6606 6607 6608 6609 6610 6611 6612
		return 0;

	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
6613 6614 6615
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
6616 6617
		if (nr_node_ids == 1)
			pflags &= ~SD_SERIALIZE;
6618 6619 6620 6621 6622 6623 6624
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

6625
static void free_rootdomain(struct rcu_head *rcu)
6626
{
6627
	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
6628

6629
	cpupri_cleanup(&rd->cpupri);
6630 6631 6632 6633 6634 6635
	free_cpumask_var(rd->rto_mask);
	free_cpumask_var(rd->online);
	free_cpumask_var(rd->span);
	kfree(rd);
}

G
Gregory Haskins 已提交
6636 6637
static void rq_attach_root(struct rq *rq, struct root_domain *rd)
{
I
Ingo Molnar 已提交
6638
	struct root_domain *old_rd = NULL;
G
Gregory Haskins 已提交
6639 6640
	unsigned long flags;

6641
	raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
6642 6643

	if (rq->rd) {
I
Ingo Molnar 已提交
6644
		old_rd = rq->rd;
G
Gregory Haskins 已提交
6645

6646
		if (cpumask_test_cpu(rq->cpu, old_rd->online))
6647
			set_rq_offline(rq);
G
Gregory Haskins 已提交
6648

6649
		cpumask_clear_cpu(rq->cpu, old_rd->span);
6650

I
Ingo Molnar 已提交
6651 6652 6653 6654 6655 6656 6657
		/*
		 * If we dont want to free the old_rt yet then
		 * set old_rd to NULL to skip the freeing later
		 * in this function:
		 */
		if (!atomic_dec_and_test(&old_rd->refcount))
			old_rd = NULL;
G
Gregory Haskins 已提交
6658 6659 6660 6661 6662
	}

	atomic_inc(&rd->refcount);
	rq->rd = rd;

6663
	cpumask_set_cpu(rq->cpu, rd->span);
6664
	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
6665
		set_rq_online(rq);
G
Gregory Haskins 已提交
6666

6667
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
6668 6669

	if (old_rd)
6670
		call_rcu_sched(&old_rd->rcu, free_rootdomain);
G
Gregory Haskins 已提交
6671 6672
}

6673
static int init_rootdomain(struct root_domain *rd)
G
Gregory Haskins 已提交
6674 6675 6676
{
	memset(rd, 0, sizeof(*rd));

6677
	if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6678
		goto out;
6679
	if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6680
		goto free_span;
6681
	if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6682
		goto free_online;
6683

6684
	if (cpupri_init(&rd->cpupri) != 0)
6685
		goto free_rto_mask;
6686
	return 0;
6687

6688 6689
free_rto_mask:
	free_cpumask_var(rd->rto_mask);
6690 6691 6692 6693
free_online:
	free_cpumask_var(rd->online);
free_span:
	free_cpumask_var(rd->span);
6694
out:
6695
	return -ENOMEM;
G
Gregory Haskins 已提交
6696 6697 6698 6699
}

static void init_defrootdomain(void)
{
6700
	init_rootdomain(&def_root_domain);
6701

G
Gregory Haskins 已提交
6702 6703 6704
	atomic_set(&def_root_domain.refcount, 1);
}

6705
static struct root_domain *alloc_rootdomain(void)
G
Gregory Haskins 已提交
6706 6707 6708 6709 6710 6711 6712
{
	struct root_domain *rd;

	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
	if (!rd)
		return NULL;

6713
	if (init_rootdomain(rd) != 0) {
6714 6715 6716
		kfree(rd);
		return NULL;
	}
G
Gregory Haskins 已提交
6717 6718 6719 6720

	return rd;
}

6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739
static void free_sched_domain(struct rcu_head *rcu)
{
	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
	if (atomic_dec_and_test(&sd->groups->ref))
		kfree(sd->groups);
	kfree(sd);
}

static void destroy_sched_domain(struct sched_domain *sd, int cpu)
{
	call_rcu(&sd->rcu, free_sched_domain);
}

static void destroy_sched_domains(struct sched_domain *sd, int cpu)
{
	for (; sd; sd = sd->parent)
		destroy_sched_domain(sd, cpu);
}

L
Linus Torvalds 已提交
6740
/*
I
Ingo Molnar 已提交
6741
 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
L
Linus Torvalds 已提交
6742 6743
 * hold the hotplug lock.
 */
I
Ingo Molnar 已提交
6744 6745
static void
cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
L
Linus Torvalds 已提交
6746
{
6747
	struct rq *rq = cpu_rq(cpu);
6748 6749 6750
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
6751
	for (tmp = sd; tmp; ) {
6752 6753 6754
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
6755

6756
		if (sd_parent_degenerate(tmp, parent)) {
6757
			tmp->parent = parent->parent;
6758 6759
			if (parent->parent)
				parent->parent->child = tmp;
6760
			destroy_sched_domain(parent, cpu);
6761 6762
		} else
			tmp = tmp->parent;
6763 6764
	}

6765
	if (sd && sd_degenerate(sd)) {
6766
		tmp = sd;
6767
		sd = sd->parent;
6768
		destroy_sched_domain(tmp, cpu);
6769 6770 6771
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
6772

6773
	sched_domain_debug(sd, cpu);
L
Linus Torvalds 已提交
6774

G
Gregory Haskins 已提交
6775
	rq_attach_root(rq, rd);
6776
	tmp = rq->sd;
N
Nick Piggin 已提交
6777
	rcu_assign_pointer(rq->sd, sd);
6778
	destroy_sched_domains(tmp, cpu);
L
Linus Torvalds 已提交
6779 6780 6781
}

/* cpus with isolated domains */
6782
static cpumask_var_t cpu_isolated_map;
L
Linus Torvalds 已提交
6783 6784 6785 6786

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
R
Rusty Russell 已提交
6787
	alloc_bootmem_cpumask_var(&cpu_isolated_map);
R
Rusty Russell 已提交
6788
	cpulist_parse(str, cpu_isolated_map);
L
Linus Torvalds 已提交
6789 6790 6791
	return 1;
}

I
Ingo Molnar 已提交
6792
__setup("isolcpus=", isolated_cpu_setup);
L
Linus Torvalds 已提交
6793

6794
#define SD_NODES_PER_DOMAIN 16
L
Linus Torvalds 已提交
6795

6796
#ifdef CONFIG_NUMA
6797

6798 6799 6800 6801 6802
/**
 * find_next_best_node - find the next node to include in a sched_domain
 * @node: node whose sched_domain we're building
 * @used_nodes: nodes already in the sched_domain
 *
I
Ingo Molnar 已提交
6803
 * Find the next node to include in a given scheduling domain. Simply
6804 6805 6806 6807
 * finds the closest node not already in the @used_nodes map.
 *
 * Should use nodemask_t.
 */
6808
static int find_next_best_node(int node, nodemask_t *used_nodes)
6809
{
6810
	int i, n, val, min_val, best_node = -1;
6811 6812 6813

	min_val = INT_MAX;

6814
	for (i = 0; i < nr_node_ids; i++) {
6815
		/* Start at @node */
6816
		n = (node + i) % nr_node_ids;
6817 6818 6819 6820 6821

		if (!nr_cpus_node(n))
			continue;

		/* Skip already used nodes */
6822
		if (node_isset(n, *used_nodes))
6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833
			continue;

		/* Simple min distance search */
		val = node_distance(node, n);

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

6834 6835
	if (best_node != -1)
		node_set(best_node, *used_nodes);
6836 6837 6838 6839 6840 6841
	return best_node;
}

/**
 * sched_domain_node_span - get a cpumask for a node's sched_domain
 * @node: node whose cpumask we're constructing
6842
 * @span: resulting cpumask
6843
 *
I
Ingo Molnar 已提交
6844
 * Given a node, construct a good cpumask for its sched_domain to span. It
6845 6846 6847
 * should be one that prevents unnecessary balancing, but also spreads tasks
 * out optimally.
 */
6848
static void sched_domain_node_span(int node, struct cpumask *span)
6849
{
6850
	nodemask_t used_nodes;
6851
	int i;
6852

6853
	cpumask_clear(span);
6854
	nodes_clear(used_nodes);
6855

6856
	cpumask_or(span, span, cpumask_of_node(node));
6857
	node_set(node, used_nodes);
6858 6859

	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
6860
		int next_node = find_next_best_node(node, &used_nodes);
6861 6862
		if (next_node < 0)
			break;
6863
		cpumask_or(span, span, cpumask_of_node(next_node));
6864 6865
	}
}
6866 6867 6868 6869 6870 6871 6872 6873 6874

static const struct cpumask *cpu_node_mask(int cpu)
{
	lockdep_assert_held(&sched_domains_mutex);

	sched_domain_node_span(cpu_to_node(cpu), sched_domains_tmpmask);

	return sched_domains_tmpmask;
}
6875 6876 6877 6878 6879

static const struct cpumask *cpu_allnodes_mask(int cpu)
{
	return cpu_possible_mask;
}
6880
#endif /* CONFIG_NUMA */
6881

6882 6883 6884 6885 6886
static const struct cpumask *cpu_cpu_mask(int cpu)
{
	return cpumask_of_node(cpu_to_node(cpu));
}

6887
int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
6888

6889 6890 6891 6892 6893
struct sd_data {
	struct sched_domain **__percpu sd;
	struct sched_group **__percpu sg;
};

6894
struct s_data {
6895
	struct sched_domain ** __percpu sd;
6896 6897 6898
	struct root_domain	*rd;
};

6899 6900
enum s_alloc {
	sa_rootdomain,
6901
	sa_sd,
6902
	sa_sd_storage,
6903 6904 6905
	sa_none,
};

6906 6907 6908
struct sched_domain_topology_level;

typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu);
6909 6910 6911
typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);

struct sched_domain_topology_level {
6912 6913
	sched_domain_init_f init;
	sched_domain_mask_f mask;
6914
	struct sd_data      data;
6915 6916
};

6917
/*
6918
 * Assumes the sched_domain tree is fully constructed
6919
 */
6920
static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
L
Linus Torvalds 已提交
6921
{
6922 6923
	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
	struct sched_domain *child = sd->child;
L
Linus Torvalds 已提交
6924

6925 6926
	if (child)
		cpu = cpumask_first(sched_domain_span(child));
6927

6928
	if (sg)
6929 6930 6931
		*sg = *per_cpu_ptr(sdd->sg, cpu);

	return cpu;
6932 6933
}

6934
/*
6935 6936 6937 6938 6939 6940 6941 6942
 * build_sched_groups takes the cpumask we wish to span, and a pointer
 * to a function which identifies what group(along with sched group) a CPU
 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
 * (due to the fact that we keep track of groups covered with a struct cpumask).
 *
 * build_sched_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
6943
 */
6944
static void
6945
build_sched_groups(struct sched_domain *sd)
L
Linus Torvalds 已提交
6946
{
6947 6948 6949
	struct sched_group *first = NULL, *last = NULL;
	struct sd_data *sdd = sd->private;
	const struct cpumask *span = sched_domain_span(sd);
6950
	struct cpumask *covered;
6951
	int i;
6952

6953 6954 6955
	lockdep_assert_held(&sched_domains_mutex);
	covered = sched_domains_tmpmask;

6956
	cpumask_clear(covered);
6957

6958 6959 6960 6961
	for_each_cpu(i, span) {
		struct sched_group *sg;
		int group = get_group(i, sdd, &sg);
		int j;
6962

6963 6964
		if (cpumask_test_cpu(i, covered))
			continue;
6965

6966 6967
		cpumask_clear(sched_group_cpus(sg));
		sg->cpu_power = 0;
6968

6969 6970 6971
		for_each_cpu(j, span) {
			if (get_group(j, sdd, NULL) != group)
				continue;
6972

6973 6974 6975
			cpumask_set_cpu(j, covered);
			cpumask_set_cpu(j, sched_group_cpus(sg));
		}
6976

6977 6978 6979 6980 6981 6982 6983
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
6984
}
6985

6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
	WARN_ON(!sd || !sd->groups);

7000
	if (cpu != group_first_cpu(sd->groups))
7001 7002
		return;

7003 7004
	sd->groups->group_weight = cpumask_weight(sched_group_cpus(sd->groups));

7005
	update_group_power(sd, cpu);
7006 7007
}

7008 7009 7010 7011 7012
/*
 * Initializers for schedule domains
 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
 */

7013 7014 7015 7016 7017 7018
#ifdef CONFIG_SCHED_DEBUG
# define SD_INIT_NAME(sd, type)		sd->name = #type
#else
# define SD_INIT_NAME(sd, type)		do { } while (0)
#endif

7019 7020 7021 7022 7023 7024 7025 7026 7027
#define SD_INIT_FUNC(type)						\
static noinline struct sched_domain *					\
sd_init_##type(struct sched_domain_topology_level *tl, int cpu) 	\
{									\
	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);	\
	*sd = SD_##type##_INIT;						\
	SD_INIT_NAME(sd, type);						\
	sd->private = &tl->data;					\
	return sd;							\
7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040
}

SD_INIT_FUNC(CPU)
#ifdef CONFIG_NUMA
 SD_INIT_FUNC(ALLNODES)
 SD_INIT_FUNC(NODE)
#endif
#ifdef CONFIG_SCHED_SMT
 SD_INIT_FUNC(SIBLING)
#endif
#ifdef CONFIG_SCHED_MC
 SD_INIT_FUNC(MC)
#endif
7041 7042 7043
#ifdef CONFIG_SCHED_BOOK
 SD_INIT_FUNC(BOOK)
#endif
7044

7045
static int default_relax_domain_level = -1;
7046
int sched_domain_level_max;
7047 7048 7049

static int __init setup_relax_domain_level(char *str)
{
7050 7051 7052
	unsigned long val;

	val = simple_strtoul(str, NULL, 0);
7053
	if (val < sched_domain_level_max)
7054 7055
		default_relax_domain_level = val;

7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073
	return 1;
}
__setup("relax_domain_level=", setup_relax_domain_level);

static void set_domain_attribute(struct sched_domain *sd,
				 struct sched_domain_attr *attr)
{
	int request;

	if (!attr || attr->relax_domain_level < 0) {
		if (default_relax_domain_level < 0)
			return;
		else
			request = default_relax_domain_level;
	} else
		request = attr->relax_domain_level;
	if (request < sd->level) {
		/* turn off idle balance on this domain */
7074
		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
7075 7076
	} else {
		/* turn on idle balance on this domain */
7077
		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
7078 7079 7080
	}
}

7081 7082 7083
static void __sdt_free(const struct cpumask *cpu_map);
static int __sdt_alloc(const struct cpumask *cpu_map);

7084 7085 7086 7087 7088
static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
				 const struct cpumask *cpu_map)
{
	switch (what) {
	case sa_rootdomain:
7089 7090
		if (!atomic_read(&d->rd->refcount))
			free_rootdomain(&d->rd->rcu); /* fall through */
7091 7092
	case sa_sd:
		free_percpu(d->sd); /* fall through */
7093
	case sa_sd_storage:
7094
		__sdt_free(cpu_map); /* fall through */
7095 7096 7097 7098
	case sa_none:
		break;
	}
}
7099

7100 7101 7102
static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
						   const struct cpumask *cpu_map)
{
7103 7104
	memset(d, 0, sizeof(*d));

7105 7106
	if (__sdt_alloc(cpu_map))
		return sa_sd_storage;
7107 7108 7109
	d->sd = alloc_percpu(struct sched_domain *);
	if (!d->sd)
		return sa_sd_storage;
7110
	d->rd = alloc_rootdomain();
7111
	if (!d->rd)
7112
		return sa_sd;
7113 7114
	return sa_rootdomain;
}
G
Gregory Haskins 已提交
7115

7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134
/*
 * NULL the sd_data elements we've used to build the sched_domain and
 * sched_group structure so that the subsequent __free_domain_allocs()
 * will not free the data we're using.
 */
static void claim_allocations(int cpu, struct sched_domain *sd)
{
	struct sd_data *sdd = sd->private;
	struct sched_group *sg = sd->groups;

	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
	*per_cpu_ptr(sdd->sd, cpu) = NULL;

	if (cpu == cpumask_first(sched_group_cpus(sg))) {
		WARN_ON_ONCE(*per_cpu_ptr(sdd->sg, cpu) != sg);
		*per_cpu_ptr(sdd->sg, cpu) = NULL;
	}
}

7135 7136
#ifdef CONFIG_SCHED_SMT
static const struct cpumask *cpu_smt_mask(int cpu)
7137
{
7138
	return topology_thread_cpumask(cpu);
7139
}
7140
#endif
7141

7142 7143 7144
/*
 * Topology list, bottom-up.
 */
7145
static struct sched_domain_topology_level default_topology[] = {
7146 7147
#ifdef CONFIG_SCHED_SMT
	{ sd_init_SIBLING, cpu_smt_mask, },
7148
#endif
7149
#ifdef CONFIG_SCHED_MC
7150
	{ sd_init_MC, cpu_coregroup_mask, },
7151
#endif
7152 7153 7154 7155 7156 7157 7158
#ifdef CONFIG_SCHED_BOOK
	{ sd_init_BOOK, cpu_book_mask, },
#endif
	{ sd_init_CPU, cpu_cpu_mask, },
#ifdef CONFIG_NUMA
	{ sd_init_NODE, cpu_node_mask, },
	{ sd_init_ALLNODES, cpu_allnodes_mask, },
L
Linus Torvalds 已提交
7159
#endif
7160 7161 7162 7163 7164
	{ NULL, },
};

static struct sched_domain_topology_level *sched_domain_topology = default_topology;

7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220
static int __sdt_alloc(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

	for (tl = sched_domain_topology; tl->init; tl++) {
		struct sd_data *sdd = &tl->data;

		sdd->sd = alloc_percpu(struct sched_domain *);
		if (!sdd->sd)
			return -ENOMEM;

		sdd->sg = alloc_percpu(struct sched_group *);
		if (!sdd->sg)
			return -ENOMEM;

		for_each_cpu(j, cpu_map) {
			struct sched_domain *sd;
			struct sched_group *sg;

		       	sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sd)
				return -ENOMEM;

			*per_cpu_ptr(sdd->sd, j) = sd;

			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sg)
				return -ENOMEM;

			*per_cpu_ptr(sdd->sg, j) = sg;
		}
	}

	return 0;
}

static void __sdt_free(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

	for (tl = sched_domain_topology; tl->init; tl++) {
		struct sd_data *sdd = &tl->data;

		for_each_cpu(j, cpu_map) {
			kfree(*per_cpu_ptr(sdd->sd, j));
			kfree(*per_cpu_ptr(sdd->sg, j));
		}
		free_percpu(sdd->sd);
		free_percpu(sdd->sg);
	}
}

7221 7222
struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
		struct s_data *d, const struct cpumask *cpu_map,
7223
		struct sched_domain_attr *attr, struct sched_domain *child,
7224 7225
		int cpu)
{
7226
	struct sched_domain *sd = tl->init(tl, cpu);
7227
	if (!sd)
7228
		return child;
7229 7230 7231

	set_domain_attribute(sd, attr);
	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
7232 7233 7234
	if (child) {
		sd->level = child->level + 1;
		sched_domain_level_max = max(sched_domain_level_max, sd->level);
7235
		child->parent = sd;
7236
	}
7237
	sd->child = child;
7238 7239 7240 7241

	return sd;
}

7242 7243 7244 7245
/*
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
 */
7246 7247
static int build_sched_domains(const struct cpumask *cpu_map,
			       struct sched_domain_attr *attr)
7248 7249
{
	enum s_alloc alloc_state = sa_none;
7250
	struct sched_domain *sd;
7251
	struct s_data d;
7252
	int i, ret = -ENOMEM;
7253

7254 7255 7256
	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
	if (alloc_state != sa_rootdomain)
		goto error;
7257

7258
	/* Set up domains for cpus specified by the cpu_map. */
7259
	for_each_cpu(i, cpu_map) {
7260 7261
		struct sched_domain_topology_level *tl;

7262
		sd = NULL;
7263 7264
		for (tl = sched_domain_topology; tl->init; tl++)
			sd = build_sched_domain(tl, &d, cpu_map, attr, sd, i);
7265

7266 7267 7268
		while (sd->child)
			sd = sd->child;

7269
		*per_cpu_ptr(d.sd, i) = sd;
7270 7271 7272 7273 7274 7275 7276 7277
	}

	/* Build the groups for the domains */
	for_each_cpu(i, cpu_map) {
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			sd->span_weight = cpumask_weight(sched_domain_span(sd));
			get_group(i, sd->private, &sd->groups);
			atomic_inc(&sd->groups->ref);
7278

7279 7280 7281
			if (i != cpumask_first(sched_domain_span(sd)))
				continue;

7282
			build_sched_groups(sd);
7283
		}
7284
	}
7285

L
Linus Torvalds 已提交
7286
	/* Calculate CPU power for physical packages and nodes */
7287 7288 7289
	for (i = nr_cpumask_bits-1; i >= 0; i--) {
		if (!cpumask_test_cpu(i, cpu_map))
			continue;
7290

7291 7292
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			claim_allocations(i, sd);
7293
			init_sched_groups_power(i, sd);
7294
		}
7295
	}
7296

L
Linus Torvalds 已提交
7297
	/* Attach the domains */
7298
	rcu_read_lock();
7299
	for_each_cpu(i, cpu_map) {
7300
		sd = *per_cpu_ptr(d.sd, i);
7301
		cpu_attach_domain(sd, d.rd, i);
L
Linus Torvalds 已提交
7302
	}
7303
	rcu_read_unlock();
7304

7305
	ret = 0;
7306
error:
7307
	__free_domain_allocs(&d, alloc_state, cpu_map);
7308
	return ret;
L
Linus Torvalds 已提交
7309
}
P
Paul Jackson 已提交
7310

7311
static cpumask_var_t *doms_cur;	/* current sched domains */
P
Paul Jackson 已提交
7312
static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
I
Ingo Molnar 已提交
7313 7314
static struct sched_domain_attr *dattr_cur;
				/* attribues of custom domains in 'doms_cur' */
P
Paul Jackson 已提交
7315 7316 7317

/*
 * Special case: If a kmalloc of a doms_cur partition (array of
7318 7319
 * cpumask) fails, then fallback to a single sched domain,
 * as determined by the single cpumask fallback_doms.
P
Paul Jackson 已提交
7320
 */
7321
static cpumask_var_t fallback_doms;
P
Paul Jackson 已提交
7322

7323 7324 7325 7326 7327 7328
/*
 * arch_update_cpu_topology lets virtualized architectures update the
 * cpu core maps. It is supposed to return 1 if the topology changed
 * or 0 if it stayed the same.
 */
int __attribute__((weak)) arch_update_cpu_topology(void)
7329
{
7330
	return 0;
7331 7332
}

7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357
cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
{
	int i;
	cpumask_var_t *doms;

	doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
	if (!doms)
		return NULL;
	for (i = 0; i < ndoms; i++) {
		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
			free_sched_domains(doms, i);
			return NULL;
		}
	}
	return doms;
}

void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
{
	unsigned int i;
	for (i = 0; i < ndoms; i++)
		free_cpumask_var(doms[i]);
	kfree(doms);
}

7358
/*
I
Ingo Molnar 已提交
7359
 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
P
Paul Jackson 已提交
7360 7361
 * For now this just excludes isolated cpus, but could be used to
 * exclude other special cases in the future.
7362
 */
7363
static int init_sched_domains(const struct cpumask *cpu_map)
7364
{
7365 7366
	int err;

7367
	arch_update_cpu_topology();
P
Paul Jackson 已提交
7368
	ndoms_cur = 1;
7369
	doms_cur = alloc_sched_domains(ndoms_cur);
P
Paul Jackson 已提交
7370
	if (!doms_cur)
7371 7372
		doms_cur = &fallback_doms;
	cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
7373
	dattr_cur = NULL;
7374
	err = build_sched_domains(doms_cur[0], NULL);
7375
	register_sched_domain_sysctl();
7376 7377

	return err;
7378 7379 7380 7381 7382 7383
}

/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
7384
static void detach_destroy_domains(const struct cpumask *cpu_map)
7385 7386 7387
{
	int i;

7388
	rcu_read_lock();
7389
	for_each_cpu(i, cpu_map)
G
Gregory Haskins 已提交
7390
		cpu_attach_domain(NULL, &def_root_domain, i);
7391
	rcu_read_unlock();
7392 7393
}

7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409
/* handle null as "default" */
static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
			struct sched_domain_attr *new, int idx_new)
{
	struct sched_domain_attr tmp;

	/* fast path */
	if (!new && !cur)
		return 1;

	tmp = SD_ATTR_INIT;
	return !memcmp(cur ? (cur + idx_cur) : &tmp,
			new ? (new + idx_new) : &tmp,
			sizeof(struct sched_domain_attr));
}

P
Paul Jackson 已提交
7410 7411
/*
 * Partition sched domains as specified by the 'ndoms_new'
I
Ingo Molnar 已提交
7412
 * cpumasks in the array doms_new[] of cpumasks. This compares
P
Paul Jackson 已提交
7413 7414 7415
 * doms_new[] to the current sched domain partitioning, doms_cur[].
 * It destroys each deleted domain and builds each new domain.
 *
7416
 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
I
Ingo Molnar 已提交
7417 7418 7419
 * The masks don't intersect (don't overlap.) We should setup one
 * sched domain for each mask. CPUs not in any of the cpumasks will
 * not be load balanced. If the same cpumask appears both in the
P
Paul Jackson 已提交
7420 7421 7422
 * current 'doms_cur' domains and in the new 'doms_new', we can leave
 * it as it is.
 *
7423 7424 7425 7426 7427 7428
 * The passed in 'doms_new' should be allocated using
 * alloc_sched_domains.  This routine takes ownership of it and will
 * free_sched_domains it when done with it. If the caller failed the
 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
 * and partition_sched_domains() will fallback to the single partition
 * 'fallback_doms', it also forces the domains to be rebuilt.
P
Paul Jackson 已提交
7429
 *
7430
 * If doms_new == NULL it will be replaced with cpu_online_mask.
7431 7432
 * ndoms_new == 0 is a special case for destroying existing domains,
 * and it will not create the default domain.
7433
 *
P
Paul Jackson 已提交
7434 7435
 * Call with hotplug lock held
 */
7436
void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
7437
			     struct sched_domain_attr *dattr_new)
P
Paul Jackson 已提交
7438
{
7439
	int i, j, n;
7440
	int new_topology;
P
Paul Jackson 已提交
7441

7442
	mutex_lock(&sched_domains_mutex);
7443

7444 7445 7446
	/* always unregister in case we don't destroy any domains */
	unregister_sched_domain_sysctl();

7447 7448 7449
	/* Let architecture update cpu core mappings. */
	new_topology = arch_update_cpu_topology();

7450
	n = doms_new ? ndoms_new : 0;
P
Paul Jackson 已提交
7451 7452 7453

	/* Destroy deleted domains */
	for (i = 0; i < ndoms_cur; i++) {
7454
		for (j = 0; j < n && !new_topology; j++) {
7455
			if (cpumask_equal(doms_cur[i], doms_new[j])
7456
			    && dattrs_equal(dattr_cur, i, dattr_new, j))
P
Paul Jackson 已提交
7457 7458 7459
				goto match1;
		}
		/* no match - a current sched domain not in new doms_new[] */
7460
		detach_destroy_domains(doms_cur[i]);
P
Paul Jackson 已提交
7461 7462 7463 7464
match1:
		;
	}

7465 7466
	if (doms_new == NULL) {
		ndoms_cur = 0;
7467
		doms_new = &fallback_doms;
7468
		cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
7469
		WARN_ON_ONCE(dattr_new);
7470 7471
	}

P
Paul Jackson 已提交
7472 7473
	/* Build new domains */
	for (i = 0; i < ndoms_new; i++) {
7474
		for (j = 0; j < ndoms_cur && !new_topology; j++) {
7475
			if (cpumask_equal(doms_new[i], doms_cur[j])
7476
			    && dattrs_equal(dattr_new, i, dattr_cur, j))
P
Paul Jackson 已提交
7477 7478 7479
				goto match2;
		}
		/* no match - add a new doms_new */
7480
		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
P
Paul Jackson 已提交
7481 7482 7483 7484 7485
match2:
		;
	}

	/* Remember the new sched domains */
7486 7487
	if (doms_cur != &fallback_doms)
		free_sched_domains(doms_cur, ndoms_cur);
7488
	kfree(dattr_cur);	/* kfree(NULL) is safe */
P
Paul Jackson 已提交
7489
	doms_cur = doms_new;
7490
	dattr_cur = dattr_new;
P
Paul Jackson 已提交
7491
	ndoms_cur = ndoms_new;
7492 7493

	register_sched_domain_sysctl();
7494

7495
	mutex_unlock(&sched_domains_mutex);
P
Paul Jackson 已提交
7496 7497
}

7498
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
7499
static void reinit_sched_domains(void)
7500
{
7501
	get_online_cpus();
7502 7503 7504 7505

	/* Destroy domains first to force the rebuild */
	partition_sched_domains(0, NULL, NULL);

7506
	rebuild_sched_domains();
7507
	put_online_cpus();
7508 7509 7510 7511
}

static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
{
7512
	unsigned int level = 0;
7513

7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524
	if (sscanf(buf, "%u", &level) != 1)
		return -EINVAL;

	/*
	 * level is always be positive so don't check for
	 * level < POWERSAVINGS_BALANCE_NONE which is 0
	 * What happens on 0 or 1 byte write,
	 * need to check for count as well?
	 */

	if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
7525 7526 7527
		return -EINVAL;

	if (smt)
7528
		sched_smt_power_savings = level;
7529
	else
7530
		sched_mc_power_savings = level;
7531

7532
	reinit_sched_domains();
7533

7534
	return count;
7535 7536 7537
}

#ifdef CONFIG_SCHED_MC
7538
static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
7539
					   struct sysdev_class_attribute *attr,
7540
					   char *page)
7541 7542 7543
{
	return sprintf(page, "%u\n", sched_mc_power_savings);
}
7544
static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
7545
					    struct sysdev_class_attribute *attr,
7546
					    const char *buf, size_t count)
7547 7548 7549
{
	return sched_power_savings_store(buf, count, 0);
}
7550 7551 7552
static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
			 sched_mc_power_savings_show,
			 sched_mc_power_savings_store);
7553 7554 7555
#endif

#ifdef CONFIG_SCHED_SMT
7556
static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
7557
					    struct sysdev_class_attribute *attr,
7558
					    char *page)
7559 7560 7561
{
	return sprintf(page, "%u\n", sched_smt_power_savings);
}
7562
static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
7563
					     struct sysdev_class_attribute *attr,
7564
					     const char *buf, size_t count)
7565 7566 7567
{
	return sched_power_savings_store(buf, count, 1);
}
7568 7569
static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
		   sched_smt_power_savings_show,
A
Adrian Bunk 已提交
7570 7571 7572
		   sched_smt_power_savings_store);
#endif

7573
int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
A
Adrian Bunk 已提交
7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588
{
	int err = 0;

#ifdef CONFIG_SCHED_SMT
	if (smt_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_smt_power_savings.attr);
#endif
#ifdef CONFIG_SCHED_MC
	if (!err && mc_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_mc_power_savings.attr);
#endif
	return err;
}
7589
#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
7590

L
Linus Torvalds 已提交
7591
/*
7592 7593 7594
 * Update cpusets according to cpu_active mask.  If cpusets are
 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
 * around partition_sched_domains().
L
Linus Torvalds 已提交
7595
 */
7596 7597
static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
			     void *hcpu)
7598
{
7599
	switch (action & ~CPU_TASKS_FROZEN) {
7600
	case CPU_ONLINE:
7601
	case CPU_DOWN_FAILED:
7602
		cpuset_update_active_cpus();
7603
		return NOTIFY_OK;
7604 7605 7606 7607
	default:
		return NOTIFY_DONE;
	}
}
7608

7609 7610
static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
			       void *hcpu)
7611 7612 7613 7614 7615
{
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_DOWN_PREPARE:
		cpuset_update_active_cpus();
		return NOTIFY_OK;
7616 7617 7618 7619 7620 7621 7622
	default:
		return NOTIFY_DONE;
	}
}

static int update_runtime(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
7623
{
P
Peter Zijlstra 已提交
7624 7625
	int cpu = (int)(long)hcpu;

L
Linus Torvalds 已提交
7626 7627
	switch (action) {
	case CPU_DOWN_PREPARE:
7628
	case CPU_DOWN_PREPARE_FROZEN:
P
Peter Zijlstra 已提交
7629
		disable_runtime(cpu_rq(cpu));
L
Linus Torvalds 已提交
7630 7631 7632
		return NOTIFY_OK;

	case CPU_DOWN_FAILED:
7633
	case CPU_DOWN_FAILED_FROZEN:
L
Linus Torvalds 已提交
7634
	case CPU_ONLINE:
7635
	case CPU_ONLINE_FROZEN:
P
Peter Zijlstra 已提交
7636
		enable_runtime(cpu_rq(cpu));
7637 7638
		return NOTIFY_OK;

L
Linus Torvalds 已提交
7639 7640 7641 7642 7643 7644 7645
	default:
		return NOTIFY_DONE;
	}
}

void __init sched_init_smp(void)
{
7646 7647 7648
	cpumask_var_t non_isolated_cpus;

	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7649
	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
7650

7651
	get_online_cpus();
7652
	mutex_lock(&sched_domains_mutex);
7653
	init_sched_domains(cpu_active_mask);
7654 7655 7656
	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
	if (cpumask_empty(non_isolated_cpus))
		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
7657
	mutex_unlock(&sched_domains_mutex);
7658
	put_online_cpus();
7659

7660 7661
	hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
	hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
7662 7663 7664 7665

	/* RT runtime code needs to handle some hotplug events */
	hotcpu_notifier(update_runtime, 0);

7666
	init_hrtick();
7667 7668

	/* Move init over to a non-isolated CPU */
7669
	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
7670
		BUG();
I
Ingo Molnar 已提交
7671
	sched_init_granularity();
7672
	free_cpumask_var(non_isolated_cpus);
7673

7674
	init_sched_rt_class();
L
Linus Torvalds 已提交
7675 7676 7677 7678
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
7679
	sched_init_granularity();
L
Linus Torvalds 已提交
7680 7681 7682
}
#endif /* CONFIG_SMP */

7683 7684
const_debug unsigned int sysctl_timer_migration = 1;

L
Linus Torvalds 已提交
7685 7686 7687 7688 7689 7690 7691
int in_sched_functions(unsigned long addr)
{
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

A
Alexey Dobriyan 已提交
7692
static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
I
Ingo Molnar 已提交
7693 7694
{
	cfs_rq->tasks_timeline = RB_ROOT;
7695
	INIT_LIST_HEAD(&cfs_rq->tasks);
I
Ingo Molnar 已提交
7696 7697
#ifdef CONFIG_FAIR_GROUP_SCHED
	cfs_rq->rq = rq;
7698
	/* allow initial update_cfs_load() to truncate */
7699
#ifdef CONFIG_SMP
7700
	cfs_rq->load_stamp = 1;
7701
#endif
I
Ingo Molnar 已提交
7702
#endif
P
Peter Zijlstra 已提交
7703
	cfs_rq->min_vruntime = (u64)(-(1LL << 20));
I
Ingo Molnar 已提交
7704 7705
}

P
Peter Zijlstra 已提交
7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718
static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
{
	struct rt_prio_array *array;
	int i;

	array = &rt_rq->active;
	for (i = 0; i < MAX_RT_PRIO; i++) {
		INIT_LIST_HEAD(array->queue + i);
		__clear_bit(i, array->bitmap);
	}
	/* delimiter for bitsearch: */
	__set_bit(MAX_RT_PRIO, array->bitmap);

7719
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
7720
	rt_rq->highest_prio.curr = MAX_RT_PRIO;
7721
#ifdef CONFIG_SMP
7722
	rt_rq->highest_prio.next = MAX_RT_PRIO;
P
Peter Zijlstra 已提交
7723 7724
#endif
#endif
P
Peter Zijlstra 已提交
7725 7726 7727
#ifdef CONFIG_SMP
	rt_rq->rt_nr_migratory = 0;
	rt_rq->overloaded = 0;
7728
	plist_head_init_raw(&rt_rq->pushable_tasks, &rq->lock);
P
Peter Zijlstra 已提交
7729 7730 7731 7732
#endif

	rt_rq->rt_time = 0;
	rt_rq->rt_throttled = 0;
P
Peter Zijlstra 已提交
7733
	rt_rq->rt_runtime = 0;
7734
	raw_spin_lock_init(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7735

7736
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
7737
	rt_rq->rt_nr_boosted = 0;
P
Peter Zijlstra 已提交
7738 7739
	rt_rq->rq = rq;
#endif
P
Peter Zijlstra 已提交
7740 7741
}

P
Peter Zijlstra 已提交
7742
#ifdef CONFIG_FAIR_GROUP_SCHED
7743
static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
7744
				struct sched_entity *se, int cpu,
7745
				struct sched_entity *parent)
P
Peter Zijlstra 已提交
7746
{
7747
	struct rq *rq = cpu_rq(cpu);
P
Peter Zijlstra 已提交
7748 7749 7750 7751 7752
	tg->cfs_rq[cpu] = cfs_rq;
	init_cfs_rq(cfs_rq, rq);
	cfs_rq->tg = tg;

	tg->se[cpu] = se;
7753
	/* se could be NULL for root_task_group */
D
Dhaval Giani 已提交
7754 7755 7756
	if (!se)
		return;

7757 7758 7759 7760 7761
	if (!parent)
		se->cfs_rq = &rq->cfs;
	else
		se->cfs_rq = parent->my_q;

P
Peter Zijlstra 已提交
7762
	se->my_q = cfs_rq;
7763
	update_load_set(&se->load, 0);
7764
	se->parent = parent;
P
Peter Zijlstra 已提交
7765
}
7766
#endif
P
Peter Zijlstra 已提交
7767

7768
#ifdef CONFIG_RT_GROUP_SCHED
7769
static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
7770
		struct sched_rt_entity *rt_se, int cpu,
7771
		struct sched_rt_entity *parent)
P
Peter Zijlstra 已提交
7772
{
7773 7774
	struct rq *rq = cpu_rq(cpu);

P
Peter Zijlstra 已提交
7775 7776 7777
	tg->rt_rq[cpu] = rt_rq;
	init_rt_rq(rt_rq, rq);
	rt_rq->tg = tg;
P
Peter Zijlstra 已提交
7778
	rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
7779 7780

	tg->rt_se[cpu] = rt_se;
D
Dhaval Giani 已提交
7781 7782 7783
	if (!rt_se)
		return;

7784 7785 7786 7787 7788
	if (!parent)
		rt_se->rt_rq = &rq->rt;
	else
		rt_se->rt_rq = parent->my_q;

P
Peter Zijlstra 已提交
7789
	rt_se->my_q = rt_rq;
7790
	rt_se->parent = parent;
P
Peter Zijlstra 已提交
7791 7792 7793 7794
	INIT_LIST_HEAD(&rt_se->run_list);
}
#endif

L
Linus Torvalds 已提交
7795 7796
void __init sched_init(void)
{
I
Ingo Molnar 已提交
7797
	int i, j;
7798 7799 7800 7801 7802 7803 7804
	unsigned long alloc_size = 0, ptr;

#ifdef CONFIG_FAIR_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7805
#endif
7806
#ifdef CONFIG_CPUMASK_OFFSTACK
7807
	alloc_size += num_possible_cpus() * cpumask_size();
7808 7809
#endif
	if (alloc_size) {
7810
		ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
7811 7812

#ifdef CONFIG_FAIR_GROUP_SCHED
7813
		root_task_group.se = (struct sched_entity **)ptr;
7814 7815
		ptr += nr_cpu_ids * sizeof(void **);

7816
		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
7817
		ptr += nr_cpu_ids * sizeof(void **);
7818

7819
#endif /* CONFIG_FAIR_GROUP_SCHED */
7820
#ifdef CONFIG_RT_GROUP_SCHED
7821
		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
7822 7823
		ptr += nr_cpu_ids * sizeof(void **);

7824
		root_task_group.rt_rq = (struct rt_rq **)ptr;
7825 7826
		ptr += nr_cpu_ids * sizeof(void **);

7827
#endif /* CONFIG_RT_GROUP_SCHED */
7828 7829 7830 7831 7832 7833
#ifdef CONFIG_CPUMASK_OFFSTACK
		for_each_possible_cpu(i) {
			per_cpu(load_balance_tmpmask, i) = (void *)ptr;
			ptr += cpumask_size();
		}
#endif /* CONFIG_CPUMASK_OFFSTACK */
7834
	}
I
Ingo Molnar 已提交
7835

G
Gregory Haskins 已提交
7836 7837 7838 7839
#ifdef CONFIG_SMP
	init_defrootdomain();
#endif

7840 7841 7842 7843
	init_rt_bandwidth(&def_rt_bandwidth,
			global_rt_period(), global_rt_runtime());

#ifdef CONFIG_RT_GROUP_SCHED
7844
	init_rt_bandwidth(&root_task_group.rt_bandwidth,
7845
			global_rt_period(), global_rt_runtime());
7846
#endif /* CONFIG_RT_GROUP_SCHED */
7847

D
Dhaval Giani 已提交
7848
#ifdef CONFIG_CGROUP_SCHED
7849 7850
	list_add(&root_task_group.list, &task_groups);
	INIT_LIST_HEAD(&root_task_group.children);
7851
	autogroup_init(&init_task);
D
Dhaval Giani 已提交
7852
#endif /* CONFIG_CGROUP_SCHED */
P
Peter Zijlstra 已提交
7853

7854
	for_each_possible_cpu(i) {
7855
		struct rq *rq;
L
Linus Torvalds 已提交
7856 7857

		rq = cpu_rq(i);
7858
		raw_spin_lock_init(&rq->lock);
N
Nick Piggin 已提交
7859
		rq->nr_running = 0;
7860 7861
		rq->calc_load_active = 0;
		rq->calc_load_update = jiffies + LOAD_FREQ;
I
Ingo Molnar 已提交
7862
		init_cfs_rq(&rq->cfs, rq);
P
Peter Zijlstra 已提交
7863
		init_rt_rq(&rq->rt, rq);
I
Ingo Molnar 已提交
7864
#ifdef CONFIG_FAIR_GROUP_SCHED
7865
		root_task_group.shares = root_task_group_load;
P
Peter Zijlstra 已提交
7866
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
D
Dhaval Giani 已提交
7867
		/*
7868
		 * How much cpu bandwidth does root_task_group get?
D
Dhaval Giani 已提交
7869 7870 7871 7872
		 *
		 * In case of task-groups formed thr' the cgroup filesystem, it
		 * gets 100% of the cpu resources in the system. This overall
		 * system cpu resource is divided among the tasks of
7873
		 * root_task_group and its child task-groups in a fair manner,
D
Dhaval Giani 已提交
7874 7875 7876
		 * based on each entity's (task or task-group's) weight
		 * (se->load.weight).
		 *
7877
		 * In other words, if root_task_group has 10 tasks of weight
D
Dhaval Giani 已提交
7878 7879 7880
		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
		 * then A0's share of the cpu resource is:
		 *
7881
		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
D
Dhaval Giani 已提交
7882
		 *
7883 7884
		 * We achieve this by letting root_task_group's tasks sit
		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
D
Dhaval Giani 已提交
7885
		 */
7886
		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
D
Dhaval Giani 已提交
7887 7888 7889
#endif /* CONFIG_FAIR_GROUP_SCHED */

		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
7890
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
7891
		INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
7892
		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
I
Ingo Molnar 已提交
7893
#endif
L
Linus Torvalds 已提交
7894

I
Ingo Molnar 已提交
7895 7896
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
7897 7898 7899

		rq->last_load_update_tick = jiffies;

L
Linus Torvalds 已提交
7900
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
7901
		rq->sd = NULL;
G
Gregory Haskins 已提交
7902
		rq->rd = NULL;
7903
		rq->cpu_power = SCHED_LOAD_SCALE;
7904
		rq->post_schedule = 0;
L
Linus Torvalds 已提交
7905
		rq->active_balance = 0;
I
Ingo Molnar 已提交
7906
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
7907
		rq->push_cpu = 0;
7908
		rq->cpu = i;
7909
		rq->online = 0;
7910 7911
		rq->idle_stamp = 0;
		rq->avg_idle = 2*sysctl_sched_migration_cost;
7912
		rq_attach_root(rq, &def_root_domain);
7913 7914 7915 7916
#ifdef CONFIG_NO_HZ
		rq->nohz_balance_kick = 0;
		init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i));
#endif
L
Linus Torvalds 已提交
7917
#endif
P
Peter Zijlstra 已提交
7918
		init_rq_hrtick(rq);
L
Linus Torvalds 已提交
7919 7920 7921
		atomic_set(&rq->nr_iowait, 0);
	}

7922
	set_load_weight(&init_task);
7923

7924 7925 7926 7927
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

7928
#ifdef CONFIG_SMP
7929
	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
7930 7931
#endif

7932
#ifdef CONFIG_RT_MUTEXES
7933
	plist_head_init_raw(&init_task.pi_waiters, &init_task.pi_lock);
7934 7935
#endif

L
Linus Torvalds 已提交
7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
7949 7950 7951

	calc_load_update = jiffies + LOAD_FREQ;

I
Ingo Molnar 已提交
7952 7953 7954 7955
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
7956

7957
	/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
7958
	zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
7959
#ifdef CONFIG_SMP
7960
	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
7961
#ifdef CONFIG_NO_HZ
7962 7963 7964 7965 7966
	zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
	alloc_cpumask_var(&nohz.grp_idle_mask, GFP_NOWAIT);
	atomic_set(&nohz.load_balancer, nr_cpu_ids);
	atomic_set(&nohz.first_pick_cpu, nr_cpu_ids);
	atomic_set(&nohz.second_pick_cpu, nr_cpu_ids);
7967
#endif
R
Rusty Russell 已提交
7968 7969 7970
	/* May be allocated at isolcpus cmdline parse time */
	if (cpu_isolated_map == NULL)
		zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
7971
#endif /* SMP */
7972

7973
	scheduler_running = 1;
L
Linus Torvalds 已提交
7974 7975 7976
}

#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
7977 7978
static inline int preempt_count_equals(int preempt_offset)
{
7979
	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
7980

A
Arnd Bergmann 已提交
7981
	return (nested == preempt_offset);
7982 7983
}

7984
void __might_sleep(const char *file, int line, int preempt_offset)
L
Linus Torvalds 已提交
7985
{
7986
#ifdef in_atomic
L
Linus Torvalds 已提交
7987 7988
	static unsigned long prev_jiffy;	/* ratelimiting */

7989 7990
	if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
	    system_state != SYSTEM_RUNNING || oops_in_progress)
I
Ingo Molnar 已提交
7991 7992 7993 7994 7995
		return;
	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
		return;
	prev_jiffy = jiffies;

P
Peter Zijlstra 已提交
7996 7997 7998 7999 8000 8001 8002
	printk(KERN_ERR
		"BUG: sleeping function called from invalid context at %s:%d\n",
			file, line);
	printk(KERN_ERR
		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
			in_atomic(), irqs_disabled(),
			current->pid, current->comm);
I
Ingo Molnar 已提交
8003 8004 8005 8006 8007

	debug_show_held_locks(current);
	if (irqs_disabled())
		print_irqtrace_events(current);
	dump_stack();
L
Linus Torvalds 已提交
8008 8009 8010 8011 8012 8013
#endif
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
8014 8015
static void normalize_task(struct rq *rq, struct task_struct *p)
{
P
Peter Zijlstra 已提交
8016 8017
	const struct sched_class *prev_class = p->sched_class;
	int old_prio = p->prio;
8018
	int on_rq;
8019

P
Peter Zijlstra 已提交
8020
	on_rq = p->on_rq;
8021 8022 8023 8024 8025 8026 8027
	if (on_rq)
		deactivate_task(rq, p, 0);
	__setscheduler(rq, p, SCHED_NORMAL, 0);
	if (on_rq) {
		activate_task(rq, p, 0);
		resched_task(rq->curr);
	}
P
Peter Zijlstra 已提交
8028 8029

	check_class_changed(rq, p, prev_class, old_prio);
8030 8031
}

L
Linus Torvalds 已提交
8032 8033
void normalize_rt_tasks(void)
{
8034
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
8035
	unsigned long flags;
8036
	struct rq *rq;
L
Linus Torvalds 已提交
8037

8038
	read_lock_irqsave(&tasklist_lock, flags);
8039
	do_each_thread(g, p) {
8040 8041 8042 8043 8044 8045
		/*
		 * Only normalize user tasks:
		 */
		if (!p->mm)
			continue;

I
Ingo Molnar 已提交
8046 8047
		p->se.exec_start		= 0;
#ifdef CONFIG_SCHEDSTATS
8048 8049 8050
		p->se.statistics.wait_start	= 0;
		p->se.statistics.sleep_start	= 0;
		p->se.statistics.block_start	= 0;
I
Ingo Molnar 已提交
8051
#endif
I
Ingo Molnar 已提交
8052 8053 8054 8055 8056 8057 8058 8059

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
8060
			continue;
I
Ingo Molnar 已提交
8061
		}
L
Linus Torvalds 已提交
8062

8063
		raw_spin_lock(&p->pi_lock);
8064
		rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
8065

8066
		normalize_task(rq, p);
8067

8068
		__task_rq_unlock(rq);
8069
		raw_spin_unlock(&p->pi_lock);
8070 8071
	} while_each_thread(g, p);

8072
	read_unlock_irqrestore(&tasklist_lock, flags);
L
Linus Torvalds 已提交
8073 8074 8075
}

#endif /* CONFIG_MAGIC_SYSRQ */
8076

8077
#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
8078
/*
8079
 * These functions are only useful for the IA64 MCA handling, or kdb.
8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
8094
struct task_struct *curr_task(int cpu)
8095 8096 8097 8098
{
	return cpu_curr(cpu);
}

8099 8100 8101
#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */

#ifdef CONFIG_IA64
8102 8103 8104 8105 8106 8107
/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
I
Ingo Molnar 已提交
8108 8109
 * are serviced on a separate stack. It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner. This function
8110 8111 8112 8113 8114 8115 8116
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
8117
void set_curr_task(int cpu, struct task_struct *p)
8118 8119 8120 8121 8122
{
	cpu_curr(cpu) = p;
}

#endif
S
Srivatsa Vaddagiri 已提交
8123

8124 8125
#ifdef CONFIG_FAIR_GROUP_SCHED
static void free_fair_sched_group(struct task_group *tg)
P
Peter Zijlstra 已提交
8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139
{
	int i;

	for_each_possible_cpu(i) {
		if (tg->cfs_rq)
			kfree(tg->cfs_rq[i]);
		if (tg->se)
			kfree(tg->se[i]);
	}

	kfree(tg->cfs_rq);
	kfree(tg->se);
}

8140 8141
static
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
S
Srivatsa Vaddagiri 已提交
8142 8143
{
	struct cfs_rq *cfs_rq;
8144
	struct sched_entity *se;
S
Srivatsa Vaddagiri 已提交
8145 8146
	int i;

8147
	tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
8148 8149
	if (!tg->cfs_rq)
		goto err;
8150
	tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
8151 8152
	if (!tg->se)
		goto err;
8153 8154

	tg->shares = NICE_0_LOAD;
S
Srivatsa Vaddagiri 已提交
8155 8156

	for_each_possible_cpu(i) {
8157 8158
		cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
				      GFP_KERNEL, cpu_to_node(i));
S
Srivatsa Vaddagiri 已提交
8159 8160 8161
		if (!cfs_rq)
			goto err;

8162 8163
		se = kzalloc_node(sizeof(struct sched_entity),
				  GFP_KERNEL, cpu_to_node(i));
S
Srivatsa Vaddagiri 已提交
8164
		if (!se)
8165
			goto err_free_rq;
S
Srivatsa Vaddagiri 已提交
8166

8167
		init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
8168 8169 8170 8171
	}

	return 1;

P
Peter Zijlstra 已提交
8172
err_free_rq:
8173
	kfree(cfs_rq);
P
Peter Zijlstra 已提交
8174
err:
8175 8176 8177 8178 8179
	return 0;
}

static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
{
8180 8181 8182 8183 8184 8185 8186 8187 8188 8189 8190
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	/*
	* Only empty task groups can be destroyed; so we can speculatively
	* check on_list without danger of it being re-added.
	*/
	if (!tg->cfs_rq[cpu]->on_list)
		return;

	raw_spin_lock_irqsave(&rq->lock, flags);
8191
	list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
8192
	raw_spin_unlock_irqrestore(&rq->lock, flags);
8193
}
8194
#else /* !CONFG_FAIR_GROUP_SCHED */
8195 8196 8197 8198
static inline void free_fair_sched_group(struct task_group *tg)
{
}

8199 8200
static inline
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8201 8202 8203 8204 8205 8206 8207
{
	return 1;
}

static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
{
}
8208
#endif /* CONFIG_FAIR_GROUP_SCHED */
8209 8210

#ifdef CONFIG_RT_GROUP_SCHED
8211 8212 8213 8214
static void free_rt_sched_group(struct task_group *tg)
{
	int i;

8215 8216
	destroy_rt_bandwidth(&tg->rt_bandwidth);

8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227
	for_each_possible_cpu(i) {
		if (tg->rt_rq)
			kfree(tg->rt_rq[i]);
		if (tg->rt_se)
			kfree(tg->rt_se[i]);
	}

	kfree(tg->rt_rq);
	kfree(tg->rt_se);
}

8228 8229
static
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8230 8231
{
	struct rt_rq *rt_rq;
8232
	struct sched_rt_entity *rt_se;
8233 8234
	int i;

8235
	tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
8236 8237
	if (!tg->rt_rq)
		goto err;
8238
	tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
8239 8240 8241
	if (!tg->rt_se)
		goto err;

8242 8243
	init_rt_bandwidth(&tg->rt_bandwidth,
			ktime_to_ns(def_rt_bandwidth.rt_period), 0);
8244 8245

	for_each_possible_cpu(i) {
8246 8247
		rt_rq = kzalloc_node(sizeof(struct rt_rq),
				     GFP_KERNEL, cpu_to_node(i));
P
Peter Zijlstra 已提交
8248 8249
		if (!rt_rq)
			goto err;
S
Srivatsa Vaddagiri 已提交
8250

8251 8252
		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
				     GFP_KERNEL, cpu_to_node(i));
P
Peter Zijlstra 已提交
8253
		if (!rt_se)
8254
			goto err_free_rq;
S
Srivatsa Vaddagiri 已提交
8255

8256
		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
S
Srivatsa Vaddagiri 已提交
8257 8258
	}

8259 8260
	return 1;

P
Peter Zijlstra 已提交
8261
err_free_rq:
8262
	kfree(rt_rq);
P
Peter Zijlstra 已提交
8263
err:
8264 8265
	return 0;
}
8266
#else /* !CONFIG_RT_GROUP_SCHED */
8267 8268 8269 8270
static inline void free_rt_sched_group(struct task_group *tg)
{
}

8271 8272
static inline
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8273 8274 8275
{
	return 1;
}
8276
#endif /* CONFIG_RT_GROUP_SCHED */
8277

D
Dhaval Giani 已提交
8278
#ifdef CONFIG_CGROUP_SCHED
8279 8280 8281 8282
static void free_sched_group(struct task_group *tg)
{
	free_fair_sched_group(tg);
	free_rt_sched_group(tg);
8283
	autogroup_free(tg);
8284 8285 8286 8287
	kfree(tg);
}

/* allocate runqueue etc for a new task group */
8288
struct task_group *sched_create_group(struct task_group *parent)
8289 8290 8291 8292 8293 8294 8295 8296
{
	struct task_group *tg;
	unsigned long flags;

	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
	if (!tg)
		return ERR_PTR(-ENOMEM);

8297
	if (!alloc_fair_sched_group(tg, parent))
8298 8299
		goto err;

8300
	if (!alloc_rt_sched_group(tg, parent))
8301 8302
		goto err;

8303
	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
8304
	list_add_rcu(&tg->list, &task_groups);
P
Peter Zijlstra 已提交
8305 8306 8307 8308 8309

	WARN_ON(!parent); /* root should already exist */

	tg->parent = parent;
	INIT_LIST_HEAD(&tg->children);
8310
	list_add_rcu(&tg->siblings, &parent->children);
8311
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
8312

8313
	return tg;
S
Srivatsa Vaddagiri 已提交
8314 8315

err:
P
Peter Zijlstra 已提交
8316
	free_sched_group(tg);
S
Srivatsa Vaddagiri 已提交
8317 8318 8319
	return ERR_PTR(-ENOMEM);
}

8320
/* rcu callback to free various structures associated with a task group */
P
Peter Zijlstra 已提交
8321
static void free_sched_group_rcu(struct rcu_head *rhp)
S
Srivatsa Vaddagiri 已提交
8322 8323
{
	/* now it should be safe to free those cfs_rqs */
P
Peter Zijlstra 已提交
8324
	free_sched_group(container_of(rhp, struct task_group, rcu));
S
Srivatsa Vaddagiri 已提交
8325 8326
}

8327
/* Destroy runqueue etc associated with a task group */
8328
void sched_destroy_group(struct task_group *tg)
S
Srivatsa Vaddagiri 已提交
8329
{
8330
	unsigned long flags;
8331
	int i;
S
Srivatsa Vaddagiri 已提交
8332

8333 8334
	/* end participation in shares distribution */
	for_each_possible_cpu(i)
8335
		unregister_fair_sched_group(tg, i);
8336 8337

	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
8338
	list_del_rcu(&tg->list);
P
Peter Zijlstra 已提交
8339
	list_del_rcu(&tg->siblings);
8340
	spin_unlock_irqrestore(&task_group_lock, flags);
8341 8342

	/* wait for possible concurrent references to cfs_rqs complete */
P
Peter Zijlstra 已提交
8343
	call_rcu(&tg->rcu, free_sched_group_rcu);
S
Srivatsa Vaddagiri 已提交
8344 8345
}

8346
/* change task's runqueue when it moves between groups.
I
Ingo Molnar 已提交
8347 8348 8349
 *	The caller of this function should have put the task in its new group
 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
 *	reflect its new group.
8350 8351
 */
void sched_move_task(struct task_struct *tsk)
S
Srivatsa Vaddagiri 已提交
8352 8353 8354 8355 8356 8357 8358
{
	int on_rq, running;
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

8359
	running = task_current(rq, tsk);
P
Peter Zijlstra 已提交
8360
	on_rq = tsk->on_rq;
S
Srivatsa Vaddagiri 已提交
8361

8362
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
8363
		dequeue_task(rq, tsk, 0);
8364 8365
	if (unlikely(running))
		tsk->sched_class->put_prev_task(rq, tsk);
S
Srivatsa Vaddagiri 已提交
8366

P
Peter Zijlstra 已提交
8367
#ifdef CONFIG_FAIR_GROUP_SCHED
8368 8369 8370
	if (tsk->sched_class->task_move_group)
		tsk->sched_class->task_move_group(tsk, on_rq);
	else
P
Peter Zijlstra 已提交
8371
#endif
8372
		set_task_rq(tsk, task_cpu(tsk));
P
Peter Zijlstra 已提交
8373

8374 8375 8376
	if (unlikely(running))
		tsk->sched_class->set_curr_task(rq);
	if (on_rq)
8377
		enqueue_task(rq, tsk, 0);
S
Srivatsa Vaddagiri 已提交
8378

8379
	task_rq_unlock(rq, tsk, &flags);
S
Srivatsa Vaddagiri 已提交
8380
}
D
Dhaval Giani 已提交
8381
#endif /* CONFIG_CGROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
8382

8383
#ifdef CONFIG_FAIR_GROUP_SCHED
8384 8385
static DEFINE_MUTEX(shares_mutex);

8386
int sched_group_set_shares(struct task_group *tg, unsigned long shares)
S
Srivatsa Vaddagiri 已提交
8387 8388
{
	int i;
8389
	unsigned long flags;
8390

8391 8392 8393 8394 8395 8396
	/*
	 * We can't change the weight of the root cgroup.
	 */
	if (!tg->se[0])
		return -EINVAL;

8397 8398
	if (shares < MIN_SHARES)
		shares = MIN_SHARES;
8399 8400
	else if (shares > MAX_SHARES)
		shares = MAX_SHARES;
8401

8402
	mutex_lock(&shares_mutex);
8403
	if (tg->shares == shares)
8404
		goto done;
S
Srivatsa Vaddagiri 已提交
8405

8406
	tg->shares = shares;
8407
	for_each_possible_cpu(i) {
8408 8409 8410 8411 8412 8413 8414
		struct rq *rq = cpu_rq(i);
		struct sched_entity *se;

		se = tg->se[i];
		/* Propagate contribution to hierarchy */
		raw_spin_lock_irqsave(&rq->lock, flags);
		for_each_sched_entity(se)
8415
			update_cfs_shares(group_cfs_rq(se));
8416
		raw_spin_unlock_irqrestore(&rq->lock, flags);
8417
	}
S
Srivatsa Vaddagiri 已提交
8418

8419
done:
8420
	mutex_unlock(&shares_mutex);
8421
	return 0;
S
Srivatsa Vaddagiri 已提交
8422 8423
}

8424 8425 8426 8427
unsigned long sched_group_shares(struct task_group *tg)
{
	return tg->shares;
}
8428
#endif
8429

8430
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8431
/*
P
Peter Zijlstra 已提交
8432
 * Ensure that the real time constraints are schedulable.
P
Peter Zijlstra 已提交
8433
 */
P
Peter Zijlstra 已提交
8434 8435 8436 8437 8438
static DEFINE_MUTEX(rt_constraints_mutex);

static unsigned long to_ratio(u64 period, u64 runtime)
{
	if (runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
8439
		return 1ULL << 20;
P
Peter Zijlstra 已提交
8440

P
Peter Zijlstra 已提交
8441
	return div64_u64(runtime << 20, period);
P
Peter Zijlstra 已提交
8442 8443
}

P
Peter Zijlstra 已提交
8444 8445
/* Must be called with tasklist_lock held */
static inline int tg_has_rt_tasks(struct task_group *tg)
8446
{
P
Peter Zijlstra 已提交
8447
	struct task_struct *g, *p;
8448

P
Peter Zijlstra 已提交
8449 8450 8451 8452
	do_each_thread(g, p) {
		if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
			return 1;
	} while_each_thread(g, p);
8453

P
Peter Zijlstra 已提交
8454 8455
	return 0;
}
8456

P
Peter Zijlstra 已提交
8457 8458 8459 8460 8461
struct rt_schedulable_data {
	struct task_group *tg;
	u64 rt_period;
	u64 rt_runtime;
};
8462

P
Peter Zijlstra 已提交
8463 8464 8465 8466 8467 8468
static int tg_schedulable(struct task_group *tg, void *data)
{
	struct rt_schedulable_data *d = data;
	struct task_group *child;
	unsigned long total, sum = 0;
	u64 period, runtime;
8469

P
Peter Zijlstra 已提交
8470 8471
	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	runtime = tg->rt_bandwidth.rt_runtime;
8472

P
Peter Zijlstra 已提交
8473 8474 8475
	if (tg == d->tg) {
		period = d->rt_period;
		runtime = d->rt_runtime;
8476 8477
	}

8478 8479 8480 8481 8482
	/*
	 * Cannot have more runtime than the period.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
P
Peter Zijlstra 已提交
8483

8484 8485 8486
	/*
	 * Ensure we don't starve existing RT tasks.
	 */
P
Peter Zijlstra 已提交
8487 8488
	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
		return -EBUSY;
P
Peter Zijlstra 已提交
8489

P
Peter Zijlstra 已提交
8490
	total = to_ratio(period, runtime);
P
Peter Zijlstra 已提交
8491

8492 8493 8494 8495 8496
	/*
	 * Nobody can have more than the global setting allows.
	 */
	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
		return -EINVAL;
P
Peter Zijlstra 已提交
8497

8498 8499 8500
	/*
	 * The sum of our children's runtime should not exceed our own.
	 */
P
Peter Zijlstra 已提交
8501 8502 8503
	list_for_each_entry_rcu(child, &tg->children, siblings) {
		period = ktime_to_ns(child->rt_bandwidth.rt_period);
		runtime = child->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
8504

P
Peter Zijlstra 已提交
8505 8506 8507 8508
		if (child == d->tg) {
			period = d->rt_period;
			runtime = d->rt_runtime;
		}
P
Peter Zijlstra 已提交
8509

P
Peter Zijlstra 已提交
8510
		sum += to_ratio(period, runtime);
P
Peter Zijlstra 已提交
8511
	}
P
Peter Zijlstra 已提交
8512

P
Peter Zijlstra 已提交
8513 8514 8515 8516
	if (sum > total)
		return -EINVAL;

	return 0;
P
Peter Zijlstra 已提交
8517 8518
}

P
Peter Zijlstra 已提交
8519
static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
8520
{
P
Peter Zijlstra 已提交
8521 8522 8523 8524 8525 8526 8527
	struct rt_schedulable_data data = {
		.tg = tg,
		.rt_period = period,
		.rt_runtime = runtime,
	};

	return walk_tg_tree(tg_schedulable, tg_nop, &data);
8528 8529
}

8530 8531
static int tg_set_bandwidth(struct task_group *tg,
		u64 rt_period, u64 rt_runtime)
P
Peter Zijlstra 已提交
8532
{
P
Peter Zijlstra 已提交
8533
	int i, err = 0;
P
Peter Zijlstra 已提交
8534 8535

	mutex_lock(&rt_constraints_mutex);
8536
	read_lock(&tasklist_lock);
P
Peter Zijlstra 已提交
8537 8538
	err = __rt_schedulable(tg, rt_period, rt_runtime);
	if (err)
P
Peter Zijlstra 已提交
8539
		goto unlock;
P
Peter Zijlstra 已提交
8540

8541
	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
8542 8543
	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
	tg->rt_bandwidth.rt_runtime = rt_runtime;
P
Peter Zijlstra 已提交
8544 8545 8546 8547

	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = tg->rt_rq[i];

8548
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
8549
		rt_rq->rt_runtime = rt_runtime;
8550
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
8551
	}
8552
	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
P
Peter Zijlstra 已提交
8553
unlock:
8554
	read_unlock(&tasklist_lock);
P
Peter Zijlstra 已提交
8555 8556 8557
	mutex_unlock(&rt_constraints_mutex);

	return err;
P
Peter Zijlstra 已提交
8558 8559
}

8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571
int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
{
	u64 rt_runtime, rt_period;

	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
	if (rt_runtime_us < 0)
		rt_runtime = RUNTIME_INF;

	return tg_set_bandwidth(tg, rt_period, rt_runtime);
}

P
Peter Zijlstra 已提交
8572 8573 8574 8575
long sched_group_rt_runtime(struct task_group *tg)
{
	u64 rt_runtime_us;

8576
	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
8577 8578
		return -1;

8579
	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
8580 8581 8582
	do_div(rt_runtime_us, NSEC_PER_USEC);
	return rt_runtime_us;
}
8583 8584 8585 8586 8587 8588 8589 8590

int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
{
	u64 rt_runtime, rt_period;

	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
	rt_runtime = tg->rt_bandwidth.rt_runtime;

8591 8592 8593
	if (rt_period == 0)
		return -EINVAL;

8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607
	return tg_set_bandwidth(tg, rt_period, rt_runtime);
}

long sched_group_rt_period(struct task_group *tg)
{
	u64 rt_period_us;

	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
	do_div(rt_period_us, NSEC_PER_USEC);
	return rt_period_us;
}

static int sched_rt_global_constraints(void)
{
8608
	u64 runtime, period;
8609 8610
	int ret = 0;

8611 8612 8613
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

8614 8615 8616 8617 8618 8619 8620 8621
	runtime = global_rt_runtime();
	period = global_rt_period();

	/*
	 * Sanity check on the sysctl variables.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
8622

8623
	mutex_lock(&rt_constraints_mutex);
P
Peter Zijlstra 已提交
8624
	read_lock(&tasklist_lock);
8625
	ret = __rt_schedulable(NULL, 0, 0);
P
Peter Zijlstra 已提交
8626
	read_unlock(&tasklist_lock);
8627 8628 8629 8630
	mutex_unlock(&rt_constraints_mutex);

	return ret;
}
8631 8632 8633 8634 8635 8636 8637 8638 8639 8640

int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
{
	/* Don't accept realtime tasks when there is no way for them to run */
	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
		return 0;

	return 1;
}

8641
#else /* !CONFIG_RT_GROUP_SCHED */
8642 8643
static int sched_rt_global_constraints(void)
{
P
Peter Zijlstra 已提交
8644 8645 8646
	unsigned long flags;
	int i;

8647 8648 8649
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

8650 8651 8652 8653 8654 8655 8656
	/*
	 * There's always some RT tasks in the root group
	 * -- migration, kstopmachine etc..
	 */
	if (sysctl_sched_rt_runtime == 0)
		return -EBUSY;

8657
	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
8658 8659 8660
	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = &cpu_rq(i)->rt;

8661
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
8662
		rt_rq->rt_runtime = global_rt_runtime();
8663
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
8664
	}
8665
	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
8666

8667 8668
	return 0;
}
8669
#endif /* CONFIG_RT_GROUP_SCHED */
8670 8671

int sched_rt_handler(struct ctl_table *table, int write,
8672
		void __user *buffer, size_t *lenp,
8673 8674 8675 8676 8677 8678 8679 8680 8681 8682
		loff_t *ppos)
{
	int ret;
	int old_period, old_runtime;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	old_period = sysctl_sched_rt_period;
	old_runtime = sysctl_sched_rt_runtime;

8683
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699

	if (!ret && write) {
		ret = sched_rt_global_constraints();
		if (ret) {
			sysctl_sched_rt_period = old_period;
			sysctl_sched_rt_runtime = old_runtime;
		} else {
			def_rt_bandwidth.rt_runtime = global_rt_runtime();
			def_rt_bandwidth.rt_period =
				ns_to_ktime(global_rt_period());
		}
	}
	mutex_unlock(&mutex);

	return ret;
}
8700

8701
#ifdef CONFIG_CGROUP_SCHED
8702 8703

/* return corresponding task_group object of a cgroup */
8704
static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
8705
{
8706 8707
	return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
			    struct task_group, css);
8708 8709 8710
}

static struct cgroup_subsys_state *
8711
cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
8712
{
8713
	struct task_group *tg, *parent;
8714

8715
	if (!cgrp->parent) {
8716
		/* This is early initialization for the top cgroup */
8717
		return &root_task_group.css;
8718 8719
	}

8720 8721
	parent = cgroup_tg(cgrp->parent);
	tg = sched_create_group(parent);
8722 8723 8724 8725 8726 8727
	if (IS_ERR(tg))
		return ERR_PTR(-ENOMEM);

	return &tg->css;
}

I
Ingo Molnar 已提交
8728 8729
static void
cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
8730
{
8731
	struct task_group *tg = cgroup_tg(cgrp);
8732 8733 8734 8735

	sched_destroy_group(tg);
}

I
Ingo Molnar 已提交
8736
static int
8737
cpu_cgroup_can_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
8738
{
8739
#ifdef CONFIG_RT_GROUP_SCHED
8740
	if (!sched_rt_can_attach(cgroup_tg(cgrp), tsk))
8741 8742
		return -EINVAL;
#else
8743 8744 8745
	/* We don't support RT-tasks being in separate groups */
	if (tsk->sched_class != &fair_sched_class)
		return -EINVAL;
8746
#endif
8747 8748
	return 0;
}
8749

8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768
static int
cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
		      struct task_struct *tsk, bool threadgroup)
{
	int retval = cpu_cgroup_can_attach_task(cgrp, tsk);
	if (retval)
		return retval;
	if (threadgroup) {
		struct task_struct *c;
		rcu_read_lock();
		list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
			retval = cpu_cgroup_can_attach_task(cgrp, c);
			if (retval) {
				rcu_read_unlock();
				return retval;
			}
		}
		rcu_read_unlock();
	}
8769 8770 8771 8772
	return 0;
}

static void
8773
cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
8774 8775
		  struct cgroup *old_cont, struct task_struct *tsk,
		  bool threadgroup)
8776 8777
{
	sched_move_task(tsk);
8778 8779 8780 8781 8782 8783 8784 8785
	if (threadgroup) {
		struct task_struct *c;
		rcu_read_lock();
		list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
			sched_move_task(c);
		}
		rcu_read_unlock();
	}
8786 8787
}

8788
static void
8789 8790
cpu_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp,
		struct cgroup *old_cgrp, struct task_struct *task)
8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802
{
	/*
	 * cgroup_exit() is called in the copy_process() failure path.
	 * Ignore this case since the task hasn't ran yet, this avoids
	 * trying to poke a half freed task state from generic code.
	 */
	if (!(task->flags & PF_EXITING))
		return;

	sched_move_task(task);
}

8803
#ifdef CONFIG_FAIR_GROUP_SCHED
8804
static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
8805
				u64 shareval)
8806
{
8807
	return sched_group_set_shares(cgroup_tg(cgrp), shareval);
8808 8809
}

8810
static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
8811
{
8812
	struct task_group *tg = cgroup_tg(cgrp);
8813 8814 8815

	return (u64) tg->shares;
}
8816
#endif /* CONFIG_FAIR_GROUP_SCHED */
8817

8818
#ifdef CONFIG_RT_GROUP_SCHED
M
Mirco Tischler 已提交
8819
static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
8820
				s64 val)
P
Peter Zijlstra 已提交
8821
{
8822
	return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
P
Peter Zijlstra 已提交
8823 8824
}

8825
static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
P
Peter Zijlstra 已提交
8826
{
8827
	return sched_group_rt_runtime(cgroup_tg(cgrp));
P
Peter Zijlstra 已提交
8828
}
8829 8830 8831 8832 8833 8834 8835 8836 8837 8838 8839

static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
		u64 rt_period_us)
{
	return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
}

static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
{
	return sched_group_rt_period(cgroup_tg(cgrp));
}
8840
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
8841

8842
static struct cftype cpu_files[] = {
8843
#ifdef CONFIG_FAIR_GROUP_SCHED
8844 8845
	{
		.name = "shares",
8846 8847
		.read_u64 = cpu_shares_read_u64,
		.write_u64 = cpu_shares_write_u64,
8848
	},
8849 8850
#endif
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8851
	{
P
Peter Zijlstra 已提交
8852
		.name = "rt_runtime_us",
8853 8854
		.read_s64 = cpu_rt_runtime_read,
		.write_s64 = cpu_rt_runtime_write,
P
Peter Zijlstra 已提交
8855
	},
8856 8857
	{
		.name = "rt_period_us",
8858 8859
		.read_u64 = cpu_rt_period_read_uint,
		.write_u64 = cpu_rt_period_write_uint,
8860
	},
8861
#endif
8862 8863 8864 8865
};

static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
{
8866
	return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
8867 8868 8869
}

struct cgroup_subsys cpu_cgroup_subsys = {
I
Ingo Molnar 已提交
8870 8871 8872 8873 8874
	.name		= "cpu",
	.create		= cpu_cgroup_create,
	.destroy	= cpu_cgroup_destroy,
	.can_attach	= cpu_cgroup_can_attach,
	.attach		= cpu_cgroup_attach,
8875
	.exit		= cpu_cgroup_exit,
I
Ingo Molnar 已提交
8876 8877
	.populate	= cpu_cgroup_populate,
	.subsys_id	= cpu_cgroup_subsys_id,
8878 8879 8880
	.early_init	= 1,
};

8881
#endif	/* CONFIG_CGROUP_SCHED */
8882 8883 8884 8885 8886 8887 8888 8889 8890 8891

#ifdef CONFIG_CGROUP_CPUACCT

/*
 * CPU accounting code for task groups.
 *
 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
 * (balbir@in.ibm.com).
 */

8892
/* track cpu usage of a group of tasks and its child groups */
8893 8894 8895
struct cpuacct {
	struct cgroup_subsys_state css;
	/* cpuusage holds pointer to a u64-type object on every cpu */
8896
	u64 __percpu *cpuusage;
8897
	struct percpu_counter cpustat[CPUACCT_STAT_NSTATS];
8898
	struct cpuacct *parent;
8899 8900 8901 8902 8903
};

struct cgroup_subsys cpuacct_subsys;

/* return cpu accounting group corresponding to this container */
8904
static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
8905
{
8906
	return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918
			    struct cpuacct, css);
}

/* return cpu accounting group to which this task belongs */
static inline struct cpuacct *task_ca(struct task_struct *tsk)
{
	return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
			    struct cpuacct, css);
}

/* create a new cpu accounting group */
static struct cgroup_subsys_state *cpuacct_create(
8919
	struct cgroup_subsys *ss, struct cgroup *cgrp)
8920 8921
{
	struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
8922
	int i;
8923 8924

	if (!ca)
8925
		goto out;
8926 8927

	ca->cpuusage = alloc_percpu(u64);
8928 8929 8930 8931 8932 8933
	if (!ca->cpuusage)
		goto out_free_ca;

	for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
		if (percpu_counter_init(&ca->cpustat[i], 0))
			goto out_free_counters;
8934

8935 8936 8937
	if (cgrp->parent)
		ca->parent = cgroup_ca(cgrp->parent);

8938
	return &ca->css;
8939 8940 8941 8942 8943 8944 8945 8946 8947

out_free_counters:
	while (--i >= 0)
		percpu_counter_destroy(&ca->cpustat[i]);
	free_percpu(ca->cpuusage);
out_free_ca:
	kfree(ca);
out:
	return ERR_PTR(-ENOMEM);
8948 8949 8950
}

/* destroy an existing cpu accounting group */
I
Ingo Molnar 已提交
8951
static void
8952
cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
8953
{
8954
	struct cpuacct *ca = cgroup_ca(cgrp);
8955
	int i;
8956

8957 8958
	for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
		percpu_counter_destroy(&ca->cpustat[i]);
8959 8960 8961 8962
	free_percpu(ca->cpuusage);
	kfree(ca);
}

8963 8964
static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
{
8965
	u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
8966 8967 8968 8969 8970 8971
	u64 data;

#ifndef CONFIG_64BIT
	/*
	 * Take rq->lock to make 64-bit read safe on 32-bit platforms.
	 */
8972
	raw_spin_lock_irq(&cpu_rq(cpu)->lock);
8973
	data = *cpuusage;
8974
	raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
8975 8976 8977 8978 8979 8980 8981 8982 8983
#else
	data = *cpuusage;
#endif

	return data;
}

static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
{
8984
	u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
8985 8986 8987 8988 8989

#ifndef CONFIG_64BIT
	/*
	 * Take rq->lock to make 64-bit write safe on 32-bit platforms.
	 */
8990
	raw_spin_lock_irq(&cpu_rq(cpu)->lock);
8991
	*cpuusage = val;
8992
	raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
8993 8994 8995 8996 8997
#else
	*cpuusage = val;
#endif
}

8998
/* return total cpu usage (in nanoseconds) of a group */
8999
static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
9000
{
9001
	struct cpuacct *ca = cgroup_ca(cgrp);
9002 9003 9004
	u64 totalcpuusage = 0;
	int i;

9005 9006
	for_each_present_cpu(i)
		totalcpuusage += cpuacct_cpuusage_read(ca, i);
9007 9008 9009 9010

	return totalcpuusage;
}

9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022
static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
								u64 reset)
{
	struct cpuacct *ca = cgroup_ca(cgrp);
	int err = 0;
	int i;

	if (reset) {
		err = -EINVAL;
		goto out;
	}

9023 9024
	for_each_present_cpu(i)
		cpuacct_cpuusage_write(ca, i, 0);
9025 9026 9027 9028 9029

out:
	return err;
}

9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044
static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
				   struct seq_file *m)
{
	struct cpuacct *ca = cgroup_ca(cgroup);
	u64 percpu;
	int i;

	for_each_present_cpu(i) {
		percpu = cpuacct_cpuusage_read(ca, i);
		seq_printf(m, "%llu ", (unsigned long long) percpu);
	}
	seq_printf(m, "\n");
	return 0;
}

9045 9046 9047 9048 9049 9050 9051 9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063
static const char *cpuacct_stat_desc[] = {
	[CPUACCT_STAT_USER] = "user",
	[CPUACCT_STAT_SYSTEM] = "system",
};

static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
		struct cgroup_map_cb *cb)
{
	struct cpuacct *ca = cgroup_ca(cgrp);
	int i;

	for (i = 0; i < CPUACCT_STAT_NSTATS; i++) {
		s64 val = percpu_counter_read(&ca->cpustat[i]);
		val = cputime64_to_clock_t(val);
		cb->fill(cb, cpuacct_stat_desc[i], val);
	}
	return 0;
}

9064 9065 9066
static struct cftype files[] = {
	{
		.name = "usage",
9067 9068
		.read_u64 = cpuusage_read,
		.write_u64 = cpuusage_write,
9069
	},
9070 9071 9072 9073
	{
		.name = "usage_percpu",
		.read_seq_string = cpuacct_percpu_seq_read,
	},
9074 9075 9076 9077
	{
		.name = "stat",
		.read_map = cpuacct_stats_show,
	},
9078 9079
};

9080
static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
9081
{
9082
	return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
9083 9084 9085 9086 9087 9088 9089 9090 9091 9092
}

/*
 * charge this task's execution time to its accounting group.
 *
 * called with rq->lock held.
 */
static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
{
	struct cpuacct *ca;
9093
	int cpu;
9094

L
Li Zefan 已提交
9095
	if (unlikely(!cpuacct_subsys.active))
9096 9097
		return;

9098
	cpu = task_cpu(tsk);
9099 9100 9101

	rcu_read_lock();

9102 9103
	ca = task_ca(tsk);

9104
	for (; ca; ca = ca->parent) {
9105
		u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
9106 9107
		*cpuusage += cputime;
	}
9108 9109

	rcu_read_unlock();
9110 9111
}

9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128
/*
 * When CONFIG_VIRT_CPU_ACCOUNTING is enabled one jiffy can be very large
 * in cputime_t units. As a result, cpuacct_update_stats calls
 * percpu_counter_add with values large enough to always overflow the
 * per cpu batch limit causing bad SMP scalability.
 *
 * To fix this we scale percpu_counter_batch by cputime_one_jiffy so we
 * batch the same amount of time with CONFIG_VIRT_CPU_ACCOUNTING disabled
 * and enabled. We cap it at INT_MAX which is the largest allowed batch value.
 */
#ifdef CONFIG_SMP
#define CPUACCT_BATCH	\
	min_t(long, percpu_counter_batch * cputime_one_jiffy, INT_MAX)
#else
#define CPUACCT_BATCH	0
#endif

9129 9130 9131 9132 9133 9134 9135
/*
 * Charge the system/user time to the task's accounting group.
 */
static void cpuacct_update_stats(struct task_struct *tsk,
		enum cpuacct_stat_index idx, cputime_t val)
{
	struct cpuacct *ca;
9136
	int batch = CPUACCT_BATCH;
9137 9138 9139 9140 9141 9142 9143 9144

	if (unlikely(!cpuacct_subsys.active))
		return;

	rcu_read_lock();
	ca = task_ca(tsk);

	do {
9145
		__percpu_counter_add(&ca->cpustat[idx], val, batch);
9146 9147 9148 9149 9150
		ca = ca->parent;
	} while (ca);
	rcu_read_unlock();
}

9151 9152 9153 9154 9155 9156 9157 9158
struct cgroup_subsys cpuacct_subsys = {
	.name = "cpuacct",
	.create = cpuacct_create,
	.destroy = cpuacct_destroy,
	.populate = cpuacct_populate,
	.subsys_id = cpuacct_subsys_id,
};
#endif	/* CONFIG_CGROUP_CPUACCT */
9159