sched.c 231.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
25 26
 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 *              Thomas Gleixner, Mike Kravetz
L
Linus Torvalds 已提交
27 28 29 30 31 32
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
33
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
34 35 36 37
#include <linux/highmem.h>
#include <linux/smp_lock.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
38
#include <linux/capability.h>
L
Linus Torvalds 已提交
39 40
#include <linux/completion.h>
#include <linux/kernel_stat.h>
41
#include <linux/debug_locks.h>
L
Linus Torvalds 已提交
42 43 44
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
45
#include <linux/freezer.h>
46
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
47 48
#include <linux/blkdev.h>
#include <linux/delay.h>
49
#include <linux/pid_namespace.h>
L
Linus Torvalds 已提交
50 51 52 53 54 55 56 57
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
58
#include <linux/proc_fs.h>
L
Linus Torvalds 已提交
59
#include <linux/seq_file.h>
60
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
61 62
#include <linux/syscalls.h>
#include <linux/times.h>
63
#include <linux/tsacct_kern.h>
64
#include <linux/kprobes.h>
65
#include <linux/delayacct.h>
66
#include <linux/reciprocal_div.h>
67
#include <linux/unistd.h>
J
Jens Axboe 已提交
68
#include <linux/pagemap.h>
P
Peter Zijlstra 已提交
69
#include <linux/hrtimer.h>
R
Reynes Philippe 已提交
70
#include <linux/tick.h>
71
#include <linux/bootmem.h>
P
Peter Zijlstra 已提交
72 73
#include <linux/debugfs.h>
#include <linux/ctype.h>
74
#include <linux/ftrace.h>
75
#include <trace/sched.h>
L
Linus Torvalds 已提交
76

77
#include <asm/tlb.h>
78
#include <asm/irq_regs.h>
L
Linus Torvalds 已提交
79

80 81
#include "sched_cpupri.h"

L
Linus Torvalds 已提交
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
101
 * Helpers for converting nanosecond timing to jiffy resolution
L
Linus Torvalds 已提交
102
 */
103
#define NS_TO_JIFFIES(TIME)	((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
L
Linus Torvalds 已提交
104

I
Ingo Molnar 已提交
105 106 107
#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

L
Linus Torvalds 已提交
108 109 110
/*
 * These are the 'tuning knobs' of the scheduler:
 *
D
Dmitry Adamushko 已提交
111
 * default timeslice is 100 msecs (used only for SCHED_RR tasks).
L
Linus Torvalds 已提交
112 113 114
 * Timeslices get refilled after they expire.
 */
#define DEF_TIMESLICE		(100 * HZ / 1000)
115

116 117 118 119 120
/*
 * single value that denotes runtime == period, ie unlimited time.
 */
#define RUNTIME_INF	((u64)~0ULL)

121 122 123 124 125 126
DEFINE_TRACE(sched_wait_task);
DEFINE_TRACE(sched_wakeup);
DEFINE_TRACE(sched_wakeup_new);
DEFINE_TRACE(sched_switch);
DEFINE_TRACE(sched_migrate_task);

127
#ifdef CONFIG_SMP
128 129 130

static void double_rq_lock(struct rq *rq1, struct rq *rq2);

131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
/*
 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 */
static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
{
	return reciprocal_divide(load, sg->reciprocal_cpu_power);
}

/*
 * Each time a sched group cpu_power is changed,
 * we must compute its reciprocal value
 */
static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
{
	sg->__cpu_power += val;
	sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
}
#endif

151 152
static inline int rt_policy(int policy)
{
153
	if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
154 155 156 157 158 159 160 161 162
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

L
Linus Torvalds 已提交
163
/*
I
Ingo Molnar 已提交
164
 * This is the priority-queue data structure of the RT scheduling class:
L
Linus Torvalds 已提交
165
 */
I
Ingo Molnar 已提交
166 167 168 169 170
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

171
struct rt_bandwidth {
I
Ingo Molnar 已提交
172 173 174 175 176
	/* nests inside the rq lock: */
	spinlock_t		rt_runtime_lock;
	ktime_t			rt_period;
	u64			rt_runtime;
	struct hrtimer		rt_period_timer;
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
};

static struct rt_bandwidth def_rt_bandwidth;

static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);

static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
{
	struct rt_bandwidth *rt_b =
		container_of(timer, struct rt_bandwidth, rt_period_timer);
	ktime_t now;
	int overrun;
	int idle = 0;

	for (;;) {
		now = hrtimer_cb_get_time(timer);
		overrun = hrtimer_forward(timer, now, rt_b->rt_period);

		if (!overrun)
			break;

		idle = do_sched_rt_period_timer(rt_b, overrun);
	}

	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
}

static
void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
{
	rt_b->rt_period = ns_to_ktime(period);
	rt_b->rt_runtime = runtime;

P
Peter Zijlstra 已提交
210 211
	spin_lock_init(&rt_b->rt_runtime_lock);

212 213 214 215 216
	hrtimer_init(&rt_b->rt_period_timer,
			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rt_b->rt_period_timer.function = sched_rt_period_timer;
}

217 218 219
static inline int rt_bandwidth_enabled(void)
{
	return sysctl_sched_rt_runtime >= 0;
220 221 222 223 224 225
}

static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	ktime_t now;

226
	if (rt_bandwidth_enabled() && rt_b->rt_runtime == RUNTIME_INF)
227 228 229 230 231 232 233 234 235 236 237 238
		return;

	if (hrtimer_active(&rt_b->rt_period_timer))
		return;

	spin_lock(&rt_b->rt_runtime_lock);
	for (;;) {
		if (hrtimer_active(&rt_b->rt_period_timer))
			break;

		now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
		hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
239 240
		hrtimer_start_expires(&rt_b->rt_period_timer,
				HRTIMER_MODE_ABS);
241 242 243 244 245 246 247 248 249 250 251
	}
	spin_unlock(&rt_b->rt_runtime_lock);
}

#ifdef CONFIG_RT_GROUP_SCHED
static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	hrtimer_cancel(&rt_b->rt_period_timer);
}
#endif

252 253 254 255 256 257
/*
 * sched_domains_mutex serializes calls to arch_init_sched_domains,
 * detach_destroy_domains and partition_sched_domains.
 */
static DEFINE_MUTEX(sched_domains_mutex);

258
#ifdef CONFIG_GROUP_SCHED
S
Srivatsa Vaddagiri 已提交
259

260 261
#include <linux/cgroup.h>

S
Srivatsa Vaddagiri 已提交
262 263
struct cfs_rq;

P
Peter Zijlstra 已提交
264 265
static LIST_HEAD(task_groups);

S
Srivatsa Vaddagiri 已提交
266
/* task group related information */
267
struct task_group {
268
#ifdef CONFIG_CGROUP_SCHED
269 270
	struct cgroup_subsys_state css;
#endif
271

272 273 274 275
#ifdef CONFIG_USER_SCHED
	uid_t uid;
#endif

276
#ifdef CONFIG_FAIR_GROUP_SCHED
S
Srivatsa Vaddagiri 已提交
277 278 279 280 281
	/* schedulable entities of this group on each cpu */
	struct sched_entity **se;
	/* runqueue "owned" by this group on each cpu */
	struct cfs_rq **cfs_rq;
	unsigned long shares;
282 283 284 285 286 287
#endif

#ifdef CONFIG_RT_GROUP_SCHED
	struct sched_rt_entity **rt_se;
	struct rt_rq **rt_rq;

288
	struct rt_bandwidth rt_bandwidth;
289
#endif
290

291
	struct rcu_head rcu;
P
Peter Zijlstra 已提交
292
	struct list_head list;
P
Peter Zijlstra 已提交
293 294 295 296

	struct task_group *parent;
	struct list_head siblings;
	struct list_head children;
S
Srivatsa Vaddagiri 已提交
297 298
};

D
Dhaval Giani 已提交
299
#ifdef CONFIG_USER_SCHED
300

301 302 303 304 305 306
/* Helper function to pass uid information to create_sched_user() */
void set_tg_uid(struct user_struct *user)
{
	user->tg->uid = user->uid;
}

307 308 309 310 311 312 313
/*
 * Root task group.
 * 	Every UID task group (including init_task_group aka UID-0) will
 * 	be a child to this group.
 */
struct task_group root_task_group;

314
#ifdef CONFIG_FAIR_GROUP_SCHED
S
Srivatsa Vaddagiri 已提交
315 316 317 318
/* Default task group's sched entity on each cpu */
static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
/* Default task group's cfs_rq on each cpu */
static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
319
#endif /* CONFIG_FAIR_GROUP_SCHED */
320 321 322 323

#ifdef CONFIG_RT_GROUP_SCHED
static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
324
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
325
#else /* !CONFIG_USER_SCHED */
326
#define root_task_group init_task_group
P
Peter Zijlstra 已提交
327
#endif /* CONFIG_USER_SCHED */
P
Peter Zijlstra 已提交
328

329
/* task_group_lock serializes add/remove of task groups and also changes to
330 331
 * a task group's cpu shares.
 */
332
static DEFINE_SPINLOCK(task_group_lock);
333

334 335 336
#ifdef CONFIG_FAIR_GROUP_SCHED
#ifdef CONFIG_USER_SCHED
# define INIT_TASK_GROUP_LOAD	(2*NICE_0_LOAD)
337
#else /* !CONFIG_USER_SCHED */
338
# define INIT_TASK_GROUP_LOAD	NICE_0_LOAD
339
#endif /* CONFIG_USER_SCHED */
340

341
/*
342 343 344 345
 * A weight of 0 or 1 can cause arithmetics problems.
 * A weight of a cfs_rq is the sum of weights of which entities
 * are queued on this cfs_rq, so a weight of a entity should not be
 * too large, so as the shares value of a task group.
346 347 348
 * (The default weight is 1024 - so there's no practical
 *  limitation from this.)
 */
349
#define MIN_SHARES	2
350
#define MAX_SHARES	(1UL << 18)
351

352 353 354
static int init_task_group_load = INIT_TASK_GROUP_LOAD;
#endif

S
Srivatsa Vaddagiri 已提交
355
/* Default task group.
I
Ingo Molnar 已提交
356
 *	Every task in system belong to this group at bootup.
S
Srivatsa Vaddagiri 已提交
357
 */
358
struct task_group init_task_group;
S
Srivatsa Vaddagiri 已提交
359 360

/* return group to which a task belongs */
361
static inline struct task_group *task_group(struct task_struct *p)
S
Srivatsa Vaddagiri 已提交
362
{
363
	struct task_group *tg;
364

365
#ifdef CONFIG_USER_SCHED
366 367 368
	rcu_read_lock();
	tg = __task_cred(p)->user->tg;
	rcu_read_unlock();
369
#elif defined(CONFIG_CGROUP_SCHED)
370 371
	tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
				struct task_group, css);
372
#else
I
Ingo Molnar 已提交
373
	tg = &init_task_group;
374
#endif
375
	return tg;
S
Srivatsa Vaddagiri 已提交
376 377 378
}

/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
P
Peter Zijlstra 已提交
379
static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
S
Srivatsa Vaddagiri 已提交
380
{
381
#ifdef CONFIG_FAIR_GROUP_SCHED
382 383
	p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
	p->se.parent = task_group(p)->se[cpu];
384
#endif
P
Peter Zijlstra 已提交
385

386
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
387 388
	p->rt.rt_rq  = task_group(p)->rt_rq[cpu];
	p->rt.parent = task_group(p)->rt_se[cpu];
389
#endif
S
Srivatsa Vaddagiri 已提交
390 391 392 393
}

#else

P
Peter Zijlstra 已提交
394
static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
395 396 397 398
static inline struct task_group *task_group(struct task_struct *p)
{
	return NULL;
}
S
Srivatsa Vaddagiri 已提交
399

400
#endif	/* CONFIG_GROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
401

I
Ingo Molnar 已提交
402 403 404 405 406 407
/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	u64 exec_clock;
I
Ingo Molnar 已提交
408
	u64 min_vruntime;
I
Ingo Molnar 已提交
409 410 411

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;
412 413 414 415 416 417

	struct list_head tasks;
	struct list_head *balance_iterator;

	/*
	 * 'curr' points to currently running entity on this cfs_rq.
I
Ingo Molnar 已提交
418 419
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
P
Peter Zijlstra 已提交
420
	struct sched_entity *curr, *next, *last;
P
Peter Zijlstra 已提交
421

P
Peter Zijlstra 已提交
422
	unsigned int nr_spread_over;
P
Peter Zijlstra 已提交
423

424
#ifdef CONFIG_FAIR_GROUP_SCHED
I
Ingo Molnar 已提交
425 426
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

I
Ingo Molnar 已提交
427 428
	/*
	 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
I
Ingo Molnar 已提交
429 430 431 432 433 434
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
I
Ingo Molnar 已提交
435 436
	struct list_head leaf_cfs_rq_list;
	struct task_group *tg;	/* group that "owns" this runqueue */
437 438 439

#ifdef CONFIG_SMP
	/*
440
	 * the part of load.weight contributed by tasks
441
	 */
442
	unsigned long task_weight;
443

444 445 446 447 448 449 450
	/*
	 *   h_load = weight * f(tg)
	 *
	 * Where f(tg) is the recursive weight fraction assigned to
	 * this group.
	 */
	unsigned long h_load;
451

452 453 454 455
	/*
	 * this cpu's part of tg->shares
	 */
	unsigned long shares;
456 457 458 459 460

	/*
	 * load.weight at the time we set shares
	 */
	unsigned long rq_weight;
461
#endif
I
Ingo Molnar 已提交
462 463
#endif
};
L
Linus Torvalds 已提交
464

I
Ingo Molnar 已提交
465 466 467
/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
468
	unsigned long rt_nr_running;
469
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
470 471
	int highest_prio; /* highest queued rt task prio */
#endif
P
Peter Zijlstra 已提交
472
#ifdef CONFIG_SMP
473
	unsigned long rt_nr_migratory;
G
Gregory Haskins 已提交
474
	int overloaded;
P
Peter Zijlstra 已提交
475
#endif
P
Peter Zijlstra 已提交
476
	int rt_throttled;
P
Peter Zijlstra 已提交
477
	u64 rt_time;
P
Peter Zijlstra 已提交
478
	u64 rt_runtime;
I
Ingo Molnar 已提交
479
	/* Nests inside the rq lock: */
P
Peter Zijlstra 已提交
480
	spinlock_t rt_runtime_lock;
P
Peter Zijlstra 已提交
481

482
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
483 484
	unsigned long rt_nr_boosted;

P
Peter Zijlstra 已提交
485 486 487 488 489
	struct rq *rq;
	struct list_head leaf_rt_rq_list;
	struct task_group *tg;
	struct sched_rt_entity *rt_se;
#endif
I
Ingo Molnar 已提交
490 491
};

G
Gregory Haskins 已提交
492 493 494 495
#ifdef CONFIG_SMP

/*
 * We add the notion of a root-domain which will be used to define per-domain
I
Ingo Molnar 已提交
496 497
 * variables. Each exclusive cpuset essentially defines an island domain by
 * fully partitioning the member cpus from any other cpuset. Whenever a new
G
Gregory Haskins 已提交
498 499 500 501 502 503
 * exclusive cpuset is created, we also create and attach a new root-domain
 * object.
 *
 */
struct root_domain {
	atomic_t refcount;
504 505
	cpumask_var_t span;
	cpumask_var_t online;
506

I
Ingo Molnar 已提交
507
	/*
508 509 510
	 * The "RT overload" flag: it gets set if a CPU has more than
	 * one runnable RT task.
	 */
511
	cpumask_var_t rto_mask;
I
Ingo Molnar 已提交
512
	atomic_t rto_count;
513 514 515
#ifdef CONFIG_SMP
	struct cpupri cpupri;
#endif
516 517 518 519 520 521 522 523
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
	/*
	 * Preferred wake up cpu nominated by sched_mc balance that will be
	 * used when most cpus are idle in the system indicating overall very
	 * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
	 */
	unsigned int sched_mc_preferred_wakeup_cpu;
#endif
G
Gregory Haskins 已提交
524 525
};

526 527 528 529
/*
 * By default the system creates a single root-domain with all cpus as
 * members (mimicking the global state we have today).
 */
G
Gregory Haskins 已提交
530 531 532 533
static struct root_domain def_root_domain;

#endif

L
Linus Torvalds 已提交
534 535 536 537 538 539 540
/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
541
struct rq {
542 543
	/* runqueue lock: */
	spinlock_t lock;
L
Linus Torvalds 已提交
544 545 546 547 548 549

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
I
Ingo Molnar 已提交
550 551
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
552
	unsigned char idle_at_tick;
553
#ifdef CONFIG_NO_HZ
554
	unsigned long last_tick_seen;
555 556
	unsigned char in_nohz_recently;
#endif
557 558
	/* capture load from *all* tasks on this cpu: */
	struct load_weight load;
I
Ingo Molnar 已提交
559 560 561 562
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
P
Peter Zijlstra 已提交
563 564
	struct rt_rq rt;

I
Ingo Molnar 已提交
565
#ifdef CONFIG_FAIR_GROUP_SCHED
566 567
	/* list of leaf cfs_rq on this cpu: */
	struct list_head leaf_cfs_rq_list;
568 569
#endif
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
570
	struct list_head leaf_rt_rq_list;
L
Linus Torvalds 已提交
571 572 573 574 575 576 577 578 579 580
#endif

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

581
	struct task_struct *curr, *idle;
582
	unsigned long next_balance;
L
Linus Torvalds 已提交
583
	struct mm_struct *prev_mm;
I
Ingo Molnar 已提交
584

585
	u64 clock;
I
Ingo Molnar 已提交
586

L
Linus Torvalds 已提交
587 588 589
	atomic_t nr_iowait;

#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
590
	struct root_domain *rd;
L
Linus Torvalds 已提交
591 592 593 594 595
	struct sched_domain *sd;

	/* For active balancing */
	int active_balance;
	int push_cpu;
596 597
	/* cpu of this runqueue: */
	int cpu;
598
	int online;
L
Linus Torvalds 已提交
599

600
	unsigned long avg_load_per_task;
L
Linus Torvalds 已提交
601

602
	struct task_struct *migration_thread;
L
Linus Torvalds 已提交
603 604 605
	struct list_head migration_queue;
#endif

P
Peter Zijlstra 已提交
606
#ifdef CONFIG_SCHED_HRTICK
607 608 609 610
#ifdef CONFIG_SMP
	int hrtick_csd_pending;
	struct call_single_data hrtick_csd;
#endif
P
Peter Zijlstra 已提交
611 612 613
	struct hrtimer hrtick_timer;
#endif

L
Linus Torvalds 已提交
614 615 616
#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;
617 618
	unsigned long long rq_cpu_time;
	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
L
Linus Torvalds 已提交
619 620

	/* sys_sched_yield() stats */
621 622 623 624
	unsigned int yld_exp_empty;
	unsigned int yld_act_empty;
	unsigned int yld_both_empty;
	unsigned int yld_count;
L
Linus Torvalds 已提交
625 626

	/* schedule() stats */
627 628 629
	unsigned int sched_switch;
	unsigned int sched_count;
	unsigned int sched_goidle;
L
Linus Torvalds 已提交
630 631

	/* try_to_wake_up() stats */
632 633
	unsigned int ttwu_count;
	unsigned int ttwu_local;
I
Ingo Molnar 已提交
634 635

	/* BKL stats */
636
	unsigned int bkl_count;
L
Linus Torvalds 已提交
637 638 639
#endif
};

640
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
L
Linus Torvalds 已提交
641

642
static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
I
Ingo Molnar 已提交
643
{
644
	rq->curr->sched_class->check_preempt_curr(rq, p, sync);
I
Ingo Molnar 已提交
645 646
}

647 648 649 650 651 652 653 654 655
static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

N
Nick Piggin 已提交
656 657
/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
658
 * See detach_destroy_domains: synchronize_sched for details.
N
Nick Piggin 已提交
659 660 661 662
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
663 664
#define for_each_domain(cpu, __sd) \
	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
L
Linus Torvalds 已提交
665 666 667 668 669 670

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

671 672 673 674 675
static inline void update_rq_clock(struct rq *rq)
{
	rq->clock = sched_clock_cpu(cpu_of(rq));
}

I
Ingo Molnar 已提交
676 677 678 679 680 681 682 683 684
/*
 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
 */
#ifdef CONFIG_SCHED_DEBUG
# define const_debug __read_mostly
#else
# define const_debug static const
#endif

I
Ingo Molnar 已提交
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
/**
 * runqueue_is_locked
 *
 * Returns true if the current cpu runqueue is locked.
 * This interface allows printk to be called with the runqueue lock
 * held and know whether or not it is OK to wake up the klogd.
 */
int runqueue_is_locked(void)
{
	int cpu = get_cpu();
	struct rq *rq = cpu_rq(cpu);
	int ret;

	ret = spin_is_locked(&rq->lock);
	put_cpu();
	return ret;
}

I
Ingo Molnar 已提交
703 704 705
/*
 * Debugging: various feature bits
 */
P
Peter Zijlstra 已提交
706 707 708 709

#define SCHED_FEAT(name, enabled)	\
	__SCHED_FEAT_##name ,

I
Ingo Molnar 已提交
710
enum {
P
Peter Zijlstra 已提交
711
#include "sched_features.h"
I
Ingo Molnar 已提交
712 713
};

P
Peter Zijlstra 已提交
714 715 716 717 718
#undef SCHED_FEAT

#define SCHED_FEAT(name, enabled)	\
	(1UL << __SCHED_FEAT_##name) * enabled |

I
Ingo Molnar 已提交
719
const_debug unsigned int sysctl_sched_features =
P
Peter Zijlstra 已提交
720 721 722 723 724 725 726 727 728
#include "sched_features.h"
	0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG
#define SCHED_FEAT(name, enabled)	\
	#name ,

729
static __read_mostly char *sched_feat_names[] = {
P
Peter Zijlstra 已提交
730 731 732 733 734 735
#include "sched_features.h"
	NULL
};

#undef SCHED_FEAT

L
Li Zefan 已提交
736
static int sched_feat_show(struct seq_file *m, void *v)
P
Peter Zijlstra 已提交
737 738 739 740
{
	int i;

	for (i = 0; sched_feat_names[i]; i++) {
L
Li Zefan 已提交
741 742 743
		if (!(sysctl_sched_features & (1UL << i)))
			seq_puts(m, "NO_");
		seq_printf(m, "%s ", sched_feat_names[i]);
P
Peter Zijlstra 已提交
744
	}
L
Li Zefan 已提交
745
	seq_puts(m, "\n");
P
Peter Zijlstra 已提交
746

L
Li Zefan 已提交
747
	return 0;
P
Peter Zijlstra 已提交
748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766
}

static ssize_t
sched_feat_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
	char buf[64];
	char *cmp = buf;
	int neg = 0;
	int i;

	if (cnt > 63)
		cnt = 63;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

I
Ingo Molnar 已提交
767
	if (strncmp(buf, "NO_", 3) == 0) {
P
Peter Zijlstra 已提交
768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
		neg = 1;
		cmp += 3;
	}

	for (i = 0; sched_feat_names[i]; i++) {
		int len = strlen(sched_feat_names[i]);

		if (strncmp(cmp, sched_feat_names[i], len) == 0) {
			if (neg)
				sysctl_sched_features &= ~(1UL << i);
			else
				sysctl_sched_features |= (1UL << i);
			break;
		}
	}

	if (!sched_feat_names[i])
		return -EINVAL;

	filp->f_pos += cnt;

	return cnt;
}

L
Li Zefan 已提交
792 793 794 795 796
static int sched_feat_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_feat_show, NULL);
}

P
Peter Zijlstra 已提交
797
static struct file_operations sched_feat_fops = {
L
Li Zefan 已提交
798 799 800 801 802
	.open		= sched_feat_open,
	.write		= sched_feat_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
P
Peter Zijlstra 已提交
803 804 805 806 807 808 809 810 811 812 813 814 815 816
};

static __init int sched_init_debug(void)
{
	debugfs_create_file("sched_features", 0644, NULL, NULL,
			&sched_feat_fops);

	return 0;
}
late_initcall(sched_init_debug);

#endif

#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
I
Ingo Molnar 已提交
817

818 819 820 821 822 823
/*
 * Number of tasks to iterate in a single balance run.
 * Limited because this is done with IRQs disabled.
 */
const_debug unsigned int sysctl_sched_nr_migrate = 32;

P
Peter Zijlstra 已提交
824 825
/*
 * ratelimit for updating the group shares.
826
 * default: 0.25ms
P
Peter Zijlstra 已提交
827
 */
828
unsigned int sysctl_sched_shares_ratelimit = 250000;
P
Peter Zijlstra 已提交
829

830 831 832 833 834 835 836
/*
 * Inject some fuzzyness into changing the per-cpu group shares
 * this avoids remote rq-locks at the expense of fairness.
 * default: 4
 */
unsigned int sysctl_sched_shares_thresh = 4;

P
Peter Zijlstra 已提交
837
/*
P
Peter Zijlstra 已提交
838
 * period over which we measure -rt task cpu usage in us.
P
Peter Zijlstra 已提交
839 840
 * default: 1s
 */
P
Peter Zijlstra 已提交
841
unsigned int sysctl_sched_rt_period = 1000000;
P
Peter Zijlstra 已提交
842

843 844
static __read_mostly int scheduler_running;

P
Peter Zijlstra 已提交
845 846 847 848 849
/*
 * part of the period that we allow rt tasks to run in us.
 * default: 0.95s
 */
int sysctl_sched_rt_runtime = 950000;
P
Peter Zijlstra 已提交
850

851 852 853 854 855 856 857
static inline u64 global_rt_period(void)
{
	return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
}

static inline u64 global_rt_runtime(void)
{
858
	if (sysctl_sched_rt_runtime < 0)
859 860 861 862
		return RUNTIME_INF;

	return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
}
P
Peter Zijlstra 已提交
863

L
Linus Torvalds 已提交
864
#ifndef prepare_arch_switch
865 866 867 868 869 870
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

871 872 873 874 875
static inline int task_current(struct rq *rq, struct task_struct *p)
{
	return rq->curr == p;
}

876
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
877
static inline int task_running(struct rq *rq, struct task_struct *p)
878
{
879
	return task_current(rq, p);
880 881
}

882
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
883 884 885
{
}

886
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
887
{
888 889 890 891
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
892 893 894 895 896 897 898
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

899 900 901 902
	spin_unlock_irq(&rq->lock);
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
903
static inline int task_running(struct rq *rq, struct task_struct *p)
904 905 906 907
{
#ifdef CONFIG_SMP
	return p->oncpu;
#else
908
	return task_current(rq, p);
909 910 911
#endif
}

912
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->oncpu = 1;
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	spin_unlock_irq(&rq->lock);
#else
	spin_unlock(&rq->lock);
#endif
}

929
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
930 931 932 933 934 935 936 937 938 939 940 941
{
#ifdef CONFIG_SMP
	/*
	 * After ->oncpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->oncpu = 0;
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
L
Linus Torvalds 已提交
942
#endif
943 944
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
L
Linus Torvalds 已提交
945

946 947 948 949
/*
 * __task_rq_lock - lock the runqueue a given task resides on.
 * Must be called interrupts disabled.
 */
950
static inline struct rq *__task_rq_lock(struct task_struct *p)
951 952
	__acquires(rq->lock)
{
953 954 955 956 957
	for (;;) {
		struct rq *rq = task_rq(p);
		spin_lock(&rq->lock);
		if (likely(rq == task_rq(p)))
			return rq;
958 959 960 961
		spin_unlock(&rq->lock);
	}
}

L
Linus Torvalds 已提交
962 963
/*
 * task_rq_lock - lock the runqueue a given task resides on and disable
I
Ingo Molnar 已提交
964
 * interrupts. Note the ordering: we can safely lookup the task_rq without
L
Linus Torvalds 已提交
965 966
 * explicitly disabling preemption.
 */
967
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
968 969
	__acquires(rq->lock)
{
970
	struct rq *rq;
L
Linus Torvalds 已提交
971

972 973 974 975 976 977
	for (;;) {
		local_irq_save(*flags);
		rq = task_rq(p);
		spin_lock(&rq->lock);
		if (likely(rq == task_rq(p)))
			return rq;
L
Linus Torvalds 已提交
978 979 980 981
		spin_unlock_irqrestore(&rq->lock, *flags);
	}
}

982 983 984 985 986 987 988 989
void task_rq_unlock_wait(struct task_struct *p)
{
	struct rq *rq = task_rq(p);

	smp_mb(); /* spin-unlock-wait is not a full memory barrier */
	spin_unlock_wait(&rq->lock);
}

A
Alexey Dobriyan 已提交
990
static void __task_rq_unlock(struct rq *rq)
991 992 993 994 995
	__releases(rq->lock)
{
	spin_unlock(&rq->lock);
}

996
static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
L
Linus Torvalds 已提交
997 998 999 1000 1001 1002
	__releases(rq->lock)
{
	spin_unlock_irqrestore(&rq->lock, *flags);
}

/*
1003
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
1004
 */
A
Alexey Dobriyan 已提交
1005
static struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
1006 1007
	__acquires(rq->lock)
{
1008
	struct rq *rq;
L
Linus Torvalds 已提交
1009 1010 1011 1012 1013 1014 1015 1016

	local_irq_disable();
	rq = this_rq();
	spin_lock(&rq->lock);

	return rq;
}

P
Peter Zijlstra 已提交
1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
#ifdef CONFIG_SCHED_HRTICK
/*
 * Use HR-timers to deliver accurate preemption points.
 *
 * Its all a bit involved since we cannot program an hrt while holding the
 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
 * reschedule event.
 *
 * When we get rescheduled we reprogram the hrtick_timer outside of the
 * rq->lock.
 */

/*
 * Use hrtick when:
 *  - enabled by features
 *  - hrtimer is actually high res
 */
static inline int hrtick_enabled(struct rq *rq)
{
	if (!sched_feat(HRTICK))
		return 0;
1038
	if (!cpu_active(cpu_of(rq)))
1039
		return 0;
P
Peter Zijlstra 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	return hrtimer_is_hres_active(&rq->hrtick_timer);
}

static void hrtick_clear(struct rq *rq)
{
	if (hrtimer_active(&rq->hrtick_timer))
		hrtimer_cancel(&rq->hrtick_timer);
}

/*
 * High-resolution timer tick.
 * Runs from hardirq context with interrupts disabled.
 */
static enum hrtimer_restart hrtick(struct hrtimer *timer)
{
	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

	spin_lock(&rq->lock);
1060
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
1061 1062 1063 1064 1065 1066
	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
	spin_unlock(&rq->lock);

	return HRTIMER_NORESTART;
}

1067
#ifdef CONFIG_SMP
1068 1069 1070 1071
/*
 * called from hardirq (IPI) context
 */
static void __hrtick_start(void *arg)
1072
{
1073
	struct rq *rq = arg;
1074

1075 1076 1077 1078
	spin_lock(&rq->lock);
	hrtimer_restart(&rq->hrtick_timer);
	rq->hrtick_csd_pending = 0;
	spin_unlock(&rq->lock);
1079 1080
}

1081 1082 1083 1084 1085 1086
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
1087
{
1088 1089
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
1090

1091
	hrtimer_set_expires(timer, time);
1092 1093 1094 1095 1096 1097 1098

	if (rq == this_rq()) {
		hrtimer_restart(timer);
	} else if (!rq->hrtick_csd_pending) {
		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
		rq->hrtick_csd_pending = 1;
	}
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
}

static int
hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
	int cpu = (int)(long)hcpu;

	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
1113
		hrtick_clear(cpu_rq(cpu));
1114 1115 1116 1117 1118 1119
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

1120
static __init void init_hrtick(void)
1121 1122 1123
{
	hotcpu_notifier(hotplug_hrtick, 0);
}
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
#else
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
{
	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
}
1134

A
Andrew Morton 已提交
1135
static inline void init_hrtick(void)
P
Peter Zijlstra 已提交
1136 1137
{
}
1138
#endif /* CONFIG_SMP */
P
Peter Zijlstra 已提交
1139

1140
static void init_rq_hrtick(struct rq *rq)
P
Peter Zijlstra 已提交
1141
{
1142 1143
#ifdef CONFIG_SMP
	rq->hrtick_csd_pending = 0;
P
Peter Zijlstra 已提交
1144

1145 1146 1147 1148
	rq->hrtick_csd.flags = 0;
	rq->hrtick_csd.func = __hrtick_start;
	rq->hrtick_csd.info = rq;
#endif
P
Peter Zijlstra 已提交
1149

1150 1151
	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rq->hrtick_timer.function = hrtick;
P
Peter Zijlstra 已提交
1152
}
A
Andrew Morton 已提交
1153
#else	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
1154 1155 1156 1157 1158 1159 1160 1161
static inline void hrtick_clear(struct rq *rq)
{
}

static inline void init_rq_hrtick(struct rq *rq)
{
}

1162 1163 1164
static inline void init_hrtick(void)
{
}
A
Andrew Morton 已提交
1165
#endif	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
1166

I
Ingo Molnar 已提交
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

1180
static void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
1181 1182 1183 1184 1185
{
	int cpu;

	assert_spin_locked(&task_rq(p)->lock);

1186
	if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
I
Ingo Molnar 已提交
1187 1188
		return;

1189
	set_tsk_thread_flag(p, TIF_NEED_RESCHED);
I
Ingo Molnar 已提交
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	if (!spin_trylock_irqsave(&rq->lock, flags))
		return;
	resched_task(cpu_curr(cpu));
	spin_unlock_irqrestore(&rq->lock, flags);
}
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251

#ifdef CONFIG_NO_HZ
/*
 * When add_timer_on() enqueues a timer into the timer wheel of an
 * idle CPU then this timer might expire before the next timer event
 * which is scheduled to wake up that CPU. In case of a completely
 * idle system the next event might even be infinite time into the
 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 * leaves the inner idle loop so the newly added timer is taken into
 * account when the CPU goes back to idle and evaluates the timer
 * wheel for the next timer event.
 */
void wake_up_idle_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);

	if (cpu == smp_processor_id())
		return;

	/*
	 * This is safe, as this function is called with the timer
	 * wheel base lock of (cpu) held. When the CPU is on the way
	 * to idle and has not yet set rq->curr to idle then it will
	 * be serialized on the timer wheel base lock and take the new
	 * timer into account automatically.
	 */
	if (rq->curr != rq->idle)
		return;

	/*
	 * We can set TIF_RESCHED on the idle task of the other CPU
	 * lockless. The worst case is that the other CPU runs the
	 * idle task through an additional NOOP schedule()
	 */
	set_tsk_thread_flag(rq->idle, TIF_NEED_RESCHED);

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(rq->idle))
		smp_send_reschedule(cpu);
}
1252
#endif /* CONFIG_NO_HZ */
1253

1254
#else /* !CONFIG_SMP */
1255
static void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
1256 1257
{
	assert_spin_locked(&task_rq(p)->lock);
1258
	set_tsk_need_resched(p);
I
Ingo Molnar 已提交
1259
}
1260
#endif /* CONFIG_SMP */
I
Ingo Molnar 已提交
1261

1262 1263 1264 1265 1266 1267 1268 1269
#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

I
Ingo Molnar 已提交
1270 1271 1272
/*
 * Shift right and round:
 */
I
Ingo Molnar 已提交
1273
#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
I
Ingo Molnar 已提交
1274

1275 1276 1277
/*
 * delta *= weight / lw
 */
1278
static unsigned long
1279 1280 1281 1282 1283
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

1284 1285 1286 1287 1288 1289 1290
	if (!lw->inv_weight) {
		if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
			lw->inv_weight = 1;
		else
			lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
				/ (lw->weight+1);
	}
1291 1292 1293 1294 1295

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
I
Ingo Molnar 已提交
1296
	if (unlikely(tmp > WMULT_CONST))
I
Ingo Molnar 已提交
1297
		tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
I
Ingo Molnar 已提交
1298 1299
			WMULT_SHIFT/2);
	else
I
Ingo Molnar 已提交
1300
		tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
1301

1302
	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
1303 1304
}

1305
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
1306 1307
{
	lw->weight += inc;
I
Ingo Molnar 已提交
1308
	lw->inv_weight = 0;
1309 1310
}

1311
static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
1312 1313
{
	lw->weight -= dec;
I
Ingo Molnar 已提交
1314
	lw->inv_weight = 0;
1315 1316
}

1317 1318 1319 1320
/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
I
Ingo Molnar 已提交
1321
 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
1322 1323 1324 1325
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

I
Ingo Molnar 已提交
1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
#define WEIGHT_IDLEPRIO		2
#define WMULT_IDLEPRIO		(1 << 31)

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
1337 1338 1339
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
I
Ingo Molnar 已提交
1340 1341
 */
static const int prio_to_weight[40] = {
1342 1343 1344 1345 1346 1347 1348 1349
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,
 /*  10 */       110,        87,        70,        56,        45,
 /*  15 */        36,        29,        23,        18,        15,
I
Ingo Molnar 已提交
1350 1351
};

1352 1353 1354 1355 1356 1357 1358
/*
 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
 *
 * In cases where the weight does not change often, we can use the
 * precalculated inverse to speed up arithmetics by turning divisions
 * into multiplications:
 */
I
Ingo Molnar 已提交
1359
static const u32 prio_to_wmult[40] = {
1360 1361 1362 1363 1364 1365 1366 1367
 /* -20 */     48388,     59856,     76040,     92818,    118348,
 /* -15 */    147320,    184698,    229616,    287308,    360437,
 /* -10 */    449829,    563644,    704093,    875809,   1099582,
 /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
 /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
 /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
 /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
 /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
I
Ingo Molnar 已提交
1368
};
1369

I
Ingo Molnar 已提交
1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);

/*
 * runqueue iterator, to support SMP load-balancing between different
 * scheduling classes, without having to expose their internal data
 * structures to the load-balancing proper:
 */
struct rq_iterator {
	void *arg;
	struct task_struct *(*start)(void *);
	struct task_struct *(*next)(void *);
};

1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
#ifdef CONFIG_SMP
static unsigned long
balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
	      unsigned long max_load_move, struct sched_domain *sd,
	      enum cpu_idle_type idle, int *all_pinned,
	      int *this_best_prio, struct rq_iterator *iterator);

static int
iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
		   struct sched_domain *sd, enum cpu_idle_type idle,
		   struct rq_iterator *iterator);
#endif
I
Ingo Molnar 已提交
1395

1396 1397 1398 1399 1400 1401
#ifdef CONFIG_CGROUP_CPUACCT
static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
#else
static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
#endif

1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
static inline void inc_cpu_load(struct rq *rq, unsigned long load)
{
	update_load_add(&rq->load, load);
}

static inline void dec_cpu_load(struct rq *rq, unsigned long load)
{
	update_load_sub(&rq->load, load);
}

I
Ingo Molnar 已提交
1412
#if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
P
Peter Zijlstra 已提交
1413
typedef int (*tg_visitor)(struct task_group *, void *);
1414 1415 1416 1417 1418

/*
 * Iterate the full tree, calling @down when first entering a node and @up when
 * leaving it for the final time.
 */
P
Peter Zijlstra 已提交
1419
static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
1420 1421
{
	struct task_group *parent, *child;
P
Peter Zijlstra 已提交
1422
	int ret;
1423 1424 1425 1426

	rcu_read_lock();
	parent = &root_task_group;
down:
P
Peter Zijlstra 已提交
1427 1428 1429
	ret = (*down)(parent, data);
	if (ret)
		goto out_unlock;
1430 1431 1432 1433 1434 1435 1436
	list_for_each_entry_rcu(child, &parent->children, siblings) {
		parent = child;
		goto down;

up:
		continue;
	}
P
Peter Zijlstra 已提交
1437 1438 1439
	ret = (*up)(parent, data);
	if (ret)
		goto out_unlock;
1440 1441 1442 1443 1444

	child = parent;
	parent = parent->parent;
	if (parent)
		goto up;
P
Peter Zijlstra 已提交
1445
out_unlock:
1446
	rcu_read_unlock();
P
Peter Zijlstra 已提交
1447 1448

	return ret;
1449 1450
}

P
Peter Zijlstra 已提交
1451 1452 1453
static int tg_nop(struct task_group *tg, void *data)
{
	return 0;
1454
}
P
Peter Zijlstra 已提交
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
#endif

#ifdef CONFIG_SMP
static unsigned long source_load(int cpu, int type);
static unsigned long target_load(int cpu, int type);
static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);

static unsigned long cpu_avg_load_per_task(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
1465
	unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
P
Peter Zijlstra 已提交
1466

1467 1468
	if (nr_running)
		rq->avg_load_per_task = rq->load.weight / nr_running;
1469 1470
	else
		rq->avg_load_per_task = 0;
P
Peter Zijlstra 已提交
1471 1472 1473 1474 1475

	return rq->avg_load_per_task;
}

#ifdef CONFIG_FAIR_GROUP_SCHED
1476 1477 1478 1479 1480 1481 1482

static void __set_se_shares(struct sched_entity *se, unsigned long shares);

/*
 * Calculate and set the cpu's group shares.
 */
static void
1483 1484
update_group_shares_cpu(struct task_group *tg, int cpu,
			unsigned long sd_shares, unsigned long sd_rq_weight)
1485
{
1486 1487 1488
	unsigned long shares;
	unsigned long rq_weight;

1489
	if (!tg->se[cpu])
1490 1491
		return;

1492
	rq_weight = tg->cfs_rq[cpu]->rq_weight;
1493

1494 1495 1496 1497 1498 1499
	/*
	 *           \Sum shares * rq_weight
	 * shares =  -----------------------
	 *               \Sum rq_weight
	 *
	 */
1500
	shares = (sd_shares * rq_weight) / sd_rq_weight;
1501
	shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1502

1503 1504 1505 1506
	if (abs(shares - tg->se[cpu]->load.weight) >
			sysctl_sched_shares_thresh) {
		struct rq *rq = cpu_rq(cpu);
		unsigned long flags;
1507

1508
		spin_lock_irqsave(&rq->lock, flags);
1509
		tg->cfs_rq[cpu]->shares = shares;
1510

1511 1512 1513
		__set_se_shares(tg->se[cpu], shares);
		spin_unlock_irqrestore(&rq->lock, flags);
	}
1514
}
1515 1516

/*
1517 1518 1519
 * Re-compute the task group their per cpu shares over the given domain.
 * This needs to be done in a bottom-up fashion because the rq weight of a
 * parent group depends on the shares of its child groups.
1520
 */
P
Peter Zijlstra 已提交
1521
static int tg_shares_up(struct task_group *tg, void *data)
1522
{
1523
	unsigned long weight, rq_weight = 0;
1524
	unsigned long shares = 0;
P
Peter Zijlstra 已提交
1525
	struct sched_domain *sd = data;
1526
	int i;
1527

1528
	for_each_cpu(i, sched_domain_span(sd)) {
1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
		/*
		 * If there are currently no tasks on the cpu pretend there
		 * is one of average load so that when a new task gets to
		 * run here it will not get delayed by group starvation.
		 */
		weight = tg->cfs_rq[i]->load.weight;
		if (!weight)
			weight = NICE_0_LOAD;

		tg->cfs_rq[i]->rq_weight = weight;
		rq_weight += weight;
1540
		shares += tg->cfs_rq[i]->shares;
1541 1542
	}

1543 1544 1545 1546 1547
	if ((!shares && rq_weight) || shares > tg->shares)
		shares = tg->shares;

	if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
		shares = tg->shares;
1548

1549
	for_each_cpu(i, sched_domain_span(sd))
1550
		update_group_shares_cpu(tg, i, shares, rq_weight);
P
Peter Zijlstra 已提交
1551 1552

	return 0;
1553 1554 1555
}

/*
1556 1557 1558
 * Compute the cpu's hierarchical load factor for each task group.
 * This needs to be done in a top-down fashion because the load of a child
 * group is a fraction of its parents load.
1559
 */
P
Peter Zijlstra 已提交
1560
static int tg_load_down(struct task_group *tg, void *data)
1561
{
1562
	unsigned long load;
P
Peter Zijlstra 已提交
1563
	long cpu = (long)data;
1564

1565 1566 1567 1568 1569 1570 1571
	if (!tg->parent) {
		load = cpu_rq(cpu)->load.weight;
	} else {
		load = tg->parent->cfs_rq[cpu]->h_load;
		load *= tg->cfs_rq[cpu]->shares;
		load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
	}
1572

1573
	tg->cfs_rq[cpu]->h_load = load;
1574

P
Peter Zijlstra 已提交
1575
	return 0;
1576 1577
}

1578
static void update_shares(struct sched_domain *sd)
1579
{
P
Peter Zijlstra 已提交
1580 1581 1582 1583 1584
	u64 now = cpu_clock(raw_smp_processor_id());
	s64 elapsed = now - sd->last_update;

	if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
		sd->last_update = now;
P
Peter Zijlstra 已提交
1585
		walk_tg_tree(tg_nop, tg_shares_up, sd);
P
Peter Zijlstra 已提交
1586
	}
1587 1588
}

1589 1590 1591 1592 1593 1594 1595
static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
{
	spin_unlock(&rq->lock);
	update_shares(sd);
	spin_lock(&rq->lock);
}

P
Peter Zijlstra 已提交
1596
static void update_h_load(long cpu)
1597
{
P
Peter Zijlstra 已提交
1598
	walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
1599 1600 1601 1602
}

#else

1603
static inline void update_shares(struct sched_domain *sd)
1604 1605 1606
{
}

1607 1608 1609 1610
static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
{
}

1611 1612
#endif

1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
/*
 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
 */
static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
	int ret = 0;

	if (unlikely(!irqs_disabled())) {
		/* printk() doesn't work good under rq->lock */
		spin_unlock(&this_rq->lock);
		BUG_ON(1);
	}
	if (unlikely(!spin_trylock(&busiest->lock))) {
		if (busiest < this_rq) {
			spin_unlock(&this_rq->lock);
			spin_lock(&busiest->lock);
			spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
			ret = 1;
		} else
			spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
	}
	return ret;
}

static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
	__releases(busiest->lock)
{
	spin_unlock(&busiest->lock);
	lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
}
1646 1647
#endif

V
Vegard Nossum 已提交
1648
#ifdef CONFIG_FAIR_GROUP_SCHED
I
Ingo Molnar 已提交
1649 1650
static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
{
V
Vegard Nossum 已提交
1651
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
1652 1653 1654
	cfs_rq->shares = shares;
#endif
}
V
Vegard Nossum 已提交
1655
#endif
1656

I
Ingo Molnar 已提交
1657 1658
#include "sched_stats.h"
#include "sched_idletask.c"
1659 1660
#include "sched_fair.c"
#include "sched_rt.c"
I
Ingo Molnar 已提交
1661 1662 1663 1664 1665
#ifdef CONFIG_SCHED_DEBUG
# include "sched_debug.c"
#endif

#define sched_class_highest (&rt_sched_class)
1666 1667
#define for_each_class(class) \
   for (class = sched_class_highest; class; class = class->next)
I
Ingo Molnar 已提交
1668

1669
static void inc_nr_running(struct rq *rq)
1670 1671 1672 1673
{
	rq->nr_running++;
}

1674
static void dec_nr_running(struct rq *rq)
1675 1676 1677 1678
{
	rq->nr_running--;
}

1679 1680 1681
static void set_load_weight(struct task_struct *p)
{
	if (task_has_rt_policy(p)) {
I
Ingo Molnar 已提交
1682 1683 1684 1685
		p->se.load.weight = prio_to_weight[0] * 2;
		p->se.load.inv_weight = prio_to_wmult[0] >> 1;
		return;
	}
1686

I
Ingo Molnar 已提交
1687 1688 1689 1690 1691 1692 1693 1694
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
		p->se.load.weight = WEIGHT_IDLEPRIO;
		p->se.load.inv_weight = WMULT_IDLEPRIO;
		return;
	}
1695

I
Ingo Molnar 已提交
1696 1697
	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
1698 1699
}

1700 1701 1702 1703 1704 1705
static void update_avg(u64 *avg, u64 sample)
{
	s64 diff = sample - *avg;
	*avg += diff >> 3;
}

1706
static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
1707
{
I
Ingo Molnar 已提交
1708
	sched_info_queued(p);
1709
	p->sched_class->enqueue_task(rq, p, wakeup);
I
Ingo Molnar 已提交
1710
	p->se.on_rq = 1;
1711 1712
}

1713
static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
1714
{
1715 1716 1717 1718 1719 1720
	if (sleep && p->se.last_wakeup) {
		update_avg(&p->se.avg_overlap,
			   p->se.sum_exec_runtime - p->se.last_wakeup);
		p->se.last_wakeup = 0;
	}

1721
	sched_info_dequeued(p);
1722
	p->sched_class->dequeue_task(rq, p, sleep);
I
Ingo Molnar 已提交
1723
	p->se.on_rq = 0;
1724 1725
}

1726
/*
I
Ingo Molnar 已提交
1727
 * __normal_prio - return the priority that is based on the static prio
1728 1729 1730
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
1731
	return p->static_prio;
1732 1733
}

1734 1735 1736 1737 1738 1739 1740
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
1741
static inline int normal_prio(struct task_struct *p)
1742 1743 1744
{
	int prio;

1745
	if (task_has_rt_policy(p))
1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
1759
static int effective_prio(struct task_struct *p)
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
1772
/*
I
Ingo Molnar 已提交
1773
 * activate_task - move a task to the runqueue.
L
Linus Torvalds 已提交
1774
 */
I
Ingo Molnar 已提交
1775
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
L
Linus Torvalds 已提交
1776
{
1777
	if (task_contributes_to_load(p))
I
Ingo Molnar 已提交
1778
		rq->nr_uninterruptible--;
L
Linus Torvalds 已提交
1779

1780
	enqueue_task(rq, p, wakeup);
1781
	inc_nr_running(rq);
L
Linus Torvalds 已提交
1782 1783 1784 1785 1786
}

/*
 * deactivate_task - remove a task from the runqueue.
 */
1787
static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
L
Linus Torvalds 已提交
1788
{
1789
	if (task_contributes_to_load(p))
I
Ingo Molnar 已提交
1790 1791
		rq->nr_uninterruptible++;

1792
	dequeue_task(rq, p, sleep);
1793
	dec_nr_running(rq);
L
Linus Torvalds 已提交
1794 1795 1796 1797 1798 1799
}

/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
1800
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
1801 1802 1803 1804
{
	return cpu_curr(task_cpu(p)) == p;
}

I
Ingo Molnar 已提交
1805 1806
static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
P
Peter Zijlstra 已提交
1807
	set_task_rq(p, cpu);
I
Ingo Molnar 已提交
1808
#ifdef CONFIG_SMP
1809 1810 1811 1812 1813 1814
	/*
	 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
	 * successfuly executed on another CPU. We must ensure that updates of
	 * per-task data have been completed by this moment.
	 */
	smp_wmb();
I
Ingo Molnar 已提交
1815 1816
	task_thread_info(p)->cpu = cpu;
#endif
1817 1818
}

1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
				       const struct sched_class *prev_class,
				       int oldprio, int running)
{
	if (prev_class != p->sched_class) {
		if (prev_class->switched_from)
			prev_class->switched_from(rq, p, running);
		p->sched_class->switched_to(rq, p, running);
	} else
		p->sched_class->prio_changed(rq, p, oldprio, running);
}

L
Linus Torvalds 已提交
1831
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
1832

1833 1834 1835 1836 1837 1838
/* Used instead of source_load when we know the type == 0 */
static unsigned long weighted_cpuload(const int cpu)
{
	return cpu_rq(cpu)->load.weight;
}

1839 1840 1841
/*
 * Is this task likely cache-hot:
 */
1842
static int
1843 1844 1845 1846
task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
{
	s64 delta;

1847 1848 1849
	/*
	 * Buddy candidates are cache hot:
	 */
P
Peter Zijlstra 已提交
1850 1851 1852
	if (sched_feat(CACHE_HOT_BUDDY) &&
			(&p->se == cfs_rq_of(&p->se)->next ||
			 &p->se == cfs_rq_of(&p->se)->last))
1853 1854
		return 1;

1855 1856 1857
	if (p->sched_class != &fair_sched_class)
		return 0;

1858 1859 1860 1861 1862
	if (sysctl_sched_migration_cost == -1)
		return 1;
	if (sysctl_sched_migration_cost == 0)
		return 0;

1863 1864 1865 1866 1867 1868
	delta = now - p->se.exec_start;

	return delta < (s64)sysctl_sched_migration_cost;
}


I
Ingo Molnar 已提交
1869
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
1870
{
I
Ingo Molnar 已提交
1871 1872
	int old_cpu = task_cpu(p);
	struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
1873 1874
	struct cfs_rq *old_cfsrq = task_cfs_rq(p),
		      *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
1875
	u64 clock_offset;
I
Ingo Molnar 已提交
1876 1877

	clock_offset = old_rq->clock - new_rq->clock;
I
Ingo Molnar 已提交
1878

1879 1880
	trace_sched_migrate_task(p, task_cpu(p), new_cpu);

I
Ingo Molnar 已提交
1881 1882 1883
#ifdef CONFIG_SCHEDSTATS
	if (p->se.wait_start)
		p->se.wait_start -= clock_offset;
I
Ingo Molnar 已提交
1884 1885 1886 1887
	if (p->se.sleep_start)
		p->se.sleep_start -= clock_offset;
	if (p->se.block_start)
		p->se.block_start -= clock_offset;
1888 1889 1890 1891 1892
	if (old_cpu != new_cpu) {
		schedstat_inc(p, se.nr_migrations);
		if (task_hot(p, old_rq->clock, NULL))
			schedstat_inc(p, se.nr_forced2_migrations);
	}
I
Ingo Molnar 已提交
1893
#endif
1894 1895
	p->se.vruntime -= old_cfsrq->min_vruntime -
					 new_cfsrq->min_vruntime;
I
Ingo Molnar 已提交
1896 1897

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1898 1899
}

1900
struct migration_req {
L
Linus Torvalds 已提交
1901 1902
	struct list_head list;

1903
	struct task_struct *task;
L
Linus Torvalds 已提交
1904 1905 1906
	int dest_cpu;

	struct completion done;
1907
};
L
Linus Torvalds 已提交
1908 1909 1910 1911 1912

/*
 * The task's runqueue lock must be held.
 * Returns true if you have to wait for migration thread.
 */
1913
static int
1914
migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
L
Linus Torvalds 已提交
1915
{
1916
	struct rq *rq = task_rq(p);
L
Linus Torvalds 已提交
1917 1918 1919 1920 1921

	/*
	 * If the task is not on a runqueue (and not running), then
	 * it is sufficient to simply update the task's cpu field.
	 */
I
Ingo Molnar 已提交
1922
	if (!p->se.on_rq && !task_running(rq, p)) {
L
Linus Torvalds 已提交
1923 1924 1925 1926 1927 1928 1929 1930
		set_task_cpu(p, dest_cpu);
		return 0;
	}

	init_completion(&req->done);
	req->task = p;
	req->dest_cpu = dest_cpu;
	list_add(&req->list, &rq->migration_queue);
1931

L
Linus Torvalds 已提交
1932 1933 1934 1935 1936 1937
	return 1;
}

/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
R
Roland McGrath 已提交
1938 1939 1940 1941 1942 1943 1944
 * If @match_state is nonzero, it's the @p->state value just checked and
 * not expected to change.  If it changes, i.e. @p might have woken up,
 * then return zero.  When we succeed in waiting for @p to be off its CPU,
 * we return a positive number (its total switch count).  If a second call
 * a short while later returns the same number, the caller can be sure that
 * @p has remained unscheduled the whole time.
 *
L
Linus Torvalds 已提交
1945 1946 1947 1948 1949 1950
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
R
Roland McGrath 已提交
1951
unsigned long wait_task_inactive(struct task_struct *p, long match_state)
L
Linus Torvalds 已提交
1952 1953
{
	unsigned long flags;
I
Ingo Molnar 已提交
1954
	int running, on_rq;
R
Roland McGrath 已提交
1955
	unsigned long ncsw;
1956
	struct rq *rq;
L
Linus Torvalds 已提交
1957

1958 1959 1960 1961 1962 1963 1964 1965
	for (;;) {
		/*
		 * We do the initial early heuristics without holding
		 * any task-queue locks at all. We'll only try to get
		 * the runqueue lock when things look like they will
		 * work out!
		 */
		rq = task_rq(p);
1966

1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
		/*
		 * If the task is actively running on another CPU
		 * still, just relax and busy-wait without holding
		 * any locks.
		 *
		 * NOTE! Since we don't hold any locks, it's not
		 * even sure that "rq" stays as the right runqueue!
		 * But we don't care, since "task_running()" will
		 * return false if the runqueue has changed and p
		 * is actually now running somewhere else!
		 */
R
Roland McGrath 已提交
1978 1979 1980
		while (task_running(rq, p)) {
			if (match_state && unlikely(p->state != match_state))
				return 0;
1981
			cpu_relax();
R
Roland McGrath 已提交
1982
		}
1983

1984 1985 1986 1987 1988 1989
		/*
		 * Ok, time to look more closely! We need the rq
		 * lock now, to be *sure*. If we're wrong, we'll
		 * just go back and repeat.
		 */
		rq = task_rq_lock(p, &flags);
1990
		trace_sched_wait_task(rq, p);
1991 1992
		running = task_running(rq, p);
		on_rq = p->se.on_rq;
R
Roland McGrath 已提交
1993
		ncsw = 0;
1994
		if (!match_state || p->state == match_state)
1995
			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1996
		task_rq_unlock(rq, &flags);
1997

R
Roland McGrath 已提交
1998 1999 2000 2001 2002 2003
		/*
		 * If it changed from the expected state, bail out now.
		 */
		if (unlikely(!ncsw))
			break;

2004 2005 2006 2007 2008 2009 2010 2011 2012 2013
		/*
		 * Was it really running after all now that we
		 * checked with the proper locks actually held?
		 *
		 * Oops. Go back and try again..
		 */
		if (unlikely(running)) {
			cpu_relax();
			continue;
		}
2014

2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
		/*
		 * It's not enough that it's not actively running,
		 * it must be off the runqueue _entirely_, and not
		 * preempted!
		 *
		 * So if it wa still runnable (but just not actively
		 * running right now), it's preempted, and we should
		 * yield - it could be a while.
		 */
		if (unlikely(on_rq)) {
			schedule_timeout_uninterruptible(1);
			continue;
		}
2028

2029 2030 2031 2032 2033 2034 2035
		/*
		 * Ahh, all good. It wasn't running, and it wasn't
		 * runnable, which means that it will never become
		 * running in the future either. We're all done!
		 */
		break;
	}
R
Roland McGrath 已提交
2036 2037

	return ncsw;
L
Linus Torvalds 已提交
2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
 * NOTE: this function doesnt have to take the runqueue lock,
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
2053
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}

/*
2065 2066
 * Return a low guess at the load of a migration-source cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
2067 2068 2069 2070
 *
 * We want to under-estimate the load of migration sources, to
 * balance conservatively.
 */
A
Alexey Dobriyan 已提交
2071
static unsigned long source_load(int cpu, int type)
L
Linus Torvalds 已提交
2072
{
2073
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
2074
	unsigned long total = weighted_cpuload(cpu);
2075

2076
	if (type == 0 || !sched_feat(LB_BIAS))
I
Ingo Molnar 已提交
2077
		return total;
2078

I
Ingo Molnar 已提交
2079
	return min(rq->cpu_load[type-1], total);
L
Linus Torvalds 已提交
2080 2081 2082
}

/*
2083 2084
 * Return a high guess at the load of a migration-target cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
2085
 */
A
Alexey Dobriyan 已提交
2086
static unsigned long target_load(int cpu, int type)
L
Linus Torvalds 已提交
2087
{
2088
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
2089
	unsigned long total = weighted_cpuload(cpu);
2090

2091
	if (type == 0 || !sched_feat(LB_BIAS))
I
Ingo Molnar 已提交
2092
		return total;
2093

I
Ingo Molnar 已提交
2094
	return max(rq->cpu_load[type-1], total);
2095 2096
}

N
Nick Piggin 已提交
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
/*
 * find_idlest_group finds and returns the least busy CPU group within the
 * domain.
 */
static struct sched_group *
find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
{
	struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
	unsigned long min_load = ULONG_MAX, this_load = 0;
	int load_idx = sd->forkexec_idx;
	int imbalance = 100 + (sd->imbalance_pct-100)/2;

	do {
		unsigned long load, avg_load;
		int local_group;
		int i;

2114
		/* Skip over this group if it has no CPUs allowed */
2115 2116
		if (!cpumask_intersects(sched_group_cpus(group),
					&p->cpus_allowed))
2117
			continue;
2118

2119 2120
		local_group = cpumask_test_cpu(this_cpu,
					       sched_group_cpus(group));
N
Nick Piggin 已提交
2121 2122 2123 2124

		/* Tally up the load of all CPUs in the group */
		avg_load = 0;

2125
		for_each_cpu(i, sched_group_cpus(group)) {
N
Nick Piggin 已提交
2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
			/* Bias balancing toward cpus of our domain */
			if (local_group)
				load = source_load(i, load_idx);
			else
				load = target_load(i, load_idx);

			avg_load += load;
		}

		/* Adjust by relative CPU power of the group */
2136 2137
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
N
Nick Piggin 已提交
2138 2139 2140 2141 2142 2143 2144 2145

		if (local_group) {
			this_load = avg_load;
			this = group;
		} else if (avg_load < min_load) {
			min_load = avg_load;
			idlest = group;
		}
2146
	} while (group = group->next, group != sd->groups);
N
Nick Piggin 已提交
2147 2148 2149 2150 2151 2152 2153

	if (!idlest || 100*this_load < imbalance*min_load)
		return NULL;
	return idlest;
}

/*
2154
 * find_idlest_cpu - find the idlest cpu among the cpus in group.
N
Nick Piggin 已提交
2155
 */
I
Ingo Molnar 已提交
2156
static int
2157
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
N
Nick Piggin 已提交
2158 2159 2160 2161 2162
{
	unsigned long load, min_load = ULONG_MAX;
	int idlest = -1;
	int i;

2163
	/* Traverse only the allowed CPUs */
2164
	for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2165
		load = weighted_cpuload(i);
N
Nick Piggin 已提交
2166 2167 2168 2169 2170 2171 2172 2173 2174 2175

		if (load < min_load || (load == min_load && i == this_cpu)) {
			min_load = load;
			idlest = i;
		}
	}

	return idlest;
}

N
Nick Piggin 已提交
2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
/*
 * sched_balance_self: balance the current task (running on cpu) in domains
 * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
 * SD_BALANCE_EXEC.
 *
 * Balance, ie. select the least loaded group.
 *
 * Returns the target CPU number, or the same CPU if no balancing is needed.
 *
 * preempt must be disabled.
 */
static int sched_balance_self(int cpu, int flag)
{
	struct task_struct *t = current;
	struct sched_domain *tmp, *sd = NULL;
N
Nick Piggin 已提交
2191

2192
	for_each_domain(cpu, tmp) {
I
Ingo Molnar 已提交
2193 2194 2195
		/*
		 * If power savings logic is enabled for a domain, stop there.
		 */
2196 2197
		if (tmp->flags & SD_POWERSAVINGS_BALANCE)
			break;
N
Nick Piggin 已提交
2198 2199
		if (tmp->flags & flag)
			sd = tmp;
2200
	}
N
Nick Piggin 已提交
2201

2202 2203 2204
	if (sd)
		update_shares(sd);

N
Nick Piggin 已提交
2205 2206
	while (sd) {
		struct sched_group *group;
2207 2208 2209 2210 2211 2212
		int new_cpu, weight;

		if (!(sd->flags & flag)) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
2213 2214

		group = find_idlest_group(sd, t, cpu);
2215 2216 2217 2218
		if (!group) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
2219

2220
		new_cpu = find_idlest_cpu(group, t, cpu);
2221 2222 2223 2224 2225
		if (new_cpu == -1 || new_cpu == cpu) {
			/* Now try balancing at a lower domain level of cpu */
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
2226

2227
		/* Now try balancing at a lower domain level of new_cpu */
N
Nick Piggin 已提交
2228
		cpu = new_cpu;
2229
		weight = cpumask_weight(sched_domain_span(sd));
N
Nick Piggin 已提交
2230 2231
		sd = NULL;
		for_each_domain(cpu, tmp) {
2232
			if (weight <= cpumask_weight(sched_domain_span(tmp)))
N
Nick Piggin 已提交
2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243
				break;
			if (tmp->flags & flag)
				sd = tmp;
		}
		/* while loop will break here if sd == NULL */
	}

	return cpu;
}

#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258

/***
 * try_to_wake_up - wake up a thread
 * @p: the to-be-woken-up thread
 * @state: the mask of task states that can be woken
 * @sync: do a synchronous wakeup?
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
 * returns failure only if the task is already active.
 */
2259
static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
L
Linus Torvalds 已提交
2260
{
2261
	int cpu, orig_cpu, this_cpu, success = 0;
L
Linus Torvalds 已提交
2262 2263
	unsigned long flags;
	long old_state;
2264
	struct rq *rq;
L
Linus Torvalds 已提交
2265

2266 2267 2268
	if (!sched_feat(SYNC_WAKEUPS))
		sync = 0;

P
Peter Zijlstra 已提交
2269 2270 2271 2272 2273 2274 2275 2276
#ifdef CONFIG_SMP
	if (sched_feat(LB_WAKEUP_UPDATE)) {
		struct sched_domain *sd;

		this_cpu = raw_smp_processor_id();
		cpu = task_cpu(p);

		for_each_domain(this_cpu, sd) {
2277
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
P
Peter Zijlstra 已提交
2278 2279 2280 2281 2282 2283 2284
				update_shares(sd);
				break;
			}
		}
	}
#endif

2285
	smp_wmb();
L
Linus Torvalds 已提交
2286
	rq = task_rq_lock(p, &flags);
2287
	update_rq_clock(rq);
L
Linus Torvalds 已提交
2288 2289 2290 2291
	old_state = p->state;
	if (!(old_state & state))
		goto out;

I
Ingo Molnar 已提交
2292
	if (p->se.on_rq)
L
Linus Torvalds 已提交
2293 2294 2295
		goto out_running;

	cpu = task_cpu(p);
2296
	orig_cpu = cpu;
L
Linus Torvalds 已提交
2297 2298 2299 2300 2301 2302
	this_cpu = smp_processor_id();

#ifdef CONFIG_SMP
	if (unlikely(task_running(rq, p)))
		goto out_activate;

2303 2304 2305
	cpu = p->sched_class->select_task_rq(p, sync);
	if (cpu != orig_cpu) {
		set_task_cpu(p, cpu);
L
Linus Torvalds 已提交
2306 2307 2308 2309 2310 2311
		task_rq_unlock(rq, &flags);
		/* might preempt at this point */
		rq = task_rq_lock(p, &flags);
		old_state = p->state;
		if (!(old_state & state))
			goto out;
I
Ingo Molnar 已提交
2312
		if (p->se.on_rq)
L
Linus Torvalds 已提交
2313 2314 2315 2316 2317 2318
			goto out_running;

		this_cpu = smp_processor_id();
		cpu = task_cpu(p);
	}

2319 2320 2321 2322 2323 2324 2325
#ifdef CONFIG_SCHEDSTATS
	schedstat_inc(rq, ttwu_count);
	if (cpu == this_cpu)
		schedstat_inc(rq, ttwu_local);
	else {
		struct sched_domain *sd;
		for_each_domain(this_cpu, sd) {
2326
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2327 2328 2329 2330 2331
				schedstat_inc(sd, ttwu_wake_remote);
				break;
			}
		}
	}
2332
#endif /* CONFIG_SCHEDSTATS */
2333

L
Linus Torvalds 已提交
2334 2335
out_activate:
#endif /* CONFIG_SMP */
2336 2337 2338 2339 2340 2341 2342 2343 2344
	schedstat_inc(p, se.nr_wakeups);
	if (sync)
		schedstat_inc(p, se.nr_wakeups_sync);
	if (orig_cpu != cpu)
		schedstat_inc(p, se.nr_wakeups_migrate);
	if (cpu == this_cpu)
		schedstat_inc(p, se.nr_wakeups_local);
	else
		schedstat_inc(p, se.nr_wakeups_remote);
I
Ingo Molnar 已提交
2345
	activate_task(rq, p, 1);
L
Linus Torvalds 已提交
2346 2347 2348
	success = 1;

out_running:
2349
	trace_sched_wakeup(rq, p, success);
2350
	check_preempt_curr(rq, p, sync);
I
Ingo Molnar 已提交
2351

L
Linus Torvalds 已提交
2352
	p->state = TASK_RUNNING;
2353 2354 2355 2356
#ifdef CONFIG_SMP
	if (p->sched_class->task_wake_up)
		p->sched_class->task_wake_up(rq, p);
#endif
L
Linus Torvalds 已提交
2357
out:
2358 2359
	current->se.last_wakeup = current->se.sum_exec_runtime;

L
Linus Torvalds 已提交
2360 2361 2362 2363 2364
	task_rq_unlock(rq, &flags);

	return success;
}

2365
int wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
2366
{
2367
	return try_to_wake_up(p, TASK_ALL, 0);
L
Linus Torvalds 已提交
2368 2369 2370
}
EXPORT_SYMBOL(wake_up_process);

2371
int wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
2372 2373 2374 2375 2376 2377 2378
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
2379 2380 2381 2382 2383 2384 2385
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
2386
	p->se.prev_sum_exec_runtime	= 0;
I
Ingo Molnar 已提交
2387 2388
	p->se.last_wakeup		= 0;
	p->se.avg_overlap		= 0;
I
Ingo Molnar 已提交
2389 2390 2391

#ifdef CONFIG_SCHEDSTATS
	p->se.wait_start		= 0;
I
Ingo Molnar 已提交
2392 2393 2394 2395 2396 2397
	p->se.sum_sleep_runtime		= 0;
	p->se.sleep_start		= 0;
	p->se.block_start		= 0;
	p->se.sleep_max			= 0;
	p->se.block_max			= 0;
	p->se.exec_max			= 0;
I
Ingo Molnar 已提交
2398
	p->se.slice_max			= 0;
I
Ingo Molnar 已提交
2399
	p->se.wait_max			= 0;
I
Ingo Molnar 已提交
2400
#endif
N
Nick Piggin 已提交
2401

P
Peter Zijlstra 已提交
2402
	INIT_LIST_HEAD(&p->rt.run_list);
I
Ingo Molnar 已提交
2403
	p->se.on_rq = 0;
2404
	INIT_LIST_HEAD(&p->se.group_node);
N
Nick Piggin 已提交
2405

2406 2407 2408 2409
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif

L
Linus Torvalds 已提交
2410 2411 2412 2413 2414 2415 2416
	/*
	 * We mark the process as running here, but have not actually
	 * inserted it onto the runqueue yet. This guarantees that
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430
}

/*
 * fork()/clone()-time setup:
 */
void sched_fork(struct task_struct *p, int clone_flags)
{
	int cpu = get_cpu();

	__sched_fork(p);

#ifdef CONFIG_SMP
	cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
#endif
I
Ingo Molnar 已提交
2431
	set_task_cpu(p, cpu);
2432 2433 2434 2435 2436

	/*
	 * Make sure we do not leak PI boosting priority to the child:
	 */
	p->prio = current->normal_prio;
H
Hiroshi Shimamoto 已提交
2437 2438
	if (!rt_prio(p->prio))
		p->sched_class = &fair_sched_class;
2439

2440
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
2441
	if (likely(sched_info_on()))
2442
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
2443
#endif
2444
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
2445 2446
	p->oncpu = 0;
#endif
L
Linus Torvalds 已提交
2447
#ifdef CONFIG_PREEMPT
2448
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
2449
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
2450
#endif
N
Nick Piggin 已提交
2451
	put_cpu();
L
Linus Torvalds 已提交
2452 2453 2454 2455 2456 2457 2458 2459 2460
}

/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
2461
void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
L
Linus Torvalds 已提交
2462 2463
{
	unsigned long flags;
I
Ingo Molnar 已提交
2464
	struct rq *rq;
L
Linus Torvalds 已提交
2465 2466

	rq = task_rq_lock(p, &flags);
N
Nick Piggin 已提交
2467
	BUG_ON(p->state != TASK_RUNNING);
I
Ingo Molnar 已提交
2468
	update_rq_clock(rq);
L
Linus Torvalds 已提交
2469 2470 2471

	p->prio = effective_prio(p);

2472
	if (!p->sched_class->task_new || !current->se.on_rq) {
I
Ingo Molnar 已提交
2473
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
2474 2475
	} else {
		/*
I
Ingo Molnar 已提交
2476 2477
		 * Let the scheduling class do new task startup
		 * management (if any):
L
Linus Torvalds 已提交
2478
		 */
2479
		p->sched_class->task_new(rq, p);
2480
		inc_nr_running(rq);
L
Linus Torvalds 已提交
2481
	}
2482
	trace_sched_wakeup_new(rq, p, 1);
2483
	check_preempt_curr(rq, p, 0);
2484 2485 2486 2487
#ifdef CONFIG_SMP
	if (p->sched_class->task_wake_up)
		p->sched_class->task_wake_up(rq, p);
#endif
I
Ingo Molnar 已提交
2488
	task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
2489 2490
}

2491 2492 2493
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
R
Randy Dunlap 已提交
2494 2495
 * preempt_notifier_register - tell me when current is being being preempted & rescheduled
 * @notifier: notifier struct to register
2496 2497 2498 2499 2500 2501 2502 2503 2504
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
2505
 * @notifier: notifier struct to unregister
2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_out(notifier, next);
}

2535
#else /* !CONFIG_PREEMPT_NOTIFIERS */
2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

2547
#endif /* CONFIG_PREEMPT_NOTIFIERS */
2548

2549 2550 2551
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
2552
 * @prev: the current task that is being switched out
2553 2554 2555 2556 2557 2558 2559 2560 2561
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
2562 2563 2564
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
2565
{
2566
	fire_sched_out_preempt_notifiers(prev, next);
2567 2568 2569 2570
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
2571 2572
/**
 * finish_task_switch - clean up after a task-switch
2573
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
2574 2575
 * @prev: the thread we just switched away from.
 *
2576 2577 2578 2579
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
2580 2581
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
I
Ingo Molnar 已提交
2582
 * so, we finish that here outside of the runqueue lock. (Doing it
L
Linus Torvalds 已提交
2583 2584 2585
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
A
Alexey Dobriyan 已提交
2586
static void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
2587 2588 2589
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
2590
	long prev_state;
L
Linus Torvalds 已提交
2591 2592 2593 2594 2595

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
2596
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
2597 2598
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
2599
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
2600 2601 2602 2603 2604
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
2605
	prev_state = prev->state;
2606 2607
	finish_arch_switch(prev);
	finish_lock_switch(rq, prev);
2608 2609 2610 2611
#ifdef CONFIG_SMP
	if (current->sched_class->post_schedule)
		current->sched_class->post_schedule(rq);
#endif
S
Steven Rostedt 已提交
2612

2613
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
2614 2615
	if (mm)
		mmdrop(mm);
2616
	if (unlikely(prev_state == TASK_DEAD)) {
2617 2618 2619
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
2620
		 */
2621
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
2622
		put_task_struct(prev);
2623
	}
L
Linus Torvalds 已提交
2624 2625 2626 2627 2628 2629
}

/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
2630
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
2631 2632
	__releases(rq->lock)
{
2633 2634
	struct rq *rq = this_rq();

2635 2636 2637 2638 2639
	finish_task_switch(rq, prev);
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
2640
	if (current->set_child_tid)
2641
		put_user(task_pid_vnr(current), current->set_child_tid);
L
Linus Torvalds 已提交
2642 2643 2644 2645 2646 2647
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
2648
static inline void
2649
context_switch(struct rq *rq, struct task_struct *prev,
2650
	       struct task_struct *next)
L
Linus Torvalds 已提交
2651
{
I
Ingo Molnar 已提交
2652
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
2653

2654
	prepare_task_switch(rq, prev, next);
2655
	trace_sched_switch(rq, prev, next);
I
Ingo Molnar 已提交
2656 2657
	mm = next->mm;
	oldmm = prev->active_mm;
2658 2659 2660 2661 2662 2663 2664
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
	arch_enter_lazy_cpu_mode();

I
Ingo Molnar 已提交
2665
	if (unlikely(!mm)) {
L
Linus Torvalds 已提交
2666 2667 2668 2669 2670 2671
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

I
Ingo Molnar 已提交
2672
	if (unlikely(!prev->mm)) {
L
Linus Torvalds 已提交
2673 2674 2675
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
2676 2677 2678 2679 2680 2681 2682
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2683
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2684
#endif
L
Linus Torvalds 已提交
2685 2686 2687 2688

	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
2689 2690 2691 2692 2693 2694 2695
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
}

/*
 * nr_running, nr_uninterruptible and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, current number of uninterruptible-sleeping threads, total
 * number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
}

unsigned long nr_uninterruptible(void)
{
	unsigned long i, sum = 0;

2719
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733
		sum += cpu_rq(i)->nr_uninterruptible;

	/*
	 * Since we read the counters lockless, it might be slightly
	 * inaccurate. Do not allow it to go below zero though:
	 */
	if (unlikely((long)sum < 0))
		sum = 0;

	return sum;
}

unsigned long long nr_context_switches(void)
{
2734 2735
	int i;
	unsigned long long sum = 0;
L
Linus Torvalds 已提交
2736

2737
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2738 2739 2740 2741 2742 2743 2744 2745 2746
		sum += cpu_rq(i)->nr_switches;

	return sum;
}

unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;

2747
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2748 2749 2750 2751 2752
		sum += atomic_read(&cpu_rq(i)->nr_iowait);

	return sum;
}

2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767
unsigned long nr_active(void)
{
	unsigned long i, running = 0, uninterruptible = 0;

	for_each_online_cpu(i) {
		running += cpu_rq(i)->nr_running;
		uninterruptible += cpu_rq(i)->nr_uninterruptible;
	}

	if (unlikely((long)uninterruptible < 0))
		uninterruptible = 0;

	return running + uninterruptible;
}

2768
/*
I
Ingo Molnar 已提交
2769 2770
 * Update rq->cpu_load[] statistics. This function is usually called every
 * scheduler tick (TICK_NSEC).
2771
 */
I
Ingo Molnar 已提交
2772
static void update_cpu_load(struct rq *this_rq)
2773
{
2774
	unsigned long this_load = this_rq->load.weight;
I
Ingo Molnar 已提交
2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
	int i, scale;

	this_rq->nr_load_updates++;

	/* Update our load: */
	for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
		unsigned long old_load, new_load;

		/* scale is effectively 1 << i now, and >> i divides by scale */

		old_load = this_rq->cpu_load[i];
		new_load = this_load;
I
Ingo Molnar 已提交
2787 2788 2789 2790 2791 2792 2793
		/*
		 * Round up the averaging division if load is increasing. This
		 * prevents us from getting stuck on 9 if the load is 10, for
		 * example.
		 */
		if (new_load > old_load)
			new_load += scale-1;
I
Ingo Molnar 已提交
2794 2795
		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
	}
2796 2797
}

I
Ingo Molnar 已提交
2798 2799
#ifdef CONFIG_SMP

L
Linus Torvalds 已提交
2800 2801 2802 2803 2804 2805
/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
2806
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2807 2808 2809
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
2810
	BUG_ON(!irqs_disabled());
L
Linus Torvalds 已提交
2811 2812 2813 2814
	if (rq1 == rq2) {
		spin_lock(&rq1->lock);
		__acquire(rq2->lock);	/* Fake it out ;) */
	} else {
2815
		if (rq1 < rq2) {
L
Linus Torvalds 已提交
2816
			spin_lock(&rq1->lock);
2817
			spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
L
Linus Torvalds 已提交
2818 2819
		} else {
			spin_lock(&rq2->lock);
2820
			spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
L
Linus Torvalds 已提交
2821 2822
		}
	}
2823 2824
	update_rq_clock(rq1);
	update_rq_clock(rq2);
L
Linus Torvalds 已提交
2825 2826 2827 2828 2829 2830 2831 2832
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
2833
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	spin_unlock(&rq1->lock);
	if (rq1 != rq2)
		spin_unlock(&rq2->lock);
	else
		__release(rq2->lock);
}

/*
 * If dest_cpu is allowed for this process, migrate the task to it.
 * This is accomplished by forcing the cpu_allowed mask to only
I
Ingo Molnar 已提交
2847
 * allow dest_cpu, which will force the cpu onto dest_cpu. Then
L
Linus Torvalds 已提交
2848 2849
 * the cpu_allowed mask is restored.
 */
2850
static void sched_migrate_task(struct task_struct *p, int dest_cpu)
L
Linus Torvalds 已提交
2851
{
2852
	struct migration_req req;
L
Linus Torvalds 已提交
2853
	unsigned long flags;
2854
	struct rq *rq;
L
Linus Torvalds 已提交
2855 2856

	rq = task_rq_lock(p, &flags);
2857
	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2858
	    || unlikely(!cpu_active(dest_cpu)))
L
Linus Torvalds 已提交
2859 2860 2861 2862 2863 2864
		goto out;

	/* force the process onto the specified CPU */
	if (migrate_task(p, dest_cpu, &req)) {
		/* Need to wait for migration thread (might exit: take ref). */
		struct task_struct *mt = rq->migration_thread;
2865

L
Linus Torvalds 已提交
2866 2867 2868 2869 2870
		get_task_struct(mt);
		task_rq_unlock(rq, &flags);
		wake_up_process(mt);
		put_task_struct(mt);
		wait_for_completion(&req.done);
2871

L
Linus Torvalds 已提交
2872 2873 2874 2875 2876 2877 2878
		return;
	}
out:
	task_rq_unlock(rq, &flags);
}

/*
N
Nick Piggin 已提交
2879 2880
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
L
Linus Torvalds 已提交
2881 2882 2883 2884
 */
void sched_exec(void)
{
	int new_cpu, this_cpu = get_cpu();
N
Nick Piggin 已提交
2885
	new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
L
Linus Torvalds 已提交
2886
	put_cpu();
N
Nick Piggin 已提交
2887 2888
	if (new_cpu != this_cpu)
		sched_migrate_task(current, new_cpu);
L
Linus Torvalds 已提交
2889 2890 2891 2892 2893 2894
}

/*
 * pull_task - move a task from a remote runqueue to the local runqueue.
 * Both runqueues must be locked.
 */
I
Ingo Molnar 已提交
2895 2896
static void pull_task(struct rq *src_rq, struct task_struct *p,
		      struct rq *this_rq, int this_cpu)
L
Linus Torvalds 已提交
2897
{
2898
	deactivate_task(src_rq, p, 0);
L
Linus Torvalds 已提交
2899
	set_task_cpu(p, this_cpu);
I
Ingo Molnar 已提交
2900
	activate_task(this_rq, p, 0);
L
Linus Torvalds 已提交
2901 2902 2903 2904
	/*
	 * Note that idle threads have a prio of MAX_PRIO, for this test
	 * to be always true for them.
	 */
2905
	check_preempt_curr(this_rq, p, 0);
L
Linus Torvalds 已提交
2906 2907 2908 2909 2910
}

/*
 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
 */
2911
static
2912
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
I
Ingo Molnar 已提交
2913
		     struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2914
		     int *all_pinned)
L
Linus Torvalds 已提交
2915 2916 2917 2918 2919 2920 2921
{
	/*
	 * We do not migrate tasks that are:
	 * 1) running (obviously), or
	 * 2) cannot be migrated to this CPU due to cpus_allowed, or
	 * 3) are cache-hot on their current CPU.
	 */
2922
	if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2923
		schedstat_inc(p, se.nr_failed_migrations_affine);
L
Linus Torvalds 已提交
2924
		return 0;
2925
	}
2926 2927
	*all_pinned = 0;

2928 2929
	if (task_running(rq, p)) {
		schedstat_inc(p, se.nr_failed_migrations_running);
2930
		return 0;
2931
	}
L
Linus Torvalds 已提交
2932

2933 2934 2935 2936 2937 2938
	/*
	 * Aggressive migration if:
	 * 1) task is cache cold, or
	 * 2) too many balance attempts have failed.
	 */

2939 2940
	if (!task_hot(p, rq->clock, sd) ||
			sd->nr_balance_failed > sd->cache_nice_tries) {
2941
#ifdef CONFIG_SCHEDSTATS
2942
		if (task_hot(p, rq->clock, sd)) {
2943
			schedstat_inc(sd, lb_hot_gained[idle]);
2944 2945
			schedstat_inc(p, se.nr_forced_migrations);
		}
2946 2947 2948 2949
#endif
		return 1;
	}

2950 2951
	if (task_hot(p, rq->clock, sd)) {
		schedstat_inc(p, se.nr_failed_migrations_hot);
2952
		return 0;
2953
	}
L
Linus Torvalds 已提交
2954 2955 2956
	return 1;
}

2957 2958 2959 2960 2961
static unsigned long
balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
	      unsigned long max_load_move, struct sched_domain *sd,
	      enum cpu_idle_type idle, int *all_pinned,
	      int *this_best_prio, struct rq_iterator *iterator)
L
Linus Torvalds 已提交
2962
{
2963
	int loops = 0, pulled = 0, pinned = 0;
I
Ingo Molnar 已提交
2964 2965
	struct task_struct *p;
	long rem_load_move = max_load_move;
L
Linus Torvalds 已提交
2966

2967
	if (max_load_move == 0)
L
Linus Torvalds 已提交
2968 2969
		goto out;

2970 2971
	pinned = 1;

L
Linus Torvalds 已提交
2972
	/*
I
Ingo Molnar 已提交
2973
	 * Start the load-balancing iterator:
L
Linus Torvalds 已提交
2974
	 */
I
Ingo Molnar 已提交
2975 2976
	p = iterator->start(iterator->arg);
next:
2977
	if (!p || loops++ > sysctl_sched_nr_migrate)
L
Linus Torvalds 已提交
2978
		goto out;
2979 2980

	if ((p->se.load.weight >> 1) > rem_load_move ||
I
Ingo Molnar 已提交
2981 2982 2983
	    !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2984 2985
	}

I
Ingo Molnar 已提交
2986
	pull_task(busiest, p, this_rq, this_cpu);
L
Linus Torvalds 已提交
2987
	pulled++;
I
Ingo Molnar 已提交
2988
	rem_load_move -= p->se.load.weight;
L
Linus Torvalds 已提交
2989

2990
	/*
2991
	 * We only want to steal up to the prescribed amount of weighted load.
2992
	 */
2993
	if (rem_load_move > 0) {
2994 2995
		if (p->prio < *this_best_prio)
			*this_best_prio = p->prio;
I
Ingo Molnar 已提交
2996 2997
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2998 2999 3000
	}
out:
	/*
3001
	 * Right now, this is one of only two places pull_task() is called,
L
Linus Torvalds 已提交
3002 3003 3004 3005
	 * so we can safely collect pull_task() stats here rather than
	 * inside pull_task().
	 */
	schedstat_add(sd, lb_gained[idle], pulled);
3006 3007 3008

	if (all_pinned)
		*all_pinned = pinned;
3009 3010

	return max_load_move - rem_load_move;
L
Linus Torvalds 已提交
3011 3012
}

I
Ingo Molnar 已提交
3013
/*
P
Peter Williams 已提交
3014 3015 3016
 * move_tasks tries to move up to max_load_move weighted load from busiest to
 * this_rq, as part of a balancing operation within domain "sd".
 * Returns 1 if successful and 0 otherwise.
I
Ingo Molnar 已提交
3017 3018 3019 3020
 *
 * Called with both runqueues locked.
 */
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
P
Peter Williams 已提交
3021
		      unsigned long max_load_move,
I
Ingo Molnar 已提交
3022 3023 3024
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned)
{
3025
	const struct sched_class *class = sched_class_highest;
P
Peter Williams 已提交
3026
	unsigned long total_load_moved = 0;
3027
	int this_best_prio = this_rq->curr->prio;
I
Ingo Molnar 已提交
3028 3029

	do {
P
Peter Williams 已提交
3030 3031
		total_load_moved +=
			class->load_balance(this_rq, this_cpu, busiest,
3032
				max_load_move - total_load_moved,
3033
				sd, idle, all_pinned, &this_best_prio);
I
Ingo Molnar 已提交
3034
		class = class->next;
3035 3036 3037 3038

		if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
			break;

P
Peter Williams 已提交
3039
	} while (class && max_load_move > total_load_moved);
I
Ingo Molnar 已提交
3040

P
Peter Williams 已提交
3041 3042 3043
	return total_load_moved > 0;
}

3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069
static int
iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
		   struct sched_domain *sd, enum cpu_idle_type idle,
		   struct rq_iterator *iterator)
{
	struct task_struct *p = iterator->start(iterator->arg);
	int pinned = 0;

	while (p) {
		if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
			pull_task(busiest, p, this_rq, this_cpu);
			/*
			 * Right now, this is only the second place pull_task()
			 * is called, so we can safely collect pull_task()
			 * stats here rather than inside pull_task().
			 */
			schedstat_inc(sd, lb_gained[idle]);

			return 1;
		}
		p = iterator->next(iterator->arg);
	}

	return 0;
}

P
Peter Williams 已提交
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079
/*
 * move_one_task tries to move exactly one task from busiest to this_rq, as
 * part of active balancing operations within "domain".
 * Returns 1 if successful and 0 otherwise.
 *
 * Called with both runqueues locked.
 */
static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
			 struct sched_domain *sd, enum cpu_idle_type idle)
{
3080
	const struct sched_class *class;
P
Peter Williams 已提交
3081 3082

	for (class = sched_class_highest; class; class = class->next)
3083
		if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
P
Peter Williams 已提交
3084 3085 3086
			return 1;

	return 0;
I
Ingo Molnar 已提交
3087 3088
}

L
Linus Torvalds 已提交
3089 3090
/*
 * find_busiest_group finds and returns the busiest CPU group within the
3091 3092
 * domain. It calculates and returns the amount of weighted load which
 * should be moved to restore balance via the imbalance parameter.
L
Linus Torvalds 已提交
3093 3094 3095
 */
static struct sched_group *
find_busiest_group(struct sched_domain *sd, int this_cpu,
I
Ingo Molnar 已提交
3096
		   unsigned long *imbalance, enum cpu_idle_type idle,
3097
		   int *sd_idle, const struct cpumask *cpus, int *balance)
L
Linus Torvalds 已提交
3098 3099 3100
{
	struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
	unsigned long max_load, avg_load, total_load, this_load, total_pwr;
3101
	unsigned long max_pull;
3102 3103
	unsigned long busiest_load_per_task, busiest_nr_running;
	unsigned long this_load_per_task, this_nr_running;
3104
	int load_idx, group_imb = 0;
3105 3106 3107 3108 3109 3110
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
	int power_savings_balance = 1;
	unsigned long leader_nr_running = 0, min_load_per_task = 0;
	unsigned long min_nr_running = ULONG_MAX;
	struct sched_group *group_min = NULL, *group_leader = NULL;
#endif
L
Linus Torvalds 已提交
3111 3112

	max_load = this_load = total_load = total_pwr = 0;
3113 3114
	busiest_load_per_task = busiest_nr_running = 0;
	this_load_per_task = this_nr_running = 0;
3115

I
Ingo Molnar 已提交
3116
	if (idle == CPU_NOT_IDLE)
N
Nick Piggin 已提交
3117
		load_idx = sd->busy_idx;
I
Ingo Molnar 已提交
3118
	else if (idle == CPU_NEWLY_IDLE)
N
Nick Piggin 已提交
3119 3120 3121
		load_idx = sd->newidle_idx;
	else
		load_idx = sd->idle_idx;
L
Linus Torvalds 已提交
3122 3123

	do {
3124
		unsigned long load, group_capacity, max_cpu_load, min_cpu_load;
L
Linus Torvalds 已提交
3125 3126
		int local_group;
		int i;
3127
		int __group_imb = 0;
3128
		unsigned int balance_cpu = -1, first_idle_cpu = 0;
3129
		unsigned long sum_nr_running, sum_weighted_load;
3130 3131
		unsigned long sum_avg_load_per_task;
		unsigned long avg_load_per_task;
L
Linus Torvalds 已提交
3132

3133 3134
		local_group = cpumask_test_cpu(this_cpu,
					       sched_group_cpus(group));
L
Linus Torvalds 已提交
3135

3136
		if (local_group)
3137
			balance_cpu = cpumask_first(sched_group_cpus(group));
3138

L
Linus Torvalds 已提交
3139
		/* Tally up the load of all CPUs in the group */
3140
		sum_weighted_load = sum_nr_running = avg_load = 0;
3141 3142
		sum_avg_load_per_task = avg_load_per_task = 0;

3143 3144
		max_cpu_load = 0;
		min_cpu_load = ~0UL;
L
Linus Torvalds 已提交
3145

3146 3147
		for_each_cpu_and(i, sched_group_cpus(group), cpus) {
			struct rq *rq = cpu_rq(i);
3148

3149
			if (*sd_idle && rq->nr_running)
N
Nick Piggin 已提交
3150 3151
				*sd_idle = 0;

L
Linus Torvalds 已提交
3152
			/* Bias balancing toward cpus of our domain */
3153 3154 3155 3156 3157 3158
			if (local_group) {
				if (idle_cpu(i) && !first_idle_cpu) {
					first_idle_cpu = 1;
					balance_cpu = i;
				}

N
Nick Piggin 已提交
3159
				load = target_load(i, load_idx);
3160
			} else {
N
Nick Piggin 已提交
3161
				load = source_load(i, load_idx);
3162 3163 3164 3165 3166
				if (load > max_cpu_load)
					max_cpu_load = load;
				if (min_cpu_load > load)
					min_cpu_load = load;
			}
L
Linus Torvalds 已提交
3167 3168

			avg_load += load;
3169
			sum_nr_running += rq->nr_running;
I
Ingo Molnar 已提交
3170
			sum_weighted_load += weighted_cpuload(i);
3171 3172

			sum_avg_load_per_task += cpu_avg_load_per_task(i);
L
Linus Torvalds 已提交
3173 3174
		}

3175 3176 3177
		/*
		 * First idle cpu or the first cpu(busiest) in this sched group
		 * is eligible for doing load balancing at this and above
3178 3179
		 * domains. In the newly idle case, we will allow all the cpu's
		 * to do the newly idle load balance.
3180
		 */
3181 3182
		if (idle != CPU_NEWLY_IDLE && local_group &&
		    balance_cpu != this_cpu && balance) {
3183 3184 3185 3186
			*balance = 0;
			goto ret;
		}

L
Linus Torvalds 已提交
3187
		total_load += avg_load;
3188
		total_pwr += group->__cpu_power;
L
Linus Torvalds 已提交
3189 3190

		/* Adjust by relative CPU power of the group */
3191 3192
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
3193

3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207

		/*
		 * Consider the group unbalanced when the imbalance is larger
		 * than the average weight of two tasks.
		 *
		 * APZ: with cgroup the avg task weight can vary wildly and
		 *      might not be a suitable number - should we keep a
		 *      normalized nr_running number somewhere that negates
		 *      the hierarchy?
		 */
		avg_load_per_task = sg_div_cpu_power(group,
				sum_avg_load_per_task * SCHED_LOAD_SCALE);

		if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
3208 3209
			__group_imb = 1;

3210
		group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
3211

L
Linus Torvalds 已提交
3212 3213 3214
		if (local_group) {
			this_load = avg_load;
			this = group;
3215 3216 3217
			this_nr_running = sum_nr_running;
			this_load_per_task = sum_weighted_load;
		} else if (avg_load > max_load &&
3218
			   (sum_nr_running > group_capacity || __group_imb)) {
L
Linus Torvalds 已提交
3219 3220
			max_load = avg_load;
			busiest = group;
3221 3222
			busiest_nr_running = sum_nr_running;
			busiest_load_per_task = sum_weighted_load;
3223
			group_imb = __group_imb;
L
Linus Torvalds 已提交
3224
		}
3225 3226 3227 3228 3229 3230

#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
		/*
		 * Busy processors will not participate in power savings
		 * balance.
		 */
I
Ingo Molnar 已提交
3231 3232 3233
		if (idle == CPU_NOT_IDLE ||
				!(sd->flags & SD_POWERSAVINGS_BALANCE))
			goto group_next;
3234 3235 3236 3237 3238 3239 3240 3241 3242

		/*
		 * If the local group is idle or completely loaded
		 * no need to do power savings balance at this domain
		 */
		if (local_group && (this_nr_running >= group_capacity ||
				    !this_nr_running))
			power_savings_balance = 0;

I
Ingo Molnar 已提交
3243
		/*
3244 3245
		 * If a group is already running at full capacity or idle,
		 * don't include that group in power savings calculations
I
Ingo Molnar 已提交
3246 3247
		 */
		if (!power_savings_balance || sum_nr_running >= group_capacity
3248
		    || !sum_nr_running)
I
Ingo Molnar 已提交
3249
			goto group_next;
3250

I
Ingo Molnar 已提交
3251
		/*
3252
		 * Calculate the group which has the least non-idle load.
I
Ingo Molnar 已提交
3253 3254 3255 3256 3257
		 * This is the group from where we need to pick up the load
		 * for saving power
		 */
		if ((sum_nr_running < min_nr_running) ||
		    (sum_nr_running == min_nr_running &&
3258
		     cpumask_first(sched_group_cpus(group)) >
3259
		     cpumask_first(sched_group_cpus(group_min)))) {
I
Ingo Molnar 已提交
3260 3261
			group_min = group;
			min_nr_running = sum_nr_running;
3262 3263
			min_load_per_task = sum_weighted_load /
						sum_nr_running;
I
Ingo Molnar 已提交
3264
		}
3265

I
Ingo Molnar 已提交
3266
		/*
3267
		 * Calculate the group which is almost near its
I
Ingo Molnar 已提交
3268 3269 3270 3271 3272 3273
		 * capacity but still has some space to pick up some load
		 * from other group and save more power
		 */
		if (sum_nr_running <= group_capacity - 1) {
			if (sum_nr_running > leader_nr_running ||
			    (sum_nr_running == leader_nr_running &&
3274
			     cpumask_first(sched_group_cpus(group)) <
3275
			     cpumask_first(sched_group_cpus(group_leader)))) {
I
Ingo Molnar 已提交
3276 3277 3278
				group_leader = group;
				leader_nr_running = sum_nr_running;
			}
3279
		}
3280 3281
group_next:
#endif
L
Linus Torvalds 已提交
3282 3283 3284
		group = group->next;
	} while (group != sd->groups);

3285
	if (!busiest || this_load >= max_load || busiest_nr_running == 0)
L
Linus Torvalds 已提交
3286 3287 3288 3289 3290 3291 3292 3293
		goto out_balanced;

	avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;

	if (this_load >= avg_load ||
			100*max_load <= sd->imbalance_pct*this_load)
		goto out_balanced;

3294
	busiest_load_per_task /= busiest_nr_running;
3295 3296 3297
	if (group_imb)
		busiest_load_per_task = min(busiest_load_per_task, avg_load);

L
Linus Torvalds 已提交
3298 3299 3300 3301 3302 3303 3304 3305
	/*
	 * We're trying to get all the cpus to the average_load, so we don't
	 * want to push ourselves above the average load, nor do we wish to
	 * reduce the max loaded cpu below the average load, as either of these
	 * actions would just result in more rebalancing later, and ping-pong
	 * tasks around. Thus we look for the minimum possible imbalance.
	 * Negative imbalances (*we* are more loaded than anyone else) will
	 * be counted as no imbalance for these purposes -- we can't fix that
I
Ingo Molnar 已提交
3306
	 * by pulling tasks to us. Be careful of negative numbers as they'll
L
Linus Torvalds 已提交
3307 3308
	 * appear as very large values with unsigned longs.
	 */
3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320
	if (max_load <= busiest_load_per_task)
		goto out_balanced;

	/*
	 * In the presence of smp nice balancing, certain scenarios can have
	 * max load less than avg load(as we skip the groups at or below
	 * its cpu_power, while calculating max_load..)
	 */
	if (max_load < avg_load) {
		*imbalance = 0;
		goto small_imbalance;
	}
3321 3322

	/* Don't want to pull so many tasks that a group would go idle */
3323
	max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
3324

L
Linus Torvalds 已提交
3325
	/* How much load to actually move to equalise the imbalance */
3326 3327
	*imbalance = min(max_pull * busiest->__cpu_power,
				(avg_load - this_load) * this->__cpu_power)
L
Linus Torvalds 已提交
3328 3329
			/ SCHED_LOAD_SCALE;

3330 3331 3332 3333 3334 3335
	/*
	 * if *imbalance is less than the average load per runnable task
	 * there is no gaurantee that any tasks will be moved so we'll have
	 * a think about bumping its value to force at least one task to be
	 * moved
	 */
3336
	if (*imbalance < busiest_load_per_task) {
3337
		unsigned long tmp, pwr_now, pwr_move;
3338 3339 3340 3341 3342 3343 3344 3345 3346 3347
		unsigned int imbn;

small_imbalance:
		pwr_move = pwr_now = 0;
		imbn = 2;
		if (this_nr_running) {
			this_load_per_task /= this_nr_running;
			if (busiest_load_per_task > this_load_per_task)
				imbn = 1;
		} else
3348
			this_load_per_task = cpu_avg_load_per_task(this_cpu);
L
Linus Torvalds 已提交
3349

3350
		if (max_load - this_load + busiest_load_per_task >=
I
Ingo Molnar 已提交
3351
					busiest_load_per_task * imbn) {
3352
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
3353 3354 3355 3356 3357 3358 3359 3360 3361
			return busiest;
		}

		/*
		 * OK, we don't have enough imbalance to justify moving tasks,
		 * however we may be able to increase total CPU power used by
		 * moving them.
		 */

3362 3363 3364 3365
		pwr_now += busiest->__cpu_power *
				min(busiest_load_per_task, max_load);
		pwr_now += this->__cpu_power *
				min(this_load_per_task, this_load);
L
Linus Torvalds 已提交
3366 3367 3368
		pwr_now /= SCHED_LOAD_SCALE;

		/* Amount of load we'd subtract */
3369 3370
		tmp = sg_div_cpu_power(busiest,
				busiest_load_per_task * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
3371
		if (max_load > tmp)
3372
			pwr_move += busiest->__cpu_power *
3373
				min(busiest_load_per_task, max_load - tmp);
L
Linus Torvalds 已提交
3374 3375

		/* Amount of load we'd add */
3376
		if (max_load * busiest->__cpu_power <
3377
				busiest_load_per_task * SCHED_LOAD_SCALE)
3378 3379
			tmp = sg_div_cpu_power(this,
					max_load * busiest->__cpu_power);
L
Linus Torvalds 已提交
3380
		else
3381 3382 3383 3384
			tmp = sg_div_cpu_power(this,
				busiest_load_per_task * SCHED_LOAD_SCALE);
		pwr_move += this->__cpu_power *
				min(this_load_per_task, this_load + tmp);
L
Linus Torvalds 已提交
3385 3386 3387
		pwr_move /= SCHED_LOAD_SCALE;

		/* Move if we gain throughput */
3388 3389
		if (pwr_move > pwr_now)
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
3390 3391 3392 3393 3394
	}

	return busiest;

out_balanced:
3395
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
3396
	if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
3397
		goto ret;
L
Linus Torvalds 已提交
3398

3399 3400
	if (this == group_leader && group_leader != group_min) {
		*imbalance = min_load_per_task;
3401 3402
		if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
			cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
I
Ingo Molnar 已提交
3403
				cpumask_first(sched_group_cpus(group_leader));
3404
		}
3405 3406 3407
		return group_min;
	}
#endif
3408
ret:
L
Linus Torvalds 已提交
3409 3410 3411 3412 3413 3414 3415
	*imbalance = 0;
	return NULL;
}

/*
 * find_busiest_queue - find the busiest runqueue among the cpus in group.
 */
3416
static struct rq *
I
Ingo Molnar 已提交
3417
find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3418
		   unsigned long imbalance, const struct cpumask *cpus)
L
Linus Torvalds 已提交
3419
{
3420
	struct rq *busiest = NULL, *rq;
3421
	unsigned long max_load = 0;
L
Linus Torvalds 已提交
3422 3423
	int i;

3424
	for_each_cpu(i, sched_group_cpus(group)) {
I
Ingo Molnar 已提交
3425
		unsigned long wl;
3426

3427
		if (!cpumask_test_cpu(i, cpus))
3428 3429
			continue;

3430
		rq = cpu_rq(i);
I
Ingo Molnar 已提交
3431
		wl = weighted_cpuload(i);
3432

I
Ingo Molnar 已提交
3433
		if (rq->nr_running == 1 && wl > imbalance)
3434
			continue;
L
Linus Torvalds 已提交
3435

I
Ingo Molnar 已提交
3436 3437
		if (wl > max_load) {
			max_load = wl;
3438
			busiest = rq;
L
Linus Torvalds 已提交
3439 3440 3441 3442 3443 3444
		}
	}

	return busiest;
}

3445 3446 3447 3448 3449 3450
/*
 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
 * so long as it is large enough.
 */
#define MAX_PINNED_INTERVAL	512

L
Linus Torvalds 已提交
3451 3452 3453 3454
/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
3455
static int load_balance(int this_cpu, struct rq *this_rq,
I
Ingo Molnar 已提交
3456
			struct sched_domain *sd, enum cpu_idle_type idle,
3457
			int *balance, struct cpumask *cpus)
L
Linus Torvalds 已提交
3458
{
P
Peter Williams 已提交
3459
	int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
L
Linus Torvalds 已提交
3460 3461
	struct sched_group *group;
	unsigned long imbalance;
3462
	struct rq *busiest;
3463
	unsigned long flags;
N
Nick Piggin 已提交
3464

3465
	cpumask_setall(cpus);
3466

3467 3468 3469
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
I
Ingo Molnar 已提交
3470
	 * let the state of idle sibling percolate up as CPU_IDLE, instead of
I
Ingo Molnar 已提交
3471
	 * portraying it as CPU_NOT_IDLE.
3472
	 */
I
Ingo Molnar 已提交
3473
	if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
3474
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3475
		sd_idle = 1;
L
Linus Torvalds 已提交
3476

3477
	schedstat_inc(sd, lb_count[idle]);
L
Linus Torvalds 已提交
3478

3479
redo:
3480
	update_shares(sd);
3481
	group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
3482
				   cpus, balance);
3483

3484
	if (*balance == 0)
3485 3486
		goto out_balanced;

L
Linus Torvalds 已提交
3487 3488 3489 3490 3491
	if (!group) {
		schedstat_inc(sd, lb_nobusyg[idle]);
		goto out_balanced;
	}

3492
	busiest = find_busiest_queue(group, idle, imbalance, cpus);
L
Linus Torvalds 已提交
3493 3494 3495 3496 3497
	if (!busiest) {
		schedstat_inc(sd, lb_nobusyq[idle]);
		goto out_balanced;
	}

N
Nick Piggin 已提交
3498
	BUG_ON(busiest == this_rq);
L
Linus Torvalds 已提交
3499 3500 3501

	schedstat_add(sd, lb_imbalance[idle], imbalance);

P
Peter Williams 已提交
3502
	ld_moved = 0;
L
Linus Torvalds 已提交
3503 3504 3505 3506
	if (busiest->nr_running > 1) {
		/*
		 * Attempt to move tasks. If find_busiest_group has found
		 * an imbalance but busiest->nr_running <= 1, the group is
P
Peter Williams 已提交
3507
		 * still unbalanced. ld_moved simply stays zero, so it is
L
Linus Torvalds 已提交
3508 3509
		 * correctly treated as an imbalance.
		 */
3510
		local_irq_save(flags);
N
Nick Piggin 已提交
3511
		double_rq_lock(this_rq, busiest);
P
Peter Williams 已提交
3512
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
3513
				      imbalance, sd, idle, &all_pinned);
N
Nick Piggin 已提交
3514
		double_rq_unlock(this_rq, busiest);
3515
		local_irq_restore(flags);
3516

3517 3518 3519
		/*
		 * some other cpu did the load balance for us.
		 */
P
Peter Williams 已提交
3520
		if (ld_moved && this_cpu != smp_processor_id())
3521 3522
			resched_cpu(this_cpu);

3523
		/* All tasks on this runqueue were pinned by CPU affinity */
3524
		if (unlikely(all_pinned)) {
3525 3526
			cpumask_clear_cpu(cpu_of(busiest), cpus);
			if (!cpumask_empty(cpus))
3527
				goto redo;
3528
			goto out_balanced;
3529
		}
L
Linus Torvalds 已提交
3530
	}
3531

P
Peter Williams 已提交
3532
	if (!ld_moved) {
L
Linus Torvalds 已提交
3533 3534 3535 3536 3537
		schedstat_inc(sd, lb_failed[idle]);
		sd->nr_balance_failed++;

		if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {

3538
			spin_lock_irqsave(&busiest->lock, flags);
3539 3540 3541 3542

			/* don't kick the migration_thread, if the curr
			 * task on busiest cpu can't be moved to this_cpu
			 */
3543 3544
			if (!cpumask_test_cpu(this_cpu,
					      &busiest->curr->cpus_allowed)) {
3545
				spin_unlock_irqrestore(&busiest->lock, flags);
3546 3547 3548 3549
				all_pinned = 1;
				goto out_one_pinned;
			}

L
Linus Torvalds 已提交
3550 3551 3552
			if (!busiest->active_balance) {
				busiest->active_balance = 1;
				busiest->push_cpu = this_cpu;
3553
				active_balance = 1;
L
Linus Torvalds 已提交
3554
			}
3555
			spin_unlock_irqrestore(&busiest->lock, flags);
3556
			if (active_balance)
L
Linus Torvalds 已提交
3557 3558 3559 3560 3561 3562
				wake_up_process(busiest->migration_thread);

			/*
			 * We've kicked active balancing, reset the failure
			 * counter.
			 */
3563
			sd->nr_balance_failed = sd->cache_nice_tries+1;
L
Linus Torvalds 已提交
3564
		}
3565
	} else
L
Linus Torvalds 已提交
3566 3567
		sd->nr_balance_failed = 0;

3568
	if (likely(!active_balance)) {
L
Linus Torvalds 已提交
3569 3570
		/* We were unbalanced, so reset the balancing interval */
		sd->balance_interval = sd->min_interval;
3571 3572 3573 3574 3575 3576 3577 3578 3579
	} else {
		/*
		 * If we've begun active balancing, start to back off. This
		 * case may not be covered by the all_pinned logic if there
		 * is only 1 task on the busy runqueue (because we don't call
		 * move_tasks).
		 */
		if (sd->balance_interval < sd->max_interval)
			sd->balance_interval *= 2;
L
Linus Torvalds 已提交
3580 3581
	}

P
Peter Williams 已提交
3582
	if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3583
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3584 3585 3586
		ld_moved = -1;

	goto out;
L
Linus Torvalds 已提交
3587 3588 3589 3590

out_balanced:
	schedstat_inc(sd, lb_balanced[idle]);

3591
	sd->nr_balance_failed = 0;
3592 3593

out_one_pinned:
L
Linus Torvalds 已提交
3594
	/* tune up the balancing interval */
3595 3596
	if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
			(sd->balance_interval < sd->max_interval))
L
Linus Torvalds 已提交
3597 3598
		sd->balance_interval *= 2;

3599
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3600
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3601 3602 3603 3604
		ld_moved = -1;
	else
		ld_moved = 0;
out:
3605 3606
	if (ld_moved)
		update_shares(sd);
3607
	return ld_moved;
L
Linus Torvalds 已提交
3608 3609 3610 3611 3612 3613
}

/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 *
I
Ingo Molnar 已提交
3614
 * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
L
Linus Torvalds 已提交
3615 3616
 * this_rq is locked.
 */
3617
static int
3618
load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3619
			struct cpumask *cpus)
L
Linus Torvalds 已提交
3620 3621
{
	struct sched_group *group;
3622
	struct rq *busiest = NULL;
L
Linus Torvalds 已提交
3623
	unsigned long imbalance;
P
Peter Williams 已提交
3624
	int ld_moved = 0;
N
Nick Piggin 已提交
3625
	int sd_idle = 0;
3626
	int all_pinned = 0;
3627

3628
	cpumask_setall(cpus);
N
Nick Piggin 已提交
3629

3630 3631 3632 3633
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
	 * let the state of idle sibling percolate up as IDLE, instead of
I
Ingo Molnar 已提交
3634
	 * portraying it as CPU_NOT_IDLE.
3635 3636 3637
	 */
	if (sd->flags & SD_SHARE_CPUPOWER &&
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3638
		sd_idle = 1;
L
Linus Torvalds 已提交
3639

3640
	schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
3641
redo:
3642
	update_shares_locked(this_rq, sd);
I
Ingo Molnar 已提交
3643
	group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
3644
				   &sd_idle, cpus, NULL);
L
Linus Torvalds 已提交
3645
	if (!group) {
I
Ingo Molnar 已提交
3646
		schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
3647
		goto out_balanced;
L
Linus Torvalds 已提交
3648 3649
	}

3650
	busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance, cpus);
N
Nick Piggin 已提交
3651
	if (!busiest) {
I
Ingo Molnar 已提交
3652
		schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
3653
		goto out_balanced;
L
Linus Torvalds 已提交
3654 3655
	}

N
Nick Piggin 已提交
3656 3657
	BUG_ON(busiest == this_rq);

I
Ingo Molnar 已提交
3658
	schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
3659

P
Peter Williams 已提交
3660
	ld_moved = 0;
3661 3662 3663
	if (busiest->nr_running > 1) {
		/* Attempt to move tasks */
		double_lock_balance(this_rq, busiest);
3664 3665
		/* this_rq->clock is already updated */
		update_rq_clock(busiest);
P
Peter Williams 已提交
3666
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
3667 3668
					imbalance, sd, CPU_NEWLY_IDLE,
					&all_pinned);
3669
		double_unlock_balance(this_rq, busiest);
3670

3671
		if (unlikely(all_pinned)) {
3672 3673
			cpumask_clear_cpu(cpu_of(busiest), cpus);
			if (!cpumask_empty(cpus))
3674 3675
				goto redo;
		}
3676 3677
	}

P
Peter Williams 已提交
3678
	if (!ld_moved) {
3679
		int active_balance = 0;
3680

I
Ingo Molnar 已提交
3681
		schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3682 3683
		if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
		    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3684
			return -1;
3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720

		if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
			return -1;

		if (sd->nr_balance_failed++ < 2)
			return -1;

		/*
		 * The only task running in a non-idle cpu can be moved to this
		 * cpu in an attempt to completely freeup the other CPU
		 * package. The same method used to move task in load_balance()
		 * have been extended for load_balance_newidle() to speedup
		 * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
		 *
		 * The package power saving logic comes from
		 * find_busiest_group().  If there are no imbalance, then
		 * f_b_g() will return NULL.  However when sched_mc={1,2} then
		 * f_b_g() will select a group from which a running task may be
		 * pulled to this cpu in order to make the other package idle.
		 * If there is no opportunity to make a package idle and if
		 * there are no imbalance, then f_b_g() will return NULL and no
		 * action will be taken in load_balance_newidle().
		 *
		 * Under normal task pull operation due to imbalance, there
		 * will be more than one task in the source run queue and
		 * move_tasks() will succeed.  ld_moved will be true and this
		 * active balance code will not be triggered.
		 */

		/* Lock busiest in correct order while this_rq is held */
		double_lock_balance(this_rq, busiest);

		/*
		 * don't kick the migration_thread, if the curr
		 * task on busiest cpu can't be moved to this_cpu
		 */
3721
		if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733
			double_unlock_balance(this_rq, busiest);
			all_pinned = 1;
			return ld_moved;
		}

		if (!busiest->active_balance) {
			busiest->active_balance = 1;
			busiest->push_cpu = this_cpu;
			active_balance = 1;
		}

		double_unlock_balance(this_rq, busiest);
3734 3735 3736 3737
		/*
		 * Should not call ttwu while holding a rq->lock
		 */
		spin_unlock(&this_rq->lock);
3738 3739
		if (active_balance)
			wake_up_process(busiest->migration_thread);
3740
		spin_lock(&this_rq->lock);
3741

N
Nick Piggin 已提交
3742
	} else
3743
		sd->nr_balance_failed = 0;
L
Linus Torvalds 已提交
3744

3745
	update_shares_locked(this_rq, sd);
P
Peter Williams 已提交
3746
	return ld_moved;
3747 3748

out_balanced:
I
Ingo Molnar 已提交
3749
	schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
3750
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3751
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3752
		return -1;
3753
	sd->nr_balance_failed = 0;
3754

3755
	return 0;
L
Linus Torvalds 已提交
3756 3757 3758 3759 3760 3761
}

/*
 * idle_balance is called by schedule() if this_cpu is about to become
 * idle. Attempts to pull tasks from other CPUs.
 */
3762
static void idle_balance(int this_cpu, struct rq *this_rq)
L
Linus Torvalds 已提交
3763 3764
{
	struct sched_domain *sd;
3765
	int pulled_task = 0;
I
Ingo Molnar 已提交
3766
	unsigned long next_balance = jiffies + HZ;
3767 3768 3769 3770
	cpumask_var_t tmpmask;

	if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
		return;
L
Linus Torvalds 已提交
3771 3772

	for_each_domain(this_cpu, sd) {
3773 3774 3775 3776 3777 3778
		unsigned long interval;

		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		if (sd->flags & SD_BALANCE_NEWIDLE)
3779
			/* If we've pulled tasks over stop searching: */
3780
			pulled_task = load_balance_newidle(this_cpu, this_rq,
3781
							   sd, tmpmask);
3782 3783 3784 3785 3786 3787

		interval = msecs_to_jiffies(sd->balance_interval);
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
		if (pulled_task)
			break;
L
Linus Torvalds 已提交
3788
	}
I
Ingo Molnar 已提交
3789
	if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
3790 3791 3792 3793 3794
		/*
		 * We are going idle. next_balance may be set based on
		 * a busy processor. So reset next_balance.
		 */
		this_rq->next_balance = next_balance;
I
Ingo Molnar 已提交
3795
	}
3796
	free_cpumask_var(tmpmask);
L
Linus Torvalds 已提交
3797 3798 3799 3800 3801 3802 3803 3804 3805 3806
}

/*
 * active_load_balance is run by migration threads. It pushes running tasks
 * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
 * running on each physical CPU where possible, and avoids physical /
 * logical imbalances.
 *
 * Called with busiest_rq locked.
 */
3807
static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
L
Linus Torvalds 已提交
3808
{
3809
	int target_cpu = busiest_rq->push_cpu;
3810 3811
	struct sched_domain *sd;
	struct rq *target_rq;
3812

3813
	/* Is there any task to move? */
3814 3815 3816 3817
	if (busiest_rq->nr_running <= 1)
		return;

	target_rq = cpu_rq(target_cpu);
L
Linus Torvalds 已提交
3818 3819

	/*
3820
	 * This condition is "impossible", if it occurs
I
Ingo Molnar 已提交
3821
	 * we need to fix it. Originally reported by
3822
	 * Bjorn Helgaas on a 128-cpu setup.
L
Linus Torvalds 已提交
3823
	 */
3824
	BUG_ON(busiest_rq == target_rq);
L
Linus Torvalds 已提交
3825

3826 3827
	/* move a task from busiest_rq to target_rq */
	double_lock_balance(busiest_rq, target_rq);
3828 3829
	update_rq_clock(busiest_rq);
	update_rq_clock(target_rq);
3830 3831

	/* Search for an sd spanning us and the target CPU. */
3832
	for_each_domain(target_cpu, sd) {
3833
		if ((sd->flags & SD_LOAD_BALANCE) &&
3834
		    cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3835
				break;
3836
	}
3837

3838
	if (likely(sd)) {
3839
		schedstat_inc(sd, alb_count);
3840

P
Peter Williams 已提交
3841 3842
		if (move_one_task(target_rq, target_cpu, busiest_rq,
				  sd, CPU_IDLE))
3843 3844 3845 3846
			schedstat_inc(sd, alb_pushed);
		else
			schedstat_inc(sd, alb_failed);
	}
3847
	double_unlock_balance(busiest_rq, target_rq);
L
Linus Torvalds 已提交
3848 3849
}

3850 3851 3852
#ifdef CONFIG_NO_HZ
static struct {
	atomic_t load_balancer;
3853
	cpumask_var_t cpu_mask;
3854 3855 3856 3857
} nohz ____cacheline_aligned = {
	.load_balancer = ATOMIC_INIT(-1),
};

3858
/*
3859 3860 3861 3862 3863 3864 3865 3866 3867 3868
 * This routine will try to nominate the ilb (idle load balancing)
 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
 * load balancing on behalf of all those cpus. If all the cpus in the system
 * go into this tickless mode, then there will be no ilb owner (as there is
 * no need for one) and all the cpus will sleep till the next wakeup event
 * arrives...
 *
 * For the ilb owner, tick is not stopped. And this tick will be used
 * for idle load balancing. ilb owner will still be part of
 * nohz.cpu_mask..
3869
 *
3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882
 * While stopping the tick, this cpu will become the ilb owner if there
 * is no other owner. And will be the owner till that cpu becomes busy
 * or if all cpus in the system stop their ticks at which point
 * there is no need for ilb owner.
 *
 * When the ilb owner becomes busy, it nominates another owner, during the
 * next busy scheduler_tick()
 */
int select_nohz_load_balancer(int stop_tick)
{
	int cpu = smp_processor_id();

	if (stop_tick) {
3883
		cpumask_set_cpu(cpu, nohz.cpu_mask);
3884 3885 3886 3887 3888
		cpu_rq(cpu)->in_nohz_recently = 1;

		/*
		 * If we are going offline and still the leader, give up!
		 */
3889
		if (!cpu_active(cpu) &&
3890 3891 3892 3893 3894 3895 3896
		    atomic_read(&nohz.load_balancer) == cpu) {
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
			return 0;
		}

		/* time for ilb owner also to sleep */
3897
		if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909
			if (atomic_read(&nohz.load_balancer) == cpu)
				atomic_set(&nohz.load_balancer, -1);
			return 0;
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/* make me the ilb owner */
			if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
				return 1;
		} else if (atomic_read(&nohz.load_balancer) == cpu)
			return 1;
	} else {
3910
		if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3911 3912
			return 0;

3913
		cpumask_clear_cpu(cpu, nohz.cpu_mask);
3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925

		if (atomic_read(&nohz.load_balancer) == cpu)
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
	}
	return 0;
}
#endif

static DEFINE_SPINLOCK(balancing);

/*
3926 3927 3928 3929 3930
 * It checks each scheduling domain to see if it is due to be balanced,
 * and initiates a balancing operation if so.
 *
 * Balancing parameters are set up in arch_init_sched_domains.
 */
A
Alexey Dobriyan 已提交
3931
static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3932
{
3933 3934
	int balance = 1;
	struct rq *rq = cpu_rq(cpu);
3935 3936
	unsigned long interval;
	struct sched_domain *sd;
3937
	/* Earliest time when we have to do rebalance again */
3938
	unsigned long next_balance = jiffies + 60*HZ;
3939
	int update_next_balance = 0;
3940
	int need_serialize;
3941 3942 3943 3944 3945
	cpumask_var_t tmp;

	/* Fails alloc?  Rebalancing probably not a priority right now. */
	if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
		return;
L
Linus Torvalds 已提交
3946

3947
	for_each_domain(cpu, sd) {
L
Linus Torvalds 已提交
3948 3949 3950 3951
		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		interval = sd->balance_interval;
I
Ingo Molnar 已提交
3952
		if (idle != CPU_IDLE)
L
Linus Torvalds 已提交
3953 3954 3955 3956 3957 3958
			interval *= sd->busy_factor;

		/* scale ms to jiffies */
		interval = msecs_to_jiffies(interval);
		if (unlikely(!interval))
			interval = 1;
I
Ingo Molnar 已提交
3959 3960 3961
		if (interval > HZ*NR_CPUS/10)
			interval = HZ*NR_CPUS/10;

3962
		need_serialize = sd->flags & SD_SERIALIZE;
L
Linus Torvalds 已提交
3963

3964
		if (need_serialize) {
3965 3966 3967 3968
			if (!spin_trylock(&balancing))
				goto out;
		}

3969
		if (time_after_eq(jiffies, sd->last_balance + interval)) {
3970
			if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3971 3972
				/*
				 * We've pulled tasks over so either we're no
N
Nick Piggin 已提交
3973 3974 3975
				 * longer idle, or one of our SMT siblings is
				 * not idle.
				 */
I
Ingo Molnar 已提交
3976
				idle = CPU_NOT_IDLE;
L
Linus Torvalds 已提交
3977
			}
3978
			sd->last_balance = jiffies;
L
Linus Torvalds 已提交
3979
		}
3980
		if (need_serialize)
3981 3982
			spin_unlock(&balancing);
out:
3983
		if (time_after(next_balance, sd->last_balance + interval)) {
3984
			next_balance = sd->last_balance + interval;
3985 3986
			update_next_balance = 1;
		}
3987 3988 3989 3990 3991 3992 3993 3994

		/*
		 * Stop the load balance at this level. There is another
		 * CPU in our sched group which is doing load balancing more
		 * actively.
		 */
		if (!balance)
			break;
L
Linus Torvalds 已提交
3995
	}
3996 3997 3998 3999 4000 4001 4002 4003

	/*
	 * next_balance will be updated only when there is a need.
	 * When the cpu is attached to null domain for ex, it will not be
	 * updated.
	 */
	if (likely(update_next_balance))
		rq->next_balance = next_balance;
4004 4005

	free_cpumask_var(tmp);
4006 4007 4008 4009 4010 4011 4012 4013 4014
}

/*
 * run_rebalance_domains is triggered when needed from the scheduler tick.
 * In CONFIG_NO_HZ case, the idle load balance owner will do the
 * rebalancing for all the cpus for whom scheduler ticks are stopped.
 */
static void run_rebalance_domains(struct softirq_action *h)
{
I
Ingo Molnar 已提交
4015 4016 4017 4018
	int this_cpu = smp_processor_id();
	struct rq *this_rq = cpu_rq(this_cpu);
	enum cpu_idle_type idle = this_rq->idle_at_tick ?
						CPU_IDLE : CPU_NOT_IDLE;
4019

I
Ingo Molnar 已提交
4020
	rebalance_domains(this_cpu, idle);
4021 4022 4023 4024 4025 4026 4027

#ifdef CONFIG_NO_HZ
	/*
	 * If this cpu is the owner for idle load balancing, then do the
	 * balancing on behalf of the other idle cpus whose ticks are
	 * stopped.
	 */
I
Ingo Molnar 已提交
4028 4029
	if (this_rq->idle_at_tick &&
	    atomic_read(&nohz.load_balancer) == this_cpu) {
4030 4031 4032
		struct rq *rq;
		int balance_cpu;

4033 4034 4035 4036
		for_each_cpu(balance_cpu, nohz.cpu_mask) {
			if (balance_cpu == this_cpu)
				continue;

4037 4038 4039 4040 4041 4042 4043 4044
			/*
			 * If this cpu gets work to do, stop the load balancing
			 * work being done for other cpus. Next load
			 * balancing owner will pick it up.
			 */
			if (need_resched())
				break;

4045
			rebalance_domains(balance_cpu, CPU_IDLE);
4046 4047

			rq = cpu_rq(balance_cpu);
I
Ingo Molnar 已提交
4048 4049
			if (time_after(this_rq->next_balance, rq->next_balance))
				this_rq->next_balance = rq->next_balance;
4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061
		}
	}
#endif
}

/*
 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
 *
 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
 * idle load balancing owner or decide to stop the periodic load balancing,
 * if the whole system is idle.
 */
I
Ingo Molnar 已提交
4062
static inline void trigger_load_balance(struct rq *rq, int cpu)
4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073
{
#ifdef CONFIG_NO_HZ
	/*
	 * If we were in the nohz mode recently and busy at the current
	 * scheduler tick, then check if we need to nominate new idle
	 * load balancer.
	 */
	if (rq->in_nohz_recently && !rq->idle_at_tick) {
		rq->in_nohz_recently = 0;

		if (atomic_read(&nohz.load_balancer) == cpu) {
4074
			cpumask_clear_cpu(cpu, nohz.cpu_mask);
4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086
			atomic_set(&nohz.load_balancer, -1);
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/*
			 * simple selection for now: Nominate the
			 * first cpu in the nohz list to be the next
			 * ilb owner.
			 *
			 * TBD: Traverse the sched domains and nominate
			 * the nearest cpu in the nohz.cpu_mask.
			 */
4087
			int ilb = cpumask_first(nohz.cpu_mask);
4088

4089
			if (ilb < nr_cpu_ids)
4090 4091 4092 4093 4094 4095 4096 4097 4098
				resched_cpu(ilb);
		}
	}

	/*
	 * If this cpu is idle and doing idle load balancing for all the
	 * cpus with ticks stopped, is it time for that to stop?
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4099
	    cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4100 4101 4102 4103 4104 4105 4106 4107 4108
		resched_cpu(cpu);
		return;
	}

	/*
	 * If this cpu is idle and the idle load balancing is done by
	 * someone else, then no need raise the SCHED_SOFTIRQ
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4109
	    cpumask_test_cpu(cpu, nohz.cpu_mask))
4110 4111 4112 4113
		return;
#endif
	if (time_after_eq(jiffies, rq->next_balance))
		raise_softirq(SCHED_SOFTIRQ);
L
Linus Torvalds 已提交
4114
}
I
Ingo Molnar 已提交
4115 4116 4117

#else	/* CONFIG_SMP */

L
Linus Torvalds 已提交
4118 4119 4120
/*
 * on UP we do not need to balance between CPUs:
 */
4121
static inline void idle_balance(int cpu, struct rq *rq)
L
Linus Torvalds 已提交
4122 4123
{
}
I
Ingo Molnar 已提交
4124

L
Linus Torvalds 已提交
4125 4126 4127 4128 4129 4130 4131
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
4132 4133
 * Return any ns on the sched_clock that have not yet been banked in
 * @p in case that task is currently running.
L
Linus Torvalds 已提交
4134
 */
4135
unsigned long long task_delta_exec(struct task_struct *p)
L
Linus Torvalds 已提交
4136 4137
{
	unsigned long flags;
4138
	struct rq *rq;
4139
	u64 ns = 0;
4140

4141
	rq = task_rq_lock(p, &flags);
4142

4143
	if (task_current(rq, p)) {
4144 4145
		u64 delta_exec;

I
Ingo Molnar 已提交
4146 4147
		update_rq_clock(rq);
		delta_exec = rq->clock - p->se.exec_start;
4148
		if ((s64)delta_exec > 0)
4149
			ns = delta_exec;
4150
	}
4151

4152
	task_rq_unlock(rq, &flags);
4153

L
Linus Torvalds 已提交
4154 4155 4156 4157 4158 4159 4160
	return ns;
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in user space since the last update
4161
 * @cputime_scaled: cputime scaled by cpu frequency
L
Linus Torvalds 已提交
4162
 */
4163 4164
void account_user_time(struct task_struct *p, cputime_t cputime,
		       cputime_t cputime_scaled)
L
Linus Torvalds 已提交
4165 4166 4167 4168
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

4169
	/* Add user time to process. */
L
Linus Torvalds 已提交
4170
	p->utime = cputime_add(p->utime, cputime);
4171
	p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4172
	account_group_user_time(p, cputime);
L
Linus Torvalds 已提交
4173 4174 4175 4176 4177 4178 4179

	/* Add user time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (TASK_NICE(p) > 0)
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
	else
		cpustat->user = cputime64_add(cpustat->user, tmp);
4180 4181
	/* Account for user time used */
	acct_update_integrals(p);
L
Linus Torvalds 已提交
4182 4183
}

4184 4185 4186 4187
/*
 * Account guest cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in virtual machine since the last update
4188
 * @cputime_scaled: cputime scaled by cpu frequency
4189
 */
4190 4191
static void account_guest_time(struct task_struct *p, cputime_t cputime,
			       cputime_t cputime_scaled)
4192 4193 4194 4195 4196 4197
{
	cputime64_t tmp;
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;

	tmp = cputime_to_cputime64(cputime);

4198
	/* Add guest time to process. */
4199
	p->utime = cputime_add(p->utime, cputime);
4200
	p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4201
	account_group_user_time(p, cputime);
4202 4203
	p->gtime = cputime_add(p->gtime, cputime);

4204
	/* Add guest time to cpustat. */
4205 4206 4207 4208
	cpustat->user = cputime64_add(cpustat->user, tmp);
	cpustat->guest = cputime64_add(cpustat->guest, tmp);
}

L
Linus Torvalds 已提交
4209 4210 4211 4212 4213
/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
4214
 * @cputime_scaled: cputime scaled by cpu frequency
L
Linus Torvalds 已提交
4215 4216
 */
void account_system_time(struct task_struct *p, int hardirq_offset,
4217
			 cputime_t cputime, cputime_t cputime_scaled)
L
Linus Torvalds 已提交
4218 4219 4220 4221
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

4222
	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
4223
		account_guest_time(p, cputime, cputime_scaled);
4224 4225
		return;
	}
4226

4227
	/* Add system time to process. */
L
Linus Torvalds 已提交
4228
	p->stime = cputime_add(p->stime, cputime);
4229
	p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
4230
	account_group_system_time(p, cputime);
L
Linus Torvalds 已提交
4231 4232 4233 4234 4235 4236 4237 4238

	/* Add system time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (hardirq_count() - hardirq_offset)
		cpustat->irq = cputime64_add(cpustat->irq, tmp);
	else if (softirq_count())
		cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
	else
4239 4240
		cpustat->system = cputime64_add(cpustat->system, tmp);

L
Linus Torvalds 已提交
4241 4242 4243 4244
	/* Account for system time used */
	acct_update_integrals(p);
}

4245
/*
L
Linus Torvalds 已提交
4246 4247
 * Account for involuntary wait time.
 * @steal: the cpu time spent in involuntary wait
4248
 */
4249
void account_steal_time(cputime_t cputime)
4250
{
4251 4252 4253 4254
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t cputime64 = cputime_to_cputime64(cputime);

	cpustat->steal = cputime64_add(cpustat->steal, cputime64);
4255 4256
}

L
Linus Torvalds 已提交
4257
/*
4258 4259
 * Account for idle time.
 * @cputime: the cpu time spent in idle wait
L
Linus Torvalds 已提交
4260
 */
4261
void account_idle_time(cputime_t cputime)
L
Linus Torvalds 已提交
4262 4263
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4264
	cputime64_t cputime64 = cputime_to_cputime64(cputime);
4265
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
4266

4267 4268 4269 4270
	if (atomic_read(&rq->nr_iowait) > 0)
		cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
	else
		cpustat->idle = cputime64_add(cpustat->idle, cputime64);
L
Linus Torvalds 已提交
4271 4272
}

4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311
#ifndef CONFIG_VIRT_CPU_ACCOUNTING

/*
 * Account a single tick of cpu time.
 * @p: the process that the cpu time gets accounted to
 * @user_tick: indicates if the tick is a user or a system tick
 */
void account_process_tick(struct task_struct *p, int user_tick)
{
	cputime_t one_jiffy = jiffies_to_cputime(1);
	cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
	struct rq *rq = this_rq();

	if (user_tick)
		account_user_time(p, one_jiffy, one_jiffy_scaled);
	else if (p != rq->idle)
		account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
				    one_jiffy_scaled);
	else
		account_idle_time(one_jiffy);
}

/*
 * Account multiple ticks of steal time.
 * @p: the process from which the cpu time has been stolen
 * @ticks: number of stolen ticks
 */
void account_steal_ticks(unsigned long ticks)
{
	account_steal_time(jiffies_to_cputime(ticks));
}

/*
 * Account multiple ticks of idle time.
 * @ticks: number of stolen ticks
 */
void account_idle_ticks(unsigned long ticks)
{
	account_idle_time(jiffies_to_cputime(ticks));
L
Linus Torvalds 已提交
4312 4313
}

4314 4315
#endif

4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374
/*
 * Use precise platform statistics if available:
 */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
cputime_t task_utime(struct task_struct *p)
{
	return p->utime;
}

cputime_t task_stime(struct task_struct *p)
{
	return p->stime;
}
#else
cputime_t task_utime(struct task_struct *p)
{
	clock_t utime = cputime_to_clock_t(p->utime),
		total = utime + cputime_to_clock_t(p->stime);
	u64 temp;

	/*
	 * Use CFS's precise accounting:
	 */
	temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime);

	if (total) {
		temp *= utime;
		do_div(temp, total);
	}
	utime = (clock_t)temp;

	p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime));
	return p->prev_utime;
}

cputime_t task_stime(struct task_struct *p)
{
	clock_t stime;

	/*
	 * Use CFS's precise accounting. (we subtract utime from
	 * the total, to make sure the total observed by userspace
	 * grows monotonically - apps rely on that):
	 */
	stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
			cputime_to_clock_t(task_utime(p));

	if (stime >= 0)
		p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime));

	return p->prev_stime;
}
#endif

inline cputime_t task_gtime(struct task_struct *p)
{
	return p->gtime;
}

4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 *
 * It also gets called by the fork code, when changing the parent's
 * timeslices.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
4386
	struct task_struct *curr = rq->curr;
4387 4388

	sched_clock_tick();
I
Ingo Molnar 已提交
4389 4390

	spin_lock(&rq->lock);
4391
	update_rq_clock(rq);
4392
	update_cpu_load(rq);
P
Peter Zijlstra 已提交
4393
	curr->sched_class->task_tick(rq, curr, 0);
I
Ingo Molnar 已提交
4394
	spin_unlock(&rq->lock);
4395

4396
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
4397 4398
	rq->idle_at_tick = idle_cpu(cpu);
	trigger_load_balance(rq, cpu);
4399
#endif
L
Linus Torvalds 已提交
4400 4401
}

4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
				defined(CONFIG_PREEMPT_TRACER))

static inline unsigned long get_parent_ip(unsigned long addr)
{
	if (in_lock_functions(addr)) {
		addr = CALLER_ADDR2;
		if (in_lock_functions(addr))
			addr = CALLER_ADDR3;
	}
	return addr;
}
L
Linus Torvalds 已提交
4414

4415
void __kprobes add_preempt_count(int val)
L
Linus Torvalds 已提交
4416
{
4417
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4418 4419 4420
	/*
	 * Underflow?
	 */
4421 4422
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
4423
#endif
L
Linus Torvalds 已提交
4424
	preempt_count() += val;
4425
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4426 4427 4428
	/*
	 * Spinlock count overflowing soon?
	 */
4429 4430
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
4431 4432 4433
#endif
	if (preempt_count() == val)
		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
4434 4435 4436
}
EXPORT_SYMBOL(add_preempt_count);

4437
void __kprobes sub_preempt_count(int val)
L
Linus Torvalds 已提交
4438
{
4439
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4440 4441 4442
	/*
	 * Underflow?
	 */
N
Nick Piggin 已提交
4443
       if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
4444
		return;
L
Linus Torvalds 已提交
4445 4446 4447
	/*
	 * Is the spinlock portion underflowing?
	 */
4448 4449 4450
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;
4451
#endif
4452

4453 4454
	if (preempt_count() == val)
		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
4455 4456 4457 4458 4459 4460 4461
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
4462
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
4463
 */
I
Ingo Molnar 已提交
4464
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
4465
{
4466 4467 4468 4469 4470
	struct pt_regs *regs = get_irq_regs();

	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
		prev->comm, prev->pid, preempt_count());

I
Ingo Molnar 已提交
4471
	debug_show_held_locks(prev);
4472
	print_modules();
I
Ingo Molnar 已提交
4473 4474
	if (irqs_disabled())
		print_irqtrace_events(prev);
4475 4476 4477 4478 4479

	if (regs)
		show_regs(regs);
	else
		dump_stack();
I
Ingo Molnar 已提交
4480
}
L
Linus Torvalds 已提交
4481

I
Ingo Molnar 已提交
4482 4483 4484 4485 4486
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
4487
	/*
I
Ingo Molnar 已提交
4488
	 * Test if we are atomic. Since do_exit() needs to call into
L
Linus Torvalds 已提交
4489 4490 4491
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
4492
	if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
I
Ingo Molnar 已提交
4493 4494
		__schedule_bug(prev);

L
Linus Torvalds 已提交
4495 4496
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

4497
	schedstat_inc(this_rq(), sched_count);
I
Ingo Molnar 已提交
4498 4499
#ifdef CONFIG_SCHEDSTATS
	if (unlikely(prev->lock_depth >= 0)) {
4500 4501
		schedstat_inc(this_rq(), bkl_count);
		schedstat_inc(prev, sched_info.bkl_count);
I
Ingo Molnar 已提交
4502 4503
	}
#endif
I
Ingo Molnar 已提交
4504 4505 4506 4507 4508 4509
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
4510
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
4511
{
4512
	const struct sched_class *class;
I
Ingo Molnar 已提交
4513
	struct task_struct *p;
L
Linus Torvalds 已提交
4514 4515

	/*
I
Ingo Molnar 已提交
4516 4517
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
4518
	 */
I
Ingo Molnar 已提交
4519
	if (likely(rq->nr_running == rq->cfs.nr_running)) {
4520
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
4521 4522
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
4523 4524
	}

I
Ingo Molnar 已提交
4525 4526
	class = sched_class_highest;
	for ( ; ; ) {
4527
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
4528 4529 4530 4531 4532 4533 4534 4535 4536
		if (p)
			return p;
		/*
		 * Will never be NULL as the idle class always
		 * returns a non-NULL p:
		 */
		class = class->next;
	}
}
L
Linus Torvalds 已提交
4537

I
Ingo Molnar 已提交
4538 4539 4540 4541 4542 4543
/*
 * schedule() is the main scheduler function.
 */
asmlinkage void __sched schedule(void)
{
	struct task_struct *prev, *next;
4544
	unsigned long *switch_count;
I
Ingo Molnar 已提交
4545
	struct rq *rq;
4546
	int cpu;
I
Ingo Molnar 已提交
4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559

need_resched:
	preempt_disable();
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
	rcu_qsctr_inc(cpu);
	prev = rq->curr;
	switch_count = &prev->nivcsw;

	release_kernel_lock(prev);
need_resched_nonpreemptible:

	schedule_debug(prev);
L
Linus Torvalds 已提交
4560

4561
	if (sched_feat(HRTICK))
M
Mike Galbraith 已提交
4562
		hrtick_clear(rq);
P
Peter Zijlstra 已提交
4563

4564
	spin_lock_irq(&rq->lock);
4565
	update_rq_clock(rq);
4566
	clear_tsk_need_resched(prev);
L
Linus Torvalds 已提交
4567 4568

	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
4569
		if (unlikely(signal_pending_state(prev->state, prev)))
L
Linus Torvalds 已提交
4570
			prev->state = TASK_RUNNING;
4571
		else
4572
			deactivate_task(rq, prev, 1);
I
Ingo Molnar 已提交
4573
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
4574 4575
	}

4576 4577 4578 4579
#ifdef CONFIG_SMP
	if (prev->sched_class->pre_schedule)
		prev->sched_class->pre_schedule(rq, prev);
#endif
4580

I
Ingo Molnar 已提交
4581
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
4582 4583
		idle_balance(cpu, rq);

4584
	prev->sched_class->put_prev_task(rq, prev);
4585
	next = pick_next_task(rq, prev);
L
Linus Torvalds 已提交
4586 4587

	if (likely(prev != next)) {
4588 4589
		sched_info_switch(prev, next);

L
Linus Torvalds 已提交
4590 4591 4592 4593
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
4594
		context_switch(rq, prev, next); /* unlocks the rq */
P
Peter Zijlstra 已提交
4595 4596 4597 4598 4599 4600
		/*
		 * the context switch might have flipped the stack from under
		 * us, hence refresh the local variables.
		 */
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4601 4602 4603
	} else
		spin_unlock_irq(&rq->lock);

P
Peter Zijlstra 已提交
4604
	if (unlikely(reacquire_kernel_lock(current) < 0))
L
Linus Torvalds 已提交
4605
		goto need_resched_nonpreemptible;
P
Peter Zijlstra 已提交
4606

L
Linus Torvalds 已提交
4607 4608 4609 4610 4611 4612 4613 4614
	preempt_enable_no_resched();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(schedule);

#ifdef CONFIG_PREEMPT
/*
4615
 * this is the entry point to schedule() from in-kernel preemption
I
Ingo Molnar 已提交
4616
 * off of preempt_enable. Kernel preemptions off return from interrupt
L
Linus Torvalds 已提交
4617 4618 4619 4620 4621
 * occur there and call schedule directly.
 */
asmlinkage void __sched preempt_schedule(void)
{
	struct thread_info *ti = current_thread_info();
4622

L
Linus Torvalds 已提交
4623 4624
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
I
Ingo Molnar 已提交
4625
	 * we do not want to preempt the current task. Just return..
L
Linus Torvalds 已提交
4626
	 */
N
Nick Piggin 已提交
4627
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
4628 4629
		return;

4630 4631 4632 4633
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4634

4635 4636 4637 4638 4639 4640
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
	} while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
L
Linus Torvalds 已提交
4641 4642 4643 4644
}
EXPORT_SYMBOL(preempt_schedule);

/*
4645
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
4646 4647 4648 4649 4650 4651 4652
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
4653

4654
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
4655 4656
	BUG_ON(ti->preempt_count || !irqs_disabled());

4657 4658 4659 4660 4661 4662
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		local_irq_enable();
		schedule();
		local_irq_disable();
		sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4663

4664 4665 4666 4667 4668 4669
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
	} while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
L
Linus Torvalds 已提交
4670 4671 4672 4673
}

#endif /* CONFIG_PREEMPT */

I
Ingo Molnar 已提交
4674 4675
int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
			  void *key)
L
Linus Torvalds 已提交
4676
{
4677
	return try_to_wake_up(curr->private, mode, sync);
L
Linus Torvalds 已提交
4678 4679 4680 4681
}
EXPORT_SYMBOL(default_wake_function);

/*
I
Ingo Molnar 已提交
4682 4683
 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
L
Linus Torvalds 已提交
4684 4685 4686
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
I
Ingo Molnar 已提交
4687
 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
L
Linus Torvalds 已提交
4688 4689 4690 4691 4692
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
			     int nr_exclusive, int sync, void *key)
{
4693
	wait_queue_t *curr, *next;
L
Linus Torvalds 已提交
4694

4695
	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
4696 4697
		unsigned flags = curr->flags;

L
Linus Torvalds 已提交
4698
		if (curr->func(curr, mode, sync, key) &&
4699
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
4700 4701 4702 4703 4704 4705 4706 4707 4708
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
4709
 * @key: is directly passed to the wakeup function
L
Linus Torvalds 已提交
4710
 */
4711
void __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
4712
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
4725
void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
L
Linus Torvalds 已提交
4726 4727 4728 4729 4730
{
	__wake_up_common(q, mode, 1, 0, NULL);
}

/**
4731
 * __wake_up_sync - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
 */
4743
void
I
Ingo Molnar 已提交
4744
__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
L
Linus Torvalds 已提交
4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760
{
	unsigned long flags;
	int sync = 1;

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
		sync = 0;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, sync, NULL);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

4761 4762 4763 4764 4765 4766 4767 4768 4769
/**
 * complete: - signals a single thread waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up a single thread waiting on this completion. Threads will be
 * awakened in the same order in which they were queued.
 *
 * See also complete_all(), wait_for_completion() and related routines.
 */
4770
void complete(struct completion *x)
L
Linus Torvalds 已提交
4771 4772 4773 4774 4775
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
4776
	__wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
L
Linus Torvalds 已提交
4777 4778 4779 4780
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

4781 4782 4783 4784 4785 4786
/**
 * complete_all: - signals all threads waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up all threads waiting on this particular completion event.
 */
4787
void complete_all(struct completion *x)
L
Linus Torvalds 已提交
4788 4789 4790 4791 4792
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
4793
	__wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
L
Linus Torvalds 已提交
4794 4795 4796 4797
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

4798 4799
static inline long __sched
do_wait_for_common(struct completion *x, long timeout, int state)
L
Linus Torvalds 已提交
4800 4801 4802 4803 4804 4805 4806
{
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
4807
			if (signal_pending_state(state, current)) {
4808 4809
				timeout = -ERESTARTSYS;
				break;
4810 4811
			}
			__set_current_state(state);
L
Linus Torvalds 已提交
4812 4813 4814
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
4815
		} while (!x->done && timeout);
L
Linus Torvalds 已提交
4816
		__remove_wait_queue(&x->wait, &wait);
4817 4818
		if (!x->done)
			return timeout;
L
Linus Torvalds 已提交
4819 4820
	}
	x->done--;
4821
	return timeout ?: 1;
L
Linus Torvalds 已提交
4822 4823
}

4824 4825
static long __sched
wait_for_common(struct completion *x, long timeout, int state)
L
Linus Torvalds 已提交
4826 4827 4828 4829
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
4830
	timeout = do_wait_for_common(x, timeout, state);
L
Linus Torvalds 已提交
4831
	spin_unlock_irq(&x->wait.lock);
4832 4833
	return timeout;
}
L
Linus Torvalds 已提交
4834

4835 4836 4837 4838 4839 4840 4841 4842 4843 4844
/**
 * wait_for_completion: - waits for completion of a task
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It is NOT
 * interruptible and there is no timeout.
 *
 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
 * and interrupt capability. Also see complete().
 */
4845
void __sched wait_for_completion(struct completion *x)
4846 4847
{
	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
4848
}
4849
EXPORT_SYMBOL(wait_for_completion);
L
Linus Torvalds 已提交
4850

4851 4852 4853 4854 4855 4856 4857 4858 4859
/**
 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. The timeout is in jiffies. It is not
 * interruptible.
 */
4860
unsigned long __sched
4861
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
L
Linus Torvalds 已提交
4862
{
4863
	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
4864
}
4865
EXPORT_SYMBOL(wait_for_completion_timeout);
L
Linus Torvalds 已提交
4866

4867 4868 4869 4870 4871 4872 4873
/**
 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
 * @x:  holds the state of this particular completion
 *
 * This waits for completion of a specific task to be signaled. It is
 * interruptible.
 */
4874
int __sched wait_for_completion_interruptible(struct completion *x)
I
Ingo Molnar 已提交
4875
{
4876 4877 4878 4879
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
I
Ingo Molnar 已提交
4880
}
4881
EXPORT_SYMBOL(wait_for_completion_interruptible);
L
Linus Torvalds 已提交
4882

4883 4884 4885 4886 4887 4888 4889 4890
/**
 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
 */
4891
unsigned long __sched
4892 4893
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
I
Ingo Molnar 已提交
4894
{
4895
	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
I
Ingo Molnar 已提交
4896
}
4897
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
L
Linus Torvalds 已提交
4898

4899 4900 4901 4902 4903 4904 4905
/**
 * wait_for_completion_killable: - waits for completion of a task (killable)
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It can be
 * interrupted by a kill signal.
 */
M
Matthew Wilcox 已提交
4906 4907 4908 4909 4910 4911 4912 4913 4914
int __sched wait_for_completion_killable(struct completion *x)
{
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
}
EXPORT_SYMBOL(wait_for_completion_killable);

4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960
/**
 *	try_wait_for_completion - try to decrement a completion without blocking
 *	@x:	completion structure
 *
 *	Returns: 0 if a decrement cannot be done without blocking
 *		 1 if a decrement succeeded.
 *
 *	If a completion is being used as a counting completion,
 *	attempt to decrement the counter without blocking. This
 *	enables us to avoid waiting if the resource the completion
 *	is protecting is not available.
 */
bool try_wait_for_completion(struct completion *x)
{
	int ret = 1;

	spin_lock_irq(&x->wait.lock);
	if (!x->done)
		ret = 0;
	else
		x->done--;
	spin_unlock_irq(&x->wait.lock);
	return ret;
}
EXPORT_SYMBOL(try_wait_for_completion);

/**
 *	completion_done - Test to see if a completion has any waiters
 *	@x:	completion structure
 *
 *	Returns: 0 if there are waiters (wait_for_completion() in progress)
 *		 1 if there are no waiters.
 *
 */
bool completion_done(struct completion *x)
{
	int ret = 1;

	spin_lock_irq(&x->wait.lock);
	if (!x->done)
		ret = 0;
	spin_unlock_irq(&x->wait.lock);
	return ret;
}
EXPORT_SYMBOL(completion_done);

4961 4962
static long __sched
sleep_on_common(wait_queue_head_t *q, int state, long timeout)
L
Linus Torvalds 已提交
4963
{
I
Ingo Molnar 已提交
4964 4965 4966 4967
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
4968

4969
	__set_current_state(state);
L
Linus Torvalds 已提交
4970

4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984
	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, &wait);
	spin_unlock(&q->lock);
	timeout = schedule_timeout(timeout);
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, &wait);
	spin_unlock_irqrestore(&q->lock, flags);

	return timeout;
}

void __sched interruptible_sleep_on(wait_queue_head_t *q)
{
	sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
4985 4986 4987
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
4988
long __sched
I
Ingo Molnar 已提交
4989
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
4990
{
4991
	return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
4992 4993 4994
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
4995
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
4996
{
4997
	sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
4998 4999 5000
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
5001
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
5002
{
5003
	return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
5004 5005 5006
}
EXPORT_SYMBOL(sleep_on_timeout);

5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
5019
void rt_mutex_setprio(struct task_struct *p, int prio)
5020 5021
{
	unsigned long flags;
5022
	int oldprio, on_rq, running;
5023
	struct rq *rq;
5024
	const struct sched_class *prev_class = p->sched_class;
5025 5026 5027 5028

	BUG_ON(prio < 0 || prio > MAX_PRIO);

	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
5029
	update_rq_clock(rq);
5030

5031
	oldprio = p->prio;
I
Ingo Molnar 已提交
5032
	on_rq = p->se.on_rq;
5033
	running = task_current(rq, p);
5034
	if (on_rq)
5035
		dequeue_task(rq, p, 0);
5036 5037
	if (running)
		p->sched_class->put_prev_task(rq, p);
I
Ingo Molnar 已提交
5038 5039 5040 5041 5042 5043

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

5044 5045
	p->prio = prio;

5046 5047
	if (running)
		p->sched_class->set_curr_task(rq);
I
Ingo Molnar 已提交
5048
	if (on_rq) {
5049
		enqueue_task(rq, p, 0);
5050 5051

		check_class_changed(rq, p, prev_class, oldprio, running);
5052 5053 5054 5055 5056 5057
	}
	task_rq_unlock(rq, &flags);
}

#endif

5058
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
5059
{
I
Ingo Molnar 已提交
5060
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
5061
	unsigned long flags;
5062
	struct rq *rq;
L
Linus Torvalds 已提交
5063 5064 5065 5066 5067 5068 5069 5070

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
5071
	update_rq_clock(rq);
L
Linus Torvalds 已提交
5072 5073 5074 5075
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
5076
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
5077
	 */
5078
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
5079 5080 5081
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
I
Ingo Molnar 已提交
5082
	on_rq = p->se.on_rq;
5083
	if (on_rq)
5084
		dequeue_task(rq, p, 0);
L
Linus Torvalds 已提交
5085 5086

	p->static_prio = NICE_TO_PRIO(nice);
5087
	set_load_weight(p);
5088 5089 5090
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
5091

I
Ingo Molnar 已提交
5092
	if (on_rq) {
5093
		enqueue_task(rq, p, 0);
L
Linus Torvalds 已提交
5094
		/*
5095 5096
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
5097
		 */
5098
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
5099 5100 5101 5102 5103 5104 5105
			resched_task(rq->curr);
	}
out_unlock:
	task_rq_unlock(rq, &flags);
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
5106 5107 5108 5109 5110
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
5111
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
5112
{
5113 5114
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
5115

M
Matt Mackall 已提交
5116 5117 5118 5119
	return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
5120 5121 5122 5123 5124 5125 5126 5127 5128
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
5129
SYSCALL_DEFINE1(nice, int, increment)
L
Linus Torvalds 已提交
5130
{
5131
	long nice, retval;
L
Linus Torvalds 已提交
5132 5133 5134 5135 5136 5137

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
5138 5139
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
5140 5141 5142 5143 5144 5145 5146 5147 5148
	if (increment > 40)
		increment = 40;

	nice = PRIO_TO_NICE(current->static_prio) + increment;
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
5149 5150 5151
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
5170
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
5171 5172 5173 5174 5175 5176 5177 5178
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
5179
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
5180 5181 5182
{
	return TASK_NICE(p);
}
P
Pavel Roskin 已提交
5183
EXPORT_SYMBOL(task_nice);
L
Linus Torvalds 已提交
5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
5198
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
5199 5200 5201 5202 5203 5204 5205 5206
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
A
Alexey Dobriyan 已提交
5207
static struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
5208
{
5209
	return pid ? find_task_by_vpid(pid) : current;
L
Linus Torvalds 已提交
5210 5211 5212
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
5213 5214
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
5215
{
I
Ingo Molnar 已提交
5216
	BUG_ON(p->se.on_rq);
5217

L
Linus Torvalds 已提交
5218
	p->policy = policy;
I
Ingo Molnar 已提交
5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230
	switch (p->policy) {
	case SCHED_NORMAL:
	case SCHED_BATCH:
	case SCHED_IDLE:
		p->sched_class = &fair_sched_class;
		break;
	case SCHED_FIFO:
	case SCHED_RR:
		p->sched_class = &rt_sched_class;
		break;
	}

L
Linus Torvalds 已提交
5231
	p->rt_priority = prio;
5232 5233 5234
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
5235
	set_load_weight(p);
L
Linus Torvalds 已提交
5236 5237
}

5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253
/*
 * check the target process has a UID that matches the current process's
 */
static bool check_same_owner(struct task_struct *p)
{
	const struct cred *cred = current_cred(), *pcred;
	bool match;

	rcu_read_lock();
	pcred = __task_cred(p);
	match = (cred->euid == pcred->euid ||
		 cred->euid == pcred->uid);
	rcu_read_unlock();
	return match;
}

5254 5255
static int __sched_setscheduler(struct task_struct *p, int policy,
				struct sched_param *param, bool user)
L
Linus Torvalds 已提交
5256
{
5257
	int retval, oldprio, oldpolicy = -1, on_rq, running;
L
Linus Torvalds 已提交
5258
	unsigned long flags;
5259
	const struct sched_class *prev_class = p->sched_class;
5260
	struct rq *rq;
L
Linus Torvalds 已提交
5261

5262 5263
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
5264 5265 5266 5267 5268
recheck:
	/* double check policy once rq lock held */
	if (policy < 0)
		policy = oldpolicy = p->policy;
	else if (policy != SCHED_FIFO && policy != SCHED_RR &&
I
Ingo Molnar 已提交
5269 5270
			policy != SCHED_NORMAL && policy != SCHED_BATCH &&
			policy != SCHED_IDLE)
5271
		return -EINVAL;
L
Linus Torvalds 已提交
5272 5273
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
5274 5275
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
5276 5277
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
5278
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
5279
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
5280
		return -EINVAL;
5281
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
5282 5283
		return -EINVAL;

5284 5285 5286
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
5287
	if (user && !capable(CAP_SYS_NICE)) {
5288
		if (rt_policy(policy)) {
5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304
			unsigned long rlim_rtprio;

			if (!lock_task_sighand(p, &flags))
				return -ESRCH;
			rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
			unlock_task_sighand(p, &flags);

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
I
Ingo Molnar 已提交
5305 5306 5307 5308 5309 5310
		/*
		 * Like positive nice levels, dont allow tasks to
		 * move out of SCHED_IDLE either:
		 */
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
			return -EPERM;
5311

5312
		/* can't change other user's priorities */
5313
		if (!check_same_owner(p))
5314 5315
			return -EPERM;
	}
L
Linus Torvalds 已提交
5316

5317
	if (user) {
5318
#ifdef CONFIG_RT_GROUP_SCHED
5319 5320 5321 5322
		/*
		 * Do not allow realtime tasks into groups that have no runtime
		 * assigned.
		 */
P
Peter Zijlstra 已提交
5323 5324
		if (rt_bandwidth_enabled() && rt_policy(policy) &&
				task_group(p)->rt_bandwidth.rt_runtime == 0)
5325
			return -EPERM;
5326 5327
#endif

5328 5329 5330 5331 5332
		retval = security_task_setscheduler(p, policy, param);
		if (retval)
			return retval;
	}

5333 5334 5335 5336 5337
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
	 */
	spin_lock_irqsave(&p->pi_lock, flags);
L
Linus Torvalds 已提交
5338 5339 5340 5341
	/*
	 * To be able to change p->policy safely, the apropriate
	 * runqueue lock must be held.
	 */
5342
	rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
5343 5344 5345
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
5346 5347
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
5348 5349
		goto recheck;
	}
I
Ingo Molnar 已提交
5350
	update_rq_clock(rq);
I
Ingo Molnar 已提交
5351
	on_rq = p->se.on_rq;
5352
	running = task_current(rq, p);
5353
	if (on_rq)
5354
		deactivate_task(rq, p, 0);
5355 5356
	if (running)
		p->sched_class->put_prev_task(rq, p);
5357

L
Linus Torvalds 已提交
5358
	oldprio = p->prio;
I
Ingo Molnar 已提交
5359
	__setscheduler(rq, p, policy, param->sched_priority);
5360

5361 5362
	if (running)
		p->sched_class->set_curr_task(rq);
I
Ingo Molnar 已提交
5363 5364
	if (on_rq) {
		activate_task(rq, p, 0);
5365 5366

		check_class_changed(rq, p, prev_class, oldprio, running);
L
Linus Torvalds 已提交
5367
	}
5368 5369 5370
	__task_rq_unlock(rq);
	spin_unlock_irqrestore(&p->pi_lock, flags);

5371 5372
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
5373 5374
	return 0;
}
5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388

/**
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * NOTE that the task may be already dead.
 */
int sched_setscheduler(struct task_struct *p, int policy,
		       struct sched_param *param)
{
	return __sched_setscheduler(p, policy, param, true);
}
L
Linus Torvalds 已提交
5389 5390
EXPORT_SYMBOL_GPL(sched_setscheduler);

5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407
/**
 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * Just like sched_setscheduler, only don't bother checking if the
 * current context has permission.  For example, this is needed in
 * stop_machine(): we create temporary high priority worker threads,
 * but our caller might not have that capability.
 */
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
			       struct sched_param *param)
{
	return __sched_setscheduler(p, policy, param, false);
}

I
Ingo Molnar 已提交
5408 5409
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
5410 5411 5412
{
	struct sched_param lparam;
	struct task_struct *p;
5413
	int retval;
L
Linus Torvalds 已提交
5414 5415 5416 5417 5418

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
5419 5420 5421

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
5422
	p = find_process_by_pid(pid);
5423 5424 5425
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
5426

L
Linus Torvalds 已提交
5427 5428 5429 5430 5431 5432 5433 5434 5435
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
5436 5437
SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
		struct sched_param __user *, param)
L
Linus Torvalds 已提交
5438
{
5439 5440 5441 5442
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
5443 5444 5445 5446 5447 5448 5449 5450
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
5451
SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
5452 5453 5454 5455 5456 5457 5458 5459
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
5460
SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
L
Linus Torvalds 已提交
5461
{
5462
	struct task_struct *p;
5463
	int retval;
L
Linus Torvalds 已提交
5464 5465

	if (pid < 0)
5466
		return -EINVAL;
L
Linus Torvalds 已提交
5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
			retval = p->policy;
	}
	read_unlock(&tasklist_lock);
	return retval;
}

/**
 * sys_sched_getscheduler - get the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
5485
SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
5486 5487
{
	struct sched_param lp;
5488
	struct task_struct *p;
5489
	int retval;
L
Linus Torvalds 已提交
5490 5491

	if (!param || pid < 0)
5492
		return -EINVAL;
L
Linus Torvalds 已提交
5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518

	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
	read_unlock(&tasklist_lock);

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

	return retval;

out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

5519
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
L
Linus Torvalds 已提交
5520
{
5521
	cpumask_var_t cpus_allowed, new_mask;
5522 5523
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
5524

5525
	get_online_cpus();
L
Linus Torvalds 已提交
5526 5527 5528 5529 5530
	read_lock(&tasklist_lock);

	p = find_process_by_pid(pid);
	if (!p) {
		read_unlock(&tasklist_lock);
5531
		put_online_cpus();
L
Linus Torvalds 已提交
5532 5533 5534 5535 5536
		return -ESRCH;
	}

	/*
	 * It is not safe to call set_cpus_allowed with the
I
Ingo Molnar 已提交
5537
	 * tasklist_lock held. We will bump the task_struct's
L
Linus Torvalds 已提交
5538 5539 5540 5541 5542
	 * usage count and then drop tasklist_lock.
	 */
	get_task_struct(p);
	read_unlock(&tasklist_lock);

5543 5544 5545 5546 5547 5548 5549 5550
	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_put_task;
	}
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_free_cpus_allowed;
	}
L
Linus Torvalds 已提交
5551
	retval = -EPERM;
5552
	if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
L
Linus Torvalds 已提交
5553 5554
		goto out_unlock;

5555 5556 5557 5558
	retval = security_task_setscheduler(p, 0, NULL);
	if (retval)
		goto out_unlock;

5559 5560
	cpuset_cpus_allowed(p, cpus_allowed);
	cpumask_and(new_mask, in_mask, cpus_allowed);
P
Paul Menage 已提交
5561
 again:
5562
	retval = set_cpus_allowed_ptr(p, new_mask);
L
Linus Torvalds 已提交
5563

P
Paul Menage 已提交
5564
	if (!retval) {
5565 5566
		cpuset_cpus_allowed(p, cpus_allowed);
		if (!cpumask_subset(new_mask, cpus_allowed)) {
P
Paul Menage 已提交
5567 5568 5569 5570 5571
			/*
			 * We must have raced with a concurrent cpuset
			 * update. Just reset the cpus_allowed to the
			 * cpuset's cpus_allowed
			 */
5572
			cpumask_copy(new_mask, cpus_allowed);
P
Paul Menage 已提交
5573 5574 5575
			goto again;
		}
	}
L
Linus Torvalds 已提交
5576
out_unlock:
5577 5578 5579 5580
	free_cpumask_var(new_mask);
out_free_cpus_allowed:
	free_cpumask_var(cpus_allowed);
out_put_task:
L
Linus Torvalds 已提交
5581
	put_task_struct(p);
5582
	put_online_cpus();
L
Linus Torvalds 已提交
5583 5584 5585 5586
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5587
			     struct cpumask *new_mask)
L
Linus Torvalds 已提交
5588
{
5589 5590 5591 5592 5593
	if (len < cpumask_size())
		cpumask_clear(new_mask);
	else if (len > cpumask_size())
		len = cpumask_size();

L
Linus Torvalds 已提交
5594 5595 5596 5597 5598 5599 5600 5601 5602
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
5603 5604
SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
5605
{
5606
	cpumask_var_t new_mask;
L
Linus Torvalds 已提交
5607 5608
	int retval;

5609 5610
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
5611

5612 5613 5614 5615 5616
	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
	if (retval == 0)
		retval = sched_setaffinity(pid, new_mask);
	free_cpumask_var(new_mask);
	return retval;
L
Linus Torvalds 已提交
5617 5618
}

5619
long sched_getaffinity(pid_t pid, struct cpumask *mask)
L
Linus Torvalds 已提交
5620
{
5621
	struct task_struct *p;
L
Linus Torvalds 已提交
5622 5623
	int retval;

5624
	get_online_cpus();
L
Linus Torvalds 已提交
5625 5626 5627 5628 5629 5630 5631
	read_lock(&tasklist_lock);

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

5632 5633 5634 5635
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

5636
	cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
L
Linus Torvalds 已提交
5637 5638 5639

out_unlock:
	read_unlock(&tasklist_lock);
5640
	put_online_cpus();
L
Linus Torvalds 已提交
5641

5642
	return retval;
L
Linus Torvalds 已提交
5643 5644 5645 5646 5647 5648 5649 5650
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
5651 5652
SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
5653 5654
{
	int ret;
5655
	cpumask_var_t mask;
L
Linus Torvalds 已提交
5656

5657
	if (len < cpumask_size())
L
Linus Torvalds 已提交
5658 5659
		return -EINVAL;

5660 5661
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
5662

5663 5664 5665 5666 5667 5668 5669 5670
	ret = sched_getaffinity(pid, mask);
	if (ret == 0) {
		if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
			ret = -EFAULT;
		else
			ret = cpumask_size();
	}
	free_cpumask_var(mask);
L
Linus Torvalds 已提交
5671

5672
	return ret;
L
Linus Torvalds 已提交
5673 5674 5675 5676 5677
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
5678 5679
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
5680
 */
5681
SYSCALL_DEFINE0(sched_yield)
L
Linus Torvalds 已提交
5682
{
5683
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
5684

5685
	schedstat_inc(rq, yld_count);
5686
	current->sched_class->yield_task(rq);
L
Linus Torvalds 已提交
5687 5688 5689 5690 5691 5692

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
5693
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
5694 5695 5696 5697 5698 5699 5700 5701
	_raw_spin_unlock(&rq->lock);
	preempt_enable_no_resched();

	schedule();

	return 0;
}

A
Andrew Morton 已提交
5702
static void __cond_resched(void)
L
Linus Torvalds 已提交
5703
{
5704 5705 5706
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
	__might_sleep(__FILE__, __LINE__);
#endif
5707 5708 5709 5710 5711
	/*
	 * The BKS might be reacquired before we have dropped
	 * PREEMPT_ACTIVE, which could trigger a second
	 * cond_resched() call.
	 */
L
Linus Torvalds 已提交
5712 5713 5714 5715 5716 5717 5718
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
	} while (need_resched());
}

5719
int __sched _cond_resched(void)
L
Linus Torvalds 已提交
5720
{
5721 5722
	if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
					system_state == SYSTEM_RUNNING) {
L
Linus Torvalds 已提交
5723 5724 5725 5726 5727
		__cond_resched();
		return 1;
	}
	return 0;
}
5728
EXPORT_SYMBOL(_cond_resched);
L
Linus Torvalds 已提交
5729 5730 5731 5732 5733

/*
 * cond_resched_lock() - if a reschedule is pending, drop the given lock,
 * call schedule, and on return reacquire the lock.
 *
I
Ingo Molnar 已提交
5734
 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
L
Linus Torvalds 已提交
5735 5736 5737
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
I
Ingo Molnar 已提交
5738
int cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
5739
{
N
Nick Piggin 已提交
5740
	int resched = need_resched() && system_state == SYSTEM_RUNNING;
J
Jan Kara 已提交
5741 5742
	int ret = 0;

N
Nick Piggin 已提交
5743
	if (spin_needbreak(lock) || resched) {
L
Linus Torvalds 已提交
5744
		spin_unlock(lock);
N
Nick Piggin 已提交
5745 5746 5747 5748
		if (resched && need_resched())
			__cond_resched();
		else
			cpu_relax();
J
Jan Kara 已提交
5749
		ret = 1;
L
Linus Torvalds 已提交
5750 5751
		spin_lock(lock);
	}
J
Jan Kara 已提交
5752
	return ret;
L
Linus Torvalds 已提交
5753 5754 5755 5756 5757 5758 5759
}
EXPORT_SYMBOL(cond_resched_lock);

int __sched cond_resched_softirq(void)
{
	BUG_ON(!in_softirq());

5760
	if (need_resched() && system_state == SYSTEM_RUNNING) {
5761
		local_bh_enable();
L
Linus Torvalds 已提交
5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched_softirq);

/**
 * yield - yield the current processor to other threads.
 *
5773
 * This is a shortcut for kernel-space yielding - it marks the
L
Linus Torvalds 已提交
5774 5775 5776 5777 5778 5779 5780 5781 5782 5783
 * thread runnable and calls sys_sched_yield().
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

/*
I
Ingo Molnar 已提交
5784
 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
L
Linus Torvalds 已提交
5785 5786 5787 5788 5789 5790 5791
 * that process accounting knows that this is a task in IO wait state.
 *
 * But don't do that if it is a deliberate, throttling IO wait (this task
 * has set its backing_dev_info: the queue against which it should throttle)
 */
void __sched io_schedule(void)
{
5792
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
5793

5794
	delayacct_blkio_start();
L
Linus Torvalds 已提交
5795 5796 5797
	atomic_inc(&rq->nr_iowait);
	schedule();
	atomic_dec(&rq->nr_iowait);
5798
	delayacct_blkio_end();
L
Linus Torvalds 已提交
5799 5800 5801 5802 5803
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
5804
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
5805 5806
	long ret;

5807
	delayacct_blkio_start();
L
Linus Torvalds 已提交
5808 5809 5810
	atomic_inc(&rq->nr_iowait);
	ret = schedule_timeout(timeout);
	atomic_dec(&rq->nr_iowait);
5811
	delayacct_blkio_end();
L
Linus Torvalds 已提交
5812 5813 5814 5815 5816 5817 5818 5819 5820 5821
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
5822
SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
L
Linus Torvalds 已提交
5823 5824 5825 5826 5827 5828 5829 5830 5831
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
5832
	case SCHED_BATCH:
I
Ingo Molnar 已提交
5833
	case SCHED_IDLE:
L
Linus Torvalds 已提交
5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
5847
SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
L
Linus Torvalds 已提交
5848 5849 5850 5851 5852 5853 5854 5855 5856
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
5857
	case SCHED_BATCH:
I
Ingo Molnar 已提交
5858
	case SCHED_IDLE:
L
Linus Torvalds 已提交
5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
5872 5873
SYSCALL_DEFINE4(sched_rr_get_interval, pid_t, pid,
		struct timespec __user *, interval)
L
Linus Torvalds 已提交
5874
{
5875
	struct task_struct *p;
D
Dmitry Adamushko 已提交
5876
	unsigned int time_slice;
5877
	int retval;
L
Linus Torvalds 已提交
5878 5879 5880
	struct timespec t;

	if (pid < 0)
5881
		return -EINVAL;
L
Linus Torvalds 已提交
5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

5893 5894 5895 5896 5897 5898
	/*
	 * Time slice is 0 for SCHED_FIFO tasks and for SCHED_OTHER
	 * tasks that are on an otherwise idle runqueue:
	 */
	time_slice = 0;
	if (p->policy == SCHED_RR) {
D
Dmitry Adamushko 已提交
5899
		time_slice = DEF_TIMESLICE;
5900
	} else if (p->policy != SCHED_FIFO) {
D
Dmitry Adamushko 已提交
5901 5902 5903 5904 5905
		struct sched_entity *se = &p->se;
		unsigned long flags;
		struct rq *rq;

		rq = task_rq_lock(p, &flags);
5906 5907
		if (rq->cfs.load.weight)
			time_slice = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
D
Dmitry Adamushko 已提交
5908 5909
		task_rq_unlock(rq, &flags);
	}
L
Linus Torvalds 已提交
5910
	read_unlock(&tasklist_lock);
D
Dmitry Adamushko 已提交
5911
	jiffies_to_timespec(time_slice, &t);
L
Linus Torvalds 已提交
5912 5913
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
	return retval;
5914

L
Linus Torvalds 已提交
5915 5916 5917 5918 5919
out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

5920
static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5921

5922
void sched_show_task(struct task_struct *p)
L
Linus Torvalds 已提交
5923 5924
{
	unsigned long free = 0;
5925
	unsigned state;
L
Linus Torvalds 已提交
5926 5927

	state = p->state ? __ffs(p->state) + 1 : 0;
I
Ingo Molnar 已提交
5928
	printk(KERN_INFO "%-13.13s %c", p->comm,
5929
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5930
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
5931
	if (state == TASK_RUNNING)
I
Ingo Molnar 已提交
5932
		printk(KERN_CONT " running  ");
L
Linus Torvalds 已提交
5933
	else
I
Ingo Molnar 已提交
5934
		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
5935 5936
#else
	if (state == TASK_RUNNING)
I
Ingo Molnar 已提交
5937
		printk(KERN_CONT "  running task    ");
L
Linus Torvalds 已提交
5938
	else
I
Ingo Molnar 已提交
5939
		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
5940 5941 5942
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
	{
5943
		unsigned long *n = end_of_stack(p);
L
Linus Torvalds 已提交
5944 5945
		while (!*n)
			n++;
5946
		free = (unsigned long)n - (unsigned long)end_of_stack(p);
L
Linus Torvalds 已提交
5947 5948
	}
#endif
5949
	printk(KERN_CONT "%5lu %5d %6d\n", free,
R
Roland McGrath 已提交
5950
		task_pid_nr(p), task_pid_nr(p->real_parent));
L
Linus Torvalds 已提交
5951

5952
	show_stack(p, NULL);
L
Linus Torvalds 已提交
5953 5954
}

I
Ingo Molnar 已提交
5955
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
5956
{
5957
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
5958

5959 5960 5961
#if BITS_PER_LONG == 32
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
5962
#else
5963 5964
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
5965 5966 5967 5968 5969 5970 5971 5972
#endif
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
		 * console might take alot of time:
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
5973
		if (!state_filter || (p->state & state_filter))
5974
			sched_show_task(p);
L
Linus Torvalds 已提交
5975 5976
	} while_each_thread(g, p);

5977 5978
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
5979 5980 5981
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
L
Linus Torvalds 已提交
5982
	read_unlock(&tasklist_lock);
I
Ingo Molnar 已提交
5983 5984 5985 5986 5987
	/*
	 * Only show locks if all tasks are dumped:
	 */
	if (state_filter == -1)
		debug_show_all_locks();
L
Linus Torvalds 已提交
5988 5989
}

I
Ingo Molnar 已提交
5990 5991
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
5992
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
5993 5994
}

5995 5996 5997 5998 5999 6000 6001 6002
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
6003
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
6004
{
6005
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
6006 6007
	unsigned long flags;

6008 6009
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
6010 6011 6012
	__sched_fork(idle);
	idle->se.exec_start = sched_clock();

6013
	idle->prio = idle->normal_prio = MAX_PRIO;
6014
	cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
I
Ingo Molnar 已提交
6015
	__set_task_cpu(idle, cpu);
L
Linus Torvalds 已提交
6016 6017

	rq->curr = rq->idle = idle;
6018 6019 6020
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
	idle->oncpu = 1;
#endif
L
Linus Torvalds 已提交
6021 6022 6023
	spin_unlock_irqrestore(&rq->lock, flags);

	/* Set the preempt count _outside_ the spinlocks! */
6024 6025 6026
#if defined(CONFIG_PREEMPT)
	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
#else
A
Al Viro 已提交
6027
	task_thread_info(idle)->preempt_count = 0;
6028
#endif
I
Ingo Molnar 已提交
6029 6030 6031 6032
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
6033
	ftrace_graph_init_task(idle);
L
Linus Torvalds 已提交
6034 6035 6036 6037 6038 6039 6040
}

/*
 * In a system that switches off the HZ timer nohz_cpu_mask
 * indicates which cpus entered this state. This is used
 * in the rcu update to wait only for active cpus. For system
 * which do not switch off the HZ timer nohz_cpu_mask should
6041
 * always be CPU_BITS_NONE.
L
Linus Torvalds 已提交
6042
 */
6043
cpumask_var_t nohz_cpu_mask;
L
Linus Torvalds 已提交
6044

I
Ingo Molnar 已提交
6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067
/*
 * Increase the granularity value when there are more CPUs,
 * because with more CPUs the 'effective latency' as visible
 * to users decreases. But the relationship is not linear,
 * so pick a second-best guess by going with the log2 of the
 * number of CPUs.
 *
 * This idea comes from the SD scheduler of Con Kolivas:
 */
static inline void sched_init_granularity(void)
{
	unsigned int factor = 1 + ilog2(num_online_cpus());
	const unsigned long limit = 200000000;

	sysctl_sched_min_granularity *= factor;
	if (sysctl_sched_min_granularity > limit)
		sysctl_sched_min_granularity = limit;

	sysctl_sched_latency *= factor;
	if (sysctl_sched_latency > limit)
		sysctl_sched_latency = limit;

	sysctl_sched_wakeup_granularity *= factor;
6068 6069

	sysctl_sched_shares_ratelimit *= factor;
I
Ingo Molnar 已提交
6070 6071
}

L
Linus Torvalds 已提交
6072 6073 6074 6075
#ifdef CONFIG_SMP
/*
 * This is how migration works:
 *
6076
 * 1) we queue a struct migration_req structure in the source CPU's
L
Linus Torvalds 已提交
6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094
 *    runqueue and wake up that CPU's migration thread.
 * 2) we down() the locked semaphore => thread blocks.
 * 3) migration thread wakes up (implicitly it forces the migrated
 *    thread off the CPU)
 * 4) it gets the migration request and checks whether the migrated
 *    task is still in the wrong runqueue.
 * 5) if it's in the wrong runqueue then the migration thread removes
 *    it and puts it into the right queue.
 * 6) migration thread up()s the semaphore.
 * 7) we wake up and the migration is done.
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
I
Ingo Molnar 已提交
6095
 * task must not exit() & deallocate itself prematurely. The
L
Linus Torvalds 已提交
6096 6097
 * call is not atomic; no spinlocks may be held.
 */
6098
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
L
Linus Torvalds 已提交
6099
{
6100
	struct migration_req req;
L
Linus Torvalds 已提交
6101
	unsigned long flags;
6102
	struct rq *rq;
6103
	int ret = 0;
L
Linus Torvalds 已提交
6104 6105

	rq = task_rq_lock(p, &flags);
6106
	if (!cpumask_intersects(new_mask, cpu_online_mask)) {
L
Linus Torvalds 已提交
6107 6108 6109 6110
		ret = -EINVAL;
		goto out;
	}

6111
	if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
6112
		     !cpumask_equal(&p->cpus_allowed, new_mask))) {
6113 6114 6115 6116
		ret = -EINVAL;
		goto out;
	}

6117
	if (p->sched_class->set_cpus_allowed)
6118
		p->sched_class->set_cpus_allowed(p, new_mask);
6119
	else {
6120 6121
		cpumask_copy(&p->cpus_allowed, new_mask);
		p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
6122 6123
	}

L
Linus Torvalds 已提交
6124
	/* Can the task run on the task's current CPU? If so, we're done */
6125
	if (cpumask_test_cpu(task_cpu(p), new_mask))
L
Linus Torvalds 已提交
6126 6127
		goto out;

R
Rusty Russell 已提交
6128
	if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
L
Linus Torvalds 已提交
6129 6130 6131 6132 6133 6134 6135 6136 6137
		/* Need help from migration thread: drop lock and wait. */
		task_rq_unlock(rq, &flags);
		wake_up_process(rq->migration_thread);
		wait_for_completion(&req.done);
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
	task_rq_unlock(rq, &flags);
6138

L
Linus Torvalds 已提交
6139 6140
	return ret;
}
6141
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
L
Linus Torvalds 已提交
6142 6143

/*
I
Ingo Molnar 已提交
6144
 * Move (not current) task off this cpu, onto dest cpu. We're doing
L
Linus Torvalds 已提交
6145 6146 6147 6148 6149 6150
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
6151 6152
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
6153
 */
6154
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
6155
{
6156
	struct rq *rq_dest, *rq_src;
I
Ingo Molnar 已提交
6157
	int ret = 0, on_rq;
L
Linus Torvalds 已提交
6158

6159
	if (unlikely(!cpu_active(dest_cpu)))
6160
		return ret;
L
Linus Torvalds 已提交
6161 6162 6163 6164 6165 6166 6167

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
L
Linus Torvalds 已提交
6168
		goto done;
L
Linus Torvalds 已提交
6169
	/* Affinity changed (again). */
6170
	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
L
Linus Torvalds 已提交
6171
		goto fail;
L
Linus Torvalds 已提交
6172

I
Ingo Molnar 已提交
6173
	on_rq = p->se.on_rq;
6174
	if (on_rq)
6175
		deactivate_task(rq_src, p, 0);
6176

L
Linus Torvalds 已提交
6177
	set_task_cpu(p, dest_cpu);
I
Ingo Molnar 已提交
6178 6179
	if (on_rq) {
		activate_task(rq_dest, p, 0);
6180
		check_preempt_curr(rq_dest, p, 0);
L
Linus Torvalds 已提交
6181
	}
L
Linus Torvalds 已提交
6182
done:
6183
	ret = 1;
L
Linus Torvalds 已提交
6184
fail:
L
Linus Torvalds 已提交
6185
	double_rq_unlock(rq_src, rq_dest);
6186
	return ret;
L
Linus Torvalds 已提交
6187 6188 6189 6190 6191 6192 6193
}

/*
 * migration_thread - this is a highprio system thread that performs
 * thread migration by bumping thread off CPU then 'pushing' onto
 * another runqueue.
 */
I
Ingo Molnar 已提交
6194
static int migration_thread(void *data)
L
Linus Torvalds 已提交
6195 6196
{
	int cpu = (long)data;
6197
	struct rq *rq;
L
Linus Torvalds 已提交
6198 6199 6200 6201 6202 6203

	rq = cpu_rq(cpu);
	BUG_ON(rq->migration_thread != current);

	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
6204
		struct migration_req *req;
L
Linus Torvalds 已提交
6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226
		struct list_head *head;

		spin_lock_irq(&rq->lock);

		if (cpu_is_offline(cpu)) {
			spin_unlock_irq(&rq->lock);
			goto wait_to_die;
		}

		if (rq->active_balance) {
			active_load_balance(rq, cpu);
			rq->active_balance = 0;
		}

		head = &rq->migration_queue;

		if (list_empty(head)) {
			spin_unlock_irq(&rq->lock);
			schedule();
			set_current_state(TASK_INTERRUPTIBLE);
			continue;
		}
6227
		req = list_entry(head->next, struct migration_req, list);
L
Linus Torvalds 已提交
6228 6229
		list_del_init(head->next);

N
Nick Piggin 已提交
6230 6231 6232
		spin_unlock(&rq->lock);
		__migrate_task(req->task, cpu, req->dest_cpu);
		local_irq_enable();
L
Linus Torvalds 已提交
6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250

		complete(&req->done);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261

static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
{
	int ret;

	local_irq_disable();
	ret = __migrate_task(p, src_cpu, dest_cpu);
	local_irq_enable();
	return ret;
}

6262
/*
6263
 * Figure out where task on dead CPU should go, use force if necessary.
6264
 */
6265
static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
6266
{
6267
	int dest_cpu;
6268
	const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284

again:
	/* Look for allowed, online CPU in same node. */
	for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
		if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
			goto move;

	/* Any allowed, online CPU? */
	dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
	if (dest_cpu < nr_cpu_ids)
		goto move;

	/* No more Mr. Nice Guy. */
	if (dest_cpu >= nr_cpu_ids) {
		cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
		dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
L
Linus Torvalds 已提交
6285

6286 6287 6288 6289 6290 6291 6292 6293 6294
		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
		if (p->mm && printk_ratelimit()) {
			printk(KERN_INFO "process %d (%s) no "
			       "longer affine to cpu%d\n",
			       task_pid_nr(p), p->comm, dead_cpu);
6295
		}
6296 6297 6298 6299 6300 6301
	}

move:
	/* It can have affinity changed while we were choosing. */
	if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
		goto again;
L
Linus Torvalds 已提交
6302 6303 6304 6305 6306 6307 6308 6309 6310
}

/*
 * While a dead CPU has no uninterruptible tasks queued at this point,
 * it might still have a nonzero ->nr_uninterruptible counter, because
 * for performance reasons the counter is not stricly tracking tasks to
 * their home CPUs. So we just add the counter to another CPU's counter,
 * to keep the global sum constant after CPU-down:
 */
6311
static void migrate_nr_uninterruptible(struct rq *rq_src)
L
Linus Torvalds 已提交
6312
{
R
Rusty Russell 已提交
6313
	struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
L
Linus Torvalds 已提交
6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326
	unsigned long flags;

	local_irq_save(flags);
	double_rq_lock(rq_src, rq_dest);
	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
	rq_src->nr_uninterruptible = 0;
	double_rq_unlock(rq_src, rq_dest);
	local_irq_restore(flags);
}

/* Run through task list and migrate tasks from the dead cpu. */
static void migrate_live_tasks(int src_cpu)
{
6327
	struct task_struct *p, *t;
L
Linus Torvalds 已提交
6328

6329
	read_lock(&tasklist_lock);
L
Linus Torvalds 已提交
6330

6331 6332
	do_each_thread(t, p) {
		if (p == current)
L
Linus Torvalds 已提交
6333 6334
			continue;

6335 6336 6337
		if (task_cpu(p) == src_cpu)
			move_task_off_dead_cpu(src_cpu, p);
	} while_each_thread(t, p);
L
Linus Torvalds 已提交
6338

6339
	read_unlock(&tasklist_lock);
L
Linus Torvalds 已提交
6340 6341
}

I
Ingo Molnar 已提交
6342 6343
/*
 * Schedules idle task to be the next runnable task on current CPU.
6344 6345
 * It does so by boosting its priority to highest possible.
 * Used by CPU offline code.
L
Linus Torvalds 已提交
6346 6347 6348
 */
void sched_idle_next(void)
{
6349
	int this_cpu = smp_processor_id();
6350
	struct rq *rq = cpu_rq(this_cpu);
L
Linus Torvalds 已提交
6351 6352 6353 6354
	struct task_struct *p = rq->idle;
	unsigned long flags;

	/* cpu has to be offline */
6355
	BUG_ON(cpu_online(this_cpu));
L
Linus Torvalds 已提交
6356

6357 6358 6359
	/*
	 * Strictly not necessary since rest of the CPUs are stopped by now
	 * and interrupts disabled on the current cpu.
L
Linus Torvalds 已提交
6360 6361 6362
	 */
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
6363
	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
6364

6365 6366
	update_rq_clock(rq);
	activate_task(rq, p, 0);
L
Linus Torvalds 已提交
6367 6368 6369 6370

	spin_unlock_irqrestore(&rq->lock, flags);
}

6371 6372
/*
 * Ensures that the idle task is using init_mm right before its cpu goes
L
Linus Torvalds 已提交
6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385
 * offline.
 */
void idle_task_exit(void)
{
	struct mm_struct *mm = current->active_mm;

	BUG_ON(cpu_online(smp_processor_id()));

	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
}

6386
/* called under rq->lock with disabled interrupts */
6387
static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
6388
{
6389
	struct rq *rq = cpu_rq(dead_cpu);
L
Linus Torvalds 已提交
6390 6391

	/* Must be exiting, otherwise would be on tasklist. */
E
Eugene Teo 已提交
6392
	BUG_ON(!p->exit_state);
L
Linus Torvalds 已提交
6393 6394

	/* Cannot have done final schedule yet: would have vanished. */
6395
	BUG_ON(p->state == TASK_DEAD);
L
Linus Torvalds 已提交
6396

6397
	get_task_struct(p);
L
Linus Torvalds 已提交
6398 6399 6400

	/*
	 * Drop lock around migration; if someone else moves it,
I
Ingo Molnar 已提交
6401
	 * that's OK. No task can be added to this CPU, so iteration is
L
Linus Torvalds 已提交
6402 6403
	 * fine.
	 */
6404
	spin_unlock_irq(&rq->lock);
6405
	move_task_off_dead_cpu(dead_cpu, p);
6406
	spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
6407

6408
	put_task_struct(p);
L
Linus Torvalds 已提交
6409 6410 6411 6412 6413
}

/* release_task() removes task from tasklist, so we won't find dead tasks. */
static void migrate_dead_tasks(unsigned int dead_cpu)
{
6414
	struct rq *rq = cpu_rq(dead_cpu);
I
Ingo Molnar 已提交
6415
	struct task_struct *next;
6416

I
Ingo Molnar 已提交
6417 6418 6419
	for ( ; ; ) {
		if (!rq->nr_running)
			break;
I
Ingo Molnar 已提交
6420
		update_rq_clock(rq);
6421
		next = pick_next_task(rq, rq->curr);
I
Ingo Molnar 已提交
6422 6423
		if (!next)
			break;
D
Dmitry Adamushko 已提交
6424
		next->sched_class->put_prev_task(rq, next);
I
Ingo Molnar 已提交
6425
		migrate_dead(dead_cpu, next);
6426

L
Linus Torvalds 已提交
6427 6428 6429 6430
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

6431 6432 6433
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
6434 6435
	{
		.procname	= "sched_domain",
6436
		.mode		= 0555,
6437
	},
I
Ingo Molnar 已提交
6438
	{0, },
6439 6440 6441
};

static struct ctl_table sd_ctl_root[] = {
6442
	{
6443
		.ctl_name	= CTL_KERN,
6444
		.procname	= "kernel",
6445
		.mode		= 0555,
6446 6447
		.child		= sd_ctl_dir,
	},
I
Ingo Molnar 已提交
6448
	{0, },
6449 6450 6451 6452 6453
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
6454
		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
6455 6456 6457 6458

	return entry;
}

6459 6460
static void sd_free_ctl_entry(struct ctl_table **tablep)
{
6461
	struct ctl_table *entry;
6462

6463 6464 6465
	/*
	 * In the intermediate directories, both the child directory and
	 * procname are dynamically allocated and could fail but the mode
I
Ingo Molnar 已提交
6466
	 * will always be set. In the lowest directory the names are
6467 6468 6469
	 * static strings and all have proc handlers.
	 */
	for (entry = *tablep; entry->mode; entry++) {
6470 6471
		if (entry->child)
			sd_free_ctl_entry(&entry->child);
6472 6473 6474
		if (entry->proc_handler == NULL)
			kfree(entry->procname);
	}
6475 6476 6477 6478 6479

	kfree(*tablep);
	*tablep = NULL;
}

6480
static void
6481
set_table_entry(struct ctl_table *entry,
6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494
		const char *procname, void *data, int maxlen,
		mode_t mode, proc_handler *proc_handler)
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
6495
	struct ctl_table *table = sd_alloc_ctl_entry(13);
6496

6497 6498 6499
	if (table == NULL)
		return NULL;

6500
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
6501
		sizeof(long), 0644, proc_doulongvec_minmax);
6502
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
6503
		sizeof(long), 0644, proc_doulongvec_minmax);
6504
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
6505
		sizeof(int), 0644, proc_dointvec_minmax);
6506
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
6507
		sizeof(int), 0644, proc_dointvec_minmax);
6508
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
6509
		sizeof(int), 0644, proc_dointvec_minmax);
6510
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
6511
		sizeof(int), 0644, proc_dointvec_minmax);
6512
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
6513
		sizeof(int), 0644, proc_dointvec_minmax);
6514
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
6515
		sizeof(int), 0644, proc_dointvec_minmax);
6516
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
6517
		sizeof(int), 0644, proc_dointvec_minmax);
6518
	set_table_entry(&table[9], "cache_nice_tries",
6519 6520
		&sd->cache_nice_tries,
		sizeof(int), 0644, proc_dointvec_minmax);
6521
	set_table_entry(&table[10], "flags", &sd->flags,
6522
		sizeof(int), 0644, proc_dointvec_minmax);
6523 6524 6525
	set_table_entry(&table[11], "name", sd->name,
		CORENAME_MAX_SIZE, 0444, proc_dostring);
	/* &table[12] is terminator */
6526 6527 6528 6529

	return table;
}

6530
static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
6531 6532 6533 6534 6535 6536 6537 6538 6539
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);
6540 6541
	if (table == NULL)
		return NULL;
6542 6543 6544 6545 6546

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
6547
		entry->mode = 0555;
6548 6549 6550 6551 6552 6553 6554 6555
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
6556
static void register_sched_domain_sysctl(void)
6557 6558 6559 6560 6561
{
	int i, cpu_num = num_online_cpus();
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

6562 6563 6564
	WARN_ON(sd_ctl_dir[0].child);
	sd_ctl_dir[0].child = entry;

6565 6566 6567
	if (entry == NULL)
		return;

6568
	for_each_online_cpu(i) {
6569 6570
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
6571
		entry->mode = 0555;
6572
		entry->child = sd_alloc_ctl_cpu_table(i);
6573
		entry++;
6574
	}
6575 6576

	WARN_ON(sd_sysctl_header);
6577 6578
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
6579

6580
/* may be called multiple times per register */
6581 6582
static void unregister_sched_domain_sysctl(void)
{
6583 6584
	if (sd_sysctl_header)
		unregister_sysctl_table(sd_sysctl_header);
6585
	sd_sysctl_header = NULL;
6586 6587
	if (sd_ctl_dir[0].child)
		sd_free_ctl_entry(&sd_ctl_dir[0].child);
6588
}
6589
#else
6590 6591 6592 6593
static void register_sched_domain_sysctl(void)
{
}
static void unregister_sched_domain_sysctl(void)
6594 6595 6596 6597
{
}
#endif

6598 6599 6600 6601 6602
static void set_rq_online(struct rq *rq)
{
	if (!rq->online) {
		const struct sched_class *class;

6603
		cpumask_set_cpu(rq->cpu, rq->rd->online);
6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622
		rq->online = 1;

		for_each_class(class) {
			if (class->rq_online)
				class->rq_online(rq);
		}
	}
}

static void set_rq_offline(struct rq *rq)
{
	if (rq->online) {
		const struct sched_class *class;

		for_each_class(class) {
			if (class->rq_offline)
				class->rq_offline(rq);
		}

6623
		cpumask_clear_cpu(rq->cpu, rq->rd->online);
6624 6625 6626 6627
		rq->online = 0;
	}
}

L
Linus Torvalds 已提交
6628 6629 6630 6631
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
6632 6633
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
6634 6635
{
	struct task_struct *p;
6636
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
6637
	unsigned long flags;
6638
	struct rq *rq;
L
Linus Torvalds 已提交
6639 6640

	switch (action) {
6641

L
Linus Torvalds 已提交
6642
	case CPU_UP_PREPARE:
6643
	case CPU_UP_PREPARE_FROZEN:
I
Ingo Molnar 已提交
6644
		p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
L
Linus Torvalds 已提交
6645 6646 6647 6648 6649
		if (IS_ERR(p))
			return NOTIFY_BAD;
		kthread_bind(p, cpu);
		/* Must be high prio: stop_machine expects to yield to it. */
		rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
6650
		__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
L
Linus Torvalds 已提交
6651 6652 6653
		task_rq_unlock(rq, &flags);
		cpu_rq(cpu)->migration_thread = p;
		break;
6654

L
Linus Torvalds 已提交
6655
	case CPU_ONLINE:
6656
	case CPU_ONLINE_FROZEN:
6657
		/* Strictly unnecessary, as first user will wake it. */
L
Linus Torvalds 已提交
6658
		wake_up_process(cpu_rq(cpu)->migration_thread);
6659 6660 6661 6662 6663

		/* Update our root-domain */
		rq = cpu_rq(cpu);
		spin_lock_irqsave(&rq->lock, flags);
		if (rq->rd) {
6664
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6665 6666

			set_rq_online(rq);
6667 6668
		}
		spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
6669
		break;
6670

L
Linus Torvalds 已提交
6671 6672
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
6673
	case CPU_UP_CANCELED_FROZEN:
6674 6675
		if (!cpu_rq(cpu)->migration_thread)
			break;
I
Ingo Molnar 已提交
6676
		/* Unbind it from offline cpu so it can run. Fall thru. */
6677
		kthread_bind(cpu_rq(cpu)->migration_thread,
R
Rusty Russell 已提交
6678
			     cpumask_any(cpu_online_mask));
L
Linus Torvalds 已提交
6679 6680 6681
		kthread_stop(cpu_rq(cpu)->migration_thread);
		cpu_rq(cpu)->migration_thread = NULL;
		break;
6682

L
Linus Torvalds 已提交
6683
	case CPU_DEAD:
6684
	case CPU_DEAD_FROZEN:
6685
		cpuset_lock(); /* around calls to cpuset_cpus_allowed_lock() */
L
Linus Torvalds 已提交
6686 6687 6688 6689 6690
		migrate_live_tasks(cpu);
		rq = cpu_rq(cpu);
		kthread_stop(rq->migration_thread);
		rq->migration_thread = NULL;
		/* Idle task back to normal (off runqueue, low prio) */
6691
		spin_lock_irq(&rq->lock);
I
Ingo Molnar 已提交
6692
		update_rq_clock(rq);
6693
		deactivate_task(rq, rq->idle, 0);
L
Linus Torvalds 已提交
6694
		rq->idle->static_prio = MAX_PRIO;
I
Ingo Molnar 已提交
6695 6696
		__setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
		rq->idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
6697
		migrate_dead_tasks(cpu);
6698
		spin_unlock_irq(&rq->lock);
6699
		cpuset_unlock();
L
Linus Torvalds 已提交
6700 6701 6702
		migrate_nr_uninterruptible(rq);
		BUG_ON(rq->nr_running != 0);

I
Ingo Molnar 已提交
6703 6704 6705 6706 6707
		/*
		 * No need to migrate the tasks: it was best-effort if
		 * they didn't take sched_hotcpu_mutex. Just wake up
		 * the requestors.
		 */
L
Linus Torvalds 已提交
6708 6709
		spin_lock_irq(&rq->lock);
		while (!list_empty(&rq->migration_queue)) {
6710 6711
			struct migration_req *req;

L
Linus Torvalds 已提交
6712
			req = list_entry(rq->migration_queue.next,
6713
					 struct migration_req, list);
L
Linus Torvalds 已提交
6714
			list_del_init(&req->list);
B
Brian King 已提交
6715
			spin_unlock_irq(&rq->lock);
L
Linus Torvalds 已提交
6716
			complete(&req->done);
B
Brian King 已提交
6717
			spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
6718 6719 6720
		}
		spin_unlock_irq(&rq->lock);
		break;
G
Gregory Haskins 已提交
6721

6722 6723
	case CPU_DYING:
	case CPU_DYING_FROZEN:
G
Gregory Haskins 已提交
6724 6725 6726 6727
		/* Update our root-domain */
		rq = cpu_rq(cpu);
		spin_lock_irqsave(&rq->lock, flags);
		if (rq->rd) {
6728
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6729
			set_rq_offline(rq);
G
Gregory Haskins 已提交
6730 6731 6732
		}
		spin_unlock_irqrestore(&rq->lock, flags);
		break;
L
Linus Torvalds 已提交
6733 6734 6735 6736 6737 6738 6739 6740
#endif
	}
	return NOTIFY_OK;
}

/* Register at highest priority so that task migration (migrate_all_tasks)
 * happens before everything else.
 */
6741
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
6742 6743 6744 6745
	.notifier_call = migration_call,
	.priority = 10
};

6746
static int __init migration_init(void)
L
Linus Torvalds 已提交
6747 6748
{
	void *cpu = (void *)(long)smp_processor_id();
6749
	int err;
6750 6751

	/* Start one for the boot CPU: */
6752 6753
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
6754 6755
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
6756 6757

	return err;
L
Linus Torvalds 已提交
6758
}
6759
early_initcall(migration_init);
L
Linus Torvalds 已提交
6760 6761 6762
#endif

#ifdef CONFIG_SMP
6763

6764
#ifdef CONFIG_SCHED_DEBUG
I
Ingo Molnar 已提交
6765

6766
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6767
				  struct cpumask *groupmask)
L
Linus Torvalds 已提交
6768
{
I
Ingo Molnar 已提交
6769
	struct sched_group *group = sd->groups;
6770
	char str[256];
L
Linus Torvalds 已提交
6771

R
Rusty Russell 已提交
6772
	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6773
	cpumask_clear(groupmask);
I
Ingo Molnar 已提交
6774 6775 6776 6777 6778 6779 6780 6781 6782

	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);

	if (!(sd->flags & SD_LOAD_BALANCE)) {
		printk("does not load-balance\n");
		if (sd->parent)
			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
					" has parent");
		return -1;
N
Nick Piggin 已提交
6783 6784
	}

6785
	printk(KERN_CONT "span %s level %s\n", str, sd->name);
I
Ingo Molnar 已提交
6786

6787
	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
I
Ingo Molnar 已提交
6788 6789 6790
		printk(KERN_ERR "ERROR: domain->span does not contain "
				"CPU%d\n", cpu);
	}
6791
	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
I
Ingo Molnar 已提交
6792 6793 6794
		printk(KERN_ERR "ERROR: domain->groups does not contain"
				" CPU%d\n", cpu);
	}
L
Linus Torvalds 已提交
6795

I
Ingo Molnar 已提交
6796
	printk(KERN_DEBUG "%*s groups:", level + 1, "");
L
Linus Torvalds 已提交
6797
	do {
I
Ingo Molnar 已提交
6798 6799 6800
		if (!group) {
			printk("\n");
			printk(KERN_ERR "ERROR: group is NULL\n");
L
Linus Torvalds 已提交
6801 6802 6803
			break;
		}

I
Ingo Molnar 已提交
6804 6805 6806 6807 6808 6809
		if (!group->__cpu_power) {
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: domain->cpu_power not "
					"set\n");
			break;
		}
L
Linus Torvalds 已提交
6810

6811
		if (!cpumask_weight(sched_group_cpus(group))) {
I
Ingo Molnar 已提交
6812 6813 6814 6815
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: empty group\n");
			break;
		}
L
Linus Torvalds 已提交
6816

6817
		if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
I
Ingo Molnar 已提交
6818 6819 6820 6821
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: repeated CPUs\n");
			break;
		}
L
Linus Torvalds 已提交
6822

6823
		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
L
Linus Torvalds 已提交
6824

R
Rusty Russell 已提交
6825
		cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
I
Ingo Molnar 已提交
6826
		printk(KERN_CONT " %s", str);
L
Linus Torvalds 已提交
6827

I
Ingo Molnar 已提交
6828 6829 6830
		group = group->next;
	} while (group != sd->groups);
	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
6831

6832
	if (!cpumask_equal(sched_domain_span(sd), groupmask))
I
Ingo Molnar 已提交
6833
		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
L
Linus Torvalds 已提交
6834

6835 6836
	if (sd->parent &&
	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
I
Ingo Molnar 已提交
6837 6838 6839 6840
		printk(KERN_ERR "ERROR: parent span is not a superset "
			"of domain->span\n");
	return 0;
}
L
Linus Torvalds 已提交
6841

I
Ingo Molnar 已提交
6842 6843
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
6844
	cpumask_var_t groupmask;
I
Ingo Molnar 已提交
6845
	int level = 0;
L
Linus Torvalds 已提交
6846

I
Ingo Molnar 已提交
6847 6848 6849 6850
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}
L
Linus Torvalds 已提交
6851

I
Ingo Molnar 已提交
6852 6853
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

6854
	if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6855 6856 6857 6858
		printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
		return;
	}

I
Ingo Molnar 已提交
6859
	for (;;) {
6860
		if (sched_domain_debug_one(sd, cpu, level, groupmask))
I
Ingo Molnar 已提交
6861
			break;
L
Linus Torvalds 已提交
6862 6863
		level++;
		sd = sd->parent;
6864
		if (!sd)
I
Ingo Molnar 已提交
6865 6866
			break;
	}
6867
	free_cpumask_var(groupmask);
L
Linus Torvalds 已提交
6868
}
6869
#else /* !CONFIG_SCHED_DEBUG */
6870
# define sched_domain_debug(sd, cpu) do { } while (0)
6871
#endif /* CONFIG_SCHED_DEBUG */
L
Linus Torvalds 已提交
6872

6873
static int sd_degenerate(struct sched_domain *sd)
6874
{
6875
	if (cpumask_weight(sched_domain_span(sd)) == 1)
6876 6877 6878 6879 6880 6881
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
6882 6883 6884
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
	if (sd->flags & (SD_WAKE_IDLE |
			 SD_WAKE_AFFINE |
			 SD_WAKE_BALANCE))
		return 0;

	return 1;
}

6898 6899
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6900 6901 6902 6903 6904 6905
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

6906
	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917
		return 0;

	/* Does parent contain flags not in child? */
	/* WAKE_BALANCE is a subset of WAKE_AFFINE */
	if (cflags & SD_WAKE_AFFINE)
		pflags &= ~SD_WAKE_BALANCE;
	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
6918 6919 6920
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
6921 6922
		if (nr_node_ids == 1)
			pflags &= ~SD_SERIALIZE;
6923 6924 6925 6926 6927 6928 6929
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

6930 6931
static void free_rootdomain(struct root_domain *rd)
{
6932 6933
	cpupri_cleanup(&rd->cpupri);

6934 6935 6936 6937 6938 6939
	free_cpumask_var(rd->rto_mask);
	free_cpumask_var(rd->online);
	free_cpumask_var(rd->span);
	kfree(rd);
}

G
Gregory Haskins 已提交
6940 6941 6942 6943 6944 6945 6946 6947 6948
static void rq_attach_root(struct rq *rq, struct root_domain *rd)
{
	unsigned long flags;

	spin_lock_irqsave(&rq->lock, flags);

	if (rq->rd) {
		struct root_domain *old_rd = rq->rd;

6949
		if (cpumask_test_cpu(rq->cpu, old_rd->online))
6950
			set_rq_offline(rq);
G
Gregory Haskins 已提交
6951

6952
		cpumask_clear_cpu(rq->cpu, old_rd->span);
6953

G
Gregory Haskins 已提交
6954
		if (atomic_dec_and_test(&old_rd->refcount))
6955
			free_rootdomain(old_rd);
G
Gregory Haskins 已提交
6956 6957 6958 6959 6960
	}

	atomic_inc(&rd->refcount);
	rq->rd = rd;

6961 6962
	cpumask_set_cpu(rq->cpu, rd->span);
	if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6963
		set_rq_online(rq);
G
Gregory Haskins 已提交
6964 6965 6966 6967

	spin_unlock_irqrestore(&rq->lock, flags);
}

L
Li Zefan 已提交
6968
static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem)
G
Gregory Haskins 已提交
6969 6970 6971
{
	memset(rd, 0, sizeof(*rd));

6972 6973 6974 6975
	if (bootmem) {
		alloc_bootmem_cpumask_var(&def_root_domain.span);
		alloc_bootmem_cpumask_var(&def_root_domain.online);
		alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6976
		cpupri_init(&rd->cpupri, true);
6977 6978 6979 6980
		return 0;
	}

	if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6981
		goto out;
6982 6983 6984 6985
	if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
		goto free_span;
	if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
		goto free_online;
6986

6987 6988
	if (cpupri_init(&rd->cpupri, false) != 0)
		goto free_rto_mask;
6989
	return 0;
6990

6991 6992
free_rto_mask:
	free_cpumask_var(rd->rto_mask);
6993 6994 6995 6996
free_online:
	free_cpumask_var(rd->online);
free_span:
	free_cpumask_var(rd->span);
6997
out:
6998
	return -ENOMEM;
G
Gregory Haskins 已提交
6999 7000 7001 7002
}

static void init_defrootdomain(void)
{
7003 7004
	init_rootdomain(&def_root_domain, true);

G
Gregory Haskins 已提交
7005 7006 7007
	atomic_set(&def_root_domain.refcount, 1);
}

7008
static struct root_domain *alloc_rootdomain(void)
G
Gregory Haskins 已提交
7009 7010 7011 7012 7013 7014 7015
{
	struct root_domain *rd;

	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
	if (!rd)
		return NULL;

7016 7017 7018 7019
	if (init_rootdomain(rd, false) != 0) {
		kfree(rd);
		return NULL;
	}
G
Gregory Haskins 已提交
7020 7021 7022 7023

	return rd;
}

L
Linus Torvalds 已提交
7024
/*
I
Ingo Molnar 已提交
7025
 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
L
Linus Torvalds 已提交
7026 7027
 * hold the hotplug lock.
 */
I
Ingo Molnar 已提交
7028 7029
static void
cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
L
Linus Torvalds 已提交
7030
{
7031
	struct rq *rq = cpu_rq(cpu);
7032 7033 7034
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
7035
	for (tmp = sd; tmp; ) {
7036 7037 7038
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
7039

7040
		if (sd_parent_degenerate(tmp, parent)) {
7041
			tmp->parent = parent->parent;
7042 7043
			if (parent->parent)
				parent->parent->child = tmp;
7044 7045
		} else
			tmp = tmp->parent;
7046 7047
	}

7048
	if (sd && sd_degenerate(sd)) {
7049
		sd = sd->parent;
7050 7051 7052
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
7053 7054 7055

	sched_domain_debug(sd, cpu);

G
Gregory Haskins 已提交
7056
	rq_attach_root(rq, rd);
N
Nick Piggin 已提交
7057
	rcu_assign_pointer(rq->sd, sd);
L
Linus Torvalds 已提交
7058 7059 7060
}

/* cpus with isolated domains */
7061
static cpumask_var_t cpu_isolated_map;
L
Linus Torvalds 已提交
7062 7063 7064 7065

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
R
Rusty Russell 已提交
7066
	cpulist_parse(str, cpu_isolated_map);
L
Linus Torvalds 已提交
7067 7068 7069
	return 1;
}

I
Ingo Molnar 已提交
7070
__setup("isolcpus=", isolated_cpu_setup);
L
Linus Torvalds 已提交
7071 7072

/*
7073 7074
 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
 * to a function which identifies what group(along with sched group) a CPU
7075 7076
 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
 * (due to the fact that we keep track of groups covered with a struct cpumask).
L
Linus Torvalds 已提交
7077 7078 7079 7080 7081
 *
 * init_sched_build_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
 */
7082
static void
7083 7084 7085
init_sched_build_groups(const struct cpumask *span,
			const struct cpumask *cpu_map,
			int (*group_fn)(int cpu, const struct cpumask *cpu_map,
7086
					struct sched_group **sg,
7087 7088
					struct cpumask *tmpmask),
			struct cpumask *covered, struct cpumask *tmpmask)
L
Linus Torvalds 已提交
7089 7090 7091 7092
{
	struct sched_group *first = NULL, *last = NULL;
	int i;

7093
	cpumask_clear(covered);
7094

7095
	for_each_cpu(i, span) {
7096
		struct sched_group *sg;
7097
		int group = group_fn(i, cpu_map, &sg, tmpmask);
L
Linus Torvalds 已提交
7098 7099
		int j;

7100
		if (cpumask_test_cpu(i, covered))
L
Linus Torvalds 已提交
7101 7102
			continue;

7103
		cpumask_clear(sched_group_cpus(sg));
7104
		sg->__cpu_power = 0;
L
Linus Torvalds 已提交
7105

7106
		for_each_cpu(j, span) {
7107
			if (group_fn(j, cpu_map, NULL, tmpmask) != group)
L
Linus Torvalds 已提交
7108 7109
				continue;

7110
			cpumask_set_cpu(j, covered);
7111
			cpumask_set_cpu(j, sched_group_cpus(sg));
L
Linus Torvalds 已提交
7112 7113 7114 7115 7116 7117 7118 7119 7120 7121
		}
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
}

7122
#define SD_NODES_PER_DOMAIN 16
L
Linus Torvalds 已提交
7123

7124
#ifdef CONFIG_NUMA
7125

7126 7127 7128 7129 7130
/**
 * find_next_best_node - find the next node to include in a sched_domain
 * @node: node whose sched_domain we're building
 * @used_nodes: nodes already in the sched_domain
 *
I
Ingo Molnar 已提交
7131
 * Find the next node to include in a given scheduling domain. Simply
7132 7133 7134 7135
 * finds the closest node not already in the @used_nodes map.
 *
 * Should use nodemask_t.
 */
7136
static int find_next_best_node(int node, nodemask_t *used_nodes)
7137 7138 7139 7140 7141
{
	int i, n, val, min_val, best_node = 0;

	min_val = INT_MAX;

7142
	for (i = 0; i < nr_node_ids; i++) {
7143
		/* Start at @node */
7144
		n = (node + i) % nr_node_ids;
7145 7146 7147 7148 7149

		if (!nr_cpus_node(n))
			continue;

		/* Skip already used nodes */
7150
		if (node_isset(n, *used_nodes))
7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161
			continue;

		/* Simple min distance search */
		val = node_distance(node, n);

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

7162
	node_set(best_node, *used_nodes);
7163 7164 7165 7166 7167 7168
	return best_node;
}

/**
 * sched_domain_node_span - get a cpumask for a node's sched_domain
 * @node: node whose cpumask we're constructing
7169
 * @span: resulting cpumask
7170
 *
I
Ingo Molnar 已提交
7171
 * Given a node, construct a good cpumask for its sched_domain to span. It
7172 7173 7174
 * should be one that prevents unnecessary balancing, but also spreads tasks
 * out optimally.
 */
7175
static void sched_domain_node_span(int node, struct cpumask *span)
7176
{
7177
	nodemask_t used_nodes;
7178
	int i;
7179

7180
	cpumask_clear(span);
7181
	nodes_clear(used_nodes);
7182

7183
	cpumask_or(span, span, cpumask_of_node(node));
7184
	node_set(node, used_nodes);
7185 7186

	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
7187
		int next_node = find_next_best_node(node, &used_nodes);
7188

7189
		cpumask_or(span, span, cpumask_of_node(next_node));
7190 7191
	}
}
7192
#endif /* CONFIG_NUMA */
7193

7194
int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7195

7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210
/*
 * The cpus mask in sched_group and sched_domain hangs off the end.
 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
 * for nr_cpu_ids < CONFIG_NR_CPUS.
 */
struct static_sched_group {
	struct sched_group sg;
	DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
};

struct static_sched_domain {
	struct sched_domain sd;
	DECLARE_BITMAP(span, CONFIG_NR_CPUS);
};

7211
/*
7212
 * SMT sched-domains:
7213
 */
L
Linus Torvalds 已提交
7214
#ifdef CONFIG_SCHED_SMT
7215 7216
static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7217

I
Ingo Molnar 已提交
7218
static int
7219 7220
cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
		 struct sched_group **sg, struct cpumask *unused)
L
Linus Torvalds 已提交
7221
{
7222
	if (sg)
7223
		*sg = &per_cpu(sched_group_cpus, cpu).sg;
L
Linus Torvalds 已提交
7224 7225
	return cpu;
}
7226
#endif /* CONFIG_SCHED_SMT */
L
Linus Torvalds 已提交
7227

7228 7229 7230
/*
 * multi-core sched-domains:
 */
7231
#ifdef CONFIG_SCHED_MC
7232 7233
static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7234
#endif /* CONFIG_SCHED_MC */
7235 7236

#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
7237
static int
7238 7239
cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
		  struct sched_group **sg, struct cpumask *mask)
7240
{
7241
	int group;
7242

7243 7244
	cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
	group = cpumask_first(mask);
7245
	if (sg)
7246
		*sg = &per_cpu(sched_group_core, group).sg;
7247
	return group;
7248 7249
}
#elif defined(CONFIG_SCHED_MC)
I
Ingo Molnar 已提交
7250
static int
7251 7252
cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
		  struct sched_group **sg, struct cpumask *unused)
7253
{
7254
	if (sg)
7255
		*sg = &per_cpu(sched_group_core, cpu).sg;
7256 7257 7258 7259
	return cpu;
}
#endif

7260 7261
static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7262

I
Ingo Molnar 已提交
7263
static int
7264 7265
cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
		  struct sched_group **sg, struct cpumask *mask)
L
Linus Torvalds 已提交
7266
{
7267
	int group;
7268
#ifdef CONFIG_SCHED_MC
7269
	cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
7270
	group = cpumask_first(mask);
7271
#elif defined(CONFIG_SCHED_SMT)
7272 7273
	cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
	group = cpumask_first(mask);
L
Linus Torvalds 已提交
7274
#else
7275
	group = cpu;
L
Linus Torvalds 已提交
7276
#endif
7277
	if (sg)
7278
		*sg = &per_cpu(sched_group_phys, group).sg;
7279
	return group;
L
Linus Torvalds 已提交
7280 7281 7282 7283
}

#ifdef CONFIG_NUMA
/*
7284 7285 7286
 * The init_sched_build_groups can't handle what we want to do with node
 * groups, so roll our own. Now each node has its own list of groups which
 * gets dynamically allocated.
L
Linus Torvalds 已提交
7287
 */
7288
static DEFINE_PER_CPU(struct static_sched_domain, node_domains);
7289
static struct sched_group ***sched_group_nodes_bycpu;
L
Linus Torvalds 已提交
7290

7291
static DEFINE_PER_CPU(struct static_sched_domain, allnodes_domains);
7292
static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7293

7294 7295 7296
static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
				 struct sched_group **sg,
				 struct cpumask *nodemask)
7297
{
7298 7299
	int group;

7300
	cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
7301
	group = cpumask_first(nodemask);
7302 7303

	if (sg)
7304
		*sg = &per_cpu(sched_group_allnodes, group).sg;
7305
	return group;
L
Linus Torvalds 已提交
7306
}
7307

7308 7309 7310 7311 7312 7313 7314
static void init_numa_sched_groups_power(struct sched_group *group_head)
{
	struct sched_group *sg = group_head;
	int j;

	if (!sg)
		return;
7315
	do {
7316
		for_each_cpu(j, sched_group_cpus(sg)) {
7317
			struct sched_domain *sd;
7318

7319
			sd = &per_cpu(phys_domains, j).sd;
7320
			if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7321 7322 7323 7324 7325 7326
				/*
				 * Only add "power" once for each
				 * physical package.
				 */
				continue;
			}
7327

7328 7329 7330 7331
			sg_inc_cpu_power(sg, sd->groups->__cpu_power);
		}
		sg = sg->next;
	} while (sg != group_head);
7332
}
7333
#endif /* CONFIG_NUMA */
L
Linus Torvalds 已提交
7334

7335
#ifdef CONFIG_NUMA
7336
/* Free memory allocated for various sched_group structures */
7337 7338
static void free_sched_groups(const struct cpumask *cpu_map,
			      struct cpumask *nodemask)
7339
{
7340
	int cpu, i;
7341

7342
	for_each_cpu(cpu, cpu_map) {
7343 7344 7345 7346 7347 7348
		struct sched_group **sched_group_nodes
			= sched_group_nodes_bycpu[cpu];

		if (!sched_group_nodes)
			continue;

7349
		for (i = 0; i < nr_node_ids; i++) {
7350 7351
			struct sched_group *oldsg, *sg = sched_group_nodes[i];

7352
			cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7353
			if (cpumask_empty(nodemask))
7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369
				continue;

			if (sg == NULL)
				continue;
			sg = sg->next;
next_sg:
			oldsg = sg;
			sg = sg->next;
			kfree(oldsg);
			if (oldsg != sched_group_nodes[i])
				goto next_sg;
		}
		kfree(sched_group_nodes);
		sched_group_nodes_bycpu[cpu] = NULL;
	}
}
7370
#else /* !CONFIG_NUMA */
7371 7372
static void free_sched_groups(const struct cpumask *cpu_map,
			      struct cpumask *nodemask)
7373 7374
{
}
7375
#endif /* CONFIG_NUMA */
7376

7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 *
 * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
 * the maximum number of tasks a group can handle in the presence of other idle
 * or lightly loaded groups in the same sched domain.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
	struct sched_domain *child;
	struct sched_group *group;

	WARN_ON(!sd || !sd->groups);

7398
	if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7399 7400 7401 7402
		return;

	child = sd->child;

7403 7404
	sd->groups->__cpu_power = 0;

7405 7406 7407 7408 7409 7410 7411 7412 7413 7414
	/*
	 * For perf policy, if the groups in child domain share resources
	 * (for example cores sharing some portions of the cache hierarchy
	 * or SMT), then set this domain groups cpu_power such that each group
	 * can handle only one task, when there are other idle groups in the
	 * same sched domain.
	 */
	if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
		       (child->flags &
			(SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
7415
		sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
7416 7417 7418 7419 7420 7421 7422 7423
		return;
	}

	/*
	 * add cpu_power of each child group to this groups cpu_power
	 */
	group = child->groups;
	do {
7424
		sg_inc_cpu_power(sd->groups, group->__cpu_power);
7425 7426 7427 7428
		group = group->next;
	} while (group != child->groups);
}

7429 7430 7431 7432 7433
/*
 * Initializers for schedule domains
 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
 */

7434 7435 7436 7437 7438 7439
#ifdef CONFIG_SCHED_DEBUG
# define SD_INIT_NAME(sd, type)		sd->name = #type
#else
# define SD_INIT_NAME(sd, type)		do { } while (0)
#endif

7440
#define	SD_INIT(sd, type)	sd_init_##type(sd)
7441

7442 7443 7444 7445 7446
#define SD_INIT_FUNC(type)	\
static noinline void sd_init_##type(struct sched_domain *sd)	\
{								\
	memset(sd, 0, sizeof(*sd));				\
	*sd = SD_##type##_INIT;					\
7447
	sd->level = SD_LV_##type;				\
7448
	SD_INIT_NAME(sd, type);					\
7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462
}

SD_INIT_FUNC(CPU)
#ifdef CONFIG_NUMA
 SD_INIT_FUNC(ALLNODES)
 SD_INIT_FUNC(NODE)
#endif
#ifdef CONFIG_SCHED_SMT
 SD_INIT_FUNC(SIBLING)
#endif
#ifdef CONFIG_SCHED_MC
 SD_INIT_FUNC(MC)
#endif

7463 7464 7465 7466
static int default_relax_domain_level = -1;

static int __init setup_relax_domain_level(char *str)
{
7467 7468 7469 7470 7471 7472
	unsigned long val;

	val = simple_strtoul(str, NULL, 0);
	if (val < SD_LV_MAX)
		default_relax_domain_level = val;

7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497
	return 1;
}
__setup("relax_domain_level=", setup_relax_domain_level);

static void set_domain_attribute(struct sched_domain *sd,
				 struct sched_domain_attr *attr)
{
	int request;

	if (!attr || attr->relax_domain_level < 0) {
		if (default_relax_domain_level < 0)
			return;
		else
			request = default_relax_domain_level;
	} else
		request = attr->relax_domain_level;
	if (request < sd->level) {
		/* turn off idle balance on this domain */
		sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
	} else {
		/* turn on idle balance on this domain */
		sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
	}
}

L
Linus Torvalds 已提交
7498
/*
7499 7500
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
L
Linus Torvalds 已提交
7501
 */
7502
static int __build_sched_domains(const struct cpumask *cpu_map,
7503
				 struct sched_domain_attr *attr)
L
Linus Torvalds 已提交
7504
{
7505
	int i, err = -ENOMEM;
G
Gregory Haskins 已提交
7506
	struct root_domain *rd;
7507 7508
	cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
		tmpmask;
7509
#ifdef CONFIG_NUMA
7510
	cpumask_var_t domainspan, covered, notcovered;
7511
	struct sched_group **sched_group_nodes = NULL;
7512
	int sd_allnodes = 0;
7513

7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533
	if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
		goto out;
	if (!alloc_cpumask_var(&covered, GFP_KERNEL))
		goto free_domainspan;
	if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
		goto free_covered;
#endif

	if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
		goto free_notcovered;
	if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
		goto free_nodemask;
	if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
		goto free_this_sibling_map;
	if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
		goto free_this_core_map;
	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
		goto free_send_covered;

#ifdef CONFIG_NUMA
7534 7535 7536
	/*
	 * Allocate the per-node list of sched groups
	 */
7537
	sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
I
Ingo Molnar 已提交
7538
				    GFP_KERNEL);
7539 7540
	if (!sched_group_nodes) {
		printk(KERN_WARNING "Can not alloc sched group node list\n");
7541
		goto free_tmpmask;
7542 7543
	}
#endif
L
Linus Torvalds 已提交
7544

7545
	rd = alloc_rootdomain();
G
Gregory Haskins 已提交
7546 7547
	if (!rd) {
		printk(KERN_WARNING "Cannot alloc root domain\n");
7548
		goto free_sched_groups;
G
Gregory Haskins 已提交
7549 7550
	}

7551
#ifdef CONFIG_NUMA
7552
	sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7553 7554
#endif

L
Linus Torvalds 已提交
7555
	/*
7556
	 * Set up domains for cpus specified by the cpu_map.
L
Linus Torvalds 已提交
7557
	 */
7558
	for_each_cpu(i, cpu_map) {
L
Linus Torvalds 已提交
7559 7560
		struct sched_domain *sd = NULL, *p;

7561
		cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
L
Linus Torvalds 已提交
7562 7563

#ifdef CONFIG_NUMA
7564 7565
		if (cpumask_weight(cpu_map) >
				SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7566
			sd = &per_cpu(allnodes_domains, i).sd;
7567
			SD_INIT(sd, ALLNODES);
7568
			set_domain_attribute(sd, attr);
7569
			cpumask_copy(sched_domain_span(sd), cpu_map);
7570
			cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7571
			p = sd;
7572
			sd_allnodes = 1;
7573 7574 7575
		} else
			p = NULL;

7576
		sd = &per_cpu(node_domains, i).sd;
7577
		SD_INIT(sd, NODE);
7578
		set_domain_attribute(sd, attr);
7579
		sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7580
		sd->parent = p;
7581 7582
		if (p)
			p->child = sd;
7583 7584
		cpumask_and(sched_domain_span(sd),
			    sched_domain_span(sd), cpu_map);
L
Linus Torvalds 已提交
7585 7586 7587
#endif

		p = sd;
7588
		sd = &per_cpu(phys_domains, i).sd;
7589
		SD_INIT(sd, CPU);
7590
		set_domain_attribute(sd, attr);
7591
		cpumask_copy(sched_domain_span(sd), nodemask);
L
Linus Torvalds 已提交
7592
		sd->parent = p;
7593 7594
		if (p)
			p->child = sd;
7595
		cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
L
Linus Torvalds 已提交
7596

7597 7598
#ifdef CONFIG_SCHED_MC
		p = sd;
7599
		sd = &per_cpu(core_domains, i).sd;
7600
		SD_INIT(sd, MC);
7601
		set_domain_attribute(sd, attr);
7602 7603
		cpumask_and(sched_domain_span(sd), cpu_map,
						   cpu_coregroup_mask(i));
7604
		sd->parent = p;
7605
		p->child = sd;
7606
		cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
7607 7608
#endif

L
Linus Torvalds 已提交
7609 7610
#ifdef CONFIG_SCHED_SMT
		p = sd;
7611
		sd = &per_cpu(cpu_domains, i).sd;
7612
		SD_INIT(sd, SIBLING);
7613
		set_domain_attribute(sd, attr);
7614 7615
		cpumask_and(sched_domain_span(sd),
			    &per_cpu(cpu_sibling_map, i), cpu_map);
L
Linus Torvalds 已提交
7616
		sd->parent = p;
7617
		p->child = sd;
7618
		cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
L
Linus Torvalds 已提交
7619 7620 7621 7622 7623
#endif
	}

#ifdef CONFIG_SCHED_SMT
	/* Set up CPU (sibling) groups */
7624
	for_each_cpu(i, cpu_map) {
7625 7626 7627
		cpumask_and(this_sibling_map,
			    &per_cpu(cpu_sibling_map, i), cpu_map);
		if (i != cpumask_first(this_sibling_map))
L
Linus Torvalds 已提交
7628 7629
			continue;

I
Ingo Molnar 已提交
7630
		init_sched_build_groups(this_sibling_map, cpu_map,
7631 7632
					&cpu_to_cpu_group,
					send_covered, tmpmask);
L
Linus Torvalds 已提交
7633 7634 7635
	}
#endif

7636 7637
#ifdef CONFIG_SCHED_MC
	/* Set up multi-core groups */
7638
	for_each_cpu(i, cpu_map) {
7639
		cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
7640
		if (i != cpumask_first(this_core_map))
7641
			continue;
7642

I
Ingo Molnar 已提交
7643
		init_sched_build_groups(this_core_map, cpu_map,
7644 7645
					&cpu_to_core_group,
					send_covered, tmpmask);
7646 7647 7648
	}
#endif

L
Linus Torvalds 已提交
7649
	/* Set up physical groups */
7650
	for (i = 0; i < nr_node_ids; i++) {
7651
		cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7652
		if (cpumask_empty(nodemask))
L
Linus Torvalds 已提交
7653 7654
			continue;

7655 7656 7657
		init_sched_build_groups(nodemask, cpu_map,
					&cpu_to_phys_group,
					send_covered, tmpmask);
L
Linus Torvalds 已提交
7658 7659 7660 7661
	}

#ifdef CONFIG_NUMA
	/* Set up node groups */
7662 7663 7664 7665 7666
	if (sd_allnodes) {
		init_sched_build_groups(cpu_map, cpu_map,
					&cpu_to_allnodes_group,
					send_covered, tmpmask);
	}
7667

7668
	for (i = 0; i < nr_node_ids; i++) {
7669 7670 7671 7672
		/* Set up node groups */
		struct sched_group *sg, *prev;
		int j;

7673
		cpumask_clear(covered);
7674
		cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7675
		if (cpumask_empty(nodemask)) {
7676
			sched_group_nodes[i] = NULL;
7677
			continue;
7678
		}
7679

7680
		sched_domain_node_span(i, domainspan);
7681
		cpumask_and(domainspan, domainspan, cpu_map);
7682

7683 7684
		sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
				  GFP_KERNEL, i);
7685 7686 7687 7688 7689
		if (!sg) {
			printk(KERN_WARNING "Can not alloc domain group for "
				"node %d\n", i);
			goto error;
		}
7690
		sched_group_nodes[i] = sg;
7691
		for_each_cpu(j, nodemask) {
7692
			struct sched_domain *sd;
I
Ingo Molnar 已提交
7693

7694
			sd = &per_cpu(node_domains, j).sd;
7695 7696
			sd->groups = sg;
		}
7697
		sg->__cpu_power = 0;
7698
		cpumask_copy(sched_group_cpus(sg), nodemask);
7699
		sg->next = sg;
7700
		cpumask_or(covered, covered, nodemask);
7701 7702
		prev = sg;

7703 7704
		for (j = 0; j < nr_node_ids; j++) {
			int n = (i + j) % nr_node_ids;
7705

7706 7707 7708 7709
			cpumask_complement(notcovered, covered);
			cpumask_and(tmpmask, notcovered, cpu_map);
			cpumask_and(tmpmask, tmpmask, domainspan);
			if (cpumask_empty(tmpmask))
7710 7711
				break;

7712
			cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
7713
			if (cpumask_empty(tmpmask))
7714 7715
				continue;

7716 7717
			sg = kmalloc_node(sizeof(struct sched_group) +
					  cpumask_size(),
7718
					  GFP_KERNEL, i);
7719 7720 7721
			if (!sg) {
				printk(KERN_WARNING
				"Can not alloc domain group for node %d\n", j);
7722
				goto error;
7723
			}
7724
			sg->__cpu_power = 0;
7725
			cpumask_copy(sched_group_cpus(sg), tmpmask);
7726
			sg->next = prev->next;
7727
			cpumask_or(covered, covered, tmpmask);
7728 7729 7730 7731
			prev->next = sg;
			prev = sg;
		}
	}
L
Linus Torvalds 已提交
7732 7733 7734
#endif

	/* Calculate CPU power for physical packages and nodes */
7735
#ifdef CONFIG_SCHED_SMT
7736
	for_each_cpu(i, cpu_map) {
7737
		struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
I
Ingo Molnar 已提交
7738

7739
		init_sched_groups_power(i, sd);
7740
	}
L
Linus Torvalds 已提交
7741
#endif
7742
#ifdef CONFIG_SCHED_MC
7743
	for_each_cpu(i, cpu_map) {
7744
		struct sched_domain *sd = &per_cpu(core_domains, i).sd;
I
Ingo Molnar 已提交
7745

7746
		init_sched_groups_power(i, sd);
7747 7748
	}
#endif
7749

7750
	for_each_cpu(i, cpu_map) {
7751
		struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
I
Ingo Molnar 已提交
7752

7753
		init_sched_groups_power(i, sd);
L
Linus Torvalds 已提交
7754 7755
	}

7756
#ifdef CONFIG_NUMA
7757
	for (i = 0; i < nr_node_ids; i++)
7758
		init_numa_sched_groups_power(sched_group_nodes[i]);
7759

7760 7761
	if (sd_allnodes) {
		struct sched_group *sg;
7762

7763
		cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7764
								tmpmask);
7765 7766
		init_numa_sched_groups_power(sg);
	}
7767 7768
#endif

L
Linus Torvalds 已提交
7769
	/* Attach the domains */
7770
	for_each_cpu(i, cpu_map) {
L
Linus Torvalds 已提交
7771 7772
		struct sched_domain *sd;
#ifdef CONFIG_SCHED_SMT
7773
		sd = &per_cpu(cpu_domains, i).sd;
7774
#elif defined(CONFIG_SCHED_MC)
7775
		sd = &per_cpu(core_domains, i).sd;
L
Linus Torvalds 已提交
7776
#else
7777
		sd = &per_cpu(phys_domains, i).sd;
L
Linus Torvalds 已提交
7778
#endif
G
Gregory Haskins 已提交
7779
		cpu_attach_domain(sd, rd, i);
L
Linus Torvalds 已提交
7780
	}
7781

7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809
	err = 0;

free_tmpmask:
	free_cpumask_var(tmpmask);
free_send_covered:
	free_cpumask_var(send_covered);
free_this_core_map:
	free_cpumask_var(this_core_map);
free_this_sibling_map:
	free_cpumask_var(this_sibling_map);
free_nodemask:
	free_cpumask_var(nodemask);
free_notcovered:
#ifdef CONFIG_NUMA
	free_cpumask_var(notcovered);
free_covered:
	free_cpumask_var(covered);
free_domainspan:
	free_cpumask_var(domainspan);
out:
#endif
	return err;

free_sched_groups:
#ifdef CONFIG_NUMA
	kfree(sched_group_nodes);
#endif
	goto free_tmpmask;
7810

7811
#ifdef CONFIG_NUMA
7812
error:
7813
	free_sched_groups(cpu_map, tmpmask);
7814
	free_rootdomain(rd);
7815
	goto free_tmpmask;
7816
#endif
L
Linus Torvalds 已提交
7817
}
P
Paul Jackson 已提交
7818

7819
static int build_sched_domains(const struct cpumask *cpu_map)
7820 7821 7822 7823
{
	return __build_sched_domains(cpu_map, NULL);
}

7824
static struct cpumask *doms_cur;	/* current sched domains */
P
Paul Jackson 已提交
7825
static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
I
Ingo Molnar 已提交
7826 7827
static struct sched_domain_attr *dattr_cur;
				/* attribues of custom domains in 'doms_cur' */
P
Paul Jackson 已提交
7828 7829 7830

/*
 * Special case: If a kmalloc of a doms_cur partition (array of
7831 7832
 * cpumask) fails, then fallback to a single sched domain,
 * as determined by the single cpumask fallback_doms.
P
Paul Jackson 已提交
7833
 */
7834
static cpumask_var_t fallback_doms;
P
Paul Jackson 已提交
7835

7836 7837 7838 7839 7840 7841
/*
 * arch_update_cpu_topology lets virtualized architectures update the
 * cpu core maps. It is supposed to return 1 if the topology changed
 * or 0 if it stayed the same.
 */
int __attribute__((weak)) arch_update_cpu_topology(void)
7842
{
7843
	return 0;
7844 7845
}

7846
/*
I
Ingo Molnar 已提交
7847
 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
P
Paul Jackson 已提交
7848 7849
 * For now this just excludes isolated cpus, but could be used to
 * exclude other special cases in the future.
7850
 */
7851
static int arch_init_sched_domains(const struct cpumask *cpu_map)
7852
{
7853 7854
	int err;

7855
	arch_update_cpu_topology();
P
Paul Jackson 已提交
7856
	ndoms_cur = 1;
7857
	doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
P
Paul Jackson 已提交
7858
	if (!doms_cur)
7859
		doms_cur = fallback_doms;
7860
	cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7861
	dattr_cur = NULL;
7862
	err = build_sched_domains(doms_cur);
7863
	register_sched_domain_sysctl();
7864 7865

	return err;
7866 7867
}

7868 7869
static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
				       struct cpumask *tmpmask)
L
Linus Torvalds 已提交
7870
{
7871
	free_sched_groups(cpu_map, tmpmask);
7872
}
L
Linus Torvalds 已提交
7873

7874 7875 7876 7877
/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
7878
static void detach_destroy_domains(const struct cpumask *cpu_map)
7879
{
7880 7881
	/* Save because hotplug lock held. */
	static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7882 7883
	int i;

7884
	for_each_cpu(i, cpu_map)
G
Gregory Haskins 已提交
7885
		cpu_attach_domain(NULL, &def_root_domain, i);
7886
	synchronize_sched();
7887
	arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7888 7889
}

7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905
/* handle null as "default" */
static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
			struct sched_domain_attr *new, int idx_new)
{
	struct sched_domain_attr tmp;

	/* fast path */
	if (!new && !cur)
		return 1;

	tmp = SD_ATTR_INIT;
	return !memcmp(cur ? (cur + idx_cur) : &tmp,
			new ? (new + idx_new) : &tmp,
			sizeof(struct sched_domain_attr));
}

P
Paul Jackson 已提交
7906 7907
/*
 * Partition sched domains as specified by the 'ndoms_new'
I
Ingo Molnar 已提交
7908
 * cpumasks in the array doms_new[] of cpumasks. This compares
P
Paul Jackson 已提交
7909 7910 7911
 * doms_new[] to the current sched domain partitioning, doms_cur[].
 * It destroys each deleted domain and builds each new domain.
 *
7912
 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
I
Ingo Molnar 已提交
7913 7914 7915
 * The masks don't intersect (don't overlap.) We should setup one
 * sched domain for each mask. CPUs not in any of the cpumasks will
 * not be load balanced. If the same cpumask appears both in the
P
Paul Jackson 已提交
7916 7917 7918
 * current 'doms_cur' domains and in the new 'doms_new', we can leave
 * it as it is.
 *
I
Ingo Molnar 已提交
7919 7920
 * The passed in 'doms_new' should be kmalloc'd. This routine takes
 * ownership of it and will kfree it when done with it. If the caller
7921 7922 7923 7924
 * failed the kmalloc call, then it can pass in doms_new == NULL &&
 * ndoms_new == 1, and partition_sched_domains() will fallback to
 * the single partition 'fallback_doms', it also forces the domains
 * to be rebuilt.
P
Paul Jackson 已提交
7925
 *
7926
 * If doms_new == NULL it will be replaced with cpu_online_mask.
7927 7928
 * ndoms_new == 0 is a special case for destroying existing domains,
 * and it will not create the default domain.
7929
 *
P
Paul Jackson 已提交
7930 7931
 * Call with hotplug lock held
 */
7932 7933
/* FIXME: Change to struct cpumask *doms_new[] */
void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7934
			     struct sched_domain_attr *dattr_new)
P
Paul Jackson 已提交
7935
{
7936
	int i, j, n;
7937
	int new_topology;
P
Paul Jackson 已提交
7938

7939
	mutex_lock(&sched_domains_mutex);
7940

7941 7942 7943
	/* always unregister in case we don't destroy any domains */
	unregister_sched_domain_sysctl();

7944 7945 7946
	/* Let architecture update cpu core mappings. */
	new_topology = arch_update_cpu_topology();

7947
	n = doms_new ? ndoms_new : 0;
P
Paul Jackson 已提交
7948 7949 7950

	/* Destroy deleted domains */
	for (i = 0; i < ndoms_cur; i++) {
7951
		for (j = 0; j < n && !new_topology; j++) {
7952
			if (cpumask_equal(&doms_cur[i], &doms_new[j])
7953
			    && dattrs_equal(dattr_cur, i, dattr_new, j))
P
Paul Jackson 已提交
7954 7955 7956 7957 7958 7959 7960 7961
				goto match1;
		}
		/* no match - a current sched domain not in new doms_new[] */
		detach_destroy_domains(doms_cur + i);
match1:
		;
	}

7962 7963
	if (doms_new == NULL) {
		ndoms_cur = 0;
7964
		doms_new = fallback_doms;
7965
		cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7966
		WARN_ON_ONCE(dattr_new);
7967 7968
	}

P
Paul Jackson 已提交
7969 7970
	/* Build new domains */
	for (i = 0; i < ndoms_new; i++) {
7971
		for (j = 0; j < ndoms_cur && !new_topology; j++) {
7972
			if (cpumask_equal(&doms_new[i], &doms_cur[j])
7973
			    && dattrs_equal(dattr_new, i, dattr_cur, j))
P
Paul Jackson 已提交
7974 7975 7976
				goto match2;
		}
		/* no match - add a new doms_new */
7977 7978
		__build_sched_domains(doms_new + i,
					dattr_new ? dattr_new + i : NULL);
P
Paul Jackson 已提交
7979 7980 7981 7982 7983
match2:
		;
	}

	/* Remember the new sched domains */
7984
	if (doms_cur != fallback_doms)
P
Paul Jackson 已提交
7985
		kfree(doms_cur);
7986
	kfree(dattr_cur);	/* kfree(NULL) is safe */
P
Paul Jackson 已提交
7987
	doms_cur = doms_new;
7988
	dattr_cur = dattr_new;
P
Paul Jackson 已提交
7989
	ndoms_cur = ndoms_new;
7990 7991

	register_sched_domain_sysctl();
7992

7993
	mutex_unlock(&sched_domains_mutex);
P
Paul Jackson 已提交
7994 7995
}

7996
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
7997
static void arch_reinit_sched_domains(void)
7998
{
7999
	get_online_cpus();
8000 8001 8002 8003

	/* Destroy domains first to force the rebuild */
	partition_sched_domains(0, NULL, NULL);

8004
	rebuild_sched_domains();
8005
	put_online_cpus();
8006 8007 8008 8009
}

static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
{
8010
	unsigned int level = 0;
8011

8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022
	if (sscanf(buf, "%u", &level) != 1)
		return -EINVAL;

	/*
	 * level is always be positive so don't check for
	 * level < POWERSAVINGS_BALANCE_NONE which is 0
	 * What happens on 0 or 1 byte write,
	 * need to check for count as well?
	 */

	if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
8023 8024 8025
		return -EINVAL;

	if (smt)
8026
		sched_smt_power_savings = level;
8027
	else
8028
		sched_mc_power_savings = level;
8029

8030
	arch_reinit_sched_domains();
8031

8032
	return count;
8033 8034 8035
}

#ifdef CONFIG_SCHED_MC
8036 8037
static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
					   char *page)
8038 8039 8040
{
	return sprintf(page, "%u\n", sched_mc_power_savings);
}
8041
static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
8042
					    const char *buf, size_t count)
8043 8044 8045
{
	return sched_power_savings_store(buf, count, 0);
}
8046 8047 8048
static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
			 sched_mc_power_savings_show,
			 sched_mc_power_savings_store);
8049 8050 8051
#endif

#ifdef CONFIG_SCHED_SMT
8052 8053
static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
					    char *page)
8054 8055 8056
{
	return sprintf(page, "%u\n", sched_smt_power_savings);
}
8057
static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
8058
					     const char *buf, size_t count)
8059 8060 8061
{
	return sched_power_savings_store(buf, count, 1);
}
8062 8063
static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
		   sched_smt_power_savings_show,
A
Adrian Bunk 已提交
8064 8065 8066
		   sched_smt_power_savings_store);
#endif

8067
int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
A
Adrian Bunk 已提交
8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082
{
	int err = 0;

#ifdef CONFIG_SCHED_SMT
	if (smt_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_smt_power_savings.attr);
#endif
#ifdef CONFIG_SCHED_MC
	if (!err && mc_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_mc_power_savings.attr);
#endif
	return err;
}
8083
#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
8084

8085
#ifndef CONFIG_CPUSETS
L
Linus Torvalds 已提交
8086
/*
8087 8088
 * Add online and remove offline CPUs from the scheduler domains.
 * When cpusets are enabled they take over this function.
L
Linus Torvalds 已提交
8089 8090 8091
 */
static int update_sched_domains(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
8092 8093 8094 8095 8096 8097
{
	switch (action) {
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
8098
		partition_sched_domains(1, NULL, NULL);
8099 8100 8101 8102 8103 8104 8105 8106 8107 8108
		return NOTIFY_OK;

	default:
		return NOTIFY_DONE;
	}
}
#endif

static int update_runtime(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
8109
{
P
Peter Zijlstra 已提交
8110 8111
	int cpu = (int)(long)hcpu;

L
Linus Torvalds 已提交
8112 8113
	switch (action) {
	case CPU_DOWN_PREPARE:
8114
	case CPU_DOWN_PREPARE_FROZEN:
P
Peter Zijlstra 已提交
8115
		disable_runtime(cpu_rq(cpu));
L
Linus Torvalds 已提交
8116 8117 8118
		return NOTIFY_OK;

	case CPU_DOWN_FAILED:
8119
	case CPU_DOWN_FAILED_FROZEN:
L
Linus Torvalds 已提交
8120
	case CPU_ONLINE:
8121
	case CPU_ONLINE_FROZEN:
P
Peter Zijlstra 已提交
8122
		enable_runtime(cpu_rq(cpu));
8123 8124
		return NOTIFY_OK;

L
Linus Torvalds 已提交
8125 8126 8127 8128 8129 8130 8131
	default:
		return NOTIFY_DONE;
	}
}

void __init sched_init_smp(void)
{
8132 8133 8134
	cpumask_var_t non_isolated_cpus;

	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
8135

8136 8137 8138 8139 8140
#if defined(CONFIG_NUMA)
	sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
								GFP_KERNEL);
	BUG_ON(sched_group_nodes_bycpu == NULL);
#endif
8141
	get_online_cpus();
8142
	mutex_lock(&sched_domains_mutex);
8143 8144 8145 8146
	arch_init_sched_domains(cpu_online_mask);
	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
	if (cpumask_empty(non_isolated_cpus))
		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8147
	mutex_unlock(&sched_domains_mutex);
8148
	put_online_cpus();
8149 8150

#ifndef CONFIG_CPUSETS
L
Linus Torvalds 已提交
8151 8152
	/* XXX: Theoretical race here - CPU may be hotplugged now */
	hotcpu_notifier(update_sched_domains, 0);
8153 8154 8155 8156 8157
#endif

	/* RT runtime code needs to handle some hotplug events */
	hotcpu_notifier(update_runtime, 0);

8158
	init_hrtick();
8159 8160

	/* Move init over to a non-isolated CPU */
8161
	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8162
		BUG();
I
Ingo Molnar 已提交
8163
	sched_init_granularity();
8164
	free_cpumask_var(non_isolated_cpus);
8165 8166

	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8167
	init_sched_rt_class();
L
Linus Torvalds 已提交
8168 8169 8170 8171
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
8172
	sched_init_granularity();
L
Linus Torvalds 已提交
8173 8174 8175 8176 8177 8178 8179 8180 8181 8182
}
#endif /* CONFIG_SMP */

int in_sched_functions(unsigned long addr)
{
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

A
Alexey Dobriyan 已提交
8183
static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
I
Ingo Molnar 已提交
8184 8185
{
	cfs_rq->tasks_timeline = RB_ROOT;
8186
	INIT_LIST_HEAD(&cfs_rq->tasks);
I
Ingo Molnar 已提交
8187 8188 8189
#ifdef CONFIG_FAIR_GROUP_SCHED
	cfs_rq->rq = rq;
#endif
P
Peter Zijlstra 已提交
8190
	cfs_rq->min_vruntime = (u64)(-(1LL << 20));
I
Ingo Molnar 已提交
8191 8192
}

P
Peter Zijlstra 已提交
8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205
static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
{
	struct rt_prio_array *array;
	int i;

	array = &rt_rq->active;
	for (i = 0; i < MAX_RT_PRIO; i++) {
		INIT_LIST_HEAD(array->queue + i);
		__clear_bit(i, array->bitmap);
	}
	/* delimiter for bitsearch: */
	__set_bit(MAX_RT_PRIO, array->bitmap);

8206
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8207 8208
	rt_rq->highest_prio = MAX_RT_PRIO;
#endif
P
Peter Zijlstra 已提交
8209 8210 8211 8212 8213 8214 8215
#ifdef CONFIG_SMP
	rt_rq->rt_nr_migratory = 0;
	rt_rq->overloaded = 0;
#endif

	rt_rq->rt_time = 0;
	rt_rq->rt_throttled = 0;
P
Peter Zijlstra 已提交
8216 8217
	rt_rq->rt_runtime = 0;
	spin_lock_init(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
8218

8219
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8220
	rt_rq->rt_nr_boosted = 0;
P
Peter Zijlstra 已提交
8221 8222
	rt_rq->rq = rq;
#endif
P
Peter Zijlstra 已提交
8223 8224
}

P
Peter Zijlstra 已提交
8225
#ifdef CONFIG_FAIR_GROUP_SCHED
8226 8227 8228
static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
				struct sched_entity *se, int cpu, int add,
				struct sched_entity *parent)
P
Peter Zijlstra 已提交
8229
{
8230
	struct rq *rq = cpu_rq(cpu);
P
Peter Zijlstra 已提交
8231 8232 8233 8234 8235 8236 8237
	tg->cfs_rq[cpu] = cfs_rq;
	init_cfs_rq(cfs_rq, rq);
	cfs_rq->tg = tg;
	if (add)
		list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);

	tg->se[cpu] = se;
D
Dhaval Giani 已提交
8238 8239 8240 8241
	/* se could be NULL for init_task_group */
	if (!se)
		return;

8242 8243 8244 8245 8246
	if (!parent)
		se->cfs_rq = &rq->cfs;
	else
		se->cfs_rq = parent->my_q;

P
Peter Zijlstra 已提交
8247 8248
	se->my_q = cfs_rq;
	se->load.weight = tg->shares;
8249
	se->load.inv_weight = 0;
8250
	se->parent = parent;
P
Peter Zijlstra 已提交
8251
}
8252
#endif
P
Peter Zijlstra 已提交
8253

8254
#ifdef CONFIG_RT_GROUP_SCHED
8255 8256 8257
static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
		struct sched_rt_entity *rt_se, int cpu, int add,
		struct sched_rt_entity *parent)
P
Peter Zijlstra 已提交
8258
{
8259 8260
	struct rq *rq = cpu_rq(cpu);

P
Peter Zijlstra 已提交
8261 8262 8263 8264
	tg->rt_rq[cpu] = rt_rq;
	init_rt_rq(rt_rq, rq);
	rt_rq->tg = tg;
	rt_rq->rt_se = rt_se;
P
Peter Zijlstra 已提交
8265
	rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
8266 8267 8268 8269
	if (add)
		list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);

	tg->rt_se[cpu] = rt_se;
D
Dhaval Giani 已提交
8270 8271 8272
	if (!rt_se)
		return;

8273 8274 8275 8276 8277
	if (!parent)
		rt_se->rt_rq = &rq->rt;
	else
		rt_se->rt_rq = parent->my_q;

P
Peter Zijlstra 已提交
8278
	rt_se->my_q = rt_rq;
8279
	rt_se->parent = parent;
P
Peter Zijlstra 已提交
8280 8281 8282 8283
	INIT_LIST_HEAD(&rt_se->run_list);
}
#endif

L
Linus Torvalds 已提交
8284 8285
void __init sched_init(void)
{
I
Ingo Molnar 已提交
8286
	int i, j;
8287 8288 8289 8290 8291 8292 8293
	unsigned long alloc_size = 0, ptr;

#ifdef CONFIG_FAIR_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
8294 8295 8296
#endif
#ifdef CONFIG_USER_SCHED
	alloc_size *= 2;
8297 8298 8299 8300 8301 8302
#endif
	/*
	 * As sched_init() is called before page_alloc is setup,
	 * we use alloc_bootmem().
	 */
	if (alloc_size) {
8303
		ptr = (unsigned long)alloc_bootmem(alloc_size);
8304 8305 8306 8307 8308 8309 8310

#ifdef CONFIG_FAIR_GROUP_SCHED
		init_task_group.se = (struct sched_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		init_task_group.cfs_rq = (struct cfs_rq **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);
8311 8312 8313 8314 8315 8316 8317

#ifdef CONFIG_USER_SCHED
		root_task_group.se = (struct sched_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);
8318 8319
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_FAIR_GROUP_SCHED */
8320 8321 8322 8323 8324
#ifdef CONFIG_RT_GROUP_SCHED
		init_task_group.rt_se = (struct sched_rt_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		init_task_group.rt_rq = (struct rt_rq **)ptr;
8325 8326 8327 8328 8329 8330 8331 8332
		ptr += nr_cpu_ids * sizeof(void **);

#ifdef CONFIG_USER_SCHED
		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		root_task_group.rt_rq = (struct rt_rq **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);
8333 8334
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_RT_GROUP_SCHED */
8335
	}
I
Ingo Molnar 已提交
8336

G
Gregory Haskins 已提交
8337 8338 8339 8340
#ifdef CONFIG_SMP
	init_defrootdomain();
#endif

8341 8342 8343 8344 8345 8346
	init_rt_bandwidth(&def_rt_bandwidth,
			global_rt_period(), global_rt_runtime());

#ifdef CONFIG_RT_GROUP_SCHED
	init_rt_bandwidth(&init_task_group.rt_bandwidth,
			global_rt_period(), global_rt_runtime());
8347 8348 8349
#ifdef CONFIG_USER_SCHED
	init_rt_bandwidth(&root_task_group.rt_bandwidth,
			global_rt_period(), RUNTIME_INF);
8350 8351
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_RT_GROUP_SCHED */
8352

8353
#ifdef CONFIG_GROUP_SCHED
P
Peter Zijlstra 已提交
8354
	list_add(&init_task_group.list, &task_groups);
P
Peter Zijlstra 已提交
8355 8356 8357 8358 8359 8360
	INIT_LIST_HEAD(&init_task_group.children);

#ifdef CONFIG_USER_SCHED
	INIT_LIST_HEAD(&root_task_group.children);
	init_task_group.parent = &root_task_group;
	list_add(&init_task_group.siblings, &root_task_group.children);
8361 8362
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_GROUP_SCHED */
P
Peter Zijlstra 已提交
8363

8364
	for_each_possible_cpu(i) {
8365
		struct rq *rq;
L
Linus Torvalds 已提交
8366 8367 8368

		rq = cpu_rq(i);
		spin_lock_init(&rq->lock);
N
Nick Piggin 已提交
8369
		rq->nr_running = 0;
I
Ingo Molnar 已提交
8370
		init_cfs_rq(&rq->cfs, rq);
P
Peter Zijlstra 已提交
8371
		init_rt_rq(&rq->rt, rq);
I
Ingo Molnar 已提交
8372
#ifdef CONFIG_FAIR_GROUP_SCHED
8373
		init_task_group.shares = init_task_group_load;
P
Peter Zijlstra 已提交
8374
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
D
Dhaval Giani 已提交
8375 8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394
#ifdef CONFIG_CGROUP_SCHED
		/*
		 * How much cpu bandwidth does init_task_group get?
		 *
		 * In case of task-groups formed thr' the cgroup filesystem, it
		 * gets 100% of the cpu resources in the system. This overall
		 * system cpu resource is divided among the tasks of
		 * init_task_group and its child task-groups in a fair manner,
		 * based on each entity's (task or task-group's) weight
		 * (se->load.weight).
		 *
		 * In other words, if init_task_group has 10 tasks of weight
		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
		 * then A0's share of the cpu resource is:
		 *
		 * 	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
		 *
		 * We achieve this by letting init_task_group's tasks sit
		 * directly in rq->cfs (i.e init_task_group->se[] = NULL).
		 */
8395
		init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
D
Dhaval Giani 已提交
8396
#elif defined CONFIG_USER_SCHED
8397 8398
		root_task_group.shares = NICE_0_LOAD;
		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, 0, NULL);
D
Dhaval Giani 已提交
8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409
		/*
		 * In case of task-groups formed thr' the user id of tasks,
		 * init_task_group represents tasks belonging to root user.
		 * Hence it forms a sibling of all subsequent groups formed.
		 * In this case, init_task_group gets only a fraction of overall
		 * system cpu resource, based on the weight assigned to root
		 * user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
		 * by letting tasks of init_task_group sit in a separate cfs_rq
		 * (init_cfs_rq) and having one entity represent this group of
		 * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
		 */
8410
		init_tg_cfs_entry(&init_task_group,
P
Peter Zijlstra 已提交
8411
				&per_cpu(init_cfs_rq, i),
8412 8413
				&per_cpu(init_sched_entity, i), i, 1,
				root_task_group.se[i]);
P
Peter Zijlstra 已提交
8414

8415
#endif
D
Dhaval Giani 已提交
8416 8417 8418
#endif /* CONFIG_FAIR_GROUP_SCHED */

		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
8419
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8420
		INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
D
Dhaval Giani 已提交
8421
#ifdef CONFIG_CGROUP_SCHED
8422
		init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
D
Dhaval Giani 已提交
8423
#elif defined CONFIG_USER_SCHED
8424
		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
8425
		init_tg_rt_entry(&init_task_group,
P
Peter Zijlstra 已提交
8426
				&per_cpu(init_rt_rq, i),
8427 8428
				&per_cpu(init_sched_rt_entity, i), i, 1,
				root_task_group.rt_se[i]);
D
Dhaval Giani 已提交
8429
#endif
I
Ingo Molnar 已提交
8430
#endif
L
Linus Torvalds 已提交
8431

I
Ingo Molnar 已提交
8432 8433
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
L
Linus Torvalds 已提交
8434
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
8435
		rq->sd = NULL;
G
Gregory Haskins 已提交
8436
		rq->rd = NULL;
L
Linus Torvalds 已提交
8437
		rq->active_balance = 0;
I
Ingo Molnar 已提交
8438
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
8439
		rq->push_cpu = 0;
8440
		rq->cpu = i;
8441
		rq->online = 0;
L
Linus Torvalds 已提交
8442 8443
		rq->migration_thread = NULL;
		INIT_LIST_HEAD(&rq->migration_queue);
8444
		rq_attach_root(rq, &def_root_domain);
L
Linus Torvalds 已提交
8445
#endif
P
Peter Zijlstra 已提交
8446
		init_rq_hrtick(rq);
L
Linus Torvalds 已提交
8447 8448 8449
		atomic_set(&rq->nr_iowait, 0);
	}

8450
	set_load_weight(&init_task);
8451

8452 8453 8454 8455
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

8456
#ifdef CONFIG_SMP
8457
	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
8458 8459
#endif

8460 8461 8462 8463
#ifdef CONFIG_RT_MUTEXES
	plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
#endif

L
Linus Torvalds 已提交
8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
I
Ingo Molnar 已提交
8477 8478 8479 8480
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
8481

8482 8483
	/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
	alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8484
#ifdef CONFIG_SMP
8485 8486 8487
#ifdef CONFIG_NO_HZ
	alloc_bootmem_cpumask_var(&nohz.cpu_mask);
#endif
8488
	alloc_bootmem_cpumask_var(&cpu_isolated_map);
8489
#endif /* SMP */
8490

8491
	scheduler_running = 1;
L
Linus Torvalds 已提交
8492 8493 8494 8495 8496
}

#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
void __might_sleep(char *file, int line)
{
8497
#ifdef in_atomic
L
Linus Torvalds 已提交
8498 8499
	static unsigned long prev_jiffy;	/* ratelimiting */

I
Ingo Molnar 已提交
8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518
	if ((!in_atomic() && !irqs_disabled()) ||
		    system_state != SYSTEM_RUNNING || oops_in_progress)
		return;
	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
		return;
	prev_jiffy = jiffies;

	printk(KERN_ERR
		"BUG: sleeping function called from invalid context at %s:%d\n",
			file, line);
	printk(KERN_ERR
		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
			in_atomic(), irqs_disabled(),
			current->pid, current->comm);

	debug_show_held_locks(current);
	if (irqs_disabled())
		print_irqtrace_events(current);
	dump_stack();
L
Linus Torvalds 已提交
8519 8520 8521 8522 8523 8524
#endif
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
8525 8526 8527
static void normalize_task(struct rq *rq, struct task_struct *p)
{
	int on_rq;
8528

8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539
	update_rq_clock(rq);
	on_rq = p->se.on_rq;
	if (on_rq)
		deactivate_task(rq, p, 0);
	__setscheduler(rq, p, SCHED_NORMAL, 0);
	if (on_rq) {
		activate_task(rq, p, 0);
		resched_task(rq->curr);
	}
}

L
Linus Torvalds 已提交
8540 8541
void normalize_rt_tasks(void)
{
8542
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
8543
	unsigned long flags;
8544
	struct rq *rq;
L
Linus Torvalds 已提交
8545

8546
	read_lock_irqsave(&tasklist_lock, flags);
8547
	do_each_thread(g, p) {
8548 8549 8550 8551 8552 8553
		/*
		 * Only normalize user tasks:
		 */
		if (!p->mm)
			continue;

I
Ingo Molnar 已提交
8554 8555
		p->se.exec_start		= 0;
#ifdef CONFIG_SCHEDSTATS
I
Ingo Molnar 已提交
8556 8557 8558
		p->se.wait_start		= 0;
		p->se.sleep_start		= 0;
		p->se.block_start		= 0;
I
Ingo Molnar 已提交
8559
#endif
I
Ingo Molnar 已提交
8560 8561 8562 8563 8564 8565 8566 8567

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
8568
			continue;
I
Ingo Molnar 已提交
8569
		}
L
Linus Torvalds 已提交
8570

8571
		spin_lock(&p->pi_lock);
8572
		rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
8573

8574
		normalize_task(rq, p);
8575

8576
		__task_rq_unlock(rq);
8577
		spin_unlock(&p->pi_lock);
8578 8579
	} while_each_thread(g, p);

8580
	read_unlock_irqrestore(&tasklist_lock, flags);
L
Linus Torvalds 已提交
8581 8582 8583
}

#endif /* CONFIG_MAGIC_SYSRQ */
8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601

#ifdef CONFIG_IA64
/*
 * These functions are only useful for the IA64 MCA handling.
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
8602
struct task_struct *curr_task(int cpu)
8603 8604 8605 8606 8607 8608 8609 8610 8611 8612
{
	return cpu_curr(cpu);
}

/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
I
Ingo Molnar 已提交
8613 8614
 * are serviced on a separate stack. It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner. This function
8615 8616 8617 8618 8619 8620 8621
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
8622
void set_curr_task(int cpu, struct task_struct *p)
8623 8624 8625 8626 8627
{
	cpu_curr(cpu) = p;
}

#endif
S
Srivatsa Vaddagiri 已提交
8628

8629 8630
#ifdef CONFIG_FAIR_GROUP_SCHED
static void free_fair_sched_group(struct task_group *tg)
P
Peter Zijlstra 已提交
8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644
{
	int i;

	for_each_possible_cpu(i) {
		if (tg->cfs_rq)
			kfree(tg->cfs_rq[i]);
		if (tg->se)
			kfree(tg->se[i]);
	}

	kfree(tg->cfs_rq);
	kfree(tg->se);
}

8645 8646
static
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
S
Srivatsa Vaddagiri 已提交
8647 8648
{
	struct cfs_rq *cfs_rq;
8649
	struct sched_entity *se;
8650
	struct rq *rq;
S
Srivatsa Vaddagiri 已提交
8651 8652
	int i;

8653
	tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
8654 8655
	if (!tg->cfs_rq)
		goto err;
8656
	tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
8657 8658
	if (!tg->se)
		goto err;
8659 8660

	tg->shares = NICE_0_LOAD;
S
Srivatsa Vaddagiri 已提交
8661 8662

	for_each_possible_cpu(i) {
8663
		rq = cpu_rq(i);
S
Srivatsa Vaddagiri 已提交
8664

8665 8666
		cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
				      GFP_KERNEL, cpu_to_node(i));
S
Srivatsa Vaddagiri 已提交
8667 8668 8669
		if (!cfs_rq)
			goto err;

8670 8671
		se = kzalloc_node(sizeof(struct sched_entity),
				  GFP_KERNEL, cpu_to_node(i));
S
Srivatsa Vaddagiri 已提交
8672 8673 8674
		if (!se)
			goto err;

8675
		init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693
	}

	return 1;

 err:
	return 0;
}

static inline void register_fair_sched_group(struct task_group *tg, int cpu)
{
	list_add_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list,
			&cpu_rq(cpu)->leaf_cfs_rq_list);
}

static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
{
	list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
}
8694
#else /* !CONFG_FAIR_GROUP_SCHED */
8695 8696 8697 8698
static inline void free_fair_sched_group(struct task_group *tg)
{
}

8699 8700
static inline
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711
{
	return 1;
}

static inline void register_fair_sched_group(struct task_group *tg, int cpu)
{
}

static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
{
}
8712
#endif /* CONFIG_FAIR_GROUP_SCHED */
8713 8714

#ifdef CONFIG_RT_GROUP_SCHED
8715 8716 8717 8718
static void free_rt_sched_group(struct task_group *tg)
{
	int i;

8719 8720
	destroy_rt_bandwidth(&tg->rt_bandwidth);

8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731
	for_each_possible_cpu(i) {
		if (tg->rt_rq)
			kfree(tg->rt_rq[i]);
		if (tg->rt_se)
			kfree(tg->rt_se[i]);
	}

	kfree(tg->rt_rq);
	kfree(tg->rt_se);
}

8732 8733
static
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8734 8735
{
	struct rt_rq *rt_rq;
8736
	struct sched_rt_entity *rt_se;
8737 8738 8739
	struct rq *rq;
	int i;

8740
	tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
8741 8742
	if (!tg->rt_rq)
		goto err;
8743
	tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
8744 8745 8746
	if (!tg->rt_se)
		goto err;

8747 8748
	init_rt_bandwidth(&tg->rt_bandwidth,
			ktime_to_ns(def_rt_bandwidth.rt_period), 0);
8749 8750 8751 8752

	for_each_possible_cpu(i) {
		rq = cpu_rq(i);

8753 8754
		rt_rq = kzalloc_node(sizeof(struct rt_rq),
				     GFP_KERNEL, cpu_to_node(i));
P
Peter Zijlstra 已提交
8755 8756
		if (!rt_rq)
			goto err;
S
Srivatsa Vaddagiri 已提交
8757

8758 8759
		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
				     GFP_KERNEL, cpu_to_node(i));
P
Peter Zijlstra 已提交
8760 8761
		if (!rt_se)
			goto err;
S
Srivatsa Vaddagiri 已提交
8762

8763
		init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
S
Srivatsa Vaddagiri 已提交
8764 8765
	}

8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781
	return 1;

 err:
	return 0;
}

static inline void register_rt_sched_group(struct task_group *tg, int cpu)
{
	list_add_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list,
			&cpu_rq(cpu)->leaf_rt_rq_list);
}

static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
{
	list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
}
8782
#else /* !CONFIG_RT_GROUP_SCHED */
8783 8784 8785 8786
static inline void free_rt_sched_group(struct task_group *tg)
{
}

8787 8788
static inline
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799
{
	return 1;
}

static inline void register_rt_sched_group(struct task_group *tg, int cpu)
{
}

static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
{
}
8800
#endif /* CONFIG_RT_GROUP_SCHED */
8801

8802
#ifdef CONFIG_GROUP_SCHED
8803 8804 8805 8806 8807 8808 8809 8810
static void free_sched_group(struct task_group *tg)
{
	free_fair_sched_group(tg);
	free_rt_sched_group(tg);
	kfree(tg);
}

/* allocate runqueue etc for a new task group */
8811
struct task_group *sched_create_group(struct task_group *parent)
8812 8813 8814 8815 8816 8817 8818 8819 8820
{
	struct task_group *tg;
	unsigned long flags;
	int i;

	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
	if (!tg)
		return ERR_PTR(-ENOMEM);

8821
	if (!alloc_fair_sched_group(tg, parent))
8822 8823
		goto err;

8824
	if (!alloc_rt_sched_group(tg, parent))
8825 8826
		goto err;

8827
	spin_lock_irqsave(&task_group_lock, flags);
8828
	for_each_possible_cpu(i) {
8829 8830
		register_fair_sched_group(tg, i);
		register_rt_sched_group(tg, i);
8831
	}
P
Peter Zijlstra 已提交
8832
	list_add_rcu(&tg->list, &task_groups);
P
Peter Zijlstra 已提交
8833 8834 8835 8836 8837

	WARN_ON(!parent); /* root should already exist */

	tg->parent = parent;
	INIT_LIST_HEAD(&tg->children);
8838
	list_add_rcu(&tg->siblings, &parent->children);
8839
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
8840

8841
	return tg;
S
Srivatsa Vaddagiri 已提交
8842 8843

err:
P
Peter Zijlstra 已提交
8844
	free_sched_group(tg);
S
Srivatsa Vaddagiri 已提交
8845 8846 8847
	return ERR_PTR(-ENOMEM);
}

8848
/* rcu callback to free various structures associated with a task group */
P
Peter Zijlstra 已提交
8849
static void free_sched_group_rcu(struct rcu_head *rhp)
S
Srivatsa Vaddagiri 已提交
8850 8851
{
	/* now it should be safe to free those cfs_rqs */
P
Peter Zijlstra 已提交
8852
	free_sched_group(container_of(rhp, struct task_group, rcu));
S
Srivatsa Vaddagiri 已提交
8853 8854
}

8855
/* Destroy runqueue etc associated with a task group */
8856
void sched_destroy_group(struct task_group *tg)
S
Srivatsa Vaddagiri 已提交
8857
{
8858
	unsigned long flags;
8859
	int i;
S
Srivatsa Vaddagiri 已提交
8860

8861
	spin_lock_irqsave(&task_group_lock, flags);
8862
	for_each_possible_cpu(i) {
8863 8864
		unregister_fair_sched_group(tg, i);
		unregister_rt_sched_group(tg, i);
8865
	}
P
Peter Zijlstra 已提交
8866
	list_del_rcu(&tg->list);
P
Peter Zijlstra 已提交
8867
	list_del_rcu(&tg->siblings);
8868
	spin_unlock_irqrestore(&task_group_lock, flags);
8869 8870

	/* wait for possible concurrent references to cfs_rqs complete */
P
Peter Zijlstra 已提交
8871
	call_rcu(&tg->rcu, free_sched_group_rcu);
S
Srivatsa Vaddagiri 已提交
8872 8873
}

8874
/* change task's runqueue when it moves between groups.
I
Ingo Molnar 已提交
8875 8876 8877
 *	The caller of this function should have put the task in its new group
 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
 *	reflect its new group.
8878 8879
 */
void sched_move_task(struct task_struct *tsk)
S
Srivatsa Vaddagiri 已提交
8880 8881 8882 8883 8884 8885 8886 8887 8888
{
	int on_rq, running;
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

	update_rq_clock(rq);

8889
	running = task_current(rq, tsk);
S
Srivatsa Vaddagiri 已提交
8890 8891
	on_rq = tsk->se.on_rq;

8892
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
8893
		dequeue_task(rq, tsk, 0);
8894 8895
	if (unlikely(running))
		tsk->sched_class->put_prev_task(rq, tsk);
S
Srivatsa Vaddagiri 已提交
8896

P
Peter Zijlstra 已提交
8897
	set_task_rq(tsk, task_cpu(tsk));
S
Srivatsa Vaddagiri 已提交
8898

P
Peter Zijlstra 已提交
8899 8900 8901 8902 8903
#ifdef CONFIG_FAIR_GROUP_SCHED
	if (tsk->sched_class->moved_group)
		tsk->sched_class->moved_group(tsk);
#endif

8904 8905 8906
	if (unlikely(running))
		tsk->sched_class->set_curr_task(rq);
	if (on_rq)
8907
		enqueue_task(rq, tsk, 0);
S
Srivatsa Vaddagiri 已提交
8908 8909 8910

	task_rq_unlock(rq, &flags);
}
8911
#endif /* CONFIG_GROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
8912

8913
#ifdef CONFIG_FAIR_GROUP_SCHED
8914
static void __set_se_shares(struct sched_entity *se, unsigned long shares)
S
Srivatsa Vaddagiri 已提交
8915 8916 8917 8918 8919
{
	struct cfs_rq *cfs_rq = se->cfs_rq;
	int on_rq;

	on_rq = se->on_rq;
8920
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
8921 8922 8923
		dequeue_entity(cfs_rq, se, 0);

	se->load.weight = shares;
8924
	se->load.inv_weight = 0;
S
Srivatsa Vaddagiri 已提交
8925

8926
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
8927
		enqueue_entity(cfs_rq, se, 0);
8928
}
8929

8930 8931 8932 8933 8934 8935 8936 8937 8938
static void set_se_shares(struct sched_entity *se, unsigned long shares)
{
	struct cfs_rq *cfs_rq = se->cfs_rq;
	struct rq *rq = cfs_rq->rq;
	unsigned long flags;

	spin_lock_irqsave(&rq->lock, flags);
	__set_se_shares(se, shares);
	spin_unlock_irqrestore(&rq->lock, flags);
S
Srivatsa Vaddagiri 已提交
8939 8940
}

8941 8942
static DEFINE_MUTEX(shares_mutex);

8943
int sched_group_set_shares(struct task_group *tg, unsigned long shares)
S
Srivatsa Vaddagiri 已提交
8944 8945
{
	int i;
8946
	unsigned long flags;
8947

8948 8949 8950 8951 8952 8953
	/*
	 * We can't change the weight of the root cgroup.
	 */
	if (!tg->se[0])
		return -EINVAL;

8954 8955
	if (shares < MIN_SHARES)
		shares = MIN_SHARES;
8956 8957
	else if (shares > MAX_SHARES)
		shares = MAX_SHARES;
8958

8959
	mutex_lock(&shares_mutex);
8960
	if (tg->shares == shares)
8961
		goto done;
S
Srivatsa Vaddagiri 已提交
8962

8963
	spin_lock_irqsave(&task_group_lock, flags);
8964 8965
	for_each_possible_cpu(i)
		unregister_fair_sched_group(tg, i);
P
Peter Zijlstra 已提交
8966
	list_del_rcu(&tg->siblings);
8967
	spin_unlock_irqrestore(&task_group_lock, flags);
8968 8969 8970 8971 8972 8973 8974 8975

	/* wait for any ongoing reference to this group to finish */
	synchronize_sched();

	/*
	 * Now we are free to modify the group's share on each cpu
	 * w/o tripping rebalance_share or load_balance_fair.
	 */
8976
	tg->shares = shares;
8977 8978 8979 8980 8981
	for_each_possible_cpu(i) {
		/*
		 * force a rebalance
		 */
		cfs_rq_set_shares(tg->cfs_rq[i], 0);
8982
		set_se_shares(tg->se[i], shares);
8983
	}
S
Srivatsa Vaddagiri 已提交
8984

8985 8986 8987 8988
	/*
	 * Enable load balance activity on this group, by inserting it back on
	 * each cpu's rq->leaf_cfs_rq_list.
	 */
8989
	spin_lock_irqsave(&task_group_lock, flags);
8990 8991
	for_each_possible_cpu(i)
		register_fair_sched_group(tg, i);
P
Peter Zijlstra 已提交
8992
	list_add_rcu(&tg->siblings, &tg->parent->children);
8993
	spin_unlock_irqrestore(&task_group_lock, flags);
8994
done:
8995
	mutex_unlock(&shares_mutex);
8996
	return 0;
S
Srivatsa Vaddagiri 已提交
8997 8998
}

8999 9000 9001 9002
unsigned long sched_group_shares(struct task_group *tg)
{
	return tg->shares;
}
9003
#endif
9004

9005
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
9006
/*
P
Peter Zijlstra 已提交
9007
 * Ensure that the real time constraints are schedulable.
P
Peter Zijlstra 已提交
9008
 */
P
Peter Zijlstra 已提交
9009 9010 9011 9012 9013
static DEFINE_MUTEX(rt_constraints_mutex);

static unsigned long to_ratio(u64 period, u64 runtime)
{
	if (runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
9014
		return 1ULL << 20;
P
Peter Zijlstra 已提交
9015

P
Peter Zijlstra 已提交
9016
	return div64_u64(runtime << 20, period);
P
Peter Zijlstra 已提交
9017 9018
}

P
Peter Zijlstra 已提交
9019 9020
/* Must be called with tasklist_lock held */
static inline int tg_has_rt_tasks(struct task_group *tg)
9021
{
P
Peter Zijlstra 已提交
9022
	struct task_struct *g, *p;
9023

P
Peter Zijlstra 已提交
9024 9025 9026 9027
	do_each_thread(g, p) {
		if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
			return 1;
	} while_each_thread(g, p);
9028

P
Peter Zijlstra 已提交
9029 9030
	return 0;
}
9031

P
Peter Zijlstra 已提交
9032 9033 9034 9035 9036
struct rt_schedulable_data {
	struct task_group *tg;
	u64 rt_period;
	u64 rt_runtime;
};
9037

P
Peter Zijlstra 已提交
9038 9039 9040 9041 9042 9043
static int tg_schedulable(struct task_group *tg, void *data)
{
	struct rt_schedulable_data *d = data;
	struct task_group *child;
	unsigned long total, sum = 0;
	u64 period, runtime;
9044

P
Peter Zijlstra 已提交
9045 9046
	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	runtime = tg->rt_bandwidth.rt_runtime;
9047

P
Peter Zijlstra 已提交
9048 9049 9050
	if (tg == d->tg) {
		period = d->rt_period;
		runtime = d->rt_runtime;
9051 9052
	}

9053 9054 9055 9056 9057
	/*
	 * Cannot have more runtime than the period.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
P
Peter Zijlstra 已提交
9058

9059 9060 9061
	/*
	 * Ensure we don't starve existing RT tasks.
	 */
P
Peter Zijlstra 已提交
9062 9063
	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
		return -EBUSY;
P
Peter Zijlstra 已提交
9064

P
Peter Zijlstra 已提交
9065
	total = to_ratio(period, runtime);
P
Peter Zijlstra 已提交
9066

9067 9068 9069 9070 9071
	/*
	 * Nobody can have more than the global setting allows.
	 */
	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
		return -EINVAL;
P
Peter Zijlstra 已提交
9072

9073 9074 9075
	/*
	 * The sum of our children's runtime should not exceed our own.
	 */
P
Peter Zijlstra 已提交
9076 9077 9078
	list_for_each_entry_rcu(child, &tg->children, siblings) {
		period = ktime_to_ns(child->rt_bandwidth.rt_period);
		runtime = child->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
9079

P
Peter Zijlstra 已提交
9080 9081 9082 9083
		if (child == d->tg) {
			period = d->rt_period;
			runtime = d->rt_runtime;
		}
P
Peter Zijlstra 已提交
9084

P
Peter Zijlstra 已提交
9085
		sum += to_ratio(period, runtime);
P
Peter Zijlstra 已提交
9086
	}
P
Peter Zijlstra 已提交
9087

P
Peter Zijlstra 已提交
9088 9089 9090 9091
	if (sum > total)
		return -EINVAL;

	return 0;
P
Peter Zijlstra 已提交
9092 9093
}

P
Peter Zijlstra 已提交
9094
static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
9095
{
P
Peter Zijlstra 已提交
9096 9097 9098 9099 9100 9101 9102
	struct rt_schedulable_data data = {
		.tg = tg,
		.rt_period = period,
		.rt_runtime = runtime,
	};

	return walk_tg_tree(tg_schedulable, tg_nop, &data);
9103 9104
}

9105 9106
static int tg_set_bandwidth(struct task_group *tg,
		u64 rt_period, u64 rt_runtime)
P
Peter Zijlstra 已提交
9107
{
P
Peter Zijlstra 已提交
9108
	int i, err = 0;
P
Peter Zijlstra 已提交
9109 9110

	mutex_lock(&rt_constraints_mutex);
9111
	read_lock(&tasklist_lock);
P
Peter Zijlstra 已提交
9112 9113
	err = __rt_schedulable(tg, rt_period, rt_runtime);
	if (err)
P
Peter Zijlstra 已提交
9114
		goto unlock;
P
Peter Zijlstra 已提交
9115 9116

	spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
9117 9118
	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
	tg->rt_bandwidth.rt_runtime = rt_runtime;
P
Peter Zijlstra 已提交
9119 9120 9121 9122 9123 9124 9125 9126 9127

	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = tg->rt_rq[i];

		spin_lock(&rt_rq->rt_runtime_lock);
		rt_rq->rt_runtime = rt_runtime;
		spin_unlock(&rt_rq->rt_runtime_lock);
	}
	spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
P
Peter Zijlstra 已提交
9128
 unlock:
9129
	read_unlock(&tasklist_lock);
P
Peter Zijlstra 已提交
9130 9131 9132
	mutex_unlock(&rt_constraints_mutex);

	return err;
P
Peter Zijlstra 已提交
9133 9134
}

9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146
int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
{
	u64 rt_runtime, rt_period;

	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
	if (rt_runtime_us < 0)
		rt_runtime = RUNTIME_INF;

	return tg_set_bandwidth(tg, rt_period, rt_runtime);
}

P
Peter Zijlstra 已提交
9147 9148 9149 9150
long sched_group_rt_runtime(struct task_group *tg)
{
	u64 rt_runtime_us;

9151
	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
9152 9153
		return -1;

9154
	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
9155 9156 9157
	do_div(rt_runtime_us, NSEC_PER_USEC);
	return rt_runtime_us;
}
9158 9159 9160 9161 9162 9163 9164 9165

int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
{
	u64 rt_runtime, rt_period;

	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
	rt_runtime = tg->rt_bandwidth.rt_runtime;

9166 9167 9168
	if (rt_period == 0)
		return -EINVAL;

9169 9170 9171 9172 9173 9174 9175 9176 9177 9178 9179 9180 9181 9182
	return tg_set_bandwidth(tg, rt_period, rt_runtime);
}

long sched_group_rt_period(struct task_group *tg)
{
	u64 rt_period_us;

	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
	do_div(rt_period_us, NSEC_PER_USEC);
	return rt_period_us;
}

static int sched_rt_global_constraints(void)
{
9183
	u64 runtime, period;
9184 9185
	int ret = 0;

9186 9187 9188
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

9189 9190 9191 9192 9193 9194 9195 9196
	runtime = global_rt_runtime();
	period = global_rt_period();

	/*
	 * Sanity check on the sysctl variables.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
9197

9198
	mutex_lock(&rt_constraints_mutex);
P
Peter Zijlstra 已提交
9199
	read_lock(&tasklist_lock);
9200
	ret = __rt_schedulable(NULL, 0, 0);
P
Peter Zijlstra 已提交
9201
	read_unlock(&tasklist_lock);
9202 9203 9204 9205
	mutex_unlock(&rt_constraints_mutex);

	return ret;
}
9206
#else /* !CONFIG_RT_GROUP_SCHED */
9207 9208
static int sched_rt_global_constraints(void)
{
P
Peter Zijlstra 已提交
9209 9210 9211
	unsigned long flags;
	int i;

9212 9213 9214
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

P
Peter Zijlstra 已提交
9215 9216 9217 9218 9219 9220 9221 9222 9223 9224
	spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = &cpu_rq(i)->rt;

		spin_lock(&rt_rq->rt_runtime_lock);
		rt_rq->rt_runtime = global_rt_runtime();
		spin_unlock(&rt_rq->rt_runtime_lock);
	}
	spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);

9225 9226
	return 0;
}
9227
#endif /* CONFIG_RT_GROUP_SCHED */
9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256 9257

int sched_rt_handler(struct ctl_table *table, int write,
		struct file *filp, void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;
	int old_period, old_runtime;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	old_period = sysctl_sched_rt_period;
	old_runtime = sysctl_sched_rt_runtime;

	ret = proc_dointvec(table, write, filp, buffer, lenp, ppos);

	if (!ret && write) {
		ret = sched_rt_global_constraints();
		if (ret) {
			sysctl_sched_rt_period = old_period;
			sysctl_sched_rt_runtime = old_runtime;
		} else {
			def_rt_bandwidth.rt_runtime = global_rt_runtime();
			def_rt_bandwidth.rt_period =
				ns_to_ktime(global_rt_period());
		}
	}
	mutex_unlock(&mutex);

	return ret;
}
9258

9259
#ifdef CONFIG_CGROUP_SCHED
9260 9261

/* return corresponding task_group object of a cgroup */
9262
static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
9263
{
9264 9265
	return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
			    struct task_group, css);
9266 9267 9268
}

static struct cgroup_subsys_state *
9269
cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
9270
{
9271
	struct task_group *tg, *parent;
9272

9273
	if (!cgrp->parent) {
9274 9275 9276 9277
		/* This is early initialization for the top cgroup */
		return &init_task_group.css;
	}

9278 9279
	parent = cgroup_tg(cgrp->parent);
	tg = sched_create_group(parent);
9280 9281 9282 9283 9284 9285
	if (IS_ERR(tg))
		return ERR_PTR(-ENOMEM);

	return &tg->css;
}

I
Ingo Molnar 已提交
9286 9287
static void
cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
9288
{
9289
	struct task_group *tg = cgroup_tg(cgrp);
9290 9291 9292 9293

	sched_destroy_group(tg);
}

I
Ingo Molnar 已提交
9294 9295 9296
static int
cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
		      struct task_struct *tsk)
9297
{
9298 9299
#ifdef CONFIG_RT_GROUP_SCHED
	/* Don't accept realtime tasks when there is no way for them to run */
9300
	if (rt_task(tsk) && cgroup_tg(cgrp)->rt_bandwidth.rt_runtime == 0)
9301 9302
		return -EINVAL;
#else
9303 9304 9305
	/* We don't support RT-tasks being in separate groups */
	if (tsk->sched_class != &fair_sched_class)
		return -EINVAL;
9306
#endif
9307 9308 9309 9310 9311

	return 0;
}

static void
9312
cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
9313 9314 9315 9316 9317
			struct cgroup *old_cont, struct task_struct *tsk)
{
	sched_move_task(tsk);
}

9318
#ifdef CONFIG_FAIR_GROUP_SCHED
9319
static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
9320
				u64 shareval)
9321
{
9322
	return sched_group_set_shares(cgroup_tg(cgrp), shareval);
9323 9324
}

9325
static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
9326
{
9327
	struct task_group *tg = cgroup_tg(cgrp);
9328 9329 9330

	return (u64) tg->shares;
}
9331
#endif /* CONFIG_FAIR_GROUP_SCHED */
9332

9333
#ifdef CONFIG_RT_GROUP_SCHED
M
Mirco Tischler 已提交
9334
static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
9335
				s64 val)
P
Peter Zijlstra 已提交
9336
{
9337
	return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
P
Peter Zijlstra 已提交
9338 9339
}

9340
static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
P
Peter Zijlstra 已提交
9341
{
9342
	return sched_group_rt_runtime(cgroup_tg(cgrp));
P
Peter Zijlstra 已提交
9343
}
9344 9345 9346 9347 9348 9349 9350 9351 9352 9353 9354

static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
		u64 rt_period_us)
{
	return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
}

static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
{
	return sched_group_rt_period(cgroup_tg(cgrp));
}
9355
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
9356

9357
static struct cftype cpu_files[] = {
9358
#ifdef CONFIG_FAIR_GROUP_SCHED
9359 9360
	{
		.name = "shares",
9361 9362
		.read_u64 = cpu_shares_read_u64,
		.write_u64 = cpu_shares_write_u64,
9363
	},
9364 9365
#endif
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
9366
	{
P
Peter Zijlstra 已提交
9367
		.name = "rt_runtime_us",
9368 9369
		.read_s64 = cpu_rt_runtime_read,
		.write_s64 = cpu_rt_runtime_write,
P
Peter Zijlstra 已提交
9370
	},
9371 9372
	{
		.name = "rt_period_us",
9373 9374
		.read_u64 = cpu_rt_period_read_uint,
		.write_u64 = cpu_rt_period_write_uint,
9375
	},
9376
#endif
9377 9378 9379 9380
};

static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
{
9381
	return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
9382 9383 9384
}

struct cgroup_subsys cpu_cgroup_subsys = {
I
Ingo Molnar 已提交
9385 9386 9387 9388 9389 9390 9391
	.name		= "cpu",
	.create		= cpu_cgroup_create,
	.destroy	= cpu_cgroup_destroy,
	.can_attach	= cpu_cgroup_can_attach,
	.attach		= cpu_cgroup_attach,
	.populate	= cpu_cgroup_populate,
	.subsys_id	= cpu_cgroup_subsys_id,
9392 9393 9394
	.early_init	= 1,
};

9395
#endif	/* CONFIG_CGROUP_SCHED */
9396 9397 9398 9399 9400 9401 9402 9403 9404 9405

#ifdef CONFIG_CGROUP_CPUACCT

/*
 * CPU accounting code for task groups.
 *
 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
 * (balbir@in.ibm.com).
 */

9406
/* track cpu usage of a group of tasks and its child groups */
9407 9408 9409 9410
struct cpuacct {
	struct cgroup_subsys_state css;
	/* cpuusage holds pointer to a u64-type object on every cpu */
	u64 *cpuusage;
9411
	struct cpuacct *parent;
9412 9413 9414 9415 9416
};

struct cgroup_subsys cpuacct_subsys;

/* return cpu accounting group corresponding to this container */
9417
static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
9418
{
9419
	return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
9420 9421 9422 9423 9424 9425 9426 9427 9428 9429 9430 9431
			    struct cpuacct, css);
}

/* return cpu accounting group to which this task belongs */
static inline struct cpuacct *task_ca(struct task_struct *tsk)
{
	return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
			    struct cpuacct, css);
}

/* create a new cpu accounting group */
static struct cgroup_subsys_state *cpuacct_create(
9432
	struct cgroup_subsys *ss, struct cgroup *cgrp)
9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444
{
	struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);

	if (!ca)
		return ERR_PTR(-ENOMEM);

	ca->cpuusage = alloc_percpu(u64);
	if (!ca->cpuusage) {
		kfree(ca);
		return ERR_PTR(-ENOMEM);
	}

9445 9446 9447
	if (cgrp->parent)
		ca->parent = cgroup_ca(cgrp->parent);

9448 9449 9450 9451
	return &ca->css;
}

/* destroy an existing cpu accounting group */
I
Ingo Molnar 已提交
9452
static void
9453
cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
9454
{
9455
	struct cpuacct *ca = cgroup_ca(cgrp);
9456 9457 9458 9459 9460

	free_percpu(ca->cpuusage);
	kfree(ca);
}

9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495
static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
{
	u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
	u64 data;

#ifndef CONFIG_64BIT
	/*
	 * Take rq->lock to make 64-bit read safe on 32-bit platforms.
	 */
	spin_lock_irq(&cpu_rq(cpu)->lock);
	data = *cpuusage;
	spin_unlock_irq(&cpu_rq(cpu)->lock);
#else
	data = *cpuusage;
#endif

	return data;
}

static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
{
	u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);

#ifndef CONFIG_64BIT
	/*
	 * Take rq->lock to make 64-bit write safe on 32-bit platforms.
	 */
	spin_lock_irq(&cpu_rq(cpu)->lock);
	*cpuusage = val;
	spin_unlock_irq(&cpu_rq(cpu)->lock);
#else
	*cpuusage = val;
#endif
}

9496
/* return total cpu usage (in nanoseconds) of a group */
9497
static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
9498
{
9499
	struct cpuacct *ca = cgroup_ca(cgrp);
9500 9501 9502
	u64 totalcpuusage = 0;
	int i;

9503 9504
	for_each_present_cpu(i)
		totalcpuusage += cpuacct_cpuusage_read(ca, i);
9505 9506 9507 9508

	return totalcpuusage;
}

9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520
static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
								u64 reset)
{
	struct cpuacct *ca = cgroup_ca(cgrp);
	int err = 0;
	int i;

	if (reset) {
		err = -EINVAL;
		goto out;
	}

9521 9522
	for_each_present_cpu(i)
		cpuacct_cpuusage_write(ca, i, 0);
9523 9524 9525 9526 9527

out:
	return err;
}

9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542
static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
				   struct seq_file *m)
{
	struct cpuacct *ca = cgroup_ca(cgroup);
	u64 percpu;
	int i;

	for_each_present_cpu(i) {
		percpu = cpuacct_cpuusage_read(ca, i);
		seq_printf(m, "%llu ", (unsigned long long) percpu);
	}
	seq_printf(m, "\n");
	return 0;
}

9543 9544 9545
static struct cftype files[] = {
	{
		.name = "usage",
9546 9547
		.read_u64 = cpuusage_read,
		.write_u64 = cpuusage_write,
9548
	},
9549 9550 9551 9552 9553
	{
		.name = "usage_percpu",
		.read_seq_string = cpuacct_percpu_seq_read,
	},

9554 9555
};

9556
static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
9557
{
9558
	return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
9559 9560 9561 9562 9563 9564 9565 9566 9567 9568
}

/*
 * charge this task's execution time to its accounting group.
 *
 * called with rq->lock held.
 */
static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
{
	struct cpuacct *ca;
9569
	int cpu;
9570 9571 9572 9573

	if (!cpuacct_subsys.active)
		return;

9574
	cpu = task_cpu(tsk);
9575 9576
	ca = task_ca(tsk);

9577 9578
	for (; ca; ca = ca->parent) {
		u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9579 9580 9581 9582 9583 9584 9585 9586 9587 9588 9589 9590
		*cpuusage += cputime;
	}
}

struct cgroup_subsys cpuacct_subsys = {
	.name = "cpuacct",
	.create = cpuacct_create,
	.destroy = cpuacct_destroy,
	.populate = cpuacct_populate,
	.subsys_id = cpuacct_subsys_id,
};
#endif	/* CONFIG_CGROUP_CPUACCT */