sched.c 228.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
25 26
 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 *              Thomas Gleixner, Mike Kravetz
L
Linus Torvalds 已提交
27 28 29 30 31 32
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
33
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
34 35 36 37
#include <linux/highmem.h>
#include <linux/smp_lock.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
38
#include <linux/capability.h>
L
Linus Torvalds 已提交
39 40
#include <linux/completion.h>
#include <linux/kernel_stat.h>
41
#include <linux/debug_locks.h>
L
Linus Torvalds 已提交
42 43 44
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
45
#include <linux/freezer.h>
46
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
47 48
#include <linux/blkdev.h>
#include <linux/delay.h>
49
#include <linux/pid_namespace.h>
L
Linus Torvalds 已提交
50 51 52 53 54 55 56 57
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
58
#include <linux/proc_fs.h>
L
Linus Torvalds 已提交
59
#include <linux/seq_file.h>
60
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
61 62
#include <linux/syscalls.h>
#include <linux/times.h>
63
#include <linux/tsacct_kern.h>
64
#include <linux/kprobes.h>
65
#include <linux/delayacct.h>
66
#include <linux/reciprocal_div.h>
67
#include <linux/unistd.h>
J
Jens Axboe 已提交
68
#include <linux/pagemap.h>
P
Peter Zijlstra 已提交
69
#include <linux/hrtimer.h>
R
Reynes Philippe 已提交
70
#include <linux/tick.h>
71
#include <linux/bootmem.h>
P
Peter Zijlstra 已提交
72 73
#include <linux/debugfs.h>
#include <linux/ctype.h>
74
#include <linux/ftrace.h>
75
#include <trace/sched.h>
L
Linus Torvalds 已提交
76

77
#include <asm/tlb.h>
78
#include <asm/irq_regs.h>
L
Linus Torvalds 已提交
79

80 81
#include "sched_cpupri.h"

L
Linus Torvalds 已提交
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
101
 * Helpers for converting nanosecond timing to jiffy resolution
L
Linus Torvalds 已提交
102
 */
103
#define NS_TO_JIFFIES(TIME)	((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
L
Linus Torvalds 已提交
104

I
Ingo Molnar 已提交
105 106 107
#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

L
Linus Torvalds 已提交
108 109 110
/*
 * These are the 'tuning knobs' of the scheduler:
 *
D
Dmitry Adamushko 已提交
111
 * default timeslice is 100 msecs (used only for SCHED_RR tasks).
L
Linus Torvalds 已提交
112 113 114
 * Timeslices get refilled after they expire.
 */
#define DEF_TIMESLICE		(100 * HZ / 1000)
115

116 117 118 119 120
/*
 * single value that denotes runtime == period, ie unlimited time.
 */
#define RUNTIME_INF	((u64)~0ULL)

121 122 123 124 125 126
DEFINE_TRACE(sched_wait_task);
DEFINE_TRACE(sched_wakeup);
DEFINE_TRACE(sched_wakeup_new);
DEFINE_TRACE(sched_switch);
DEFINE_TRACE(sched_migrate_task);

127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
#ifdef CONFIG_SMP
/*
 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 */
static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
{
	return reciprocal_divide(load, sg->reciprocal_cpu_power);
}

/*
 * Each time a sched group cpu_power is changed,
 * we must compute its reciprocal value
 */
static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
{
	sg->__cpu_power += val;
	sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
}
#endif

148 149
static inline int rt_policy(int policy)
{
150
	if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
151 152 153 154 155 156 157 158 159
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

L
Linus Torvalds 已提交
160
/*
I
Ingo Molnar 已提交
161
 * This is the priority-queue data structure of the RT scheduling class:
L
Linus Torvalds 已提交
162
 */
I
Ingo Molnar 已提交
163 164 165 166 167
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

168
struct rt_bandwidth {
I
Ingo Molnar 已提交
169 170 171 172 173
	/* nests inside the rq lock: */
	spinlock_t		rt_runtime_lock;
	ktime_t			rt_period;
	u64			rt_runtime;
	struct hrtimer		rt_period_timer;
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
};

static struct rt_bandwidth def_rt_bandwidth;

static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);

static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
{
	struct rt_bandwidth *rt_b =
		container_of(timer, struct rt_bandwidth, rt_period_timer);
	ktime_t now;
	int overrun;
	int idle = 0;

	for (;;) {
		now = hrtimer_cb_get_time(timer);
		overrun = hrtimer_forward(timer, now, rt_b->rt_period);

		if (!overrun)
			break;

		idle = do_sched_rt_period_timer(rt_b, overrun);
	}

	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
}

static
void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
{
	rt_b->rt_period = ns_to_ktime(period);
	rt_b->rt_runtime = runtime;

P
Peter Zijlstra 已提交
207 208
	spin_lock_init(&rt_b->rt_runtime_lock);

209 210 211
	hrtimer_init(&rt_b->rt_period_timer,
			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rt_b->rt_period_timer.function = sched_rt_period_timer;
212
	rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
213 214
}

215 216 217
static inline int rt_bandwidth_enabled(void)
{
	return sysctl_sched_rt_runtime >= 0;
218 219 220 221 222 223
}

static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	ktime_t now;

224
	if (rt_bandwidth_enabled() && rt_b->rt_runtime == RUNTIME_INF)
225 226 227 228 229 230 231 232 233 234 235 236
		return;

	if (hrtimer_active(&rt_b->rt_period_timer))
		return;

	spin_lock(&rt_b->rt_runtime_lock);
	for (;;) {
		if (hrtimer_active(&rt_b->rt_period_timer))
			break;

		now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
		hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
237 238
		hrtimer_start_expires(&rt_b->rt_period_timer,
				HRTIMER_MODE_ABS);
239 240 241 242 243 244 245 246 247 248 249
	}
	spin_unlock(&rt_b->rt_runtime_lock);
}

#ifdef CONFIG_RT_GROUP_SCHED
static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	hrtimer_cancel(&rt_b->rt_period_timer);
}
#endif

250 251 252 253 254 255
/*
 * sched_domains_mutex serializes calls to arch_init_sched_domains,
 * detach_destroy_domains and partition_sched_domains.
 */
static DEFINE_MUTEX(sched_domains_mutex);

256
#ifdef CONFIG_GROUP_SCHED
S
Srivatsa Vaddagiri 已提交
257

258 259
#include <linux/cgroup.h>

S
Srivatsa Vaddagiri 已提交
260 261
struct cfs_rq;

P
Peter Zijlstra 已提交
262 263
static LIST_HEAD(task_groups);

S
Srivatsa Vaddagiri 已提交
264
/* task group related information */
265
struct task_group {
266
#ifdef CONFIG_CGROUP_SCHED
267 268
	struct cgroup_subsys_state css;
#endif
269

270 271 272 273
#ifdef CONFIG_USER_SCHED
	uid_t uid;
#endif

274
#ifdef CONFIG_FAIR_GROUP_SCHED
S
Srivatsa Vaddagiri 已提交
275 276 277 278 279
	/* schedulable entities of this group on each cpu */
	struct sched_entity **se;
	/* runqueue "owned" by this group on each cpu */
	struct cfs_rq **cfs_rq;
	unsigned long shares;
280 281 282 283 284 285
#endif

#ifdef CONFIG_RT_GROUP_SCHED
	struct sched_rt_entity **rt_se;
	struct rt_rq **rt_rq;

286
	struct rt_bandwidth rt_bandwidth;
287
#endif
288

289
	struct rcu_head rcu;
P
Peter Zijlstra 已提交
290
	struct list_head list;
P
Peter Zijlstra 已提交
291 292 293 294

	struct task_group *parent;
	struct list_head siblings;
	struct list_head children;
S
Srivatsa Vaddagiri 已提交
295 296
};

D
Dhaval Giani 已提交
297
#ifdef CONFIG_USER_SCHED
298

299 300 301 302 303 304
/* Helper function to pass uid information to create_sched_user() */
void set_tg_uid(struct user_struct *user)
{
	user->tg->uid = user->uid;
}

305 306 307 308 309 310 311
/*
 * Root task group.
 * 	Every UID task group (including init_task_group aka UID-0) will
 * 	be a child to this group.
 */
struct task_group root_task_group;

312
#ifdef CONFIG_FAIR_GROUP_SCHED
S
Srivatsa Vaddagiri 已提交
313 314 315 316
/* Default task group's sched entity on each cpu */
static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
/* Default task group's cfs_rq on each cpu */
static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
317
#endif /* CONFIG_FAIR_GROUP_SCHED */
318 319 320 321

#ifdef CONFIG_RT_GROUP_SCHED
static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
322
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
323
#else /* !CONFIG_USER_SCHED */
324
#define root_task_group init_task_group
P
Peter Zijlstra 已提交
325
#endif /* CONFIG_USER_SCHED */
P
Peter Zijlstra 已提交
326

327
/* task_group_lock serializes add/remove of task groups and also changes to
328 329
 * a task group's cpu shares.
 */
330
static DEFINE_SPINLOCK(task_group_lock);
331

332 333 334
#ifdef CONFIG_FAIR_GROUP_SCHED
#ifdef CONFIG_USER_SCHED
# define INIT_TASK_GROUP_LOAD	(2*NICE_0_LOAD)
335
#else /* !CONFIG_USER_SCHED */
336
# define INIT_TASK_GROUP_LOAD	NICE_0_LOAD
337
#endif /* CONFIG_USER_SCHED */
338

339
/*
340 341 342 343
 * A weight of 0 or 1 can cause arithmetics problems.
 * A weight of a cfs_rq is the sum of weights of which entities
 * are queued on this cfs_rq, so a weight of a entity should not be
 * too large, so as the shares value of a task group.
344 345 346
 * (The default weight is 1024 - so there's no practical
 *  limitation from this.)
 */
347
#define MIN_SHARES	2
348
#define MAX_SHARES	(1UL << 18)
349

350 351 352
static int init_task_group_load = INIT_TASK_GROUP_LOAD;
#endif

S
Srivatsa Vaddagiri 已提交
353
/* Default task group.
I
Ingo Molnar 已提交
354
 *	Every task in system belong to this group at bootup.
S
Srivatsa Vaddagiri 已提交
355
 */
356
struct task_group init_task_group;
S
Srivatsa Vaddagiri 已提交
357 358

/* return group to which a task belongs */
359
static inline struct task_group *task_group(struct task_struct *p)
S
Srivatsa Vaddagiri 已提交
360
{
361
	struct task_group *tg;
362

363
#ifdef CONFIG_USER_SCHED
364
	tg = p->user->tg;
365
#elif defined(CONFIG_CGROUP_SCHED)
366 367
	tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
				struct task_group, css);
368
#else
I
Ingo Molnar 已提交
369
	tg = &init_task_group;
370
#endif
371
	return tg;
S
Srivatsa Vaddagiri 已提交
372 373 374
}

/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
P
Peter Zijlstra 已提交
375
static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
S
Srivatsa Vaddagiri 已提交
376
{
377
#ifdef CONFIG_FAIR_GROUP_SCHED
378 379
	p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
	p->se.parent = task_group(p)->se[cpu];
380
#endif
P
Peter Zijlstra 已提交
381

382
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
383 384
	p->rt.rt_rq  = task_group(p)->rt_rq[cpu];
	p->rt.parent = task_group(p)->rt_se[cpu];
385
#endif
S
Srivatsa Vaddagiri 已提交
386 387 388 389
}

#else

P
Peter Zijlstra 已提交
390
static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
391 392 393 394
static inline struct task_group *task_group(struct task_struct *p)
{
	return NULL;
}
S
Srivatsa Vaddagiri 已提交
395

396
#endif	/* CONFIG_GROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
397

I
Ingo Molnar 已提交
398 399 400 401 402 403
/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	u64 exec_clock;
I
Ingo Molnar 已提交
404
	u64 min_vruntime;
I
Ingo Molnar 已提交
405 406 407

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;
408 409 410 411 412 413

	struct list_head tasks;
	struct list_head *balance_iterator;

	/*
	 * 'curr' points to currently running entity on this cfs_rq.
I
Ingo Molnar 已提交
414 415
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
P
Peter Zijlstra 已提交
416
	struct sched_entity *curr, *next, *last;
P
Peter Zijlstra 已提交
417

P
Peter Zijlstra 已提交
418
	unsigned int nr_spread_over;
P
Peter Zijlstra 已提交
419

420
#ifdef CONFIG_FAIR_GROUP_SCHED
I
Ingo Molnar 已提交
421 422
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

I
Ingo Molnar 已提交
423 424
	/*
	 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
I
Ingo Molnar 已提交
425 426 427 428 429 430
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
I
Ingo Molnar 已提交
431 432
	struct list_head leaf_cfs_rq_list;
	struct task_group *tg;	/* group that "owns" this runqueue */
433 434 435

#ifdef CONFIG_SMP
	/*
436
	 * the part of load.weight contributed by tasks
437
	 */
438
	unsigned long task_weight;
439

440 441 442 443 444 445 446
	/*
	 *   h_load = weight * f(tg)
	 *
	 * Where f(tg) is the recursive weight fraction assigned to
	 * this group.
	 */
	unsigned long h_load;
447

448 449 450 451
	/*
	 * this cpu's part of tg->shares
	 */
	unsigned long shares;
452 453 454 455 456

	/*
	 * load.weight at the time we set shares
	 */
	unsigned long rq_weight;
457
#endif
I
Ingo Molnar 已提交
458 459
#endif
};
L
Linus Torvalds 已提交
460

I
Ingo Molnar 已提交
461 462 463
/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
464
	unsigned long rt_nr_running;
465
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
466 467
	int highest_prio; /* highest queued rt task prio */
#endif
P
Peter Zijlstra 已提交
468
#ifdef CONFIG_SMP
469
	unsigned long rt_nr_migratory;
G
Gregory Haskins 已提交
470
	int overloaded;
P
Peter Zijlstra 已提交
471
#endif
P
Peter Zijlstra 已提交
472
	int rt_throttled;
P
Peter Zijlstra 已提交
473
	u64 rt_time;
P
Peter Zijlstra 已提交
474
	u64 rt_runtime;
I
Ingo Molnar 已提交
475
	/* Nests inside the rq lock: */
P
Peter Zijlstra 已提交
476
	spinlock_t rt_runtime_lock;
P
Peter Zijlstra 已提交
477

478
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
479 480
	unsigned long rt_nr_boosted;

P
Peter Zijlstra 已提交
481 482 483 484 485
	struct rq *rq;
	struct list_head leaf_rt_rq_list;
	struct task_group *tg;
	struct sched_rt_entity *rt_se;
#endif
I
Ingo Molnar 已提交
486 487
};

G
Gregory Haskins 已提交
488 489 490 491
#ifdef CONFIG_SMP

/*
 * We add the notion of a root-domain which will be used to define per-domain
I
Ingo Molnar 已提交
492 493
 * variables. Each exclusive cpuset essentially defines an island domain by
 * fully partitioning the member cpus from any other cpuset. Whenever a new
G
Gregory Haskins 已提交
494 495 496 497 498 499
 * exclusive cpuset is created, we also create and attach a new root-domain
 * object.
 *
 */
struct root_domain {
	atomic_t refcount;
500 501
	cpumask_var_t span;
	cpumask_var_t online;
502

I
Ingo Molnar 已提交
503
	/*
504 505 506
	 * The "RT overload" flag: it gets set if a CPU has more than
	 * one runnable RT task.
	 */
507
	cpumask_var_t rto_mask;
I
Ingo Molnar 已提交
508
	atomic_t rto_count;
509 510 511
#ifdef CONFIG_SMP
	struct cpupri cpupri;
#endif
G
Gregory Haskins 已提交
512 513
};

514 515 516 517
/*
 * By default the system creates a single root-domain with all cpus as
 * members (mimicking the global state we have today).
 */
G
Gregory Haskins 已提交
518 519 520 521
static struct root_domain def_root_domain;

#endif

L
Linus Torvalds 已提交
522 523 524 525 526 527 528
/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
529
struct rq {
530 531
	/* runqueue lock: */
	spinlock_t lock;
L
Linus Torvalds 已提交
532 533 534 535 536 537

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
I
Ingo Molnar 已提交
538 539
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
540
	unsigned char idle_at_tick;
541
#ifdef CONFIG_NO_HZ
542
	unsigned long last_tick_seen;
543 544
	unsigned char in_nohz_recently;
#endif
545 546
	/* capture load from *all* tasks on this cpu: */
	struct load_weight load;
I
Ingo Molnar 已提交
547 548 549 550
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
P
Peter Zijlstra 已提交
551 552
	struct rt_rq rt;

I
Ingo Molnar 已提交
553
#ifdef CONFIG_FAIR_GROUP_SCHED
554 555
	/* list of leaf cfs_rq on this cpu: */
	struct list_head leaf_cfs_rq_list;
556 557
#endif
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
558
	struct list_head leaf_rt_rq_list;
L
Linus Torvalds 已提交
559 560 561 562 563 564 565 566 567 568
#endif

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

569
	struct task_struct *curr, *idle;
570
	unsigned long next_balance;
L
Linus Torvalds 已提交
571
	struct mm_struct *prev_mm;
I
Ingo Molnar 已提交
572

573
	u64 clock;
I
Ingo Molnar 已提交
574

L
Linus Torvalds 已提交
575 576 577
	atomic_t nr_iowait;

#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
578
	struct root_domain *rd;
L
Linus Torvalds 已提交
579 580 581 582 583
	struct sched_domain *sd;

	/* For active balancing */
	int active_balance;
	int push_cpu;
584 585
	/* cpu of this runqueue: */
	int cpu;
586
	int online;
L
Linus Torvalds 已提交
587

588
	unsigned long avg_load_per_task;
L
Linus Torvalds 已提交
589

590
	struct task_struct *migration_thread;
L
Linus Torvalds 已提交
591 592 593
	struct list_head migration_queue;
#endif

P
Peter Zijlstra 已提交
594
#ifdef CONFIG_SCHED_HRTICK
595 596 597 598
#ifdef CONFIG_SMP
	int hrtick_csd_pending;
	struct call_single_data hrtick_csd;
#endif
P
Peter Zijlstra 已提交
599 600 601
	struct hrtimer hrtick_timer;
#endif

L
Linus Torvalds 已提交
602 603 604 605 606
#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;

	/* sys_sched_yield() stats */
607 608 609 610
	unsigned int yld_exp_empty;
	unsigned int yld_act_empty;
	unsigned int yld_both_empty;
	unsigned int yld_count;
L
Linus Torvalds 已提交
611 612

	/* schedule() stats */
613 614 615
	unsigned int sched_switch;
	unsigned int sched_count;
	unsigned int sched_goidle;
L
Linus Torvalds 已提交
616 617

	/* try_to_wake_up() stats */
618 619
	unsigned int ttwu_count;
	unsigned int ttwu_local;
I
Ingo Molnar 已提交
620 621

	/* BKL stats */
622
	unsigned int bkl_count;
L
Linus Torvalds 已提交
623 624 625
#endif
};

626
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
L
Linus Torvalds 已提交
627

628
static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
I
Ingo Molnar 已提交
629
{
630
	rq->curr->sched_class->check_preempt_curr(rq, p, sync);
I
Ingo Molnar 已提交
631 632
}

633 634 635 636 637 638 639 640 641
static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

N
Nick Piggin 已提交
642 643
/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
644
 * See detach_destroy_domains: synchronize_sched for details.
N
Nick Piggin 已提交
645 646 647 648
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
649 650
#define for_each_domain(cpu, __sd) \
	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
L
Linus Torvalds 已提交
651 652 653 654 655 656

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

657 658 659 660 661
static inline void update_rq_clock(struct rq *rq)
{
	rq->clock = sched_clock_cpu(cpu_of(rq));
}

I
Ingo Molnar 已提交
662 663 664 665 666 667 668 669 670
/*
 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
 */
#ifdef CONFIG_SCHED_DEBUG
# define const_debug __read_mostly
#else
# define const_debug static const
#endif

I
Ingo Molnar 已提交
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
/**
 * runqueue_is_locked
 *
 * Returns true if the current cpu runqueue is locked.
 * This interface allows printk to be called with the runqueue lock
 * held and know whether or not it is OK to wake up the klogd.
 */
int runqueue_is_locked(void)
{
	int cpu = get_cpu();
	struct rq *rq = cpu_rq(cpu);
	int ret;

	ret = spin_is_locked(&rq->lock);
	put_cpu();
	return ret;
}

I
Ingo Molnar 已提交
689 690 691
/*
 * Debugging: various feature bits
 */
P
Peter Zijlstra 已提交
692 693 694 695

#define SCHED_FEAT(name, enabled)	\
	__SCHED_FEAT_##name ,

I
Ingo Molnar 已提交
696
enum {
P
Peter Zijlstra 已提交
697
#include "sched_features.h"
I
Ingo Molnar 已提交
698 699
};

P
Peter Zijlstra 已提交
700 701 702 703 704
#undef SCHED_FEAT

#define SCHED_FEAT(name, enabled)	\
	(1UL << __SCHED_FEAT_##name) * enabled |

I
Ingo Molnar 已提交
705
const_debug unsigned int sysctl_sched_features =
P
Peter Zijlstra 已提交
706 707 708 709 710 711 712 713 714
#include "sched_features.h"
	0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG
#define SCHED_FEAT(name, enabled)	\
	#name ,

715
static __read_mostly char *sched_feat_names[] = {
P
Peter Zijlstra 已提交
716 717 718 719 720 721
#include "sched_features.h"
	NULL
};

#undef SCHED_FEAT

L
Li Zefan 已提交
722
static int sched_feat_show(struct seq_file *m, void *v)
P
Peter Zijlstra 已提交
723 724 725 726
{
	int i;

	for (i = 0; sched_feat_names[i]; i++) {
L
Li Zefan 已提交
727 728 729
		if (!(sysctl_sched_features & (1UL << i)))
			seq_puts(m, "NO_");
		seq_printf(m, "%s ", sched_feat_names[i]);
P
Peter Zijlstra 已提交
730
	}
L
Li Zefan 已提交
731
	seq_puts(m, "\n");
P
Peter Zijlstra 已提交
732

L
Li Zefan 已提交
733
	return 0;
P
Peter Zijlstra 已提交
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
}

static ssize_t
sched_feat_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
	char buf[64];
	char *cmp = buf;
	int neg = 0;
	int i;

	if (cnt > 63)
		cnt = 63;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

I
Ingo Molnar 已提交
753
	if (strncmp(buf, "NO_", 3) == 0) {
P
Peter Zijlstra 已提交
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
		neg = 1;
		cmp += 3;
	}

	for (i = 0; sched_feat_names[i]; i++) {
		int len = strlen(sched_feat_names[i]);

		if (strncmp(cmp, sched_feat_names[i], len) == 0) {
			if (neg)
				sysctl_sched_features &= ~(1UL << i);
			else
				sysctl_sched_features |= (1UL << i);
			break;
		}
	}

	if (!sched_feat_names[i])
		return -EINVAL;

	filp->f_pos += cnt;

	return cnt;
}

L
Li Zefan 已提交
778 779 780 781 782
static int sched_feat_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_feat_show, NULL);
}

P
Peter Zijlstra 已提交
783
static struct file_operations sched_feat_fops = {
L
Li Zefan 已提交
784 785 786 787 788
	.open		= sched_feat_open,
	.write		= sched_feat_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
P
Peter Zijlstra 已提交
789 790 791 792 793 794 795 796 797 798 799 800 801 802
};

static __init int sched_init_debug(void)
{
	debugfs_create_file("sched_features", 0644, NULL, NULL,
			&sched_feat_fops);

	return 0;
}
late_initcall(sched_init_debug);

#endif

#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
I
Ingo Molnar 已提交
803

804 805 806 807 808 809
/*
 * Number of tasks to iterate in a single balance run.
 * Limited because this is done with IRQs disabled.
 */
const_debug unsigned int sysctl_sched_nr_migrate = 32;

P
Peter Zijlstra 已提交
810 811
/*
 * ratelimit for updating the group shares.
812
 * default: 0.25ms
P
Peter Zijlstra 已提交
813
 */
814
unsigned int sysctl_sched_shares_ratelimit = 250000;
P
Peter Zijlstra 已提交
815

816 817 818 819 820 821 822
/*
 * Inject some fuzzyness into changing the per-cpu group shares
 * this avoids remote rq-locks at the expense of fairness.
 * default: 4
 */
unsigned int sysctl_sched_shares_thresh = 4;

P
Peter Zijlstra 已提交
823
/*
P
Peter Zijlstra 已提交
824
 * period over which we measure -rt task cpu usage in us.
P
Peter Zijlstra 已提交
825 826
 * default: 1s
 */
P
Peter Zijlstra 已提交
827
unsigned int sysctl_sched_rt_period = 1000000;
P
Peter Zijlstra 已提交
828

829 830
static __read_mostly int scheduler_running;

P
Peter Zijlstra 已提交
831 832 833 834 835
/*
 * part of the period that we allow rt tasks to run in us.
 * default: 0.95s
 */
int sysctl_sched_rt_runtime = 950000;
P
Peter Zijlstra 已提交
836

837 838 839 840 841 842 843
static inline u64 global_rt_period(void)
{
	return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
}

static inline u64 global_rt_runtime(void)
{
844
	if (sysctl_sched_rt_runtime < 0)
845 846 847 848
		return RUNTIME_INF;

	return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
}
P
Peter Zijlstra 已提交
849

L
Linus Torvalds 已提交
850
#ifndef prepare_arch_switch
851 852 853 854 855 856
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

857 858 859 860 861
static inline int task_current(struct rq *rq, struct task_struct *p)
{
	return rq->curr == p;
}

862
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
863
static inline int task_running(struct rq *rq, struct task_struct *p)
864
{
865
	return task_current(rq, p);
866 867
}

868
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
869 870 871
{
}

872
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
873
{
874 875 876 877
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
878 879 880 881 882 883 884
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

885 886 887 888
	spin_unlock_irq(&rq->lock);
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
889
static inline int task_running(struct rq *rq, struct task_struct *p)
890 891 892 893
{
#ifdef CONFIG_SMP
	return p->oncpu;
#else
894
	return task_current(rq, p);
895 896 897
#endif
}

898
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->oncpu = 1;
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	spin_unlock_irq(&rq->lock);
#else
	spin_unlock(&rq->lock);
#endif
}

915
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
916 917 918 919 920 921 922 923 924 925 926 927
{
#ifdef CONFIG_SMP
	/*
	 * After ->oncpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->oncpu = 0;
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
L
Linus Torvalds 已提交
928
#endif
929 930
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
L
Linus Torvalds 已提交
931

932 933 934 935
/*
 * __task_rq_lock - lock the runqueue a given task resides on.
 * Must be called interrupts disabled.
 */
936
static inline struct rq *__task_rq_lock(struct task_struct *p)
937 938
	__acquires(rq->lock)
{
939 940 941 942 943
	for (;;) {
		struct rq *rq = task_rq(p);
		spin_lock(&rq->lock);
		if (likely(rq == task_rq(p)))
			return rq;
944 945 946 947
		spin_unlock(&rq->lock);
	}
}

L
Linus Torvalds 已提交
948 949
/*
 * task_rq_lock - lock the runqueue a given task resides on and disable
I
Ingo Molnar 已提交
950
 * interrupts. Note the ordering: we can safely lookup the task_rq without
L
Linus Torvalds 已提交
951 952
 * explicitly disabling preemption.
 */
953
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
954 955
	__acquires(rq->lock)
{
956
	struct rq *rq;
L
Linus Torvalds 已提交
957

958 959 960 961 962 963
	for (;;) {
		local_irq_save(*flags);
		rq = task_rq(p);
		spin_lock(&rq->lock);
		if (likely(rq == task_rq(p)))
			return rq;
L
Linus Torvalds 已提交
964 965 966 967
		spin_unlock_irqrestore(&rq->lock, *flags);
	}
}

968 969 970 971 972 973 974 975
void task_rq_unlock_wait(struct task_struct *p)
{
	struct rq *rq = task_rq(p);

	smp_mb(); /* spin-unlock-wait is not a full memory barrier */
	spin_unlock_wait(&rq->lock);
}

A
Alexey Dobriyan 已提交
976
static void __task_rq_unlock(struct rq *rq)
977 978 979 980 981
	__releases(rq->lock)
{
	spin_unlock(&rq->lock);
}

982
static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
L
Linus Torvalds 已提交
983 984 985 986 987 988
	__releases(rq->lock)
{
	spin_unlock_irqrestore(&rq->lock, *flags);
}

/*
989
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
990
 */
A
Alexey Dobriyan 已提交
991
static struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
992 993
	__acquires(rq->lock)
{
994
	struct rq *rq;
L
Linus Torvalds 已提交
995 996 997 998 999 1000 1001 1002

	local_irq_disable();
	rq = this_rq();
	spin_lock(&rq->lock);

	return rq;
}

P
Peter Zijlstra 已提交
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
#ifdef CONFIG_SCHED_HRTICK
/*
 * Use HR-timers to deliver accurate preemption points.
 *
 * Its all a bit involved since we cannot program an hrt while holding the
 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
 * reschedule event.
 *
 * When we get rescheduled we reprogram the hrtick_timer outside of the
 * rq->lock.
 */

/*
 * Use hrtick when:
 *  - enabled by features
 *  - hrtimer is actually high res
 */
static inline int hrtick_enabled(struct rq *rq)
{
	if (!sched_feat(HRTICK))
		return 0;
1024
	if (!cpu_active(cpu_of(rq)))
1025
		return 0;
P
Peter Zijlstra 已提交
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
	return hrtimer_is_hres_active(&rq->hrtick_timer);
}

static void hrtick_clear(struct rq *rq)
{
	if (hrtimer_active(&rq->hrtick_timer))
		hrtimer_cancel(&rq->hrtick_timer);
}

/*
 * High-resolution timer tick.
 * Runs from hardirq context with interrupts disabled.
 */
static enum hrtimer_restart hrtick(struct hrtimer *timer)
{
	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

	spin_lock(&rq->lock);
1046
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
1047 1048 1049 1050 1051 1052
	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
	spin_unlock(&rq->lock);

	return HRTIMER_NORESTART;
}

1053
#ifdef CONFIG_SMP
1054 1055 1056 1057
/*
 * called from hardirq (IPI) context
 */
static void __hrtick_start(void *arg)
1058
{
1059
	struct rq *rq = arg;
1060

1061 1062 1063 1064
	spin_lock(&rq->lock);
	hrtimer_restart(&rq->hrtick_timer);
	rq->hrtick_csd_pending = 0;
	spin_unlock(&rq->lock);
1065 1066
}

1067 1068 1069 1070 1071 1072
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
1073
{
1074 1075
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
1076

1077
	hrtimer_set_expires(timer, time);
1078 1079 1080 1081 1082 1083 1084

	if (rq == this_rq()) {
		hrtimer_restart(timer);
	} else if (!rq->hrtick_csd_pending) {
		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
		rq->hrtick_csd_pending = 1;
	}
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
}

static int
hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
	int cpu = (int)(long)hcpu;

	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
1099
		hrtick_clear(cpu_rq(cpu));
1100 1101 1102 1103 1104 1105
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

1106
static __init void init_hrtick(void)
1107 1108 1109
{
	hotcpu_notifier(hotplug_hrtick, 0);
}
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
#else
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
{
	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
}
1120

A
Andrew Morton 已提交
1121
static inline void init_hrtick(void)
P
Peter Zijlstra 已提交
1122 1123
{
}
1124
#endif /* CONFIG_SMP */
P
Peter Zijlstra 已提交
1125

1126
static void init_rq_hrtick(struct rq *rq)
P
Peter Zijlstra 已提交
1127
{
1128 1129
#ifdef CONFIG_SMP
	rq->hrtick_csd_pending = 0;
P
Peter Zijlstra 已提交
1130

1131 1132 1133 1134
	rq->hrtick_csd.flags = 0;
	rq->hrtick_csd.func = __hrtick_start;
	rq->hrtick_csd.info = rq;
#endif
P
Peter Zijlstra 已提交
1135

1136 1137
	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rq->hrtick_timer.function = hrtick;
1138
	rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
P
Peter Zijlstra 已提交
1139
}
A
Andrew Morton 已提交
1140
#else	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
1141 1142 1143 1144 1145 1146 1147 1148
static inline void hrtick_clear(struct rq *rq)
{
}

static inline void init_rq_hrtick(struct rq *rq)
{
}

1149 1150 1151
static inline void init_hrtick(void)
{
}
A
Andrew Morton 已提交
1152
#endif	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
1153

I
Ingo Molnar 已提交
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

1167
static void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
1168 1169 1170 1171 1172
{
	int cpu;

	assert_spin_locked(&task_rq(p)->lock);

1173
	if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
I
Ingo Molnar 已提交
1174 1175
		return;

1176
	set_tsk_thread_flag(p, TIF_NEED_RESCHED);
I
Ingo Molnar 已提交
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	if (!spin_trylock_irqsave(&rq->lock, flags))
		return;
	resched_task(cpu_curr(cpu));
	spin_unlock_irqrestore(&rq->lock, flags);
}
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238

#ifdef CONFIG_NO_HZ
/*
 * When add_timer_on() enqueues a timer into the timer wheel of an
 * idle CPU then this timer might expire before the next timer event
 * which is scheduled to wake up that CPU. In case of a completely
 * idle system the next event might even be infinite time into the
 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 * leaves the inner idle loop so the newly added timer is taken into
 * account when the CPU goes back to idle and evaluates the timer
 * wheel for the next timer event.
 */
void wake_up_idle_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);

	if (cpu == smp_processor_id())
		return;

	/*
	 * This is safe, as this function is called with the timer
	 * wheel base lock of (cpu) held. When the CPU is on the way
	 * to idle and has not yet set rq->curr to idle then it will
	 * be serialized on the timer wheel base lock and take the new
	 * timer into account automatically.
	 */
	if (rq->curr != rq->idle)
		return;

	/*
	 * We can set TIF_RESCHED on the idle task of the other CPU
	 * lockless. The worst case is that the other CPU runs the
	 * idle task through an additional NOOP schedule()
	 */
	set_tsk_thread_flag(rq->idle, TIF_NEED_RESCHED);

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(rq->idle))
		smp_send_reschedule(cpu);
}
1239
#endif /* CONFIG_NO_HZ */
1240

1241
#else /* !CONFIG_SMP */
1242
static void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
1243 1244
{
	assert_spin_locked(&task_rq(p)->lock);
1245
	set_tsk_need_resched(p);
I
Ingo Molnar 已提交
1246
}
1247
#endif /* CONFIG_SMP */
I
Ingo Molnar 已提交
1248

1249 1250 1251 1252 1253 1254 1255 1256
#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

I
Ingo Molnar 已提交
1257 1258 1259
/*
 * Shift right and round:
 */
I
Ingo Molnar 已提交
1260
#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
I
Ingo Molnar 已提交
1261

1262 1263 1264
/*
 * delta *= weight / lw
 */
1265
static unsigned long
1266 1267 1268 1269 1270
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

1271 1272 1273 1274 1275 1276 1277
	if (!lw->inv_weight) {
		if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
			lw->inv_weight = 1;
		else
			lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
				/ (lw->weight+1);
	}
1278 1279 1280 1281 1282

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
I
Ingo Molnar 已提交
1283
	if (unlikely(tmp > WMULT_CONST))
I
Ingo Molnar 已提交
1284
		tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
I
Ingo Molnar 已提交
1285 1286
			WMULT_SHIFT/2);
	else
I
Ingo Molnar 已提交
1287
		tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
1288

1289
	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
1290 1291
}

1292
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
1293 1294
{
	lw->weight += inc;
I
Ingo Molnar 已提交
1295
	lw->inv_weight = 0;
1296 1297
}

1298
static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
1299 1300
{
	lw->weight -= dec;
I
Ingo Molnar 已提交
1301
	lw->inv_weight = 0;
1302 1303
}

1304 1305 1306 1307
/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
I
Ingo Molnar 已提交
1308
 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
1309 1310 1311 1312
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

I
Ingo Molnar 已提交
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
#define WEIGHT_IDLEPRIO		2
#define WMULT_IDLEPRIO		(1 << 31)

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
1324 1325 1326
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
I
Ingo Molnar 已提交
1327 1328
 */
static const int prio_to_weight[40] = {
1329 1330 1331 1332 1333 1334 1335 1336
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,
 /*  10 */       110,        87,        70,        56,        45,
 /*  15 */        36,        29,        23,        18,        15,
I
Ingo Molnar 已提交
1337 1338
};

1339 1340 1341 1342 1343 1344 1345
/*
 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
 *
 * In cases where the weight does not change often, we can use the
 * precalculated inverse to speed up arithmetics by turning divisions
 * into multiplications:
 */
I
Ingo Molnar 已提交
1346
static const u32 prio_to_wmult[40] = {
1347 1348 1349 1350 1351 1352 1353 1354
 /* -20 */     48388,     59856,     76040,     92818,    118348,
 /* -15 */    147320,    184698,    229616,    287308,    360437,
 /* -10 */    449829,    563644,    704093,    875809,   1099582,
 /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
 /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
 /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
 /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
 /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
I
Ingo Molnar 已提交
1355
};
1356

I
Ingo Molnar 已提交
1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);

/*
 * runqueue iterator, to support SMP load-balancing between different
 * scheduling classes, without having to expose their internal data
 * structures to the load-balancing proper:
 */
struct rq_iterator {
	void *arg;
	struct task_struct *(*start)(void *);
	struct task_struct *(*next)(void *);
};

1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381
#ifdef CONFIG_SMP
static unsigned long
balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
	      unsigned long max_load_move, struct sched_domain *sd,
	      enum cpu_idle_type idle, int *all_pinned,
	      int *this_best_prio, struct rq_iterator *iterator);

static int
iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
		   struct sched_domain *sd, enum cpu_idle_type idle,
		   struct rq_iterator *iterator);
#endif
I
Ingo Molnar 已提交
1382

1383 1384 1385 1386 1387 1388
#ifdef CONFIG_CGROUP_CPUACCT
static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
#else
static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
#endif

1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
static inline void inc_cpu_load(struct rq *rq, unsigned long load)
{
	update_load_add(&rq->load, load);
}

static inline void dec_cpu_load(struct rq *rq, unsigned long load)
{
	update_load_sub(&rq->load, load);
}

I
Ingo Molnar 已提交
1399
#if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
P
Peter Zijlstra 已提交
1400
typedef int (*tg_visitor)(struct task_group *, void *);
1401 1402 1403 1404 1405

/*
 * Iterate the full tree, calling @down when first entering a node and @up when
 * leaving it for the final time.
 */
P
Peter Zijlstra 已提交
1406
static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
1407 1408
{
	struct task_group *parent, *child;
P
Peter Zijlstra 已提交
1409
	int ret;
1410 1411 1412 1413

	rcu_read_lock();
	parent = &root_task_group;
down:
P
Peter Zijlstra 已提交
1414 1415 1416
	ret = (*down)(parent, data);
	if (ret)
		goto out_unlock;
1417 1418 1419 1420 1421 1422 1423
	list_for_each_entry_rcu(child, &parent->children, siblings) {
		parent = child;
		goto down;

up:
		continue;
	}
P
Peter Zijlstra 已提交
1424 1425 1426
	ret = (*up)(parent, data);
	if (ret)
		goto out_unlock;
1427 1428 1429 1430 1431

	child = parent;
	parent = parent->parent;
	if (parent)
		goto up;
P
Peter Zijlstra 已提交
1432
out_unlock:
1433
	rcu_read_unlock();
P
Peter Zijlstra 已提交
1434 1435

	return ret;
1436 1437
}

P
Peter Zijlstra 已提交
1438 1439 1440
static int tg_nop(struct task_group *tg, void *data)
{
	return 0;
1441
}
P
Peter Zijlstra 已提交
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
#endif

#ifdef CONFIG_SMP
static unsigned long source_load(int cpu, int type);
static unsigned long target_load(int cpu, int type);
static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);

static unsigned long cpu_avg_load_per_task(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
1452
	unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
P
Peter Zijlstra 已提交
1453

1454 1455
	if (nr_running)
		rq->avg_load_per_task = rq->load.weight / nr_running;
1456 1457
	else
		rq->avg_load_per_task = 0;
P
Peter Zijlstra 已提交
1458 1459 1460 1461 1462

	return rq->avg_load_per_task;
}

#ifdef CONFIG_FAIR_GROUP_SCHED
1463 1464 1465 1466 1467 1468 1469

static void __set_se_shares(struct sched_entity *se, unsigned long shares);

/*
 * Calculate and set the cpu's group shares.
 */
static void
1470 1471
update_group_shares_cpu(struct task_group *tg, int cpu,
			unsigned long sd_shares, unsigned long sd_rq_weight)
1472
{
1473 1474 1475
	unsigned long shares;
	unsigned long rq_weight;

1476
	if (!tg->se[cpu])
1477 1478
		return;

1479
	rq_weight = tg->cfs_rq[cpu]->rq_weight;
1480

1481 1482 1483 1484 1485 1486
	/*
	 *           \Sum shares * rq_weight
	 * shares =  -----------------------
	 *               \Sum rq_weight
	 *
	 */
1487
	shares = (sd_shares * rq_weight) / sd_rq_weight;
1488
	shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1489

1490 1491 1492 1493
	if (abs(shares - tg->se[cpu]->load.weight) >
			sysctl_sched_shares_thresh) {
		struct rq *rq = cpu_rq(cpu);
		unsigned long flags;
1494

1495
		spin_lock_irqsave(&rq->lock, flags);
1496
		tg->cfs_rq[cpu]->shares = shares;
1497

1498 1499 1500
		__set_se_shares(tg->se[cpu], shares);
		spin_unlock_irqrestore(&rq->lock, flags);
	}
1501
}
1502 1503

/*
1504 1505 1506
 * Re-compute the task group their per cpu shares over the given domain.
 * This needs to be done in a bottom-up fashion because the rq weight of a
 * parent group depends on the shares of its child groups.
1507
 */
P
Peter Zijlstra 已提交
1508
static int tg_shares_up(struct task_group *tg, void *data)
1509
{
1510
	unsigned long weight, rq_weight = 0;
1511
	unsigned long shares = 0;
P
Peter Zijlstra 已提交
1512
	struct sched_domain *sd = data;
1513
	int i;
1514

1515
	for_each_cpu(i, sched_domain_span(sd)) {
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526
		/*
		 * If there are currently no tasks on the cpu pretend there
		 * is one of average load so that when a new task gets to
		 * run here it will not get delayed by group starvation.
		 */
		weight = tg->cfs_rq[i]->load.weight;
		if (!weight)
			weight = NICE_0_LOAD;

		tg->cfs_rq[i]->rq_weight = weight;
		rq_weight += weight;
1527
		shares += tg->cfs_rq[i]->shares;
1528 1529
	}

1530 1531 1532 1533 1534
	if ((!shares && rq_weight) || shares > tg->shares)
		shares = tg->shares;

	if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
		shares = tg->shares;
1535

1536
	for_each_cpu(i, sched_domain_span(sd))
1537
		update_group_shares_cpu(tg, i, shares, rq_weight);
P
Peter Zijlstra 已提交
1538 1539

	return 0;
1540 1541 1542
}

/*
1543 1544 1545
 * Compute the cpu's hierarchical load factor for each task group.
 * This needs to be done in a top-down fashion because the load of a child
 * group is a fraction of its parents load.
1546
 */
P
Peter Zijlstra 已提交
1547
static int tg_load_down(struct task_group *tg, void *data)
1548
{
1549
	unsigned long load;
P
Peter Zijlstra 已提交
1550
	long cpu = (long)data;
1551

1552 1553 1554 1555 1556 1557 1558
	if (!tg->parent) {
		load = cpu_rq(cpu)->load.weight;
	} else {
		load = tg->parent->cfs_rq[cpu]->h_load;
		load *= tg->cfs_rq[cpu]->shares;
		load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
	}
1559

1560
	tg->cfs_rq[cpu]->h_load = load;
1561

P
Peter Zijlstra 已提交
1562
	return 0;
1563 1564
}

1565
static void update_shares(struct sched_domain *sd)
1566
{
P
Peter Zijlstra 已提交
1567 1568 1569 1570 1571
	u64 now = cpu_clock(raw_smp_processor_id());
	s64 elapsed = now - sd->last_update;

	if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
		sd->last_update = now;
P
Peter Zijlstra 已提交
1572
		walk_tg_tree(tg_nop, tg_shares_up, sd);
P
Peter Zijlstra 已提交
1573
	}
1574 1575
}

1576 1577 1578 1579 1580 1581 1582
static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
{
	spin_unlock(&rq->lock);
	update_shares(sd);
	spin_lock(&rq->lock);
}

P
Peter Zijlstra 已提交
1583
static void update_h_load(long cpu)
1584
{
P
Peter Zijlstra 已提交
1585
	walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
1586 1587 1588 1589
}

#else

1590
static inline void update_shares(struct sched_domain *sd)
1591 1592 1593
{
}

1594 1595 1596 1597
static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
{
}

1598 1599
#endif

1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632
/*
 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
 */
static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
	int ret = 0;

	if (unlikely(!irqs_disabled())) {
		/* printk() doesn't work good under rq->lock */
		spin_unlock(&this_rq->lock);
		BUG_ON(1);
	}
	if (unlikely(!spin_trylock(&busiest->lock))) {
		if (busiest < this_rq) {
			spin_unlock(&this_rq->lock);
			spin_lock(&busiest->lock);
			spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
			ret = 1;
		} else
			spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
	}
	return ret;
}

static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
	__releases(busiest->lock)
{
	spin_unlock(&busiest->lock);
	lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
}
1633 1634
#endif

V
Vegard Nossum 已提交
1635
#ifdef CONFIG_FAIR_GROUP_SCHED
I
Ingo Molnar 已提交
1636 1637
static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
{
V
Vegard Nossum 已提交
1638
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
1639 1640 1641
	cfs_rq->shares = shares;
#endif
}
V
Vegard Nossum 已提交
1642
#endif
1643

I
Ingo Molnar 已提交
1644 1645
#include "sched_stats.h"
#include "sched_idletask.c"
1646 1647
#include "sched_fair.c"
#include "sched_rt.c"
I
Ingo Molnar 已提交
1648 1649 1650 1651 1652
#ifdef CONFIG_SCHED_DEBUG
# include "sched_debug.c"
#endif

#define sched_class_highest (&rt_sched_class)
1653 1654
#define for_each_class(class) \
   for (class = sched_class_highest; class; class = class->next)
I
Ingo Molnar 已提交
1655

1656
static void inc_nr_running(struct rq *rq)
1657 1658 1659 1660
{
	rq->nr_running++;
}

1661
static void dec_nr_running(struct rq *rq)
1662 1663 1664 1665
{
	rq->nr_running--;
}

1666 1667 1668
static void set_load_weight(struct task_struct *p)
{
	if (task_has_rt_policy(p)) {
I
Ingo Molnar 已提交
1669 1670 1671 1672
		p->se.load.weight = prio_to_weight[0] * 2;
		p->se.load.inv_weight = prio_to_wmult[0] >> 1;
		return;
	}
1673

I
Ingo Molnar 已提交
1674 1675 1676 1677 1678 1679 1680 1681
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
		p->se.load.weight = WEIGHT_IDLEPRIO;
		p->se.load.inv_weight = WMULT_IDLEPRIO;
		return;
	}
1682

I
Ingo Molnar 已提交
1683 1684
	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
1685 1686
}

1687 1688 1689 1690 1691 1692
static void update_avg(u64 *avg, u64 sample)
{
	s64 diff = sample - *avg;
	*avg += diff >> 3;
}

1693
static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
1694
{
I
Ingo Molnar 已提交
1695
	sched_info_queued(p);
1696
	p->sched_class->enqueue_task(rq, p, wakeup);
I
Ingo Molnar 已提交
1697
	p->se.on_rq = 1;
1698 1699
}

1700
static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
1701
{
1702 1703 1704 1705 1706 1707
	if (sleep && p->se.last_wakeup) {
		update_avg(&p->se.avg_overlap,
			   p->se.sum_exec_runtime - p->se.last_wakeup);
		p->se.last_wakeup = 0;
	}

1708
	sched_info_dequeued(p);
1709
	p->sched_class->dequeue_task(rq, p, sleep);
I
Ingo Molnar 已提交
1710
	p->se.on_rq = 0;
1711 1712
}

1713
/*
I
Ingo Molnar 已提交
1714
 * __normal_prio - return the priority that is based on the static prio
1715 1716 1717
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
1718
	return p->static_prio;
1719 1720
}

1721 1722 1723 1724 1725 1726 1727
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
1728
static inline int normal_prio(struct task_struct *p)
1729 1730 1731
{
	int prio;

1732
	if (task_has_rt_policy(p))
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
1746
static int effective_prio(struct task_struct *p)
1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
1759
/*
I
Ingo Molnar 已提交
1760
 * activate_task - move a task to the runqueue.
L
Linus Torvalds 已提交
1761
 */
I
Ingo Molnar 已提交
1762
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
L
Linus Torvalds 已提交
1763
{
1764
	if (task_contributes_to_load(p))
I
Ingo Molnar 已提交
1765
		rq->nr_uninterruptible--;
L
Linus Torvalds 已提交
1766

1767
	enqueue_task(rq, p, wakeup);
1768
	inc_nr_running(rq);
L
Linus Torvalds 已提交
1769 1770 1771 1772 1773
}

/*
 * deactivate_task - remove a task from the runqueue.
 */
1774
static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
L
Linus Torvalds 已提交
1775
{
1776
	if (task_contributes_to_load(p))
I
Ingo Molnar 已提交
1777 1778
		rq->nr_uninterruptible++;

1779
	dequeue_task(rq, p, sleep);
1780
	dec_nr_running(rq);
L
Linus Torvalds 已提交
1781 1782 1783 1784 1785 1786
}

/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
1787
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
1788 1789 1790 1791
{
	return cpu_curr(task_cpu(p)) == p;
}

I
Ingo Molnar 已提交
1792 1793
static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
P
Peter Zijlstra 已提交
1794
	set_task_rq(p, cpu);
I
Ingo Molnar 已提交
1795
#ifdef CONFIG_SMP
1796 1797 1798 1799 1800 1801
	/*
	 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
	 * successfuly executed on another CPU. We must ensure that updates of
	 * per-task data have been completed by this moment.
	 */
	smp_wmb();
I
Ingo Molnar 已提交
1802 1803
	task_thread_info(p)->cpu = cpu;
#endif
1804 1805
}

1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
				       const struct sched_class *prev_class,
				       int oldprio, int running)
{
	if (prev_class != p->sched_class) {
		if (prev_class->switched_from)
			prev_class->switched_from(rq, p, running);
		p->sched_class->switched_to(rq, p, running);
	} else
		p->sched_class->prio_changed(rq, p, oldprio, running);
}

L
Linus Torvalds 已提交
1818
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
1819

1820 1821 1822 1823 1824 1825
/* Used instead of source_load when we know the type == 0 */
static unsigned long weighted_cpuload(const int cpu)
{
	return cpu_rq(cpu)->load.weight;
}

1826 1827 1828
/*
 * Is this task likely cache-hot:
 */
1829
static int
1830 1831 1832 1833
task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
{
	s64 delta;

1834 1835 1836
	/*
	 * Buddy candidates are cache hot:
	 */
P
Peter Zijlstra 已提交
1837 1838 1839
	if (sched_feat(CACHE_HOT_BUDDY) &&
			(&p->se == cfs_rq_of(&p->se)->next ||
			 &p->se == cfs_rq_of(&p->se)->last))
1840 1841
		return 1;

1842 1843 1844
	if (p->sched_class != &fair_sched_class)
		return 0;

1845 1846 1847 1848 1849
	if (sysctl_sched_migration_cost == -1)
		return 1;
	if (sysctl_sched_migration_cost == 0)
		return 0;

1850 1851 1852 1853 1854 1855
	delta = now - p->se.exec_start;

	return delta < (s64)sysctl_sched_migration_cost;
}


I
Ingo Molnar 已提交
1856
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
1857
{
I
Ingo Molnar 已提交
1858 1859
	int old_cpu = task_cpu(p);
	struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
1860 1861
	struct cfs_rq *old_cfsrq = task_cfs_rq(p),
		      *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
1862
	u64 clock_offset;
I
Ingo Molnar 已提交
1863 1864

	clock_offset = old_rq->clock - new_rq->clock;
I
Ingo Molnar 已提交
1865 1866 1867 1868

#ifdef CONFIG_SCHEDSTATS
	if (p->se.wait_start)
		p->se.wait_start -= clock_offset;
I
Ingo Molnar 已提交
1869 1870 1871 1872
	if (p->se.sleep_start)
		p->se.sleep_start -= clock_offset;
	if (p->se.block_start)
		p->se.block_start -= clock_offset;
1873 1874 1875 1876 1877
	if (old_cpu != new_cpu) {
		schedstat_inc(p, se.nr_migrations);
		if (task_hot(p, old_rq->clock, NULL))
			schedstat_inc(p, se.nr_forced2_migrations);
	}
I
Ingo Molnar 已提交
1878
#endif
1879 1880
	p->se.vruntime -= old_cfsrq->min_vruntime -
					 new_cfsrq->min_vruntime;
I
Ingo Molnar 已提交
1881 1882

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1883 1884
}

1885
struct migration_req {
L
Linus Torvalds 已提交
1886 1887
	struct list_head list;

1888
	struct task_struct *task;
L
Linus Torvalds 已提交
1889 1890 1891
	int dest_cpu;

	struct completion done;
1892
};
L
Linus Torvalds 已提交
1893 1894 1895 1896 1897

/*
 * The task's runqueue lock must be held.
 * Returns true if you have to wait for migration thread.
 */
1898
static int
1899
migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
L
Linus Torvalds 已提交
1900
{
1901
	struct rq *rq = task_rq(p);
L
Linus Torvalds 已提交
1902 1903 1904 1905 1906

	/*
	 * If the task is not on a runqueue (and not running), then
	 * it is sufficient to simply update the task's cpu field.
	 */
I
Ingo Molnar 已提交
1907
	if (!p->se.on_rq && !task_running(rq, p)) {
L
Linus Torvalds 已提交
1908 1909 1910 1911 1912 1913 1914 1915
		set_task_cpu(p, dest_cpu);
		return 0;
	}

	init_completion(&req->done);
	req->task = p;
	req->dest_cpu = dest_cpu;
	list_add(&req->list, &rq->migration_queue);
1916

L
Linus Torvalds 已提交
1917 1918 1919 1920 1921 1922
	return 1;
}

/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
R
Roland McGrath 已提交
1923 1924 1925 1926 1927 1928 1929
 * If @match_state is nonzero, it's the @p->state value just checked and
 * not expected to change.  If it changes, i.e. @p might have woken up,
 * then return zero.  When we succeed in waiting for @p to be off its CPU,
 * we return a positive number (its total switch count).  If a second call
 * a short while later returns the same number, the caller can be sure that
 * @p has remained unscheduled the whole time.
 *
L
Linus Torvalds 已提交
1930 1931 1932 1933 1934 1935
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
R
Roland McGrath 已提交
1936
unsigned long wait_task_inactive(struct task_struct *p, long match_state)
L
Linus Torvalds 已提交
1937 1938
{
	unsigned long flags;
I
Ingo Molnar 已提交
1939
	int running, on_rq;
R
Roland McGrath 已提交
1940
	unsigned long ncsw;
1941
	struct rq *rq;
L
Linus Torvalds 已提交
1942

1943 1944 1945 1946 1947 1948 1949 1950
	for (;;) {
		/*
		 * We do the initial early heuristics without holding
		 * any task-queue locks at all. We'll only try to get
		 * the runqueue lock when things look like they will
		 * work out!
		 */
		rq = task_rq(p);
1951

1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962
		/*
		 * If the task is actively running on another CPU
		 * still, just relax and busy-wait without holding
		 * any locks.
		 *
		 * NOTE! Since we don't hold any locks, it's not
		 * even sure that "rq" stays as the right runqueue!
		 * But we don't care, since "task_running()" will
		 * return false if the runqueue has changed and p
		 * is actually now running somewhere else!
		 */
R
Roland McGrath 已提交
1963 1964 1965
		while (task_running(rq, p)) {
			if (match_state && unlikely(p->state != match_state))
				return 0;
1966
			cpu_relax();
R
Roland McGrath 已提交
1967
		}
1968

1969 1970 1971 1972 1973 1974
		/*
		 * Ok, time to look more closely! We need the rq
		 * lock now, to be *sure*. If we're wrong, we'll
		 * just go back and repeat.
		 */
		rq = task_rq_lock(p, &flags);
1975
		trace_sched_wait_task(rq, p);
1976 1977
		running = task_running(rq, p);
		on_rq = p->se.on_rq;
R
Roland McGrath 已提交
1978
		ncsw = 0;
1979
		if (!match_state || p->state == match_state)
1980
			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1981
		task_rq_unlock(rq, &flags);
1982

R
Roland McGrath 已提交
1983 1984 1985 1986 1987 1988
		/*
		 * If it changed from the expected state, bail out now.
		 */
		if (unlikely(!ncsw))
			break;

1989 1990 1991 1992 1993 1994 1995 1996 1997 1998
		/*
		 * Was it really running after all now that we
		 * checked with the proper locks actually held?
		 *
		 * Oops. Go back and try again..
		 */
		if (unlikely(running)) {
			cpu_relax();
			continue;
		}
1999

2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012
		/*
		 * It's not enough that it's not actively running,
		 * it must be off the runqueue _entirely_, and not
		 * preempted!
		 *
		 * So if it wa still runnable (but just not actively
		 * running right now), it's preempted, and we should
		 * yield - it could be a while.
		 */
		if (unlikely(on_rq)) {
			schedule_timeout_uninterruptible(1);
			continue;
		}
2013

2014 2015 2016 2017 2018 2019 2020
		/*
		 * Ahh, all good. It wasn't running, and it wasn't
		 * runnable, which means that it will never become
		 * running in the future either. We're all done!
		 */
		break;
	}
R
Roland McGrath 已提交
2021 2022

	return ncsw;
L
Linus Torvalds 已提交
2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
 * NOTE: this function doesnt have to take the runqueue lock,
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
2038
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}

/*
2050 2051
 * Return a low guess at the load of a migration-source cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
2052 2053 2054 2055
 *
 * We want to under-estimate the load of migration sources, to
 * balance conservatively.
 */
A
Alexey Dobriyan 已提交
2056
static unsigned long source_load(int cpu, int type)
L
Linus Torvalds 已提交
2057
{
2058
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
2059
	unsigned long total = weighted_cpuload(cpu);
2060

2061
	if (type == 0 || !sched_feat(LB_BIAS))
I
Ingo Molnar 已提交
2062
		return total;
2063

I
Ingo Molnar 已提交
2064
	return min(rq->cpu_load[type-1], total);
L
Linus Torvalds 已提交
2065 2066 2067
}

/*
2068 2069
 * Return a high guess at the load of a migration-target cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
2070
 */
A
Alexey Dobriyan 已提交
2071
static unsigned long target_load(int cpu, int type)
L
Linus Torvalds 已提交
2072
{
2073
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
2074
	unsigned long total = weighted_cpuload(cpu);
2075

2076
	if (type == 0 || !sched_feat(LB_BIAS))
I
Ingo Molnar 已提交
2077
		return total;
2078

I
Ingo Molnar 已提交
2079
	return max(rq->cpu_load[type-1], total);
2080 2081
}

N
Nick Piggin 已提交
2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
/*
 * find_idlest_group finds and returns the least busy CPU group within the
 * domain.
 */
static struct sched_group *
find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
{
	struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
	unsigned long min_load = ULONG_MAX, this_load = 0;
	int load_idx = sd->forkexec_idx;
	int imbalance = 100 + (sd->imbalance_pct-100)/2;

	do {
		unsigned long load, avg_load;
		int local_group;
		int i;

2099
		/* Skip over this group if it has no CPUs allowed */
2100 2101
		if (!cpumask_intersects(sched_group_cpus(group),
					&p->cpus_allowed))
2102
			continue;
2103

2104 2105
		local_group = cpumask_test_cpu(this_cpu,
					       sched_group_cpus(group));
N
Nick Piggin 已提交
2106 2107 2108 2109

		/* Tally up the load of all CPUs in the group */
		avg_load = 0;

2110
		for_each_cpu(i, sched_group_cpus(group)) {
N
Nick Piggin 已提交
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120
			/* Bias balancing toward cpus of our domain */
			if (local_group)
				load = source_load(i, load_idx);
			else
				load = target_load(i, load_idx);

			avg_load += load;
		}

		/* Adjust by relative CPU power of the group */
2121 2122
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
N
Nick Piggin 已提交
2123 2124 2125 2126 2127 2128 2129 2130

		if (local_group) {
			this_load = avg_load;
			this = group;
		} else if (avg_load < min_load) {
			min_load = avg_load;
			idlest = group;
		}
2131
	} while (group = group->next, group != sd->groups);
N
Nick Piggin 已提交
2132 2133 2134 2135 2136 2137 2138

	if (!idlest || 100*this_load < imbalance*min_load)
		return NULL;
	return idlest;
}

/*
2139
 * find_idlest_cpu - find the idlest cpu among the cpus in group.
N
Nick Piggin 已提交
2140
 */
I
Ingo Molnar 已提交
2141
static int
2142
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
N
Nick Piggin 已提交
2143 2144 2145 2146 2147
{
	unsigned long load, min_load = ULONG_MAX;
	int idlest = -1;
	int i;

2148
	/* Traverse only the allowed CPUs */
2149
	for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2150
		load = weighted_cpuload(i);
N
Nick Piggin 已提交
2151 2152 2153 2154 2155 2156 2157 2158 2159 2160

		if (load < min_load || (load == min_load && i == this_cpu)) {
			min_load = load;
			idlest = i;
		}
	}

	return idlest;
}

N
Nick Piggin 已提交
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175
/*
 * sched_balance_self: balance the current task (running on cpu) in domains
 * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
 * SD_BALANCE_EXEC.
 *
 * Balance, ie. select the least loaded group.
 *
 * Returns the target CPU number, or the same CPU if no balancing is needed.
 *
 * preempt must be disabled.
 */
static int sched_balance_self(int cpu, int flag)
{
	struct task_struct *t = current;
	struct sched_domain *tmp, *sd = NULL;
N
Nick Piggin 已提交
2176

2177
	for_each_domain(cpu, tmp) {
I
Ingo Molnar 已提交
2178 2179 2180
		/*
		 * If power savings logic is enabled for a domain, stop there.
		 */
2181 2182
		if (tmp->flags & SD_POWERSAVINGS_BALANCE)
			break;
N
Nick Piggin 已提交
2183 2184
		if (tmp->flags & flag)
			sd = tmp;
2185
	}
N
Nick Piggin 已提交
2186

2187 2188 2189
	if (sd)
		update_shares(sd);

N
Nick Piggin 已提交
2190 2191
	while (sd) {
		struct sched_group *group;
2192 2193 2194 2195 2196 2197
		int new_cpu, weight;

		if (!(sd->flags & flag)) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
2198 2199

		group = find_idlest_group(sd, t, cpu);
2200 2201 2202 2203
		if (!group) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
2204

2205
		new_cpu = find_idlest_cpu(group, t, cpu);
2206 2207 2208 2209 2210
		if (new_cpu == -1 || new_cpu == cpu) {
			/* Now try balancing at a lower domain level of cpu */
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
2211

2212
		/* Now try balancing at a lower domain level of new_cpu */
N
Nick Piggin 已提交
2213
		cpu = new_cpu;
2214
		weight = cpumask_weight(sched_domain_span(sd));
N
Nick Piggin 已提交
2215 2216
		sd = NULL;
		for_each_domain(cpu, tmp) {
2217
			if (weight <= cpumask_weight(sched_domain_span(tmp)))
N
Nick Piggin 已提交
2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228
				break;
			if (tmp->flags & flag)
				sd = tmp;
		}
		/* while loop will break here if sd == NULL */
	}

	return cpu;
}

#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243

/***
 * try_to_wake_up - wake up a thread
 * @p: the to-be-woken-up thread
 * @state: the mask of task states that can be woken
 * @sync: do a synchronous wakeup?
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
 * returns failure only if the task is already active.
 */
2244
static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
L
Linus Torvalds 已提交
2245
{
2246
	int cpu, orig_cpu, this_cpu, success = 0;
L
Linus Torvalds 已提交
2247 2248
	unsigned long flags;
	long old_state;
2249
	struct rq *rq;
L
Linus Torvalds 已提交
2250

2251 2252 2253
	if (!sched_feat(SYNC_WAKEUPS))
		sync = 0;

P
Peter Zijlstra 已提交
2254 2255 2256 2257 2258 2259 2260 2261
#ifdef CONFIG_SMP
	if (sched_feat(LB_WAKEUP_UPDATE)) {
		struct sched_domain *sd;

		this_cpu = raw_smp_processor_id();
		cpu = task_cpu(p);

		for_each_domain(this_cpu, sd) {
2262
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
P
Peter Zijlstra 已提交
2263 2264 2265 2266 2267 2268 2269
				update_shares(sd);
				break;
			}
		}
	}
#endif

2270
	smp_wmb();
L
Linus Torvalds 已提交
2271 2272 2273 2274 2275
	rq = task_rq_lock(p, &flags);
	old_state = p->state;
	if (!(old_state & state))
		goto out;

I
Ingo Molnar 已提交
2276
	if (p->se.on_rq)
L
Linus Torvalds 已提交
2277 2278 2279
		goto out_running;

	cpu = task_cpu(p);
2280
	orig_cpu = cpu;
L
Linus Torvalds 已提交
2281 2282 2283 2284 2285 2286
	this_cpu = smp_processor_id();

#ifdef CONFIG_SMP
	if (unlikely(task_running(rq, p)))
		goto out_activate;

2287 2288 2289
	cpu = p->sched_class->select_task_rq(p, sync);
	if (cpu != orig_cpu) {
		set_task_cpu(p, cpu);
L
Linus Torvalds 已提交
2290 2291 2292 2293 2294 2295
		task_rq_unlock(rq, &flags);
		/* might preempt at this point */
		rq = task_rq_lock(p, &flags);
		old_state = p->state;
		if (!(old_state & state))
			goto out;
I
Ingo Molnar 已提交
2296
		if (p->se.on_rq)
L
Linus Torvalds 已提交
2297 2298 2299 2300 2301 2302
			goto out_running;

		this_cpu = smp_processor_id();
		cpu = task_cpu(p);
	}

2303 2304 2305 2306 2307 2308 2309
#ifdef CONFIG_SCHEDSTATS
	schedstat_inc(rq, ttwu_count);
	if (cpu == this_cpu)
		schedstat_inc(rq, ttwu_local);
	else {
		struct sched_domain *sd;
		for_each_domain(this_cpu, sd) {
2310
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2311 2312 2313 2314 2315
				schedstat_inc(sd, ttwu_wake_remote);
				break;
			}
		}
	}
2316
#endif /* CONFIG_SCHEDSTATS */
2317

L
Linus Torvalds 已提交
2318 2319
out_activate:
#endif /* CONFIG_SMP */
2320 2321 2322 2323 2324 2325 2326 2327 2328
	schedstat_inc(p, se.nr_wakeups);
	if (sync)
		schedstat_inc(p, se.nr_wakeups_sync);
	if (orig_cpu != cpu)
		schedstat_inc(p, se.nr_wakeups_migrate);
	if (cpu == this_cpu)
		schedstat_inc(p, se.nr_wakeups_local);
	else
		schedstat_inc(p, se.nr_wakeups_remote);
I
Ingo Molnar 已提交
2329
	update_rq_clock(rq);
I
Ingo Molnar 已提交
2330
	activate_task(rq, p, 1);
L
Linus Torvalds 已提交
2331 2332 2333
	success = 1;

out_running:
2334
	trace_sched_wakeup(rq, p);
2335
	check_preempt_curr(rq, p, sync);
I
Ingo Molnar 已提交
2336

L
Linus Torvalds 已提交
2337
	p->state = TASK_RUNNING;
2338 2339 2340 2341
#ifdef CONFIG_SMP
	if (p->sched_class->task_wake_up)
		p->sched_class->task_wake_up(rq, p);
#endif
L
Linus Torvalds 已提交
2342
out:
2343 2344
	current->se.last_wakeup = current->se.sum_exec_runtime;

L
Linus Torvalds 已提交
2345 2346 2347 2348 2349
	task_rq_unlock(rq, &flags);

	return success;
}

2350
int wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
2351
{
2352
	return try_to_wake_up(p, TASK_ALL, 0);
L
Linus Torvalds 已提交
2353 2354 2355
}
EXPORT_SYMBOL(wake_up_process);

2356
int wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
2357 2358 2359 2360 2361 2362 2363
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
2364 2365 2366 2367 2368 2369 2370
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
2371
	p->se.prev_sum_exec_runtime	= 0;
I
Ingo Molnar 已提交
2372 2373
	p->se.last_wakeup		= 0;
	p->se.avg_overlap		= 0;
I
Ingo Molnar 已提交
2374 2375 2376

#ifdef CONFIG_SCHEDSTATS
	p->se.wait_start		= 0;
I
Ingo Molnar 已提交
2377 2378 2379 2380 2381 2382
	p->se.sum_sleep_runtime		= 0;
	p->se.sleep_start		= 0;
	p->se.block_start		= 0;
	p->se.sleep_max			= 0;
	p->se.block_max			= 0;
	p->se.exec_max			= 0;
I
Ingo Molnar 已提交
2383
	p->se.slice_max			= 0;
I
Ingo Molnar 已提交
2384
	p->se.wait_max			= 0;
I
Ingo Molnar 已提交
2385
#endif
N
Nick Piggin 已提交
2386

P
Peter Zijlstra 已提交
2387
	INIT_LIST_HEAD(&p->rt.run_list);
I
Ingo Molnar 已提交
2388
	p->se.on_rq = 0;
2389
	INIT_LIST_HEAD(&p->se.group_node);
N
Nick Piggin 已提交
2390

2391 2392 2393 2394
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif

L
Linus Torvalds 已提交
2395 2396 2397 2398 2399 2400 2401
	/*
	 * We mark the process as running here, but have not actually
	 * inserted it onto the runqueue yet. This guarantees that
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415
}

/*
 * fork()/clone()-time setup:
 */
void sched_fork(struct task_struct *p, int clone_flags)
{
	int cpu = get_cpu();

	__sched_fork(p);

#ifdef CONFIG_SMP
	cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
#endif
I
Ingo Molnar 已提交
2416
	set_task_cpu(p, cpu);
2417 2418 2419 2420 2421

	/*
	 * Make sure we do not leak PI boosting priority to the child:
	 */
	p->prio = current->normal_prio;
H
Hiroshi Shimamoto 已提交
2422 2423
	if (!rt_prio(p->prio))
		p->sched_class = &fair_sched_class;
2424

2425
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
2426
	if (likely(sched_info_on()))
2427
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
2428
#endif
2429
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
2430 2431
	p->oncpu = 0;
#endif
L
Linus Torvalds 已提交
2432
#ifdef CONFIG_PREEMPT
2433
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
2434
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
2435
#endif
N
Nick Piggin 已提交
2436
	put_cpu();
L
Linus Torvalds 已提交
2437 2438 2439 2440 2441 2442 2443 2444 2445
}

/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
2446
void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
L
Linus Torvalds 已提交
2447 2448
{
	unsigned long flags;
I
Ingo Molnar 已提交
2449
	struct rq *rq;
L
Linus Torvalds 已提交
2450 2451

	rq = task_rq_lock(p, &flags);
N
Nick Piggin 已提交
2452
	BUG_ON(p->state != TASK_RUNNING);
I
Ingo Molnar 已提交
2453
	update_rq_clock(rq);
L
Linus Torvalds 已提交
2454 2455 2456

	p->prio = effective_prio(p);

2457
	if (!p->sched_class->task_new || !current->se.on_rq) {
I
Ingo Molnar 已提交
2458
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
2459 2460
	} else {
		/*
I
Ingo Molnar 已提交
2461 2462
		 * Let the scheduling class do new task startup
		 * management (if any):
L
Linus Torvalds 已提交
2463
		 */
2464
		p->sched_class->task_new(rq, p);
2465
		inc_nr_running(rq);
L
Linus Torvalds 已提交
2466
	}
2467
	trace_sched_wakeup_new(rq, p);
2468
	check_preempt_curr(rq, p, 0);
2469 2470 2471 2472
#ifdef CONFIG_SMP
	if (p->sched_class->task_wake_up)
		p->sched_class->task_wake_up(rq, p);
#endif
I
Ingo Molnar 已提交
2473
	task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
2474 2475
}

2476 2477 2478
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
R
Randy Dunlap 已提交
2479 2480
 * preempt_notifier_register - tell me when current is being being preempted & rescheduled
 * @notifier: notifier struct to register
2481 2482 2483 2484 2485 2486 2487 2488 2489
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
2490
 * @notifier: notifier struct to unregister
2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_out(notifier, next);
}

2520
#else /* !CONFIG_PREEMPT_NOTIFIERS */
2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

2532
#endif /* CONFIG_PREEMPT_NOTIFIERS */
2533

2534 2535 2536
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
2537
 * @prev: the current task that is being switched out
2538 2539 2540 2541 2542 2543 2544 2545 2546
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
2547 2548 2549
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
2550
{
2551
	fire_sched_out_preempt_notifiers(prev, next);
2552 2553 2554 2555
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
2556 2557
/**
 * finish_task_switch - clean up after a task-switch
2558
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
2559 2560
 * @prev: the thread we just switched away from.
 *
2561 2562 2563 2564
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
2565 2566
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
I
Ingo Molnar 已提交
2567
 * so, we finish that here outside of the runqueue lock. (Doing it
L
Linus Torvalds 已提交
2568 2569 2570
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
A
Alexey Dobriyan 已提交
2571
static void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
2572 2573 2574
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
2575
	long prev_state;
L
Linus Torvalds 已提交
2576 2577 2578 2579 2580

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
2581
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
2582 2583
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
2584
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
2585 2586 2587 2588 2589
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
2590
	prev_state = prev->state;
2591 2592
	finish_arch_switch(prev);
	finish_lock_switch(rq, prev);
2593 2594 2595 2596
#ifdef CONFIG_SMP
	if (current->sched_class->post_schedule)
		current->sched_class->post_schedule(rq);
#endif
S
Steven Rostedt 已提交
2597

2598
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
2599 2600
	if (mm)
		mmdrop(mm);
2601
	if (unlikely(prev_state == TASK_DEAD)) {
2602 2603 2604
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
2605
		 */
2606
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
2607
		put_task_struct(prev);
2608
	}
L
Linus Torvalds 已提交
2609 2610 2611 2612 2613 2614
}

/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
2615
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
2616 2617
	__releases(rq->lock)
{
2618 2619
	struct rq *rq = this_rq();

2620 2621 2622 2623 2624
	finish_task_switch(rq, prev);
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
2625
	if (current->set_child_tid)
2626
		put_user(task_pid_vnr(current), current->set_child_tid);
L
Linus Torvalds 已提交
2627 2628 2629 2630 2631 2632
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
2633
static inline void
2634
context_switch(struct rq *rq, struct task_struct *prev,
2635
	       struct task_struct *next)
L
Linus Torvalds 已提交
2636
{
I
Ingo Molnar 已提交
2637
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
2638

2639
	prepare_task_switch(rq, prev, next);
2640
	trace_sched_switch(rq, prev, next);
I
Ingo Molnar 已提交
2641 2642
	mm = next->mm;
	oldmm = prev->active_mm;
2643 2644 2645 2646 2647 2648 2649
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
	arch_enter_lazy_cpu_mode();

I
Ingo Molnar 已提交
2650
	if (unlikely(!mm)) {
L
Linus Torvalds 已提交
2651 2652 2653 2654 2655 2656
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

I
Ingo Molnar 已提交
2657
	if (unlikely(!prev->mm)) {
L
Linus Torvalds 已提交
2658 2659 2660
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
2661 2662 2663 2664 2665 2666 2667
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2668
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2669
#endif
L
Linus Torvalds 已提交
2670 2671 2672 2673

	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
2674 2675 2676 2677 2678 2679 2680
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703
}

/*
 * nr_running, nr_uninterruptible and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, current number of uninterruptible-sleeping threads, total
 * number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
}

unsigned long nr_uninterruptible(void)
{
	unsigned long i, sum = 0;

2704
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
		sum += cpu_rq(i)->nr_uninterruptible;

	/*
	 * Since we read the counters lockless, it might be slightly
	 * inaccurate. Do not allow it to go below zero though:
	 */
	if (unlikely((long)sum < 0))
		sum = 0;

	return sum;
}

unsigned long long nr_context_switches(void)
{
2719 2720
	int i;
	unsigned long long sum = 0;
L
Linus Torvalds 已提交
2721

2722
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2723 2724 2725 2726 2727 2728 2729 2730 2731
		sum += cpu_rq(i)->nr_switches;

	return sum;
}

unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;

2732
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2733 2734 2735 2736 2737
		sum += atomic_read(&cpu_rq(i)->nr_iowait);

	return sum;
}

2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752
unsigned long nr_active(void)
{
	unsigned long i, running = 0, uninterruptible = 0;

	for_each_online_cpu(i) {
		running += cpu_rq(i)->nr_running;
		uninterruptible += cpu_rq(i)->nr_uninterruptible;
	}

	if (unlikely((long)uninterruptible < 0))
		uninterruptible = 0;

	return running + uninterruptible;
}

2753
/*
I
Ingo Molnar 已提交
2754 2755
 * Update rq->cpu_load[] statistics. This function is usually called every
 * scheduler tick (TICK_NSEC).
2756
 */
I
Ingo Molnar 已提交
2757
static void update_cpu_load(struct rq *this_rq)
2758
{
2759
	unsigned long this_load = this_rq->load.weight;
I
Ingo Molnar 已提交
2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771
	int i, scale;

	this_rq->nr_load_updates++;

	/* Update our load: */
	for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
		unsigned long old_load, new_load;

		/* scale is effectively 1 << i now, and >> i divides by scale */

		old_load = this_rq->cpu_load[i];
		new_load = this_load;
I
Ingo Molnar 已提交
2772 2773 2774 2775 2776 2777 2778
		/*
		 * Round up the averaging division if load is increasing. This
		 * prevents us from getting stuck on 9 if the load is 10, for
		 * example.
		 */
		if (new_load > old_load)
			new_load += scale-1;
I
Ingo Molnar 已提交
2779 2780
		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
	}
2781 2782
}

I
Ingo Molnar 已提交
2783 2784
#ifdef CONFIG_SMP

L
Linus Torvalds 已提交
2785 2786 2787 2788 2789 2790
/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
2791
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2792 2793 2794
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
2795
	BUG_ON(!irqs_disabled());
L
Linus Torvalds 已提交
2796 2797 2798 2799
	if (rq1 == rq2) {
		spin_lock(&rq1->lock);
		__acquire(rq2->lock);	/* Fake it out ;) */
	} else {
2800
		if (rq1 < rq2) {
L
Linus Torvalds 已提交
2801
			spin_lock(&rq1->lock);
2802
			spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
L
Linus Torvalds 已提交
2803 2804
		} else {
			spin_lock(&rq2->lock);
2805
			spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
L
Linus Torvalds 已提交
2806 2807
		}
	}
2808 2809
	update_rq_clock(rq1);
	update_rq_clock(rq2);
L
Linus Torvalds 已提交
2810 2811 2812 2813 2814 2815 2816 2817
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
2818
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	spin_unlock(&rq1->lock);
	if (rq1 != rq2)
		spin_unlock(&rq2->lock);
	else
		__release(rq2->lock);
}

/*
 * If dest_cpu is allowed for this process, migrate the task to it.
 * This is accomplished by forcing the cpu_allowed mask to only
I
Ingo Molnar 已提交
2832
 * allow dest_cpu, which will force the cpu onto dest_cpu. Then
L
Linus Torvalds 已提交
2833 2834
 * the cpu_allowed mask is restored.
 */
2835
static void sched_migrate_task(struct task_struct *p, int dest_cpu)
L
Linus Torvalds 已提交
2836
{
2837
	struct migration_req req;
L
Linus Torvalds 已提交
2838
	unsigned long flags;
2839
	struct rq *rq;
L
Linus Torvalds 已提交
2840 2841

	rq = task_rq_lock(p, &flags);
2842
	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2843
	    || unlikely(!cpu_active(dest_cpu)))
L
Linus Torvalds 已提交
2844 2845
		goto out;

2846
	trace_sched_migrate_task(rq, p, dest_cpu);
L
Linus Torvalds 已提交
2847 2848 2849 2850
	/* force the process onto the specified CPU */
	if (migrate_task(p, dest_cpu, &req)) {
		/* Need to wait for migration thread (might exit: take ref). */
		struct task_struct *mt = rq->migration_thread;
2851

L
Linus Torvalds 已提交
2852 2853 2854 2855 2856
		get_task_struct(mt);
		task_rq_unlock(rq, &flags);
		wake_up_process(mt);
		put_task_struct(mt);
		wait_for_completion(&req.done);
2857

L
Linus Torvalds 已提交
2858 2859 2860 2861 2862 2863 2864
		return;
	}
out:
	task_rq_unlock(rq, &flags);
}

/*
N
Nick Piggin 已提交
2865 2866
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
L
Linus Torvalds 已提交
2867 2868 2869 2870
 */
void sched_exec(void)
{
	int new_cpu, this_cpu = get_cpu();
N
Nick Piggin 已提交
2871
	new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
L
Linus Torvalds 已提交
2872
	put_cpu();
N
Nick Piggin 已提交
2873 2874
	if (new_cpu != this_cpu)
		sched_migrate_task(current, new_cpu);
L
Linus Torvalds 已提交
2875 2876 2877 2878 2879 2880
}

/*
 * pull_task - move a task from a remote runqueue to the local runqueue.
 * Both runqueues must be locked.
 */
I
Ingo Molnar 已提交
2881 2882
static void pull_task(struct rq *src_rq, struct task_struct *p,
		      struct rq *this_rq, int this_cpu)
L
Linus Torvalds 已提交
2883
{
2884
	deactivate_task(src_rq, p, 0);
L
Linus Torvalds 已提交
2885
	set_task_cpu(p, this_cpu);
I
Ingo Molnar 已提交
2886
	activate_task(this_rq, p, 0);
L
Linus Torvalds 已提交
2887 2888 2889 2890
	/*
	 * Note that idle threads have a prio of MAX_PRIO, for this test
	 * to be always true for them.
	 */
2891
	check_preempt_curr(this_rq, p, 0);
L
Linus Torvalds 已提交
2892 2893 2894 2895 2896
}

/*
 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
 */
2897
static
2898
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
I
Ingo Molnar 已提交
2899
		     struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2900
		     int *all_pinned)
L
Linus Torvalds 已提交
2901 2902 2903 2904 2905 2906 2907
{
	/*
	 * We do not migrate tasks that are:
	 * 1) running (obviously), or
	 * 2) cannot be migrated to this CPU due to cpus_allowed, or
	 * 3) are cache-hot on their current CPU.
	 */
2908
	if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2909
		schedstat_inc(p, se.nr_failed_migrations_affine);
L
Linus Torvalds 已提交
2910
		return 0;
2911
	}
2912 2913
	*all_pinned = 0;

2914 2915
	if (task_running(rq, p)) {
		schedstat_inc(p, se.nr_failed_migrations_running);
2916
		return 0;
2917
	}
L
Linus Torvalds 已提交
2918

2919 2920 2921 2922 2923 2924
	/*
	 * Aggressive migration if:
	 * 1) task is cache cold, or
	 * 2) too many balance attempts have failed.
	 */

2925 2926
	if (!task_hot(p, rq->clock, sd) ||
			sd->nr_balance_failed > sd->cache_nice_tries) {
2927
#ifdef CONFIG_SCHEDSTATS
2928
		if (task_hot(p, rq->clock, sd)) {
2929
			schedstat_inc(sd, lb_hot_gained[idle]);
2930 2931
			schedstat_inc(p, se.nr_forced_migrations);
		}
2932 2933 2934 2935
#endif
		return 1;
	}

2936 2937
	if (task_hot(p, rq->clock, sd)) {
		schedstat_inc(p, se.nr_failed_migrations_hot);
2938
		return 0;
2939
	}
L
Linus Torvalds 已提交
2940 2941 2942
	return 1;
}

2943 2944 2945 2946 2947
static unsigned long
balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
	      unsigned long max_load_move, struct sched_domain *sd,
	      enum cpu_idle_type idle, int *all_pinned,
	      int *this_best_prio, struct rq_iterator *iterator)
L
Linus Torvalds 已提交
2948
{
2949
	int loops = 0, pulled = 0, pinned = 0;
I
Ingo Molnar 已提交
2950 2951
	struct task_struct *p;
	long rem_load_move = max_load_move;
L
Linus Torvalds 已提交
2952

2953
	if (max_load_move == 0)
L
Linus Torvalds 已提交
2954 2955
		goto out;

2956 2957
	pinned = 1;

L
Linus Torvalds 已提交
2958
	/*
I
Ingo Molnar 已提交
2959
	 * Start the load-balancing iterator:
L
Linus Torvalds 已提交
2960
	 */
I
Ingo Molnar 已提交
2961 2962
	p = iterator->start(iterator->arg);
next:
2963
	if (!p || loops++ > sysctl_sched_nr_migrate)
L
Linus Torvalds 已提交
2964
		goto out;
2965 2966

	if ((p->se.load.weight >> 1) > rem_load_move ||
I
Ingo Molnar 已提交
2967 2968 2969
	    !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2970 2971
	}

I
Ingo Molnar 已提交
2972
	pull_task(busiest, p, this_rq, this_cpu);
L
Linus Torvalds 已提交
2973
	pulled++;
I
Ingo Molnar 已提交
2974
	rem_load_move -= p->se.load.weight;
L
Linus Torvalds 已提交
2975

2976
	/*
2977
	 * We only want to steal up to the prescribed amount of weighted load.
2978
	 */
2979
	if (rem_load_move > 0) {
2980 2981
		if (p->prio < *this_best_prio)
			*this_best_prio = p->prio;
I
Ingo Molnar 已提交
2982 2983
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2984 2985 2986
	}
out:
	/*
2987
	 * Right now, this is one of only two places pull_task() is called,
L
Linus Torvalds 已提交
2988 2989 2990 2991
	 * so we can safely collect pull_task() stats here rather than
	 * inside pull_task().
	 */
	schedstat_add(sd, lb_gained[idle], pulled);
2992 2993 2994

	if (all_pinned)
		*all_pinned = pinned;
2995 2996

	return max_load_move - rem_load_move;
L
Linus Torvalds 已提交
2997 2998
}

I
Ingo Molnar 已提交
2999
/*
P
Peter Williams 已提交
3000 3001 3002
 * move_tasks tries to move up to max_load_move weighted load from busiest to
 * this_rq, as part of a balancing operation within domain "sd".
 * Returns 1 if successful and 0 otherwise.
I
Ingo Molnar 已提交
3003 3004 3005 3006
 *
 * Called with both runqueues locked.
 */
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
P
Peter Williams 已提交
3007
		      unsigned long max_load_move,
I
Ingo Molnar 已提交
3008 3009 3010
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned)
{
3011
	const struct sched_class *class = sched_class_highest;
P
Peter Williams 已提交
3012
	unsigned long total_load_moved = 0;
3013
	int this_best_prio = this_rq->curr->prio;
I
Ingo Molnar 已提交
3014 3015

	do {
P
Peter Williams 已提交
3016 3017
		total_load_moved +=
			class->load_balance(this_rq, this_cpu, busiest,
3018
				max_load_move - total_load_moved,
3019
				sd, idle, all_pinned, &this_best_prio);
I
Ingo Molnar 已提交
3020
		class = class->next;
3021 3022 3023 3024

		if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
			break;

P
Peter Williams 已提交
3025
	} while (class && max_load_move > total_load_moved);
I
Ingo Molnar 已提交
3026

P
Peter Williams 已提交
3027 3028 3029
	return total_load_moved > 0;
}

3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055
static int
iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
		   struct sched_domain *sd, enum cpu_idle_type idle,
		   struct rq_iterator *iterator)
{
	struct task_struct *p = iterator->start(iterator->arg);
	int pinned = 0;

	while (p) {
		if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
			pull_task(busiest, p, this_rq, this_cpu);
			/*
			 * Right now, this is only the second place pull_task()
			 * is called, so we can safely collect pull_task()
			 * stats here rather than inside pull_task().
			 */
			schedstat_inc(sd, lb_gained[idle]);

			return 1;
		}
		p = iterator->next(iterator->arg);
	}

	return 0;
}

P
Peter Williams 已提交
3056 3057 3058 3059 3060 3061 3062 3063 3064 3065
/*
 * move_one_task tries to move exactly one task from busiest to this_rq, as
 * part of active balancing operations within "domain".
 * Returns 1 if successful and 0 otherwise.
 *
 * Called with both runqueues locked.
 */
static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
			 struct sched_domain *sd, enum cpu_idle_type idle)
{
3066
	const struct sched_class *class;
P
Peter Williams 已提交
3067 3068

	for (class = sched_class_highest; class; class = class->next)
3069
		if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
P
Peter Williams 已提交
3070 3071 3072
			return 1;

	return 0;
I
Ingo Molnar 已提交
3073 3074
}

L
Linus Torvalds 已提交
3075 3076
/*
 * find_busiest_group finds and returns the busiest CPU group within the
3077 3078
 * domain. It calculates and returns the amount of weighted load which
 * should be moved to restore balance via the imbalance parameter.
L
Linus Torvalds 已提交
3079 3080 3081
 */
static struct sched_group *
find_busiest_group(struct sched_domain *sd, int this_cpu,
I
Ingo Molnar 已提交
3082
		   unsigned long *imbalance, enum cpu_idle_type idle,
3083
		   int *sd_idle, const struct cpumask *cpus, int *balance)
L
Linus Torvalds 已提交
3084 3085 3086
{
	struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
	unsigned long max_load, avg_load, total_load, this_load, total_pwr;
3087
	unsigned long max_pull;
3088 3089
	unsigned long busiest_load_per_task, busiest_nr_running;
	unsigned long this_load_per_task, this_nr_running;
3090
	int load_idx, group_imb = 0;
3091 3092 3093 3094 3095 3096
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
	int power_savings_balance = 1;
	unsigned long leader_nr_running = 0, min_load_per_task = 0;
	unsigned long min_nr_running = ULONG_MAX;
	struct sched_group *group_min = NULL, *group_leader = NULL;
#endif
L
Linus Torvalds 已提交
3097 3098

	max_load = this_load = total_load = total_pwr = 0;
3099 3100
	busiest_load_per_task = busiest_nr_running = 0;
	this_load_per_task = this_nr_running = 0;
3101

I
Ingo Molnar 已提交
3102
	if (idle == CPU_NOT_IDLE)
N
Nick Piggin 已提交
3103
		load_idx = sd->busy_idx;
I
Ingo Molnar 已提交
3104
	else if (idle == CPU_NEWLY_IDLE)
N
Nick Piggin 已提交
3105 3106 3107
		load_idx = sd->newidle_idx;
	else
		load_idx = sd->idle_idx;
L
Linus Torvalds 已提交
3108 3109

	do {
3110
		unsigned long load, group_capacity, max_cpu_load, min_cpu_load;
L
Linus Torvalds 已提交
3111 3112
		int local_group;
		int i;
3113
		int __group_imb = 0;
3114
		unsigned int balance_cpu = -1, first_idle_cpu = 0;
3115
		unsigned long sum_nr_running, sum_weighted_load;
3116 3117
		unsigned long sum_avg_load_per_task;
		unsigned long avg_load_per_task;
L
Linus Torvalds 已提交
3118

3119 3120
		local_group = cpumask_test_cpu(this_cpu,
					       sched_group_cpus(group));
L
Linus Torvalds 已提交
3121

3122
		if (local_group)
3123
			balance_cpu = cpumask_first(sched_group_cpus(group));
3124

L
Linus Torvalds 已提交
3125
		/* Tally up the load of all CPUs in the group */
3126
		sum_weighted_load = sum_nr_running = avg_load = 0;
3127 3128
		sum_avg_load_per_task = avg_load_per_task = 0;

3129 3130
		max_cpu_load = 0;
		min_cpu_load = ~0UL;
L
Linus Torvalds 已提交
3131

3132 3133
		for_each_cpu_and(i, sched_group_cpus(group), cpus) {
			struct rq *rq = cpu_rq(i);
3134

3135
			if (*sd_idle && rq->nr_running)
N
Nick Piggin 已提交
3136 3137
				*sd_idle = 0;

L
Linus Torvalds 已提交
3138
			/* Bias balancing toward cpus of our domain */
3139 3140 3141 3142 3143 3144
			if (local_group) {
				if (idle_cpu(i) && !first_idle_cpu) {
					first_idle_cpu = 1;
					balance_cpu = i;
				}

N
Nick Piggin 已提交
3145
				load = target_load(i, load_idx);
3146
			} else {
N
Nick Piggin 已提交
3147
				load = source_load(i, load_idx);
3148 3149 3150 3151 3152
				if (load > max_cpu_load)
					max_cpu_load = load;
				if (min_cpu_load > load)
					min_cpu_load = load;
			}
L
Linus Torvalds 已提交
3153 3154

			avg_load += load;
3155
			sum_nr_running += rq->nr_running;
I
Ingo Molnar 已提交
3156
			sum_weighted_load += weighted_cpuload(i);
3157 3158

			sum_avg_load_per_task += cpu_avg_load_per_task(i);
L
Linus Torvalds 已提交
3159 3160
		}

3161 3162 3163
		/*
		 * First idle cpu or the first cpu(busiest) in this sched group
		 * is eligible for doing load balancing at this and above
3164 3165
		 * domains. In the newly idle case, we will allow all the cpu's
		 * to do the newly idle load balance.
3166
		 */
3167 3168
		if (idle != CPU_NEWLY_IDLE && local_group &&
		    balance_cpu != this_cpu && balance) {
3169 3170 3171 3172
			*balance = 0;
			goto ret;
		}

L
Linus Torvalds 已提交
3173
		total_load += avg_load;
3174
		total_pwr += group->__cpu_power;
L
Linus Torvalds 已提交
3175 3176

		/* Adjust by relative CPU power of the group */
3177 3178
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
3179

3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193

		/*
		 * Consider the group unbalanced when the imbalance is larger
		 * than the average weight of two tasks.
		 *
		 * APZ: with cgroup the avg task weight can vary wildly and
		 *      might not be a suitable number - should we keep a
		 *      normalized nr_running number somewhere that negates
		 *      the hierarchy?
		 */
		avg_load_per_task = sg_div_cpu_power(group,
				sum_avg_load_per_task * SCHED_LOAD_SCALE);

		if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
3194 3195
			__group_imb = 1;

3196
		group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
3197

L
Linus Torvalds 已提交
3198 3199 3200
		if (local_group) {
			this_load = avg_load;
			this = group;
3201 3202 3203
			this_nr_running = sum_nr_running;
			this_load_per_task = sum_weighted_load;
		} else if (avg_load > max_load &&
3204
			   (sum_nr_running > group_capacity || __group_imb)) {
L
Linus Torvalds 已提交
3205 3206
			max_load = avg_load;
			busiest = group;
3207 3208
			busiest_nr_running = sum_nr_running;
			busiest_load_per_task = sum_weighted_load;
3209
			group_imb = __group_imb;
L
Linus Torvalds 已提交
3210
		}
3211 3212 3213 3214 3215 3216

#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
		/*
		 * Busy processors will not participate in power savings
		 * balance.
		 */
I
Ingo Molnar 已提交
3217 3218 3219
		if (idle == CPU_NOT_IDLE ||
				!(sd->flags & SD_POWERSAVINGS_BALANCE))
			goto group_next;
3220 3221 3222 3223 3224 3225 3226 3227 3228

		/*
		 * If the local group is idle or completely loaded
		 * no need to do power savings balance at this domain
		 */
		if (local_group && (this_nr_running >= group_capacity ||
				    !this_nr_running))
			power_savings_balance = 0;

I
Ingo Molnar 已提交
3229
		/*
3230 3231
		 * If a group is already running at full capacity or idle,
		 * don't include that group in power savings calculations
I
Ingo Molnar 已提交
3232 3233
		 */
		if (!power_savings_balance || sum_nr_running >= group_capacity
3234
		    || !sum_nr_running)
I
Ingo Molnar 已提交
3235
			goto group_next;
3236

I
Ingo Molnar 已提交
3237
		/*
3238
		 * Calculate the group which has the least non-idle load.
I
Ingo Molnar 已提交
3239 3240 3241 3242 3243
		 * This is the group from where we need to pick up the load
		 * for saving power
		 */
		if ((sum_nr_running < min_nr_running) ||
		    (sum_nr_running == min_nr_running &&
3244 3245
		     cpumask_first(sched_group_cpus(group)) <
		     cpumask_first(sched_group_cpus(group_min)))) {
I
Ingo Molnar 已提交
3246 3247
			group_min = group;
			min_nr_running = sum_nr_running;
3248 3249
			min_load_per_task = sum_weighted_load /
						sum_nr_running;
I
Ingo Molnar 已提交
3250
		}
3251

I
Ingo Molnar 已提交
3252
		/*
3253
		 * Calculate the group which is almost near its
I
Ingo Molnar 已提交
3254 3255 3256 3257 3258 3259
		 * capacity but still has some space to pick up some load
		 * from other group and save more power
		 */
		if (sum_nr_running <= group_capacity - 1) {
			if (sum_nr_running > leader_nr_running ||
			    (sum_nr_running == leader_nr_running &&
3260 3261
			     cpumask_first(sched_group_cpus(group)) >
			     cpumask_first(sched_group_cpus(group_leader)))) {
I
Ingo Molnar 已提交
3262 3263 3264
				group_leader = group;
				leader_nr_running = sum_nr_running;
			}
3265
		}
3266 3267
group_next:
#endif
L
Linus Torvalds 已提交
3268 3269 3270
		group = group->next;
	} while (group != sd->groups);

3271
	if (!busiest || this_load >= max_load || busiest_nr_running == 0)
L
Linus Torvalds 已提交
3272 3273 3274 3275 3276 3277 3278 3279
		goto out_balanced;

	avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;

	if (this_load >= avg_load ||
			100*max_load <= sd->imbalance_pct*this_load)
		goto out_balanced;

3280
	busiest_load_per_task /= busiest_nr_running;
3281 3282 3283
	if (group_imb)
		busiest_load_per_task = min(busiest_load_per_task, avg_load);

L
Linus Torvalds 已提交
3284 3285 3286 3287 3288 3289 3290 3291
	/*
	 * We're trying to get all the cpus to the average_load, so we don't
	 * want to push ourselves above the average load, nor do we wish to
	 * reduce the max loaded cpu below the average load, as either of these
	 * actions would just result in more rebalancing later, and ping-pong
	 * tasks around. Thus we look for the minimum possible imbalance.
	 * Negative imbalances (*we* are more loaded than anyone else) will
	 * be counted as no imbalance for these purposes -- we can't fix that
I
Ingo Molnar 已提交
3292
	 * by pulling tasks to us. Be careful of negative numbers as they'll
L
Linus Torvalds 已提交
3293 3294
	 * appear as very large values with unsigned longs.
	 */
3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
	if (max_load <= busiest_load_per_task)
		goto out_balanced;

	/*
	 * In the presence of smp nice balancing, certain scenarios can have
	 * max load less than avg load(as we skip the groups at or below
	 * its cpu_power, while calculating max_load..)
	 */
	if (max_load < avg_load) {
		*imbalance = 0;
		goto small_imbalance;
	}
3307 3308

	/* Don't want to pull so many tasks that a group would go idle */
3309
	max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
3310

L
Linus Torvalds 已提交
3311
	/* How much load to actually move to equalise the imbalance */
3312 3313
	*imbalance = min(max_pull * busiest->__cpu_power,
				(avg_load - this_load) * this->__cpu_power)
L
Linus Torvalds 已提交
3314 3315
			/ SCHED_LOAD_SCALE;

3316 3317 3318 3319 3320 3321
	/*
	 * if *imbalance is less than the average load per runnable task
	 * there is no gaurantee that any tasks will be moved so we'll have
	 * a think about bumping its value to force at least one task to be
	 * moved
	 */
3322
	if (*imbalance < busiest_load_per_task) {
3323
		unsigned long tmp, pwr_now, pwr_move;
3324 3325 3326 3327 3328 3329 3330 3331 3332 3333
		unsigned int imbn;

small_imbalance:
		pwr_move = pwr_now = 0;
		imbn = 2;
		if (this_nr_running) {
			this_load_per_task /= this_nr_running;
			if (busiest_load_per_task > this_load_per_task)
				imbn = 1;
		} else
3334
			this_load_per_task = cpu_avg_load_per_task(this_cpu);
L
Linus Torvalds 已提交
3335

3336
		if (max_load - this_load + busiest_load_per_task >=
I
Ingo Molnar 已提交
3337
					busiest_load_per_task * imbn) {
3338
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
3339 3340 3341 3342 3343 3344 3345 3346 3347
			return busiest;
		}

		/*
		 * OK, we don't have enough imbalance to justify moving tasks,
		 * however we may be able to increase total CPU power used by
		 * moving them.
		 */

3348 3349 3350 3351
		pwr_now += busiest->__cpu_power *
				min(busiest_load_per_task, max_load);
		pwr_now += this->__cpu_power *
				min(this_load_per_task, this_load);
L
Linus Torvalds 已提交
3352 3353 3354
		pwr_now /= SCHED_LOAD_SCALE;

		/* Amount of load we'd subtract */
3355 3356
		tmp = sg_div_cpu_power(busiest,
				busiest_load_per_task * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
3357
		if (max_load > tmp)
3358
			pwr_move += busiest->__cpu_power *
3359
				min(busiest_load_per_task, max_load - tmp);
L
Linus Torvalds 已提交
3360 3361

		/* Amount of load we'd add */
3362
		if (max_load * busiest->__cpu_power <
3363
				busiest_load_per_task * SCHED_LOAD_SCALE)
3364 3365
			tmp = sg_div_cpu_power(this,
					max_load * busiest->__cpu_power);
L
Linus Torvalds 已提交
3366
		else
3367 3368 3369 3370
			tmp = sg_div_cpu_power(this,
				busiest_load_per_task * SCHED_LOAD_SCALE);
		pwr_move += this->__cpu_power *
				min(this_load_per_task, this_load + tmp);
L
Linus Torvalds 已提交
3371 3372 3373
		pwr_move /= SCHED_LOAD_SCALE;

		/* Move if we gain throughput */
3374 3375
		if (pwr_move > pwr_now)
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
3376 3377 3378 3379 3380
	}

	return busiest;

out_balanced:
3381
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
3382
	if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
3383
		goto ret;
L
Linus Torvalds 已提交
3384

3385 3386 3387 3388 3389
	if (this == group_leader && group_leader != group_min) {
		*imbalance = min_load_per_task;
		return group_min;
	}
#endif
3390
ret:
L
Linus Torvalds 已提交
3391 3392 3393 3394 3395 3396 3397
	*imbalance = 0;
	return NULL;
}

/*
 * find_busiest_queue - find the busiest runqueue among the cpus in group.
 */
3398
static struct rq *
I
Ingo Molnar 已提交
3399
find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3400
		   unsigned long imbalance, const struct cpumask *cpus)
L
Linus Torvalds 已提交
3401
{
3402
	struct rq *busiest = NULL, *rq;
3403
	unsigned long max_load = 0;
L
Linus Torvalds 已提交
3404 3405
	int i;

3406
	for_each_cpu(i, sched_group_cpus(group)) {
I
Ingo Molnar 已提交
3407
		unsigned long wl;
3408

3409
		if (!cpumask_test_cpu(i, cpus))
3410 3411
			continue;

3412
		rq = cpu_rq(i);
I
Ingo Molnar 已提交
3413
		wl = weighted_cpuload(i);
3414

I
Ingo Molnar 已提交
3415
		if (rq->nr_running == 1 && wl > imbalance)
3416
			continue;
L
Linus Torvalds 已提交
3417

I
Ingo Molnar 已提交
3418 3419
		if (wl > max_load) {
			max_load = wl;
3420
			busiest = rq;
L
Linus Torvalds 已提交
3421 3422 3423 3424 3425 3426
		}
	}

	return busiest;
}

3427 3428 3429 3430 3431 3432
/*
 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
 * so long as it is large enough.
 */
#define MAX_PINNED_INTERVAL	512

L
Linus Torvalds 已提交
3433 3434 3435 3436
/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
3437
static int load_balance(int this_cpu, struct rq *this_rq,
I
Ingo Molnar 已提交
3438
			struct sched_domain *sd, enum cpu_idle_type idle,
3439
			int *balance, struct cpumask *cpus)
L
Linus Torvalds 已提交
3440
{
P
Peter Williams 已提交
3441
	int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
L
Linus Torvalds 已提交
3442 3443
	struct sched_group *group;
	unsigned long imbalance;
3444
	struct rq *busiest;
3445
	unsigned long flags;
N
Nick Piggin 已提交
3446

3447
	cpumask_setall(cpus);
3448

3449 3450 3451
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
I
Ingo Molnar 已提交
3452
	 * let the state of idle sibling percolate up as CPU_IDLE, instead of
I
Ingo Molnar 已提交
3453
	 * portraying it as CPU_NOT_IDLE.
3454
	 */
I
Ingo Molnar 已提交
3455
	if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
3456
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3457
		sd_idle = 1;
L
Linus Torvalds 已提交
3458

3459
	schedstat_inc(sd, lb_count[idle]);
L
Linus Torvalds 已提交
3460

3461
redo:
3462
	update_shares(sd);
3463
	group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
3464
				   cpus, balance);
3465

3466
	if (*balance == 0)
3467 3468
		goto out_balanced;

L
Linus Torvalds 已提交
3469 3470 3471 3472 3473
	if (!group) {
		schedstat_inc(sd, lb_nobusyg[idle]);
		goto out_balanced;
	}

3474
	busiest = find_busiest_queue(group, idle, imbalance, cpus);
L
Linus Torvalds 已提交
3475 3476 3477 3478 3479
	if (!busiest) {
		schedstat_inc(sd, lb_nobusyq[idle]);
		goto out_balanced;
	}

N
Nick Piggin 已提交
3480
	BUG_ON(busiest == this_rq);
L
Linus Torvalds 已提交
3481 3482 3483

	schedstat_add(sd, lb_imbalance[idle], imbalance);

P
Peter Williams 已提交
3484
	ld_moved = 0;
L
Linus Torvalds 已提交
3485 3486 3487 3488
	if (busiest->nr_running > 1) {
		/*
		 * Attempt to move tasks. If find_busiest_group has found
		 * an imbalance but busiest->nr_running <= 1, the group is
P
Peter Williams 已提交
3489
		 * still unbalanced. ld_moved simply stays zero, so it is
L
Linus Torvalds 已提交
3490 3491
		 * correctly treated as an imbalance.
		 */
3492
		local_irq_save(flags);
N
Nick Piggin 已提交
3493
		double_rq_lock(this_rq, busiest);
P
Peter Williams 已提交
3494
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
3495
				      imbalance, sd, idle, &all_pinned);
N
Nick Piggin 已提交
3496
		double_rq_unlock(this_rq, busiest);
3497
		local_irq_restore(flags);
3498

3499 3500 3501
		/*
		 * some other cpu did the load balance for us.
		 */
P
Peter Williams 已提交
3502
		if (ld_moved && this_cpu != smp_processor_id())
3503 3504
			resched_cpu(this_cpu);

3505
		/* All tasks on this runqueue were pinned by CPU affinity */
3506
		if (unlikely(all_pinned)) {
3507 3508
			cpumask_clear_cpu(cpu_of(busiest), cpus);
			if (!cpumask_empty(cpus))
3509
				goto redo;
3510
			goto out_balanced;
3511
		}
L
Linus Torvalds 已提交
3512
	}
3513

P
Peter Williams 已提交
3514
	if (!ld_moved) {
L
Linus Torvalds 已提交
3515 3516 3517 3518 3519
		schedstat_inc(sd, lb_failed[idle]);
		sd->nr_balance_failed++;

		if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {

3520
			spin_lock_irqsave(&busiest->lock, flags);
3521 3522 3523 3524

			/* don't kick the migration_thread, if the curr
			 * task on busiest cpu can't be moved to this_cpu
			 */
3525 3526
			if (!cpumask_test_cpu(this_cpu,
					      &busiest->curr->cpus_allowed)) {
3527
				spin_unlock_irqrestore(&busiest->lock, flags);
3528 3529 3530 3531
				all_pinned = 1;
				goto out_one_pinned;
			}

L
Linus Torvalds 已提交
3532 3533 3534
			if (!busiest->active_balance) {
				busiest->active_balance = 1;
				busiest->push_cpu = this_cpu;
3535
				active_balance = 1;
L
Linus Torvalds 已提交
3536
			}
3537
			spin_unlock_irqrestore(&busiest->lock, flags);
3538
			if (active_balance)
L
Linus Torvalds 已提交
3539 3540 3541 3542 3543 3544
				wake_up_process(busiest->migration_thread);

			/*
			 * We've kicked active balancing, reset the failure
			 * counter.
			 */
3545
			sd->nr_balance_failed = sd->cache_nice_tries+1;
L
Linus Torvalds 已提交
3546
		}
3547
	} else
L
Linus Torvalds 已提交
3548 3549
		sd->nr_balance_failed = 0;

3550
	if (likely(!active_balance)) {
L
Linus Torvalds 已提交
3551 3552
		/* We were unbalanced, so reset the balancing interval */
		sd->balance_interval = sd->min_interval;
3553 3554 3555 3556 3557 3558 3559 3560 3561
	} else {
		/*
		 * If we've begun active balancing, start to back off. This
		 * case may not be covered by the all_pinned logic if there
		 * is only 1 task on the busy runqueue (because we don't call
		 * move_tasks).
		 */
		if (sd->balance_interval < sd->max_interval)
			sd->balance_interval *= 2;
L
Linus Torvalds 已提交
3562 3563
	}

P
Peter Williams 已提交
3564
	if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3565
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3566 3567 3568
		ld_moved = -1;

	goto out;
L
Linus Torvalds 已提交
3569 3570 3571 3572

out_balanced:
	schedstat_inc(sd, lb_balanced[idle]);

3573
	sd->nr_balance_failed = 0;
3574 3575

out_one_pinned:
L
Linus Torvalds 已提交
3576
	/* tune up the balancing interval */
3577 3578
	if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
			(sd->balance_interval < sd->max_interval))
L
Linus Torvalds 已提交
3579 3580
		sd->balance_interval *= 2;

3581
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3582
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3583 3584 3585 3586
		ld_moved = -1;
	else
		ld_moved = 0;
out:
3587 3588
	if (ld_moved)
		update_shares(sd);
3589
	return ld_moved;
L
Linus Torvalds 已提交
3590 3591 3592 3593 3594 3595
}

/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 *
I
Ingo Molnar 已提交
3596
 * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
L
Linus Torvalds 已提交
3597 3598
 * this_rq is locked.
 */
3599
static int
3600
load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3601
			struct cpumask *cpus)
L
Linus Torvalds 已提交
3602 3603
{
	struct sched_group *group;
3604
	struct rq *busiest = NULL;
L
Linus Torvalds 已提交
3605
	unsigned long imbalance;
P
Peter Williams 已提交
3606
	int ld_moved = 0;
N
Nick Piggin 已提交
3607
	int sd_idle = 0;
3608
	int all_pinned = 0;
3609

3610
	cpumask_setall(cpus);
N
Nick Piggin 已提交
3611

3612 3613 3614 3615
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
	 * let the state of idle sibling percolate up as IDLE, instead of
I
Ingo Molnar 已提交
3616
	 * portraying it as CPU_NOT_IDLE.
3617 3618 3619
	 */
	if (sd->flags & SD_SHARE_CPUPOWER &&
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3620
		sd_idle = 1;
L
Linus Torvalds 已提交
3621

3622
	schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
3623
redo:
3624
	update_shares_locked(this_rq, sd);
I
Ingo Molnar 已提交
3625
	group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
3626
				   &sd_idle, cpus, NULL);
L
Linus Torvalds 已提交
3627
	if (!group) {
I
Ingo Molnar 已提交
3628
		schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
3629
		goto out_balanced;
L
Linus Torvalds 已提交
3630 3631
	}

3632
	busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance, cpus);
N
Nick Piggin 已提交
3633
	if (!busiest) {
I
Ingo Molnar 已提交
3634
		schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
3635
		goto out_balanced;
L
Linus Torvalds 已提交
3636 3637
	}

N
Nick Piggin 已提交
3638 3639
	BUG_ON(busiest == this_rq);

I
Ingo Molnar 已提交
3640
	schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
3641

P
Peter Williams 已提交
3642
	ld_moved = 0;
3643 3644 3645
	if (busiest->nr_running > 1) {
		/* Attempt to move tasks */
		double_lock_balance(this_rq, busiest);
3646 3647
		/* this_rq->clock is already updated */
		update_rq_clock(busiest);
P
Peter Williams 已提交
3648
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
3649 3650
					imbalance, sd, CPU_NEWLY_IDLE,
					&all_pinned);
3651
		double_unlock_balance(this_rq, busiest);
3652

3653
		if (unlikely(all_pinned)) {
3654 3655
			cpumask_clear_cpu(cpu_of(busiest), cpus);
			if (!cpumask_empty(cpus))
3656 3657
				goto redo;
		}
3658 3659
	}

P
Peter Williams 已提交
3660
	if (!ld_moved) {
I
Ingo Molnar 已提交
3661
		schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3662 3663
		if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
		    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3664 3665
			return -1;
	} else
3666
		sd->nr_balance_failed = 0;
L
Linus Torvalds 已提交
3667

3668
	update_shares_locked(this_rq, sd);
P
Peter Williams 已提交
3669
	return ld_moved;
3670 3671

out_balanced:
I
Ingo Molnar 已提交
3672
	schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
3673
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3674
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
3675
		return -1;
3676
	sd->nr_balance_failed = 0;
3677

3678
	return 0;
L
Linus Torvalds 已提交
3679 3680 3681 3682 3683 3684
}

/*
 * idle_balance is called by schedule() if this_cpu is about to become
 * idle. Attempts to pull tasks from other CPUs.
 */
3685
static void idle_balance(int this_cpu, struct rq *this_rq)
L
Linus Torvalds 已提交
3686 3687
{
	struct sched_domain *sd;
3688
	int pulled_task = 0;
I
Ingo Molnar 已提交
3689
	unsigned long next_balance = jiffies + HZ;
3690 3691 3692 3693
	cpumask_var_t tmpmask;

	if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
		return;
L
Linus Torvalds 已提交
3694 3695

	for_each_domain(this_cpu, sd) {
3696 3697 3698 3699 3700 3701
		unsigned long interval;

		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		if (sd->flags & SD_BALANCE_NEWIDLE)
3702
			/* If we've pulled tasks over stop searching: */
3703
			pulled_task = load_balance_newidle(this_cpu, this_rq,
3704
							   sd, tmpmask);
3705 3706 3707 3708 3709 3710

		interval = msecs_to_jiffies(sd->balance_interval);
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
		if (pulled_task)
			break;
L
Linus Torvalds 已提交
3711
	}
I
Ingo Molnar 已提交
3712
	if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
3713 3714 3715 3716 3717
		/*
		 * We are going idle. next_balance may be set based on
		 * a busy processor. So reset next_balance.
		 */
		this_rq->next_balance = next_balance;
I
Ingo Molnar 已提交
3718
	}
3719
	free_cpumask_var(tmpmask);
L
Linus Torvalds 已提交
3720 3721 3722 3723 3724 3725 3726 3727 3728 3729
}

/*
 * active_load_balance is run by migration threads. It pushes running tasks
 * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
 * running on each physical CPU where possible, and avoids physical /
 * logical imbalances.
 *
 * Called with busiest_rq locked.
 */
3730
static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
L
Linus Torvalds 已提交
3731
{
3732
	int target_cpu = busiest_rq->push_cpu;
3733 3734
	struct sched_domain *sd;
	struct rq *target_rq;
3735

3736
	/* Is there any task to move? */
3737 3738 3739 3740
	if (busiest_rq->nr_running <= 1)
		return;

	target_rq = cpu_rq(target_cpu);
L
Linus Torvalds 已提交
3741 3742

	/*
3743
	 * This condition is "impossible", if it occurs
I
Ingo Molnar 已提交
3744
	 * we need to fix it. Originally reported by
3745
	 * Bjorn Helgaas on a 128-cpu setup.
L
Linus Torvalds 已提交
3746
	 */
3747
	BUG_ON(busiest_rq == target_rq);
L
Linus Torvalds 已提交
3748

3749 3750
	/* move a task from busiest_rq to target_rq */
	double_lock_balance(busiest_rq, target_rq);
3751 3752
	update_rq_clock(busiest_rq);
	update_rq_clock(target_rq);
3753 3754

	/* Search for an sd spanning us and the target CPU. */
3755
	for_each_domain(target_cpu, sd) {
3756
		if ((sd->flags & SD_LOAD_BALANCE) &&
3757
		    cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3758
				break;
3759
	}
3760

3761
	if (likely(sd)) {
3762
		schedstat_inc(sd, alb_count);
3763

P
Peter Williams 已提交
3764 3765
		if (move_one_task(target_rq, target_cpu, busiest_rq,
				  sd, CPU_IDLE))
3766 3767 3768 3769
			schedstat_inc(sd, alb_pushed);
		else
			schedstat_inc(sd, alb_failed);
	}
3770
	double_unlock_balance(busiest_rq, target_rq);
L
Linus Torvalds 已提交
3771 3772
}

3773 3774 3775
#ifdef CONFIG_NO_HZ
static struct {
	atomic_t load_balancer;
3776
	cpumask_var_t cpu_mask;
3777 3778 3779 3780
} nohz ____cacheline_aligned = {
	.load_balancer = ATOMIC_INIT(-1),
};

3781
/*
3782 3783 3784 3785 3786 3787 3788 3789 3790 3791
 * This routine will try to nominate the ilb (idle load balancing)
 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
 * load balancing on behalf of all those cpus. If all the cpus in the system
 * go into this tickless mode, then there will be no ilb owner (as there is
 * no need for one) and all the cpus will sleep till the next wakeup event
 * arrives...
 *
 * For the ilb owner, tick is not stopped. And this tick will be used
 * for idle load balancing. ilb owner will still be part of
 * nohz.cpu_mask..
3792
 *
3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
 * While stopping the tick, this cpu will become the ilb owner if there
 * is no other owner. And will be the owner till that cpu becomes busy
 * or if all cpus in the system stop their ticks at which point
 * there is no need for ilb owner.
 *
 * When the ilb owner becomes busy, it nominates another owner, during the
 * next busy scheduler_tick()
 */
int select_nohz_load_balancer(int stop_tick)
{
	int cpu = smp_processor_id();

	if (stop_tick) {
3806
		cpumask_set_cpu(cpu, nohz.cpu_mask);
3807 3808 3809 3810 3811
		cpu_rq(cpu)->in_nohz_recently = 1;

		/*
		 * If we are going offline and still the leader, give up!
		 */
3812
		if (!cpu_active(cpu) &&
3813 3814 3815 3816 3817 3818 3819
		    atomic_read(&nohz.load_balancer) == cpu) {
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
			return 0;
		}

		/* time for ilb owner also to sleep */
3820
		if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832
			if (atomic_read(&nohz.load_balancer) == cpu)
				atomic_set(&nohz.load_balancer, -1);
			return 0;
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/* make me the ilb owner */
			if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
				return 1;
		} else if (atomic_read(&nohz.load_balancer) == cpu)
			return 1;
	} else {
3833
		if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3834 3835
			return 0;

3836
		cpumask_clear_cpu(cpu, nohz.cpu_mask);
3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848

		if (atomic_read(&nohz.load_balancer) == cpu)
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
	}
	return 0;
}
#endif

static DEFINE_SPINLOCK(balancing);

/*
3849 3850 3851 3852 3853
 * It checks each scheduling domain to see if it is due to be balanced,
 * and initiates a balancing operation if so.
 *
 * Balancing parameters are set up in arch_init_sched_domains.
 */
A
Alexey Dobriyan 已提交
3854
static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3855
{
3856 3857
	int balance = 1;
	struct rq *rq = cpu_rq(cpu);
3858 3859
	unsigned long interval;
	struct sched_domain *sd;
3860
	/* Earliest time when we have to do rebalance again */
3861
	unsigned long next_balance = jiffies + 60*HZ;
3862
	int update_next_balance = 0;
3863
	int need_serialize;
3864 3865 3866 3867 3868
	cpumask_var_t tmp;

	/* Fails alloc?  Rebalancing probably not a priority right now. */
	if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
		return;
L
Linus Torvalds 已提交
3869

3870
	for_each_domain(cpu, sd) {
L
Linus Torvalds 已提交
3871 3872 3873 3874
		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		interval = sd->balance_interval;
I
Ingo Molnar 已提交
3875
		if (idle != CPU_IDLE)
L
Linus Torvalds 已提交
3876 3877 3878 3879 3880 3881
			interval *= sd->busy_factor;

		/* scale ms to jiffies */
		interval = msecs_to_jiffies(interval);
		if (unlikely(!interval))
			interval = 1;
I
Ingo Molnar 已提交
3882 3883 3884
		if (interval > HZ*NR_CPUS/10)
			interval = HZ*NR_CPUS/10;

3885
		need_serialize = sd->flags & SD_SERIALIZE;
L
Linus Torvalds 已提交
3886

3887
		if (need_serialize) {
3888 3889 3890 3891
			if (!spin_trylock(&balancing))
				goto out;
		}

3892
		if (time_after_eq(jiffies, sd->last_balance + interval)) {
3893
			if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3894 3895
				/*
				 * We've pulled tasks over so either we're no
N
Nick Piggin 已提交
3896 3897 3898
				 * longer idle, or one of our SMT siblings is
				 * not idle.
				 */
I
Ingo Molnar 已提交
3899
				idle = CPU_NOT_IDLE;
L
Linus Torvalds 已提交
3900
			}
3901
			sd->last_balance = jiffies;
L
Linus Torvalds 已提交
3902
		}
3903
		if (need_serialize)
3904 3905
			spin_unlock(&balancing);
out:
3906
		if (time_after(next_balance, sd->last_balance + interval)) {
3907
			next_balance = sd->last_balance + interval;
3908 3909
			update_next_balance = 1;
		}
3910 3911 3912 3913 3914 3915 3916 3917

		/*
		 * Stop the load balance at this level. There is another
		 * CPU in our sched group which is doing load balancing more
		 * actively.
		 */
		if (!balance)
			break;
L
Linus Torvalds 已提交
3918
	}
3919 3920 3921 3922 3923 3924 3925 3926

	/*
	 * next_balance will be updated only when there is a need.
	 * When the cpu is attached to null domain for ex, it will not be
	 * updated.
	 */
	if (likely(update_next_balance))
		rq->next_balance = next_balance;
3927 3928

	free_cpumask_var(tmp);
3929 3930 3931 3932 3933 3934 3935 3936 3937
}

/*
 * run_rebalance_domains is triggered when needed from the scheduler tick.
 * In CONFIG_NO_HZ case, the idle load balance owner will do the
 * rebalancing for all the cpus for whom scheduler ticks are stopped.
 */
static void run_rebalance_domains(struct softirq_action *h)
{
I
Ingo Molnar 已提交
3938 3939 3940 3941
	int this_cpu = smp_processor_id();
	struct rq *this_rq = cpu_rq(this_cpu);
	enum cpu_idle_type idle = this_rq->idle_at_tick ?
						CPU_IDLE : CPU_NOT_IDLE;
3942

I
Ingo Molnar 已提交
3943
	rebalance_domains(this_cpu, idle);
3944 3945 3946 3947 3948 3949 3950

#ifdef CONFIG_NO_HZ
	/*
	 * If this cpu is the owner for idle load balancing, then do the
	 * balancing on behalf of the other idle cpus whose ticks are
	 * stopped.
	 */
I
Ingo Molnar 已提交
3951 3952
	if (this_rq->idle_at_tick &&
	    atomic_read(&nohz.load_balancer) == this_cpu) {
3953 3954 3955
		struct rq *rq;
		int balance_cpu;

3956 3957 3958 3959
		for_each_cpu(balance_cpu, nohz.cpu_mask) {
			if (balance_cpu == this_cpu)
				continue;

3960 3961 3962 3963 3964 3965 3966 3967
			/*
			 * If this cpu gets work to do, stop the load balancing
			 * work being done for other cpus. Next load
			 * balancing owner will pick it up.
			 */
			if (need_resched())
				break;

3968
			rebalance_domains(balance_cpu, CPU_IDLE);
3969 3970

			rq = cpu_rq(balance_cpu);
I
Ingo Molnar 已提交
3971 3972
			if (time_after(this_rq->next_balance, rq->next_balance))
				this_rq->next_balance = rq->next_balance;
3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984
		}
	}
#endif
}

/*
 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
 *
 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
 * idle load balancing owner or decide to stop the periodic load balancing,
 * if the whole system is idle.
 */
I
Ingo Molnar 已提交
3985
static inline void trigger_load_balance(struct rq *rq, int cpu)
3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996
{
#ifdef CONFIG_NO_HZ
	/*
	 * If we were in the nohz mode recently and busy at the current
	 * scheduler tick, then check if we need to nominate new idle
	 * load balancer.
	 */
	if (rq->in_nohz_recently && !rq->idle_at_tick) {
		rq->in_nohz_recently = 0;

		if (atomic_read(&nohz.load_balancer) == cpu) {
3997
			cpumask_clear_cpu(cpu, nohz.cpu_mask);
3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009
			atomic_set(&nohz.load_balancer, -1);
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/*
			 * simple selection for now: Nominate the
			 * first cpu in the nohz list to be the next
			 * ilb owner.
			 *
			 * TBD: Traverse the sched domains and nominate
			 * the nearest cpu in the nohz.cpu_mask.
			 */
4010
			int ilb = cpumask_first(nohz.cpu_mask);
4011

4012
			if (ilb < nr_cpu_ids)
4013 4014 4015 4016 4017 4018 4019 4020 4021
				resched_cpu(ilb);
		}
	}

	/*
	 * If this cpu is idle and doing idle load balancing for all the
	 * cpus with ticks stopped, is it time for that to stop?
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4022
	    cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4023 4024 4025 4026 4027 4028 4029 4030 4031
		resched_cpu(cpu);
		return;
	}

	/*
	 * If this cpu is idle and the idle load balancing is done by
	 * someone else, then no need raise the SCHED_SOFTIRQ
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4032
	    cpumask_test_cpu(cpu, nohz.cpu_mask))
4033 4034 4035 4036
		return;
#endif
	if (time_after_eq(jiffies, rq->next_balance))
		raise_softirq(SCHED_SOFTIRQ);
L
Linus Torvalds 已提交
4037
}
I
Ingo Molnar 已提交
4038 4039 4040

#else	/* CONFIG_SMP */

L
Linus Torvalds 已提交
4041 4042 4043
/*
 * on UP we do not need to balance between CPUs:
 */
4044
static inline void idle_balance(int cpu, struct rq *rq)
L
Linus Torvalds 已提交
4045 4046
{
}
I
Ingo Molnar 已提交
4047

L
Linus Torvalds 已提交
4048 4049 4050 4051 4052 4053 4054
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
4055 4056
 * Return any ns on the sched_clock that have not yet been banked in
 * @p in case that task is currently running.
L
Linus Torvalds 已提交
4057
 */
4058
unsigned long long task_delta_exec(struct task_struct *p)
L
Linus Torvalds 已提交
4059 4060
{
	unsigned long flags;
4061
	struct rq *rq;
4062
	u64 ns = 0;
4063

4064
	rq = task_rq_lock(p, &flags);
4065

4066
	if (task_current(rq, p)) {
4067 4068
		u64 delta_exec;

I
Ingo Molnar 已提交
4069 4070
		update_rq_clock(rq);
		delta_exec = rq->clock - p->se.exec_start;
4071
		if ((s64)delta_exec > 0)
4072
			ns = delta_exec;
4073
	}
4074

4075
	task_rq_unlock(rq, &flags);
4076

L
Linus Torvalds 已提交
4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090
	return ns;
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in user space since the last update
 */
void account_user_time(struct task_struct *p, cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

	p->utime = cputime_add(p->utime, cputime);
4091
	account_group_user_time(p, cputime);
L
Linus Torvalds 已提交
4092 4093 4094 4095 4096 4097 4098

	/* Add user time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (TASK_NICE(p) > 0)
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
	else
		cpustat->user = cputime64_add(cpustat->user, tmp);
4099 4100
	/* Account for user time used */
	acct_update_integrals(p);
L
Linus Torvalds 已提交
4101 4102
}

4103 4104 4105 4106 4107
/*
 * Account guest cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in virtual machine since the last update
 */
4108
static void account_guest_time(struct task_struct *p, cputime_t cputime)
4109 4110 4111 4112 4113 4114 4115
{
	cputime64_t tmp;
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;

	tmp = cputime_to_cputime64(cputime);

	p->utime = cputime_add(p->utime, cputime);
4116
	account_group_user_time(p, cputime);
4117 4118 4119 4120 4121 4122
	p->gtime = cputime_add(p->gtime, cputime);

	cpustat->user = cputime64_add(cpustat->user, tmp);
	cpustat->guest = cputime64_add(cpustat->guest, tmp);
}

4123 4124 4125 4126 4127 4128 4129 4130 4131 4132
/*
 * Account scaled user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in user space since the last update
 */
void account_user_time_scaled(struct task_struct *p, cputime_t cputime)
{
	p->utimescaled = cputime_add(p->utimescaled, cputime);
}

L
Linus Torvalds 已提交
4133 4134 4135 4136 4137 4138 4139 4140 4141 4142
/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
 */
void account_system_time(struct task_struct *p, int hardirq_offset,
			 cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4143
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
4144 4145
	cputime64_t tmp;

4146 4147 4148 4149
	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
		account_guest_time(p, cputime);
		return;
	}
4150

L
Linus Torvalds 已提交
4151
	p->stime = cputime_add(p->stime, cputime);
4152
	account_group_system_time(p, cputime);
L
Linus Torvalds 已提交
4153 4154 4155 4156 4157 4158 4159

	/* Add system time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (hardirq_count() - hardirq_offset)
		cpustat->irq = cputime64_add(cpustat->irq, tmp);
	else if (softirq_count())
		cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
4160
	else if (p != rq->idle)
L
Linus Torvalds 已提交
4161
		cpustat->system = cputime64_add(cpustat->system, tmp);
4162
	else if (atomic_read(&rq->nr_iowait) > 0)
L
Linus Torvalds 已提交
4163 4164 4165 4166 4167 4168 4169
		cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
	else
		cpustat->idle = cputime64_add(cpustat->idle, tmp);
	/* Account for system time used */
	acct_update_integrals(p);
}

4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180
/*
 * Account scaled system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
 */
void account_system_time_scaled(struct task_struct *p, cputime_t cputime)
{
	p->stimescaled = cputime_add(p->stimescaled, cputime);
}

L
Linus Torvalds 已提交
4181 4182 4183 4184 4185 4186 4187 4188 4189
/*
 * Account for involuntary wait time.
 * @p: the process from which the cpu time has been stolen
 * @steal: the cpu time spent in involuntary wait
 */
void account_steal_time(struct task_struct *p, cputime_t steal)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp = cputime_to_cputime64(steal);
4190
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
4191 4192 4193 4194 4195 4196 4197

	if (p == rq->idle) {
		p->stime = cputime_add(p->stime, steal);
		if (atomic_read(&rq->nr_iowait) > 0)
			cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
		else
			cpustat->idle = cputime64_add(cpustat->idle, tmp);
4198
	} else
L
Linus Torvalds 已提交
4199 4200 4201
		cpustat->steal = cputime64_add(cpustat->steal, tmp);
}

4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260
/*
 * Use precise platform statistics if available:
 */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
cputime_t task_utime(struct task_struct *p)
{
	return p->utime;
}

cputime_t task_stime(struct task_struct *p)
{
	return p->stime;
}
#else
cputime_t task_utime(struct task_struct *p)
{
	clock_t utime = cputime_to_clock_t(p->utime),
		total = utime + cputime_to_clock_t(p->stime);
	u64 temp;

	/*
	 * Use CFS's precise accounting:
	 */
	temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime);

	if (total) {
		temp *= utime;
		do_div(temp, total);
	}
	utime = (clock_t)temp;

	p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime));
	return p->prev_utime;
}

cputime_t task_stime(struct task_struct *p)
{
	clock_t stime;

	/*
	 * Use CFS's precise accounting. (we subtract utime from
	 * the total, to make sure the total observed by userspace
	 * grows monotonically - apps rely on that):
	 */
	stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
			cputime_to_clock_t(task_utime(p));

	if (stime >= 0)
		p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime));

	return p->prev_stime;
}
#endif

inline cputime_t task_gtime(struct task_struct *p)
{
	return p->gtime;
}

4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 *
 * It also gets called by the fork code, when changing the parent's
 * timeslices.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
4272
	struct task_struct *curr = rq->curr;
4273 4274

	sched_clock_tick();
I
Ingo Molnar 已提交
4275 4276

	spin_lock(&rq->lock);
4277
	update_rq_clock(rq);
4278
	update_cpu_load(rq);
P
Peter Zijlstra 已提交
4279
	curr->sched_class->task_tick(rq, curr, 0);
I
Ingo Molnar 已提交
4280
	spin_unlock(&rq->lock);
4281

4282
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
4283 4284
	rq->idle_at_tick = idle_cpu(cpu);
	trigger_load_balance(rq, cpu);
4285
#endif
L
Linus Torvalds 已提交
4286 4287
}

4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
				defined(CONFIG_PREEMPT_TRACER))

static inline unsigned long get_parent_ip(unsigned long addr)
{
	if (in_lock_functions(addr)) {
		addr = CALLER_ADDR2;
		if (in_lock_functions(addr))
			addr = CALLER_ADDR3;
	}
	return addr;
}
L
Linus Torvalds 已提交
4300

4301
void __kprobes add_preempt_count(int val)
L
Linus Torvalds 已提交
4302
{
4303
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4304 4305 4306
	/*
	 * Underflow?
	 */
4307 4308
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
4309
#endif
L
Linus Torvalds 已提交
4310
	preempt_count() += val;
4311
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4312 4313 4314
	/*
	 * Spinlock count overflowing soon?
	 */
4315 4316
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
4317 4318 4319
#endif
	if (preempt_count() == val)
		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
4320 4321 4322
}
EXPORT_SYMBOL(add_preempt_count);

4323
void __kprobes sub_preempt_count(int val)
L
Linus Torvalds 已提交
4324
{
4325
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
4326 4327 4328
	/*
	 * Underflow?
	 */
N
Nick Piggin 已提交
4329
       if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
4330
		return;
L
Linus Torvalds 已提交
4331 4332 4333
	/*
	 * Is the spinlock portion underflowing?
	 */
4334 4335 4336
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;
4337
#endif
4338

4339 4340
	if (preempt_count() == val)
		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
4341 4342 4343 4344 4345 4346 4347
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
4348
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
4349
 */
I
Ingo Molnar 已提交
4350
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
4351
{
4352 4353 4354 4355 4356
	struct pt_regs *regs = get_irq_regs();

	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
		prev->comm, prev->pid, preempt_count());

I
Ingo Molnar 已提交
4357
	debug_show_held_locks(prev);
4358
	print_modules();
I
Ingo Molnar 已提交
4359 4360
	if (irqs_disabled())
		print_irqtrace_events(prev);
4361 4362 4363 4364 4365

	if (regs)
		show_regs(regs);
	else
		dump_stack();
I
Ingo Molnar 已提交
4366
}
L
Linus Torvalds 已提交
4367

I
Ingo Molnar 已提交
4368 4369 4370 4371 4372
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
4373
	/*
I
Ingo Molnar 已提交
4374
	 * Test if we are atomic. Since do_exit() needs to call into
L
Linus Torvalds 已提交
4375 4376 4377
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
4378
	if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
I
Ingo Molnar 已提交
4379 4380
		__schedule_bug(prev);

L
Linus Torvalds 已提交
4381 4382
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

4383
	schedstat_inc(this_rq(), sched_count);
I
Ingo Molnar 已提交
4384 4385
#ifdef CONFIG_SCHEDSTATS
	if (unlikely(prev->lock_depth >= 0)) {
4386 4387
		schedstat_inc(this_rq(), bkl_count);
		schedstat_inc(prev, sched_info.bkl_count);
I
Ingo Molnar 已提交
4388 4389
	}
#endif
I
Ingo Molnar 已提交
4390 4391 4392 4393 4394 4395
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
4396
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
4397
{
4398
	const struct sched_class *class;
I
Ingo Molnar 已提交
4399
	struct task_struct *p;
L
Linus Torvalds 已提交
4400 4401

	/*
I
Ingo Molnar 已提交
4402 4403
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
4404
	 */
I
Ingo Molnar 已提交
4405
	if (likely(rq->nr_running == rq->cfs.nr_running)) {
4406
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
4407 4408
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
4409 4410
	}

I
Ingo Molnar 已提交
4411 4412
	class = sched_class_highest;
	for ( ; ; ) {
4413
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
4414 4415 4416 4417 4418 4419 4420 4421 4422
		if (p)
			return p;
		/*
		 * Will never be NULL as the idle class always
		 * returns a non-NULL p:
		 */
		class = class->next;
	}
}
L
Linus Torvalds 已提交
4423

I
Ingo Molnar 已提交
4424 4425 4426 4427 4428 4429
/*
 * schedule() is the main scheduler function.
 */
asmlinkage void __sched schedule(void)
{
	struct task_struct *prev, *next;
4430
	unsigned long *switch_count;
I
Ingo Molnar 已提交
4431
	struct rq *rq;
4432
	int cpu;
I
Ingo Molnar 已提交
4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445

need_resched:
	preempt_disable();
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
	rcu_qsctr_inc(cpu);
	prev = rq->curr;
	switch_count = &prev->nivcsw;

	release_kernel_lock(prev);
need_resched_nonpreemptible:

	schedule_debug(prev);
L
Linus Torvalds 已提交
4446

4447
	if (sched_feat(HRTICK))
M
Mike Galbraith 已提交
4448
		hrtick_clear(rq);
P
Peter Zijlstra 已提交
4449

4450
	spin_lock_irq(&rq->lock);
4451
	update_rq_clock(rq);
4452
	clear_tsk_need_resched(prev);
L
Linus Torvalds 已提交
4453 4454

	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
4455
		if (unlikely(signal_pending_state(prev->state, prev)))
L
Linus Torvalds 已提交
4456
			prev->state = TASK_RUNNING;
4457
		else
4458
			deactivate_task(rq, prev, 1);
I
Ingo Molnar 已提交
4459
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
4460 4461
	}

4462 4463 4464 4465
#ifdef CONFIG_SMP
	if (prev->sched_class->pre_schedule)
		prev->sched_class->pre_schedule(rq, prev);
#endif
4466

I
Ingo Molnar 已提交
4467
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
4468 4469
		idle_balance(cpu, rq);

4470
	prev->sched_class->put_prev_task(rq, prev);
4471
	next = pick_next_task(rq, prev);
L
Linus Torvalds 已提交
4472 4473

	if (likely(prev != next)) {
4474 4475
		sched_info_switch(prev, next);

L
Linus Torvalds 已提交
4476 4477 4478 4479
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
4480
		context_switch(rq, prev, next); /* unlocks the rq */
P
Peter Zijlstra 已提交
4481 4482 4483 4484 4485 4486
		/*
		 * the context switch might have flipped the stack from under
		 * us, hence refresh the local variables.
		 */
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4487 4488 4489
	} else
		spin_unlock_irq(&rq->lock);

P
Peter Zijlstra 已提交
4490
	if (unlikely(reacquire_kernel_lock(current) < 0))
L
Linus Torvalds 已提交
4491
		goto need_resched_nonpreemptible;
P
Peter Zijlstra 已提交
4492

L
Linus Torvalds 已提交
4493 4494 4495 4496 4497 4498 4499 4500
	preempt_enable_no_resched();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(schedule);

#ifdef CONFIG_PREEMPT
/*
4501
 * this is the entry point to schedule() from in-kernel preemption
I
Ingo Molnar 已提交
4502
 * off of preempt_enable. Kernel preemptions off return from interrupt
L
Linus Torvalds 已提交
4503 4504 4505 4506 4507
 * occur there and call schedule directly.
 */
asmlinkage void __sched preempt_schedule(void)
{
	struct thread_info *ti = current_thread_info();
4508

L
Linus Torvalds 已提交
4509 4510
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
I
Ingo Molnar 已提交
4511
	 * we do not want to preempt the current task. Just return..
L
Linus Torvalds 已提交
4512
	 */
N
Nick Piggin 已提交
4513
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
4514 4515
		return;

4516 4517 4518 4519
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4520

4521 4522 4523 4524 4525 4526
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
	} while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
L
Linus Torvalds 已提交
4527 4528 4529 4530
}
EXPORT_SYMBOL(preempt_schedule);

/*
4531
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
4532 4533 4534 4535 4536 4537 4538
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
4539

4540
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
4541 4542
	BUG_ON(ti->preempt_count || !irqs_disabled());

4543 4544 4545 4546 4547 4548
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		local_irq_enable();
		schedule();
		local_irq_disable();
		sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4549

4550 4551 4552 4553 4554 4555
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
	} while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
L
Linus Torvalds 已提交
4556 4557 4558 4559
}

#endif /* CONFIG_PREEMPT */

I
Ingo Molnar 已提交
4560 4561
int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
			  void *key)
L
Linus Torvalds 已提交
4562
{
4563
	return try_to_wake_up(curr->private, mode, sync);
L
Linus Torvalds 已提交
4564 4565 4566 4567
}
EXPORT_SYMBOL(default_wake_function);

/*
I
Ingo Molnar 已提交
4568 4569
 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
L
Linus Torvalds 已提交
4570 4571 4572
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
I
Ingo Molnar 已提交
4573
 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
L
Linus Torvalds 已提交
4574 4575 4576 4577 4578
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
			     int nr_exclusive, int sync, void *key)
{
4579
	wait_queue_t *curr, *next;
L
Linus Torvalds 已提交
4580

4581
	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
4582 4583
		unsigned flags = curr->flags;

L
Linus Torvalds 已提交
4584
		if (curr->func(curr, mode, sync, key) &&
4585
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
4586 4587 4588 4589 4590 4591 4592 4593 4594
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
4595
 * @key: is directly passed to the wakeup function
L
Linus Torvalds 已提交
4596
 */
4597
void __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
4598
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
4611
void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
L
Linus Torvalds 已提交
4612 4613 4614 4615 4616
{
	__wake_up_common(q, mode, 1, 0, NULL);
}

/**
4617
 * __wake_up_sync - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
 */
4629
void
I
Ingo Molnar 已提交
4630
__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
L
Linus Torvalds 已提交
4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646
{
	unsigned long flags;
	int sync = 1;

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
		sync = 0;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, sync, NULL);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

4647 4648 4649 4650 4651 4652 4653 4654 4655
/**
 * complete: - signals a single thread waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up a single thread waiting on this completion. Threads will be
 * awakened in the same order in which they were queued.
 *
 * See also complete_all(), wait_for_completion() and related routines.
 */
4656
void complete(struct completion *x)
L
Linus Torvalds 已提交
4657 4658 4659 4660 4661
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
4662
	__wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
L
Linus Torvalds 已提交
4663 4664 4665 4666
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

4667 4668 4669 4670 4671 4672
/**
 * complete_all: - signals all threads waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up all threads waiting on this particular completion event.
 */
4673
void complete_all(struct completion *x)
L
Linus Torvalds 已提交
4674 4675 4676 4677 4678
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
4679
	__wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
L
Linus Torvalds 已提交
4680 4681 4682 4683
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

4684 4685
static inline long __sched
do_wait_for_common(struct completion *x, long timeout, int state)
L
Linus Torvalds 已提交
4686 4687 4688 4689 4690 4691 4692
{
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
4693
			if (signal_pending_state(state, current)) {
4694 4695
				timeout = -ERESTARTSYS;
				break;
4696 4697
			}
			__set_current_state(state);
L
Linus Torvalds 已提交
4698 4699 4700
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
4701
		} while (!x->done && timeout);
L
Linus Torvalds 已提交
4702
		__remove_wait_queue(&x->wait, &wait);
4703 4704
		if (!x->done)
			return timeout;
L
Linus Torvalds 已提交
4705 4706
	}
	x->done--;
4707
	return timeout ?: 1;
L
Linus Torvalds 已提交
4708 4709
}

4710 4711
static long __sched
wait_for_common(struct completion *x, long timeout, int state)
L
Linus Torvalds 已提交
4712 4713 4714 4715
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
4716
	timeout = do_wait_for_common(x, timeout, state);
L
Linus Torvalds 已提交
4717
	spin_unlock_irq(&x->wait.lock);
4718 4719
	return timeout;
}
L
Linus Torvalds 已提交
4720

4721 4722 4723 4724 4725 4726 4727 4728 4729 4730
/**
 * wait_for_completion: - waits for completion of a task
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It is NOT
 * interruptible and there is no timeout.
 *
 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
 * and interrupt capability. Also see complete().
 */
4731
void __sched wait_for_completion(struct completion *x)
4732 4733
{
	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
4734
}
4735
EXPORT_SYMBOL(wait_for_completion);
L
Linus Torvalds 已提交
4736

4737 4738 4739 4740 4741 4742 4743 4744 4745
/**
 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. The timeout is in jiffies. It is not
 * interruptible.
 */
4746
unsigned long __sched
4747
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
L
Linus Torvalds 已提交
4748
{
4749
	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
4750
}
4751
EXPORT_SYMBOL(wait_for_completion_timeout);
L
Linus Torvalds 已提交
4752

4753 4754 4755 4756 4757 4758 4759
/**
 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
 * @x:  holds the state of this particular completion
 *
 * This waits for completion of a specific task to be signaled. It is
 * interruptible.
 */
4760
int __sched wait_for_completion_interruptible(struct completion *x)
I
Ingo Molnar 已提交
4761
{
4762 4763 4764 4765
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
I
Ingo Molnar 已提交
4766
}
4767
EXPORT_SYMBOL(wait_for_completion_interruptible);
L
Linus Torvalds 已提交
4768

4769 4770 4771 4772 4773 4774 4775 4776
/**
 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
 */
4777
unsigned long __sched
4778 4779
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
I
Ingo Molnar 已提交
4780
{
4781
	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
I
Ingo Molnar 已提交
4782
}
4783
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
L
Linus Torvalds 已提交
4784

4785 4786 4787 4788 4789 4790 4791
/**
 * wait_for_completion_killable: - waits for completion of a task (killable)
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It can be
 * interrupted by a kill signal.
 */
M
Matthew Wilcox 已提交
4792 4793 4794 4795 4796 4797 4798 4799 4800
int __sched wait_for_completion_killable(struct completion *x)
{
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
}
EXPORT_SYMBOL(wait_for_completion_killable);

4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846
/**
 *	try_wait_for_completion - try to decrement a completion without blocking
 *	@x:	completion structure
 *
 *	Returns: 0 if a decrement cannot be done without blocking
 *		 1 if a decrement succeeded.
 *
 *	If a completion is being used as a counting completion,
 *	attempt to decrement the counter without blocking. This
 *	enables us to avoid waiting if the resource the completion
 *	is protecting is not available.
 */
bool try_wait_for_completion(struct completion *x)
{
	int ret = 1;

	spin_lock_irq(&x->wait.lock);
	if (!x->done)
		ret = 0;
	else
		x->done--;
	spin_unlock_irq(&x->wait.lock);
	return ret;
}
EXPORT_SYMBOL(try_wait_for_completion);

/**
 *	completion_done - Test to see if a completion has any waiters
 *	@x:	completion structure
 *
 *	Returns: 0 if there are waiters (wait_for_completion() in progress)
 *		 1 if there are no waiters.
 *
 */
bool completion_done(struct completion *x)
{
	int ret = 1;

	spin_lock_irq(&x->wait.lock);
	if (!x->done)
		ret = 0;
	spin_unlock_irq(&x->wait.lock);
	return ret;
}
EXPORT_SYMBOL(completion_done);

4847 4848
static long __sched
sleep_on_common(wait_queue_head_t *q, int state, long timeout)
L
Linus Torvalds 已提交
4849
{
I
Ingo Molnar 已提交
4850 4851 4852 4853
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
4854

4855
	__set_current_state(state);
L
Linus Torvalds 已提交
4856

4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870
	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, &wait);
	spin_unlock(&q->lock);
	timeout = schedule_timeout(timeout);
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, &wait);
	spin_unlock_irqrestore(&q->lock, flags);

	return timeout;
}

void __sched interruptible_sleep_on(wait_queue_head_t *q)
{
	sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
4871 4872 4873
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
4874
long __sched
I
Ingo Molnar 已提交
4875
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
4876
{
4877
	return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
4878 4879 4880
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
4881
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
4882
{
4883
	sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
4884 4885 4886
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
4887
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
4888
{
4889
	return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
4890 4891 4892
}
EXPORT_SYMBOL(sleep_on_timeout);

4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
4905
void rt_mutex_setprio(struct task_struct *p, int prio)
4906 4907
{
	unsigned long flags;
4908
	int oldprio, on_rq, running;
4909
	struct rq *rq;
4910
	const struct sched_class *prev_class = p->sched_class;
4911 4912 4913 4914

	BUG_ON(prio < 0 || prio > MAX_PRIO);

	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
4915
	update_rq_clock(rq);
4916

4917
	oldprio = p->prio;
I
Ingo Molnar 已提交
4918
	on_rq = p->se.on_rq;
4919
	running = task_current(rq, p);
4920
	if (on_rq)
4921
		dequeue_task(rq, p, 0);
4922 4923
	if (running)
		p->sched_class->put_prev_task(rq, p);
I
Ingo Molnar 已提交
4924 4925 4926 4927 4928 4929

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

4930 4931
	p->prio = prio;

4932 4933
	if (running)
		p->sched_class->set_curr_task(rq);
I
Ingo Molnar 已提交
4934
	if (on_rq) {
4935
		enqueue_task(rq, p, 0);
4936 4937

		check_class_changed(rq, p, prev_class, oldprio, running);
4938 4939 4940 4941 4942 4943
	}
	task_rq_unlock(rq, &flags);
}

#endif

4944
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
4945
{
I
Ingo Molnar 已提交
4946
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
4947
	unsigned long flags;
4948
	struct rq *rq;
L
Linus Torvalds 已提交
4949 4950 4951 4952 4953 4954 4955 4956

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
4957
	update_rq_clock(rq);
L
Linus Torvalds 已提交
4958 4959 4960 4961
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
4962
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
4963
	 */
4964
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
4965 4966 4967
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
I
Ingo Molnar 已提交
4968
	on_rq = p->se.on_rq;
4969
	if (on_rq)
4970
		dequeue_task(rq, p, 0);
L
Linus Torvalds 已提交
4971 4972

	p->static_prio = NICE_TO_PRIO(nice);
4973
	set_load_weight(p);
4974 4975 4976
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
4977

I
Ingo Molnar 已提交
4978
	if (on_rq) {
4979
		enqueue_task(rq, p, 0);
L
Linus Torvalds 已提交
4980
		/*
4981 4982
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
4983
		 */
4984
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
4985 4986 4987 4988 4989 4990 4991
			resched_task(rq->curr);
	}
out_unlock:
	task_rq_unlock(rq, &flags);
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
4992 4993 4994 4995 4996
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
4997
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
4998
{
4999 5000
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
5001

M
Matt Mackall 已提交
5002 5003 5004 5005
	return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
asmlinkage long sys_nice(int increment)
{
5017
	long nice, retval;
L
Linus Torvalds 已提交
5018 5019 5020 5021 5022 5023

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
5024 5025
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
5026 5027 5028 5029 5030 5031 5032 5033 5034
	if (increment > 40)
		increment = 40;

	nice = PRIO_TO_NICE(current->static_prio) + increment;
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
5035 5036 5037
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
5056
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
5057 5058 5059 5060 5061 5062 5063 5064
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
5065
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
5066 5067 5068
{
	return TASK_NICE(p);
}
P
Pavel Roskin 已提交
5069
EXPORT_SYMBOL(task_nice);
L
Linus Torvalds 已提交
5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
5084
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
5085 5086 5087 5088 5089 5090 5091 5092
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
A
Alexey Dobriyan 已提交
5093
static struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
5094
{
5095
	return pid ? find_task_by_vpid(pid) : current;
L
Linus Torvalds 已提交
5096 5097 5098
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
5099 5100
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
5101
{
I
Ingo Molnar 已提交
5102
	BUG_ON(p->se.on_rq);
5103

L
Linus Torvalds 已提交
5104
	p->policy = policy;
I
Ingo Molnar 已提交
5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116
	switch (p->policy) {
	case SCHED_NORMAL:
	case SCHED_BATCH:
	case SCHED_IDLE:
		p->sched_class = &fair_sched_class;
		break;
	case SCHED_FIFO:
	case SCHED_RR:
		p->sched_class = &rt_sched_class;
		break;
	}

L
Linus Torvalds 已提交
5117
	p->rt_priority = prio;
5118 5119 5120
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
5121
	set_load_weight(p);
L
Linus Torvalds 已提交
5122 5123
}

5124 5125
static int __sched_setscheduler(struct task_struct *p, int policy,
				struct sched_param *param, bool user)
L
Linus Torvalds 已提交
5126
{
5127
	int retval, oldprio, oldpolicy = -1, on_rq, running;
L
Linus Torvalds 已提交
5128
	unsigned long flags;
5129
	const struct sched_class *prev_class = p->sched_class;
5130
	struct rq *rq;
L
Linus Torvalds 已提交
5131

5132 5133
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
5134 5135 5136 5137 5138
recheck:
	/* double check policy once rq lock held */
	if (policy < 0)
		policy = oldpolicy = p->policy;
	else if (policy != SCHED_FIFO && policy != SCHED_RR &&
I
Ingo Molnar 已提交
5139 5140
			policy != SCHED_NORMAL && policy != SCHED_BATCH &&
			policy != SCHED_IDLE)
5141
		return -EINVAL;
L
Linus Torvalds 已提交
5142 5143
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
5144 5145
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
5146 5147
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
5148
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
5149
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
5150
		return -EINVAL;
5151
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
5152 5153
		return -EINVAL;

5154 5155 5156
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
5157
	if (user && !capable(CAP_SYS_NICE)) {
5158
		if (rt_policy(policy)) {
5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174
			unsigned long rlim_rtprio;

			if (!lock_task_sighand(p, &flags))
				return -ESRCH;
			rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
			unlock_task_sighand(p, &flags);

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
I
Ingo Molnar 已提交
5175 5176 5177 5178 5179 5180
		/*
		 * Like positive nice levels, dont allow tasks to
		 * move out of SCHED_IDLE either:
		 */
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
			return -EPERM;
5181

5182 5183 5184 5185 5186
		/* can't change other user's priorities */
		if ((current->euid != p->euid) &&
		    (current->euid != p->uid))
			return -EPERM;
	}
L
Linus Torvalds 已提交
5187

5188
	if (user) {
5189
#ifdef CONFIG_RT_GROUP_SCHED
5190 5191 5192 5193
		/*
		 * Do not allow realtime tasks into groups that have no runtime
		 * assigned.
		 */
P
Peter Zijlstra 已提交
5194 5195
		if (rt_bandwidth_enabled() && rt_policy(policy) &&
				task_group(p)->rt_bandwidth.rt_runtime == 0)
5196
			return -EPERM;
5197 5198
#endif

5199 5200 5201 5202 5203
		retval = security_task_setscheduler(p, policy, param);
		if (retval)
			return retval;
	}

5204 5205 5206 5207 5208
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
	 */
	spin_lock_irqsave(&p->pi_lock, flags);
L
Linus Torvalds 已提交
5209 5210 5211 5212
	/*
	 * To be able to change p->policy safely, the apropriate
	 * runqueue lock must be held.
	 */
5213
	rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
5214 5215 5216
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
5217 5218
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
5219 5220
		goto recheck;
	}
I
Ingo Molnar 已提交
5221
	update_rq_clock(rq);
I
Ingo Molnar 已提交
5222
	on_rq = p->se.on_rq;
5223
	running = task_current(rq, p);
5224
	if (on_rq)
5225
		deactivate_task(rq, p, 0);
5226 5227
	if (running)
		p->sched_class->put_prev_task(rq, p);
5228

L
Linus Torvalds 已提交
5229
	oldprio = p->prio;
I
Ingo Molnar 已提交
5230
	__setscheduler(rq, p, policy, param->sched_priority);
5231

5232 5233
	if (running)
		p->sched_class->set_curr_task(rq);
I
Ingo Molnar 已提交
5234 5235
	if (on_rq) {
		activate_task(rq, p, 0);
5236 5237

		check_class_changed(rq, p, prev_class, oldprio, running);
L
Linus Torvalds 已提交
5238
	}
5239 5240 5241
	__task_rq_unlock(rq);
	spin_unlock_irqrestore(&p->pi_lock, flags);

5242 5243
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
5244 5245
	return 0;
}
5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259

/**
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * NOTE that the task may be already dead.
 */
int sched_setscheduler(struct task_struct *p, int policy,
		       struct sched_param *param)
{
	return __sched_setscheduler(p, policy, param, true);
}
L
Linus Torvalds 已提交
5260 5261
EXPORT_SYMBOL_GPL(sched_setscheduler);

5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278
/**
 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * Just like sched_setscheduler, only don't bother checking if the
 * current context has permission.  For example, this is needed in
 * stop_machine(): we create temporary high priority worker threads,
 * but our caller might not have that capability.
 */
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
			       struct sched_param *param)
{
	return __sched_setscheduler(p, policy, param, false);
}

I
Ingo Molnar 已提交
5279 5280
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
5281 5282 5283
{
	struct sched_param lparam;
	struct task_struct *p;
5284
	int retval;
L
Linus Torvalds 已提交
5285 5286 5287 5288 5289

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
5290 5291 5292

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
5293
	p = find_process_by_pid(pid);
5294 5295 5296
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
5297

L
Linus Torvalds 已提交
5298 5299 5300 5301 5302 5303 5304 5305 5306
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
I
Ingo Molnar 已提交
5307 5308
asmlinkage long
sys_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
5309
{
5310 5311 5312 5313
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
asmlinkage long sys_sched_getscheduler(pid_t pid)
{
5333
	struct task_struct *p;
5334
	int retval;
L
Linus Torvalds 已提交
5335 5336

	if (pid < 0)
5337
		return -EINVAL;
L
Linus Torvalds 已提交
5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
			retval = p->policy;
	}
	read_unlock(&tasklist_lock);
	return retval;
}

/**
 * sys_sched_getscheduler - get the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
{
	struct sched_param lp;
5359
	struct task_struct *p;
5360
	int retval;
L
Linus Torvalds 已提交
5361 5362

	if (!param || pid < 0)
5363
		return -EINVAL;
L
Linus Torvalds 已提交
5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389

	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
	read_unlock(&tasklist_lock);

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

	return retval;

out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

5390
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
L
Linus Torvalds 已提交
5391
{
5392
	cpumask_var_t cpus_allowed, new_mask;
5393 5394
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
5395

5396
	get_online_cpus();
L
Linus Torvalds 已提交
5397 5398 5399 5400 5401
	read_lock(&tasklist_lock);

	p = find_process_by_pid(pid);
	if (!p) {
		read_unlock(&tasklist_lock);
5402
		put_online_cpus();
L
Linus Torvalds 已提交
5403 5404 5405 5406 5407
		return -ESRCH;
	}

	/*
	 * It is not safe to call set_cpus_allowed with the
I
Ingo Molnar 已提交
5408
	 * tasklist_lock held. We will bump the task_struct's
L
Linus Torvalds 已提交
5409 5410 5411 5412 5413
	 * usage count and then drop tasklist_lock.
	 */
	get_task_struct(p);
	read_unlock(&tasklist_lock);

5414 5415 5416 5417 5418 5419 5420 5421
	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_put_task;
	}
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_free_cpus_allowed;
	}
L
Linus Torvalds 已提交
5422 5423 5424 5425 5426
	retval = -EPERM;
	if ((current->euid != p->euid) && (current->euid != p->uid) &&
			!capable(CAP_SYS_NICE))
		goto out_unlock;

5427 5428 5429 5430
	retval = security_task_setscheduler(p, 0, NULL);
	if (retval)
		goto out_unlock;

5431 5432
	cpuset_cpus_allowed(p, cpus_allowed);
	cpumask_and(new_mask, in_mask, cpus_allowed);
P
Paul Menage 已提交
5433
 again:
5434
	retval = set_cpus_allowed_ptr(p, new_mask);
L
Linus Torvalds 已提交
5435

P
Paul Menage 已提交
5436
	if (!retval) {
5437 5438
		cpuset_cpus_allowed(p, cpus_allowed);
		if (!cpumask_subset(new_mask, cpus_allowed)) {
P
Paul Menage 已提交
5439 5440 5441 5442 5443
			/*
			 * We must have raced with a concurrent cpuset
			 * update. Just reset the cpus_allowed to the
			 * cpuset's cpus_allowed
			 */
5444
			cpumask_copy(new_mask, cpus_allowed);
P
Paul Menage 已提交
5445 5446 5447
			goto again;
		}
	}
L
Linus Torvalds 已提交
5448
out_unlock:
5449 5450 5451 5452
	free_cpumask_var(new_mask);
out_free_cpus_allowed:
	free_cpumask_var(cpus_allowed);
out_put_task:
L
Linus Torvalds 已提交
5453
	put_task_struct(p);
5454
	put_online_cpus();
L
Linus Torvalds 已提交
5455 5456 5457 5458
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5459
			     struct cpumask *new_mask)
L
Linus Torvalds 已提交
5460
{
5461 5462 5463 5464 5465
	if (len < cpumask_size())
		cpumask_clear(new_mask);
	else if (len > cpumask_size())
		len = cpumask_size();

L
Linus Torvalds 已提交
5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
5478
	cpumask_var_t new_mask;
L
Linus Torvalds 已提交
5479 5480
	int retval;

5481 5482
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
5483

5484 5485 5486 5487 5488
	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
	if (retval == 0)
		retval = sched_setaffinity(pid, new_mask);
	free_cpumask_var(new_mask);
	return retval;
L
Linus Torvalds 已提交
5489 5490
}

5491
long sched_getaffinity(pid_t pid, struct cpumask *mask)
L
Linus Torvalds 已提交
5492
{
5493
	struct task_struct *p;
L
Linus Torvalds 已提交
5494 5495
	int retval;

5496
	get_online_cpus();
L
Linus Torvalds 已提交
5497 5498 5499 5500 5501 5502 5503
	read_lock(&tasklist_lock);

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

5504 5505 5506 5507
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

5508
	cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
L
Linus Torvalds 已提交
5509 5510 5511

out_unlock:
	read_unlock(&tasklist_lock);
5512
	put_online_cpus();
L
Linus Torvalds 已提交
5513

5514
	return retval;
L
Linus Torvalds 已提交
5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	int ret;
5527
	cpumask_var_t mask;
L
Linus Torvalds 已提交
5528

5529
	if (len < cpumask_size())
L
Linus Torvalds 已提交
5530 5531
		return -EINVAL;

5532 5533
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
5534

5535 5536 5537 5538 5539 5540 5541 5542
	ret = sched_getaffinity(pid, mask);
	if (ret == 0) {
		if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
			ret = -EFAULT;
		else
			ret = cpumask_size();
	}
	free_cpumask_var(mask);
L
Linus Torvalds 已提交
5543

5544
	return ret;
L
Linus Torvalds 已提交
5545 5546 5547 5548 5549
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
5550 5551
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
5552 5553 5554
 */
asmlinkage long sys_sched_yield(void)
{
5555
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
5556

5557
	schedstat_inc(rq, yld_count);
5558
	current->sched_class->yield_task(rq);
L
Linus Torvalds 已提交
5559 5560 5561 5562 5563 5564

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
5565
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
5566 5567 5568 5569 5570 5571 5572 5573
	_raw_spin_unlock(&rq->lock);
	preempt_enable_no_resched();

	schedule();

	return 0;
}

A
Andrew Morton 已提交
5574
static void __cond_resched(void)
L
Linus Torvalds 已提交
5575
{
5576 5577 5578
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
	__might_sleep(__FILE__, __LINE__);
#endif
5579 5580 5581 5582 5583
	/*
	 * The BKS might be reacquired before we have dropped
	 * PREEMPT_ACTIVE, which could trigger a second
	 * cond_resched() call.
	 */
L
Linus Torvalds 已提交
5584 5585 5586 5587 5588 5589 5590
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
	} while (need_resched());
}

5591
int __sched _cond_resched(void)
L
Linus Torvalds 已提交
5592
{
5593 5594
	if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
					system_state == SYSTEM_RUNNING) {
L
Linus Torvalds 已提交
5595 5596 5597 5598 5599
		__cond_resched();
		return 1;
	}
	return 0;
}
5600
EXPORT_SYMBOL(_cond_resched);
L
Linus Torvalds 已提交
5601 5602 5603 5604 5605

/*
 * cond_resched_lock() - if a reschedule is pending, drop the given lock,
 * call schedule, and on return reacquire the lock.
 *
I
Ingo Molnar 已提交
5606
 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
L
Linus Torvalds 已提交
5607 5608 5609
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
I
Ingo Molnar 已提交
5610
int cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
5611
{
N
Nick Piggin 已提交
5612
	int resched = need_resched() && system_state == SYSTEM_RUNNING;
J
Jan Kara 已提交
5613 5614
	int ret = 0;

N
Nick Piggin 已提交
5615
	if (spin_needbreak(lock) || resched) {
L
Linus Torvalds 已提交
5616
		spin_unlock(lock);
N
Nick Piggin 已提交
5617 5618 5619 5620
		if (resched && need_resched())
			__cond_resched();
		else
			cpu_relax();
J
Jan Kara 已提交
5621
		ret = 1;
L
Linus Torvalds 已提交
5622 5623
		spin_lock(lock);
	}
J
Jan Kara 已提交
5624
	return ret;
L
Linus Torvalds 已提交
5625 5626 5627 5628 5629 5630 5631
}
EXPORT_SYMBOL(cond_resched_lock);

int __sched cond_resched_softirq(void)
{
	BUG_ON(!in_softirq());

5632
	if (need_resched() && system_state == SYSTEM_RUNNING) {
5633
		local_bh_enable();
L
Linus Torvalds 已提交
5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched_softirq);

/**
 * yield - yield the current processor to other threads.
 *
5645
 * This is a shortcut for kernel-space yielding - it marks the
L
Linus Torvalds 已提交
5646 5647 5648 5649 5650 5651 5652 5653 5654 5655
 * thread runnable and calls sys_sched_yield().
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

/*
I
Ingo Molnar 已提交
5656
 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
L
Linus Torvalds 已提交
5657 5658 5659 5660 5661 5662 5663
 * that process accounting knows that this is a task in IO wait state.
 *
 * But don't do that if it is a deliberate, throttling IO wait (this task
 * has set its backing_dev_info: the queue against which it should throttle)
 */
void __sched io_schedule(void)
{
5664
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
5665

5666
	delayacct_blkio_start();
L
Linus Torvalds 已提交
5667 5668 5669
	atomic_inc(&rq->nr_iowait);
	schedule();
	atomic_dec(&rq->nr_iowait);
5670
	delayacct_blkio_end();
L
Linus Torvalds 已提交
5671 5672 5673 5674 5675
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
5676
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
5677 5678
	long ret;

5679
	delayacct_blkio_start();
L
Linus Torvalds 已提交
5680 5681 5682
	atomic_inc(&rq->nr_iowait);
	ret = schedule_timeout(timeout);
	atomic_dec(&rq->nr_iowait);
5683
	delayacct_blkio_end();
L
Linus Torvalds 已提交
5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_max(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
5704
	case SCHED_BATCH:
I
Ingo Molnar 已提交
5705
	case SCHED_IDLE:
L
Linus Torvalds 已提交
5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_min(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
5729
	case SCHED_BATCH:
I
Ingo Molnar 已提交
5730
	case SCHED_IDLE:
L
Linus Torvalds 已提交
5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
asmlinkage
long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
{
5747
	struct task_struct *p;
D
Dmitry Adamushko 已提交
5748
	unsigned int time_slice;
5749
	int retval;
L
Linus Torvalds 已提交
5750 5751 5752
	struct timespec t;

	if (pid < 0)
5753
		return -EINVAL;
L
Linus Torvalds 已提交
5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

5765 5766 5767 5768 5769 5770
	/*
	 * Time slice is 0 for SCHED_FIFO tasks and for SCHED_OTHER
	 * tasks that are on an otherwise idle runqueue:
	 */
	time_slice = 0;
	if (p->policy == SCHED_RR) {
D
Dmitry Adamushko 已提交
5771
		time_slice = DEF_TIMESLICE;
5772
	} else if (p->policy != SCHED_FIFO) {
D
Dmitry Adamushko 已提交
5773 5774 5775 5776 5777
		struct sched_entity *se = &p->se;
		unsigned long flags;
		struct rq *rq;

		rq = task_rq_lock(p, &flags);
5778 5779
		if (rq->cfs.load.weight)
			time_slice = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
D
Dmitry Adamushko 已提交
5780 5781
		task_rq_unlock(rq, &flags);
	}
L
Linus Torvalds 已提交
5782
	read_unlock(&tasklist_lock);
D
Dmitry Adamushko 已提交
5783
	jiffies_to_timespec(time_slice, &t);
L
Linus Torvalds 已提交
5784 5785
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
	return retval;
5786

L
Linus Torvalds 已提交
5787 5788 5789 5790 5791
out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

5792
static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5793

5794
void sched_show_task(struct task_struct *p)
L
Linus Torvalds 已提交
5795 5796
{
	unsigned long free = 0;
5797
	unsigned state;
L
Linus Torvalds 已提交
5798 5799

	state = p->state ? __ffs(p->state) + 1 : 0;
I
Ingo Molnar 已提交
5800
	printk(KERN_INFO "%-13.13s %c", p->comm,
5801
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5802
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
5803
	if (state == TASK_RUNNING)
I
Ingo Molnar 已提交
5804
		printk(KERN_CONT " running  ");
L
Linus Torvalds 已提交
5805
	else
I
Ingo Molnar 已提交
5806
		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
5807 5808
#else
	if (state == TASK_RUNNING)
I
Ingo Molnar 已提交
5809
		printk(KERN_CONT "  running task    ");
L
Linus Torvalds 已提交
5810
	else
I
Ingo Molnar 已提交
5811
		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
5812 5813 5814
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
	{
5815
		unsigned long *n = end_of_stack(p);
L
Linus Torvalds 已提交
5816 5817
		while (!*n)
			n++;
5818
		free = (unsigned long)n - (unsigned long)end_of_stack(p);
L
Linus Torvalds 已提交
5819 5820
	}
#endif
5821
	printk(KERN_CONT "%5lu %5d %6d\n", free,
R
Roland McGrath 已提交
5822
		task_pid_nr(p), task_pid_nr(p->real_parent));
L
Linus Torvalds 已提交
5823

5824
	show_stack(p, NULL);
L
Linus Torvalds 已提交
5825 5826
}

I
Ingo Molnar 已提交
5827
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
5828
{
5829
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
5830

5831 5832 5833
#if BITS_PER_LONG == 32
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
5834
#else
5835 5836
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
5837 5838 5839 5840 5841 5842 5843 5844
#endif
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
		 * console might take alot of time:
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
5845
		if (!state_filter || (p->state & state_filter))
5846
			sched_show_task(p);
L
Linus Torvalds 已提交
5847 5848
	} while_each_thread(g, p);

5849 5850
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
5851 5852 5853
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
L
Linus Torvalds 已提交
5854
	read_unlock(&tasklist_lock);
I
Ingo Molnar 已提交
5855 5856 5857 5858 5859
	/*
	 * Only show locks if all tasks are dumped:
	 */
	if (state_filter == -1)
		debug_show_all_locks();
L
Linus Torvalds 已提交
5860 5861
}

I
Ingo Molnar 已提交
5862 5863
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
5864
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
5865 5866
}

5867 5868 5869 5870 5871 5872 5873 5874
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
5875
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
5876
{
5877
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
5878 5879
	unsigned long flags;

5880 5881
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
5882 5883 5884
	__sched_fork(idle);
	idle->se.exec_start = sched_clock();

5885
	idle->prio = idle->normal_prio = MAX_PRIO;
5886
	cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
I
Ingo Molnar 已提交
5887
	__set_task_cpu(idle, cpu);
L
Linus Torvalds 已提交
5888 5889

	rq->curr = rq->idle = idle;
5890 5891 5892
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
	idle->oncpu = 1;
#endif
L
Linus Torvalds 已提交
5893 5894 5895
	spin_unlock_irqrestore(&rq->lock, flags);

	/* Set the preempt count _outside_ the spinlocks! */
5896 5897 5898
#if defined(CONFIG_PREEMPT)
	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
#else
A
Al Viro 已提交
5899
	task_thread_info(idle)->preempt_count = 0;
5900
#endif
I
Ingo Molnar 已提交
5901 5902 5903 5904
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
5905
	ftrace_graph_init_task(idle);
L
Linus Torvalds 已提交
5906 5907 5908 5909 5910 5911 5912
}

/*
 * In a system that switches off the HZ timer nohz_cpu_mask
 * indicates which cpus entered this state. This is used
 * in the rcu update to wait only for active cpus. For system
 * which do not switch off the HZ timer nohz_cpu_mask should
5913
 * always be CPU_BITS_NONE.
L
Linus Torvalds 已提交
5914
 */
5915
cpumask_var_t nohz_cpu_mask;
L
Linus Torvalds 已提交
5916

I
Ingo Molnar 已提交
5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939
/*
 * Increase the granularity value when there are more CPUs,
 * because with more CPUs the 'effective latency' as visible
 * to users decreases. But the relationship is not linear,
 * so pick a second-best guess by going with the log2 of the
 * number of CPUs.
 *
 * This idea comes from the SD scheduler of Con Kolivas:
 */
static inline void sched_init_granularity(void)
{
	unsigned int factor = 1 + ilog2(num_online_cpus());
	const unsigned long limit = 200000000;

	sysctl_sched_min_granularity *= factor;
	if (sysctl_sched_min_granularity > limit)
		sysctl_sched_min_granularity = limit;

	sysctl_sched_latency *= factor;
	if (sysctl_sched_latency > limit)
		sysctl_sched_latency = limit;

	sysctl_sched_wakeup_granularity *= factor;
5940 5941

	sysctl_sched_shares_ratelimit *= factor;
I
Ingo Molnar 已提交
5942 5943
}

L
Linus Torvalds 已提交
5944 5945 5946 5947
#ifdef CONFIG_SMP
/*
 * This is how migration works:
 *
5948
 * 1) we queue a struct migration_req structure in the source CPU's
L
Linus Torvalds 已提交
5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966
 *    runqueue and wake up that CPU's migration thread.
 * 2) we down() the locked semaphore => thread blocks.
 * 3) migration thread wakes up (implicitly it forces the migrated
 *    thread off the CPU)
 * 4) it gets the migration request and checks whether the migrated
 *    task is still in the wrong runqueue.
 * 5) if it's in the wrong runqueue then the migration thread removes
 *    it and puts it into the right queue.
 * 6) migration thread up()s the semaphore.
 * 7) we wake up and the migration is done.
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
I
Ingo Molnar 已提交
5967
 * task must not exit() & deallocate itself prematurely. The
L
Linus Torvalds 已提交
5968 5969
 * call is not atomic; no spinlocks may be held.
 */
5970
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
L
Linus Torvalds 已提交
5971
{
5972
	struct migration_req req;
L
Linus Torvalds 已提交
5973
	unsigned long flags;
5974
	struct rq *rq;
5975
	int ret = 0;
L
Linus Torvalds 已提交
5976 5977

	rq = task_rq_lock(p, &flags);
5978
	if (!cpumask_intersects(new_mask, cpu_online_mask)) {
L
Linus Torvalds 已提交
5979 5980 5981 5982
		ret = -EINVAL;
		goto out;
	}

5983
	if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5984
		     !cpumask_equal(&p->cpus_allowed, new_mask))) {
5985 5986 5987 5988
		ret = -EINVAL;
		goto out;
	}

5989
	if (p->sched_class->set_cpus_allowed)
5990
		p->sched_class->set_cpus_allowed(p, new_mask);
5991
	else {
5992 5993
		cpumask_copy(&p->cpus_allowed, new_mask);
		p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5994 5995
	}

L
Linus Torvalds 已提交
5996
	/* Can the task run on the task's current CPU? If so, we're done */
5997
	if (cpumask_test_cpu(task_cpu(p), new_mask))
L
Linus Torvalds 已提交
5998 5999
		goto out;

R
Rusty Russell 已提交
6000
	if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
L
Linus Torvalds 已提交
6001 6002 6003 6004 6005 6006 6007 6008 6009
		/* Need help from migration thread: drop lock and wait. */
		task_rq_unlock(rq, &flags);
		wake_up_process(rq->migration_thread);
		wait_for_completion(&req.done);
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
	task_rq_unlock(rq, &flags);
6010

L
Linus Torvalds 已提交
6011 6012
	return ret;
}
6013
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
L
Linus Torvalds 已提交
6014 6015

/*
I
Ingo Molnar 已提交
6016
 * Move (not current) task off this cpu, onto dest cpu. We're doing
L
Linus Torvalds 已提交
6017 6018 6019 6020 6021 6022
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
6023 6024
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
6025
 */
6026
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
6027
{
6028
	struct rq *rq_dest, *rq_src;
I
Ingo Molnar 已提交
6029
	int ret = 0, on_rq;
L
Linus Torvalds 已提交
6030

6031
	if (unlikely(!cpu_active(dest_cpu)))
6032
		return ret;
L
Linus Torvalds 已提交
6033 6034 6035 6036 6037 6038 6039

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
L
Linus Torvalds 已提交
6040
		goto done;
L
Linus Torvalds 已提交
6041
	/* Affinity changed (again). */
6042
	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
L
Linus Torvalds 已提交
6043
		goto fail;
L
Linus Torvalds 已提交
6044

I
Ingo Molnar 已提交
6045
	on_rq = p->se.on_rq;
6046
	if (on_rq)
6047
		deactivate_task(rq_src, p, 0);
6048

L
Linus Torvalds 已提交
6049
	set_task_cpu(p, dest_cpu);
I
Ingo Molnar 已提交
6050 6051
	if (on_rq) {
		activate_task(rq_dest, p, 0);
6052
		check_preempt_curr(rq_dest, p, 0);
L
Linus Torvalds 已提交
6053
	}
L
Linus Torvalds 已提交
6054
done:
6055
	ret = 1;
L
Linus Torvalds 已提交
6056
fail:
L
Linus Torvalds 已提交
6057
	double_rq_unlock(rq_src, rq_dest);
6058
	return ret;
L
Linus Torvalds 已提交
6059 6060 6061 6062 6063 6064 6065
}

/*
 * migration_thread - this is a highprio system thread that performs
 * thread migration by bumping thread off CPU then 'pushing' onto
 * another runqueue.
 */
I
Ingo Molnar 已提交
6066
static int migration_thread(void *data)
L
Linus Torvalds 已提交
6067 6068
{
	int cpu = (long)data;
6069
	struct rq *rq;
L
Linus Torvalds 已提交
6070 6071 6072 6073 6074 6075

	rq = cpu_rq(cpu);
	BUG_ON(rq->migration_thread != current);

	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
6076
		struct migration_req *req;
L
Linus Torvalds 已提交
6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098
		struct list_head *head;

		spin_lock_irq(&rq->lock);

		if (cpu_is_offline(cpu)) {
			spin_unlock_irq(&rq->lock);
			goto wait_to_die;
		}

		if (rq->active_balance) {
			active_load_balance(rq, cpu);
			rq->active_balance = 0;
		}

		head = &rq->migration_queue;

		if (list_empty(head)) {
			spin_unlock_irq(&rq->lock);
			schedule();
			set_current_state(TASK_INTERRUPTIBLE);
			continue;
		}
6099
		req = list_entry(head->next, struct migration_req, list);
L
Linus Torvalds 已提交
6100 6101
		list_del_init(head->next);

N
Nick Piggin 已提交
6102 6103 6104
		spin_unlock(&rq->lock);
		__migrate_task(req->task, cpu, req->dest_cpu);
		local_irq_enable();
L
Linus Torvalds 已提交
6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122

		complete(&req->done);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133

static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
{
	int ret;

	local_irq_disable();
	ret = __migrate_task(p, src_cpu, dest_cpu);
	local_irq_enable();
	return ret;
}

6134
/*
6135
 * Figure out where task on dead CPU should go, use force if necessary.
6136
 */
6137
static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
6138
{
6139
	int dest_cpu;
6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158
	/* FIXME: Use cpumask_of_node here. */
	cpumask_t _nodemask = node_to_cpumask(cpu_to_node(dead_cpu));
	const struct cpumask *nodemask = &_nodemask;

again:
	/* Look for allowed, online CPU in same node. */
	for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
		if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
			goto move;

	/* Any allowed, online CPU? */
	dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
	if (dest_cpu < nr_cpu_ids)
		goto move;

	/* No more Mr. Nice Guy. */
	if (dest_cpu >= nr_cpu_ids) {
		cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
		dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
L
Linus Torvalds 已提交
6159

6160 6161 6162 6163 6164 6165 6166 6167 6168
		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
		if (p->mm && printk_ratelimit()) {
			printk(KERN_INFO "process %d (%s) no "
			       "longer affine to cpu%d\n",
			       task_pid_nr(p), p->comm, dead_cpu);
6169
		}
6170 6171 6172 6173 6174 6175
	}

move:
	/* It can have affinity changed while we were choosing. */
	if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
		goto again;
L
Linus Torvalds 已提交
6176 6177 6178 6179 6180 6181 6182 6183 6184
}

/*
 * While a dead CPU has no uninterruptible tasks queued at this point,
 * it might still have a nonzero ->nr_uninterruptible counter, because
 * for performance reasons the counter is not stricly tracking tasks to
 * their home CPUs. So we just add the counter to another CPU's counter,
 * to keep the global sum constant after CPU-down:
 */
6185
static void migrate_nr_uninterruptible(struct rq *rq_src)
L
Linus Torvalds 已提交
6186
{
R
Rusty Russell 已提交
6187
	struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
L
Linus Torvalds 已提交
6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200
	unsigned long flags;

	local_irq_save(flags);
	double_rq_lock(rq_src, rq_dest);
	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
	rq_src->nr_uninterruptible = 0;
	double_rq_unlock(rq_src, rq_dest);
	local_irq_restore(flags);
}

/* Run through task list and migrate tasks from the dead cpu. */
static void migrate_live_tasks(int src_cpu)
{
6201
	struct task_struct *p, *t;
L
Linus Torvalds 已提交
6202

6203
	read_lock(&tasklist_lock);
L
Linus Torvalds 已提交
6204

6205 6206
	do_each_thread(t, p) {
		if (p == current)
L
Linus Torvalds 已提交
6207 6208
			continue;

6209 6210 6211
		if (task_cpu(p) == src_cpu)
			move_task_off_dead_cpu(src_cpu, p);
	} while_each_thread(t, p);
L
Linus Torvalds 已提交
6212

6213
	read_unlock(&tasklist_lock);
L
Linus Torvalds 已提交
6214 6215
}

I
Ingo Molnar 已提交
6216 6217
/*
 * Schedules idle task to be the next runnable task on current CPU.
6218 6219
 * It does so by boosting its priority to highest possible.
 * Used by CPU offline code.
L
Linus Torvalds 已提交
6220 6221 6222
 */
void sched_idle_next(void)
{
6223
	int this_cpu = smp_processor_id();
6224
	struct rq *rq = cpu_rq(this_cpu);
L
Linus Torvalds 已提交
6225 6226 6227 6228
	struct task_struct *p = rq->idle;
	unsigned long flags;

	/* cpu has to be offline */
6229
	BUG_ON(cpu_online(this_cpu));
L
Linus Torvalds 已提交
6230

6231 6232 6233
	/*
	 * Strictly not necessary since rest of the CPUs are stopped by now
	 * and interrupts disabled on the current cpu.
L
Linus Torvalds 已提交
6234 6235 6236
	 */
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
6237
	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
6238

6239 6240
	update_rq_clock(rq);
	activate_task(rq, p, 0);
L
Linus Torvalds 已提交
6241 6242 6243 6244

	spin_unlock_irqrestore(&rq->lock, flags);
}

6245 6246
/*
 * Ensures that the idle task is using init_mm right before its cpu goes
L
Linus Torvalds 已提交
6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259
 * offline.
 */
void idle_task_exit(void)
{
	struct mm_struct *mm = current->active_mm;

	BUG_ON(cpu_online(smp_processor_id()));

	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
}

6260
/* called under rq->lock with disabled interrupts */
6261
static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
6262
{
6263
	struct rq *rq = cpu_rq(dead_cpu);
L
Linus Torvalds 已提交
6264 6265

	/* Must be exiting, otherwise would be on tasklist. */
E
Eugene Teo 已提交
6266
	BUG_ON(!p->exit_state);
L
Linus Torvalds 已提交
6267 6268

	/* Cannot have done final schedule yet: would have vanished. */
6269
	BUG_ON(p->state == TASK_DEAD);
L
Linus Torvalds 已提交
6270

6271
	get_task_struct(p);
L
Linus Torvalds 已提交
6272 6273 6274

	/*
	 * Drop lock around migration; if someone else moves it,
I
Ingo Molnar 已提交
6275
	 * that's OK. No task can be added to this CPU, so iteration is
L
Linus Torvalds 已提交
6276 6277
	 * fine.
	 */
6278
	spin_unlock_irq(&rq->lock);
6279
	move_task_off_dead_cpu(dead_cpu, p);
6280
	spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
6281

6282
	put_task_struct(p);
L
Linus Torvalds 已提交
6283 6284 6285 6286 6287
}

/* release_task() removes task from tasklist, so we won't find dead tasks. */
static void migrate_dead_tasks(unsigned int dead_cpu)
{
6288
	struct rq *rq = cpu_rq(dead_cpu);
I
Ingo Molnar 已提交
6289
	struct task_struct *next;
6290

I
Ingo Molnar 已提交
6291 6292 6293
	for ( ; ; ) {
		if (!rq->nr_running)
			break;
I
Ingo Molnar 已提交
6294
		update_rq_clock(rq);
6295
		next = pick_next_task(rq, rq->curr);
I
Ingo Molnar 已提交
6296 6297
		if (!next)
			break;
D
Dmitry Adamushko 已提交
6298
		next->sched_class->put_prev_task(rq, next);
I
Ingo Molnar 已提交
6299
		migrate_dead(dead_cpu, next);
6300

L
Linus Torvalds 已提交
6301 6302 6303 6304
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

6305 6306 6307
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
6308 6309
	{
		.procname	= "sched_domain",
6310
		.mode		= 0555,
6311
	},
I
Ingo Molnar 已提交
6312
	{0, },
6313 6314 6315
};

static struct ctl_table sd_ctl_root[] = {
6316
	{
6317
		.ctl_name	= CTL_KERN,
6318
		.procname	= "kernel",
6319
		.mode		= 0555,
6320 6321
		.child		= sd_ctl_dir,
	},
I
Ingo Molnar 已提交
6322
	{0, },
6323 6324 6325 6326 6327
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
6328
		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
6329 6330 6331 6332

	return entry;
}

6333 6334
static void sd_free_ctl_entry(struct ctl_table **tablep)
{
6335
	struct ctl_table *entry;
6336

6337 6338 6339
	/*
	 * In the intermediate directories, both the child directory and
	 * procname are dynamically allocated and could fail but the mode
I
Ingo Molnar 已提交
6340
	 * will always be set. In the lowest directory the names are
6341 6342 6343
	 * static strings and all have proc handlers.
	 */
	for (entry = *tablep; entry->mode; entry++) {
6344 6345
		if (entry->child)
			sd_free_ctl_entry(&entry->child);
6346 6347 6348
		if (entry->proc_handler == NULL)
			kfree(entry->procname);
	}
6349 6350 6351 6352 6353

	kfree(*tablep);
	*tablep = NULL;
}

6354
static void
6355
set_table_entry(struct ctl_table *entry,
6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368
		const char *procname, void *data, int maxlen,
		mode_t mode, proc_handler *proc_handler)
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
6369
	struct ctl_table *table = sd_alloc_ctl_entry(13);
6370

6371 6372 6373
	if (table == NULL)
		return NULL;

6374
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
6375
		sizeof(long), 0644, proc_doulongvec_minmax);
6376
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
6377
		sizeof(long), 0644, proc_doulongvec_minmax);
6378
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
6379
		sizeof(int), 0644, proc_dointvec_minmax);
6380
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
6381
		sizeof(int), 0644, proc_dointvec_minmax);
6382
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
6383
		sizeof(int), 0644, proc_dointvec_minmax);
6384
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
6385
		sizeof(int), 0644, proc_dointvec_minmax);
6386
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
6387
		sizeof(int), 0644, proc_dointvec_minmax);
6388
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
6389
		sizeof(int), 0644, proc_dointvec_minmax);
6390
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
6391
		sizeof(int), 0644, proc_dointvec_minmax);
6392
	set_table_entry(&table[9], "cache_nice_tries",
6393 6394
		&sd->cache_nice_tries,
		sizeof(int), 0644, proc_dointvec_minmax);
6395
	set_table_entry(&table[10], "flags", &sd->flags,
6396
		sizeof(int), 0644, proc_dointvec_minmax);
6397 6398 6399
	set_table_entry(&table[11], "name", sd->name,
		CORENAME_MAX_SIZE, 0444, proc_dostring);
	/* &table[12] is terminator */
6400 6401 6402 6403

	return table;
}

6404
static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
6405 6406 6407 6408 6409 6410 6411 6412 6413
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);
6414 6415
	if (table == NULL)
		return NULL;
6416 6417 6418 6419 6420

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
6421
		entry->mode = 0555;
6422 6423 6424 6425 6426 6427 6428 6429
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
6430
static void register_sched_domain_sysctl(void)
6431 6432 6433 6434 6435
{
	int i, cpu_num = num_online_cpus();
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

6436 6437 6438
	WARN_ON(sd_ctl_dir[0].child);
	sd_ctl_dir[0].child = entry;

6439 6440 6441
	if (entry == NULL)
		return;

6442
	for_each_online_cpu(i) {
6443 6444
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
6445
		entry->mode = 0555;
6446
		entry->child = sd_alloc_ctl_cpu_table(i);
6447
		entry++;
6448
	}
6449 6450

	WARN_ON(sd_sysctl_header);
6451 6452
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
6453

6454
/* may be called multiple times per register */
6455 6456
static void unregister_sched_domain_sysctl(void)
{
6457 6458
	if (sd_sysctl_header)
		unregister_sysctl_table(sd_sysctl_header);
6459
	sd_sysctl_header = NULL;
6460 6461
	if (sd_ctl_dir[0].child)
		sd_free_ctl_entry(&sd_ctl_dir[0].child);
6462
}
6463
#else
6464 6465 6466 6467
static void register_sched_domain_sysctl(void)
{
}
static void unregister_sched_domain_sysctl(void)
6468 6469 6470 6471
{
}
#endif

6472 6473 6474 6475 6476
static void set_rq_online(struct rq *rq)
{
	if (!rq->online) {
		const struct sched_class *class;

6477
		cpumask_set_cpu(rq->cpu, rq->rd->online);
6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496
		rq->online = 1;

		for_each_class(class) {
			if (class->rq_online)
				class->rq_online(rq);
		}
	}
}

static void set_rq_offline(struct rq *rq)
{
	if (rq->online) {
		const struct sched_class *class;

		for_each_class(class) {
			if (class->rq_offline)
				class->rq_offline(rq);
		}

6497
		cpumask_clear_cpu(rq->cpu, rq->rd->online);
6498 6499 6500 6501
		rq->online = 0;
	}
}

L
Linus Torvalds 已提交
6502 6503 6504 6505
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
6506 6507
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
6508 6509
{
	struct task_struct *p;
6510
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
6511
	unsigned long flags;
6512
	struct rq *rq;
L
Linus Torvalds 已提交
6513 6514

	switch (action) {
6515

L
Linus Torvalds 已提交
6516
	case CPU_UP_PREPARE:
6517
	case CPU_UP_PREPARE_FROZEN:
I
Ingo Molnar 已提交
6518
		p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
L
Linus Torvalds 已提交
6519 6520 6521 6522 6523
		if (IS_ERR(p))
			return NOTIFY_BAD;
		kthread_bind(p, cpu);
		/* Must be high prio: stop_machine expects to yield to it. */
		rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
6524
		__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
L
Linus Torvalds 已提交
6525 6526 6527
		task_rq_unlock(rq, &flags);
		cpu_rq(cpu)->migration_thread = p;
		break;
6528

L
Linus Torvalds 已提交
6529
	case CPU_ONLINE:
6530
	case CPU_ONLINE_FROZEN:
6531
		/* Strictly unnecessary, as first user will wake it. */
L
Linus Torvalds 已提交
6532
		wake_up_process(cpu_rq(cpu)->migration_thread);
6533 6534 6535 6536 6537

		/* Update our root-domain */
		rq = cpu_rq(cpu);
		spin_lock_irqsave(&rq->lock, flags);
		if (rq->rd) {
6538
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6539 6540

			set_rq_online(rq);
6541 6542
		}
		spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
6543
		break;
6544

L
Linus Torvalds 已提交
6545 6546
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
6547
	case CPU_UP_CANCELED_FROZEN:
6548 6549
		if (!cpu_rq(cpu)->migration_thread)
			break;
I
Ingo Molnar 已提交
6550
		/* Unbind it from offline cpu so it can run. Fall thru. */
6551
		kthread_bind(cpu_rq(cpu)->migration_thread,
R
Rusty Russell 已提交
6552
			     cpumask_any(cpu_online_mask));
L
Linus Torvalds 已提交
6553 6554 6555
		kthread_stop(cpu_rq(cpu)->migration_thread);
		cpu_rq(cpu)->migration_thread = NULL;
		break;
6556

L
Linus Torvalds 已提交
6557
	case CPU_DEAD:
6558
	case CPU_DEAD_FROZEN:
6559
		cpuset_lock(); /* around calls to cpuset_cpus_allowed_lock() */
L
Linus Torvalds 已提交
6560 6561 6562 6563 6564
		migrate_live_tasks(cpu);
		rq = cpu_rq(cpu);
		kthread_stop(rq->migration_thread);
		rq->migration_thread = NULL;
		/* Idle task back to normal (off runqueue, low prio) */
6565
		spin_lock_irq(&rq->lock);
I
Ingo Molnar 已提交
6566
		update_rq_clock(rq);
6567
		deactivate_task(rq, rq->idle, 0);
L
Linus Torvalds 已提交
6568
		rq->idle->static_prio = MAX_PRIO;
I
Ingo Molnar 已提交
6569 6570
		__setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
		rq->idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
6571
		migrate_dead_tasks(cpu);
6572
		spin_unlock_irq(&rq->lock);
6573
		cpuset_unlock();
L
Linus Torvalds 已提交
6574 6575 6576
		migrate_nr_uninterruptible(rq);
		BUG_ON(rq->nr_running != 0);

I
Ingo Molnar 已提交
6577 6578 6579 6580 6581
		/*
		 * No need to migrate the tasks: it was best-effort if
		 * they didn't take sched_hotcpu_mutex. Just wake up
		 * the requestors.
		 */
L
Linus Torvalds 已提交
6582 6583
		spin_lock_irq(&rq->lock);
		while (!list_empty(&rq->migration_queue)) {
6584 6585
			struct migration_req *req;

L
Linus Torvalds 已提交
6586
			req = list_entry(rq->migration_queue.next,
6587
					 struct migration_req, list);
L
Linus Torvalds 已提交
6588
			list_del_init(&req->list);
B
Brian King 已提交
6589
			spin_unlock_irq(&rq->lock);
L
Linus Torvalds 已提交
6590
			complete(&req->done);
B
Brian King 已提交
6591
			spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
6592 6593 6594
		}
		spin_unlock_irq(&rq->lock);
		break;
G
Gregory Haskins 已提交
6595

6596 6597
	case CPU_DYING:
	case CPU_DYING_FROZEN:
G
Gregory Haskins 已提交
6598 6599 6600 6601
		/* Update our root-domain */
		rq = cpu_rq(cpu);
		spin_lock_irqsave(&rq->lock, flags);
		if (rq->rd) {
6602
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6603
			set_rq_offline(rq);
G
Gregory Haskins 已提交
6604 6605 6606
		}
		spin_unlock_irqrestore(&rq->lock, flags);
		break;
L
Linus Torvalds 已提交
6607 6608 6609 6610 6611 6612 6613 6614
#endif
	}
	return NOTIFY_OK;
}

/* Register at highest priority so that task migration (migrate_all_tasks)
 * happens before everything else.
 */
6615
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
6616 6617 6618 6619
	.notifier_call = migration_call,
	.priority = 10
};

6620
static int __init migration_init(void)
L
Linus Torvalds 已提交
6621 6622
{
	void *cpu = (void *)(long)smp_processor_id();
6623
	int err;
6624 6625

	/* Start one for the boot CPU: */
6626 6627
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
6628 6629
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
6630 6631

	return err;
L
Linus Torvalds 已提交
6632
}
6633
early_initcall(migration_init);
L
Linus Torvalds 已提交
6634 6635 6636
#endif

#ifdef CONFIG_SMP
6637

6638
#ifdef CONFIG_SCHED_DEBUG
I
Ingo Molnar 已提交
6639

6640
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6641
				  struct cpumask *groupmask)
L
Linus Torvalds 已提交
6642
{
I
Ingo Molnar 已提交
6643
	struct sched_group *group = sd->groups;
6644
	char str[256];
L
Linus Torvalds 已提交
6645

R
Rusty Russell 已提交
6646
	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6647
	cpumask_clear(groupmask);
I
Ingo Molnar 已提交
6648 6649 6650 6651 6652 6653 6654 6655 6656

	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);

	if (!(sd->flags & SD_LOAD_BALANCE)) {
		printk("does not load-balance\n");
		if (sd->parent)
			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
					" has parent");
		return -1;
N
Nick Piggin 已提交
6657 6658
	}

6659
	printk(KERN_CONT "span %s level %s\n", str, sd->name);
I
Ingo Molnar 已提交
6660

6661
	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
I
Ingo Molnar 已提交
6662 6663 6664
		printk(KERN_ERR "ERROR: domain->span does not contain "
				"CPU%d\n", cpu);
	}
6665
	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
I
Ingo Molnar 已提交
6666 6667 6668
		printk(KERN_ERR "ERROR: domain->groups does not contain"
				" CPU%d\n", cpu);
	}
L
Linus Torvalds 已提交
6669

I
Ingo Molnar 已提交
6670
	printk(KERN_DEBUG "%*s groups:", level + 1, "");
L
Linus Torvalds 已提交
6671
	do {
I
Ingo Molnar 已提交
6672 6673 6674
		if (!group) {
			printk("\n");
			printk(KERN_ERR "ERROR: group is NULL\n");
L
Linus Torvalds 已提交
6675 6676 6677
			break;
		}

I
Ingo Molnar 已提交
6678 6679 6680 6681 6682 6683
		if (!group->__cpu_power) {
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: domain->cpu_power not "
					"set\n");
			break;
		}
L
Linus Torvalds 已提交
6684

6685
		if (!cpumask_weight(sched_group_cpus(group))) {
I
Ingo Molnar 已提交
6686 6687 6688 6689
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: empty group\n");
			break;
		}
L
Linus Torvalds 已提交
6690

6691
		if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
I
Ingo Molnar 已提交
6692 6693 6694 6695
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: repeated CPUs\n");
			break;
		}
L
Linus Torvalds 已提交
6696

6697
		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
L
Linus Torvalds 已提交
6698

R
Rusty Russell 已提交
6699
		cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
I
Ingo Molnar 已提交
6700
		printk(KERN_CONT " %s", str);
L
Linus Torvalds 已提交
6701

I
Ingo Molnar 已提交
6702 6703 6704
		group = group->next;
	} while (group != sd->groups);
	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
6705

6706
	if (!cpumask_equal(sched_domain_span(sd), groupmask))
I
Ingo Molnar 已提交
6707
		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
L
Linus Torvalds 已提交
6708

6709 6710
	if (sd->parent &&
	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
I
Ingo Molnar 已提交
6711 6712 6713 6714
		printk(KERN_ERR "ERROR: parent span is not a superset "
			"of domain->span\n");
	return 0;
}
L
Linus Torvalds 已提交
6715

I
Ingo Molnar 已提交
6716 6717
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
6718
	cpumask_var_t groupmask;
I
Ingo Molnar 已提交
6719
	int level = 0;
L
Linus Torvalds 已提交
6720

I
Ingo Molnar 已提交
6721 6722 6723 6724
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}
L
Linus Torvalds 已提交
6725

I
Ingo Molnar 已提交
6726 6727
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

6728
	if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6729 6730 6731 6732
		printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
		return;
	}

I
Ingo Molnar 已提交
6733
	for (;;) {
6734
		if (sched_domain_debug_one(sd, cpu, level, groupmask))
I
Ingo Molnar 已提交
6735
			break;
L
Linus Torvalds 已提交
6736 6737
		level++;
		sd = sd->parent;
6738
		if (!sd)
I
Ingo Molnar 已提交
6739 6740
			break;
	}
6741
	free_cpumask_var(groupmask);
L
Linus Torvalds 已提交
6742
}
6743
#else /* !CONFIG_SCHED_DEBUG */
6744
# define sched_domain_debug(sd, cpu) do { } while (0)
6745
#endif /* CONFIG_SCHED_DEBUG */
L
Linus Torvalds 已提交
6746

6747
static int sd_degenerate(struct sched_domain *sd)
6748
{
6749
	if (cpumask_weight(sched_domain_span(sd)) == 1)
6750 6751 6752 6753 6754 6755
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
6756 6757 6758
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
	if (sd->flags & (SD_WAKE_IDLE |
			 SD_WAKE_AFFINE |
			 SD_WAKE_BALANCE))
		return 0;

	return 1;
}

6772 6773
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6774 6775 6776 6777 6778 6779
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

6780
	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791
		return 0;

	/* Does parent contain flags not in child? */
	/* WAKE_BALANCE is a subset of WAKE_AFFINE */
	if (cflags & SD_WAKE_AFFINE)
		pflags &= ~SD_WAKE_BALANCE;
	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
6792 6793 6794
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
6795 6796
		if (nr_node_ids == 1)
			pflags &= ~SD_SERIALIZE;
6797 6798 6799 6800 6801 6802 6803
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

6804 6805
static void free_rootdomain(struct root_domain *rd)
{
6806 6807
	cpupri_cleanup(&rd->cpupri);

6808 6809 6810 6811 6812 6813
	free_cpumask_var(rd->rto_mask);
	free_cpumask_var(rd->online);
	free_cpumask_var(rd->span);
	kfree(rd);
}

G
Gregory Haskins 已提交
6814 6815 6816 6817 6818 6819 6820 6821 6822
static void rq_attach_root(struct rq *rq, struct root_domain *rd)
{
	unsigned long flags;

	spin_lock_irqsave(&rq->lock, flags);

	if (rq->rd) {
		struct root_domain *old_rd = rq->rd;

6823
		if (cpumask_test_cpu(rq->cpu, old_rd->online))
6824
			set_rq_offline(rq);
G
Gregory Haskins 已提交
6825

6826
		cpumask_clear_cpu(rq->cpu, old_rd->span);
6827

G
Gregory Haskins 已提交
6828
		if (atomic_dec_and_test(&old_rd->refcount))
6829
			free_rootdomain(old_rd);
G
Gregory Haskins 已提交
6830 6831 6832 6833 6834
	}

	atomic_inc(&rd->refcount);
	rq->rd = rd;

6835 6836
	cpumask_set_cpu(rq->cpu, rd->span);
	if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6837
		set_rq_online(rq);
G
Gregory Haskins 已提交
6838 6839 6840 6841

	spin_unlock_irqrestore(&rq->lock, flags);
}

6842
static int init_rootdomain(struct root_domain *rd, bool bootmem)
G
Gregory Haskins 已提交
6843 6844 6845
{
	memset(rd, 0, sizeof(*rd));

6846 6847 6848 6849
	if (bootmem) {
		alloc_bootmem_cpumask_var(&def_root_domain.span);
		alloc_bootmem_cpumask_var(&def_root_domain.online);
		alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6850
		cpupri_init(&rd->cpupri, true);
6851 6852 6853 6854 6855 6856 6857 6858 6859
		return 0;
	}

	if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
		goto free_rd;
	if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
		goto free_span;
	if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
		goto free_online;
6860

6861 6862
	if (cpupri_init(&rd->cpupri, false) != 0)
		goto free_rto_mask;
6863
	return 0;
6864

6865 6866
free_rto_mask:
	free_cpumask_var(rd->rto_mask);
6867 6868 6869 6870 6871 6872 6873
free_online:
	free_cpumask_var(rd->online);
free_span:
	free_cpumask_var(rd->span);
free_rd:
	kfree(rd);
	return -ENOMEM;
G
Gregory Haskins 已提交
6874 6875 6876 6877
}

static void init_defrootdomain(void)
{
6878 6879
	init_rootdomain(&def_root_domain, true);

G
Gregory Haskins 已提交
6880 6881 6882
	atomic_set(&def_root_domain.refcount, 1);
}

6883
static struct root_domain *alloc_rootdomain(void)
G
Gregory Haskins 已提交
6884 6885 6886 6887 6888 6889 6890
{
	struct root_domain *rd;

	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
	if (!rd)
		return NULL;

6891 6892 6893 6894
	if (init_rootdomain(rd, false) != 0) {
		kfree(rd);
		return NULL;
	}
G
Gregory Haskins 已提交
6895 6896 6897 6898

	return rd;
}

L
Linus Torvalds 已提交
6899
/*
I
Ingo Molnar 已提交
6900
 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
L
Linus Torvalds 已提交
6901 6902
 * hold the hotplug lock.
 */
I
Ingo Molnar 已提交
6903 6904
static void
cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
L
Linus Torvalds 已提交
6905
{
6906
	struct rq *rq = cpu_rq(cpu);
6907 6908 6909
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
6910
	for (tmp = sd; tmp; ) {
6911 6912 6913
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
6914

6915
		if (sd_parent_degenerate(tmp, parent)) {
6916
			tmp->parent = parent->parent;
6917 6918
			if (parent->parent)
				parent->parent->child = tmp;
6919 6920
		} else
			tmp = tmp->parent;
6921 6922
	}

6923
	if (sd && sd_degenerate(sd)) {
6924
		sd = sd->parent;
6925 6926 6927
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
6928 6929 6930

	sched_domain_debug(sd, cpu);

G
Gregory Haskins 已提交
6931
	rq_attach_root(rq, rd);
N
Nick Piggin 已提交
6932
	rcu_assign_pointer(rq->sd, sd);
L
Linus Torvalds 已提交
6933 6934 6935
}

/* cpus with isolated domains */
6936
static cpumask_var_t cpu_isolated_map;
L
Linus Torvalds 已提交
6937 6938 6939 6940

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
R
Rusty Russell 已提交
6941
	cpulist_parse(str, cpu_isolated_map);
L
Linus Torvalds 已提交
6942 6943 6944
	return 1;
}

I
Ingo Molnar 已提交
6945
__setup("isolcpus=", isolated_cpu_setup);
L
Linus Torvalds 已提交
6946 6947

/*
6948 6949
 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
 * to a function which identifies what group(along with sched group) a CPU
6950 6951
 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
 * (due to the fact that we keep track of groups covered with a struct cpumask).
L
Linus Torvalds 已提交
6952 6953 6954 6955 6956
 *
 * init_sched_build_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
 */
6957
static void
6958 6959 6960
init_sched_build_groups(const struct cpumask *span,
			const struct cpumask *cpu_map,
			int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6961
					struct sched_group **sg,
6962 6963
					struct cpumask *tmpmask),
			struct cpumask *covered, struct cpumask *tmpmask)
L
Linus Torvalds 已提交
6964 6965 6966 6967
{
	struct sched_group *first = NULL, *last = NULL;
	int i;

6968
	cpumask_clear(covered);
6969

6970
	for_each_cpu(i, span) {
6971
		struct sched_group *sg;
6972
		int group = group_fn(i, cpu_map, &sg, tmpmask);
L
Linus Torvalds 已提交
6973 6974
		int j;

6975
		if (cpumask_test_cpu(i, covered))
L
Linus Torvalds 已提交
6976 6977
			continue;

6978
		cpumask_clear(sched_group_cpus(sg));
6979
		sg->__cpu_power = 0;
L
Linus Torvalds 已提交
6980

6981
		for_each_cpu(j, span) {
6982
			if (group_fn(j, cpu_map, NULL, tmpmask) != group)
L
Linus Torvalds 已提交
6983 6984
				continue;

6985
			cpumask_set_cpu(j, covered);
6986
			cpumask_set_cpu(j, sched_group_cpus(sg));
L
Linus Torvalds 已提交
6987 6988 6989 6990 6991 6992 6993 6994 6995 6996
		}
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
}

6997
#define SD_NODES_PER_DOMAIN 16
L
Linus Torvalds 已提交
6998

6999
#ifdef CONFIG_NUMA
7000

7001 7002 7003 7004 7005
/**
 * find_next_best_node - find the next node to include in a sched_domain
 * @node: node whose sched_domain we're building
 * @used_nodes: nodes already in the sched_domain
 *
I
Ingo Molnar 已提交
7006
 * Find the next node to include in a given scheduling domain. Simply
7007 7008 7009 7010
 * finds the closest node not already in the @used_nodes map.
 *
 * Should use nodemask_t.
 */
7011
static int find_next_best_node(int node, nodemask_t *used_nodes)
7012 7013 7014 7015 7016
{
	int i, n, val, min_val, best_node = 0;

	min_val = INT_MAX;

7017
	for (i = 0; i < nr_node_ids; i++) {
7018
		/* Start at @node */
7019
		n = (node + i) % nr_node_ids;
7020 7021 7022 7023 7024

		if (!nr_cpus_node(n))
			continue;

		/* Skip already used nodes */
7025
		if (node_isset(n, *used_nodes))
7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036
			continue;

		/* Simple min distance search */
		val = node_distance(node, n);

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

7037
	node_set(best_node, *used_nodes);
7038 7039 7040 7041 7042 7043
	return best_node;
}

/**
 * sched_domain_node_span - get a cpumask for a node's sched_domain
 * @node: node whose cpumask we're constructing
7044
 * @span: resulting cpumask
7045
 *
I
Ingo Molnar 已提交
7046
 * Given a node, construct a good cpumask for its sched_domain to span. It
7047 7048 7049
 * should be one that prevents unnecessary balancing, but also spreads tasks
 * out optimally.
 */
7050
static void sched_domain_node_span(int node, struct cpumask *span)
7051
{
7052
	nodemask_t used_nodes;
7053
	/* FIXME: use cpumask_of_node() */
7054
	node_to_cpumask_ptr(nodemask, node);
7055
	int i;
7056

7057
	cpus_clear(*span);
7058
	nodes_clear(used_nodes);
7059

7060
	cpus_or(*span, *span, *nodemask);
7061
	node_set(node, used_nodes);
7062 7063

	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
7064
		int next_node = find_next_best_node(node, &used_nodes);
7065

7066
		node_to_cpumask_ptr_next(nodemask, next_node);
7067
		cpus_or(*span, *span, *nodemask);
7068 7069
	}
}
7070
#endif /* CONFIG_NUMA */
7071

7072
int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7073

7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088
/*
 * The cpus mask in sched_group and sched_domain hangs off the end.
 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
 * for nr_cpu_ids < CONFIG_NR_CPUS.
 */
struct static_sched_group {
	struct sched_group sg;
	DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
};

struct static_sched_domain {
	struct sched_domain sd;
	DECLARE_BITMAP(span, CONFIG_NR_CPUS);
};

7089
/*
7090
 * SMT sched-domains:
7091
 */
L
Linus Torvalds 已提交
7092
#ifdef CONFIG_SCHED_SMT
7093 7094
static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7095

I
Ingo Molnar 已提交
7096
static int
7097 7098
cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
		 struct sched_group **sg, struct cpumask *unused)
L
Linus Torvalds 已提交
7099
{
7100
	if (sg)
7101
		*sg = &per_cpu(sched_group_cpus, cpu).sg;
L
Linus Torvalds 已提交
7102 7103
	return cpu;
}
7104
#endif /* CONFIG_SCHED_SMT */
L
Linus Torvalds 已提交
7105

7106 7107 7108
/*
 * multi-core sched-domains:
 */
7109
#ifdef CONFIG_SCHED_MC
7110 7111
static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7112
#endif /* CONFIG_SCHED_MC */
7113 7114

#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
7115
static int
7116 7117
cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
		  struct sched_group **sg, struct cpumask *mask)
7118
{
7119
	int group;
7120

7121 7122
	cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
	group = cpumask_first(mask);
7123
	if (sg)
7124
		*sg = &per_cpu(sched_group_core, group).sg;
7125
	return group;
7126 7127
}
#elif defined(CONFIG_SCHED_MC)
I
Ingo Molnar 已提交
7128
static int
7129 7130
cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
		  struct sched_group **sg, struct cpumask *unused)
7131
{
7132
	if (sg)
7133
		*sg = &per_cpu(sched_group_core, cpu).sg;
7134 7135 7136 7137
	return cpu;
}
#endif

7138 7139
static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7140

I
Ingo Molnar 已提交
7141
static int
7142 7143
cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
		  struct sched_group **sg, struct cpumask *mask)
L
Linus Torvalds 已提交
7144
{
7145
	int group;
7146
#ifdef CONFIG_SCHED_MC
7147
	/* FIXME: Use cpu_coregroup_mask. */
7148 7149
	*mask = cpu_coregroup_map(cpu);
	cpus_and(*mask, *mask, *cpu_map);
7150
	group = cpumask_first(mask);
7151
#elif defined(CONFIG_SCHED_SMT)
7152 7153
	cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
	group = cpumask_first(mask);
L
Linus Torvalds 已提交
7154
#else
7155
	group = cpu;
L
Linus Torvalds 已提交
7156
#endif
7157
	if (sg)
7158
		*sg = &per_cpu(sched_group_phys, group).sg;
7159
	return group;
L
Linus Torvalds 已提交
7160 7161 7162 7163
}

#ifdef CONFIG_NUMA
/*
7164 7165 7166
 * The init_sched_build_groups can't handle what we want to do with node
 * groups, so roll our own. Now each node has its own list of groups which
 * gets dynamically allocated.
L
Linus Torvalds 已提交
7167
 */
7168
static DEFINE_PER_CPU(struct sched_domain, node_domains);
7169
static struct sched_group ***sched_group_nodes_bycpu;
L
Linus Torvalds 已提交
7170

7171
static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7172
static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7173

7174 7175 7176
static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
				 struct sched_group **sg,
				 struct cpumask *nodemask)
7177
{
7178
	int group;
7179
	/* FIXME: use cpumask_of_node */
7180
	node_to_cpumask_ptr(pnodemask, cpu_to_node(cpu));
7181

7182 7183
	cpumask_and(nodemask, pnodemask, cpu_map);
	group = cpumask_first(nodemask);
7184 7185

	if (sg)
7186
		*sg = &per_cpu(sched_group_allnodes, group).sg;
7187
	return group;
L
Linus Torvalds 已提交
7188
}
7189

7190 7191 7192 7193 7194 7195 7196
static void init_numa_sched_groups_power(struct sched_group *group_head)
{
	struct sched_group *sg = group_head;
	int j;

	if (!sg)
		return;
7197
	do {
7198
		for_each_cpu(j, sched_group_cpus(sg)) {
7199
			struct sched_domain *sd;
7200

7201
			sd = &per_cpu(phys_domains, j).sd;
7202
			if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7203 7204 7205 7206 7207 7208
				/*
				 * Only add "power" once for each
				 * physical package.
				 */
				continue;
			}
7209

7210 7211 7212 7213
			sg_inc_cpu_power(sg, sd->groups->__cpu_power);
		}
		sg = sg->next;
	} while (sg != group_head);
7214
}
7215
#endif /* CONFIG_NUMA */
L
Linus Torvalds 已提交
7216

7217
#ifdef CONFIG_NUMA
7218
/* Free memory allocated for various sched_group structures */
7219 7220
static void free_sched_groups(const struct cpumask *cpu_map,
			      struct cpumask *nodemask)
7221
{
7222
	int cpu, i;
7223

7224
	for_each_cpu(cpu, cpu_map) {
7225 7226 7227 7228 7229 7230
		struct sched_group **sched_group_nodes
			= sched_group_nodes_bycpu[cpu];

		if (!sched_group_nodes)
			continue;

7231
		for (i = 0; i < nr_node_ids; i++) {
7232
			struct sched_group *oldsg, *sg = sched_group_nodes[i];
7233
			/* FIXME: Use cpumask_of_node */
7234
			node_to_cpumask_ptr(pnodemask, i);
7235

7236
			cpus_and(*nodemask, *pnodemask, *cpu_map);
7237
			if (cpumask_empty(nodemask))
7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253
				continue;

			if (sg == NULL)
				continue;
			sg = sg->next;
next_sg:
			oldsg = sg;
			sg = sg->next;
			kfree(oldsg);
			if (oldsg != sched_group_nodes[i])
				goto next_sg;
		}
		kfree(sched_group_nodes);
		sched_group_nodes_bycpu[cpu] = NULL;
	}
}
7254
#else /* !CONFIG_NUMA */
7255 7256
static void free_sched_groups(const struct cpumask *cpu_map,
			      struct cpumask *nodemask)
7257 7258
{
}
7259
#endif /* CONFIG_NUMA */
7260

7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 *
 * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
 * the maximum number of tasks a group can handle in the presence of other idle
 * or lightly loaded groups in the same sched domain.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
	struct sched_domain *child;
	struct sched_group *group;

	WARN_ON(!sd || !sd->groups);

7282
	if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7283 7284 7285 7286
		return;

	child = sd->child;

7287 7288
	sd->groups->__cpu_power = 0;

7289 7290 7291 7292 7293 7294 7295 7296 7297 7298
	/*
	 * For perf policy, if the groups in child domain share resources
	 * (for example cores sharing some portions of the cache hierarchy
	 * or SMT), then set this domain groups cpu_power such that each group
	 * can handle only one task, when there are other idle groups in the
	 * same sched domain.
	 */
	if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
		       (child->flags &
			(SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
7299
		sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
7300 7301 7302 7303 7304 7305 7306 7307
		return;
	}

	/*
	 * add cpu_power of each child group to this groups cpu_power
	 */
	group = child->groups;
	do {
7308
		sg_inc_cpu_power(sd->groups, group->__cpu_power);
7309 7310 7311 7312
		group = group->next;
	} while (group != child->groups);
}

7313 7314 7315 7316 7317
/*
 * Initializers for schedule domains
 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
 */

7318 7319 7320 7321 7322 7323
#ifdef CONFIG_SCHED_DEBUG
# define SD_INIT_NAME(sd, type)		sd->name = #type
#else
# define SD_INIT_NAME(sd, type)		do { } while (0)
#endif

7324
#define	SD_INIT(sd, type)	sd_init_##type(sd)
7325

7326 7327 7328 7329 7330
#define SD_INIT_FUNC(type)	\
static noinline void sd_init_##type(struct sched_domain *sd)	\
{								\
	memset(sd, 0, sizeof(*sd));				\
	*sd = SD_##type##_INIT;					\
7331
	sd->level = SD_LV_##type;				\
7332
	SD_INIT_NAME(sd, type);					\
7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346
}

SD_INIT_FUNC(CPU)
#ifdef CONFIG_NUMA
 SD_INIT_FUNC(ALLNODES)
 SD_INIT_FUNC(NODE)
#endif
#ifdef CONFIG_SCHED_SMT
 SD_INIT_FUNC(SIBLING)
#endif
#ifdef CONFIG_SCHED_MC
 SD_INIT_FUNC(MC)
#endif

7347 7348 7349 7350
static int default_relax_domain_level = -1;

static int __init setup_relax_domain_level(char *str)
{
7351 7352 7353 7354 7355 7356
	unsigned long val;

	val = simple_strtoul(str, NULL, 0);
	if (val < SD_LV_MAX)
		default_relax_domain_level = val;

7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381
	return 1;
}
__setup("relax_domain_level=", setup_relax_domain_level);

static void set_domain_attribute(struct sched_domain *sd,
				 struct sched_domain_attr *attr)
{
	int request;

	if (!attr || attr->relax_domain_level < 0) {
		if (default_relax_domain_level < 0)
			return;
		else
			request = default_relax_domain_level;
	} else
		request = attr->relax_domain_level;
	if (request < sd->level) {
		/* turn off idle balance on this domain */
		sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
	} else {
		/* turn on idle balance on this domain */
		sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
	}
}

L
Linus Torvalds 已提交
7382
/*
7383 7384
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
L
Linus Torvalds 已提交
7385
 */
7386
static int __build_sched_domains(const struct cpumask *cpu_map,
7387
				 struct sched_domain_attr *attr)
L
Linus Torvalds 已提交
7388
{
7389
	int i, err = -ENOMEM;
G
Gregory Haskins 已提交
7390
	struct root_domain *rd;
7391 7392
	cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
		tmpmask;
7393
#ifdef CONFIG_NUMA
7394
	cpumask_var_t domainspan, covered, notcovered;
7395
	struct sched_group **sched_group_nodes = NULL;
7396
	int sd_allnodes = 0;
7397

7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417
	if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
		goto out;
	if (!alloc_cpumask_var(&covered, GFP_KERNEL))
		goto free_domainspan;
	if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
		goto free_covered;
#endif

	if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
		goto free_notcovered;
	if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
		goto free_nodemask;
	if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
		goto free_this_sibling_map;
	if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
		goto free_this_core_map;
	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
		goto free_send_covered;

#ifdef CONFIG_NUMA
7418 7419 7420
	/*
	 * Allocate the per-node list of sched groups
	 */
7421
	sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
I
Ingo Molnar 已提交
7422
				    GFP_KERNEL);
7423 7424
	if (!sched_group_nodes) {
		printk(KERN_WARNING "Can not alloc sched group node list\n");
7425
		goto free_tmpmask;
7426 7427
	}
#endif
L
Linus Torvalds 已提交
7428

7429
	rd = alloc_rootdomain();
G
Gregory Haskins 已提交
7430 7431
	if (!rd) {
		printk(KERN_WARNING "Cannot alloc root domain\n");
7432
		goto free_sched_groups;
G
Gregory Haskins 已提交
7433 7434
	}

7435
#ifdef CONFIG_NUMA
7436
	sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7437 7438
#endif

L
Linus Torvalds 已提交
7439
	/*
7440
	 * Set up domains for cpus specified by the cpu_map.
L
Linus Torvalds 已提交
7441
	 */
7442
	for_each_cpu(i, cpu_map) {
L
Linus Torvalds 已提交
7443 7444
		struct sched_domain *sd = NULL, *p;

7445
		/* FIXME: use cpumask_of_node */
7446 7447
		*nodemask = node_to_cpumask(cpu_to_node(i));
		cpus_and(*nodemask, *nodemask, *cpu_map);
L
Linus Torvalds 已提交
7448 7449

#ifdef CONFIG_NUMA
7450 7451
		if (cpumask_weight(cpu_map) >
				SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7452
			sd = &per_cpu(allnodes_domains, i);
7453
			SD_INIT(sd, ALLNODES);
7454
			set_domain_attribute(sd, attr);
7455
			cpumask_copy(sched_domain_span(sd), cpu_map);
7456
			cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7457
			p = sd;
7458
			sd_allnodes = 1;
7459 7460 7461
		} else
			p = NULL;

L
Linus Torvalds 已提交
7462
		sd = &per_cpu(node_domains, i);
7463
		SD_INIT(sd, NODE);
7464
		set_domain_attribute(sd, attr);
7465
		sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7466
		sd->parent = p;
7467 7468
		if (p)
			p->child = sd;
7469 7470
		cpumask_and(sched_domain_span(sd),
			    sched_domain_span(sd), cpu_map);
L
Linus Torvalds 已提交
7471 7472 7473
#endif

		p = sd;
7474
		sd = &per_cpu(phys_domains, i).sd;
7475
		SD_INIT(sd, CPU);
7476
		set_domain_attribute(sd, attr);
7477
		cpumask_copy(sched_domain_span(sd), nodemask);
L
Linus Torvalds 已提交
7478
		sd->parent = p;
7479 7480
		if (p)
			p->child = sd;
7481
		cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
L
Linus Torvalds 已提交
7482

7483 7484
#ifdef CONFIG_SCHED_MC
		p = sd;
7485
		sd = &per_cpu(core_domains, i).sd;
7486
		SD_INIT(sd, MC);
7487
		set_domain_attribute(sd, attr);
7488 7489 7490
		*sched_domain_span(sd) = cpu_coregroup_map(i);
		cpumask_and(sched_domain_span(sd),
			    sched_domain_span(sd), cpu_map);
7491
		sd->parent = p;
7492
		p->child = sd;
7493
		cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
7494 7495
#endif

L
Linus Torvalds 已提交
7496 7497
#ifdef CONFIG_SCHED_SMT
		p = sd;
7498
		sd = &per_cpu(cpu_domains, i).sd;
7499
		SD_INIT(sd, SIBLING);
7500
		set_domain_attribute(sd, attr);
7501 7502
		cpumask_and(sched_domain_span(sd),
			    &per_cpu(cpu_sibling_map, i), cpu_map);
L
Linus Torvalds 已提交
7503
		sd->parent = p;
7504
		p->child = sd;
7505
		cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
L
Linus Torvalds 已提交
7506 7507 7508 7509 7510
#endif
	}

#ifdef CONFIG_SCHED_SMT
	/* Set up CPU (sibling) groups */
7511
	for_each_cpu(i, cpu_map) {
7512 7513 7514
		cpumask_and(this_sibling_map,
			    &per_cpu(cpu_sibling_map, i), cpu_map);
		if (i != cpumask_first(this_sibling_map))
L
Linus Torvalds 已提交
7515 7516
			continue;

I
Ingo Molnar 已提交
7517
		init_sched_build_groups(this_sibling_map, cpu_map,
7518 7519
					&cpu_to_cpu_group,
					send_covered, tmpmask);
L
Linus Torvalds 已提交
7520 7521 7522
	}
#endif

7523 7524
#ifdef CONFIG_SCHED_MC
	/* Set up multi-core groups */
7525
	for_each_cpu(i, cpu_map) {
7526
		/* FIXME: Use cpu_coregroup_mask */
7527 7528
		*this_core_map = cpu_coregroup_map(i);
		cpus_and(*this_core_map, *this_core_map, *cpu_map);
7529
		if (i != cpumask_first(this_core_map))
7530
			continue;
7531

I
Ingo Molnar 已提交
7532
		init_sched_build_groups(this_core_map, cpu_map,
7533 7534
					&cpu_to_core_group,
					send_covered, tmpmask);
7535 7536 7537
	}
#endif

L
Linus Torvalds 已提交
7538
	/* Set up physical groups */
7539
	for (i = 0; i < nr_node_ids; i++) {
7540
		/* FIXME: Use cpumask_of_node */
7541 7542
		*nodemask = node_to_cpumask(i);
		cpus_and(*nodemask, *nodemask, *cpu_map);
7543
		if (cpumask_empty(nodemask))
L
Linus Torvalds 已提交
7544 7545
			continue;

7546 7547 7548
		init_sched_build_groups(nodemask, cpu_map,
					&cpu_to_phys_group,
					send_covered, tmpmask);
L
Linus Torvalds 已提交
7549 7550 7551 7552
	}

#ifdef CONFIG_NUMA
	/* Set up node groups */
7553 7554 7555 7556 7557
	if (sd_allnodes) {
		init_sched_build_groups(cpu_map, cpu_map,
					&cpu_to_allnodes_group,
					send_covered, tmpmask);
	}
7558

7559
	for (i = 0; i < nr_node_ids; i++) {
7560 7561 7562 7563
		/* Set up node groups */
		struct sched_group *sg, *prev;
		int j;

7564
		/* FIXME: Use cpumask_of_node */
7565
		*nodemask = node_to_cpumask(i);
7566
		cpumask_clear(covered);
7567 7568

		cpus_and(*nodemask, *nodemask, *cpu_map);
7569
		if (cpumask_empty(nodemask)) {
7570
			sched_group_nodes[i] = NULL;
7571
			continue;
7572
		}
7573

7574
		sched_domain_node_span(i, domainspan);
7575
		cpumask_and(domainspan, domainspan, cpu_map);
7576

7577 7578
		sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
				  GFP_KERNEL, i);
7579 7580 7581 7582 7583
		if (!sg) {
			printk(KERN_WARNING "Can not alloc domain group for "
				"node %d\n", i);
			goto error;
		}
7584
		sched_group_nodes[i] = sg;
7585
		for_each_cpu(j, nodemask) {
7586
			struct sched_domain *sd;
I
Ingo Molnar 已提交
7587

7588 7589 7590
			sd = &per_cpu(node_domains, j);
			sd->groups = sg;
		}
7591
		sg->__cpu_power = 0;
7592
		cpumask_copy(sched_group_cpus(sg), nodemask);
7593
		sg->next = sg;
7594
		cpumask_or(covered, covered, nodemask);
7595 7596
		prev = sg;

7597 7598
		for (j = 0; j < nr_node_ids; j++) {
			int n = (i + j) % nr_node_ids;
7599
			/* FIXME: Use cpumask_of_node */
7600
			node_to_cpumask_ptr(pnodemask, n);
7601

7602 7603 7604 7605
			cpumask_complement(notcovered, covered);
			cpumask_and(tmpmask, notcovered, cpu_map);
			cpumask_and(tmpmask, tmpmask, domainspan);
			if (cpumask_empty(tmpmask))
7606 7607
				break;

7608 7609
			cpumask_and(tmpmask, tmpmask, pnodemask);
			if (cpumask_empty(tmpmask))
7610 7611
				continue;

7612 7613
			sg = kmalloc_node(sizeof(struct sched_group) +
					  cpumask_size(),
7614
					  GFP_KERNEL, i);
7615 7616 7617
			if (!sg) {
				printk(KERN_WARNING
				"Can not alloc domain group for node %d\n", j);
7618
				goto error;
7619
			}
7620
			sg->__cpu_power = 0;
7621
			cpumask_copy(sched_group_cpus(sg), tmpmask);
7622
			sg->next = prev->next;
7623
			cpumask_or(covered, covered, tmpmask);
7624 7625 7626 7627
			prev->next = sg;
			prev = sg;
		}
	}
L
Linus Torvalds 已提交
7628 7629 7630
#endif

	/* Calculate CPU power for physical packages and nodes */
7631
#ifdef CONFIG_SCHED_SMT
7632
	for_each_cpu(i, cpu_map) {
7633
		struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
I
Ingo Molnar 已提交
7634

7635
		init_sched_groups_power(i, sd);
7636
	}
L
Linus Torvalds 已提交
7637
#endif
7638
#ifdef CONFIG_SCHED_MC
7639
	for_each_cpu(i, cpu_map) {
7640
		struct sched_domain *sd = &per_cpu(core_domains, i).sd;
I
Ingo Molnar 已提交
7641

7642
		init_sched_groups_power(i, sd);
7643 7644
	}
#endif
7645

7646
	for_each_cpu(i, cpu_map) {
7647
		struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
I
Ingo Molnar 已提交
7648

7649
		init_sched_groups_power(i, sd);
L
Linus Torvalds 已提交
7650 7651
	}

7652
#ifdef CONFIG_NUMA
7653
	for (i = 0; i < nr_node_ids; i++)
7654
		init_numa_sched_groups_power(sched_group_nodes[i]);
7655

7656 7657
	if (sd_allnodes) {
		struct sched_group *sg;
7658

7659
		cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7660
								tmpmask);
7661 7662
		init_numa_sched_groups_power(sg);
	}
7663 7664
#endif

L
Linus Torvalds 已提交
7665
	/* Attach the domains */
7666
	for_each_cpu(i, cpu_map) {
L
Linus Torvalds 已提交
7667 7668
		struct sched_domain *sd;
#ifdef CONFIG_SCHED_SMT
7669
		sd = &per_cpu(cpu_domains, i).sd;
7670
#elif defined(CONFIG_SCHED_MC)
7671
		sd = &per_cpu(core_domains, i).sd;
L
Linus Torvalds 已提交
7672
#else
7673
		sd = &per_cpu(phys_domains, i).sd;
L
Linus Torvalds 已提交
7674
#endif
G
Gregory Haskins 已提交
7675
		cpu_attach_domain(sd, rd, i);
L
Linus Torvalds 已提交
7676
	}
7677

7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705
	err = 0;

free_tmpmask:
	free_cpumask_var(tmpmask);
free_send_covered:
	free_cpumask_var(send_covered);
free_this_core_map:
	free_cpumask_var(this_core_map);
free_this_sibling_map:
	free_cpumask_var(this_sibling_map);
free_nodemask:
	free_cpumask_var(nodemask);
free_notcovered:
#ifdef CONFIG_NUMA
	free_cpumask_var(notcovered);
free_covered:
	free_cpumask_var(covered);
free_domainspan:
	free_cpumask_var(domainspan);
out:
#endif
	return err;

free_sched_groups:
#ifdef CONFIG_NUMA
	kfree(sched_group_nodes);
#endif
	goto free_tmpmask;
7706

7707
#ifdef CONFIG_NUMA
7708
error:
7709
	free_sched_groups(cpu_map, tmpmask);
7710
	free_rootdomain(rd);
7711
	goto free_tmpmask;
7712
#endif
L
Linus Torvalds 已提交
7713
}
P
Paul Jackson 已提交
7714

7715
static int build_sched_domains(const struct cpumask *cpu_map)
7716 7717 7718 7719
{
	return __build_sched_domains(cpu_map, NULL);
}

7720
static struct cpumask *doms_cur;	/* current sched domains */
P
Paul Jackson 已提交
7721
static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
I
Ingo Molnar 已提交
7722 7723
static struct sched_domain_attr *dattr_cur;
				/* attribues of custom domains in 'doms_cur' */
P
Paul Jackson 已提交
7724 7725 7726

/*
 * Special case: If a kmalloc of a doms_cur partition (array of
7727 7728
 * cpumask) fails, then fallback to a single sched domain,
 * as determined by the single cpumask fallback_doms.
P
Paul Jackson 已提交
7729
 */
7730
static cpumask_var_t fallback_doms;
P
Paul Jackson 已提交
7731

7732 7733 7734 7735 7736 7737
/*
 * arch_update_cpu_topology lets virtualized architectures update the
 * cpu core maps. It is supposed to return 1 if the topology changed
 * or 0 if it stayed the same.
 */
int __attribute__((weak)) arch_update_cpu_topology(void)
7738
{
7739
	return 0;
7740 7741
}

7742
/*
I
Ingo Molnar 已提交
7743
 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
P
Paul Jackson 已提交
7744 7745
 * For now this just excludes isolated cpus, but could be used to
 * exclude other special cases in the future.
7746
 */
7747
static int arch_init_sched_domains(const struct cpumask *cpu_map)
7748
{
7749 7750
	int err;

7751
	arch_update_cpu_topology();
P
Paul Jackson 已提交
7752
	ndoms_cur = 1;
7753
	doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
P
Paul Jackson 已提交
7754
	if (!doms_cur)
7755
		doms_cur = fallback_doms;
7756
	cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7757
	dattr_cur = NULL;
7758
	err = build_sched_domains(doms_cur);
7759
	register_sched_domain_sysctl();
7760 7761

	return err;
7762 7763
}

7764 7765
static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
				       struct cpumask *tmpmask)
L
Linus Torvalds 已提交
7766
{
7767
	free_sched_groups(cpu_map, tmpmask);
7768
}
L
Linus Torvalds 已提交
7769

7770 7771 7772 7773
/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
7774
static void detach_destroy_domains(const struct cpumask *cpu_map)
7775
{
7776 7777
	/* Save because hotplug lock held. */
	static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7778 7779
	int i;

7780
	for_each_cpu(i, cpu_map)
G
Gregory Haskins 已提交
7781
		cpu_attach_domain(NULL, &def_root_domain, i);
7782
	synchronize_sched();
7783
	arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7784 7785
}

7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801
/* handle null as "default" */
static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
			struct sched_domain_attr *new, int idx_new)
{
	struct sched_domain_attr tmp;

	/* fast path */
	if (!new && !cur)
		return 1;

	tmp = SD_ATTR_INIT;
	return !memcmp(cur ? (cur + idx_cur) : &tmp,
			new ? (new + idx_new) : &tmp,
			sizeof(struct sched_domain_attr));
}

P
Paul Jackson 已提交
7802 7803
/*
 * Partition sched domains as specified by the 'ndoms_new'
I
Ingo Molnar 已提交
7804
 * cpumasks in the array doms_new[] of cpumasks. This compares
P
Paul Jackson 已提交
7805 7806 7807
 * doms_new[] to the current sched domain partitioning, doms_cur[].
 * It destroys each deleted domain and builds each new domain.
 *
7808
 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
I
Ingo Molnar 已提交
7809 7810 7811
 * The masks don't intersect (don't overlap.) We should setup one
 * sched domain for each mask. CPUs not in any of the cpumasks will
 * not be load balanced. If the same cpumask appears both in the
P
Paul Jackson 已提交
7812 7813 7814
 * current 'doms_cur' domains and in the new 'doms_new', we can leave
 * it as it is.
 *
I
Ingo Molnar 已提交
7815 7816
 * The passed in 'doms_new' should be kmalloc'd. This routine takes
 * ownership of it and will kfree it when done with it. If the caller
7817 7818 7819 7820
 * failed the kmalloc call, then it can pass in doms_new == NULL &&
 * ndoms_new == 1, and partition_sched_domains() will fallback to
 * the single partition 'fallback_doms', it also forces the domains
 * to be rebuilt.
P
Paul Jackson 已提交
7821
 *
7822
 * If doms_new == NULL it will be replaced with cpu_online_mask.
7823 7824
 * ndoms_new == 0 is a special case for destroying existing domains,
 * and it will not create the default domain.
7825
 *
P
Paul Jackson 已提交
7826 7827
 * Call with hotplug lock held
 */
7828 7829
/* FIXME: Change to struct cpumask *doms_new[] */
void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7830
			     struct sched_domain_attr *dattr_new)
P
Paul Jackson 已提交
7831
{
7832
	int i, j, n;
7833
	int new_topology;
P
Paul Jackson 已提交
7834

7835
	mutex_lock(&sched_domains_mutex);
7836

7837 7838 7839
	/* always unregister in case we don't destroy any domains */
	unregister_sched_domain_sysctl();

7840 7841 7842
	/* Let architecture update cpu core mappings. */
	new_topology = arch_update_cpu_topology();

7843
	n = doms_new ? ndoms_new : 0;
P
Paul Jackson 已提交
7844 7845 7846

	/* Destroy deleted domains */
	for (i = 0; i < ndoms_cur; i++) {
7847
		for (j = 0; j < n && !new_topology; j++) {
7848
			if (cpumask_equal(&doms_cur[i], &doms_new[j])
7849
			    && dattrs_equal(dattr_cur, i, dattr_new, j))
P
Paul Jackson 已提交
7850 7851 7852 7853 7854 7855 7856 7857
				goto match1;
		}
		/* no match - a current sched domain not in new doms_new[] */
		detach_destroy_domains(doms_cur + i);
match1:
		;
	}

7858 7859
	if (doms_new == NULL) {
		ndoms_cur = 0;
7860
		doms_new = fallback_doms;
7861
		cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7862
		WARN_ON_ONCE(dattr_new);
7863 7864
	}

P
Paul Jackson 已提交
7865 7866
	/* Build new domains */
	for (i = 0; i < ndoms_new; i++) {
7867
		for (j = 0; j < ndoms_cur && !new_topology; j++) {
7868
			if (cpumask_equal(&doms_new[i], &doms_cur[j])
7869
			    && dattrs_equal(dattr_new, i, dattr_cur, j))
P
Paul Jackson 已提交
7870 7871 7872
				goto match2;
		}
		/* no match - add a new doms_new */
7873 7874
		__build_sched_domains(doms_new + i,
					dattr_new ? dattr_new + i : NULL);
P
Paul Jackson 已提交
7875 7876 7877 7878 7879
match2:
		;
	}

	/* Remember the new sched domains */
7880
	if (doms_cur != fallback_doms)
P
Paul Jackson 已提交
7881
		kfree(doms_cur);
7882
	kfree(dattr_cur);	/* kfree(NULL) is safe */
P
Paul Jackson 已提交
7883
	doms_cur = doms_new;
7884
	dattr_cur = dattr_new;
P
Paul Jackson 已提交
7885
	ndoms_cur = ndoms_new;
7886 7887

	register_sched_domain_sysctl();
7888

7889
	mutex_unlock(&sched_domains_mutex);
P
Paul Jackson 已提交
7890 7891
}

7892
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
7893
int arch_reinit_sched_domains(void)
7894
{
7895
	get_online_cpus();
7896 7897 7898 7899

	/* Destroy domains first to force the rebuild */
	partition_sched_domains(0, NULL, NULL);

7900
	rebuild_sched_domains();
7901
	put_online_cpus();
7902

7903
	return 0;
7904 7905 7906 7907 7908
}

static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
{
	int ret;
7909
	unsigned int level = 0;
7910

7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921
	if (sscanf(buf, "%u", &level) != 1)
		return -EINVAL;

	/*
	 * level is always be positive so don't check for
	 * level < POWERSAVINGS_BALANCE_NONE which is 0
	 * What happens on 0 or 1 byte write,
	 * need to check for count as well?
	 */

	if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
7922 7923 7924
		return -EINVAL;

	if (smt)
7925
		sched_smt_power_savings = level;
7926
	else
7927
		sched_mc_power_savings = level;
7928 7929 7930 7931 7932 7933 7934

	ret = arch_reinit_sched_domains();

	return ret ? ret : count;
}

#ifdef CONFIG_SCHED_MC
7935 7936
static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
					   char *page)
7937 7938 7939
{
	return sprintf(page, "%u\n", sched_mc_power_savings);
}
7940
static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
7941
					    const char *buf, size_t count)
7942 7943 7944
{
	return sched_power_savings_store(buf, count, 0);
}
7945 7946 7947
static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
			 sched_mc_power_savings_show,
			 sched_mc_power_savings_store);
7948 7949 7950
#endif

#ifdef CONFIG_SCHED_SMT
7951 7952
static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
					    char *page)
7953 7954 7955
{
	return sprintf(page, "%u\n", sched_smt_power_savings);
}
7956
static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
7957
					     const char *buf, size_t count)
7958 7959 7960
{
	return sched_power_savings_store(buf, count, 1);
}
7961 7962
static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
		   sched_smt_power_savings_show,
A
Adrian Bunk 已提交
7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981
		   sched_smt_power_savings_store);
#endif

int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
{
	int err = 0;

#ifdef CONFIG_SCHED_SMT
	if (smt_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_smt_power_savings.attr);
#endif
#ifdef CONFIG_SCHED_MC
	if (!err && mc_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_mc_power_savings.attr);
#endif
	return err;
}
7982
#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
7983

7984
#ifndef CONFIG_CPUSETS
L
Linus Torvalds 已提交
7985
/*
7986 7987
 * Add online and remove offline CPUs from the scheduler domains.
 * When cpusets are enabled they take over this function.
L
Linus Torvalds 已提交
7988 7989 7990
 */
static int update_sched_domains(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
7991 7992 7993 7994 7995 7996
{
	switch (action) {
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
7997
		partition_sched_domains(1, NULL, NULL);
7998 7999 8000 8001 8002 8003 8004 8005 8006 8007
		return NOTIFY_OK;

	default:
		return NOTIFY_DONE;
	}
}
#endif

static int update_runtime(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
8008
{
P
Peter Zijlstra 已提交
8009 8010
	int cpu = (int)(long)hcpu;

L
Linus Torvalds 已提交
8011 8012
	switch (action) {
	case CPU_DOWN_PREPARE:
8013
	case CPU_DOWN_PREPARE_FROZEN:
P
Peter Zijlstra 已提交
8014
		disable_runtime(cpu_rq(cpu));
L
Linus Torvalds 已提交
8015 8016 8017
		return NOTIFY_OK;

	case CPU_DOWN_FAILED:
8018
	case CPU_DOWN_FAILED_FROZEN:
L
Linus Torvalds 已提交
8019
	case CPU_ONLINE:
8020
	case CPU_ONLINE_FROZEN:
P
Peter Zijlstra 已提交
8021
		enable_runtime(cpu_rq(cpu));
8022 8023
		return NOTIFY_OK;

L
Linus Torvalds 已提交
8024 8025 8026 8027 8028 8029 8030
	default:
		return NOTIFY_DONE;
	}
}

void __init sched_init_smp(void)
{
8031 8032 8033
	cpumask_var_t non_isolated_cpus;

	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
8034

8035 8036 8037 8038 8039
#if defined(CONFIG_NUMA)
	sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
								GFP_KERNEL);
	BUG_ON(sched_group_nodes_bycpu == NULL);
#endif
8040
	get_online_cpus();
8041
	mutex_lock(&sched_domains_mutex);
8042 8043 8044 8045
	arch_init_sched_domains(cpu_online_mask);
	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
	if (cpumask_empty(non_isolated_cpus))
		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8046
	mutex_unlock(&sched_domains_mutex);
8047
	put_online_cpus();
8048 8049

#ifndef CONFIG_CPUSETS
L
Linus Torvalds 已提交
8050 8051
	/* XXX: Theoretical race here - CPU may be hotplugged now */
	hotcpu_notifier(update_sched_domains, 0);
8052 8053 8054 8055 8056
#endif

	/* RT runtime code needs to handle some hotplug events */
	hotcpu_notifier(update_runtime, 0);

8057
	init_hrtick();
8058 8059

	/* Move init over to a non-isolated CPU */
8060
	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8061
		BUG();
I
Ingo Molnar 已提交
8062
	sched_init_granularity();
8063
	free_cpumask_var(non_isolated_cpus);
8064 8065

	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8066
	init_sched_rt_class();
L
Linus Torvalds 已提交
8067 8068 8069 8070
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
8071
	sched_init_granularity();
L
Linus Torvalds 已提交
8072 8073 8074 8075 8076 8077 8078 8079 8080 8081
}
#endif /* CONFIG_SMP */

int in_sched_functions(unsigned long addr)
{
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

A
Alexey Dobriyan 已提交
8082
static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
I
Ingo Molnar 已提交
8083 8084
{
	cfs_rq->tasks_timeline = RB_ROOT;
8085
	INIT_LIST_HEAD(&cfs_rq->tasks);
I
Ingo Molnar 已提交
8086 8087 8088
#ifdef CONFIG_FAIR_GROUP_SCHED
	cfs_rq->rq = rq;
#endif
P
Peter Zijlstra 已提交
8089
	cfs_rq->min_vruntime = (u64)(-(1LL << 20));
I
Ingo Molnar 已提交
8090 8091
}

P
Peter Zijlstra 已提交
8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104
static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
{
	struct rt_prio_array *array;
	int i;

	array = &rt_rq->active;
	for (i = 0; i < MAX_RT_PRIO; i++) {
		INIT_LIST_HEAD(array->queue + i);
		__clear_bit(i, array->bitmap);
	}
	/* delimiter for bitsearch: */
	__set_bit(MAX_RT_PRIO, array->bitmap);

8105
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8106 8107
	rt_rq->highest_prio = MAX_RT_PRIO;
#endif
P
Peter Zijlstra 已提交
8108 8109 8110 8111 8112 8113 8114
#ifdef CONFIG_SMP
	rt_rq->rt_nr_migratory = 0;
	rt_rq->overloaded = 0;
#endif

	rt_rq->rt_time = 0;
	rt_rq->rt_throttled = 0;
P
Peter Zijlstra 已提交
8115 8116
	rt_rq->rt_runtime = 0;
	spin_lock_init(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
8117

8118
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8119
	rt_rq->rt_nr_boosted = 0;
P
Peter Zijlstra 已提交
8120 8121
	rt_rq->rq = rq;
#endif
P
Peter Zijlstra 已提交
8122 8123
}

P
Peter Zijlstra 已提交
8124
#ifdef CONFIG_FAIR_GROUP_SCHED
8125 8126 8127
static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
				struct sched_entity *se, int cpu, int add,
				struct sched_entity *parent)
P
Peter Zijlstra 已提交
8128
{
8129
	struct rq *rq = cpu_rq(cpu);
P
Peter Zijlstra 已提交
8130 8131 8132 8133 8134 8135 8136
	tg->cfs_rq[cpu] = cfs_rq;
	init_cfs_rq(cfs_rq, rq);
	cfs_rq->tg = tg;
	if (add)
		list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);

	tg->se[cpu] = se;
D
Dhaval Giani 已提交
8137 8138 8139 8140
	/* se could be NULL for init_task_group */
	if (!se)
		return;

8141 8142 8143 8144 8145
	if (!parent)
		se->cfs_rq = &rq->cfs;
	else
		se->cfs_rq = parent->my_q;

P
Peter Zijlstra 已提交
8146 8147
	se->my_q = cfs_rq;
	se->load.weight = tg->shares;
8148
	se->load.inv_weight = 0;
8149
	se->parent = parent;
P
Peter Zijlstra 已提交
8150
}
8151
#endif
P
Peter Zijlstra 已提交
8152

8153
#ifdef CONFIG_RT_GROUP_SCHED
8154 8155 8156
static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
		struct sched_rt_entity *rt_se, int cpu, int add,
		struct sched_rt_entity *parent)
P
Peter Zijlstra 已提交
8157
{
8158 8159
	struct rq *rq = cpu_rq(cpu);

P
Peter Zijlstra 已提交
8160 8161 8162 8163
	tg->rt_rq[cpu] = rt_rq;
	init_rt_rq(rt_rq, rq);
	rt_rq->tg = tg;
	rt_rq->rt_se = rt_se;
P
Peter Zijlstra 已提交
8164
	rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
8165 8166 8167 8168
	if (add)
		list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);

	tg->rt_se[cpu] = rt_se;
D
Dhaval Giani 已提交
8169 8170 8171
	if (!rt_se)
		return;

8172 8173 8174 8175 8176
	if (!parent)
		rt_se->rt_rq = &rq->rt;
	else
		rt_se->rt_rq = parent->my_q;

P
Peter Zijlstra 已提交
8177
	rt_se->my_q = rt_rq;
8178
	rt_se->parent = parent;
P
Peter Zijlstra 已提交
8179 8180 8181 8182
	INIT_LIST_HEAD(&rt_se->run_list);
}
#endif

L
Linus Torvalds 已提交
8183 8184
void __init sched_init(void)
{
I
Ingo Molnar 已提交
8185
	int i, j;
8186 8187 8188 8189 8190 8191 8192
	unsigned long alloc_size = 0, ptr;

#ifdef CONFIG_FAIR_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
8193 8194 8195
#endif
#ifdef CONFIG_USER_SCHED
	alloc_size *= 2;
8196 8197 8198 8199 8200 8201
#endif
	/*
	 * As sched_init() is called before page_alloc is setup,
	 * we use alloc_bootmem().
	 */
	if (alloc_size) {
8202
		ptr = (unsigned long)alloc_bootmem(alloc_size);
8203 8204 8205 8206 8207 8208 8209

#ifdef CONFIG_FAIR_GROUP_SCHED
		init_task_group.se = (struct sched_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		init_task_group.cfs_rq = (struct cfs_rq **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);
8210 8211 8212 8213 8214 8215 8216

#ifdef CONFIG_USER_SCHED
		root_task_group.se = (struct sched_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);
8217 8218
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_FAIR_GROUP_SCHED */
8219 8220 8221 8222 8223
#ifdef CONFIG_RT_GROUP_SCHED
		init_task_group.rt_se = (struct sched_rt_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		init_task_group.rt_rq = (struct rt_rq **)ptr;
8224 8225 8226 8227 8228 8229 8230 8231
		ptr += nr_cpu_ids * sizeof(void **);

#ifdef CONFIG_USER_SCHED
		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);

		root_task_group.rt_rq = (struct rt_rq **)ptr;
		ptr += nr_cpu_ids * sizeof(void **);
8232 8233
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_RT_GROUP_SCHED */
8234
	}
I
Ingo Molnar 已提交
8235

G
Gregory Haskins 已提交
8236 8237 8238 8239
#ifdef CONFIG_SMP
	init_defrootdomain();
#endif

8240 8241 8242 8243 8244 8245
	init_rt_bandwidth(&def_rt_bandwidth,
			global_rt_period(), global_rt_runtime());

#ifdef CONFIG_RT_GROUP_SCHED
	init_rt_bandwidth(&init_task_group.rt_bandwidth,
			global_rt_period(), global_rt_runtime());
8246 8247 8248
#ifdef CONFIG_USER_SCHED
	init_rt_bandwidth(&root_task_group.rt_bandwidth,
			global_rt_period(), RUNTIME_INF);
8249 8250
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_RT_GROUP_SCHED */
8251

8252
#ifdef CONFIG_GROUP_SCHED
P
Peter Zijlstra 已提交
8253
	list_add(&init_task_group.list, &task_groups);
P
Peter Zijlstra 已提交
8254 8255 8256 8257 8258 8259
	INIT_LIST_HEAD(&init_task_group.children);

#ifdef CONFIG_USER_SCHED
	INIT_LIST_HEAD(&root_task_group.children);
	init_task_group.parent = &root_task_group;
	list_add(&init_task_group.siblings, &root_task_group.children);
8260 8261
#endif /* CONFIG_USER_SCHED */
#endif /* CONFIG_GROUP_SCHED */
P
Peter Zijlstra 已提交
8262

8263
	for_each_possible_cpu(i) {
8264
		struct rq *rq;
L
Linus Torvalds 已提交
8265 8266 8267

		rq = cpu_rq(i);
		spin_lock_init(&rq->lock);
N
Nick Piggin 已提交
8268
		rq->nr_running = 0;
I
Ingo Molnar 已提交
8269
		init_cfs_rq(&rq->cfs, rq);
P
Peter Zijlstra 已提交
8270
		init_rt_rq(&rq->rt, rq);
I
Ingo Molnar 已提交
8271
#ifdef CONFIG_FAIR_GROUP_SCHED
8272
		init_task_group.shares = init_task_group_load;
P
Peter Zijlstra 已提交
8273
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
D
Dhaval Giani 已提交
8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293
#ifdef CONFIG_CGROUP_SCHED
		/*
		 * How much cpu bandwidth does init_task_group get?
		 *
		 * In case of task-groups formed thr' the cgroup filesystem, it
		 * gets 100% of the cpu resources in the system. This overall
		 * system cpu resource is divided among the tasks of
		 * init_task_group and its child task-groups in a fair manner,
		 * based on each entity's (task or task-group's) weight
		 * (se->load.weight).
		 *
		 * In other words, if init_task_group has 10 tasks of weight
		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
		 * then A0's share of the cpu resource is:
		 *
		 * 	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
		 *
		 * We achieve this by letting init_task_group's tasks sit
		 * directly in rq->cfs (i.e init_task_group->se[] = NULL).
		 */
8294
		init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
D
Dhaval Giani 已提交
8295
#elif defined CONFIG_USER_SCHED
8296 8297
		root_task_group.shares = NICE_0_LOAD;
		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, 0, NULL);
D
Dhaval Giani 已提交
8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308
		/*
		 * In case of task-groups formed thr' the user id of tasks,
		 * init_task_group represents tasks belonging to root user.
		 * Hence it forms a sibling of all subsequent groups formed.
		 * In this case, init_task_group gets only a fraction of overall
		 * system cpu resource, based on the weight assigned to root
		 * user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
		 * by letting tasks of init_task_group sit in a separate cfs_rq
		 * (init_cfs_rq) and having one entity represent this group of
		 * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
		 */
8309
		init_tg_cfs_entry(&init_task_group,
P
Peter Zijlstra 已提交
8310
				&per_cpu(init_cfs_rq, i),
8311 8312
				&per_cpu(init_sched_entity, i), i, 1,
				root_task_group.se[i]);
P
Peter Zijlstra 已提交
8313

8314
#endif
D
Dhaval Giani 已提交
8315 8316 8317
#endif /* CONFIG_FAIR_GROUP_SCHED */

		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
8318
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8319
		INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
D
Dhaval Giani 已提交
8320
#ifdef CONFIG_CGROUP_SCHED
8321
		init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
D
Dhaval Giani 已提交
8322
#elif defined CONFIG_USER_SCHED
8323
		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
8324
		init_tg_rt_entry(&init_task_group,
P
Peter Zijlstra 已提交
8325
				&per_cpu(init_rt_rq, i),
8326 8327
				&per_cpu(init_sched_rt_entity, i), i, 1,
				root_task_group.rt_se[i]);
D
Dhaval Giani 已提交
8328
#endif
I
Ingo Molnar 已提交
8329
#endif
L
Linus Torvalds 已提交
8330

I
Ingo Molnar 已提交
8331 8332
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
L
Linus Torvalds 已提交
8333
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
8334
		rq->sd = NULL;
G
Gregory Haskins 已提交
8335
		rq->rd = NULL;
L
Linus Torvalds 已提交
8336
		rq->active_balance = 0;
I
Ingo Molnar 已提交
8337
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
8338
		rq->push_cpu = 0;
8339
		rq->cpu = i;
8340
		rq->online = 0;
L
Linus Torvalds 已提交
8341 8342
		rq->migration_thread = NULL;
		INIT_LIST_HEAD(&rq->migration_queue);
8343
		rq_attach_root(rq, &def_root_domain);
L
Linus Torvalds 已提交
8344
#endif
P
Peter Zijlstra 已提交
8345
		init_rq_hrtick(rq);
L
Linus Torvalds 已提交
8346 8347 8348
		atomic_set(&rq->nr_iowait, 0);
	}

8349
	set_load_weight(&init_task);
8350

8351 8352 8353 8354
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

8355
#ifdef CONFIG_SMP
8356
	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
8357 8358
#endif

8359 8360 8361 8362
#ifdef CONFIG_RT_MUTEXES
	plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
#endif

L
Linus Torvalds 已提交
8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
I
Ingo Molnar 已提交
8376 8377 8378 8379
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
8380

8381 8382
	/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
	alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8383
#ifdef CONFIG_SMP
8384 8385 8386
#ifdef CONFIG_NO_HZ
	alloc_bootmem_cpumask_var(&nohz.cpu_mask);
#endif
8387
	alloc_bootmem_cpumask_var(&cpu_isolated_map);
8388
#endif /* SMP */
8389

8390
	scheduler_running = 1;
L
Linus Torvalds 已提交
8391 8392 8393 8394 8395
}

#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
void __might_sleep(char *file, int line)
{
8396
#ifdef in_atomic
L
Linus Torvalds 已提交
8397 8398
	static unsigned long prev_jiffy;	/* ratelimiting */

I
Ingo Molnar 已提交
8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417
	if ((!in_atomic() && !irqs_disabled()) ||
		    system_state != SYSTEM_RUNNING || oops_in_progress)
		return;
	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
		return;
	prev_jiffy = jiffies;

	printk(KERN_ERR
		"BUG: sleeping function called from invalid context at %s:%d\n",
			file, line);
	printk(KERN_ERR
		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
			in_atomic(), irqs_disabled(),
			current->pid, current->comm);

	debug_show_held_locks(current);
	if (irqs_disabled())
		print_irqtrace_events(current);
	dump_stack();
L
Linus Torvalds 已提交
8418 8419 8420 8421 8422 8423
#endif
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
8424 8425 8426
static void normalize_task(struct rq *rq, struct task_struct *p)
{
	int on_rq;
8427

8428 8429 8430 8431 8432 8433 8434 8435 8436 8437 8438
	update_rq_clock(rq);
	on_rq = p->se.on_rq;
	if (on_rq)
		deactivate_task(rq, p, 0);
	__setscheduler(rq, p, SCHED_NORMAL, 0);
	if (on_rq) {
		activate_task(rq, p, 0);
		resched_task(rq->curr);
	}
}

L
Linus Torvalds 已提交
8439 8440
void normalize_rt_tasks(void)
{
8441
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
8442
	unsigned long flags;
8443
	struct rq *rq;
L
Linus Torvalds 已提交
8444

8445
	read_lock_irqsave(&tasklist_lock, flags);
8446
	do_each_thread(g, p) {
8447 8448 8449 8450 8451 8452
		/*
		 * Only normalize user tasks:
		 */
		if (!p->mm)
			continue;

I
Ingo Molnar 已提交
8453 8454
		p->se.exec_start		= 0;
#ifdef CONFIG_SCHEDSTATS
I
Ingo Molnar 已提交
8455 8456 8457
		p->se.wait_start		= 0;
		p->se.sleep_start		= 0;
		p->se.block_start		= 0;
I
Ingo Molnar 已提交
8458
#endif
I
Ingo Molnar 已提交
8459 8460 8461 8462 8463 8464 8465 8466

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
8467
			continue;
I
Ingo Molnar 已提交
8468
		}
L
Linus Torvalds 已提交
8469

8470
		spin_lock(&p->pi_lock);
8471
		rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
8472

8473
		normalize_task(rq, p);
8474

8475
		__task_rq_unlock(rq);
8476
		spin_unlock(&p->pi_lock);
8477 8478
	} while_each_thread(g, p);

8479
	read_unlock_irqrestore(&tasklist_lock, flags);
L
Linus Torvalds 已提交
8480 8481 8482
}

#endif /* CONFIG_MAGIC_SYSRQ */
8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495 8496 8497 8498 8499 8500

#ifdef CONFIG_IA64
/*
 * These functions are only useful for the IA64 MCA handling.
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
8501
struct task_struct *curr_task(int cpu)
8502 8503 8504 8505 8506 8507 8508 8509 8510 8511
{
	return cpu_curr(cpu);
}

/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
I
Ingo Molnar 已提交
8512 8513
 * are serviced on a separate stack. It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner. This function
8514 8515 8516 8517 8518 8519 8520
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
8521
void set_curr_task(int cpu, struct task_struct *p)
8522 8523 8524 8525 8526
{
	cpu_curr(cpu) = p;
}

#endif
S
Srivatsa Vaddagiri 已提交
8527

8528 8529
#ifdef CONFIG_FAIR_GROUP_SCHED
static void free_fair_sched_group(struct task_group *tg)
P
Peter Zijlstra 已提交
8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543
{
	int i;

	for_each_possible_cpu(i) {
		if (tg->cfs_rq)
			kfree(tg->cfs_rq[i]);
		if (tg->se)
			kfree(tg->se[i]);
	}

	kfree(tg->cfs_rq);
	kfree(tg->se);
}

8544 8545
static
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
S
Srivatsa Vaddagiri 已提交
8546 8547
{
	struct cfs_rq *cfs_rq;
8548
	struct sched_entity *se;
8549
	struct rq *rq;
S
Srivatsa Vaddagiri 已提交
8550 8551
	int i;

8552
	tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
8553 8554
	if (!tg->cfs_rq)
		goto err;
8555
	tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
8556 8557
	if (!tg->se)
		goto err;
8558 8559

	tg->shares = NICE_0_LOAD;
S
Srivatsa Vaddagiri 已提交
8560 8561

	for_each_possible_cpu(i) {
8562
		rq = cpu_rq(i);
S
Srivatsa Vaddagiri 已提交
8563

8564 8565
		cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
				      GFP_KERNEL, cpu_to_node(i));
S
Srivatsa Vaddagiri 已提交
8566 8567 8568
		if (!cfs_rq)
			goto err;

8569 8570
		se = kzalloc_node(sizeof(struct sched_entity),
				  GFP_KERNEL, cpu_to_node(i));
S
Srivatsa Vaddagiri 已提交
8571 8572 8573
		if (!se)
			goto err;

8574
		init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592
	}

	return 1;

 err:
	return 0;
}

static inline void register_fair_sched_group(struct task_group *tg, int cpu)
{
	list_add_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list,
			&cpu_rq(cpu)->leaf_cfs_rq_list);
}

static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
{
	list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
}
8593
#else /* !CONFG_FAIR_GROUP_SCHED */
8594 8595 8596 8597
static inline void free_fair_sched_group(struct task_group *tg)
{
}

8598 8599
static inline
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610
{
	return 1;
}

static inline void register_fair_sched_group(struct task_group *tg, int cpu)
{
}

static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
{
}
8611
#endif /* CONFIG_FAIR_GROUP_SCHED */
8612 8613

#ifdef CONFIG_RT_GROUP_SCHED
8614 8615 8616 8617
static void free_rt_sched_group(struct task_group *tg)
{
	int i;

8618 8619
	destroy_rt_bandwidth(&tg->rt_bandwidth);

8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630
	for_each_possible_cpu(i) {
		if (tg->rt_rq)
			kfree(tg->rt_rq[i]);
		if (tg->rt_se)
			kfree(tg->rt_se[i]);
	}

	kfree(tg->rt_rq);
	kfree(tg->rt_se);
}

8631 8632
static
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8633 8634
{
	struct rt_rq *rt_rq;
8635
	struct sched_rt_entity *rt_se;
8636 8637 8638
	struct rq *rq;
	int i;

8639
	tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
8640 8641
	if (!tg->rt_rq)
		goto err;
8642
	tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
8643 8644 8645
	if (!tg->rt_se)
		goto err;

8646 8647
	init_rt_bandwidth(&tg->rt_bandwidth,
			ktime_to_ns(def_rt_bandwidth.rt_period), 0);
8648 8649 8650 8651

	for_each_possible_cpu(i) {
		rq = cpu_rq(i);

8652 8653
		rt_rq = kzalloc_node(sizeof(struct rt_rq),
				     GFP_KERNEL, cpu_to_node(i));
P
Peter Zijlstra 已提交
8654 8655
		if (!rt_rq)
			goto err;
S
Srivatsa Vaddagiri 已提交
8656

8657 8658
		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
				     GFP_KERNEL, cpu_to_node(i));
P
Peter Zijlstra 已提交
8659 8660
		if (!rt_se)
			goto err;
S
Srivatsa Vaddagiri 已提交
8661

8662
		init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
S
Srivatsa Vaddagiri 已提交
8663 8664
	}

8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680
	return 1;

 err:
	return 0;
}

static inline void register_rt_sched_group(struct task_group *tg, int cpu)
{
	list_add_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list,
			&cpu_rq(cpu)->leaf_rt_rq_list);
}

static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
{
	list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
}
8681
#else /* !CONFIG_RT_GROUP_SCHED */
8682 8683 8684 8685
static inline void free_rt_sched_group(struct task_group *tg)
{
}

8686 8687
static inline
int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698
{
	return 1;
}

static inline void register_rt_sched_group(struct task_group *tg, int cpu)
{
}

static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
{
}
8699
#endif /* CONFIG_RT_GROUP_SCHED */
8700

8701
#ifdef CONFIG_GROUP_SCHED
8702 8703 8704 8705 8706 8707 8708 8709
static void free_sched_group(struct task_group *tg)
{
	free_fair_sched_group(tg);
	free_rt_sched_group(tg);
	kfree(tg);
}

/* allocate runqueue etc for a new task group */
8710
struct task_group *sched_create_group(struct task_group *parent)
8711 8712 8713 8714 8715 8716 8717 8718 8719
{
	struct task_group *tg;
	unsigned long flags;
	int i;

	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
	if (!tg)
		return ERR_PTR(-ENOMEM);

8720
	if (!alloc_fair_sched_group(tg, parent))
8721 8722
		goto err;

8723
	if (!alloc_rt_sched_group(tg, parent))
8724 8725
		goto err;

8726
	spin_lock_irqsave(&task_group_lock, flags);
8727
	for_each_possible_cpu(i) {
8728 8729
		register_fair_sched_group(tg, i);
		register_rt_sched_group(tg, i);
8730
	}
P
Peter Zijlstra 已提交
8731
	list_add_rcu(&tg->list, &task_groups);
P
Peter Zijlstra 已提交
8732 8733 8734 8735 8736

	WARN_ON(!parent); /* root should already exist */

	tg->parent = parent;
	INIT_LIST_HEAD(&tg->children);
8737
	list_add_rcu(&tg->siblings, &parent->children);
8738
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
8739

8740
	return tg;
S
Srivatsa Vaddagiri 已提交
8741 8742

err:
P
Peter Zijlstra 已提交
8743
	free_sched_group(tg);
S
Srivatsa Vaddagiri 已提交
8744 8745 8746
	return ERR_PTR(-ENOMEM);
}

8747
/* rcu callback to free various structures associated with a task group */
P
Peter Zijlstra 已提交
8748
static void free_sched_group_rcu(struct rcu_head *rhp)
S
Srivatsa Vaddagiri 已提交
8749 8750
{
	/* now it should be safe to free those cfs_rqs */
P
Peter Zijlstra 已提交
8751
	free_sched_group(container_of(rhp, struct task_group, rcu));
S
Srivatsa Vaddagiri 已提交
8752 8753
}

8754
/* Destroy runqueue etc associated with a task group */
8755
void sched_destroy_group(struct task_group *tg)
S
Srivatsa Vaddagiri 已提交
8756
{
8757
	unsigned long flags;
8758
	int i;
S
Srivatsa Vaddagiri 已提交
8759

8760
	spin_lock_irqsave(&task_group_lock, flags);
8761
	for_each_possible_cpu(i) {
8762 8763
		unregister_fair_sched_group(tg, i);
		unregister_rt_sched_group(tg, i);
8764
	}
P
Peter Zijlstra 已提交
8765
	list_del_rcu(&tg->list);
P
Peter Zijlstra 已提交
8766
	list_del_rcu(&tg->siblings);
8767
	spin_unlock_irqrestore(&task_group_lock, flags);
8768 8769

	/* wait for possible concurrent references to cfs_rqs complete */
P
Peter Zijlstra 已提交
8770
	call_rcu(&tg->rcu, free_sched_group_rcu);
S
Srivatsa Vaddagiri 已提交
8771 8772
}

8773
/* change task's runqueue when it moves between groups.
I
Ingo Molnar 已提交
8774 8775 8776
 *	The caller of this function should have put the task in its new group
 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
 *	reflect its new group.
8777 8778
 */
void sched_move_task(struct task_struct *tsk)
S
Srivatsa Vaddagiri 已提交
8779 8780 8781 8782 8783 8784 8785 8786 8787
{
	int on_rq, running;
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

	update_rq_clock(rq);

8788
	running = task_current(rq, tsk);
S
Srivatsa Vaddagiri 已提交
8789 8790
	on_rq = tsk->se.on_rq;

8791
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
8792
		dequeue_task(rq, tsk, 0);
8793 8794
	if (unlikely(running))
		tsk->sched_class->put_prev_task(rq, tsk);
S
Srivatsa Vaddagiri 已提交
8795

P
Peter Zijlstra 已提交
8796
	set_task_rq(tsk, task_cpu(tsk));
S
Srivatsa Vaddagiri 已提交
8797

P
Peter Zijlstra 已提交
8798 8799 8800 8801 8802
#ifdef CONFIG_FAIR_GROUP_SCHED
	if (tsk->sched_class->moved_group)
		tsk->sched_class->moved_group(tsk);
#endif

8803 8804 8805
	if (unlikely(running))
		tsk->sched_class->set_curr_task(rq);
	if (on_rq)
8806
		enqueue_task(rq, tsk, 0);
S
Srivatsa Vaddagiri 已提交
8807 8808 8809

	task_rq_unlock(rq, &flags);
}
8810
#endif /* CONFIG_GROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
8811

8812
#ifdef CONFIG_FAIR_GROUP_SCHED
8813
static void __set_se_shares(struct sched_entity *se, unsigned long shares)
S
Srivatsa Vaddagiri 已提交
8814 8815 8816 8817 8818
{
	struct cfs_rq *cfs_rq = se->cfs_rq;
	int on_rq;

	on_rq = se->on_rq;
8819
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
8820 8821 8822
		dequeue_entity(cfs_rq, se, 0);

	se->load.weight = shares;
8823
	se->load.inv_weight = 0;
S
Srivatsa Vaddagiri 已提交
8824

8825
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
8826
		enqueue_entity(cfs_rq, se, 0);
8827
}
8828

8829 8830 8831 8832 8833 8834 8835 8836 8837
static void set_se_shares(struct sched_entity *se, unsigned long shares)
{
	struct cfs_rq *cfs_rq = se->cfs_rq;
	struct rq *rq = cfs_rq->rq;
	unsigned long flags;

	spin_lock_irqsave(&rq->lock, flags);
	__set_se_shares(se, shares);
	spin_unlock_irqrestore(&rq->lock, flags);
S
Srivatsa Vaddagiri 已提交
8838 8839
}

8840 8841
static DEFINE_MUTEX(shares_mutex);

8842
int sched_group_set_shares(struct task_group *tg, unsigned long shares)
S
Srivatsa Vaddagiri 已提交
8843 8844
{
	int i;
8845
	unsigned long flags;
8846

8847 8848 8849 8850 8851 8852
	/*
	 * We can't change the weight of the root cgroup.
	 */
	if (!tg->se[0])
		return -EINVAL;

8853 8854
	if (shares < MIN_SHARES)
		shares = MIN_SHARES;
8855 8856
	else if (shares > MAX_SHARES)
		shares = MAX_SHARES;
8857

8858
	mutex_lock(&shares_mutex);
8859
	if (tg->shares == shares)
8860
		goto done;
S
Srivatsa Vaddagiri 已提交
8861

8862
	spin_lock_irqsave(&task_group_lock, flags);
8863 8864
	for_each_possible_cpu(i)
		unregister_fair_sched_group(tg, i);
P
Peter Zijlstra 已提交
8865
	list_del_rcu(&tg->siblings);
8866
	spin_unlock_irqrestore(&task_group_lock, flags);
8867 8868 8869 8870 8871 8872 8873 8874

	/* wait for any ongoing reference to this group to finish */
	synchronize_sched();

	/*
	 * Now we are free to modify the group's share on each cpu
	 * w/o tripping rebalance_share or load_balance_fair.
	 */
8875
	tg->shares = shares;
8876 8877 8878 8879 8880
	for_each_possible_cpu(i) {
		/*
		 * force a rebalance
		 */
		cfs_rq_set_shares(tg->cfs_rq[i], 0);
8881
		set_se_shares(tg->se[i], shares);
8882
	}
S
Srivatsa Vaddagiri 已提交
8883

8884 8885 8886 8887
	/*
	 * Enable load balance activity on this group, by inserting it back on
	 * each cpu's rq->leaf_cfs_rq_list.
	 */
8888
	spin_lock_irqsave(&task_group_lock, flags);
8889 8890
	for_each_possible_cpu(i)
		register_fair_sched_group(tg, i);
P
Peter Zijlstra 已提交
8891
	list_add_rcu(&tg->siblings, &tg->parent->children);
8892
	spin_unlock_irqrestore(&task_group_lock, flags);
8893
done:
8894
	mutex_unlock(&shares_mutex);
8895
	return 0;
S
Srivatsa Vaddagiri 已提交
8896 8897
}

8898 8899 8900 8901
unsigned long sched_group_shares(struct task_group *tg)
{
	return tg->shares;
}
8902
#endif
8903

8904
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8905
/*
P
Peter Zijlstra 已提交
8906
 * Ensure that the real time constraints are schedulable.
P
Peter Zijlstra 已提交
8907
 */
P
Peter Zijlstra 已提交
8908 8909 8910 8911 8912
static DEFINE_MUTEX(rt_constraints_mutex);

static unsigned long to_ratio(u64 period, u64 runtime)
{
	if (runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
8913
		return 1ULL << 20;
P
Peter Zijlstra 已提交
8914

P
Peter Zijlstra 已提交
8915
	return div64_u64(runtime << 20, period);
P
Peter Zijlstra 已提交
8916 8917
}

P
Peter Zijlstra 已提交
8918 8919
/* Must be called with tasklist_lock held */
static inline int tg_has_rt_tasks(struct task_group *tg)
8920
{
P
Peter Zijlstra 已提交
8921
	struct task_struct *g, *p;
8922

P
Peter Zijlstra 已提交
8923 8924 8925 8926
	do_each_thread(g, p) {
		if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
			return 1;
	} while_each_thread(g, p);
8927

P
Peter Zijlstra 已提交
8928 8929
	return 0;
}
8930

P
Peter Zijlstra 已提交
8931 8932 8933 8934 8935
struct rt_schedulable_data {
	struct task_group *tg;
	u64 rt_period;
	u64 rt_runtime;
};
8936

P
Peter Zijlstra 已提交
8937 8938 8939 8940 8941 8942
static int tg_schedulable(struct task_group *tg, void *data)
{
	struct rt_schedulable_data *d = data;
	struct task_group *child;
	unsigned long total, sum = 0;
	u64 period, runtime;
8943

P
Peter Zijlstra 已提交
8944 8945
	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	runtime = tg->rt_bandwidth.rt_runtime;
8946

P
Peter Zijlstra 已提交
8947 8948 8949
	if (tg == d->tg) {
		period = d->rt_period;
		runtime = d->rt_runtime;
8950 8951
	}

8952 8953 8954 8955 8956
	/*
	 * Cannot have more runtime than the period.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
P
Peter Zijlstra 已提交
8957

8958 8959 8960
	/*
	 * Ensure we don't starve existing RT tasks.
	 */
P
Peter Zijlstra 已提交
8961 8962
	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
		return -EBUSY;
P
Peter Zijlstra 已提交
8963

P
Peter Zijlstra 已提交
8964
	total = to_ratio(period, runtime);
P
Peter Zijlstra 已提交
8965

8966 8967 8968 8969 8970
	/*
	 * Nobody can have more than the global setting allows.
	 */
	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
		return -EINVAL;
P
Peter Zijlstra 已提交
8971

8972 8973 8974
	/*
	 * The sum of our children's runtime should not exceed our own.
	 */
P
Peter Zijlstra 已提交
8975 8976 8977
	list_for_each_entry_rcu(child, &tg->children, siblings) {
		period = ktime_to_ns(child->rt_bandwidth.rt_period);
		runtime = child->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
8978

P
Peter Zijlstra 已提交
8979 8980 8981 8982
		if (child == d->tg) {
			period = d->rt_period;
			runtime = d->rt_runtime;
		}
P
Peter Zijlstra 已提交
8983

P
Peter Zijlstra 已提交
8984
		sum += to_ratio(period, runtime);
P
Peter Zijlstra 已提交
8985
	}
P
Peter Zijlstra 已提交
8986

P
Peter Zijlstra 已提交
8987 8988 8989 8990
	if (sum > total)
		return -EINVAL;

	return 0;
P
Peter Zijlstra 已提交
8991 8992
}

P
Peter Zijlstra 已提交
8993
static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
8994
{
P
Peter Zijlstra 已提交
8995 8996 8997 8998 8999 9000 9001
	struct rt_schedulable_data data = {
		.tg = tg,
		.rt_period = period,
		.rt_runtime = runtime,
	};

	return walk_tg_tree(tg_schedulable, tg_nop, &data);
9002 9003
}

9004 9005
static int tg_set_bandwidth(struct task_group *tg,
		u64 rt_period, u64 rt_runtime)
P
Peter Zijlstra 已提交
9006
{
P
Peter Zijlstra 已提交
9007
	int i, err = 0;
P
Peter Zijlstra 已提交
9008 9009

	mutex_lock(&rt_constraints_mutex);
9010
	read_lock(&tasklist_lock);
P
Peter Zijlstra 已提交
9011 9012
	err = __rt_schedulable(tg, rt_period, rt_runtime);
	if (err)
P
Peter Zijlstra 已提交
9013
		goto unlock;
P
Peter Zijlstra 已提交
9014 9015

	spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
9016 9017
	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
	tg->rt_bandwidth.rt_runtime = rt_runtime;
P
Peter Zijlstra 已提交
9018 9019 9020 9021 9022 9023 9024 9025 9026

	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = tg->rt_rq[i];

		spin_lock(&rt_rq->rt_runtime_lock);
		rt_rq->rt_runtime = rt_runtime;
		spin_unlock(&rt_rq->rt_runtime_lock);
	}
	spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
P
Peter Zijlstra 已提交
9027
 unlock:
9028
	read_unlock(&tasklist_lock);
P
Peter Zijlstra 已提交
9029 9030 9031
	mutex_unlock(&rt_constraints_mutex);

	return err;
P
Peter Zijlstra 已提交
9032 9033
}

9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044 9045
int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
{
	u64 rt_runtime, rt_period;

	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
	if (rt_runtime_us < 0)
		rt_runtime = RUNTIME_INF;

	return tg_set_bandwidth(tg, rt_period, rt_runtime);
}

P
Peter Zijlstra 已提交
9046 9047 9048 9049
long sched_group_rt_runtime(struct task_group *tg)
{
	u64 rt_runtime_us;

9050
	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
9051 9052
		return -1;

9053
	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
9054 9055 9056
	do_div(rt_runtime_us, NSEC_PER_USEC);
	return rt_runtime_us;
}
9057 9058 9059 9060 9061 9062 9063 9064

int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
{
	u64 rt_runtime, rt_period;

	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
	rt_runtime = tg->rt_bandwidth.rt_runtime;

9065 9066 9067
	if (rt_period == 0)
		return -EINVAL;

9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078 9079 9080 9081
	return tg_set_bandwidth(tg, rt_period, rt_runtime);
}

long sched_group_rt_period(struct task_group *tg)
{
	u64 rt_period_us;

	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
	do_div(rt_period_us, NSEC_PER_USEC);
	return rt_period_us;
}

static int sched_rt_global_constraints(void)
{
9082
	u64 runtime, period;
9083 9084
	int ret = 0;

9085 9086 9087
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

9088 9089 9090 9091 9092 9093 9094 9095
	runtime = global_rt_runtime();
	period = global_rt_period();

	/*
	 * Sanity check on the sysctl variables.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
9096

9097
	mutex_lock(&rt_constraints_mutex);
P
Peter Zijlstra 已提交
9098
	read_lock(&tasklist_lock);
9099
	ret = __rt_schedulable(NULL, 0, 0);
P
Peter Zijlstra 已提交
9100
	read_unlock(&tasklist_lock);
9101 9102 9103 9104
	mutex_unlock(&rt_constraints_mutex);

	return ret;
}
9105
#else /* !CONFIG_RT_GROUP_SCHED */
9106 9107
static int sched_rt_global_constraints(void)
{
P
Peter Zijlstra 已提交
9108 9109 9110
	unsigned long flags;
	int i;

9111 9112 9113
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

P
Peter Zijlstra 已提交
9114 9115 9116 9117 9118 9119 9120 9121 9122 9123
	spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = &cpu_rq(i)->rt;

		spin_lock(&rt_rq->rt_runtime_lock);
		rt_rq->rt_runtime = global_rt_runtime();
		spin_unlock(&rt_rq->rt_runtime_lock);
	}
	spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);

9124 9125
	return 0;
}
9126
#endif /* CONFIG_RT_GROUP_SCHED */
9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156

int sched_rt_handler(struct ctl_table *table, int write,
		struct file *filp, void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;
	int old_period, old_runtime;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	old_period = sysctl_sched_rt_period;
	old_runtime = sysctl_sched_rt_runtime;

	ret = proc_dointvec(table, write, filp, buffer, lenp, ppos);

	if (!ret && write) {
		ret = sched_rt_global_constraints();
		if (ret) {
			sysctl_sched_rt_period = old_period;
			sysctl_sched_rt_runtime = old_runtime;
		} else {
			def_rt_bandwidth.rt_runtime = global_rt_runtime();
			def_rt_bandwidth.rt_period =
				ns_to_ktime(global_rt_period());
		}
	}
	mutex_unlock(&mutex);

	return ret;
}
9157

9158
#ifdef CONFIG_CGROUP_SCHED
9159 9160

/* return corresponding task_group object of a cgroup */
9161
static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
9162
{
9163 9164
	return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
			    struct task_group, css);
9165 9166 9167
}

static struct cgroup_subsys_state *
9168
cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
9169
{
9170
	struct task_group *tg, *parent;
9171

9172
	if (!cgrp->parent) {
9173 9174 9175 9176
		/* This is early initialization for the top cgroup */
		return &init_task_group.css;
	}

9177 9178
	parent = cgroup_tg(cgrp->parent);
	tg = sched_create_group(parent);
9179 9180 9181 9182 9183 9184
	if (IS_ERR(tg))
		return ERR_PTR(-ENOMEM);

	return &tg->css;
}

I
Ingo Molnar 已提交
9185 9186
static void
cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
9187
{
9188
	struct task_group *tg = cgroup_tg(cgrp);
9189 9190 9191 9192

	sched_destroy_group(tg);
}

I
Ingo Molnar 已提交
9193 9194 9195
static int
cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
		      struct task_struct *tsk)
9196
{
9197 9198
#ifdef CONFIG_RT_GROUP_SCHED
	/* Don't accept realtime tasks when there is no way for them to run */
9199
	if (rt_task(tsk) && cgroup_tg(cgrp)->rt_bandwidth.rt_runtime == 0)
9200 9201
		return -EINVAL;
#else
9202 9203 9204
	/* We don't support RT-tasks being in separate groups */
	if (tsk->sched_class != &fair_sched_class)
		return -EINVAL;
9205
#endif
9206 9207 9208 9209 9210

	return 0;
}

static void
9211
cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
9212 9213 9214 9215 9216
			struct cgroup *old_cont, struct task_struct *tsk)
{
	sched_move_task(tsk);
}

9217
#ifdef CONFIG_FAIR_GROUP_SCHED
9218
static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
9219
				u64 shareval)
9220
{
9221
	return sched_group_set_shares(cgroup_tg(cgrp), shareval);
9222 9223
}

9224
static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
9225
{
9226
	struct task_group *tg = cgroup_tg(cgrp);
9227 9228 9229

	return (u64) tg->shares;
}
9230
#endif /* CONFIG_FAIR_GROUP_SCHED */
9231

9232
#ifdef CONFIG_RT_GROUP_SCHED
M
Mirco Tischler 已提交
9233
static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
9234
				s64 val)
P
Peter Zijlstra 已提交
9235
{
9236
	return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
P
Peter Zijlstra 已提交
9237 9238
}

9239
static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
P
Peter Zijlstra 已提交
9240
{
9241
	return sched_group_rt_runtime(cgroup_tg(cgrp));
P
Peter Zijlstra 已提交
9242
}
9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253

static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
		u64 rt_period_us)
{
	return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
}

static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
{
	return sched_group_rt_period(cgroup_tg(cgrp));
}
9254
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
9255

9256
static struct cftype cpu_files[] = {
9257
#ifdef CONFIG_FAIR_GROUP_SCHED
9258 9259
	{
		.name = "shares",
9260 9261
		.read_u64 = cpu_shares_read_u64,
		.write_u64 = cpu_shares_write_u64,
9262
	},
9263 9264
#endif
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
9265
	{
P
Peter Zijlstra 已提交
9266
		.name = "rt_runtime_us",
9267 9268
		.read_s64 = cpu_rt_runtime_read,
		.write_s64 = cpu_rt_runtime_write,
P
Peter Zijlstra 已提交
9269
	},
9270 9271
	{
		.name = "rt_period_us",
9272 9273
		.read_u64 = cpu_rt_period_read_uint,
		.write_u64 = cpu_rt_period_write_uint,
9274
	},
9275
#endif
9276 9277 9278 9279
};

static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
{
9280
	return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
9281 9282 9283
}

struct cgroup_subsys cpu_cgroup_subsys = {
I
Ingo Molnar 已提交
9284 9285 9286 9287 9288 9289 9290
	.name		= "cpu",
	.create		= cpu_cgroup_create,
	.destroy	= cpu_cgroup_destroy,
	.can_attach	= cpu_cgroup_can_attach,
	.attach		= cpu_cgroup_attach,
	.populate	= cpu_cgroup_populate,
	.subsys_id	= cpu_cgroup_subsys_id,
9291 9292 9293
	.early_init	= 1,
};

9294
#endif	/* CONFIG_CGROUP_SCHED */
9295 9296 9297 9298 9299 9300 9301 9302 9303 9304

#ifdef CONFIG_CGROUP_CPUACCT

/*
 * CPU accounting code for task groups.
 *
 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
 * (balbir@in.ibm.com).
 */

9305
/* track cpu usage of a group of tasks and its child groups */
9306 9307 9308 9309
struct cpuacct {
	struct cgroup_subsys_state css;
	/* cpuusage holds pointer to a u64-type object on every cpu */
	u64 *cpuusage;
9310
	struct cpuacct *parent;
9311 9312 9313 9314 9315
};

struct cgroup_subsys cpuacct_subsys;

/* return cpu accounting group corresponding to this container */
9316
static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
9317
{
9318
	return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330
			    struct cpuacct, css);
}

/* return cpu accounting group to which this task belongs */
static inline struct cpuacct *task_ca(struct task_struct *tsk)
{
	return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
			    struct cpuacct, css);
}

/* create a new cpu accounting group */
static struct cgroup_subsys_state *cpuacct_create(
9331
	struct cgroup_subsys *ss, struct cgroup *cgrp)
9332 9333 9334 9335 9336 9337 9338 9339 9340 9341 9342 9343
{
	struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);

	if (!ca)
		return ERR_PTR(-ENOMEM);

	ca->cpuusage = alloc_percpu(u64);
	if (!ca->cpuusage) {
		kfree(ca);
		return ERR_PTR(-ENOMEM);
	}

9344 9345 9346
	if (cgrp->parent)
		ca->parent = cgroup_ca(cgrp->parent);

9347 9348 9349 9350
	return &ca->css;
}

/* destroy an existing cpu accounting group */
I
Ingo Molnar 已提交
9351
static void
9352
cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
9353
{
9354
	struct cpuacct *ca = cgroup_ca(cgrp);
9355 9356 9357 9358 9359 9360

	free_percpu(ca->cpuusage);
	kfree(ca);
}

/* return total cpu usage (in nanoseconds) of a group */
9361
static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
9362
{
9363
	struct cpuacct *ca = cgroup_ca(cgrp);
9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381
	u64 totalcpuusage = 0;
	int i;

	for_each_possible_cpu(i) {
		u64 *cpuusage = percpu_ptr(ca->cpuusage, i);

		/*
		 * Take rq->lock to make 64-bit addition safe on 32-bit
		 * platforms.
		 */
		spin_lock_irq(&cpu_rq(i)->lock);
		totalcpuusage += *cpuusage;
		spin_unlock_irq(&cpu_rq(i)->lock);
	}

	return totalcpuusage;
}

9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404
static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
								u64 reset)
{
	struct cpuacct *ca = cgroup_ca(cgrp);
	int err = 0;
	int i;

	if (reset) {
		err = -EINVAL;
		goto out;
	}

	for_each_possible_cpu(i) {
		u64 *cpuusage = percpu_ptr(ca->cpuusage, i);

		spin_lock_irq(&cpu_rq(i)->lock);
		*cpuusage = 0;
		spin_unlock_irq(&cpu_rq(i)->lock);
	}
out:
	return err;
}

9405 9406 9407
static struct cftype files[] = {
	{
		.name = "usage",
9408 9409
		.read_u64 = cpuusage_read,
		.write_u64 = cpuusage_write,
9410 9411 9412
	},
};

9413
static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
9414
{
9415
	return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
9416 9417 9418 9419 9420 9421 9422 9423 9424 9425
}

/*
 * charge this task's execution time to its accounting group.
 *
 * called with rq->lock held.
 */
static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
{
	struct cpuacct *ca;
9426
	int cpu;
9427 9428 9429 9430

	if (!cpuacct_subsys.active)
		return;

9431
	cpu = task_cpu(tsk);
9432 9433
	ca = task_ca(tsk);

9434 9435
	for (; ca; ca = ca->parent) {
		u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447
		*cpuusage += cputime;
	}
}

struct cgroup_subsys cpuacct_subsys = {
	.name = "cpuacct",
	.create = cpuacct_create,
	.destroy = cpuacct_destroy,
	.populate = cpuacct_populate,
	.subsys_id = cpuacct_subsys_id,
};
#endif	/* CONFIG_CGROUP_CPUACCT */