sched.c 169.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
L
Linus Torvalds 已提交
25 26 27 28 29 30
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
31
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
32 33 34 35
#include <linux/highmem.h>
#include <linux/smp_lock.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
36
#include <linux/capability.h>
L
Linus Torvalds 已提交
37 38
#include <linux/completion.h>
#include <linux/kernel_stat.h>
39
#include <linux/debug_locks.h>
L
Linus Torvalds 已提交
40 41 42
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
43
#include <linux/freezer.h>
44
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
45 46 47 48 49 50 51 52 53 54 55
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
#include <linux/seq_file.h>
56
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
57 58
#include <linux/syscalls.h>
#include <linux/times.h>
59
#include <linux/tsacct_kern.h>
60
#include <linux/kprobes.h>
61
#include <linux/delayacct.h>
62
#include <linux/reciprocal_div.h>
63
#include <linux/unistd.h>
J
Jens Axboe 已提交
64
#include <linux/pagemap.h>
L
Linus Torvalds 已提交
65

66
#include <asm/tlb.h>
L
Linus Torvalds 已提交
67

68 69 70 71 72 73 74 75 76 77
/*
 * Scheduler clock - returns current time in nanosec units.
 * This is default implementation.
 * Architectures and sub-architectures can override this.
 */
unsigned long long __attribute__((weak)) sched_clock(void)
{
	return (unsigned long long)jiffies * (1000000000 / HZ);
}

L
Linus Torvalds 已提交
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
 * Some helpers for converting nanosecond timing to jiffy resolution
 */
D
Dmitry Adamushko 已提交
99
#define NS_TO_JIFFIES(TIME)	((unsigned long)(TIME) / (1000000000 / HZ))
L
Linus Torvalds 已提交
100 101
#define JIFFIES_TO_NS(TIME)	((TIME) * (1000000000 / HZ))

I
Ingo Molnar 已提交
102 103 104
#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

L
Linus Torvalds 已提交
105 106 107
/*
 * These are the 'tuning knobs' of the scheduler:
 *
D
Dmitry Adamushko 已提交
108
 * default timeslice is 100 msecs (used only for SCHED_RR tasks).
L
Linus Torvalds 已提交
109 110 111
 * Timeslices get refilled after they expire.
 */
#define DEF_TIMESLICE		(100 * HZ / 1000)
112

113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
#ifdef CONFIG_SMP
/*
 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 */
static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
{
	return reciprocal_divide(load, sg->reciprocal_cpu_power);
}

/*
 * Each time a sched group cpu_power is changed,
 * we must compute its reciprocal value
 */
static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
{
	sg->__cpu_power += val;
	sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
}
#endif

134 135 136 137 138 139 140 141 142 143 144 145
static inline int rt_policy(int policy)
{
	if (unlikely(policy == SCHED_FIFO) || unlikely(policy == SCHED_RR))
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

L
Linus Torvalds 已提交
146
/*
I
Ingo Molnar 已提交
147
 * This is the priority-queue data structure of the RT scheduling class:
L
Linus Torvalds 已提交
148
 */
I
Ingo Molnar 已提交
149 150 151 152 153
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

S
Srivatsa Vaddagiri 已提交
154 155 156 157 158
#ifdef CONFIG_FAIR_GROUP_SCHED

struct cfs_rq;

/* task group related information */
159
struct task_group {
S
Srivatsa Vaddagiri 已提交
160 161 162 163 164
	/* schedulable entities of this group on each cpu */
	struct sched_entity **se;
	/* runqueue "owned" by this group on each cpu */
	struct cfs_rq **cfs_rq;
	unsigned long shares;
165 166
	/* spinlock to serialize modification to shares */
	spinlock_t lock;
S
Srivatsa Vaddagiri 已提交
167 168 169 170 171 172 173
};

/* Default task group's sched entity on each cpu */
static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
/* Default task group's cfs_rq on each cpu */
static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;

174 175
static struct sched_entity *init_sched_entity_p[NR_CPUS];
static struct cfs_rq *init_cfs_rq_p[NR_CPUS];
S
Srivatsa Vaddagiri 已提交
176 177

/* Default task group.
I
Ingo Molnar 已提交
178
 *	Every task in system belong to this group at bootup.
S
Srivatsa Vaddagiri 已提交
179
 */
180
struct task_group init_task_group = {
I
Ingo Molnar 已提交
181 182 183
	.se     = init_sched_entity_p,
	.cfs_rq = init_cfs_rq_p,
};
184

185
#ifdef CONFIG_FAIR_USER_SCHED
I
Ingo Molnar 已提交
186
# define INIT_TASK_GRP_LOAD	2*NICE_0_LOAD
187
#else
I
Ingo Molnar 已提交
188
# define INIT_TASK_GRP_LOAD	NICE_0_LOAD
189 190
#endif

191
static int init_task_group_load = INIT_TASK_GRP_LOAD;
S
Srivatsa Vaddagiri 已提交
192 193

/* return group to which a task belongs */
194
static inline struct task_group *task_group(struct task_struct *p)
S
Srivatsa Vaddagiri 已提交
195
{
196
	struct task_group *tg;
197

198 199 200
#ifdef CONFIG_FAIR_USER_SCHED
	tg = p->user->tg;
#else
201
	tg  = &init_task_group;
202
#endif
203 204

	return tg;
S
Srivatsa Vaddagiri 已提交
205 206 207 208 209
}

/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
static inline void set_task_cfs_rq(struct task_struct *p)
{
210 211
	p->se.cfs_rq = task_group(p)->cfs_rq[task_cpu(p)];
	p->se.parent = task_group(p)->se[task_cpu(p)];
S
Srivatsa Vaddagiri 已提交
212 213 214 215 216 217 218 219
}

#else

static inline void set_task_cfs_rq(struct task_struct *p) { }

#endif	/* CONFIG_FAIR_GROUP_SCHED */

I
Ingo Molnar 已提交
220 221 222 223 224 225
/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	u64 exec_clock;
I
Ingo Molnar 已提交
226
	u64 min_vruntime;
I
Ingo Molnar 已提交
227 228 229 230 231 232 233 234

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;
	struct rb_node *rb_load_balance_curr;
	/* 'curr' points to currently running entity on this cfs_rq.
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
	struct sched_entity *curr;
P
Peter Zijlstra 已提交
235 236 237

	unsigned long nr_spread_over;

238
#ifdef CONFIG_FAIR_GROUP_SCHED
I
Ingo Molnar 已提交
239 240 241 242 243 244 245 246 247 248
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

	/* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
	struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */
249
	struct task_group *tg;    /* group that "owns" this runqueue */
250
	struct rcu_head rcu;
I
Ingo Molnar 已提交
251 252
#endif
};
L
Linus Torvalds 已提交
253

I
Ingo Molnar 已提交
254 255 256 257 258 259 260
/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
	int rt_load_balance_idx;
	struct list_head *rt_load_balance_head, *rt_load_balance_curr;
};

L
Linus Torvalds 已提交
261 262 263 264 265 266 267
/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
268
struct rq {
I
Ingo Molnar 已提交
269
	spinlock_t lock;	/* runqueue lock */
L
Linus Torvalds 已提交
270 271 272 273 274 275

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
I
Ingo Molnar 已提交
276 277
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
278
	unsigned char idle_at_tick;
279 280 281
#ifdef CONFIG_NO_HZ
	unsigned char in_nohz_recently;
#endif
282
	struct load_weight load;	/* capture load from *all* tasks on this cpu */
I
Ingo Molnar 已提交
283 284 285 286 287 288
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
#ifdef CONFIG_FAIR_GROUP_SCHED
	struct list_head leaf_cfs_rq_list; /* list of leaf cfs_rq on this cpu */
L
Linus Torvalds 已提交
289
#endif
I
Ingo Molnar 已提交
290
	struct rt_rq  rt;
L
Linus Torvalds 已提交
291 292 293 294 295 296 297 298 299

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

300
	struct task_struct *curr, *idle;
301
	unsigned long next_balance;
L
Linus Torvalds 已提交
302
	struct mm_struct *prev_mm;
I
Ingo Molnar 已提交
303 304 305 306 307

	u64 clock, prev_clock_raw;
	s64 clock_max_delta;

	unsigned int clock_warps, clock_overflows;
308 309
	u64 idle_clock;
	unsigned int clock_deep_idle_events;
310
	u64 tick_timestamp;
I
Ingo Molnar 已提交
311

L
Linus Torvalds 已提交
312 313 314 315 316 317 318 319
	atomic_t nr_iowait;

#ifdef CONFIG_SMP
	struct sched_domain *sd;

	/* For active balancing */
	int active_balance;
	int push_cpu;
320
	int cpu;		/* cpu of this runqueue */
L
Linus Torvalds 已提交
321

322
	struct task_struct *migration_thread;
L
Linus Torvalds 已提交
323 324 325 326 327 328 329 330 331 332 333
	struct list_head migration_queue;
#endif

#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;

	/* sys_sched_yield() stats */
	unsigned long yld_exp_empty;
	unsigned long yld_act_empty;
	unsigned long yld_both_empty;
334
	unsigned long yld_count;
L
Linus Torvalds 已提交
335 336 337

	/* schedule() stats */
	unsigned long sched_switch;
338
	unsigned long sched_count;
L
Linus Torvalds 已提交
339 340 341
	unsigned long sched_goidle;

	/* try_to_wake_up() stats */
342
	unsigned long ttwu_count;
L
Linus Torvalds 已提交
343
	unsigned long ttwu_local;
I
Ingo Molnar 已提交
344 345

	/* BKL stats */
346
	unsigned long bkl_count;
L
Linus Torvalds 已提交
347
#endif
348
	struct lock_class_key rq_lock_key;
L
Linus Torvalds 已提交
349 350
};

351
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
352
static DEFINE_MUTEX(sched_hotcpu_mutex);
L
Linus Torvalds 已提交
353

I
Ingo Molnar 已提交
354 355 356 357 358
static inline void check_preempt_curr(struct rq *rq, struct task_struct *p)
{
	rq->curr->sched_class->check_preempt_curr(rq, p);
}

359 360 361 362 363 364 365 366 367
static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

I
Ingo Molnar 已提交
368
/*
I
Ingo Molnar 已提交
369 370
 * Update the per-runqueue clock, as finegrained as the platform can give
 * us, but without assuming monotonicity, etc.:
I
Ingo Molnar 已提交
371
 */
I
Ingo Molnar 已提交
372
static void __update_rq_clock(struct rq *rq)
I
Ingo Molnar 已提交
373 374 375 376 377 378
{
	u64 prev_raw = rq->prev_clock_raw;
	u64 now = sched_clock();
	s64 delta = now - prev_raw;
	u64 clock = rq->clock;

I
Ingo Molnar 已提交
379 380 381
#ifdef CONFIG_SCHED_DEBUG
	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
#endif
I
Ingo Molnar 已提交
382 383 384 385 386 387 388 389 390 391
	/*
	 * Protect against sched_clock() occasionally going backwards:
	 */
	if (unlikely(delta < 0)) {
		clock++;
		rq->clock_warps++;
	} else {
		/*
		 * Catch too large forward jumps too:
		 */
392 393 394 395 396
		if (unlikely(clock + delta > rq->tick_timestamp + TICK_NSEC)) {
			if (clock < rq->tick_timestamp + TICK_NSEC)
				clock = rq->tick_timestamp + TICK_NSEC;
			else
				clock++;
I
Ingo Molnar 已提交
397 398 399 400 401 402 403 404 405 406
			rq->clock_overflows++;
		} else {
			if (unlikely(delta > rq->clock_max_delta))
				rq->clock_max_delta = delta;
			clock += delta;
		}
	}

	rq->prev_clock_raw = now;
	rq->clock = clock;
I
Ingo Molnar 已提交
407
}
I
Ingo Molnar 已提交
408

I
Ingo Molnar 已提交
409 410 411 412
static void update_rq_clock(struct rq *rq)
{
	if (likely(smp_processor_id() == cpu_of(rq)))
		__update_rq_clock(rq);
I
Ingo Molnar 已提交
413 414
}

N
Nick Piggin 已提交
415 416
/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
417
 * See detach_destroy_domains: synchronize_sched for details.
N
Nick Piggin 已提交
418 419 420 421
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
422 423
#define for_each_domain(cpu, __sd) \
	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
L
Linus Torvalds 已提交
424 425 426 427 428 429

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

I
Ingo Molnar 已提交
430 431 432 433 434 435 436 437 438 439 440 441 442
/*
 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
 */
#ifdef CONFIG_SCHED_DEBUG
# define const_debug __read_mostly
#else
# define const_debug static const
#endif

/*
 * Debugging: various feature bits
 */
enum {
443 444
	SCHED_FEAT_NEW_FAIR_SLEEPERS	= 1,
	SCHED_FEAT_START_DEBIT		= 2,
445
	SCHED_FEAT_TREE_AVG             = 4,
446
	SCHED_FEAT_APPROX_AVG           = 8,
447
	SCHED_FEAT_WAKEUP_PREEMPT	= 16,
448
	SCHED_FEAT_PREEMPT_RESTRICT	= 32,
I
Ingo Molnar 已提交
449 450 451 452
};

const_debug unsigned int sysctl_sched_features =
		SCHED_FEAT_NEW_FAIR_SLEEPERS	*1 |
P
Peter Zijlstra 已提交
453
		SCHED_FEAT_START_DEBIT		*1 |
454
		SCHED_FEAT_TREE_AVG		*0 |
455
		SCHED_FEAT_APPROX_AVG		*0 |
456 457
		SCHED_FEAT_WAKEUP_PREEMPT	*1 |
		SCHED_FEAT_PREEMPT_RESTRICT	*1;
I
Ingo Molnar 已提交
458 459 460

#define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x)

461 462 463 464 465 466 467 468
/*
 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
 * clock constructed from sched_clock():
 */
unsigned long long cpu_clock(int cpu)
{
	unsigned long long now;
	unsigned long flags;
I
Ingo Molnar 已提交
469
	struct rq *rq;
470

471
	local_irq_save(flags);
I
Ingo Molnar 已提交
472 473 474
	rq = cpu_rq(cpu);
	update_rq_clock(rq);
	now = rq->clock;
475
	local_irq_restore(flags);
476 477 478

	return now;
}
P
Paul E. McKenney 已提交
479
EXPORT_SYMBOL_GPL(cpu_clock);
480

L
Linus Torvalds 已提交
481
#ifndef prepare_arch_switch
482 483 484 485 486 487 488
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

#ifndef __ARCH_WANT_UNLOCKED_CTXSW
489
static inline int task_running(struct rq *rq, struct task_struct *p)
490 491 492 493
{
	return rq->curr == p;
}

494
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
495 496 497
{
}

498
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
499
{
500 501 502 503
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
504 505 506 507 508 509 510
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

511 512 513 514
	spin_unlock_irq(&rq->lock);
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
515
static inline int task_running(struct rq *rq, struct task_struct *p)
516 517 518 519 520 521 522 523
{
#ifdef CONFIG_SMP
	return p->oncpu;
#else
	return rq->curr == p;
#endif
}

524
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->oncpu = 1;
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	spin_unlock_irq(&rq->lock);
#else
	spin_unlock(&rq->lock);
#endif
}

541
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
542 543 544 545 546 547 548 549 550 551 552 553
{
#ifdef CONFIG_SMP
	/*
	 * After ->oncpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->oncpu = 0;
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
L
Linus Torvalds 已提交
554
#endif
555 556
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
L
Linus Torvalds 已提交
557

558 559 560 561
/*
 * __task_rq_lock - lock the runqueue a given task resides on.
 * Must be called interrupts disabled.
 */
562
static inline struct rq *__task_rq_lock(struct task_struct *p)
563 564
	__acquires(rq->lock)
{
565
	struct rq *rq;
566 567 568 569 570 571 572 573 574 575 576

repeat_lock_task:
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock(&rq->lock);
		goto repeat_lock_task;
	}
	return rq;
}

L
Linus Torvalds 已提交
577 578 579 580 581
/*
 * task_rq_lock - lock the runqueue a given task resides on and disable
 * interrupts.  Note the ordering: we can safely lookup the task_rq without
 * explicitly disabling preemption.
 */
582
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
583 584
	__acquires(rq->lock)
{
585
	struct rq *rq;
L
Linus Torvalds 已提交
586 587 588 589 590 591 592 593 594 595 596 597

repeat_lock_task:
	local_irq_save(*flags);
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock_irqrestore(&rq->lock, *flags);
		goto repeat_lock_task;
	}
	return rq;
}

A
Alexey Dobriyan 已提交
598
static void __task_rq_unlock(struct rq *rq)
599 600 601 602 603
	__releases(rq->lock)
{
	spin_unlock(&rq->lock);
}

604
static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
L
Linus Torvalds 已提交
605 606 607 608 609 610
	__releases(rq->lock)
{
	spin_unlock_irqrestore(&rq->lock, *flags);
}

/*
611
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
612
 */
A
Alexey Dobriyan 已提交
613
static struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
614 615
	__acquires(rq->lock)
{
616
	struct rq *rq;
L
Linus Torvalds 已提交
617 618 619 620 621 622 623 624

	local_irq_disable();
	rq = this_rq();
	spin_lock(&rq->lock);

	return rq;
}

625
/*
626
 * We are going deep-idle (irqs are disabled):
627
 */
628
void sched_clock_idle_sleep_event(void)
629
{
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
	struct rq *rq = cpu_rq(smp_processor_id());

	spin_lock(&rq->lock);
	__update_rq_clock(rq);
	spin_unlock(&rq->lock);
	rq->clock_deep_idle_events++;
}
EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);

/*
 * We just idled delta nanoseconds (called with irqs disabled):
 */
void sched_clock_idle_wakeup_event(u64 delta_ns)
{
	struct rq *rq = cpu_rq(smp_processor_id());
	u64 now = sched_clock();
646

647 648 649 650 651 652 653 654 655 656 657
	rq->idle_clock += delta_ns;
	/*
	 * Override the previous timestamp and ignore all
	 * sched_clock() deltas that occured while we idled,
	 * and use the PM-provided delta_ns to advance the
	 * rq clock:
	 */
	spin_lock(&rq->lock);
	rq->prev_clock_raw = now;
	rq->clock += delta_ns;
	spin_unlock(&rq->lock);
658
}
659
EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
660

I
Ingo Molnar 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

static void resched_task(struct task_struct *p)
{
	int cpu;

	assert_spin_locked(&task_rq(p)->lock);

	if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
		return;

	set_tsk_thread_flag(p, TIF_NEED_RESCHED);

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	if (!spin_trylock_irqsave(&rq->lock, flags))
		return;
	resched_task(cpu_curr(cpu));
	spin_unlock_irqrestore(&rq->lock, flags);
}
#else
static inline void resched_task(struct task_struct *p)
{
	assert_spin_locked(&task_rq(p)->lock);
	set_tsk_need_resched(p);
}
#endif

713 714 715 716 717 718 719 720
#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

I
Ingo Molnar 已提交
721 722 723
/*
 * Shift right and round:
 */
I
Ingo Molnar 已提交
724
#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
I
Ingo Molnar 已提交
725

726
static unsigned long
727 728 729 730 731 732
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

	if (unlikely(!lw->inv_weight))
I
Ingo Molnar 已提交
733
		lw->inv_weight = (WMULT_CONST - lw->weight/2) / lw->weight + 1;
734 735 736 737 738

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
I
Ingo Molnar 已提交
739
	if (unlikely(tmp > WMULT_CONST))
I
Ingo Molnar 已提交
740
		tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
I
Ingo Molnar 已提交
741 742
			WMULT_SHIFT/2);
	else
I
Ingo Molnar 已提交
743
		tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
744

745
	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
746 747 748 749 750 751 752 753
}

static inline unsigned long
calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
{
	return calc_delta_mine(delta_exec, NICE_0_LOAD, lw);
}

754
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
755 756 757 758
{
	lw->weight += inc;
}

759
static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
760 761 762 763
{
	lw->weight -= dec;
}

764 765 766 767 768 769 770 771 772
/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
 * scheduling class and "nice" value.  For SCHED_NORMAL tasks this is just a
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

I
Ingo Molnar 已提交
773 774 775 776 777 778 779 780 781 782 783
#define WEIGHT_IDLEPRIO		2
#define WMULT_IDLEPRIO		(1 << 31)

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
784 785 786
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
I
Ingo Molnar 已提交
787 788
 */
static const int prio_to_weight[40] = {
789 790 791 792 793 794 795 796
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,
 /*  10 */       110,        87,        70,        56,        45,
 /*  15 */        36,        29,        23,        18,        15,
I
Ingo Molnar 已提交
797 798
};

799 800 801 802 803 804 805
/*
 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
 *
 * In cases where the weight does not change often, we can use the
 * precalculated inverse to speed up arithmetics by turning divisions
 * into multiplications:
 */
I
Ingo Molnar 已提交
806
static const u32 prio_to_wmult[40] = {
807 808 809 810 811 812 813 814
 /* -20 */     48388,     59856,     76040,     92818,    118348,
 /* -15 */    147320,    184698,    229616,    287308,    360437,
 /* -10 */    449829,    563644,    704093,    875809,   1099582,
 /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
 /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
 /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
 /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
 /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
I
Ingo Molnar 已提交
815
};
816

I
Ingo Molnar 已提交
817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);

/*
 * runqueue iterator, to support SMP load-balancing between different
 * scheduling classes, without having to expose their internal data
 * structures to the load-balancing proper:
 */
struct rq_iterator {
	void *arg;
	struct task_struct *(*start)(void *);
	struct task_struct *(*next)(void *);
};

static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
834
		      int *this_best_prio, struct rq_iterator *iterator);
I
Ingo Molnar 已提交
835 836 837

#include "sched_stats.h"
#include "sched_idletask.c"
838 839
#include "sched_fair.c"
#include "sched_rt.c"
I
Ingo Molnar 已提交
840 841 842 843 844 845
#ifdef CONFIG_SCHED_DEBUG
# include "sched_debug.c"
#endif

#define sched_class_highest (&rt_sched_class)

846 847 848 849
/*
 * Update delta_exec, delta_fair fields for rq.
 *
 * delta_fair clock advances at a rate inversely proportional to
850
 * total load (rq->load.weight) on the runqueue, while
851 852 853 854 855 856 857
 * delta_exec advances at the same rate as wall-clock (provided
 * cpu is not idle).
 *
 * delta_exec / delta_fair is a measure of the (smoothened) load on this
 * runqueue over any given interval. This (smoothened) load is used
 * during load balance.
 *
858
 * This function is called /before/ updating rq->load
859 860
 * and when switching tasks.
 */
861
static inline void inc_load(struct rq *rq, const struct task_struct *p)
862
{
863
	update_load_add(&rq->load, p->se.load.weight);
864 865
}

866
static inline void dec_load(struct rq *rq, const struct task_struct *p)
867
{
868
	update_load_sub(&rq->load, p->se.load.weight);
869 870
}

871
static void inc_nr_running(struct task_struct *p, struct rq *rq)
872 873
{
	rq->nr_running++;
874
	inc_load(rq, p);
875 876
}

877
static void dec_nr_running(struct task_struct *p, struct rq *rq)
878 879
{
	rq->nr_running--;
880
	dec_load(rq, p);
881 882
}

883 884 885
static void set_load_weight(struct task_struct *p)
{
	if (task_has_rt_policy(p)) {
I
Ingo Molnar 已提交
886 887 888 889
		p->se.load.weight = prio_to_weight[0] * 2;
		p->se.load.inv_weight = prio_to_wmult[0] >> 1;
		return;
	}
890

I
Ingo Molnar 已提交
891 892 893 894 895 896 897 898
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
		p->se.load.weight = WEIGHT_IDLEPRIO;
		p->se.load.inv_weight = WMULT_IDLEPRIO;
		return;
	}
899

I
Ingo Molnar 已提交
900 901
	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
902 903
}

904
static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
905
{
I
Ingo Molnar 已提交
906
	sched_info_queued(p);
907
	p->sched_class->enqueue_task(rq, p, wakeup);
I
Ingo Molnar 已提交
908
	p->se.on_rq = 1;
909 910
}

911
static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
912
{
913
	p->sched_class->dequeue_task(rq, p, sleep);
I
Ingo Molnar 已提交
914
	p->se.on_rq = 0;
915 916
}

917
/*
I
Ingo Molnar 已提交
918
 * __normal_prio - return the priority that is based on the static prio
919 920 921
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
922
	return p->static_prio;
923 924
}

925 926 927 928 929 930 931
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
932
static inline int normal_prio(struct task_struct *p)
933 934 935
{
	int prio;

936
	if (task_has_rt_policy(p))
937 938 939 940 941 942 943 944 945 946 947 948 949
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
950
static int effective_prio(struct task_struct *p)
951 952 953 954 955 956 957 958 959 960 961 962
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
963
/*
I
Ingo Molnar 已提交
964
 * activate_task - move a task to the runqueue.
L
Linus Torvalds 已提交
965
 */
I
Ingo Molnar 已提交
966
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
L
Linus Torvalds 已提交
967
{
I
Ingo Molnar 已提交
968 969
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;
L
Linus Torvalds 已提交
970

971
	enqueue_task(rq, p, wakeup);
972
	inc_nr_running(p, rq);
L
Linus Torvalds 已提交
973 974 975 976 977
}

/*
 * deactivate_task - remove a task from the runqueue.
 */
978
static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
L
Linus Torvalds 已提交
979
{
I
Ingo Molnar 已提交
980 981 982
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible++;

983
	dequeue_task(rq, p, sleep);
984
	dec_nr_running(p, rq);
L
Linus Torvalds 已提交
985 986 987 988 989 990
}

/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
991
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
992 993 994 995
{
	return cpu_curr(task_cpu(p)) == p;
}

996 997 998
/* Used instead of source_load when we know the type == 0 */
unsigned long weighted_cpuload(const int cpu)
{
999
	return cpu_rq(cpu)->load.weight;
I
Ingo Molnar 已提交
1000 1001 1002 1003 1004 1005 1006
}

static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
#ifdef CONFIG_SMP
	task_thread_info(p)->cpu = cpu;
#endif
S
Srivatsa Vaddagiri 已提交
1007
	set_task_cfs_rq(p);
1008 1009
}

L
Linus Torvalds 已提交
1010
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
1011

I
Ingo Molnar 已提交
1012
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
1013
{
I
Ingo Molnar 已提交
1014 1015
	int old_cpu = task_cpu(p);
	struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
1016 1017
	struct cfs_rq *old_cfsrq = task_cfs_rq(p),
		      *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
1018
	u64 clock_offset;
I
Ingo Molnar 已提交
1019 1020

	clock_offset = old_rq->clock - new_rq->clock;
I
Ingo Molnar 已提交
1021 1022 1023 1024

#ifdef CONFIG_SCHEDSTATS
	if (p->se.wait_start)
		p->se.wait_start -= clock_offset;
I
Ingo Molnar 已提交
1025 1026 1027 1028
	if (p->se.sleep_start)
		p->se.sleep_start -= clock_offset;
	if (p->se.block_start)
		p->se.block_start -= clock_offset;
I
Ingo Molnar 已提交
1029
#endif
1030 1031
	p->se.vruntime -= old_cfsrq->min_vruntime -
					 new_cfsrq->min_vruntime;
I
Ingo Molnar 已提交
1032 1033

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1034 1035
}

1036
struct migration_req {
L
Linus Torvalds 已提交
1037 1038
	struct list_head list;

1039
	struct task_struct *task;
L
Linus Torvalds 已提交
1040 1041 1042
	int dest_cpu;

	struct completion done;
1043
};
L
Linus Torvalds 已提交
1044 1045 1046 1047 1048

/*
 * The task's runqueue lock must be held.
 * Returns true if you have to wait for migration thread.
 */
1049
static int
1050
migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
L
Linus Torvalds 已提交
1051
{
1052
	struct rq *rq = task_rq(p);
L
Linus Torvalds 已提交
1053 1054 1055 1056 1057

	/*
	 * If the task is not on a runqueue (and not running), then
	 * it is sufficient to simply update the task's cpu field.
	 */
I
Ingo Molnar 已提交
1058
	if (!p->se.on_rq && !task_running(rq, p)) {
L
Linus Torvalds 已提交
1059 1060 1061 1062 1063 1064 1065 1066
		set_task_cpu(p, dest_cpu);
		return 0;
	}

	init_completion(&req->done);
	req->task = p;
	req->dest_cpu = dest_cpu;
	list_add(&req->list, &rq->migration_queue);
1067

L
Linus Torvalds 已提交
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
	return 1;
}

/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
1080
void wait_task_inactive(struct task_struct *p)
L
Linus Torvalds 已提交
1081 1082
{
	unsigned long flags;
I
Ingo Molnar 已提交
1083
	int running, on_rq;
1084
	struct rq *rq;
L
Linus Torvalds 已提交
1085 1086

repeat:
1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
	/*
	 * We do the initial early heuristics without holding
	 * any task-queue locks at all. We'll only try to get
	 * the runqueue lock when things look like they will
	 * work out!
	 */
	rq = task_rq(p);

	/*
	 * If the task is actively running on another CPU
	 * still, just relax and busy-wait without holding
	 * any locks.
	 *
	 * NOTE! Since we don't hold any locks, it's not
	 * even sure that "rq" stays as the right runqueue!
	 * But we don't care, since "task_running()" will
	 * return false if the runqueue has changed and p
	 * is actually now running somewhere else!
	 */
	while (task_running(rq, p))
		cpu_relax();

	/*
	 * Ok, time to look more closely! We need the rq
	 * lock now, to be *sure*. If we're wrong, we'll
	 * just go back and repeat.
	 */
L
Linus Torvalds 已提交
1114
	rq = task_rq_lock(p, &flags);
1115
	running = task_running(rq, p);
I
Ingo Molnar 已提交
1116
	on_rq = p->se.on_rq;
1117 1118 1119 1120 1121 1122 1123 1124 1125
	task_rq_unlock(rq, &flags);

	/*
	 * Was it really running after all now that we
	 * checked with the proper locks actually held?
	 *
	 * Oops. Go back and try again..
	 */
	if (unlikely(running)) {
L
Linus Torvalds 已提交
1126 1127 1128
		cpu_relax();
		goto repeat;
	}
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138

	/*
	 * It's not enough that it's not actively running,
	 * it must be off the runqueue _entirely_, and not
	 * preempted!
	 *
	 * So if it wa still runnable (but just not actively
	 * running right now), it's preempted, and we should
	 * yield - it could be a while.
	 */
I
Ingo Molnar 已提交
1139
	if (unlikely(on_rq)) {
1140
		schedule_timeout_uninterruptible(1);
1141 1142 1143 1144 1145 1146 1147 1148
		goto repeat;
	}

	/*
	 * Ahh, all good. It wasn't running, and it wasn't
	 * runnable, which means that it will never become
	 * running in the future either. We're all done!
	 */
L
Linus Torvalds 已提交
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
 * NOTE: this function doesnt have to take the runqueue lock,
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
1164
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}

/*
1176 1177
 * Return a low guess at the load of a migration-source cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1178 1179 1180 1181
 *
 * We want to under-estimate the load of migration sources, to
 * balance conservatively.
 */
A
Alexey Dobriyan 已提交
1182
static unsigned long source_load(int cpu, int type)
L
Linus Torvalds 已提交
1183
{
1184
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1185
	unsigned long total = weighted_cpuload(cpu);
1186

1187
	if (type == 0)
I
Ingo Molnar 已提交
1188
		return total;
1189

I
Ingo Molnar 已提交
1190
	return min(rq->cpu_load[type-1], total);
L
Linus Torvalds 已提交
1191 1192 1193
}

/*
1194 1195
 * Return a high guess at the load of a migration-target cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1196
 */
A
Alexey Dobriyan 已提交
1197
static unsigned long target_load(int cpu, int type)
L
Linus Torvalds 已提交
1198
{
1199
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1200
	unsigned long total = weighted_cpuload(cpu);
1201

N
Nick Piggin 已提交
1202
	if (type == 0)
I
Ingo Molnar 已提交
1203
		return total;
1204

I
Ingo Molnar 已提交
1205
	return max(rq->cpu_load[type-1], total);
1206 1207 1208 1209 1210 1211 1212
}

/*
 * Return the average load per task on the cpu's run queue
 */
static inline unsigned long cpu_avg_load_per_task(int cpu)
{
1213
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1214
	unsigned long total = weighted_cpuload(cpu);
1215 1216
	unsigned long n = rq->nr_running;

I
Ingo Molnar 已提交
1217
	return n ? total / n : SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
1218 1219
}

N
Nick Piggin 已提交
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
/*
 * find_idlest_group finds and returns the least busy CPU group within the
 * domain.
 */
static struct sched_group *
find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
{
	struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
	unsigned long min_load = ULONG_MAX, this_load = 0;
	int load_idx = sd->forkexec_idx;
	int imbalance = 100 + (sd->imbalance_pct-100)/2;

	do {
		unsigned long load, avg_load;
		int local_group;
		int i;

1237 1238 1239 1240
		/* Skip over this group if it has no CPUs allowed */
		if (!cpus_intersects(group->cpumask, p->cpus_allowed))
			goto nextgroup;

N
Nick Piggin 已提交
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
		local_group = cpu_isset(this_cpu, group->cpumask);

		/* Tally up the load of all CPUs in the group */
		avg_load = 0;

		for_each_cpu_mask(i, group->cpumask) {
			/* Bias balancing toward cpus of our domain */
			if (local_group)
				load = source_load(i, load_idx);
			else
				load = target_load(i, load_idx);

			avg_load += load;
		}

		/* Adjust by relative CPU power of the group */
1257 1258
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
N
Nick Piggin 已提交
1259 1260 1261 1262 1263 1264 1265 1266

		if (local_group) {
			this_load = avg_load;
			this = group;
		} else if (avg_load < min_load) {
			min_load = avg_load;
			idlest = group;
		}
1267
nextgroup:
N
Nick Piggin 已提交
1268 1269 1270 1271 1272 1273 1274 1275 1276
		group = group->next;
	} while (group != sd->groups);

	if (!idlest || 100*this_load < imbalance*min_load)
		return NULL;
	return idlest;
}

/*
1277
 * find_idlest_cpu - find the idlest cpu among the cpus in group.
N
Nick Piggin 已提交
1278
 */
I
Ingo Molnar 已提交
1279 1280
static int
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
N
Nick Piggin 已提交
1281
{
1282
	cpumask_t tmp;
N
Nick Piggin 已提交
1283 1284 1285 1286
	unsigned long load, min_load = ULONG_MAX;
	int idlest = -1;
	int i;

1287 1288 1289 1290
	/* Traverse only the allowed CPUs */
	cpus_and(tmp, group->cpumask, p->cpus_allowed);

	for_each_cpu_mask(i, tmp) {
1291
		load = weighted_cpuload(i);
N
Nick Piggin 已提交
1292 1293 1294 1295 1296 1297 1298 1299 1300 1301

		if (load < min_load || (load == min_load && i == this_cpu)) {
			min_load = load;
			idlest = i;
		}
	}

	return idlest;
}

N
Nick Piggin 已提交
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
/*
 * sched_balance_self: balance the current task (running on cpu) in domains
 * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
 * SD_BALANCE_EXEC.
 *
 * Balance, ie. select the least loaded group.
 *
 * Returns the target CPU number, or the same CPU if no balancing is needed.
 *
 * preempt must be disabled.
 */
static int sched_balance_self(int cpu, int flag)
{
	struct task_struct *t = current;
	struct sched_domain *tmp, *sd = NULL;
N
Nick Piggin 已提交
1317

1318
	for_each_domain(cpu, tmp) {
I
Ingo Molnar 已提交
1319 1320 1321
		/*
		 * If power savings logic is enabled for a domain, stop there.
		 */
1322 1323
		if (tmp->flags & SD_POWERSAVINGS_BALANCE)
			break;
N
Nick Piggin 已提交
1324 1325
		if (tmp->flags & flag)
			sd = tmp;
1326
	}
N
Nick Piggin 已提交
1327 1328 1329 1330

	while (sd) {
		cpumask_t span;
		struct sched_group *group;
1331 1332 1333 1334 1335 1336
		int new_cpu, weight;

		if (!(sd->flags & flag)) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1337 1338 1339

		span = sd->span;
		group = find_idlest_group(sd, t, cpu);
1340 1341 1342 1343
		if (!group) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1344

1345
		new_cpu = find_idlest_cpu(group, t, cpu);
1346 1347 1348 1349 1350
		if (new_cpu == -1 || new_cpu == cpu) {
			/* Now try balancing at a lower domain level of cpu */
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1351

1352
		/* Now try balancing at a lower domain level of new_cpu */
N
Nick Piggin 已提交
1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
		cpu = new_cpu;
		sd = NULL;
		weight = cpus_weight(span);
		for_each_domain(cpu, tmp) {
			if (weight <= cpus_weight(tmp->span))
				break;
			if (tmp->flags & flag)
				sd = tmp;
		}
		/* while loop will break here if sd == NULL */
	}

	return cpu;
}

#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378

/*
 * wake_idle() will wake a task on an idle cpu if task->cpu is
 * not idle and an idle cpu is available.  The span of cpus to
 * search starts with cpus closest then further out as needed,
 * so we always favor a closer, idle cpu.
 *
 * Returns the CPU we should wake onto.
 */
#if defined(ARCH_HAS_SCHED_WAKE_IDLE)
1379
static int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1380 1381 1382 1383 1384
{
	cpumask_t tmp;
	struct sched_domain *sd;
	int i;

1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
	/*
	 * If it is idle, then it is the best cpu to run this task.
	 *
	 * This cpu is also the best, if it has more than one task already.
	 * Siblings must be also busy(in most cases) as they didn't already
	 * pickup the extra load from this cpu and hence we need not check
	 * sibling runqueue info. This will avoid the checks and cache miss
	 * penalities associated with that.
	 */
	if (idle_cpu(cpu) || cpu_rq(cpu)->nr_running > 1)
L
Linus Torvalds 已提交
1395 1396 1397 1398
		return cpu;

	for_each_domain(cpu, sd) {
		if (sd->flags & SD_WAKE_IDLE) {
N
Nick Piggin 已提交
1399
			cpus_and(tmp, sd->span, p->cpus_allowed);
L
Linus Torvalds 已提交
1400 1401 1402 1403
			for_each_cpu_mask(i, tmp) {
				if (idle_cpu(i))
					return i;
			}
I
Ingo Molnar 已提交
1404
		} else {
N
Nick Piggin 已提交
1405
			break;
I
Ingo Molnar 已提交
1406
		}
L
Linus Torvalds 已提交
1407 1408 1409 1410
	}
	return cpu;
}
#else
1411
static inline int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
{
	return cpu;
}
#endif

/***
 * try_to_wake_up - wake up a thread
 * @p: the to-be-woken-up thread
 * @state: the mask of task states that can be woken
 * @sync: do a synchronous wakeup?
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
 * returns failure only if the task is already active.
 */
1431
static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
L
Linus Torvalds 已提交
1432 1433 1434 1435
{
	int cpu, this_cpu, success = 0;
	unsigned long flags;
	long old_state;
1436
	struct rq *rq;
L
Linus Torvalds 已提交
1437
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
1438
	struct sched_domain *sd, *this_sd = NULL;
1439
	unsigned long load, this_load;
L
Linus Torvalds 已提交
1440 1441 1442 1443 1444 1445 1446 1447
	int new_cpu;
#endif

	rq = task_rq_lock(p, &flags);
	old_state = p->state;
	if (!(old_state & state))
		goto out;

I
Ingo Molnar 已提交
1448
	if (p->se.on_rq)
L
Linus Torvalds 已提交
1449 1450 1451 1452 1453 1454 1455 1456 1457
		goto out_running;

	cpu = task_cpu(p);
	this_cpu = smp_processor_id();

#ifdef CONFIG_SMP
	if (unlikely(task_running(rq, p)))
		goto out_activate;

N
Nick Piggin 已提交
1458 1459
	new_cpu = cpu;

1460
	schedstat_inc(rq, ttwu_count);
L
Linus Torvalds 已提交
1461 1462
	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
N
Nick Piggin 已提交
1463 1464 1465 1466 1467 1468 1469 1470
		goto out_set_cpu;
	}

	for_each_domain(this_cpu, sd) {
		if (cpu_isset(cpu, sd->span)) {
			schedstat_inc(sd, ttwu_wake_remote);
			this_sd = sd;
			break;
L
Linus Torvalds 已提交
1471 1472 1473
		}
	}

N
Nick Piggin 已提交
1474
	if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed)))
L
Linus Torvalds 已提交
1475 1476 1477
		goto out_set_cpu;

	/*
N
Nick Piggin 已提交
1478
	 * Check for affine wakeup and passive balancing possibilities.
L
Linus Torvalds 已提交
1479
	 */
N
Nick Piggin 已提交
1480 1481 1482
	if (this_sd) {
		int idx = this_sd->wake_idx;
		unsigned int imbalance;
L
Linus Torvalds 已提交
1483

1484 1485
		imbalance = 100 + (this_sd->imbalance_pct - 100) / 2;

N
Nick Piggin 已提交
1486 1487
		load = source_load(cpu, idx);
		this_load = target_load(this_cpu, idx);
L
Linus Torvalds 已提交
1488

N
Nick Piggin 已提交
1489 1490
		new_cpu = this_cpu; /* Wake to this CPU if we can */

1491 1492
		if (this_sd->flags & SD_WAKE_AFFINE) {
			unsigned long tl = this_load;
1493 1494 1495
			unsigned long tl_per_task;

			tl_per_task = cpu_avg_load_per_task(this_cpu);
1496

L
Linus Torvalds 已提交
1497
			/*
1498 1499 1500
			 * If sync wakeup then subtract the (maximum possible)
			 * effect of the currently running task from the load
			 * of the current CPU:
L
Linus Torvalds 已提交
1501
			 */
1502
			if (sync)
I
Ingo Molnar 已提交
1503
				tl -= current->se.load.weight;
1504 1505

			if ((tl <= load &&
1506
				tl + target_load(cpu, idx) <= tl_per_task) ||
I
Ingo Molnar 已提交
1507
			       100*(tl + p->se.load.weight) <= imbalance*load) {
1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526
				/*
				 * This domain has SD_WAKE_AFFINE and
				 * p is cache cold in this domain, and
				 * there is no bad imbalance.
				 */
				schedstat_inc(this_sd, ttwu_move_affine);
				goto out_set_cpu;
			}
		}

		/*
		 * Start passive balancing when half the imbalance_pct
		 * limit is reached.
		 */
		if (this_sd->flags & SD_WAKE_BALANCE) {
			if (imbalance*this_load <= 100*load) {
				schedstat_inc(this_sd, ttwu_move_balance);
				goto out_set_cpu;
			}
L
Linus Torvalds 已提交
1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540
		}
	}

	new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */
out_set_cpu:
	new_cpu = wake_idle(new_cpu, p);
	if (new_cpu != cpu) {
		set_task_cpu(p, new_cpu);
		task_rq_unlock(rq, &flags);
		/* might preempt at this point */
		rq = task_rq_lock(p, &flags);
		old_state = p->state;
		if (!(old_state & state))
			goto out;
I
Ingo Molnar 已提交
1541
		if (p->se.on_rq)
L
Linus Torvalds 已提交
1542 1543 1544 1545 1546 1547 1548 1549
			goto out_running;

		this_cpu = smp_processor_id();
		cpu = task_cpu(p);
	}

out_activate:
#endif /* CONFIG_SMP */
I
Ingo Molnar 已提交
1550
	update_rq_clock(rq);
I
Ingo Molnar 已提交
1551
	activate_task(rq, p, 1);
L
Linus Torvalds 已提交
1552 1553 1554 1555 1556 1557 1558 1559
	/*
	 * Sync wakeups (i.e. those types of wakeups where the waker
	 * has indicated that it will leave the CPU in short order)
	 * don't trigger a preemption, if the woken up task will run on
	 * this cpu. (in this case the 'I will reschedule' promise of
	 * the waker guarantees that the freshly woken up task is going
	 * to be considered on this CPU.)
	 */
I
Ingo Molnar 已提交
1560 1561
	if (!sync || cpu != this_cpu)
		check_preempt_curr(rq, p);
L
Linus Torvalds 已提交
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
	success = 1;

out_running:
	p->state = TASK_RUNNING;
out:
	task_rq_unlock(rq, &flags);

	return success;
}

1572
int fastcall wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
1573 1574 1575 1576 1577 1578
{
	return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED |
				 TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0);
}
EXPORT_SYMBOL(wake_up_process);

1579
int fastcall wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
1580 1581 1582 1583 1584 1585 1586
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
1587 1588 1589 1590 1591 1592 1593
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
1594
	p->se.prev_sum_exec_runtime	= 0;
I
Ingo Molnar 已提交
1595 1596 1597

#ifdef CONFIG_SCHEDSTATS
	p->se.wait_start		= 0;
I
Ingo Molnar 已提交
1598 1599 1600 1601 1602 1603
	p->se.sum_sleep_runtime		= 0;
	p->se.sleep_start		= 0;
	p->se.block_start		= 0;
	p->se.sleep_max			= 0;
	p->se.block_max			= 0;
	p->se.exec_max			= 0;
I
Ingo Molnar 已提交
1604
	p->se.slice_max			= 0;
I
Ingo Molnar 已提交
1605
	p->se.wait_max			= 0;
I
Ingo Molnar 已提交
1606
#endif
N
Nick Piggin 已提交
1607

I
Ingo Molnar 已提交
1608 1609
	INIT_LIST_HEAD(&p->run_list);
	p->se.on_rq = 0;
N
Nick Piggin 已提交
1610

1611 1612 1613 1614
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif

L
Linus Torvalds 已提交
1615 1616 1617 1618 1619 1620 1621
	/*
	 * We mark the process as running here, but have not actually
	 * inserted it onto the runqueue yet. This guarantees that
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
}

/*
 * fork()/clone()-time setup:
 */
void sched_fork(struct task_struct *p, int clone_flags)
{
	int cpu = get_cpu();

	__sched_fork(p);

#ifdef CONFIG_SMP
	cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
#endif
I
Ingo Molnar 已提交
1636
	set_task_cpu(p, cpu);
1637 1638 1639 1640 1641

	/*
	 * Make sure we do not leak PI boosting priority to the child:
	 */
	p->prio = current->normal_prio;
H
Hiroshi Shimamoto 已提交
1642 1643
	if (!rt_prio(p->prio))
		p->sched_class = &fair_sched_class;
1644

1645
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
1646
	if (likely(sched_info_on()))
1647
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
1648
#endif
1649
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
1650 1651
	p->oncpu = 0;
#endif
L
Linus Torvalds 已提交
1652
#ifdef CONFIG_PREEMPT
1653
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
1654
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
1655
#endif
N
Nick Piggin 已提交
1656
	put_cpu();
L
Linus Torvalds 已提交
1657 1658 1659 1660 1661 1662 1663 1664 1665
}

/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
1666
void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
L
Linus Torvalds 已提交
1667 1668
{
	unsigned long flags;
I
Ingo Molnar 已提交
1669
	struct rq *rq;
L
Linus Torvalds 已提交
1670 1671

	rq = task_rq_lock(p, &flags);
N
Nick Piggin 已提交
1672
	BUG_ON(p->state != TASK_RUNNING);
I
Ingo Molnar 已提交
1673
	update_rq_clock(rq);
L
Linus Torvalds 已提交
1674 1675 1676

	p->prio = effective_prio(p);

1677
	if (!p->sched_class->task_new || !current->se.on_rq || !rq->cfs.curr) {
I
Ingo Molnar 已提交
1678
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
1679 1680
	} else {
		/*
I
Ingo Molnar 已提交
1681 1682
		 * Let the scheduling class do new task startup
		 * management (if any):
L
Linus Torvalds 已提交
1683
		 */
1684
		p->sched_class->task_new(rq, p);
1685
		inc_nr_running(p, rq);
L
Linus Torvalds 已提交
1686
	}
I
Ingo Molnar 已提交
1687 1688
	check_preempt_curr(rq, p);
	task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
1689 1690
}

1691 1692 1693
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
R
Randy Dunlap 已提交
1694 1695
 * preempt_notifier_register - tell me when current is being being preempted & rescheduled
 * @notifier: notifier struct to register
1696 1697 1698 1699 1700 1701 1702 1703 1704
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
1705
 * @notifier: notifier struct to unregister
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_out(notifier, next);
}

#else

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

#endif

1749 1750 1751
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
1752
 * @prev: the current task that is being switched out
1753 1754 1755 1756 1757 1758 1759 1760 1761
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
1762 1763 1764
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
1765
{
1766
	fire_sched_out_preempt_notifiers(prev, next);
1767 1768 1769 1770
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
1771 1772
/**
 * finish_task_switch - clean up after a task-switch
1773
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
1774 1775
 * @prev: the thread we just switched away from.
 *
1776 1777 1778 1779
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
1780 1781 1782 1783 1784 1785
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
 * so, we finish that here outside of the runqueue lock.  (Doing it
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
A
Alexey Dobriyan 已提交
1786
static void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
1787 1788 1789
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
1790
	long prev_state;
L
Linus Torvalds 已提交
1791 1792 1793 1794 1795

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
1796
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
1797 1798
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
1799
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
1800 1801 1802 1803 1804
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
1805
	prev_state = prev->state;
1806 1807
	finish_arch_switch(prev);
	finish_lock_switch(rq, prev);
1808
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
1809 1810
	if (mm)
		mmdrop(mm);
1811
	if (unlikely(prev_state == TASK_DEAD)) {
1812 1813 1814
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
1815
		 */
1816
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
1817
		put_task_struct(prev);
1818
	}
L
Linus Torvalds 已提交
1819 1820 1821 1822 1823 1824
}

/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
1825
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
1826 1827
	__releases(rq->lock)
{
1828 1829
	struct rq *rq = this_rq();

1830 1831 1832 1833 1834
	finish_task_switch(rq, prev);
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
1835 1836 1837 1838 1839 1840 1841 1842
	if (current->set_child_tid)
		put_user(current->pid, current->set_child_tid);
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
1843
static inline void
1844
context_switch(struct rq *rq, struct task_struct *prev,
1845
	       struct task_struct *next)
L
Linus Torvalds 已提交
1846
{
I
Ingo Molnar 已提交
1847
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
1848

1849
	prepare_task_switch(rq, prev, next);
I
Ingo Molnar 已提交
1850 1851
	mm = next->mm;
	oldmm = prev->active_mm;
1852 1853 1854 1855 1856 1857 1858
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
	arch_enter_lazy_cpu_mode();

I
Ingo Molnar 已提交
1859
	if (unlikely(!mm)) {
L
Linus Torvalds 已提交
1860 1861 1862 1863 1864 1865
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

I
Ingo Molnar 已提交
1866
	if (unlikely(!prev->mm)) {
L
Linus Torvalds 已提交
1867 1868 1869
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
1870 1871 1872 1873 1874 1875 1876
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
1877
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
1878
#endif
L
Linus Torvalds 已提交
1879 1880 1881 1882

	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
1883 1884 1885 1886 1887 1888 1889
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912
}

/*
 * nr_running, nr_uninterruptible and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, current number of uninterruptible-sleeping threads, total
 * number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
}

unsigned long nr_uninterruptible(void)
{
	unsigned long i, sum = 0;

1913
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927
		sum += cpu_rq(i)->nr_uninterruptible;

	/*
	 * Since we read the counters lockless, it might be slightly
	 * inaccurate. Do not allow it to go below zero though:
	 */
	if (unlikely((long)sum < 0))
		sum = 0;

	return sum;
}

unsigned long long nr_context_switches(void)
{
1928 1929
	int i;
	unsigned long long sum = 0;
L
Linus Torvalds 已提交
1930

1931
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1932 1933 1934 1935 1936 1937 1938 1939 1940
		sum += cpu_rq(i)->nr_switches;

	return sum;
}

unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;

1941
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1942 1943 1944 1945 1946
		sum += atomic_read(&cpu_rq(i)->nr_iowait);

	return sum;
}

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
unsigned long nr_active(void)
{
	unsigned long i, running = 0, uninterruptible = 0;

	for_each_online_cpu(i) {
		running += cpu_rq(i)->nr_running;
		uninterruptible += cpu_rq(i)->nr_uninterruptible;
	}

	if (unlikely((long)uninterruptible < 0))
		uninterruptible = 0;

	return running + uninterruptible;
}

1962
/*
I
Ingo Molnar 已提交
1963 1964
 * Update rq->cpu_load[] statistics. This function is usually called every
 * scheduler tick (TICK_NSEC).
1965
 */
I
Ingo Molnar 已提交
1966
static void update_cpu_load(struct rq *this_rq)
1967
{
1968
	unsigned long this_load = this_rq->load.weight;
I
Ingo Molnar 已提交
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980
	int i, scale;

	this_rq->nr_load_updates++;

	/* Update our load: */
	for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
		unsigned long old_load, new_load;

		/* scale is effectively 1 << i now, and >> i divides by scale */

		old_load = this_rq->cpu_load[i];
		new_load = this_load;
I
Ingo Molnar 已提交
1981 1982 1983 1984 1985 1986 1987
		/*
		 * Round up the averaging division if load is increasing. This
		 * prevents us from getting stuck on 9 if the load is 10, for
		 * example.
		 */
		if (new_load > old_load)
			new_load += scale-1;
I
Ingo Molnar 已提交
1988 1989
		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
	}
1990 1991
}

I
Ingo Molnar 已提交
1992 1993
#ifdef CONFIG_SMP

L
Linus Torvalds 已提交
1994 1995 1996 1997 1998 1999
/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
2000
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2001 2002 2003
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
2004
	BUG_ON(!irqs_disabled());
L
Linus Torvalds 已提交
2005 2006 2007 2008
	if (rq1 == rq2) {
		spin_lock(&rq1->lock);
		__acquire(rq2->lock);	/* Fake it out ;) */
	} else {
2009
		if (rq1 < rq2) {
L
Linus Torvalds 已提交
2010 2011 2012 2013 2014 2015 2016
			spin_lock(&rq1->lock);
			spin_lock(&rq2->lock);
		} else {
			spin_lock(&rq2->lock);
			spin_lock(&rq1->lock);
		}
	}
2017 2018
	update_rq_clock(rq1);
	update_rq_clock(rq2);
L
Linus Torvalds 已提交
2019 2020 2021 2022 2023 2024 2025 2026
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
2027
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	spin_unlock(&rq1->lock);
	if (rq1 != rq2)
		spin_unlock(&rq2->lock);
	else
		__release(rq2->lock);
}

/*
 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
 */
2041
static void double_lock_balance(struct rq *this_rq, struct rq *busiest)
L
Linus Torvalds 已提交
2042 2043 2044 2045
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
2046 2047 2048 2049 2050
	if (unlikely(!irqs_disabled())) {
		/* printk() doesn't work good under rq->lock */
		spin_unlock(&this_rq->lock);
		BUG_ON(1);
	}
L
Linus Torvalds 已提交
2051
	if (unlikely(!spin_trylock(&busiest->lock))) {
2052
		if (busiest < this_rq) {
L
Linus Torvalds 已提交
2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
			spin_unlock(&this_rq->lock);
			spin_lock(&busiest->lock);
			spin_lock(&this_rq->lock);
		} else
			spin_lock(&busiest->lock);
	}
}

/*
 * If dest_cpu is allowed for this process, migrate the task to it.
 * This is accomplished by forcing the cpu_allowed mask to only
 * allow dest_cpu, which will force the cpu onto dest_cpu.  Then
 * the cpu_allowed mask is restored.
 */
2067
static void sched_migrate_task(struct task_struct *p, int dest_cpu)
L
Linus Torvalds 已提交
2068
{
2069
	struct migration_req req;
L
Linus Torvalds 已提交
2070
	unsigned long flags;
2071
	struct rq *rq;
L
Linus Torvalds 已提交
2072 2073 2074 2075 2076 2077 2078 2079 2080 2081

	rq = task_rq_lock(p, &flags);
	if (!cpu_isset(dest_cpu, p->cpus_allowed)
	    || unlikely(cpu_is_offline(dest_cpu)))
		goto out;

	/* force the process onto the specified CPU */
	if (migrate_task(p, dest_cpu, &req)) {
		/* Need to wait for migration thread (might exit: take ref). */
		struct task_struct *mt = rq->migration_thread;
2082

L
Linus Torvalds 已提交
2083 2084 2085 2086 2087
		get_task_struct(mt);
		task_rq_unlock(rq, &flags);
		wake_up_process(mt);
		put_task_struct(mt);
		wait_for_completion(&req.done);
2088

L
Linus Torvalds 已提交
2089 2090 2091 2092 2093 2094 2095
		return;
	}
out:
	task_rq_unlock(rq, &flags);
}

/*
N
Nick Piggin 已提交
2096 2097
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
L
Linus Torvalds 已提交
2098 2099 2100 2101
 */
void sched_exec(void)
{
	int new_cpu, this_cpu = get_cpu();
N
Nick Piggin 已提交
2102
	new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
L
Linus Torvalds 已提交
2103
	put_cpu();
N
Nick Piggin 已提交
2104 2105
	if (new_cpu != this_cpu)
		sched_migrate_task(current, new_cpu);
L
Linus Torvalds 已提交
2106 2107 2108 2109 2110 2111
}

/*
 * pull_task - move a task from a remote runqueue to the local runqueue.
 * Both runqueues must be locked.
 */
I
Ingo Molnar 已提交
2112 2113
static void pull_task(struct rq *src_rq, struct task_struct *p,
		      struct rq *this_rq, int this_cpu)
L
Linus Torvalds 已提交
2114
{
2115
	deactivate_task(src_rq, p, 0);
L
Linus Torvalds 已提交
2116
	set_task_cpu(p, this_cpu);
I
Ingo Molnar 已提交
2117
	activate_task(this_rq, p, 0);
L
Linus Torvalds 已提交
2118 2119 2120 2121
	/*
	 * Note that idle threads have a prio of MAX_PRIO, for this test
	 * to be always true for them.
	 */
I
Ingo Molnar 已提交
2122
	check_preempt_curr(this_rq, p);
L
Linus Torvalds 已提交
2123 2124 2125 2126 2127
}

/*
 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
 */
2128
static
2129
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
I
Ingo Molnar 已提交
2130
		     struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2131
		     int *all_pinned)
L
Linus Torvalds 已提交
2132 2133 2134 2135 2136 2137 2138 2139 2140
{
	/*
	 * We do not migrate tasks that are:
	 * 1) running (obviously), or
	 * 2) cannot be migrated to this CPU due to cpus_allowed, or
	 * 3) are cache-hot on their current CPU.
	 */
	if (!cpu_isset(this_cpu, p->cpus_allowed))
		return 0;
2141 2142 2143 2144
	*all_pinned = 0;

	if (task_running(rq, p))
		return 0;
L
Linus Torvalds 已提交
2145 2146 2147 2148

	return 1;
}

I
Ingo Molnar 已提交
2149
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2150
		      unsigned long max_nr_move, unsigned long max_load_move,
I
Ingo Molnar 已提交
2151
		      struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2152
		      int *all_pinned, unsigned long *load_moved,
2153
		      int *this_best_prio, struct rq_iterator *iterator)
L
Linus Torvalds 已提交
2154
{
I
Ingo Molnar 已提交
2155 2156 2157
	int pulled = 0, pinned = 0, skip_for_load;
	struct task_struct *p;
	long rem_load_move = max_load_move;
L
Linus Torvalds 已提交
2158

2159
	if (max_nr_move == 0 || max_load_move == 0)
L
Linus Torvalds 已提交
2160 2161
		goto out;

2162 2163
	pinned = 1;

L
Linus Torvalds 已提交
2164
	/*
I
Ingo Molnar 已提交
2165
	 * Start the load-balancing iterator:
L
Linus Torvalds 已提交
2166
	 */
I
Ingo Molnar 已提交
2167 2168 2169
	p = iterator->start(iterator->arg);
next:
	if (!p)
L
Linus Torvalds 已提交
2170
		goto out;
2171 2172 2173 2174 2175
	/*
	 * To help distribute high priority tasks accross CPUs we don't
	 * skip a task if it will be the highest priority task (i.e. smallest
	 * prio value) on its new queue regardless of its load weight
	 */
I
Ingo Molnar 已提交
2176 2177
	skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
							 SCHED_LOAD_SCALE_FUZZ;
2178
	if ((skip_for_load && p->prio >= *this_best_prio) ||
I
Ingo Molnar 已提交
2179 2180 2181
	    !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2182 2183
	}

I
Ingo Molnar 已提交
2184
	pull_task(busiest, p, this_rq, this_cpu);
L
Linus Torvalds 已提交
2185
	pulled++;
I
Ingo Molnar 已提交
2186
	rem_load_move -= p->se.load.weight;
L
Linus Torvalds 已提交
2187

2188 2189 2190 2191 2192
	/*
	 * We only want to steal up to the prescribed number of tasks
	 * and the prescribed amount of weighted load.
	 */
	if (pulled < max_nr_move && rem_load_move > 0) {
2193 2194
		if (p->prio < *this_best_prio)
			*this_best_prio = p->prio;
I
Ingo Molnar 已提交
2195 2196
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2197 2198 2199 2200 2201 2202 2203 2204
	}
out:
	/*
	 * Right now, this is the only place pull_task() is called,
	 * so we can safely collect pull_task() stats here rather than
	 * inside pull_task().
	 */
	schedstat_add(sd, lb_gained[idle], pulled);
2205 2206 2207

	if (all_pinned)
		*all_pinned = pinned;
I
Ingo Molnar 已提交
2208
	*load_moved = max_load_move - rem_load_move;
L
Linus Torvalds 已提交
2209 2210 2211
	return pulled;
}

I
Ingo Molnar 已提交
2212
/*
P
Peter Williams 已提交
2213 2214 2215
 * move_tasks tries to move up to max_load_move weighted load from busiest to
 * this_rq, as part of a balancing operation within domain "sd".
 * Returns 1 if successful and 0 otherwise.
I
Ingo Molnar 已提交
2216 2217 2218 2219
 *
 * Called with both runqueues locked.
 */
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
P
Peter Williams 已提交
2220
		      unsigned long max_load_move,
I
Ingo Molnar 已提交
2221 2222 2223
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned)
{
2224
	const struct sched_class *class = sched_class_highest;
P
Peter Williams 已提交
2225
	unsigned long total_load_moved = 0;
2226
	int this_best_prio = this_rq->curr->prio;
I
Ingo Molnar 已提交
2227 2228

	do {
P
Peter Williams 已提交
2229 2230 2231
		total_load_moved +=
			class->load_balance(this_rq, this_cpu, busiest,
				ULONG_MAX, max_load_move - total_load_moved,
2232
				sd, idle, all_pinned, &this_best_prio);
I
Ingo Molnar 已提交
2233
		class = class->next;
P
Peter Williams 已提交
2234
	} while (class && max_load_move > total_load_moved);
I
Ingo Molnar 已提交
2235

P
Peter Williams 已提交
2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248
	return total_load_moved > 0;
}

/*
 * move_one_task tries to move exactly one task from busiest to this_rq, as
 * part of active balancing operations within "domain".
 * Returns 1 if successful and 0 otherwise.
 *
 * Called with both runqueues locked.
 */
static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
			 struct sched_domain *sd, enum cpu_idle_type idle)
{
2249
	const struct sched_class *class;
2250
	int this_best_prio = MAX_PRIO;
P
Peter Williams 已提交
2251 2252 2253

	for (class = sched_class_highest; class; class = class->next)
		if (class->load_balance(this_rq, this_cpu, busiest,
2254 2255
					1, ULONG_MAX, sd, idle, NULL,
					&this_best_prio))
P
Peter Williams 已提交
2256 2257 2258
			return 1;

	return 0;
I
Ingo Molnar 已提交
2259 2260
}

L
Linus Torvalds 已提交
2261 2262
/*
 * find_busiest_group finds and returns the busiest CPU group within the
2263 2264
 * domain. It calculates and returns the amount of weighted load which
 * should be moved to restore balance via the imbalance parameter.
L
Linus Torvalds 已提交
2265 2266 2267
 */
static struct sched_group *
find_busiest_group(struct sched_domain *sd, int this_cpu,
I
Ingo Molnar 已提交
2268 2269
		   unsigned long *imbalance, enum cpu_idle_type idle,
		   int *sd_idle, cpumask_t *cpus, int *balance)
L
Linus Torvalds 已提交
2270 2271 2272
{
	struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
	unsigned long max_load, avg_load, total_load, this_load, total_pwr;
2273
	unsigned long max_pull;
2274 2275
	unsigned long busiest_load_per_task, busiest_nr_running;
	unsigned long this_load_per_task, this_nr_running;
N
Nick Piggin 已提交
2276
	int load_idx;
2277 2278 2279 2280 2281 2282
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
	int power_savings_balance = 1;
	unsigned long leader_nr_running = 0, min_load_per_task = 0;
	unsigned long min_nr_running = ULONG_MAX;
	struct sched_group *group_min = NULL, *group_leader = NULL;
#endif
L
Linus Torvalds 已提交
2283 2284

	max_load = this_load = total_load = total_pwr = 0;
2285 2286
	busiest_load_per_task = busiest_nr_running = 0;
	this_load_per_task = this_nr_running = 0;
I
Ingo Molnar 已提交
2287
	if (idle == CPU_NOT_IDLE)
N
Nick Piggin 已提交
2288
		load_idx = sd->busy_idx;
I
Ingo Molnar 已提交
2289
	else if (idle == CPU_NEWLY_IDLE)
N
Nick Piggin 已提交
2290 2291 2292
		load_idx = sd->newidle_idx;
	else
		load_idx = sd->idle_idx;
L
Linus Torvalds 已提交
2293 2294

	do {
2295
		unsigned long load, group_capacity;
L
Linus Torvalds 已提交
2296 2297
		int local_group;
		int i;
2298
		unsigned int balance_cpu = -1, first_idle_cpu = 0;
2299
		unsigned long sum_nr_running, sum_weighted_load;
L
Linus Torvalds 已提交
2300 2301 2302

		local_group = cpu_isset(this_cpu, group->cpumask);

2303 2304 2305
		if (local_group)
			balance_cpu = first_cpu(group->cpumask);

L
Linus Torvalds 已提交
2306
		/* Tally up the load of all CPUs in the group */
2307
		sum_weighted_load = sum_nr_running = avg_load = 0;
L
Linus Torvalds 已提交
2308 2309

		for_each_cpu_mask(i, group->cpumask) {
2310 2311 2312 2313 2314 2315
			struct rq *rq;

			if (!cpu_isset(i, *cpus))
				continue;

			rq = cpu_rq(i);
2316

2317
			if (*sd_idle && rq->nr_running)
N
Nick Piggin 已提交
2318 2319
				*sd_idle = 0;

L
Linus Torvalds 已提交
2320
			/* Bias balancing toward cpus of our domain */
2321 2322 2323 2324 2325 2326
			if (local_group) {
				if (idle_cpu(i) && !first_idle_cpu) {
					first_idle_cpu = 1;
					balance_cpu = i;
				}

N
Nick Piggin 已提交
2327
				load = target_load(i, load_idx);
2328
			} else
N
Nick Piggin 已提交
2329
				load = source_load(i, load_idx);
L
Linus Torvalds 已提交
2330 2331

			avg_load += load;
2332
			sum_nr_running += rq->nr_running;
I
Ingo Molnar 已提交
2333
			sum_weighted_load += weighted_cpuload(i);
L
Linus Torvalds 已提交
2334 2335
		}

2336 2337 2338
		/*
		 * First idle cpu or the first cpu(busiest) in this sched group
		 * is eligible for doing load balancing at this and above
2339 2340
		 * domains. In the newly idle case, we will allow all the cpu's
		 * to do the newly idle load balance.
2341
		 */
2342 2343
		if (idle != CPU_NEWLY_IDLE && local_group &&
		    balance_cpu != this_cpu && balance) {
2344 2345 2346 2347
			*balance = 0;
			goto ret;
		}

L
Linus Torvalds 已提交
2348
		total_load += avg_load;
2349
		total_pwr += group->__cpu_power;
L
Linus Torvalds 已提交
2350 2351

		/* Adjust by relative CPU power of the group */
2352 2353
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2354

2355
		group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
2356

L
Linus Torvalds 已提交
2357 2358 2359
		if (local_group) {
			this_load = avg_load;
			this = group;
2360 2361 2362
			this_nr_running = sum_nr_running;
			this_load_per_task = sum_weighted_load;
		} else if (avg_load > max_load &&
2363
			   sum_nr_running > group_capacity) {
L
Linus Torvalds 已提交
2364 2365
			max_load = avg_load;
			busiest = group;
2366 2367
			busiest_nr_running = sum_nr_running;
			busiest_load_per_task = sum_weighted_load;
L
Linus Torvalds 已提交
2368
		}
2369 2370 2371 2372 2373 2374

#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
		/*
		 * Busy processors will not participate in power savings
		 * balance.
		 */
I
Ingo Molnar 已提交
2375 2376 2377
		if (idle == CPU_NOT_IDLE ||
				!(sd->flags & SD_POWERSAVINGS_BALANCE))
			goto group_next;
2378 2379 2380 2381 2382 2383 2384 2385 2386

		/*
		 * If the local group is idle or completely loaded
		 * no need to do power savings balance at this domain
		 */
		if (local_group && (this_nr_running >= group_capacity ||
				    !this_nr_running))
			power_savings_balance = 0;

I
Ingo Molnar 已提交
2387
		/*
2388 2389
		 * If a group is already running at full capacity or idle,
		 * don't include that group in power savings calculations
I
Ingo Molnar 已提交
2390 2391
		 */
		if (!power_savings_balance || sum_nr_running >= group_capacity
2392
		    || !sum_nr_running)
I
Ingo Molnar 已提交
2393
			goto group_next;
2394

I
Ingo Molnar 已提交
2395
		/*
2396
		 * Calculate the group which has the least non-idle load.
I
Ingo Molnar 已提交
2397 2398 2399 2400 2401
		 * This is the group from where we need to pick up the load
		 * for saving power
		 */
		if ((sum_nr_running < min_nr_running) ||
		    (sum_nr_running == min_nr_running &&
2402 2403
		     first_cpu(group->cpumask) <
		     first_cpu(group_min->cpumask))) {
I
Ingo Molnar 已提交
2404 2405
			group_min = group;
			min_nr_running = sum_nr_running;
2406 2407
			min_load_per_task = sum_weighted_load /
						sum_nr_running;
I
Ingo Molnar 已提交
2408
		}
2409

I
Ingo Molnar 已提交
2410
		/*
2411
		 * Calculate the group which is almost near its
I
Ingo Molnar 已提交
2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422
		 * capacity but still has some space to pick up some load
		 * from other group and save more power
		 */
		if (sum_nr_running <= group_capacity - 1) {
			if (sum_nr_running > leader_nr_running ||
			    (sum_nr_running == leader_nr_running &&
			     first_cpu(group->cpumask) >
			      first_cpu(group_leader->cpumask))) {
				group_leader = group;
				leader_nr_running = sum_nr_running;
			}
2423
		}
2424 2425
group_next:
#endif
L
Linus Torvalds 已提交
2426 2427 2428
		group = group->next;
	} while (group != sd->groups);

2429
	if (!busiest || this_load >= max_load || busiest_nr_running == 0)
L
Linus Torvalds 已提交
2430 2431 2432 2433 2434 2435 2436 2437
		goto out_balanced;

	avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;

	if (this_load >= avg_load ||
			100*max_load <= sd->imbalance_pct*this_load)
		goto out_balanced;

2438
	busiest_load_per_task /= busiest_nr_running;
L
Linus Torvalds 已提交
2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449
	/*
	 * We're trying to get all the cpus to the average_load, so we don't
	 * want to push ourselves above the average load, nor do we wish to
	 * reduce the max loaded cpu below the average load, as either of these
	 * actions would just result in more rebalancing later, and ping-pong
	 * tasks around. Thus we look for the minimum possible imbalance.
	 * Negative imbalances (*we* are more loaded than anyone else) will
	 * be counted as no imbalance for these purposes -- we can't fix that
	 * by pulling tasks to us.  Be careful of negative numbers as they'll
	 * appear as very large values with unsigned longs.
	 */
2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461
	if (max_load <= busiest_load_per_task)
		goto out_balanced;

	/*
	 * In the presence of smp nice balancing, certain scenarios can have
	 * max load less than avg load(as we skip the groups at or below
	 * its cpu_power, while calculating max_load..)
	 */
	if (max_load < avg_load) {
		*imbalance = 0;
		goto small_imbalance;
	}
2462 2463

	/* Don't want to pull so many tasks that a group would go idle */
2464
	max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
2465

L
Linus Torvalds 已提交
2466
	/* How much load to actually move to equalise the imbalance */
2467 2468
	*imbalance = min(max_pull * busiest->__cpu_power,
				(avg_load - this_load) * this->__cpu_power)
L
Linus Torvalds 已提交
2469 2470
			/ SCHED_LOAD_SCALE;

2471 2472 2473 2474 2475 2476
	/*
	 * if *imbalance is less than the average load per runnable task
	 * there is no gaurantee that any tasks will be moved so we'll have
	 * a think about bumping its value to force at least one task to be
	 * moved
	 */
2477
	if (*imbalance < busiest_load_per_task) {
2478
		unsigned long tmp, pwr_now, pwr_move;
2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489
		unsigned int imbn;

small_imbalance:
		pwr_move = pwr_now = 0;
		imbn = 2;
		if (this_nr_running) {
			this_load_per_task /= this_nr_running;
			if (busiest_load_per_task > this_load_per_task)
				imbn = 1;
		} else
			this_load_per_task = SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
2490

I
Ingo Molnar 已提交
2491 2492
		if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >=
					busiest_load_per_task * imbn) {
2493
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2494 2495 2496 2497 2498 2499 2500 2501 2502
			return busiest;
		}

		/*
		 * OK, we don't have enough imbalance to justify moving tasks,
		 * however we may be able to increase total CPU power used by
		 * moving them.
		 */

2503 2504 2505 2506
		pwr_now += busiest->__cpu_power *
				min(busiest_load_per_task, max_load);
		pwr_now += this->__cpu_power *
				min(this_load_per_task, this_load);
L
Linus Torvalds 已提交
2507 2508 2509
		pwr_now /= SCHED_LOAD_SCALE;

		/* Amount of load we'd subtract */
2510 2511
		tmp = sg_div_cpu_power(busiest,
				busiest_load_per_task * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2512
		if (max_load > tmp)
2513
			pwr_move += busiest->__cpu_power *
2514
				min(busiest_load_per_task, max_load - tmp);
L
Linus Torvalds 已提交
2515 2516

		/* Amount of load we'd add */
2517
		if (max_load * busiest->__cpu_power <
2518
				busiest_load_per_task * SCHED_LOAD_SCALE)
2519 2520
			tmp = sg_div_cpu_power(this,
					max_load * busiest->__cpu_power);
L
Linus Torvalds 已提交
2521
		else
2522 2523 2524 2525
			tmp = sg_div_cpu_power(this,
				busiest_load_per_task * SCHED_LOAD_SCALE);
		pwr_move += this->__cpu_power *
				min(this_load_per_task, this_load + tmp);
L
Linus Torvalds 已提交
2526 2527 2528
		pwr_move /= SCHED_LOAD_SCALE;

		/* Move if we gain throughput */
2529 2530
		if (pwr_move > pwr_now)
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2531 2532 2533 2534 2535
	}

	return busiest;

out_balanced:
2536
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
2537
	if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
2538
		goto ret;
L
Linus Torvalds 已提交
2539

2540 2541 2542 2543 2544
	if (this == group_leader && group_leader != group_min) {
		*imbalance = min_load_per_task;
		return group_min;
	}
#endif
2545
ret:
L
Linus Torvalds 已提交
2546 2547 2548 2549 2550 2551 2552
	*imbalance = 0;
	return NULL;
}

/*
 * find_busiest_queue - find the busiest runqueue among the cpus in group.
 */
2553
static struct rq *
I
Ingo Molnar 已提交
2554
find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
2555
		   unsigned long imbalance, cpumask_t *cpus)
L
Linus Torvalds 已提交
2556
{
2557
	struct rq *busiest = NULL, *rq;
2558
	unsigned long max_load = 0;
L
Linus Torvalds 已提交
2559 2560 2561
	int i;

	for_each_cpu_mask(i, group->cpumask) {
I
Ingo Molnar 已提交
2562
		unsigned long wl;
2563 2564 2565 2566

		if (!cpu_isset(i, *cpus))
			continue;

2567
		rq = cpu_rq(i);
I
Ingo Molnar 已提交
2568
		wl = weighted_cpuload(i);
2569

I
Ingo Molnar 已提交
2570
		if (rq->nr_running == 1 && wl > imbalance)
2571
			continue;
L
Linus Torvalds 已提交
2572

I
Ingo Molnar 已提交
2573 2574
		if (wl > max_load) {
			max_load = wl;
2575
			busiest = rq;
L
Linus Torvalds 已提交
2576 2577 2578 2579 2580 2581
		}
	}

	return busiest;
}

2582 2583 2584 2585 2586 2587
/*
 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
 * so long as it is large enough.
 */
#define MAX_PINNED_INTERVAL	512

L
Linus Torvalds 已提交
2588 2589 2590 2591
/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
2592
static int load_balance(int this_cpu, struct rq *this_rq,
I
Ingo Molnar 已提交
2593
			struct sched_domain *sd, enum cpu_idle_type idle,
2594
			int *balance)
L
Linus Torvalds 已提交
2595
{
P
Peter Williams 已提交
2596
	int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
L
Linus Torvalds 已提交
2597 2598
	struct sched_group *group;
	unsigned long imbalance;
2599
	struct rq *busiest;
2600
	cpumask_t cpus = CPU_MASK_ALL;
2601
	unsigned long flags;
N
Nick Piggin 已提交
2602

2603 2604 2605
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
I
Ingo Molnar 已提交
2606
	 * let the state of idle sibling percolate up as CPU_IDLE, instead of
I
Ingo Molnar 已提交
2607
	 * portraying it as CPU_NOT_IDLE.
2608
	 */
I
Ingo Molnar 已提交
2609
	if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
2610
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2611
		sd_idle = 1;
L
Linus Torvalds 已提交
2612

2613
	schedstat_inc(sd, lb_count[idle]);
L
Linus Torvalds 已提交
2614

2615 2616
redo:
	group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
2617 2618
				   &cpus, balance);

2619
	if (*balance == 0)
2620 2621
		goto out_balanced;

L
Linus Torvalds 已提交
2622 2623 2624 2625 2626
	if (!group) {
		schedstat_inc(sd, lb_nobusyg[idle]);
		goto out_balanced;
	}

2627
	busiest = find_busiest_queue(group, idle, imbalance, &cpus);
L
Linus Torvalds 已提交
2628 2629 2630 2631 2632
	if (!busiest) {
		schedstat_inc(sd, lb_nobusyq[idle]);
		goto out_balanced;
	}

N
Nick Piggin 已提交
2633
	BUG_ON(busiest == this_rq);
L
Linus Torvalds 已提交
2634 2635 2636

	schedstat_add(sd, lb_imbalance[idle], imbalance);

P
Peter Williams 已提交
2637
	ld_moved = 0;
L
Linus Torvalds 已提交
2638 2639 2640 2641
	if (busiest->nr_running > 1) {
		/*
		 * Attempt to move tasks. If find_busiest_group has found
		 * an imbalance but busiest->nr_running <= 1, the group is
P
Peter Williams 已提交
2642
		 * still unbalanced. ld_moved simply stays zero, so it is
L
Linus Torvalds 已提交
2643 2644
		 * correctly treated as an imbalance.
		 */
2645
		local_irq_save(flags);
N
Nick Piggin 已提交
2646
		double_rq_lock(this_rq, busiest);
P
Peter Williams 已提交
2647
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2648
				      imbalance, sd, idle, &all_pinned);
N
Nick Piggin 已提交
2649
		double_rq_unlock(this_rq, busiest);
2650
		local_irq_restore(flags);
2651

2652 2653 2654
		/*
		 * some other cpu did the load balance for us.
		 */
P
Peter Williams 已提交
2655
		if (ld_moved && this_cpu != smp_processor_id())
2656 2657
			resched_cpu(this_cpu);

2658
		/* All tasks on this runqueue were pinned by CPU affinity */
2659 2660 2661 2662
		if (unlikely(all_pinned)) {
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
2663
			goto out_balanced;
2664
		}
L
Linus Torvalds 已提交
2665
	}
2666

P
Peter Williams 已提交
2667
	if (!ld_moved) {
L
Linus Torvalds 已提交
2668 2669 2670 2671 2672
		schedstat_inc(sd, lb_failed[idle]);
		sd->nr_balance_failed++;

		if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {

2673
			spin_lock_irqsave(&busiest->lock, flags);
2674 2675 2676 2677 2678

			/* don't kick the migration_thread, if the curr
			 * task on busiest cpu can't be moved to this_cpu
			 */
			if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
2679
				spin_unlock_irqrestore(&busiest->lock, flags);
2680 2681 2682 2683
				all_pinned = 1;
				goto out_one_pinned;
			}

L
Linus Torvalds 已提交
2684 2685 2686
			if (!busiest->active_balance) {
				busiest->active_balance = 1;
				busiest->push_cpu = this_cpu;
2687
				active_balance = 1;
L
Linus Torvalds 已提交
2688
			}
2689
			spin_unlock_irqrestore(&busiest->lock, flags);
2690
			if (active_balance)
L
Linus Torvalds 已提交
2691 2692 2693 2694 2695 2696
				wake_up_process(busiest->migration_thread);

			/*
			 * We've kicked active balancing, reset the failure
			 * counter.
			 */
2697
			sd->nr_balance_failed = sd->cache_nice_tries+1;
L
Linus Torvalds 已提交
2698
		}
2699
	} else
L
Linus Torvalds 已提交
2700 2701
		sd->nr_balance_failed = 0;

2702
	if (likely(!active_balance)) {
L
Linus Torvalds 已提交
2703 2704
		/* We were unbalanced, so reset the balancing interval */
		sd->balance_interval = sd->min_interval;
2705 2706 2707 2708 2709 2710 2711 2712 2713
	} else {
		/*
		 * If we've begun active balancing, start to back off. This
		 * case may not be covered by the all_pinned logic if there
		 * is only 1 task on the busy runqueue (because we don't call
		 * move_tasks).
		 */
		if (sd->balance_interval < sd->max_interval)
			sd->balance_interval *= 2;
L
Linus Torvalds 已提交
2714 2715
	}

P
Peter Williams 已提交
2716
	if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2717
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2718
		return -1;
P
Peter Williams 已提交
2719
	return ld_moved;
L
Linus Torvalds 已提交
2720 2721 2722 2723

out_balanced:
	schedstat_inc(sd, lb_balanced[idle]);

2724
	sd->nr_balance_failed = 0;
2725 2726

out_one_pinned:
L
Linus Torvalds 已提交
2727
	/* tune up the balancing interval */
2728 2729
	if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
			(sd->balance_interval < sd->max_interval))
L
Linus Torvalds 已提交
2730 2731
		sd->balance_interval *= 2;

2732
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2733
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2734
		return -1;
L
Linus Torvalds 已提交
2735 2736 2737 2738 2739 2740 2741
	return 0;
}

/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 *
I
Ingo Molnar 已提交
2742
 * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
L
Linus Torvalds 已提交
2743 2744
 * this_rq is locked.
 */
2745
static int
2746
load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
L
Linus Torvalds 已提交
2747 2748
{
	struct sched_group *group;
2749
	struct rq *busiest = NULL;
L
Linus Torvalds 已提交
2750
	unsigned long imbalance;
P
Peter Williams 已提交
2751
	int ld_moved = 0;
N
Nick Piggin 已提交
2752
	int sd_idle = 0;
2753
	int all_pinned = 0;
2754
	cpumask_t cpus = CPU_MASK_ALL;
N
Nick Piggin 已提交
2755

2756 2757 2758 2759
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
	 * let the state of idle sibling percolate up as IDLE, instead of
I
Ingo Molnar 已提交
2760
	 * portraying it as CPU_NOT_IDLE.
2761 2762 2763
	 */
	if (sd->flags & SD_SHARE_CPUPOWER &&
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2764
		sd_idle = 1;
L
Linus Torvalds 已提交
2765

2766
	schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
2767
redo:
I
Ingo Molnar 已提交
2768
	group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
2769
				   &sd_idle, &cpus, NULL);
L
Linus Torvalds 已提交
2770
	if (!group) {
I
Ingo Molnar 已提交
2771
		schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
2772
		goto out_balanced;
L
Linus Torvalds 已提交
2773 2774
	}

I
Ingo Molnar 已提交
2775
	busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance,
2776
				&cpus);
N
Nick Piggin 已提交
2777
	if (!busiest) {
I
Ingo Molnar 已提交
2778
		schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
2779
		goto out_balanced;
L
Linus Torvalds 已提交
2780 2781
	}

N
Nick Piggin 已提交
2782 2783
	BUG_ON(busiest == this_rq);

I
Ingo Molnar 已提交
2784
	schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
2785

P
Peter Williams 已提交
2786
	ld_moved = 0;
2787 2788 2789
	if (busiest->nr_running > 1) {
		/* Attempt to move tasks */
		double_lock_balance(this_rq, busiest);
2790 2791
		/* this_rq->clock is already updated */
		update_rq_clock(busiest);
P
Peter Williams 已提交
2792
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2793 2794
					imbalance, sd, CPU_NEWLY_IDLE,
					&all_pinned);
2795
		spin_unlock(&busiest->lock);
2796

2797
		if (unlikely(all_pinned)) {
2798 2799 2800 2801
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
		}
2802 2803
	}

P
Peter Williams 已提交
2804
	if (!ld_moved) {
I
Ingo Molnar 已提交
2805
		schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
2806 2807
		if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
		    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2808 2809
			return -1;
	} else
2810
		sd->nr_balance_failed = 0;
L
Linus Torvalds 已提交
2811

P
Peter Williams 已提交
2812
	return ld_moved;
2813 2814

out_balanced:
I
Ingo Molnar 已提交
2815
	schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
2816
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2817
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2818
		return -1;
2819
	sd->nr_balance_failed = 0;
2820

2821
	return 0;
L
Linus Torvalds 已提交
2822 2823 2824 2825 2826 2827
}

/*
 * idle_balance is called by schedule() if this_cpu is about to become
 * idle. Attempts to pull tasks from other CPUs.
 */
2828
static void idle_balance(int this_cpu, struct rq *this_rq)
L
Linus Torvalds 已提交
2829 2830
{
	struct sched_domain *sd;
I
Ingo Molnar 已提交
2831 2832
	int pulled_task = -1;
	unsigned long next_balance = jiffies + HZ;
L
Linus Torvalds 已提交
2833 2834

	for_each_domain(this_cpu, sd) {
2835 2836 2837 2838 2839 2840
		unsigned long interval;

		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		if (sd->flags & SD_BALANCE_NEWIDLE)
2841
			/* If we've pulled tasks over stop searching: */
2842
			pulled_task = load_balance_newidle(this_cpu,
2843 2844 2845 2846 2847 2848 2849
								this_rq, sd);

		interval = msecs_to_jiffies(sd->balance_interval);
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
		if (pulled_task)
			break;
L
Linus Torvalds 已提交
2850
	}
I
Ingo Molnar 已提交
2851
	if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
2852 2853 2854 2855 2856
		/*
		 * We are going idle. next_balance may be set based on
		 * a busy processor. So reset next_balance.
		 */
		this_rq->next_balance = next_balance;
I
Ingo Molnar 已提交
2857
	}
L
Linus Torvalds 已提交
2858 2859 2860 2861 2862 2863 2864 2865 2866 2867
}

/*
 * active_load_balance is run by migration threads. It pushes running tasks
 * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
 * running on each physical CPU where possible, and avoids physical /
 * logical imbalances.
 *
 * Called with busiest_rq locked.
 */
2868
static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
L
Linus Torvalds 已提交
2869
{
2870
	int target_cpu = busiest_rq->push_cpu;
2871 2872
	struct sched_domain *sd;
	struct rq *target_rq;
2873

2874
	/* Is there any task to move? */
2875 2876 2877 2878
	if (busiest_rq->nr_running <= 1)
		return;

	target_rq = cpu_rq(target_cpu);
L
Linus Torvalds 已提交
2879 2880

	/*
2881 2882 2883
	 * This condition is "impossible", if it occurs
	 * we need to fix it.  Originally reported by
	 * Bjorn Helgaas on a 128-cpu setup.
L
Linus Torvalds 已提交
2884
	 */
2885
	BUG_ON(busiest_rq == target_rq);
L
Linus Torvalds 已提交
2886

2887 2888
	/* move a task from busiest_rq to target_rq */
	double_lock_balance(busiest_rq, target_rq);
2889 2890
	update_rq_clock(busiest_rq);
	update_rq_clock(target_rq);
2891 2892

	/* Search for an sd spanning us and the target CPU. */
2893
	for_each_domain(target_cpu, sd) {
2894
		if ((sd->flags & SD_LOAD_BALANCE) &&
2895
		    cpu_isset(busiest_cpu, sd->span))
2896
				break;
2897
	}
2898

2899
	if (likely(sd)) {
2900
		schedstat_inc(sd, alb_count);
2901

P
Peter Williams 已提交
2902 2903
		if (move_one_task(target_rq, target_cpu, busiest_rq,
				  sd, CPU_IDLE))
2904 2905 2906 2907
			schedstat_inc(sd, alb_pushed);
		else
			schedstat_inc(sd, alb_failed);
	}
2908
	spin_unlock(&target_rq->lock);
L
Linus Torvalds 已提交
2909 2910
}

2911 2912 2913 2914 2915 2916 2917 2918 2919
#ifdef CONFIG_NO_HZ
static struct {
	atomic_t load_balancer;
	cpumask_t  cpu_mask;
} nohz ____cacheline_aligned = {
	.load_balancer = ATOMIC_INIT(-1),
	.cpu_mask = CPU_MASK_NONE,
};

2920
/*
2921 2922 2923 2924 2925 2926 2927 2928 2929 2930
 * This routine will try to nominate the ilb (idle load balancing)
 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
 * load balancing on behalf of all those cpus. If all the cpus in the system
 * go into this tickless mode, then there will be no ilb owner (as there is
 * no need for one) and all the cpus will sleep till the next wakeup event
 * arrives...
 *
 * For the ilb owner, tick is not stopped. And this tick will be used
 * for idle load balancing. ilb owner will still be part of
 * nohz.cpu_mask..
2931
 *
2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987
 * While stopping the tick, this cpu will become the ilb owner if there
 * is no other owner. And will be the owner till that cpu becomes busy
 * or if all cpus in the system stop their ticks at which point
 * there is no need for ilb owner.
 *
 * When the ilb owner becomes busy, it nominates another owner, during the
 * next busy scheduler_tick()
 */
int select_nohz_load_balancer(int stop_tick)
{
	int cpu = smp_processor_id();

	if (stop_tick) {
		cpu_set(cpu, nohz.cpu_mask);
		cpu_rq(cpu)->in_nohz_recently = 1;

		/*
		 * If we are going offline and still the leader, give up!
		 */
		if (cpu_is_offline(cpu) &&
		    atomic_read(&nohz.load_balancer) == cpu) {
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
			return 0;
		}

		/* time for ilb owner also to sleep */
		if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
			if (atomic_read(&nohz.load_balancer) == cpu)
				atomic_set(&nohz.load_balancer, -1);
			return 0;
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/* make me the ilb owner */
			if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
				return 1;
		} else if (atomic_read(&nohz.load_balancer) == cpu)
			return 1;
	} else {
		if (!cpu_isset(cpu, nohz.cpu_mask))
			return 0;

		cpu_clear(cpu, nohz.cpu_mask);

		if (atomic_read(&nohz.load_balancer) == cpu)
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
	}
	return 0;
}
#endif

static DEFINE_SPINLOCK(balancing);

/*
2988 2989 2990 2991 2992
 * It checks each scheduling domain to see if it is due to be balanced,
 * and initiates a balancing operation if so.
 *
 * Balancing parameters are set up in arch_init_sched_domains.
 */
A
Alexey Dobriyan 已提交
2993
static void rebalance_domains(int cpu, enum cpu_idle_type idle)
2994
{
2995 2996
	int balance = 1;
	struct rq *rq = cpu_rq(cpu);
2997 2998
	unsigned long interval;
	struct sched_domain *sd;
2999
	/* Earliest time when we have to do rebalance again */
3000
	unsigned long next_balance = jiffies + 60*HZ;
3001
	int update_next_balance = 0;
L
Linus Torvalds 已提交
3002

3003
	for_each_domain(cpu, sd) {
L
Linus Torvalds 已提交
3004 3005 3006 3007
		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		interval = sd->balance_interval;
I
Ingo Molnar 已提交
3008
		if (idle != CPU_IDLE)
L
Linus Torvalds 已提交
3009 3010 3011 3012 3013 3014
			interval *= sd->busy_factor;

		/* scale ms to jiffies */
		interval = msecs_to_jiffies(interval);
		if (unlikely(!interval))
			interval = 1;
I
Ingo Molnar 已提交
3015 3016 3017
		if (interval > HZ*NR_CPUS/10)
			interval = HZ*NR_CPUS/10;

L
Linus Torvalds 已提交
3018

3019 3020 3021 3022 3023
		if (sd->flags & SD_SERIALIZE) {
			if (!spin_trylock(&balancing))
				goto out;
		}

3024
		if (time_after_eq(jiffies, sd->last_balance + interval)) {
3025
			if (load_balance(cpu, rq, sd, idle, &balance)) {
3026 3027
				/*
				 * We've pulled tasks over so either we're no
N
Nick Piggin 已提交
3028 3029 3030
				 * longer idle, or one of our SMT siblings is
				 * not idle.
				 */
I
Ingo Molnar 已提交
3031
				idle = CPU_NOT_IDLE;
L
Linus Torvalds 已提交
3032
			}
3033
			sd->last_balance = jiffies;
L
Linus Torvalds 已提交
3034
		}
3035 3036 3037
		if (sd->flags & SD_SERIALIZE)
			spin_unlock(&balancing);
out:
3038
		if (time_after(next_balance, sd->last_balance + interval)) {
3039
			next_balance = sd->last_balance + interval;
3040 3041
			update_next_balance = 1;
		}
3042 3043 3044 3045 3046 3047 3048 3049

		/*
		 * Stop the load balance at this level. There is another
		 * CPU in our sched group which is doing load balancing more
		 * actively.
		 */
		if (!balance)
			break;
L
Linus Torvalds 已提交
3050
	}
3051 3052 3053 3054 3055 3056 3057 3058

	/*
	 * next_balance will be updated only when there is a need.
	 * When the cpu is attached to null domain for ex, it will not be
	 * updated.
	 */
	if (likely(update_next_balance))
		rq->next_balance = next_balance;
3059 3060 3061 3062 3063 3064 3065 3066 3067
}

/*
 * run_rebalance_domains is triggered when needed from the scheduler tick.
 * In CONFIG_NO_HZ case, the idle load balance owner will do the
 * rebalancing for all the cpus for whom scheduler ticks are stopped.
 */
static void run_rebalance_domains(struct softirq_action *h)
{
I
Ingo Molnar 已提交
3068 3069 3070 3071
	int this_cpu = smp_processor_id();
	struct rq *this_rq = cpu_rq(this_cpu);
	enum cpu_idle_type idle = this_rq->idle_at_tick ?
						CPU_IDLE : CPU_NOT_IDLE;
3072

I
Ingo Molnar 已提交
3073
	rebalance_domains(this_cpu, idle);
3074 3075 3076 3077 3078 3079 3080

#ifdef CONFIG_NO_HZ
	/*
	 * If this cpu is the owner for idle load balancing, then do the
	 * balancing on behalf of the other idle cpus whose ticks are
	 * stopped.
	 */
I
Ingo Molnar 已提交
3081 3082
	if (this_rq->idle_at_tick &&
	    atomic_read(&nohz.load_balancer) == this_cpu) {
3083 3084 3085 3086
		cpumask_t cpus = nohz.cpu_mask;
		struct rq *rq;
		int balance_cpu;

I
Ingo Molnar 已提交
3087
		cpu_clear(this_cpu, cpus);
3088 3089 3090 3091 3092 3093 3094 3095 3096
		for_each_cpu_mask(balance_cpu, cpus) {
			/*
			 * If this cpu gets work to do, stop the load balancing
			 * work being done for other cpus. Next load
			 * balancing owner will pick it up.
			 */
			if (need_resched())
				break;

3097
			rebalance_domains(balance_cpu, CPU_IDLE);
3098 3099

			rq = cpu_rq(balance_cpu);
I
Ingo Molnar 已提交
3100 3101
			if (time_after(this_rq->next_balance, rq->next_balance))
				this_rq->next_balance = rq->next_balance;
3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113
		}
	}
#endif
}

/*
 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
 *
 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
 * idle load balancing owner or decide to stop the periodic load balancing,
 * if the whole system is idle.
 */
I
Ingo Molnar 已提交
3114
static inline void trigger_load_balance(struct rq *rq, int cpu)
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
{
#ifdef CONFIG_NO_HZ
	/*
	 * If we were in the nohz mode recently and busy at the current
	 * scheduler tick, then check if we need to nominate new idle
	 * load balancer.
	 */
	if (rq->in_nohz_recently && !rq->idle_at_tick) {
		rq->in_nohz_recently = 0;

		if (atomic_read(&nohz.load_balancer) == cpu) {
			cpu_clear(cpu, nohz.cpu_mask);
			atomic_set(&nohz.load_balancer, -1);
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/*
			 * simple selection for now: Nominate the
			 * first cpu in the nohz list to be the next
			 * ilb owner.
			 *
			 * TBD: Traverse the sched domains and nominate
			 * the nearest cpu in the nohz.cpu_mask.
			 */
			int ilb = first_cpu(nohz.cpu_mask);

			if (ilb != NR_CPUS)
				resched_cpu(ilb);
		}
	}

	/*
	 * If this cpu is idle and doing idle load balancing for all the
	 * cpus with ticks stopped, is it time for that to stop?
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
	    cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
		resched_cpu(cpu);
		return;
	}

	/*
	 * If this cpu is idle and the idle load balancing is done by
	 * someone else, then no need raise the SCHED_SOFTIRQ
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
	    cpu_isset(cpu, nohz.cpu_mask))
		return;
#endif
	if (time_after_eq(jiffies, rq->next_balance))
		raise_softirq(SCHED_SOFTIRQ);
L
Linus Torvalds 已提交
3166
}
I
Ingo Molnar 已提交
3167 3168 3169

#else	/* CONFIG_SMP */

L
Linus Torvalds 已提交
3170 3171 3172
/*
 * on UP we do not need to balance between CPUs:
 */
3173
static inline void idle_balance(int cpu, struct rq *rq)
L
Linus Torvalds 已提交
3174 3175
{
}
I
Ingo Molnar 已提交
3176 3177 3178 3179 3180 3181

/* Avoid "used but not defined" warning on UP */
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
3182
		      int *this_best_prio, struct rq_iterator *iterator)
I
Ingo Molnar 已提交
3183 3184 3185 3186 3187 3188
{
	*load_moved = 0;

	return 0;
}

L
Linus Torvalds 已提交
3189 3190 3191 3192 3193 3194 3195
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
3196 3197
 * Return p->sum_exec_runtime plus any more ns on the sched_clock
 * that have not yet been banked in case the task is currently running.
L
Linus Torvalds 已提交
3198
 */
3199
unsigned long long task_sched_runtime(struct task_struct *p)
L
Linus Torvalds 已提交
3200 3201
{
	unsigned long flags;
3202 3203
	u64 ns, delta_exec;
	struct rq *rq;
3204

3205 3206 3207
	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime;
	if (rq->curr == p) {
I
Ingo Molnar 已提交
3208 3209
		update_rq_clock(rq);
		delta_exec = rq->clock - p->se.exec_start;
3210 3211 3212 3213
		if ((s64)delta_exec > 0)
			ns += delta_exec;
	}
	task_rq_unlock(rq, &flags);
3214

L
Linus Torvalds 已提交
3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248
	return ns;
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in user space since the last update
 */
void account_user_time(struct task_struct *p, cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

	p->utime = cputime_add(p->utime, cputime);

	/* Add user time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (TASK_NICE(p) > 0)
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
	else
		cpustat->user = cputime64_add(cpustat->user, tmp);
}

/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
 */
void account_system_time(struct task_struct *p, int hardirq_offset,
			 cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3249
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
	cputime64_t tmp;

	p->stime = cputime_add(p->stime, cputime);

	/* Add system time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (hardirq_count() - hardirq_offset)
		cpustat->irq = cputime64_add(cpustat->irq, tmp);
	else if (softirq_count())
		cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
	else if (p != rq->idle)
		cpustat->system = cputime64_add(cpustat->system, tmp);
	else if (atomic_read(&rq->nr_iowait) > 0)
		cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
	else
		cpustat->idle = cputime64_add(cpustat->idle, tmp);
	/* Account for system time used */
	acct_update_integrals(p);
}

/*
 * Account for involuntary wait time.
 * @p: the process from which the cpu time has been stolen
 * @steal: the cpu time spent in involuntary wait
 */
void account_steal_time(struct task_struct *p, cputime_t steal)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp = cputime_to_cputime64(steal);
3279
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290

	if (p == rq->idle) {
		p->stime = cputime_add(p->stime, steal);
		if (atomic_read(&rq->nr_iowait) > 0)
			cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
		else
			cpustat->idle = cputime64_add(cpustat->idle, tmp);
	} else
		cpustat->steal = cputime64_add(cpustat->steal, tmp);
}

3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 *
 * It also gets called by the fork code, when changing the parent's
 * timeslices.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
3302
	struct task_struct *curr = rq->curr;
3303
	u64 next_tick = rq->tick_timestamp + TICK_NSEC;
I
Ingo Molnar 已提交
3304 3305

	spin_lock(&rq->lock);
3306
	__update_rq_clock(rq);
3307 3308 3309 3310 3311 3312
	/*
	 * Let rq->clock advance by at least TICK_NSEC:
	 */
	if (unlikely(rq->clock < next_tick))
		rq->clock = next_tick;
	rq->tick_timestamp = rq->clock;
3313
	update_cpu_load(rq);
I
Ingo Molnar 已提交
3314 3315 3316
	if (curr != rq->idle) /* FIXME: needed? */
		curr->sched_class->task_tick(rq, curr);
	spin_unlock(&rq->lock);
3317

3318
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
3319 3320
	rq->idle_at_tick = idle_cpu(cpu);
	trigger_load_balance(rq, cpu);
3321
#endif
L
Linus Torvalds 已提交
3322 3323 3324 3325 3326 3327 3328 3329 3330
}

#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)

void fastcall add_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3331 3332
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
L
Linus Torvalds 已提交
3333 3334 3335 3336
	preempt_count() += val;
	/*
	 * Spinlock count overflowing soon?
	 */
3337 3338
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
L
Linus Torvalds 已提交
3339 3340 3341 3342 3343 3344 3345 3346
}
EXPORT_SYMBOL(add_preempt_count);

void fastcall sub_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3347 3348
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
		return;
L
Linus Torvalds 已提交
3349 3350 3351
	/*
	 * Is the spinlock portion underflowing?
	 */
3352 3353 3354 3355
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;

L
Linus Torvalds 已提交
3356 3357 3358 3359 3360 3361 3362
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
3363
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
3364
 */
I
Ingo Molnar 已提交
3365
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
3366
{
I
Ingo Molnar 已提交
3367 3368 3369 3370 3371 3372 3373
	printk(KERN_ERR "BUG: scheduling while atomic: %s/0x%08x/%d\n",
		prev->comm, preempt_count(), prev->pid);
	debug_show_held_locks(prev);
	if (irqs_disabled())
		print_irqtrace_events(prev);
	dump_stack();
}
L
Linus Torvalds 已提交
3374

I
Ingo Molnar 已提交
3375 3376 3377 3378 3379
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
3380 3381 3382 3383 3384
	/*
	 * Test if we are atomic.  Since do_exit() needs to call into
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
I
Ingo Molnar 已提交
3385 3386 3387
	if (unlikely(in_atomic_preempt_off()) && unlikely(!prev->exit_state))
		__schedule_bug(prev);

L
Linus Torvalds 已提交
3388 3389
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

3390
	schedstat_inc(this_rq(), sched_count);
I
Ingo Molnar 已提交
3391 3392
#ifdef CONFIG_SCHEDSTATS
	if (unlikely(prev->lock_depth >= 0)) {
3393 3394
		schedstat_inc(this_rq(), bkl_count);
		schedstat_inc(prev, sched_info.bkl_count);
I
Ingo Molnar 已提交
3395 3396
	}
#endif
I
Ingo Molnar 已提交
3397 3398 3399 3400 3401 3402
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
3403
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
3404
{
3405
	const struct sched_class *class;
I
Ingo Molnar 已提交
3406
	struct task_struct *p;
L
Linus Torvalds 已提交
3407 3408

	/*
I
Ingo Molnar 已提交
3409 3410
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
3411
	 */
I
Ingo Molnar 已提交
3412
	if (likely(rq->nr_running == rq->cfs.nr_running)) {
3413
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
3414 3415
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
3416 3417
	}

I
Ingo Molnar 已提交
3418 3419
	class = sched_class_highest;
	for ( ; ; ) {
3420
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
3421 3422 3423 3424 3425 3426 3427 3428 3429
		if (p)
			return p;
		/*
		 * Will never be NULL as the idle class always
		 * returns a non-NULL p:
		 */
		class = class->next;
	}
}
L
Linus Torvalds 已提交
3430

I
Ingo Molnar 已提交
3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452
/*
 * schedule() is the main scheduler function.
 */
asmlinkage void __sched schedule(void)
{
	struct task_struct *prev, *next;
	long *switch_count;
	struct rq *rq;
	int cpu;

need_resched:
	preempt_disable();
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
	rcu_qsctr_inc(cpu);
	prev = rq->curr;
	switch_count = &prev->nivcsw;

	release_kernel_lock(prev);
need_resched_nonpreemptible:

	schedule_debug(prev);
L
Linus Torvalds 已提交
3453

3454 3455 3456 3457
	/*
	 * Do the rq-clock update outside the rq lock:
	 */
	local_irq_disable();
I
Ingo Molnar 已提交
3458
	__update_rq_clock(rq);
3459 3460
	spin_lock(&rq->lock);
	clear_tsk_need_resched(prev);
L
Linus Torvalds 已提交
3461 3462 3463

	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
		if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
I
Ingo Molnar 已提交
3464
				unlikely(signal_pending(prev)))) {
L
Linus Torvalds 已提交
3465
			prev->state = TASK_RUNNING;
I
Ingo Molnar 已提交
3466
		} else {
3467
			deactivate_task(rq, prev, 1);
L
Linus Torvalds 已提交
3468
		}
I
Ingo Molnar 已提交
3469
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
3470 3471
	}

I
Ingo Molnar 已提交
3472
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
3473 3474
		idle_balance(cpu, rq);

3475
	prev->sched_class->put_prev_task(rq, prev);
3476
	next = pick_next_task(rq, prev);
L
Linus Torvalds 已提交
3477 3478

	sched_info_switch(prev, next);
I
Ingo Molnar 已提交
3479

L
Linus Torvalds 已提交
3480 3481 3482 3483 3484
	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
3485
		context_switch(rq, prev, next); /* unlocks the rq */
L
Linus Torvalds 已提交
3486 3487 3488
	} else
		spin_unlock_irq(&rq->lock);

I
Ingo Molnar 已提交
3489 3490 3491
	if (unlikely(reacquire_kernel_lock(current) < 0)) {
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
3492
		goto need_resched_nonpreemptible;
I
Ingo Molnar 已提交
3493
	}
L
Linus Torvalds 已提交
3494 3495 3496 3497 3498 3499 3500 3501
	preempt_enable_no_resched();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(schedule);

#ifdef CONFIG_PREEMPT
/*
3502
 * this is the entry point to schedule() from in-kernel preemption
L
Linus Torvalds 已提交
3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516
 * off of preempt_enable.  Kernel preemptions off return from interrupt
 * occur there and call schedule directly.
 */
asmlinkage void __sched preempt_schedule(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
	 * we do not want to preempt the current task.  Just return..
	 */
N
Nick Piggin 已提交
3517
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544
		return;

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	schedule();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(preempt_schedule);

/*
3545
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
3557
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586
	BUG_ON(ti->preempt_count || !irqs_disabled());

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	local_irq_enable();
	schedule();
	local_irq_disable();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}

#endif /* CONFIG_PREEMPT */

I
Ingo Molnar 已提交
3587 3588
int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
			  void *key)
L
Linus Torvalds 已提交
3589
{
3590
	return try_to_wake_up(curr->private, mode, sync);
L
Linus Torvalds 已提交
3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605
}
EXPORT_SYMBOL(default_wake_function);

/*
 * The core wakeup function.  Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up.  If it's an exclusive wakeup (nr_exclusive == small +ve
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
 * started to run but is not in state TASK_RUNNING.  try_to_wake_up() returns
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
			     int nr_exclusive, int sync, void *key)
{
3606
	wait_queue_t *curr, *next;
L
Linus Torvalds 已提交
3607

3608
	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
3609 3610
		unsigned flags = curr->flags;

L
Linus Torvalds 已提交
3611
		if (curr->func(curr, mode, sync, key) &&
3612
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
3613 3614 3615 3616 3617 3618 3619 3620 3621
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
3622
 * @key: is directly passed to the wakeup function
L
Linus Torvalds 已提交
3623 3624
 */
void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
3625
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
void fastcall __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
{
	__wake_up_common(q, mode, 1, 0, NULL);
}

/**
3644
 * __wake_up_sync - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
 */
I
Ingo Molnar 已提交
3656 3657
void fastcall
__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
L
Linus Torvalds 已提交
3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700
{
	unsigned long flags;
	int sync = 1;

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
		sync = 0;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, sync, NULL);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

void fastcall complete(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 1, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

void fastcall complete_all(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 0, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

void fastcall __sched wait_for_completion(struct completion *x)
{
	might_sleep();
3701

L
Linus Torvalds 已提交
3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819
	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
	spin_unlock_irq(&x->wait.lock);
}
EXPORT_SYMBOL(wait_for_completion);

unsigned long fastcall __sched
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_timeout);

int fastcall __sched wait_for_completion_interruptible(struct completion *x)
{
	int ret = 0;

	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				ret = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);

	return ret;
}
EXPORT_SYMBOL(wait_for_completion_interruptible);

unsigned long fastcall __sched
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				timeout = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);

I
Ingo Molnar 已提交
3820 3821 3822 3823 3824
static inline void
sleep_on_head(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irqsave(&q->lock, *flags);
	__add_wait_queue(q, wait);
L
Linus Torvalds 已提交
3825
	spin_unlock(&q->lock);
I
Ingo Molnar 已提交
3826
}
L
Linus Torvalds 已提交
3827

I
Ingo Molnar 已提交
3828 3829 3830 3831 3832 3833 3834
static inline void
sleep_on_tail(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, wait);
	spin_unlock_irqrestore(&q->lock, *flags);
}
L
Linus Torvalds 已提交
3835

I
Ingo Molnar 已提交
3836
void __sched interruptible_sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3837
{
I
Ingo Molnar 已提交
3838 3839 3840 3841
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3842 3843 3844

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3845
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3846
	schedule();
I
Ingo Molnar 已提交
3847
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3848 3849 3850
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
3851
long __sched
I
Ingo Molnar 已提交
3852
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3853
{
I
Ingo Molnar 已提交
3854 3855 3856 3857
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3858 3859 3860

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3861
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3862
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3863
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3864 3865 3866 3867 3868

	return timeout;
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
3869
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3870
{
I
Ingo Molnar 已提交
3871 3872 3873 3874
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3875 3876 3877

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3878
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3879
	schedule();
I
Ingo Molnar 已提交
3880
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3881 3882 3883
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
3884
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3885
{
I
Ingo Molnar 已提交
3886 3887 3888 3889
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3890 3891 3892

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3893
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3894
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3895
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3896 3897 3898 3899 3900

	return timeout;
}
EXPORT_SYMBOL(sleep_on_timeout);

3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
3913
void rt_mutex_setprio(struct task_struct *p, int prio)
3914 3915
{
	unsigned long flags;
3916
	int oldprio, on_rq, running;
3917
	struct rq *rq;
3918 3919 3920 3921

	BUG_ON(prio < 0 || prio > MAX_PRIO);

	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3922
	update_rq_clock(rq);
3923

3924
	oldprio = p->prio;
I
Ingo Molnar 已提交
3925
	on_rq = p->se.on_rq;
3926 3927
	running = task_running(rq, p);
	if (on_rq) {
3928
		dequeue_task(rq, p, 0);
3929 3930 3931
		if (running)
			p->sched_class->put_prev_task(rq, p);
	}
I
Ingo Molnar 已提交
3932 3933 3934 3935 3936 3937

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

3938 3939
	p->prio = prio;

I
Ingo Molnar 已提交
3940
	if (on_rq) {
3941 3942
		if (running)
			p->sched_class->set_curr_task(rq);
3943
		enqueue_task(rq, p, 0);
3944 3945
		/*
		 * Reschedule if we are currently running on this runqueue and
3946 3947
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
3948
		 */
3949
		if (running) {
3950 3951
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
3952 3953 3954
		} else {
			check_preempt_curr(rq, p);
		}
3955 3956 3957 3958 3959 3960
	}
	task_rq_unlock(rq, &flags);
}

#endif

3961
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
3962
{
I
Ingo Molnar 已提交
3963
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
3964
	unsigned long flags;
3965
	struct rq *rq;
L
Linus Torvalds 已提交
3966 3967 3968 3969 3970 3971 3972 3973

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3974
	update_rq_clock(rq);
L
Linus Torvalds 已提交
3975 3976 3977 3978
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
3979
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
3980
	 */
3981
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
3982 3983 3984
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
I
Ingo Molnar 已提交
3985 3986
	on_rq = p->se.on_rq;
	if (on_rq) {
3987
		dequeue_task(rq, p, 0);
3988
		dec_load(rq, p);
3989
	}
L
Linus Torvalds 已提交
3990 3991

	p->static_prio = NICE_TO_PRIO(nice);
3992
	set_load_weight(p);
3993 3994 3995
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
3996

I
Ingo Molnar 已提交
3997
	if (on_rq) {
3998
		enqueue_task(rq, p, 0);
3999
		inc_load(rq, p);
L
Linus Torvalds 已提交
4000
		/*
4001 4002
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
4003
		 */
4004
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
4005 4006 4007 4008 4009 4010 4011
			resched_task(rq->curr);
	}
out_unlock:
	task_rq_unlock(rq, &flags);
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
4012 4013 4014 4015 4016
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
4017
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
4018
{
4019 4020
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
4021

M
Matt Mackall 已提交
4022 4023 4024 4025
	return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
asmlinkage long sys_nice(int increment)
{
4037
	long nice, retval;
L
Linus Torvalds 已提交
4038 4039 4040 4041 4042 4043

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
4044 4045
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
4046 4047 4048 4049 4050 4051 4052 4053 4054
	if (increment > 40)
		increment = 40;

	nice = PRIO_TO_NICE(current->static_prio) + increment;
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
4055 4056 4057
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
4076
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
4077 4078 4079 4080 4081 4082 4083 4084
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
4085
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103
{
	return TASK_NICE(p);
}
EXPORT_SYMBOL_GPL(task_nice);

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
4104
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
4105 4106 4107 4108 4109 4110 4111 4112
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
A
Alexey Dobriyan 已提交
4113
static struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
4114 4115 4116 4117 4118
{
	return pid ? find_task_by_pid(pid) : current;
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
4119 4120
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
4121
{
I
Ingo Molnar 已提交
4122
	BUG_ON(p->se.on_rq);
4123

L
Linus Torvalds 已提交
4124
	p->policy = policy;
I
Ingo Molnar 已提交
4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136
	switch (p->policy) {
	case SCHED_NORMAL:
	case SCHED_BATCH:
	case SCHED_IDLE:
		p->sched_class = &fair_sched_class;
		break;
	case SCHED_FIFO:
	case SCHED_RR:
		p->sched_class = &rt_sched_class;
		break;
	}

L
Linus Torvalds 已提交
4137
	p->rt_priority = prio;
4138 4139 4140
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
4141
	set_load_weight(p);
L
Linus Torvalds 已提交
4142 4143 4144
}

/**
4145
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
L
Linus Torvalds 已提交
4146 4147 4148
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
4149
 *
4150
 * NOTE that the task may be already dead.
L
Linus Torvalds 已提交
4151
 */
I
Ingo Molnar 已提交
4152 4153
int sched_setscheduler(struct task_struct *p, int policy,
		       struct sched_param *param)
L
Linus Torvalds 已提交
4154
{
4155
	int retval, oldprio, oldpolicy = -1, on_rq, running;
L
Linus Torvalds 已提交
4156
	unsigned long flags;
4157
	struct rq *rq;
L
Linus Torvalds 已提交
4158

4159 4160
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
4161 4162 4163 4164 4165
recheck:
	/* double check policy once rq lock held */
	if (policy < 0)
		policy = oldpolicy = p->policy;
	else if (policy != SCHED_FIFO && policy != SCHED_RR &&
I
Ingo Molnar 已提交
4166 4167
			policy != SCHED_NORMAL && policy != SCHED_BATCH &&
			policy != SCHED_IDLE)
4168
		return -EINVAL;
L
Linus Torvalds 已提交
4169 4170
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
4171 4172
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
4173 4174
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
4175
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
4176
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
4177
		return -EINVAL;
4178
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
4179 4180
		return -EINVAL;

4181 4182 4183 4184
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
	if (!capable(CAP_SYS_NICE)) {
4185
		if (rt_policy(policy)) {
4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201
			unsigned long rlim_rtprio;

			if (!lock_task_sighand(p, &flags))
				return -ESRCH;
			rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
			unlock_task_sighand(p, &flags);

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
I
Ingo Molnar 已提交
4202 4203 4204 4205 4206 4207
		/*
		 * Like positive nice levels, dont allow tasks to
		 * move out of SCHED_IDLE either:
		 */
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
			return -EPERM;
4208

4209 4210 4211 4212 4213
		/* can't change other user's priorities */
		if ((current->euid != p->euid) &&
		    (current->euid != p->uid))
			return -EPERM;
	}
L
Linus Torvalds 已提交
4214 4215 4216 4217

	retval = security_task_setscheduler(p, policy, param);
	if (retval)
		return retval;
4218 4219 4220 4221 4222
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
	 */
	spin_lock_irqsave(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4223 4224 4225 4226
	/*
	 * To be able to change p->policy safely, the apropriate
	 * runqueue lock must be held.
	 */
4227
	rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
4228 4229 4230
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
4231 4232
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4233 4234
		goto recheck;
	}
I
Ingo Molnar 已提交
4235
	update_rq_clock(rq);
I
Ingo Molnar 已提交
4236
	on_rq = p->se.on_rq;
4237 4238
	running = task_running(rq, p);
	if (on_rq) {
4239
		deactivate_task(rq, p, 0);
4240 4241 4242
		if (running)
			p->sched_class->put_prev_task(rq, p);
	}
4243

L
Linus Torvalds 已提交
4244
	oldprio = p->prio;
I
Ingo Molnar 已提交
4245
	__setscheduler(rq, p, policy, param->sched_priority);
4246

I
Ingo Molnar 已提交
4247
	if (on_rq) {
4248 4249
		if (running)
			p->sched_class->set_curr_task(rq);
I
Ingo Molnar 已提交
4250
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
4251 4252
		/*
		 * Reschedule if we are currently running on this runqueue and
4253 4254
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
L
Linus Torvalds 已提交
4255
		 */
4256
		if (running) {
4257 4258
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
4259 4260 4261
		} else {
			check_preempt_curr(rq, p);
		}
L
Linus Torvalds 已提交
4262
	}
4263 4264 4265
	__task_rq_unlock(rq);
	spin_unlock_irqrestore(&p->pi_lock, flags);

4266 4267
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
4268 4269 4270 4271
	return 0;
}
EXPORT_SYMBOL_GPL(sched_setscheduler);

I
Ingo Molnar 已提交
4272 4273
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
4274 4275 4276
{
	struct sched_param lparam;
	struct task_struct *p;
4277
	int retval;
L
Linus Torvalds 已提交
4278 4279 4280 4281 4282

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
4283 4284 4285

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
4286
	p = find_process_by_pid(pid);
4287 4288 4289
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
4290

L
Linus Torvalds 已提交
4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
				       struct sched_param __user *param)
{
4303 4304 4305 4306
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
asmlinkage long sys_sched_getscheduler(pid_t pid)
{
4326
	struct task_struct *p;
L
Linus Torvalds 已提交
4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353
	int retval = -EINVAL;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
			retval = p->policy;
	}
	read_unlock(&tasklist_lock);

out_nounlock:
	return retval;
}

/**
 * sys_sched_getscheduler - get the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
{
	struct sched_param lp;
4354
	struct task_struct *p;
L
Linus Torvalds 已提交
4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388
	int retval = -EINVAL;

	if (!param || pid < 0)
		goto out_nounlock;

	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
	read_unlock(&tasklist_lock);

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

out_nounlock:
	return retval;

out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

long sched_setaffinity(pid_t pid, cpumask_t new_mask)
{
	cpumask_t cpus_allowed;
4389 4390
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
4391

4392
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4393 4394 4395 4396 4397
	read_lock(&tasklist_lock);

	p = find_process_by_pid(pid);
	if (!p) {
		read_unlock(&tasklist_lock);
4398
		mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414
		return -ESRCH;
	}

	/*
	 * It is not safe to call set_cpus_allowed with the
	 * tasklist_lock held.  We will bump the task_struct's
	 * usage count and then drop tasklist_lock.
	 */
	get_task_struct(p);
	read_unlock(&tasklist_lock);

	retval = -EPERM;
	if ((current->euid != p->euid) && (current->euid != p->uid) &&
			!capable(CAP_SYS_NICE))
		goto out_unlock;

4415 4416 4417 4418
	retval = security_task_setscheduler(p, 0, NULL);
	if (retval)
		goto out_unlock;

L
Linus Torvalds 已提交
4419 4420 4421 4422 4423 4424
	cpus_allowed = cpuset_cpus_allowed(p);
	cpus_and(new_mask, new_mask, cpus_allowed);
	retval = set_cpus_allowed(p, new_mask);

out_unlock:
	put_task_struct(p);
4425
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
			     cpumask_t *new_mask)
{
	if (len < sizeof(cpumask_t)) {
		memset(new_mask, 0, sizeof(cpumask_t));
	} else if (len > sizeof(cpumask_t)) {
		len = sizeof(cpumask_t);
	}
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	cpumask_t new_mask;
	int retval;

	retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask);
	if (retval)
		return retval;

	return sched_setaffinity(pid, new_mask);
}

/*
 * Represents all cpu's present in the system
 * In systems capable of hotplug, this map could dynamically grow
 * as new cpu's are detected in the system via any platform specific
 * method, such as ACPI for e.g.
 */

4466
cpumask_t cpu_present_map __read_mostly;
L
Linus Torvalds 已提交
4467 4468 4469
EXPORT_SYMBOL(cpu_present_map);

#ifndef CONFIG_SMP
4470
cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
4471 4472
EXPORT_SYMBOL(cpu_online_map);

4473
cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
4474
EXPORT_SYMBOL(cpu_possible_map);
L
Linus Torvalds 已提交
4475 4476 4477 4478
#endif

long sched_getaffinity(pid_t pid, cpumask_t *mask)
{
4479
	struct task_struct *p;
L
Linus Torvalds 已提交
4480 4481
	int retval;

4482
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4483 4484 4485 4486 4487 4488 4489
	read_lock(&tasklist_lock);

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

4490 4491 4492 4493
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4494
	cpus_and(*mask, p->cpus_allowed, cpu_online_map);
L
Linus Torvalds 已提交
4495 4496 4497

out_unlock:
	read_unlock(&tasklist_lock);
4498
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4499

4500
	return retval;
L
Linus Torvalds 已提交
4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	int ret;
	cpumask_t mask;

	if (len < sizeof(cpumask_t))
		return -EINVAL;

	ret = sched_getaffinity(pid, &mask);
	if (ret < 0)
		return ret;

	if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t)))
		return -EFAULT;

	return sizeof(cpumask_t);
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
4531 4532
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
4533 4534 4535
 */
asmlinkage long sys_sched_yield(void)
{
4536
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
4537

4538
	schedstat_inc(rq, yld_count);
4539
	current->sched_class->yield_task(rq);
L
Linus Torvalds 已提交
4540 4541 4542 4543 4544 4545

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
4546
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4547 4548 4549 4550 4551 4552 4553 4554
	_raw_spin_unlock(&rq->lock);
	preempt_enable_no_resched();

	schedule();

	return 0;
}

A
Andrew Morton 已提交
4555
static void __cond_resched(void)
L
Linus Torvalds 已提交
4556
{
4557 4558 4559
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
	__might_sleep(__FILE__, __LINE__);
#endif
4560 4561 4562 4563 4564
	/*
	 * The BKS might be reacquired before we have dropped
	 * PREEMPT_ACTIVE, which could trigger a second
	 * cond_resched() call.
	 */
L
Linus Torvalds 已提交
4565 4566 4567 4568 4569 4570 4571 4572 4573
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
	} while (need_resched());
}

int __sched cond_resched(void)
{
4574 4575
	if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
					system_state == SYSTEM_RUNNING) {
L
Linus Torvalds 已提交
4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590
		__cond_resched();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched);

/*
 * cond_resched_lock() - if a reschedule is pending, drop the given lock,
 * call schedule, and on return reacquire the lock.
 *
 * This works OK both with and without CONFIG_PREEMPT.  We do strange low-level
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
I
Ingo Molnar 已提交
4591
int cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
4592
{
J
Jan Kara 已提交
4593 4594
	int ret = 0;

L
Linus Torvalds 已提交
4595 4596 4597
	if (need_lockbreak(lock)) {
		spin_unlock(lock);
		cpu_relax();
J
Jan Kara 已提交
4598
		ret = 1;
L
Linus Torvalds 已提交
4599 4600
		spin_lock(lock);
	}
4601
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4602
		spin_release(&lock->dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4603 4604 4605
		_raw_spin_unlock(lock);
		preempt_enable_no_resched();
		__cond_resched();
J
Jan Kara 已提交
4606
		ret = 1;
L
Linus Torvalds 已提交
4607 4608
		spin_lock(lock);
	}
J
Jan Kara 已提交
4609
	return ret;
L
Linus Torvalds 已提交
4610 4611 4612 4613 4614 4615 4616
}
EXPORT_SYMBOL(cond_resched_lock);

int __sched cond_resched_softirq(void)
{
	BUG_ON(!in_softirq());

4617
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4618
		local_bh_enable();
L
Linus Torvalds 已提交
4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched_softirq);

/**
 * yield - yield the current processor to other threads.
 *
4630
 * This is a shortcut for kernel-space yielding - it marks the
L
Linus Torvalds 已提交
4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648
 * thread runnable and calls sys_sched_yield().
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

/*
 * This task is about to go to sleep on IO.  Increment rq->nr_iowait so
 * that process accounting knows that this is a task in IO wait state.
 *
 * But don't do that if it is a deliberate, throttling IO wait (this task
 * has set its backing_dev_info: the queue against which it should throttle)
 */
void __sched io_schedule(void)
{
4649
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4650

4651
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4652 4653 4654
	atomic_inc(&rq->nr_iowait);
	schedule();
	atomic_dec(&rq->nr_iowait);
4655
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4656 4657 4658 4659 4660
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
4661
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4662 4663
	long ret;

4664
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4665 4666 4667
	atomic_inc(&rq->nr_iowait);
	ret = schedule_timeout(timeout);
	atomic_dec(&rq->nr_iowait);
4668
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_max(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
4689
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4690
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_min(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
4714
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4715
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
asmlinkage
long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
{
4732
	struct task_struct *p;
D
Dmitry Adamushko 已提交
4733
	unsigned int time_slice;
L
Linus Torvalds 已提交
4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749
	int retval = -EINVAL;
	struct timespec t;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

D
Dmitry Adamushko 已提交
4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762
	if (p->policy == SCHED_FIFO)
		time_slice = 0;
	else if (p->policy == SCHED_RR)
		time_slice = DEF_TIMESLICE;
	else {
		struct sched_entity *se = &p->se;
		unsigned long flags;
		struct rq *rq;

		rq = task_rq_lock(p, &flags);
		time_slice = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se));
		task_rq_unlock(rq, &flags);
	}
L
Linus Torvalds 已提交
4763
	read_unlock(&tasklist_lock);
D
Dmitry Adamushko 已提交
4764
	jiffies_to_timespec(time_slice, &t);
L
Linus Torvalds 已提交
4765 4766 4767 4768 4769 4770 4771 4772
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
out_nounlock:
	return retval;
out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

4773
static const char stat_nam[] = "RSDTtZX";
4774 4775

static void show_task(struct task_struct *p)
L
Linus Torvalds 已提交
4776 4777
{
	unsigned long free = 0;
4778
	unsigned state;
L
Linus Torvalds 已提交
4779 4780

	state = p->state ? __ffs(p->state) + 1 : 0;
4781 4782
	printk("%-13.13s %c", p->comm,
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4783
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
4784
	if (state == TASK_RUNNING)
4785
		printk(" running  ");
L
Linus Torvalds 已提交
4786
	else
4787
		printk(" %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4788 4789
#else
	if (state == TASK_RUNNING)
4790
		printk("  running task    ");
L
Linus Torvalds 已提交
4791 4792 4793 4794 4795
	else
		printk(" %016lx ", thread_saved_pc(p));
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
	{
4796
		unsigned long *n = end_of_stack(p);
L
Linus Torvalds 已提交
4797 4798
		while (!*n)
			n++;
4799
		free = (unsigned long)n - (unsigned long)end_of_stack(p);
L
Linus Torvalds 已提交
4800 4801
	}
#endif
4802
	printk("%5lu %5d %6d\n", free, p->pid, p->parent->pid);
L
Linus Torvalds 已提交
4803 4804 4805 4806 4807

	if (state != TASK_RUNNING)
		show_stack(p, NULL);
}

I
Ingo Molnar 已提交
4808
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
4809
{
4810
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
4811

4812 4813 4814
#if BITS_PER_LONG == 32
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
4815
#else
4816 4817
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
4818 4819 4820 4821 4822 4823 4824 4825
#endif
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
		 * console might take alot of time:
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
4826
		if (!state_filter || (p->state & state_filter))
I
Ingo Molnar 已提交
4827
			show_task(p);
L
Linus Torvalds 已提交
4828 4829
	} while_each_thread(g, p);

4830 4831
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
4832 4833 4834
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
L
Linus Torvalds 已提交
4835
	read_unlock(&tasklist_lock);
I
Ingo Molnar 已提交
4836 4837 4838 4839 4840
	/*
	 * Only show locks if all tasks are dumped:
	 */
	if (state_filter == -1)
		debug_show_all_locks();
L
Linus Torvalds 已提交
4841 4842
}

I
Ingo Molnar 已提交
4843 4844
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
4845
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
4846 4847
}

4848 4849 4850 4851 4852 4853 4854 4855
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
4856
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
4857
{
4858
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4859 4860
	unsigned long flags;

I
Ingo Molnar 已提交
4861 4862 4863
	__sched_fork(idle);
	idle->se.exec_start = sched_clock();

4864
	idle->prio = idle->normal_prio = MAX_PRIO;
L
Linus Torvalds 已提交
4865
	idle->cpus_allowed = cpumask_of_cpu(cpu);
I
Ingo Molnar 已提交
4866
	__set_task_cpu(idle, cpu);
L
Linus Torvalds 已提交
4867 4868 4869

	spin_lock_irqsave(&rq->lock, flags);
	rq->curr = rq->idle = idle;
4870 4871 4872
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
	idle->oncpu = 1;
#endif
L
Linus Torvalds 已提交
4873 4874 4875 4876
	spin_unlock_irqrestore(&rq->lock, flags);

	/* Set the preempt count _outside_ the spinlocks! */
#if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_BKL)
A
Al Viro 已提交
4877
	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
L
Linus Torvalds 已提交
4878
#else
A
Al Viro 已提交
4879
	task_thread_info(idle)->preempt_count = 0;
L
Linus Torvalds 已提交
4880
#endif
I
Ingo Molnar 已提交
4881 4882 4883 4884
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899
}

/*
 * In a system that switches off the HZ timer nohz_cpu_mask
 * indicates which cpus entered this state. This is used
 * in the rcu update to wait only for active cpus. For system
 * which do not switch off the HZ timer nohz_cpu_mask should
 * always be CPU_MASK_NONE.
 */
cpumask_t nohz_cpu_mask = CPU_MASK_NONE;

#ifdef CONFIG_SMP
/*
 * This is how migration works:
 *
4900
 * 1) we queue a struct migration_req structure in the source CPU's
L
Linus Torvalds 已提交
4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921
 *    runqueue and wake up that CPU's migration thread.
 * 2) we down() the locked semaphore => thread blocks.
 * 3) migration thread wakes up (implicitly it forces the migrated
 *    thread off the CPU)
 * 4) it gets the migration request and checks whether the migrated
 *    task is still in the wrong runqueue.
 * 5) if it's in the wrong runqueue then the migration thread removes
 *    it and puts it into the right queue.
 * 6) migration thread up()s the semaphore.
 * 7) we wake up and the migration is done.
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
 * task must not exit() & deallocate itself prematurely.  The
 * call is not atomic; no spinlocks may be held.
 */
4922
int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
L
Linus Torvalds 已提交
4923
{
4924
	struct migration_req req;
L
Linus Torvalds 已提交
4925
	unsigned long flags;
4926
	struct rq *rq;
4927
	int ret = 0;
L
Linus Torvalds 已提交
4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949

	rq = task_rq_lock(p, &flags);
	if (!cpus_intersects(new_mask, cpu_online_map)) {
		ret = -EINVAL;
		goto out;
	}

	p->cpus_allowed = new_mask;
	/* Can the task run on the task's current CPU? If so, we're done */
	if (cpu_isset(task_cpu(p), new_mask))
		goto out;

	if (migrate_task(p, any_online_cpu(new_mask), &req)) {
		/* Need help from migration thread: drop lock and wait. */
		task_rq_unlock(rq, &flags);
		wake_up_process(rq->migration_thread);
		wait_for_completion(&req.done);
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
	task_rq_unlock(rq, &flags);
4950

L
Linus Torvalds 已提交
4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962
	return ret;
}
EXPORT_SYMBOL_GPL(set_cpus_allowed);

/*
 * Move (not current) task off this cpu, onto dest cpu.  We're doing
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
4963 4964
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
4965
 */
4966
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
4967
{
4968
	struct rq *rq_dest, *rq_src;
I
Ingo Molnar 已提交
4969
	int ret = 0, on_rq;
L
Linus Torvalds 已提交
4970 4971

	if (unlikely(cpu_is_offline(dest_cpu)))
4972
		return ret;
L
Linus Torvalds 已提交
4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
		goto out;
	/* Affinity changed (again). */
	if (!cpu_isset(dest_cpu, p->cpus_allowed))
		goto out;

I
Ingo Molnar 已提交
4985
	on_rq = p->se.on_rq;
4986
	if (on_rq)
4987
		deactivate_task(rq_src, p, 0);
4988

L
Linus Torvalds 已提交
4989
	set_task_cpu(p, dest_cpu);
I
Ingo Molnar 已提交
4990 4991 4992
	if (on_rq) {
		activate_task(rq_dest, p, 0);
		check_preempt_curr(rq_dest, p);
L
Linus Torvalds 已提交
4993
	}
4994
	ret = 1;
L
Linus Torvalds 已提交
4995 4996
out:
	double_rq_unlock(rq_src, rq_dest);
4997
	return ret;
L
Linus Torvalds 已提交
4998 4999 5000 5001 5002 5003 5004
}

/*
 * migration_thread - this is a highprio system thread that performs
 * thread migration by bumping thread off CPU then 'pushing' onto
 * another runqueue.
 */
I
Ingo Molnar 已提交
5005
static int migration_thread(void *data)
L
Linus Torvalds 已提交
5006 5007
{
	int cpu = (long)data;
5008
	struct rq *rq;
L
Linus Torvalds 已提交
5009 5010 5011 5012 5013 5014

	rq = cpu_rq(cpu);
	BUG_ON(rq->migration_thread != current);

	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
5015
		struct migration_req *req;
L
Linus Torvalds 已提交
5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037
		struct list_head *head;

		spin_lock_irq(&rq->lock);

		if (cpu_is_offline(cpu)) {
			spin_unlock_irq(&rq->lock);
			goto wait_to_die;
		}

		if (rq->active_balance) {
			active_load_balance(rq, cpu);
			rq->active_balance = 0;
		}

		head = &rq->migration_queue;

		if (list_empty(head)) {
			spin_unlock_irq(&rq->lock);
			schedule();
			set_current_state(TASK_INTERRUPTIBLE);
			continue;
		}
5038
		req = list_entry(head->next, struct migration_req, list);
L
Linus Torvalds 已提交
5039 5040
		list_del_init(head->next);

N
Nick Piggin 已提交
5041 5042 5043
		spin_unlock(&rq->lock);
		__migrate_task(req->task, cpu, req->dest_cpu);
		local_irq_enable();
L
Linus Torvalds 已提交
5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061

		complete(&req->done);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
5062 5063 5064 5065
/*
 * Figure out where task on dead CPU should go, use force if neccessary.
 * NOTE: interrupts should be disabled by the caller
 */
5066
static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5067
{
5068
	unsigned long flags;
L
Linus Torvalds 已提交
5069
	cpumask_t mask;
5070 5071
	struct rq *rq;
	int dest_cpu;
L
Linus Torvalds 已提交
5072

5073
restart:
L
Linus Torvalds 已提交
5074 5075
	/* On same node? */
	mask = node_to_cpumask(cpu_to_node(dead_cpu));
5076
	cpus_and(mask, mask, p->cpus_allowed);
L
Linus Torvalds 已提交
5077 5078 5079 5080
	dest_cpu = any_online_cpu(mask);

	/* On any allowed CPU? */
	if (dest_cpu == NR_CPUS)
5081
		dest_cpu = any_online_cpu(p->cpus_allowed);
L
Linus Torvalds 已提交
5082 5083 5084

	/* No more Mr. Nice Guy. */
	if (dest_cpu == NR_CPUS) {
5085 5086 5087
		rq = task_rq_lock(p, &flags);
		cpus_setall(p->cpus_allowed);
		dest_cpu = any_online_cpu(p->cpus_allowed);
5088
		task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
5089 5090 5091 5092 5093 5094

		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
5095
		if (p->mm && printk_ratelimit())
L
Linus Torvalds 已提交
5096 5097
			printk(KERN_INFO "process %d (%s) no "
			       "longer affine to cpu%d\n",
5098
			       p->pid, p->comm, dead_cpu);
L
Linus Torvalds 已提交
5099
	}
5100
	if (!__migrate_task(p, dead_cpu, dest_cpu))
5101
		goto restart;
L
Linus Torvalds 已提交
5102 5103 5104 5105 5106 5107 5108 5109 5110
}

/*
 * While a dead CPU has no uninterruptible tasks queued at this point,
 * it might still have a nonzero ->nr_uninterruptible counter, because
 * for performance reasons the counter is not stricly tracking tasks to
 * their home CPUs. So we just add the counter to another CPU's counter,
 * to keep the global sum constant after CPU-down:
 */
5111
static void migrate_nr_uninterruptible(struct rq *rq_src)
L
Linus Torvalds 已提交
5112
{
5113
	struct rq *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL));
L
Linus Torvalds 已提交
5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126
	unsigned long flags;

	local_irq_save(flags);
	double_rq_lock(rq_src, rq_dest);
	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
	rq_src->nr_uninterruptible = 0;
	double_rq_unlock(rq_src, rq_dest);
	local_irq_restore(flags);
}

/* Run through task list and migrate tasks from the dead cpu. */
static void migrate_live_tasks(int src_cpu)
{
5127
	struct task_struct *p, *t;
L
Linus Torvalds 已提交
5128 5129 5130

	write_lock_irq(&tasklist_lock);

5131 5132
	do_each_thread(t, p) {
		if (p == current)
L
Linus Torvalds 已提交
5133 5134
			continue;

5135 5136 5137
		if (task_cpu(p) == src_cpu)
			move_task_off_dead_cpu(src_cpu, p);
	} while_each_thread(t, p);
L
Linus Torvalds 已提交
5138 5139 5140 5141

	write_unlock_irq(&tasklist_lock);
}

A
Alexey Dobriyan 已提交
5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155
/*
 * activate_idle_task - move idle task to the _front_ of runqueue.
 */
static void activate_idle_task(struct task_struct *p, struct rq *rq)
{
	update_rq_clock(rq);

	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;

	enqueue_task(rq, p, 0);
	inc_nr_running(p, rq);
}

I
Ingo Molnar 已提交
5156 5157
/*
 * Schedules idle task to be the next runnable task on current CPU.
L
Linus Torvalds 已提交
5158
 * It does so by boosting its priority to highest possible and adding it to
5159
 * the _front_ of the runqueue. Used by CPU offline code.
L
Linus Torvalds 已提交
5160 5161 5162
 */
void sched_idle_next(void)
{
5163
	int this_cpu = smp_processor_id();
5164
	struct rq *rq = cpu_rq(this_cpu);
L
Linus Torvalds 已提交
5165 5166 5167 5168
	struct task_struct *p = rq->idle;
	unsigned long flags;

	/* cpu has to be offline */
5169
	BUG_ON(cpu_online(this_cpu));
L
Linus Torvalds 已提交
5170

5171 5172 5173
	/*
	 * Strictly not necessary since rest of the CPUs are stopped by now
	 * and interrupts disabled on the current cpu.
L
Linus Torvalds 已提交
5174 5175 5176
	 */
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
5177
	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
5178 5179

	/* Add idle task to the _front_ of its priority queue: */
I
Ingo Molnar 已提交
5180
	activate_idle_task(p, rq);
L
Linus Torvalds 已提交
5181 5182 5183 5184

	spin_unlock_irqrestore(&rq->lock, flags);
}

5185 5186
/*
 * Ensures that the idle task is using init_mm right before its cpu goes
L
Linus Torvalds 已提交
5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199
 * offline.
 */
void idle_task_exit(void)
{
	struct mm_struct *mm = current->active_mm;

	BUG_ON(cpu_online(smp_processor_id()));

	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
}

5200
/* called under rq->lock with disabled interrupts */
5201
static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5202
{
5203
	struct rq *rq = cpu_rq(dead_cpu);
L
Linus Torvalds 已提交
5204 5205

	/* Must be exiting, otherwise would be on tasklist. */
5206
	BUG_ON(p->exit_state != EXIT_ZOMBIE && p->exit_state != EXIT_DEAD);
L
Linus Torvalds 已提交
5207 5208

	/* Cannot have done final schedule yet: would have vanished. */
5209
	BUG_ON(p->state == TASK_DEAD);
L
Linus Torvalds 已提交
5210

5211
	get_task_struct(p);
L
Linus Torvalds 已提交
5212 5213 5214 5215 5216

	/*
	 * Drop lock around migration; if someone else moves it,
	 * that's OK.  No task can be added to this CPU, so iteration is
	 * fine.
5217
	 * NOTE: interrupts should be left disabled  --dev@
L
Linus Torvalds 已提交
5218
	 */
5219
	spin_unlock(&rq->lock);
5220
	move_task_off_dead_cpu(dead_cpu, p);
5221
	spin_lock(&rq->lock);
L
Linus Torvalds 已提交
5222

5223
	put_task_struct(p);
L
Linus Torvalds 已提交
5224 5225 5226 5227 5228
}

/* release_task() removes task from tasklist, so we won't find dead tasks. */
static void migrate_dead_tasks(unsigned int dead_cpu)
{
5229
	struct rq *rq = cpu_rq(dead_cpu);
I
Ingo Molnar 已提交
5230
	struct task_struct *next;
5231

I
Ingo Molnar 已提交
5232 5233 5234
	for ( ; ; ) {
		if (!rq->nr_running)
			break;
I
Ingo Molnar 已提交
5235
		update_rq_clock(rq);
5236
		next = pick_next_task(rq, rq->curr);
I
Ingo Molnar 已提交
5237 5238 5239
		if (!next)
			break;
		migrate_dead(dead_cpu, next);
5240

L
Linus Torvalds 已提交
5241 5242 5243 5244
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

5245 5246 5247
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
5248 5249
	{
		.procname	= "sched_domain",
5250
		.mode		= 0555,
5251
	},
5252 5253 5254 5255
	{0,},
};

static struct ctl_table sd_ctl_root[] = {
5256
	{
5257
		.ctl_name	= CTL_KERN,
5258
		.procname	= "kernel",
5259
		.mode		= 0555,
5260 5261
		.child		= sd_ctl_dir,
	},
5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276
	{0,},
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
		kmalloc(n * sizeof(struct ctl_table), GFP_KERNEL);

	BUG_ON(!entry);
	memset(entry, 0, n * sizeof(struct ctl_table));

	return entry;
}

static void
5277
set_table_entry(struct ctl_table *entry,
5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290
		const char *procname, void *data, int maxlen,
		mode_t mode, proc_handler *proc_handler)
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
5291
	struct ctl_table *table = sd_alloc_ctl_entry(12);
5292

5293
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
5294
		sizeof(long), 0644, proc_doulongvec_minmax);
5295
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
5296
		sizeof(long), 0644, proc_doulongvec_minmax);
5297
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
5298
		sizeof(int), 0644, proc_dointvec_minmax);
5299
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
5300
		sizeof(int), 0644, proc_dointvec_minmax);
5301
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
5302
		sizeof(int), 0644, proc_dointvec_minmax);
5303
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
5304
		sizeof(int), 0644, proc_dointvec_minmax);
5305
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
5306
		sizeof(int), 0644, proc_dointvec_minmax);
5307
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
5308
		sizeof(int), 0644, proc_dointvec_minmax);
5309
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
5310
		sizeof(int), 0644, proc_dointvec_minmax);
5311
	set_table_entry(&table[9], "cache_nice_tries",
5312 5313
		&sd->cache_nice_tries,
		sizeof(int), 0644, proc_dointvec_minmax);
5314
	set_table_entry(&table[10], "flags", &sd->flags,
5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334
		sizeof(int), 0644, proc_dointvec_minmax);

	return table;
}

static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
5335
		entry->mode = 0555;
5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
static void init_sched_domain_sysctl(void)
{
	int i, cpu_num = num_online_cpus();
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

	sd_ctl_dir[0].child = entry;

	for (i = 0; i < cpu_num; i++, entry++) {
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
5355
		entry->mode = 0555;
5356 5357 5358 5359 5360 5361 5362 5363 5364 5365
		entry->child = sd_alloc_ctl_cpu_table(i);
	}
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
#else
static void init_sched_domain_sysctl(void)
{
}
#endif

L
Linus Torvalds 已提交
5366 5367 5368 5369
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
5370 5371
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
5372 5373
{
	struct task_struct *p;
5374
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
5375
	unsigned long flags;
5376
	struct rq *rq;
L
Linus Torvalds 已提交
5377 5378

	switch (action) {
5379 5380 5381 5382
	case CPU_LOCK_ACQUIRE:
		mutex_lock(&sched_hotcpu_mutex);
		break;

L
Linus Torvalds 已提交
5383
	case CPU_UP_PREPARE:
5384
	case CPU_UP_PREPARE_FROZEN:
I
Ingo Molnar 已提交
5385
		p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
L
Linus Torvalds 已提交
5386 5387 5388 5389 5390
		if (IS_ERR(p))
			return NOTIFY_BAD;
		kthread_bind(p, cpu);
		/* Must be high prio: stop_machine expects to yield to it. */
		rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
5391
		__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
L
Linus Torvalds 已提交
5392 5393 5394
		task_rq_unlock(rq, &flags);
		cpu_rq(cpu)->migration_thread = p;
		break;
5395

L
Linus Torvalds 已提交
5396
	case CPU_ONLINE:
5397
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
5398 5399 5400
		/* Strictly unneccessary, as first user will wake it. */
		wake_up_process(cpu_rq(cpu)->migration_thread);
		break;
5401

L
Linus Torvalds 已提交
5402 5403
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
5404
	case CPU_UP_CANCELED_FROZEN:
5405 5406
		if (!cpu_rq(cpu)->migration_thread)
			break;
L
Linus Torvalds 已提交
5407
		/* Unbind it from offline cpu so it can run.  Fall thru. */
5408 5409
		kthread_bind(cpu_rq(cpu)->migration_thread,
			     any_online_cpu(cpu_online_map));
L
Linus Torvalds 已提交
5410 5411 5412
		kthread_stop(cpu_rq(cpu)->migration_thread);
		cpu_rq(cpu)->migration_thread = NULL;
		break;
5413

L
Linus Torvalds 已提交
5414
	case CPU_DEAD:
5415
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
5416 5417 5418 5419 5420 5421
		migrate_live_tasks(cpu);
		rq = cpu_rq(cpu);
		kthread_stop(rq->migration_thread);
		rq->migration_thread = NULL;
		/* Idle task back to normal (off runqueue, low prio) */
		rq = task_rq_lock(rq->idle, &flags);
I
Ingo Molnar 已提交
5422
		update_rq_clock(rq);
5423
		deactivate_task(rq, rq->idle, 0);
L
Linus Torvalds 已提交
5424
		rq->idle->static_prio = MAX_PRIO;
I
Ingo Molnar 已提交
5425 5426
		__setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
		rq->idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
5427 5428 5429 5430 5431 5432
		migrate_dead_tasks(cpu);
		task_rq_unlock(rq, &flags);
		migrate_nr_uninterruptible(rq);
		BUG_ON(rq->nr_running != 0);

		/* No need to migrate the tasks: it was best-effort if
5433
		 * they didn't take sched_hotcpu_mutex.  Just wake up
L
Linus Torvalds 已提交
5434 5435 5436
		 * the requestors. */
		spin_lock_irq(&rq->lock);
		while (!list_empty(&rq->migration_queue)) {
5437 5438
			struct migration_req *req;

L
Linus Torvalds 已提交
5439
			req = list_entry(rq->migration_queue.next,
5440
					 struct migration_req, list);
L
Linus Torvalds 已提交
5441 5442 5443 5444 5445 5446
			list_del_init(&req->list);
			complete(&req->done);
		}
		spin_unlock_irq(&rq->lock);
		break;
#endif
5447 5448 5449
	case CPU_LOCK_RELEASE:
		mutex_unlock(&sched_hotcpu_mutex);
		break;
L
Linus Torvalds 已提交
5450 5451 5452 5453 5454 5455 5456
	}
	return NOTIFY_OK;
}

/* Register at highest priority so that task migration (migrate_all_tasks)
 * happens before everything else.
 */
5457
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
5458 5459 5460 5461 5462 5463 5464
	.notifier_call = migration_call,
	.priority = 10
};

int __init migration_init(void)
{
	void *cpu = (void *)(long)smp_processor_id();
5465
	int err;
5466 5467

	/* Start one for the boot CPU: */
5468 5469
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
5470 5471
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
5472

L
Linus Torvalds 已提交
5473 5474 5475 5476 5477
	return 0;
}
#endif

#ifdef CONFIG_SMP
5478 5479 5480 5481 5482

/* Number of possible processor ids */
int nr_cpu_ids __read_mostly = NR_CPUS;
EXPORT_SYMBOL(nr_cpu_ids);

5483
#ifdef CONFIG_SCHED_DEBUG
L
Linus Torvalds 已提交
5484 5485 5486 5487
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;

N
Nick Piggin 已提交
5488 5489 5490 5491 5492
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}

L
Linus Torvalds 已提交
5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	do {
		int i;
		char str[NR_CPUS];
		struct sched_group *group = sd->groups;
		cpumask_t groupmask;

		cpumask_scnprintf(str, NR_CPUS, sd->span);
		cpus_clear(groupmask);

		printk(KERN_DEBUG);
		for (i = 0; i < level + 1; i++)
			printk(" ");
		printk("domain %d: ", level);

		if (!(sd->flags & SD_LOAD_BALANCE)) {
			printk("does not load-balance\n");
			if (sd->parent)
5512 5513
				printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
						" has parent");
L
Linus Torvalds 已提交
5514 5515 5516 5517 5518 5519
			break;
		}

		printk("span %s\n", str);

		if (!cpu_isset(cpu, sd->span))
5520 5521
			printk(KERN_ERR "ERROR: domain->span does not contain "
					"CPU%d\n", cpu);
L
Linus Torvalds 已提交
5522
		if (!cpu_isset(cpu, group->cpumask))
5523 5524
			printk(KERN_ERR "ERROR: domain->groups does not contain"
					" CPU%d\n", cpu);
L
Linus Torvalds 已提交
5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536

		printk(KERN_DEBUG);
		for (i = 0; i < level + 2; i++)
			printk(" ");
		printk("groups:");
		do {
			if (!group) {
				printk("\n");
				printk(KERN_ERR "ERROR: group is NULL\n");
				break;
			}

5537
			if (!group->__cpu_power) {
L
Linus Torvalds 已提交
5538
				printk("\n");
5539 5540
				printk(KERN_ERR "ERROR: domain->cpu_power not "
						"set\n");
5541
				break;
L
Linus Torvalds 已提交
5542 5543 5544 5545 5546
			}

			if (!cpus_weight(group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: empty group\n");
5547
				break;
L
Linus Torvalds 已提交
5548 5549 5550 5551 5552
			}

			if (cpus_intersects(groupmask, group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: repeated CPUs\n");
5553
				break;
L
Linus Torvalds 已提交
5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565
			}

			cpus_or(groupmask, groupmask, group->cpumask);

			cpumask_scnprintf(str, NR_CPUS, group->cpumask);
			printk(" %s", str);

			group = group->next;
		} while (group != sd->groups);
		printk("\n");

		if (!cpus_equal(sd->span, groupmask))
5566 5567
			printk(KERN_ERR "ERROR: groups don't span "
					"domain->span\n");
L
Linus Torvalds 已提交
5568 5569 5570

		level++;
		sd = sd->parent;
5571 5572
		if (!sd)
			continue;
L
Linus Torvalds 已提交
5573

5574 5575 5576
		if (!cpus_subset(groupmask, sd->span))
			printk(KERN_ERR "ERROR: parent span is not a superset "
				"of domain->span\n");
L
Linus Torvalds 已提交
5577 5578 5579 5580

	} while (sd);
}
#else
5581
# define sched_domain_debug(sd, cpu) do { } while (0)
L
Linus Torvalds 已提交
5582 5583
#endif

5584
static int sd_degenerate(struct sched_domain *sd)
5585 5586 5587 5588 5589 5590 5591 5592
{
	if (cpus_weight(sd->span) == 1)
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
5593 5594 5595
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
	if (sd->flags & (SD_WAKE_IDLE |
			 SD_WAKE_AFFINE |
			 SD_WAKE_BALANCE))
		return 0;

	return 1;
}

5609 5610
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

	if (!cpus_equal(sd->span, parent->span))
		return 0;

	/* Does parent contain flags not in child? */
	/* WAKE_BALANCE is a subset of WAKE_AFFINE */
	if (cflags & SD_WAKE_AFFINE)
		pflags &= ~SD_WAKE_BALANCE;
	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
5629 5630 5631
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
5632 5633 5634 5635 5636 5637 5638
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

L
Linus Torvalds 已提交
5639 5640 5641 5642
/*
 * Attach the domain 'sd' to 'cpu' as its base domain.  Callers must
 * hold the hotplug lock.
 */
5643
static void cpu_attach_domain(struct sched_domain *sd, int cpu)
L
Linus Torvalds 已提交
5644
{
5645
	struct rq *rq = cpu_rq(cpu);
5646 5647 5648 5649 5650 5651 5652
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
	for (tmp = sd; tmp; tmp = tmp->parent) {
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
5653
		if (sd_parent_degenerate(tmp, parent)) {
5654
			tmp->parent = parent->parent;
5655 5656 5657
			if (parent->parent)
				parent->parent->child = tmp;
		}
5658 5659
	}

5660
	if (sd && sd_degenerate(sd)) {
5661
		sd = sd->parent;
5662 5663 5664
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
5665 5666 5667

	sched_domain_debug(sd, cpu);

N
Nick Piggin 已提交
5668
	rcu_assign_pointer(rq->sd, sd);
L
Linus Torvalds 已提交
5669 5670 5671
}

/* cpus with isolated domains */
5672
static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
L
Linus Torvalds 已提交
5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
	int ints[NR_CPUS], i;

	str = get_options(str, ARRAY_SIZE(ints), ints);
	cpus_clear(cpu_isolated_map);
	for (i = 1; i <= ints[0]; i++)
		if (ints[i] < NR_CPUS)
			cpu_set(ints[i], cpu_isolated_map);
	return 1;
}

I
Ingo Molnar 已提交
5687
__setup("isolcpus=", isolated_cpu_setup);
L
Linus Torvalds 已提交
5688 5689

/*
5690 5691 5692 5693
 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
 * to a function which identifies what group(along with sched group) a CPU
 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS
 * (due to the fact that we keep track of groups covered with a cpumask_t).
L
Linus Torvalds 已提交
5694 5695 5696 5697 5698
 *
 * init_sched_build_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
 */
5699
static void
5700 5701 5702
init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map,
			int (*group_fn)(int cpu, const cpumask_t *cpu_map,
					struct sched_group **sg))
L
Linus Torvalds 已提交
5703 5704 5705 5706 5707 5708
{
	struct sched_group *first = NULL, *last = NULL;
	cpumask_t covered = CPU_MASK_NONE;
	int i;

	for_each_cpu_mask(i, span) {
5709 5710
		struct sched_group *sg;
		int group = group_fn(i, cpu_map, &sg);
L
Linus Torvalds 已提交
5711 5712 5713 5714 5715 5716
		int j;

		if (cpu_isset(i, covered))
			continue;

		sg->cpumask = CPU_MASK_NONE;
5717
		sg->__cpu_power = 0;
L
Linus Torvalds 已提交
5718 5719

		for_each_cpu_mask(j, span) {
5720
			if (group_fn(j, cpu_map, NULL) != group)
L
Linus Torvalds 已提交
5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734
				continue;

			cpu_set(j, covered);
			cpu_set(j, sg->cpumask);
		}
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
}

5735
#define SD_NODES_PER_DOMAIN 16
L
Linus Torvalds 已提交
5736

5737
#ifdef CONFIG_NUMA
5738

5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790
/**
 * find_next_best_node - find the next node to include in a sched_domain
 * @node: node whose sched_domain we're building
 * @used_nodes: nodes already in the sched_domain
 *
 * Find the next node to include in a given scheduling domain.  Simply
 * finds the closest node not already in the @used_nodes map.
 *
 * Should use nodemask_t.
 */
static int find_next_best_node(int node, unsigned long *used_nodes)
{
	int i, n, val, min_val, best_node = 0;

	min_val = INT_MAX;

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Start at @node */
		n = (node + i) % MAX_NUMNODES;

		if (!nr_cpus_node(n))
			continue;

		/* Skip already used nodes */
		if (test_bit(n, used_nodes))
			continue;

		/* Simple min distance search */
		val = node_distance(node, n);

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

	set_bit(best_node, used_nodes);
	return best_node;
}

/**
 * sched_domain_node_span - get a cpumask for a node's sched_domain
 * @node: node whose cpumask we're constructing
 * @size: number of nodes to include in this span
 *
 * Given a node, construct a good cpumask for its sched_domain to span.  It
 * should be one that prevents unnecessary balancing, but also spreads tasks
 * out optimally.
 */
static cpumask_t sched_domain_node_span(int node)
{
	DECLARE_BITMAP(used_nodes, MAX_NUMNODES);
5791 5792
	cpumask_t span, nodemask;
	int i;
5793 5794 5795 5796 5797 5798 5799 5800 5801 5802

	cpus_clear(span);
	bitmap_zero(used_nodes, MAX_NUMNODES);

	nodemask = node_to_cpumask(node);
	cpus_or(span, span, nodemask);
	set_bit(node, used_nodes);

	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
		int next_node = find_next_best_node(node, used_nodes);
5803

5804 5805 5806 5807 5808 5809 5810 5811
		nodemask = node_to_cpumask(next_node);
		cpus_or(span, span, nodemask);
	}

	return span;
}
#endif

5812
int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
5813

5814
/*
5815
 * SMT sched-domains:
5816
 */
L
Linus Torvalds 已提交
5817 5818
#ifdef CONFIG_SCHED_SMT
static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
5819
static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
5820

5821 5822
static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map,
			    struct sched_group **sg)
L
Linus Torvalds 已提交
5823
{
5824 5825
	if (sg)
		*sg = &per_cpu(sched_group_cpus, cpu);
L
Linus Torvalds 已提交
5826 5827 5828 5829
	return cpu;
}
#endif

5830 5831 5832
/*
 * multi-core sched-domains:
 */
5833 5834
#ifdef CONFIG_SCHED_MC
static DEFINE_PER_CPU(struct sched_domain, core_domains);
5835
static DEFINE_PER_CPU(struct sched_group, sched_group_core);
5836 5837 5838
#endif

#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
5839 5840
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5841
{
5842
	int group;
5843 5844
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5845 5846 5847 5848
	group = first_cpu(mask);
	if (sg)
		*sg = &per_cpu(sched_group_core, group);
	return group;
5849 5850
}
#elif defined(CONFIG_SCHED_MC)
5851 5852
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5853
{
5854 5855
	if (sg)
		*sg = &per_cpu(sched_group_core, cpu);
5856 5857 5858 5859
	return cpu;
}
#endif

L
Linus Torvalds 已提交
5860
static DEFINE_PER_CPU(struct sched_domain, phys_domains);
5861
static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
5862

5863 5864
static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
L
Linus Torvalds 已提交
5865
{
5866
	int group;
5867
#ifdef CONFIG_SCHED_MC
5868
	cpumask_t mask = cpu_coregroup_map(cpu);
5869
	cpus_and(mask, mask, *cpu_map);
5870
	group = first_cpu(mask);
5871
#elif defined(CONFIG_SCHED_SMT)
5872 5873
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5874
	group = first_cpu(mask);
L
Linus Torvalds 已提交
5875
#else
5876
	group = cpu;
L
Linus Torvalds 已提交
5877
#endif
5878 5879 5880
	if (sg)
		*sg = &per_cpu(sched_group_phys, group);
	return group;
L
Linus Torvalds 已提交
5881 5882 5883 5884
}

#ifdef CONFIG_NUMA
/*
5885 5886 5887
 * The init_sched_build_groups can't handle what we want to do with node
 * groups, so roll our own. Now each node has its own list of groups which
 * gets dynamically allocated.
L
Linus Torvalds 已提交
5888
 */
5889
static DEFINE_PER_CPU(struct sched_domain, node_domains);
5890
static struct sched_group **sched_group_nodes_bycpu[NR_CPUS];
L
Linus Torvalds 已提交
5891

5892
static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
5893
static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
5894

5895 5896
static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
				 struct sched_group **sg)
5897
{
5898 5899 5900 5901 5902 5903 5904 5905 5906
	cpumask_t nodemask = node_to_cpumask(cpu_to_node(cpu));
	int group;

	cpus_and(nodemask, nodemask, *cpu_map);
	group = first_cpu(nodemask);

	if (sg)
		*sg = &per_cpu(sched_group_allnodes, group);
	return group;
L
Linus Torvalds 已提交
5907
}
5908

5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928
static void init_numa_sched_groups_power(struct sched_group *group_head)
{
	struct sched_group *sg = group_head;
	int j;

	if (!sg)
		return;
next_sg:
	for_each_cpu_mask(j, sg->cpumask) {
		struct sched_domain *sd;

		sd = &per_cpu(phys_domains, j);
		if (j != first_cpu(sd->groups->cpumask)) {
			/*
			 * Only add "power" once for each
			 * physical package.
			 */
			continue;
		}

5929
		sg_inc_cpu_power(sg, sd->groups->__cpu_power);
5930 5931 5932 5933 5934
	}
	sg = sg->next;
	if (sg != group_head)
		goto next_sg;
}
L
Linus Torvalds 已提交
5935 5936
#endif

5937
#ifdef CONFIG_NUMA
5938 5939 5940
/* Free memory allocated for various sched_group structures */
static void free_sched_groups(const cpumask_t *cpu_map)
{
5941
	int cpu, i;
5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971

	for_each_cpu_mask(cpu, *cpu_map) {
		struct sched_group **sched_group_nodes
			= sched_group_nodes_bycpu[cpu];

		if (!sched_group_nodes)
			continue;

		for (i = 0; i < MAX_NUMNODES; i++) {
			cpumask_t nodemask = node_to_cpumask(i);
			struct sched_group *oldsg, *sg = sched_group_nodes[i];

			cpus_and(nodemask, nodemask, *cpu_map);
			if (cpus_empty(nodemask))
				continue;

			if (sg == NULL)
				continue;
			sg = sg->next;
next_sg:
			oldsg = sg;
			sg = sg->next;
			kfree(oldsg);
			if (oldsg != sched_group_nodes[i])
				goto next_sg;
		}
		kfree(sched_group_nodes);
		sched_group_nodes_bycpu[cpu] = NULL;
	}
}
5972 5973 5974 5975 5976
#else
static void free_sched_groups(const cpumask_t *cpu_map)
{
}
#endif
5977

5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 *
 * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
 * the maximum number of tasks a group can handle in the presence of other idle
 * or lightly loaded groups in the same sched domain.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
	struct sched_domain *child;
	struct sched_group *group;

	WARN_ON(!sd || !sd->groups);

	if (cpu != first_cpu(sd->groups->cpumask))
		return;

	child = sd->child;

6004 6005
	sd->groups->__cpu_power = 0;

6006 6007 6008 6009 6010 6011 6012 6013 6014 6015
	/*
	 * For perf policy, if the groups in child domain share resources
	 * (for example cores sharing some portions of the cache hierarchy
	 * or SMT), then set this domain groups cpu_power such that each group
	 * can handle only one task, when there are other idle groups in the
	 * same sched domain.
	 */
	if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
		       (child->flags &
			(SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
6016
		sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
6017 6018 6019 6020 6021 6022 6023 6024
		return;
	}

	/*
	 * add cpu_power of each child group to this groups cpu_power
	 */
	group = child->groups;
	do {
6025
		sg_inc_cpu_power(sd->groups, group->__cpu_power);
6026 6027 6028 6029
		group = group->next;
	} while (group != child->groups);
}

L
Linus Torvalds 已提交
6030
/*
6031 6032
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
L
Linus Torvalds 已提交
6033
 */
6034
static int build_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6035 6036
{
	int i;
6037 6038
#ifdef CONFIG_NUMA
	struct sched_group **sched_group_nodes = NULL;
6039
	int sd_allnodes = 0;
6040 6041 6042 6043

	/*
	 * Allocate the per-node list of sched groups
	 */
I
Ingo Molnar 已提交
6044
	sched_group_nodes = kzalloc(sizeof(struct sched_group *)*MAX_NUMNODES,
6045
					   GFP_KERNEL);
6046 6047
	if (!sched_group_nodes) {
		printk(KERN_WARNING "Can not alloc sched group node list\n");
6048
		return -ENOMEM;
6049 6050 6051
	}
	sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
#endif
L
Linus Torvalds 已提交
6052 6053

	/*
6054
	 * Set up domains for cpus specified by the cpu_map.
L
Linus Torvalds 已提交
6055
	 */
6056
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6057 6058 6059
		struct sched_domain *sd = NULL, *p;
		cpumask_t nodemask = node_to_cpumask(cpu_to_node(i));

6060
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6061 6062

#ifdef CONFIG_NUMA
I
Ingo Molnar 已提交
6063 6064
		if (cpus_weight(*cpu_map) >
				SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) {
6065 6066 6067
			sd = &per_cpu(allnodes_domains, i);
			*sd = SD_ALLNODES_INIT;
			sd->span = *cpu_map;
6068
			cpu_to_allnodes_group(i, cpu_map, &sd->groups);
6069
			p = sd;
6070
			sd_allnodes = 1;
6071 6072 6073
		} else
			p = NULL;

L
Linus Torvalds 已提交
6074 6075
		sd = &per_cpu(node_domains, i);
		*sd = SD_NODE_INIT;
6076 6077
		sd->span = sched_domain_node_span(cpu_to_node(i));
		sd->parent = p;
6078 6079
		if (p)
			p->child = sd;
6080
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6081 6082 6083 6084 6085 6086 6087
#endif

		p = sd;
		sd = &per_cpu(phys_domains, i);
		*sd = SD_CPU_INIT;
		sd->span = nodemask;
		sd->parent = p;
6088 6089
		if (p)
			p->child = sd;
6090
		cpu_to_phys_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6091

6092 6093 6094 6095 6096 6097 6098
#ifdef CONFIG_SCHED_MC
		p = sd;
		sd = &per_cpu(core_domains, i);
		*sd = SD_MC_INIT;
		sd->span = cpu_coregroup_map(i);
		cpus_and(sd->span, sd->span, *cpu_map);
		sd->parent = p;
6099
		p->child = sd;
6100
		cpu_to_core_group(i, cpu_map, &sd->groups);
6101 6102
#endif

L
Linus Torvalds 已提交
6103 6104 6105 6106 6107
#ifdef CONFIG_SCHED_SMT
		p = sd;
		sd = &per_cpu(cpu_domains, i);
		*sd = SD_SIBLING_INIT;
		sd->span = cpu_sibling_map[i];
6108
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6109
		sd->parent = p;
6110
		p->child = sd;
6111
		cpu_to_cpu_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6112 6113 6114 6115 6116
#endif
	}

#ifdef CONFIG_SCHED_SMT
	/* Set up CPU (sibling) groups */
6117
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6118
		cpumask_t this_sibling_map = cpu_sibling_map[i];
6119
		cpus_and(this_sibling_map, this_sibling_map, *cpu_map);
L
Linus Torvalds 已提交
6120 6121 6122
		if (i != first_cpu(this_sibling_map))
			continue;

I
Ingo Molnar 已提交
6123 6124
		init_sched_build_groups(this_sibling_map, cpu_map,
					&cpu_to_cpu_group);
L
Linus Torvalds 已提交
6125 6126 6127
	}
#endif

6128 6129 6130 6131 6132 6133 6134
#ifdef CONFIG_SCHED_MC
	/* Set up multi-core groups */
	for_each_cpu_mask(i, *cpu_map) {
		cpumask_t this_core_map = cpu_coregroup_map(i);
		cpus_and(this_core_map, this_core_map, *cpu_map);
		if (i != first_cpu(this_core_map))
			continue;
I
Ingo Molnar 已提交
6135 6136
		init_sched_build_groups(this_core_map, cpu_map,
					&cpu_to_core_group);
6137 6138 6139
	}
#endif

L
Linus Torvalds 已提交
6140 6141 6142 6143
	/* Set up physical groups */
	for (i = 0; i < MAX_NUMNODES; i++) {
		cpumask_t nodemask = node_to_cpumask(i);

6144
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6145 6146 6147
		if (cpus_empty(nodemask))
			continue;

6148
		init_sched_build_groups(nodemask, cpu_map, &cpu_to_phys_group);
L
Linus Torvalds 已提交
6149 6150 6151 6152
	}

#ifdef CONFIG_NUMA
	/* Set up node groups */
6153
	if (sd_allnodes)
I
Ingo Molnar 已提交
6154 6155
		init_sched_build_groups(*cpu_map, cpu_map,
					&cpu_to_allnodes_group);
6156 6157 6158 6159 6160 6161 6162 6163 6164 6165

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Set up node groups */
		struct sched_group *sg, *prev;
		cpumask_t nodemask = node_to_cpumask(i);
		cpumask_t domainspan;
		cpumask_t covered = CPU_MASK_NONE;
		int j;

		cpus_and(nodemask, nodemask, *cpu_map);
6166 6167
		if (cpus_empty(nodemask)) {
			sched_group_nodes[i] = NULL;
6168
			continue;
6169
		}
6170 6171 6172 6173

		domainspan = sched_domain_node_span(i);
		cpus_and(domainspan, domainspan, *cpu_map);

6174
		sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);
6175 6176 6177 6178 6179
		if (!sg) {
			printk(KERN_WARNING "Can not alloc domain group for "
				"node %d\n", i);
			goto error;
		}
6180 6181 6182
		sched_group_nodes[i] = sg;
		for_each_cpu_mask(j, nodemask) {
			struct sched_domain *sd;
I
Ingo Molnar 已提交
6183

6184 6185 6186
			sd = &per_cpu(node_domains, j);
			sd->groups = sg;
		}
6187
		sg->__cpu_power = 0;
6188
		sg->cpumask = nodemask;
6189
		sg->next = sg;
6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207
		cpus_or(covered, covered, nodemask);
		prev = sg;

		for (j = 0; j < MAX_NUMNODES; j++) {
			cpumask_t tmp, notcovered;
			int n = (i + j) % MAX_NUMNODES;

			cpus_complement(notcovered, covered);
			cpus_and(tmp, notcovered, *cpu_map);
			cpus_and(tmp, tmp, domainspan);
			if (cpus_empty(tmp))
				break;

			nodemask = node_to_cpumask(n);
			cpus_and(tmp, tmp, nodemask);
			if (cpus_empty(tmp))
				continue;

6208 6209
			sg = kmalloc_node(sizeof(struct sched_group),
					  GFP_KERNEL, i);
6210 6211 6212
			if (!sg) {
				printk(KERN_WARNING
				"Can not alloc domain group for node %d\n", j);
6213
				goto error;
6214
			}
6215
			sg->__cpu_power = 0;
6216
			sg->cpumask = tmp;
6217
			sg->next = prev->next;
6218 6219 6220 6221 6222
			cpus_or(covered, covered, tmp);
			prev->next = sg;
			prev = sg;
		}
	}
L
Linus Torvalds 已提交
6223 6224 6225
#endif

	/* Calculate CPU power for physical packages and nodes */
6226
#ifdef CONFIG_SCHED_SMT
6227
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6228 6229
		struct sched_domain *sd = &per_cpu(cpu_domains, i);

6230
		init_sched_groups_power(i, sd);
6231
	}
L
Linus Torvalds 已提交
6232
#endif
6233
#ifdef CONFIG_SCHED_MC
6234
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6235 6236
		struct sched_domain *sd = &per_cpu(core_domains, i);

6237
		init_sched_groups_power(i, sd);
6238 6239
	}
#endif
6240

6241
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6242 6243
		struct sched_domain *sd = &per_cpu(phys_domains, i);

6244
		init_sched_groups_power(i, sd);
L
Linus Torvalds 已提交
6245 6246
	}

6247
#ifdef CONFIG_NUMA
6248 6249
	for (i = 0; i < MAX_NUMNODES; i++)
		init_numa_sched_groups_power(sched_group_nodes[i]);
6250

6251 6252
	if (sd_allnodes) {
		struct sched_group *sg;
6253

6254
		cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg);
6255 6256
		init_numa_sched_groups_power(sg);
	}
6257 6258
#endif

L
Linus Torvalds 已提交
6259
	/* Attach the domains */
6260
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6261 6262 6263
		struct sched_domain *sd;
#ifdef CONFIG_SCHED_SMT
		sd = &per_cpu(cpu_domains, i);
6264 6265
#elif defined(CONFIG_SCHED_MC)
		sd = &per_cpu(core_domains, i);
L
Linus Torvalds 已提交
6266 6267 6268 6269 6270
#else
		sd = &per_cpu(phys_domains, i);
#endif
		cpu_attach_domain(sd, i);
	}
6271 6272 6273

	return 0;

6274
#ifdef CONFIG_NUMA
6275 6276 6277
error:
	free_sched_groups(cpu_map);
	return -ENOMEM;
6278
#endif
L
Linus Torvalds 已提交
6279
}
6280 6281 6282
/*
 * Set up scheduler domains and groups.  Callers must hold the hotplug lock.
 */
6283
static int arch_init_sched_domains(const cpumask_t *cpu_map)
6284 6285
{
	cpumask_t cpu_default_map;
6286
	int err;
L
Linus Torvalds 已提交
6287

6288 6289 6290 6291 6292 6293 6294
	/*
	 * Setup mask for cpus without special case scheduling requirements.
	 * For now this just excludes isolated cpus, but could be used to
	 * exclude other special cases in the future.
	 */
	cpus_andnot(cpu_default_map, *cpu_map, cpu_isolated_map);

6295 6296 6297
	err = build_sched_domains(&cpu_default_map);

	return err;
6298 6299 6300
}

static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6301
{
6302
	free_sched_groups(cpu_map);
6303
}
L
Linus Torvalds 已提交
6304

6305 6306 6307 6308
/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
6309
static void detach_destroy_domains(const cpumask_t *cpu_map)
6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326
{
	int i;

	for_each_cpu_mask(i, *cpu_map)
		cpu_attach_domain(NULL, i);
	synchronize_sched();
	arch_destroy_sched_domains(cpu_map);
}

/*
 * Partition sched domains as specified by the cpumasks below.
 * This attaches all cpus from the cpumasks to the NULL domain,
 * waits for a RCU quiescent period, recalculates sched
 * domain information and then attaches them back to the
 * correct sched domains
 * Call with hotplug lock held
 */
6327
int partition_sched_domains(cpumask_t *partition1, cpumask_t *partition2)
6328 6329
{
	cpumask_t change_map;
6330
	int err = 0;
6331 6332 6333 6334 6335 6336 6337 6338

	cpus_and(*partition1, *partition1, cpu_online_map);
	cpus_and(*partition2, *partition2, cpu_online_map);
	cpus_or(change_map, *partition1, *partition2);

	/* Detach sched domains from all of the affected cpus */
	detach_destroy_domains(&change_map);
	if (!cpus_empty(*partition1))
6339 6340 6341 6342 6343
		err = build_sched_domains(partition1);
	if (!err && !cpus_empty(*partition2))
		err = build_sched_domains(partition2);

	return err;
6344 6345
}

6346
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
A
Adrian Bunk 已提交
6347
static int arch_reinit_sched_domains(void)
6348 6349 6350
{
	int err;

6351
	mutex_lock(&sched_hotcpu_mutex);
6352 6353
	detach_destroy_domains(&cpu_online_map);
	err = arch_init_sched_domains(&cpu_online_map);
6354
	mutex_unlock(&sched_hotcpu_mutex);
6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380

	return err;
}

static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
{
	int ret;

	if (buf[0] != '0' && buf[0] != '1')
		return -EINVAL;

	if (smt)
		sched_smt_power_savings = (buf[0] == '1');
	else
		sched_mc_power_savings = (buf[0] == '1');

	ret = arch_reinit_sched_domains();

	return ret ? ret : count;
}

#ifdef CONFIG_SCHED_MC
static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_mc_power_savings);
}
6381 6382
static ssize_t sched_mc_power_savings_store(struct sys_device *dev,
					    const char *buf, size_t count)
6383 6384 6385
{
	return sched_power_savings_store(buf, count, 0);
}
A
Adrian Bunk 已提交
6386 6387
static SYSDEV_ATTR(sched_mc_power_savings, 0644, sched_mc_power_savings_show,
		   sched_mc_power_savings_store);
6388 6389 6390 6391 6392 6393 6394
#endif

#ifdef CONFIG_SCHED_SMT
static ssize_t sched_smt_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_smt_power_savings);
}
6395 6396
static ssize_t sched_smt_power_savings_store(struct sys_device *dev,
					     const char *buf, size_t count)
6397 6398 6399
{
	return sched_power_savings_store(buf, count, 1);
}
A
Adrian Bunk 已提交
6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419
static SYSDEV_ATTR(sched_smt_power_savings, 0644, sched_smt_power_savings_show,
		   sched_smt_power_savings_store);
#endif

int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
{
	int err = 0;

#ifdef CONFIG_SCHED_SMT
	if (smt_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_smt_power_savings.attr);
#endif
#ifdef CONFIG_SCHED_MC
	if (!err && mc_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_mc_power_savings.attr);
#endif
	return err;
}
6420 6421
#endif

L
Linus Torvalds 已提交
6422 6423 6424
/*
 * Force a reinitialization of the sched domains hierarchy.  The domains
 * and groups cannot be updated in place without racing with the balancing
N
Nick Piggin 已提交
6425
 * code, so we temporarily attach all running cpus to the NULL domain
L
Linus Torvalds 已提交
6426 6427 6428 6429 6430 6431 6432
 * which will prevent rebalancing while the sched domains are recalculated.
 */
static int update_sched_domains(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
{
	switch (action) {
	case CPU_UP_PREPARE:
6433
	case CPU_UP_PREPARE_FROZEN:
L
Linus Torvalds 已提交
6434
	case CPU_DOWN_PREPARE:
6435
	case CPU_DOWN_PREPARE_FROZEN:
6436
		detach_destroy_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6437 6438 6439
		return NOTIFY_OK;

	case CPU_UP_CANCELED:
6440
	case CPU_UP_CANCELED_FROZEN:
L
Linus Torvalds 已提交
6441
	case CPU_DOWN_FAILED:
6442
	case CPU_DOWN_FAILED_FROZEN:
L
Linus Torvalds 已提交
6443
	case CPU_ONLINE:
6444
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
6445
	case CPU_DEAD:
6446
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
6447 6448 6449 6450 6451 6452 6453 6454 6455
		/*
		 * Fall through and re-initialise the domains.
		 */
		break;
	default:
		return NOTIFY_DONE;
	}

	/* The hotplug lock is already held by cpu_up/cpu_down */
6456
	arch_init_sched_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6457 6458 6459 6460 6461 6462

	return NOTIFY_OK;
}

void __init sched_init_smp(void)
{
6463 6464
	cpumask_t non_isolated_cpus;

6465
	mutex_lock(&sched_hotcpu_mutex);
6466
	arch_init_sched_domains(&cpu_online_map);
6467
	cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map);
6468 6469
	if (cpus_empty(non_isolated_cpus))
		cpu_set(smp_processor_id(), non_isolated_cpus);
6470
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
6471 6472
	/* XXX: Theoretical race here - CPU may be hotplugged now */
	hotcpu_notifier(update_sched_domains, 0);
6473

6474 6475
	init_sched_domain_sysctl();

6476 6477 6478
	/* Move init over to a non-isolated CPU */
	if (set_cpus_allowed(current, non_isolated_cpus) < 0)
		BUG();
L
Linus Torvalds 已提交
6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489
}
#else
void __init sched_init_smp(void)
{
}
#endif /* CONFIG_SMP */

int in_sched_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __sched functions */
	extern char __sched_text_start[], __sched_text_end[];
6490

L
Linus Torvalds 已提交
6491 6492 6493 6494 6495
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

A
Alexey Dobriyan 已提交
6496
static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
I
Ingo Molnar 已提交
6497 6498 6499 6500 6501
{
	cfs_rq->tasks_timeline = RB_ROOT;
#ifdef CONFIG_FAIR_GROUP_SCHED
	cfs_rq->rq = rq;
#endif
P
Peter Zijlstra 已提交
6502
	cfs_rq->min_vruntime = (u64)(-(1LL << 20));
I
Ingo Molnar 已提交
6503 6504
}

L
Linus Torvalds 已提交
6505 6506
void __init sched_init(void)
{
6507
	int highest_cpu = 0;
I
Ingo Molnar 已提交
6508 6509
	int i, j;

6510
	for_each_possible_cpu(i) {
I
Ingo Molnar 已提交
6511
		struct rt_prio_array *array;
6512
		struct rq *rq;
L
Linus Torvalds 已提交
6513 6514 6515

		rq = cpu_rq(i);
		spin_lock_init(&rq->lock);
6516
		lockdep_set_class(&rq->lock, &rq->rq_lock_key);
N
Nick Piggin 已提交
6517
		rq->nr_running = 0;
I
Ingo Molnar 已提交
6518 6519 6520 6521
		rq->clock = 1;
		init_cfs_rq(&rq->cfs, rq);
#ifdef CONFIG_FAIR_GROUP_SCHED
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
I
Ingo Molnar 已提交
6522 6523 6524 6525 6526 6527 6528
		{
			struct cfs_rq *cfs_rq = &per_cpu(init_cfs_rq, i);
			struct sched_entity *se =
					 &per_cpu(init_sched_entity, i);

			init_cfs_rq_p[i] = cfs_rq;
			init_cfs_rq(cfs_rq, rq);
6529
			cfs_rq->tg = &init_task_group;
I
Ingo Molnar 已提交
6530
			list_add(&cfs_rq->leaf_cfs_rq_list,
S
Srivatsa Vaddagiri 已提交
6531 6532
							 &rq->leaf_cfs_rq_list);

I
Ingo Molnar 已提交
6533 6534 6535
			init_sched_entity_p[i] = se;
			se->cfs_rq = &rq->cfs;
			se->my_q = cfs_rq;
6536
			se->load.weight = init_task_group_load;
6537
			se->load.inv_weight =
6538
				 div64_64(1ULL<<32, init_task_group_load);
I
Ingo Molnar 已提交
6539 6540
			se->parent = NULL;
		}
6541
		init_task_group.shares = init_task_group_load;
6542
		spin_lock_init(&init_task_group.lock);
I
Ingo Molnar 已提交
6543
#endif
L
Linus Torvalds 已提交
6544

I
Ingo Molnar 已提交
6545 6546
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
L
Linus Torvalds 已提交
6547
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
6548
		rq->sd = NULL;
L
Linus Torvalds 已提交
6549
		rq->active_balance = 0;
I
Ingo Molnar 已提交
6550
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
6551
		rq->push_cpu = 0;
6552
		rq->cpu = i;
L
Linus Torvalds 已提交
6553 6554 6555 6556 6557
		rq->migration_thread = NULL;
		INIT_LIST_HEAD(&rq->migration_queue);
#endif
		atomic_set(&rq->nr_iowait, 0);

I
Ingo Molnar 已提交
6558 6559 6560 6561
		array = &rq->rt.active;
		for (j = 0; j < MAX_RT_PRIO; j++) {
			INIT_LIST_HEAD(array->queue + j);
			__clear_bit(j, array->bitmap);
L
Linus Torvalds 已提交
6562
		}
6563
		highest_cpu = i;
I
Ingo Molnar 已提交
6564 6565
		/* delimiter for bitsearch: */
		__set_bit(MAX_RT_PRIO, array->bitmap);
L
Linus Torvalds 已提交
6566 6567
	}

6568
	set_load_weight(&init_task);
6569

6570 6571 6572 6573
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

6574
#ifdef CONFIG_SMP
6575
	nr_cpu_ids = highest_cpu + 1;
6576 6577 6578
	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
#endif

6579 6580 6581 6582
#ifdef CONFIG_RT_MUTEXES
	plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
#endif

L
Linus Torvalds 已提交
6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
I
Ingo Molnar 已提交
6596 6597 6598 6599
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
L
Linus Torvalds 已提交
6600 6601 6602 6603 6604
}

#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
void __might_sleep(char *file, int line)
{
6605
#ifdef in_atomic
L
Linus Torvalds 已提交
6606 6607 6608 6609 6610 6611 6612
	static unsigned long prev_jiffy;	/* ratelimiting */

	if ((in_atomic() || irqs_disabled()) &&
	    system_state == SYSTEM_RUNNING && !oops_in_progress) {
		if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
			return;
		prev_jiffy = jiffies;
6613
		printk(KERN_ERR "BUG: sleeping function called from invalid"
L
Linus Torvalds 已提交
6614 6615 6616
				" context at %s:%d\n", file, line);
		printk("in_atomic():%d, irqs_disabled():%d\n",
			in_atomic(), irqs_disabled());
6617
		debug_show_held_locks(current);
6618 6619
		if (irqs_disabled())
			print_irqtrace_events(current);
L
Linus Torvalds 已提交
6620 6621 6622 6623 6624 6625 6626 6627 6628 6629
		dump_stack();
	}
#endif
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
void normalize_rt_tasks(void)
{
6630
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
6631
	unsigned long flags;
6632
	struct rq *rq;
I
Ingo Molnar 已提交
6633
	int on_rq;
L
Linus Torvalds 已提交
6634 6635

	read_lock_irq(&tasklist_lock);
6636
	do_each_thread(g, p) {
I
Ingo Molnar 已提交
6637 6638
		p->se.exec_start		= 0;
#ifdef CONFIG_SCHEDSTATS
I
Ingo Molnar 已提交
6639 6640 6641
		p->se.wait_start		= 0;
		p->se.sleep_start		= 0;
		p->se.block_start		= 0;
I
Ingo Molnar 已提交
6642
#endif
I
Ingo Molnar 已提交
6643 6644 6645 6646 6647 6648 6649 6650 6651
		task_rq(p)->clock		= 0;

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
6652
			continue;
I
Ingo Molnar 已提交
6653
		}
L
Linus Torvalds 已提交
6654

6655 6656
		spin_lock_irqsave(&p->pi_lock, flags);
		rq = __task_rq_lock(p);
I
Ingo Molnar 已提交
6657 6658 6659 6660 6661 6662 6663
#ifdef CONFIG_SMP
		/*
		 * Do not touch the migration thread:
		 */
		if (p == rq->migration_thread)
			goto out_unlock;
#endif
L
Linus Torvalds 已提交
6664

I
Ingo Molnar 已提交
6665
		update_rq_clock(rq);
I
Ingo Molnar 已提交
6666
		on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
6667 6668
		if (on_rq)
			deactivate_task(rq, p, 0);
I
Ingo Molnar 已提交
6669 6670
		__setscheduler(rq, p, SCHED_NORMAL, 0);
		if (on_rq) {
I
Ingo Molnar 已提交
6671
			activate_task(rq, p, 0);
L
Linus Torvalds 已提交
6672 6673
			resched_task(rq->curr);
		}
I
Ingo Molnar 已提交
6674 6675 6676
#ifdef CONFIG_SMP
 out_unlock:
#endif
6677 6678
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
6679 6680
	} while_each_thread(g, p);

L
Linus Torvalds 已提交
6681 6682 6683 6684
	read_unlock_irq(&tasklist_lock);
}

#endif /* CONFIG_MAGIC_SYSRQ */
6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702

#ifdef CONFIG_IA64
/*
 * These functions are only useful for the IA64 MCA handling.
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6703
struct task_struct *curr_task(int cpu)
6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722
{
	return cpu_curr(cpu);
}

/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
 * are serviced on a separate stack.  It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner.  This function
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6723
void set_curr_task(int cpu, struct task_struct *p)
6724 6725 6726 6727 6728
{
	cpu_curr(cpu) = p;
}

#endif
S
Srivatsa Vaddagiri 已提交
6729 6730 6731 6732

#ifdef CONFIG_FAIR_GROUP_SCHED

/* allocate runqueue etc for a new task group */
6733
struct task_group *sched_create_group(void)
S
Srivatsa Vaddagiri 已提交
6734
{
6735
	struct task_group *tg;
S
Srivatsa Vaddagiri 已提交
6736 6737
	struct cfs_rq *cfs_rq;
	struct sched_entity *se;
6738
	struct rq *rq;
S
Srivatsa Vaddagiri 已提交
6739 6740 6741 6742 6743 6744
	int i;

	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
	if (!tg)
		return ERR_PTR(-ENOMEM);

6745
	tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
6746 6747
	if (!tg->cfs_rq)
		goto err;
6748
	tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL);
S
Srivatsa Vaddagiri 已提交
6749 6750 6751 6752
	if (!tg->se)
		goto err;

	for_each_possible_cpu(i) {
6753
		rq = cpu_rq(i);
S
Srivatsa Vaddagiri 已提交
6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779

		cfs_rq = kmalloc_node(sizeof(struct cfs_rq), GFP_KERNEL,
							 cpu_to_node(i));
		if (!cfs_rq)
			goto err;

		se = kmalloc_node(sizeof(struct sched_entity), GFP_KERNEL,
							cpu_to_node(i));
		if (!se)
			goto err;

		memset(cfs_rq, 0, sizeof(struct cfs_rq));
		memset(se, 0, sizeof(struct sched_entity));

		tg->cfs_rq[i] = cfs_rq;
		init_cfs_rq(cfs_rq, rq);
		cfs_rq->tg = tg;

		tg->se[i] = se;
		se->cfs_rq = &rq->cfs;
		se->my_q = cfs_rq;
		se->load.weight = NICE_0_LOAD;
		se->load.inv_weight = div64_64(1ULL<<32, NICE_0_LOAD);
		se->parent = NULL;
	}

6780 6781 6782 6783 6784
	for_each_possible_cpu(i) {
		rq = cpu_rq(i);
		cfs_rq = tg->cfs_rq[i];
		list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
	}
S
Srivatsa Vaddagiri 已提交
6785

6786
	tg->shares = NICE_0_LOAD;
6787
	spin_lock_init(&tg->lock);
S
Srivatsa Vaddagiri 已提交
6788

6789
	return tg;
S
Srivatsa Vaddagiri 已提交
6790 6791 6792

err:
	for_each_possible_cpu(i) {
I
Ingo Molnar 已提交
6793
		if (tg->cfs_rq)
S
Srivatsa Vaddagiri 已提交
6794
			kfree(tg->cfs_rq[i]);
I
Ingo Molnar 已提交
6795
		if (tg->se)
S
Srivatsa Vaddagiri 已提交
6796 6797
			kfree(tg->se[i]);
	}
I
Ingo Molnar 已提交
6798 6799 6800
	kfree(tg->cfs_rq);
	kfree(tg->se);
	kfree(tg);
S
Srivatsa Vaddagiri 已提交
6801 6802 6803 6804

	return ERR_PTR(-ENOMEM);
}

6805 6806
/* rcu callback to free various structures associated with a task group */
static void free_sched_group(struct rcu_head *rhp)
S
Srivatsa Vaddagiri 已提交
6807
{
6808
	struct cfs_rq *cfs_rq = container_of(rhp, struct cfs_rq, rcu);
6809
	struct task_group *tg = cfs_rq->tg;
S
Srivatsa Vaddagiri 已提交
6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826
	struct sched_entity *se;
	int i;

	/* now it should be safe to free those cfs_rqs */
	for_each_possible_cpu(i) {
		cfs_rq = tg->cfs_rq[i];
		kfree(cfs_rq);

		se = tg->se[i];
		kfree(se);
	}

	kfree(tg->cfs_rq);
	kfree(tg->se);
	kfree(tg);
}

6827
/* Destroy runqueue etc associated with a task group */
6828
void sched_destroy_group(struct task_group *tg)
S
Srivatsa Vaddagiri 已提交
6829
{
6830 6831
	struct cfs_rq *cfs_rq;
	int i;
S
Srivatsa Vaddagiri 已提交
6832

6833 6834 6835 6836 6837 6838 6839 6840 6841
	for_each_possible_cpu(i) {
		cfs_rq = tg->cfs_rq[i];
		list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
	}

	cfs_rq = tg->cfs_rq[0];

	/* wait for possible concurrent references to cfs_rqs complete */
	call_rcu(&cfs_rq->rcu, free_sched_group);
S
Srivatsa Vaddagiri 已提交
6842 6843
}

6844
/* change task's runqueue when it moves between groups.
I
Ingo Molnar 已提交
6845 6846 6847
 *	The caller of this function should have put the task in its new group
 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
 *	reflect its new group.
6848 6849
 */
void sched_move_task(struct task_struct *tsk)
S
Srivatsa Vaddagiri 已提交
6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864
{
	int on_rq, running;
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

	if (tsk->sched_class != &fair_sched_class)
		goto done;

	update_rq_clock(rq);

	running = task_running(rq, tsk);
	on_rq = tsk->se.on_rq;

6865
	if (on_rq) {
S
Srivatsa Vaddagiri 已提交
6866
		dequeue_task(rq, tsk, 0);
6867 6868 6869
		if (unlikely(running))
			tsk->sched_class->put_prev_task(rq, tsk);
	}
S
Srivatsa Vaddagiri 已提交
6870 6871 6872

	set_task_cfs_rq(tsk);

6873 6874 6875
	if (on_rq) {
		if (unlikely(running))
			tsk->sched_class->set_curr_task(rq);
6876
		enqueue_task(rq, tsk, 0);
6877
	}
S
Srivatsa Vaddagiri 已提交
6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903

done:
	task_rq_unlock(rq, &flags);
}

static void set_se_shares(struct sched_entity *se, unsigned long shares)
{
	struct cfs_rq *cfs_rq = se->cfs_rq;
	struct rq *rq = cfs_rq->rq;
	int on_rq;

	spin_lock_irq(&rq->lock);

	on_rq = se->on_rq;
	if (on_rq)
		dequeue_entity(cfs_rq, se, 0);

	se->load.weight = shares;
	se->load.inv_weight = div64_64((1ULL<<32), shares);

	if (on_rq)
		enqueue_entity(cfs_rq, se, 0);

	spin_unlock_irq(&rq->lock);
}

6904
int sched_group_set_shares(struct task_group *tg, unsigned long shares)
S
Srivatsa Vaddagiri 已提交
6905 6906 6907
{
	int i;

6908
	spin_lock(&tg->lock);
6909
	if (tg->shares == shares)
6910
		goto done;
S
Srivatsa Vaddagiri 已提交
6911

6912
	/* return -EINVAL if the new value is not sane */
S
Srivatsa Vaddagiri 已提交
6913

6914
	tg->shares = shares;
S
Srivatsa Vaddagiri 已提交
6915
	for_each_possible_cpu(i)
6916
		set_se_shares(tg->se[i], shares);
S
Srivatsa Vaddagiri 已提交
6917

6918 6919
done:
	spin_unlock(&tg->lock);
6920
	return 0;
S
Srivatsa Vaddagiri 已提交
6921 6922
}

6923 6924 6925 6926 6927
unsigned long sched_group_shares(struct task_group *tg)
{
	return tg->shares;
}

I
Ingo Molnar 已提交
6928
#endif	/* CONFIG_FAIR_GROUP_SCHED */