sched.c 164.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
L
Linus Torvalds 已提交
25 26 27 28 29 30
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
31
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
32 33 34 35
#include <linux/highmem.h>
#include <linux/smp_lock.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
36
#include <linux/capability.h>
L
Linus Torvalds 已提交
37 38
#include <linux/completion.h>
#include <linux/kernel_stat.h>
39
#include <linux/debug_locks.h>
L
Linus Torvalds 已提交
40 41 42
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
43
#include <linux/freezer.h>
44
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
45 46 47 48 49 50 51 52 53 54 55
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
#include <linux/seq_file.h>
56
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
57 58
#include <linux/syscalls.h>
#include <linux/times.h>
59
#include <linux/tsacct_kern.h>
60
#include <linux/kprobes.h>
61
#include <linux/delayacct.h>
62
#include <linux/reciprocal_div.h>
63
#include <linux/unistd.h>
L
Linus Torvalds 已提交
64

65
#include <asm/tlb.h>
L
Linus Torvalds 已提交
66

67 68 69 70 71 72 73 74 75 76
/*
 * Scheduler clock - returns current time in nanosec units.
 * This is default implementation.
 * Architectures and sub-architectures can override this.
 */
unsigned long long __attribute__((weak)) sched_clock(void)
{
	return (unsigned long long)jiffies * (1000000000 / HZ);
}

L
Linus Torvalds 已提交
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
 * Some helpers for converting nanosecond timing to jiffy resolution
 */
#define NS_TO_JIFFIES(TIME)	((TIME) / (1000000000 / HZ))
#define JIFFIES_TO_NS(TIME)	((TIME) * (1000000000 / HZ))

I
Ingo Molnar 已提交
101 102 103
#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

L
Linus Torvalds 已提交
104 105 106 107 108 109 110 111 112
/*
 * These are the 'tuning knobs' of the scheduler:
 *
 * Minimum timeslice is 5 msecs (or 1 jiffy, whichever is larger),
 * default timeslice is 100 msecs, maximum timeslice is 800 msecs.
 * Timeslices get refilled after they expire.
 */
#define MIN_TIMESLICE		max(5 * HZ / 1000, 1)
#define DEF_TIMESLICE		(100 * HZ / 1000)
113

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
#ifdef CONFIG_SMP
/*
 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 */
static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
{
	return reciprocal_divide(load, sg->reciprocal_cpu_power);
}

/*
 * Each time a sched group cpu_power is changed,
 * we must compute its reciprocal value
 */
static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
{
	sg->__cpu_power += val;
	sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
}
#endif

I
Ingo Molnar 已提交
135 136 137
#define SCALE_PRIO(x, prio) \
	max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_TIMESLICE)

138
/*
I
Ingo Molnar 已提交
139
 * static_prio_timeslice() scales user-nice values [ -20 ... 0 ... 19 ]
140 141
 * to time slice values: [800ms ... 100ms ... 5ms]
 */
I
Ingo Molnar 已提交
142
static unsigned int static_prio_timeslice(int static_prio)
143
{
I
Ingo Molnar 已提交
144 145 146 147 148 149 150
	if (static_prio == NICE_TO_PRIO(19))
		return 1;

	if (static_prio < NICE_TO_PRIO(0))
		return SCALE_PRIO(DEF_TIMESLICE * 4, static_prio);
	else
		return SCALE_PRIO(DEF_TIMESLICE, static_prio);
151 152
}

153 154 155 156 157 158 159 160 161 162 163 164
static inline int rt_policy(int policy)
{
	if (unlikely(policy == SCHED_FIFO) || unlikely(policy == SCHED_RR))
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

L
Linus Torvalds 已提交
165
/*
I
Ingo Molnar 已提交
166
 * This is the priority-queue data structure of the RT scheduling class:
L
Linus Torvalds 已提交
167
 */
I
Ingo Molnar 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

struct load_stat {
	struct load_weight load;
	u64 load_update_start, load_update_last;
	unsigned long delta_fair, delta_exec, delta_stat;
};

/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	s64 fair_clock;
	u64 exec_clock;
	s64 wait_runtime;
	u64 sleeper_bonus;
	unsigned long wait_runtime_overruns, wait_runtime_underruns;

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;
	struct rb_node *rb_load_balance_curr;
#ifdef CONFIG_FAIR_GROUP_SCHED
	/* 'curr' points to currently running entity on this cfs_rq.
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
	struct sched_entity *curr;
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

	/* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
	struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */
#endif
};
L
Linus Torvalds 已提交
210

I
Ingo Molnar 已提交
211 212 213 214 215 216 217
/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
	int rt_load_balance_idx;
	struct list_head *rt_load_balance_head, *rt_load_balance_curr;
};

L
Linus Torvalds 已提交
218 219 220 221 222 223 224
/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
225
struct rq {
I
Ingo Molnar 已提交
226
	spinlock_t lock;	/* runqueue lock */
L
Linus Torvalds 已提交
227 228 229 230 231 232

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
I
Ingo Molnar 已提交
233 234
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
235
	unsigned char idle_at_tick;
236 237 238
#ifdef CONFIG_NO_HZ
	unsigned char in_nohz_recently;
#endif
I
Ingo Molnar 已提交
239 240 241 242 243 244 245
	struct load_stat ls;	/* capture load from *all* tasks on this cpu */
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
#ifdef CONFIG_FAIR_GROUP_SCHED
	struct list_head leaf_cfs_rq_list; /* list of leaf cfs_rq on this cpu */
L
Linus Torvalds 已提交
246
#endif
I
Ingo Molnar 已提交
247
	struct rt_rq  rt;
L
Linus Torvalds 已提交
248 249 250 251 252 253 254 255 256

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

257
	struct task_struct *curr, *idle;
258
	unsigned long next_balance;
L
Linus Torvalds 已提交
259
	struct mm_struct *prev_mm;
I
Ingo Molnar 已提交
260 261 262 263 264 265 266

	u64 clock, prev_clock_raw;
	s64 clock_max_delta;

	unsigned int clock_warps, clock_overflows;
	unsigned int clock_unstable_events;

L
Linus Torvalds 已提交
267 268 269 270 271 272 273 274
	atomic_t nr_iowait;

#ifdef CONFIG_SMP
	struct sched_domain *sd;

	/* For active balancing */
	int active_balance;
	int push_cpu;
275
	int cpu;		/* cpu of this runqueue */
L
Linus Torvalds 已提交
276

277
	struct task_struct *migration_thread;
L
Linus Torvalds 已提交
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
	struct list_head migration_queue;
#endif

#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;

	/* sys_sched_yield() stats */
	unsigned long yld_exp_empty;
	unsigned long yld_act_empty;
	unsigned long yld_both_empty;
	unsigned long yld_cnt;

	/* schedule() stats */
	unsigned long sched_switch;
	unsigned long sched_cnt;
	unsigned long sched_goidle;

	/* try_to_wake_up() stats */
	unsigned long ttwu_cnt;
	unsigned long ttwu_local;
#endif
300
	struct lock_class_key rq_lock_key;
L
Linus Torvalds 已提交
301 302
};

303
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
304
static DEFINE_MUTEX(sched_hotcpu_mutex);
L
Linus Torvalds 已提交
305

I
Ingo Molnar 已提交
306 307 308 309 310
static inline void check_preempt_curr(struct rq *rq, struct task_struct *p)
{
	rq->curr->sched_class->check_preempt_curr(rq, p);
}

311 312 313 314 315 316 317 318 319
static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

I
Ingo Molnar 已提交
320
/*
I
Ingo Molnar 已提交
321 322
 * Update the per-runqueue clock, as finegrained as the platform can give
 * us, but without assuming monotonicity, etc.:
I
Ingo Molnar 已提交
323
 */
I
Ingo Molnar 已提交
324
static void __update_rq_clock(struct rq *rq)
I
Ingo Molnar 已提交
325 326 327 328 329 330
{
	u64 prev_raw = rq->prev_clock_raw;
	u64 now = sched_clock();
	s64 delta = now - prev_raw;
	u64 clock = rq->clock;

I
Ingo Molnar 已提交
331 332 333
#ifdef CONFIG_SCHED_DEBUG
	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
#endif
I
Ingo Molnar 已提交
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
	/*
	 * Protect against sched_clock() occasionally going backwards:
	 */
	if (unlikely(delta < 0)) {
		clock++;
		rq->clock_warps++;
	} else {
		/*
		 * Catch too large forward jumps too:
		 */
		if (unlikely(delta > 2*TICK_NSEC)) {
			clock++;
			rq->clock_overflows++;
		} else {
			if (unlikely(delta > rq->clock_max_delta))
				rq->clock_max_delta = delta;
			clock += delta;
		}
	}

	rq->prev_clock_raw = now;
	rq->clock = clock;
I
Ingo Molnar 已提交
356
}
I
Ingo Molnar 已提交
357

I
Ingo Molnar 已提交
358 359 360 361
static void update_rq_clock(struct rq *rq)
{
	if (likely(smp_processor_id() == cpu_of(rq)))
		__update_rq_clock(rq);
I
Ingo Molnar 已提交
362 363
}

N
Nick Piggin 已提交
364 365
/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
366
 * See detach_destroy_domains: synchronize_sched for details.
N
Nick Piggin 已提交
367 368 369 370
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
371 372
#define for_each_domain(cpu, __sd) \
	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
L
Linus Torvalds 已提交
373 374 375 376 377 378

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

379 380 381 382 383 384 385 386
/*
 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
 * clock constructed from sched_clock():
 */
unsigned long long cpu_clock(int cpu)
{
	unsigned long long now;
	unsigned long flags;
I
Ingo Molnar 已提交
387
	struct rq *rq;
388

389
	local_irq_save(flags);
I
Ingo Molnar 已提交
390 391 392
	rq = cpu_rq(cpu);
	update_rq_clock(rq);
	now = rq->clock;
393
	local_irq_restore(flags);
394 395 396 397

	return now;
}

I
Ingo Molnar 已提交
398 399 400 401 402 403 404 405 406 407 408 409
#ifdef CONFIG_FAIR_GROUP_SCHED
/* Change a task's ->cfs_rq if it moves across CPUs */
static inline void set_task_cfs_rq(struct task_struct *p)
{
	p->se.cfs_rq = &task_rq(p)->cfs;
}
#else
static inline void set_task_cfs_rq(struct task_struct *p)
{
}
#endif

L
Linus Torvalds 已提交
410
#ifndef prepare_arch_switch
411 412 413 414 415 416 417
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

#ifndef __ARCH_WANT_UNLOCKED_CTXSW
418
static inline int task_running(struct rq *rq, struct task_struct *p)
419 420 421 422
{
	return rq->curr == p;
}

423
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
424 425 426
{
}

427
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
428
{
429 430 431 432
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
433 434 435 436 437 438 439
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

440 441 442 443
	spin_unlock_irq(&rq->lock);
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
444
static inline int task_running(struct rq *rq, struct task_struct *p)
445 446 447 448 449 450 451 452
{
#ifdef CONFIG_SMP
	return p->oncpu;
#else
	return rq->curr == p;
#endif
}

453
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->oncpu = 1;
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	spin_unlock_irq(&rq->lock);
#else
	spin_unlock(&rq->lock);
#endif
}

470
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
471 472 473 474 475 476 477 478 479 480 481 482
{
#ifdef CONFIG_SMP
	/*
	 * After ->oncpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->oncpu = 0;
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
L
Linus Torvalds 已提交
483
#endif
484 485
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
L
Linus Torvalds 已提交
486

487 488 489 490
/*
 * __task_rq_lock - lock the runqueue a given task resides on.
 * Must be called interrupts disabled.
 */
491
static inline struct rq *__task_rq_lock(struct task_struct *p)
492 493
	__acquires(rq->lock)
{
494
	struct rq *rq;
495 496 497 498 499 500 501 502 503 504 505

repeat_lock_task:
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock(&rq->lock);
		goto repeat_lock_task;
	}
	return rq;
}

L
Linus Torvalds 已提交
506 507 508 509 510
/*
 * task_rq_lock - lock the runqueue a given task resides on and disable
 * interrupts.  Note the ordering: we can safely lookup the task_rq without
 * explicitly disabling preemption.
 */
511
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
512 513
	__acquires(rq->lock)
{
514
	struct rq *rq;
L
Linus Torvalds 已提交
515 516 517 518 519 520 521 522 523 524 525 526

repeat_lock_task:
	local_irq_save(*flags);
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock_irqrestore(&rq->lock, *flags);
		goto repeat_lock_task;
	}
	return rq;
}

527
static inline void __task_rq_unlock(struct rq *rq)
528 529 530 531 532
	__releases(rq->lock)
{
	spin_unlock(&rq->lock);
}

533
static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
L
Linus Torvalds 已提交
534 535 536 537 538 539
	__releases(rq->lock)
{
	spin_unlock_irqrestore(&rq->lock, *flags);
}

/*
540
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
541
 */
542
static inline struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
543 544
	__acquires(rq->lock)
{
545
	struct rq *rq;
L
Linus Torvalds 已提交
546 547 548 549 550 551 552 553

	local_irq_disable();
	rq = this_rq();
	spin_lock(&rq->lock);

	return rq;
}

554 555 556 557 558 559 560 561 562 563 564 565 566 567
/*
 * CPU frequency is/was unstable - start new by setting prev_clock_raw:
 */
void sched_clock_unstable_event(void)
{
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(current, &flags);
	rq->prev_clock_raw = sched_clock();
	rq->clock_unstable_events++;
	task_rq_unlock(rq, &flags);
}

I
Ingo Molnar 已提交
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

static void resched_task(struct task_struct *p)
{
	int cpu;

	assert_spin_locked(&task_rq(p)->lock);

	if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
		return;

	set_tsk_thread_flag(p, TIF_NEED_RESCHED);

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	if (!spin_trylock_irqsave(&rq->lock, flags))
		return;
	resched_task(cpu_curr(cpu));
	spin_unlock_irqrestore(&rq->lock, flags);
}
#else
static inline void resched_task(struct task_struct *p)
{
	assert_spin_locked(&task_rq(p)->lock);
	set_tsk_need_resched(p);
}
#endif

620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
static u64 div64_likely32(u64 divident, unsigned long divisor)
{
#if BITS_PER_LONG == 32
	if (likely(divident <= 0xffffffffULL))
		return (u32)divident / divisor;
	do_div(divident, divisor);

	return divident;
#else
	return divident / divisor;
#endif
}

#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

641
static unsigned long
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

	if (unlikely(!lw->inv_weight))
		lw->inv_weight = WMULT_CONST / lw->weight;

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
	if (unlikely(tmp > WMULT_CONST)) {
		tmp = ((tmp >> WMULT_SHIFT/2) * lw->inv_weight)
				>> (WMULT_SHIFT/2);
	} else {
		tmp = (tmp * lw->inv_weight) >> WMULT_SHIFT;
	}

661
	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
}

static inline unsigned long
calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
{
	return calc_delta_mine(delta_exec, NICE_0_LOAD, lw);
}

static void update_load_add(struct load_weight *lw, unsigned long inc)
{
	lw->weight += inc;
	lw->inv_weight = 0;
}

static void update_load_sub(struct load_weight *lw, unsigned long dec)
{
	lw->weight -= dec;
	lw->inv_weight = 0;
}

682 683 684 685 686 687 688 689 690
/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
 * scheduling class and "nice" value.  For SCHED_NORMAL tasks this is just a
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

I
Ingo Molnar 已提交
691 692 693 694 695 696 697 698 699 700 701
#define WEIGHT_IDLEPRIO		2
#define WMULT_IDLEPRIO		(1 << 31)

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
702 703 704
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
I
Ingo Molnar 已提交
705 706 707 708 709 710 711 712 713
 */
static const int prio_to_weight[40] = {
/* -20 */ 88818, 71054, 56843, 45475, 36380, 29104, 23283, 18626, 14901, 11921,
/* -10 */  9537,  7629,  6103,  4883,  3906,  3125,  2500,  2000,  1600,  1280,
/*   0 */  NICE_0_LOAD /* 1024 */,
/*   1 */          819,   655,   524,   419,   336,   268,   215,   172,   137,
/*  10 */   110,    87,    70,    56,    45,    36,    29,    23,    18,    15,
};

714 715 716 717 718 719 720
/*
 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
 *
 * In cases where the weight does not change often, we can use the
 * precalculated inverse to speed up arithmetics by turning divisions
 * into multiplications:
 */
I
Ingo Molnar 已提交
721
static const u32 prio_to_wmult[40] = {
722 723 724 725 726 727 728 729
/* -20 */     48356,     60446,     75558,     94446,    118058,
/* -15 */    147573,    184467,    230589,    288233,    360285,
/* -10 */    450347,    562979,    703746,    879575,   1099582,
/*  -5 */   1374389,   1717986,   2147483,   2684354,   3355443,
/*   0 */   4194304,   5244160,   6557201,   8196502,  10250518,
/*   5 */  12782640,  16025997,  19976592,  24970740,  31350126,
/*  10 */  39045157,  49367440,  61356675,  76695844,  95443717,
/*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
I
Ingo Molnar 已提交
730
};
731

I
Ingo Molnar 已提交
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);

/*
 * runqueue iterator, to support SMP load-balancing between different
 * scheduling classes, without having to expose their internal data
 * structures to the load-balancing proper:
 */
struct rq_iterator {
	void *arg;
	struct task_struct *(*start)(void *);
	struct task_struct *(*next)(void *);
};

static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
749
		      int *this_best_prio, struct rq_iterator *iterator);
I
Ingo Molnar 已提交
750 751 752 753 754 755 756 757 758 759 760

#include "sched_stats.h"
#include "sched_rt.c"
#include "sched_fair.c"
#include "sched_idletask.c"
#ifdef CONFIG_SCHED_DEBUG
# include "sched_debug.c"
#endif

#define sched_class_highest (&rt_sched_class)

761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
static void __update_curr_load(struct rq *rq, struct load_stat *ls)
{
	if (rq->curr != rq->idle && ls->load.weight) {
		ls->delta_exec += ls->delta_stat;
		ls->delta_fair += calc_delta_fair(ls->delta_stat, &ls->load);
		ls->delta_stat = 0;
	}
}

/*
 * Update delta_exec, delta_fair fields for rq.
 *
 * delta_fair clock advances at a rate inversely proportional to
 * total load (rq->ls.load.weight) on the runqueue, while
 * delta_exec advances at the same rate as wall-clock (provided
 * cpu is not idle).
 *
 * delta_exec / delta_fair is a measure of the (smoothened) load on this
 * runqueue over any given interval. This (smoothened) load is used
 * during load balance.
 *
 * This function is called /before/ updating rq->ls.load
 * and when switching tasks.
 */
785
static void update_curr_load(struct rq *rq)
786 787 788 789 790
{
	struct load_stat *ls = &rq->ls;
	u64 start;

	start = ls->load_update_start;
791 792
	ls->load_update_start = rq->clock;
	ls->delta_stat += rq->clock - start;
793 794 795 796 797 798 799 800 801 802 803
	/*
	 * Stagger updates to ls->delta_fair. Very frequent updates
	 * can be expensive.
	 */
	if (ls->delta_stat >= sysctl_sched_stat_granularity)
		__update_curr_load(rq, ls);
}

static inline void
inc_load(struct rq *rq, const struct task_struct *p, u64 now)
{
804
	update_curr_load(rq);
805 806 807 808 809 810
	update_load_add(&rq->ls.load, p->se.load.weight);
}

static inline void
dec_load(struct rq *rq, const struct task_struct *p, u64 now)
{
811
	update_curr_load(rq);
812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
	update_load_sub(&rq->ls.load, p->se.load.weight);
}

static void inc_nr_running(struct task_struct *p, struct rq *rq, u64 now)
{
	rq->nr_running++;
	inc_load(rq, p, now);
}

static void dec_nr_running(struct task_struct *p, struct rq *rq, u64 now)
{
	rq->nr_running--;
	dec_load(rq, p, now);
}

827 828
static void set_load_weight(struct task_struct *p)
{
I
Ingo Molnar 已提交
829 830 831
	task_rq(p)->cfs.wait_runtime -= p->se.wait_runtime;
	p->se.wait_runtime = 0;

832
	if (task_has_rt_policy(p)) {
I
Ingo Molnar 已提交
833 834 835 836
		p->se.load.weight = prio_to_weight[0] * 2;
		p->se.load.inv_weight = prio_to_wmult[0] >> 1;
		return;
	}
837

I
Ingo Molnar 已提交
838 839 840 841 842 843 844 845
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
		p->se.load.weight = WEIGHT_IDLEPRIO;
		p->se.load.inv_weight = WMULT_IDLEPRIO;
		return;
	}
846

I
Ingo Molnar 已提交
847 848
	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
849 850
}

I
Ingo Molnar 已提交
851 852
static void
enqueue_task(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
853
{
I
Ingo Molnar 已提交
854
	sched_info_queued(p);
855
	p->sched_class->enqueue_task(rq, p, wakeup);
I
Ingo Molnar 已提交
856
	p->se.on_rq = 1;
857 858
}

I
Ingo Molnar 已提交
859 860
static void
dequeue_task(struct rq *rq, struct task_struct *p, int sleep, u64 now)
861
{
862
	p->sched_class->dequeue_task(rq, p, sleep);
I
Ingo Molnar 已提交
863
	p->se.on_rq = 0;
864 865
}

866
/*
I
Ingo Molnar 已提交
867
 * __normal_prio - return the priority that is based on the static prio
868 869 870
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
871
	return p->static_prio;
872 873
}

874 875 876 877 878 879 880
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
881
static inline int normal_prio(struct task_struct *p)
882 883 884
{
	int prio;

885
	if (task_has_rt_policy(p))
886 887 888 889 890 891 892 893 894 895 896 897 898
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
899
static int effective_prio(struct task_struct *p)
900 901 902 903 904 905 906 907 908 909 910 911
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
912
/*
I
Ingo Molnar 已提交
913
 * activate_task - move a task to the runqueue.
L
Linus Torvalds 已提交
914
 */
I
Ingo Molnar 已提交
915
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
L
Linus Torvalds 已提交
916
{
I
Ingo Molnar 已提交
917 918 919 920
	u64 now;

	update_rq_clock(rq);
	now = rq->clock;
921

I
Ingo Molnar 已提交
922 923
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;
L
Linus Torvalds 已提交
924

I
Ingo Molnar 已提交
925 926
	enqueue_task(rq, p, wakeup, now);
	inc_nr_running(p, rq, now);
L
Linus Torvalds 已提交
927 928 929
}

/*
I
Ingo Molnar 已提交
930
 * activate_idle_task - move idle task to the _front_ of runqueue.
L
Linus Torvalds 已提交
931
 */
I
Ingo Molnar 已提交
932
static inline void activate_idle_task(struct task_struct *p, struct rq *rq)
L
Linus Torvalds 已提交
933
{
I
Ingo Molnar 已提交
934 935 936 937
	u64 now;

	update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
938

I
Ingo Molnar 已提交
939 940
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;
I
Ingo Molnar 已提交
941

I
Ingo Molnar 已提交
942 943
	enqueue_task(rq, p, 0, now);
	inc_nr_running(p, rq, now);
L
Linus Torvalds 已提交
944 945 946 947 948
}

/*
 * deactivate_task - remove a task from the runqueue.
 */
I
Ingo Molnar 已提交
949 950
static void
deactivate_task(struct rq *rq, struct task_struct *p, int sleep, u64 now)
L
Linus Torvalds 已提交
951
{
I
Ingo Molnar 已提交
952 953 954 955 956
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible++;

	dequeue_task(rq, p, sleep, now);
	dec_nr_running(p, rq, now);
L
Linus Torvalds 已提交
957 958 959 960 961 962
}

/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
963
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
964 965 966 967
{
	return cpu_curr(task_cpu(p)) == p;
}

968 969 970
/* Used instead of source_load when we know the type == 0 */
unsigned long weighted_cpuload(const int cpu)
{
I
Ingo Molnar 已提交
971 972 973 974 975 976 977 978 979
	return cpu_rq(cpu)->ls.load.weight;
}

static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
#ifdef CONFIG_SMP
	task_thread_info(p)->cpu = cpu;
	set_task_cfs_rq(p);
#endif
980 981
}

L
Linus Torvalds 已提交
982
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
983

I
Ingo Molnar 已提交
984
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
985
{
I
Ingo Molnar 已提交
986 987 988 989 990
	int old_cpu = task_cpu(p);
	struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
	u64 clock_offset, fair_clock_offset;

	clock_offset = old_rq->clock - new_rq->clock;
I
Ingo Molnar 已提交
991 992
	fair_clock_offset = old_rq->cfs.fair_clock - new_rq->cfs.fair_clock;

I
Ingo Molnar 已提交
993 994
	if (p->se.wait_start_fair)
		p->se.wait_start_fair -= fair_clock_offset;
I
Ingo Molnar 已提交
995 996 997 998 999 1000
	if (p->se.sleep_start_fair)
		p->se.sleep_start_fair -= fair_clock_offset;

#ifdef CONFIG_SCHEDSTATS
	if (p->se.wait_start)
		p->se.wait_start -= clock_offset;
I
Ingo Molnar 已提交
1001 1002 1003 1004
	if (p->se.sleep_start)
		p->se.sleep_start -= clock_offset;
	if (p->se.block_start)
		p->se.block_start -= clock_offset;
I
Ingo Molnar 已提交
1005
#endif
I
Ingo Molnar 已提交
1006 1007

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1008 1009
}

1010
struct migration_req {
L
Linus Torvalds 已提交
1011 1012
	struct list_head list;

1013
	struct task_struct *task;
L
Linus Torvalds 已提交
1014 1015 1016
	int dest_cpu;

	struct completion done;
1017
};
L
Linus Torvalds 已提交
1018 1019 1020 1021 1022

/*
 * The task's runqueue lock must be held.
 * Returns true if you have to wait for migration thread.
 */
1023
static int
1024
migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
L
Linus Torvalds 已提交
1025
{
1026
	struct rq *rq = task_rq(p);
L
Linus Torvalds 已提交
1027 1028 1029 1030 1031

	/*
	 * If the task is not on a runqueue (and not running), then
	 * it is sufficient to simply update the task's cpu field.
	 */
I
Ingo Molnar 已提交
1032
	if (!p->se.on_rq && !task_running(rq, p)) {
L
Linus Torvalds 已提交
1033 1034 1035 1036 1037 1038 1039 1040
		set_task_cpu(p, dest_cpu);
		return 0;
	}

	init_completion(&req->done);
	req->task = p;
	req->dest_cpu = dest_cpu;
	list_add(&req->list, &rq->migration_queue);
1041

L
Linus Torvalds 已提交
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
	return 1;
}

/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
1054
void wait_task_inactive(struct task_struct *p)
L
Linus Torvalds 已提交
1055 1056
{
	unsigned long flags;
I
Ingo Molnar 已提交
1057
	int running, on_rq;
1058
	struct rq *rq;
L
Linus Torvalds 已提交
1059 1060

repeat:
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
	/*
	 * We do the initial early heuristics without holding
	 * any task-queue locks at all. We'll only try to get
	 * the runqueue lock when things look like they will
	 * work out!
	 */
	rq = task_rq(p);

	/*
	 * If the task is actively running on another CPU
	 * still, just relax and busy-wait without holding
	 * any locks.
	 *
	 * NOTE! Since we don't hold any locks, it's not
	 * even sure that "rq" stays as the right runqueue!
	 * But we don't care, since "task_running()" will
	 * return false if the runqueue has changed and p
	 * is actually now running somewhere else!
	 */
	while (task_running(rq, p))
		cpu_relax();

	/*
	 * Ok, time to look more closely! We need the rq
	 * lock now, to be *sure*. If we're wrong, we'll
	 * just go back and repeat.
	 */
L
Linus Torvalds 已提交
1088
	rq = task_rq_lock(p, &flags);
1089
	running = task_running(rq, p);
I
Ingo Molnar 已提交
1090
	on_rq = p->se.on_rq;
1091 1092 1093 1094 1095 1096 1097 1098 1099
	task_rq_unlock(rq, &flags);

	/*
	 * Was it really running after all now that we
	 * checked with the proper locks actually held?
	 *
	 * Oops. Go back and try again..
	 */
	if (unlikely(running)) {
L
Linus Torvalds 已提交
1100 1101 1102
		cpu_relax();
		goto repeat;
	}
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112

	/*
	 * It's not enough that it's not actively running,
	 * it must be off the runqueue _entirely_, and not
	 * preempted!
	 *
	 * So if it wa still runnable (but just not actively
	 * running right now), it's preempted, and we should
	 * yield - it could be a while.
	 */
I
Ingo Molnar 已提交
1113
	if (unlikely(on_rq)) {
1114 1115 1116 1117 1118 1119 1120 1121 1122
		yield();
		goto repeat;
	}

	/*
	 * Ahh, all good. It wasn't running, and it wasn't
	 * runnable, which means that it will never become
	 * running in the future either. We're all done!
	 */
L
Linus Torvalds 已提交
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
 * NOTE: this function doesnt have to take the runqueue lock,
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
1138
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}

/*
1150 1151
 * Return a low guess at the load of a migration-source cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1152 1153 1154 1155
 *
 * We want to under-estimate the load of migration sources, to
 * balance conservatively.
 */
N
Nick Piggin 已提交
1156
static inline unsigned long source_load(int cpu, int type)
L
Linus Torvalds 已提交
1157
{
1158
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1159
	unsigned long total = weighted_cpuload(cpu);
1160

1161
	if (type == 0)
I
Ingo Molnar 已提交
1162
		return total;
1163

I
Ingo Molnar 已提交
1164
	return min(rq->cpu_load[type-1], total);
L
Linus Torvalds 已提交
1165 1166 1167
}

/*
1168 1169
 * Return a high guess at the load of a migration-target cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1170
 */
N
Nick Piggin 已提交
1171
static inline unsigned long target_load(int cpu, int type)
L
Linus Torvalds 已提交
1172
{
1173
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1174
	unsigned long total = weighted_cpuload(cpu);
1175

N
Nick Piggin 已提交
1176
	if (type == 0)
I
Ingo Molnar 已提交
1177
		return total;
1178

I
Ingo Molnar 已提交
1179
	return max(rq->cpu_load[type-1], total);
1180 1181 1182 1183 1184 1185 1186
}

/*
 * Return the average load per task on the cpu's run queue
 */
static inline unsigned long cpu_avg_load_per_task(int cpu)
{
1187
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1188
	unsigned long total = weighted_cpuload(cpu);
1189 1190
	unsigned long n = rq->nr_running;

I
Ingo Molnar 已提交
1191
	return n ? total / n : SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
1192 1193
}

N
Nick Piggin 已提交
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
/*
 * find_idlest_group finds and returns the least busy CPU group within the
 * domain.
 */
static struct sched_group *
find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
{
	struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
	unsigned long min_load = ULONG_MAX, this_load = 0;
	int load_idx = sd->forkexec_idx;
	int imbalance = 100 + (sd->imbalance_pct-100)/2;

	do {
		unsigned long load, avg_load;
		int local_group;
		int i;

1211 1212 1213 1214
		/* Skip over this group if it has no CPUs allowed */
		if (!cpus_intersects(group->cpumask, p->cpus_allowed))
			goto nextgroup;

N
Nick Piggin 已提交
1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
		local_group = cpu_isset(this_cpu, group->cpumask);

		/* Tally up the load of all CPUs in the group */
		avg_load = 0;

		for_each_cpu_mask(i, group->cpumask) {
			/* Bias balancing toward cpus of our domain */
			if (local_group)
				load = source_load(i, load_idx);
			else
				load = target_load(i, load_idx);

			avg_load += load;
		}

		/* Adjust by relative CPU power of the group */
1231 1232
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
N
Nick Piggin 已提交
1233 1234 1235 1236 1237 1238 1239 1240

		if (local_group) {
			this_load = avg_load;
			this = group;
		} else if (avg_load < min_load) {
			min_load = avg_load;
			idlest = group;
		}
1241
nextgroup:
N
Nick Piggin 已提交
1242 1243 1244 1245 1246 1247 1248 1249 1250
		group = group->next;
	} while (group != sd->groups);

	if (!idlest || 100*this_load < imbalance*min_load)
		return NULL;
	return idlest;
}

/*
1251
 * find_idlest_cpu - find the idlest cpu among the cpus in group.
N
Nick Piggin 已提交
1252
 */
I
Ingo Molnar 已提交
1253 1254
static int
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
N
Nick Piggin 已提交
1255
{
1256
	cpumask_t tmp;
N
Nick Piggin 已提交
1257 1258 1259 1260
	unsigned long load, min_load = ULONG_MAX;
	int idlest = -1;
	int i;

1261 1262 1263 1264
	/* Traverse only the allowed CPUs */
	cpus_and(tmp, group->cpumask, p->cpus_allowed);

	for_each_cpu_mask(i, tmp) {
1265
		load = weighted_cpuload(i);
N
Nick Piggin 已提交
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275

		if (load < min_load || (load == min_load && i == this_cpu)) {
			min_load = load;
			idlest = i;
		}
	}

	return idlest;
}

N
Nick Piggin 已提交
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
/*
 * sched_balance_self: balance the current task (running on cpu) in domains
 * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
 * SD_BALANCE_EXEC.
 *
 * Balance, ie. select the least loaded group.
 *
 * Returns the target CPU number, or the same CPU if no balancing is needed.
 *
 * preempt must be disabled.
 */
static int sched_balance_self(int cpu, int flag)
{
	struct task_struct *t = current;
	struct sched_domain *tmp, *sd = NULL;
N
Nick Piggin 已提交
1291

1292
	for_each_domain(cpu, tmp) {
I
Ingo Molnar 已提交
1293 1294 1295
		/*
		 * If power savings logic is enabled for a domain, stop there.
		 */
1296 1297
		if (tmp->flags & SD_POWERSAVINGS_BALANCE)
			break;
N
Nick Piggin 已提交
1298 1299
		if (tmp->flags & flag)
			sd = tmp;
1300
	}
N
Nick Piggin 已提交
1301 1302 1303 1304

	while (sd) {
		cpumask_t span;
		struct sched_group *group;
1305 1306 1307 1308 1309 1310
		int new_cpu, weight;

		if (!(sd->flags & flag)) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1311 1312 1313

		span = sd->span;
		group = find_idlest_group(sd, t, cpu);
1314 1315 1316 1317
		if (!group) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1318

1319
		new_cpu = find_idlest_cpu(group, t, cpu);
1320 1321 1322 1323 1324
		if (new_cpu == -1 || new_cpu == cpu) {
			/* Now try balancing at a lower domain level of cpu */
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1325

1326
		/* Now try balancing at a lower domain level of new_cpu */
N
Nick Piggin 已提交
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
		cpu = new_cpu;
		sd = NULL;
		weight = cpus_weight(span);
		for_each_domain(cpu, tmp) {
			if (weight <= cpus_weight(tmp->span))
				break;
			if (tmp->flags & flag)
				sd = tmp;
		}
		/* while loop will break here if sd == NULL */
	}

	return cpu;
}

#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352

/*
 * wake_idle() will wake a task on an idle cpu if task->cpu is
 * not idle and an idle cpu is available.  The span of cpus to
 * search starts with cpus closest then further out as needed,
 * so we always favor a closer, idle cpu.
 *
 * Returns the CPU we should wake onto.
 */
#if defined(ARCH_HAS_SCHED_WAKE_IDLE)
1353
static int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1354 1355 1356 1357 1358
{
	cpumask_t tmp;
	struct sched_domain *sd;
	int i;

1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
	/*
	 * If it is idle, then it is the best cpu to run this task.
	 *
	 * This cpu is also the best, if it has more than one task already.
	 * Siblings must be also busy(in most cases) as they didn't already
	 * pickup the extra load from this cpu and hence we need not check
	 * sibling runqueue info. This will avoid the checks and cache miss
	 * penalities associated with that.
	 */
	if (idle_cpu(cpu) || cpu_rq(cpu)->nr_running > 1)
L
Linus Torvalds 已提交
1369 1370 1371 1372
		return cpu;

	for_each_domain(cpu, sd) {
		if (sd->flags & SD_WAKE_IDLE) {
N
Nick Piggin 已提交
1373
			cpus_and(tmp, sd->span, p->cpus_allowed);
L
Linus Torvalds 已提交
1374 1375 1376 1377
			for_each_cpu_mask(i, tmp) {
				if (idle_cpu(i))
					return i;
			}
I
Ingo Molnar 已提交
1378
		} else {
N
Nick Piggin 已提交
1379
			break;
I
Ingo Molnar 已提交
1380
		}
L
Linus Torvalds 已提交
1381 1382 1383 1384
	}
	return cpu;
}
#else
1385
static inline int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
{
	return cpu;
}
#endif

/***
 * try_to_wake_up - wake up a thread
 * @p: the to-be-woken-up thread
 * @state: the mask of task states that can be woken
 * @sync: do a synchronous wakeup?
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
 * returns failure only if the task is already active.
 */
1405
static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
L
Linus Torvalds 已提交
1406 1407 1408 1409
{
	int cpu, this_cpu, success = 0;
	unsigned long flags;
	long old_state;
1410
	struct rq *rq;
L
Linus Torvalds 已提交
1411
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
1412
	struct sched_domain *sd, *this_sd = NULL;
1413
	unsigned long load, this_load;
L
Linus Torvalds 已提交
1414 1415 1416 1417 1418 1419 1420 1421
	int new_cpu;
#endif

	rq = task_rq_lock(p, &flags);
	old_state = p->state;
	if (!(old_state & state))
		goto out;

I
Ingo Molnar 已提交
1422
	if (p->se.on_rq)
L
Linus Torvalds 已提交
1423 1424 1425 1426 1427 1428 1429 1430 1431
		goto out_running;

	cpu = task_cpu(p);
	this_cpu = smp_processor_id();

#ifdef CONFIG_SMP
	if (unlikely(task_running(rq, p)))
		goto out_activate;

N
Nick Piggin 已提交
1432 1433
	new_cpu = cpu;

L
Linus Torvalds 已提交
1434 1435 1436
	schedstat_inc(rq, ttwu_cnt);
	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
N
Nick Piggin 已提交
1437 1438 1439 1440 1441 1442 1443 1444
		goto out_set_cpu;
	}

	for_each_domain(this_cpu, sd) {
		if (cpu_isset(cpu, sd->span)) {
			schedstat_inc(sd, ttwu_wake_remote);
			this_sd = sd;
			break;
L
Linus Torvalds 已提交
1445 1446 1447
		}
	}

N
Nick Piggin 已提交
1448
	if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed)))
L
Linus Torvalds 已提交
1449 1450 1451
		goto out_set_cpu;

	/*
N
Nick Piggin 已提交
1452
	 * Check for affine wakeup and passive balancing possibilities.
L
Linus Torvalds 已提交
1453
	 */
N
Nick Piggin 已提交
1454 1455 1456
	if (this_sd) {
		int idx = this_sd->wake_idx;
		unsigned int imbalance;
L
Linus Torvalds 已提交
1457

1458 1459
		imbalance = 100 + (this_sd->imbalance_pct - 100) / 2;

N
Nick Piggin 已提交
1460 1461
		load = source_load(cpu, idx);
		this_load = target_load(this_cpu, idx);
L
Linus Torvalds 已提交
1462

N
Nick Piggin 已提交
1463 1464
		new_cpu = this_cpu; /* Wake to this CPU if we can */

1465 1466
		if (this_sd->flags & SD_WAKE_AFFINE) {
			unsigned long tl = this_load;
1467 1468 1469
			unsigned long tl_per_task;

			tl_per_task = cpu_avg_load_per_task(this_cpu);
1470

L
Linus Torvalds 已提交
1471
			/*
1472 1473 1474
			 * If sync wakeup then subtract the (maximum possible)
			 * effect of the currently running task from the load
			 * of the current CPU:
L
Linus Torvalds 已提交
1475
			 */
1476
			if (sync)
I
Ingo Molnar 已提交
1477
				tl -= current->se.load.weight;
1478 1479

			if ((tl <= load &&
1480
				tl + target_load(cpu, idx) <= tl_per_task) ||
I
Ingo Molnar 已提交
1481
			       100*(tl + p->se.load.weight) <= imbalance*load) {
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
				/*
				 * This domain has SD_WAKE_AFFINE and
				 * p is cache cold in this domain, and
				 * there is no bad imbalance.
				 */
				schedstat_inc(this_sd, ttwu_move_affine);
				goto out_set_cpu;
			}
		}

		/*
		 * Start passive balancing when half the imbalance_pct
		 * limit is reached.
		 */
		if (this_sd->flags & SD_WAKE_BALANCE) {
			if (imbalance*this_load <= 100*load) {
				schedstat_inc(this_sd, ttwu_move_balance);
				goto out_set_cpu;
			}
L
Linus Torvalds 已提交
1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514
		}
	}

	new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */
out_set_cpu:
	new_cpu = wake_idle(new_cpu, p);
	if (new_cpu != cpu) {
		set_task_cpu(p, new_cpu);
		task_rq_unlock(rq, &flags);
		/* might preempt at this point */
		rq = task_rq_lock(p, &flags);
		old_state = p->state;
		if (!(old_state & state))
			goto out;
I
Ingo Molnar 已提交
1515
		if (p->se.on_rq)
L
Linus Torvalds 已提交
1516 1517 1518 1519 1520 1521 1522 1523
			goto out_running;

		this_cpu = smp_processor_id();
		cpu = task_cpu(p);
	}

out_activate:
#endif /* CONFIG_SMP */
I
Ingo Molnar 已提交
1524
	activate_task(rq, p, 1);
L
Linus Torvalds 已提交
1525 1526 1527 1528 1529 1530 1531 1532
	/*
	 * Sync wakeups (i.e. those types of wakeups where the waker
	 * has indicated that it will leave the CPU in short order)
	 * don't trigger a preemption, if the woken up task will run on
	 * this cpu. (in this case the 'I will reschedule' promise of
	 * the waker guarantees that the freshly woken up task is going
	 * to be considered on this CPU.)
	 */
I
Ingo Molnar 已提交
1533 1534
	if (!sync || cpu != this_cpu)
		check_preempt_curr(rq, p);
L
Linus Torvalds 已提交
1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
	success = 1;

out_running:
	p->state = TASK_RUNNING;
out:
	task_rq_unlock(rq, &flags);

	return success;
}

1545
int fastcall wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
1546 1547 1548 1549 1550 1551
{
	return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED |
				 TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0);
}
EXPORT_SYMBOL(wake_up_process);

1552
int fastcall wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
1553 1554 1555 1556 1557 1558 1559
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
	p->se.wait_start_fair		= 0;
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
	p->se.delta_exec		= 0;
	p->se.delta_fair_run		= 0;
	p->se.delta_fair_sleep		= 0;
	p->se.wait_runtime		= 0;
I
Ingo Molnar 已提交
1572 1573 1574 1575
	p->se.sleep_start_fair		= 0;

#ifdef CONFIG_SCHEDSTATS
	p->se.wait_start		= 0;
I
Ingo Molnar 已提交
1576 1577 1578 1579 1580 1581 1582 1583 1584 1585
	p->se.sum_wait_runtime		= 0;
	p->se.sum_sleep_runtime		= 0;
	p->se.sleep_start		= 0;
	p->se.block_start		= 0;
	p->se.sleep_max			= 0;
	p->se.block_max			= 0;
	p->se.exec_max			= 0;
	p->se.wait_max			= 0;
	p->se.wait_runtime_overruns	= 0;
	p->se.wait_runtime_underruns	= 0;
I
Ingo Molnar 已提交
1586
#endif
N
Nick Piggin 已提交
1587

I
Ingo Molnar 已提交
1588 1589
	INIT_LIST_HEAD(&p->run_list);
	p->se.on_rq = 0;
N
Nick Piggin 已提交
1590

1591 1592 1593 1594
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif

L
Linus Torvalds 已提交
1595 1596 1597 1598 1599 1600 1601
	/*
	 * We mark the process as running here, but have not actually
	 * inserted it onto the runqueue yet. This guarantees that
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
}

/*
 * fork()/clone()-time setup:
 */
void sched_fork(struct task_struct *p, int clone_flags)
{
	int cpu = get_cpu();

	__sched_fork(p);

#ifdef CONFIG_SMP
	cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
#endif
	__set_task_cpu(p, cpu);
1617 1618 1619 1620 1621 1622

	/*
	 * Make sure we do not leak PI boosting priority to the child:
	 */
	p->prio = current->normal_prio;

1623
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
1624
	if (likely(sched_info_on()))
1625
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
1626
#endif
1627
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
1628 1629
	p->oncpu = 0;
#endif
L
Linus Torvalds 已提交
1630
#ifdef CONFIG_PREEMPT
1631
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
1632
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
1633
#endif
N
Nick Piggin 已提交
1634
	put_cpu();
L
Linus Torvalds 已提交
1635 1636
}

I
Ingo Molnar 已提交
1637 1638 1639 1640 1641 1642
/*
 * After fork, child runs first. (default) If set to 0 then
 * parent will (try to) run first.
 */
unsigned int __read_mostly sysctl_sched_child_runs_first = 1;

L
Linus Torvalds 已提交
1643 1644 1645 1646 1647 1648 1649
/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
1650
void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
L
Linus Torvalds 已提交
1651 1652
{
	unsigned long flags;
I
Ingo Molnar 已提交
1653 1654
	struct rq *rq;
	int this_cpu;
I
Ingo Molnar 已提交
1655
	u64 now;
L
Linus Torvalds 已提交
1656 1657

	rq = task_rq_lock(p, &flags);
N
Nick Piggin 已提交
1658
	BUG_ON(p->state != TASK_RUNNING);
I
Ingo Molnar 已提交
1659
	this_cpu = smp_processor_id(); /* parent's CPU */
I
Ingo Molnar 已提交
1660 1661
	update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
1662 1663 1664

	p->prio = effective_prio(p);

I
Ingo Molnar 已提交
1665 1666 1667 1668
	if (!p->sched_class->task_new || !sysctl_sched_child_runs_first ||
			(clone_flags & CLONE_VM) || task_cpu(p) != this_cpu ||
			!current->se.on_rq) {

I
Ingo Molnar 已提交
1669
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
1670 1671
	} else {
		/*
I
Ingo Molnar 已提交
1672 1673
		 * Let the scheduling class do new task startup
		 * management (if any):
L
Linus Torvalds 已提交
1674
		 */
1675
		p->sched_class->task_new(rq, p);
I
Ingo Molnar 已提交
1676
		inc_nr_running(p, rq, now);
L
Linus Torvalds 已提交
1677
	}
I
Ingo Molnar 已提交
1678 1679
	check_preempt_curr(rq, p);
	task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
1680 1681
}

1682 1683 1684
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
R
Randy Dunlap 已提交
1685 1686
 * preempt_notifier_register - tell me when current is being being preempted & rescheduled
 * @notifier: notifier struct to register
1687 1688 1689 1690 1691 1692 1693 1694 1695
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
1696
 * @notifier: notifier struct to unregister
1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_out(notifier, next);
}

#else

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

#endif

1740 1741 1742
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
1743
 * @prev: the current task that is being switched out
1744 1745 1746 1747 1748 1749 1750 1751 1752
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
1753 1754 1755
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
1756
{
1757
	fire_sched_out_preempt_notifiers(prev, next);
1758 1759 1760 1761
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
1762 1763
/**
 * finish_task_switch - clean up after a task-switch
1764
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
1765 1766
 * @prev: the thread we just switched away from.
 *
1767 1768 1769 1770
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
1771 1772 1773 1774 1775 1776
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
 * so, we finish that here outside of the runqueue lock.  (Doing it
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
1777
static inline void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
1778 1779 1780
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
1781
	long prev_state;
L
Linus Torvalds 已提交
1782 1783 1784 1785 1786

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
1787
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
1788 1789
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
1790
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
1791 1792 1793 1794 1795
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
1796
	prev_state = prev->state;
1797 1798
	finish_arch_switch(prev);
	finish_lock_switch(rq, prev);
1799
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
1800 1801
	if (mm)
		mmdrop(mm);
1802
	if (unlikely(prev_state == TASK_DEAD)) {
1803 1804 1805
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
1806
		 */
1807
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
1808
		put_task_struct(prev);
1809
	}
L
Linus Torvalds 已提交
1810 1811 1812 1813 1814 1815
}

/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
1816
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
1817 1818
	__releases(rq->lock)
{
1819 1820
	struct rq *rq = this_rq();

1821 1822 1823 1824 1825
	finish_task_switch(rq, prev);
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
1826 1827 1828 1829 1830 1831 1832 1833
	if (current->set_child_tid)
		put_user(current->pid, current->set_child_tid);
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
1834
static inline void
1835
context_switch(struct rq *rq, struct task_struct *prev,
1836
	       struct task_struct *next)
L
Linus Torvalds 已提交
1837
{
I
Ingo Molnar 已提交
1838
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
1839

1840
	prepare_task_switch(rq, prev, next);
I
Ingo Molnar 已提交
1841 1842
	mm = next->mm;
	oldmm = prev->active_mm;
1843 1844 1845 1846 1847 1848 1849
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
	arch_enter_lazy_cpu_mode();

I
Ingo Molnar 已提交
1850
	if (unlikely(!mm)) {
L
Linus Torvalds 已提交
1851 1852 1853 1854 1855 1856
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

I
Ingo Molnar 已提交
1857
	if (unlikely(!prev->mm)) {
L
Linus Torvalds 已提交
1858 1859 1860
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
1861 1862 1863 1864 1865 1866 1867
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
1868
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
1869
#endif
L
Linus Torvalds 已提交
1870 1871 1872 1873

	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
1874 1875 1876 1877 1878 1879 1880
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
}

/*
 * nr_running, nr_uninterruptible and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, current number of uninterruptible-sleeping threads, total
 * number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
}

unsigned long nr_uninterruptible(void)
{
	unsigned long i, sum = 0;

1904
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
		sum += cpu_rq(i)->nr_uninterruptible;

	/*
	 * Since we read the counters lockless, it might be slightly
	 * inaccurate. Do not allow it to go below zero though:
	 */
	if (unlikely((long)sum < 0))
		sum = 0;

	return sum;
}

unsigned long long nr_context_switches(void)
{
1919 1920
	int i;
	unsigned long long sum = 0;
L
Linus Torvalds 已提交
1921

1922
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1923 1924 1925 1926 1927 1928 1929 1930 1931
		sum += cpu_rq(i)->nr_switches;

	return sum;
}

unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;

1932
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1933 1934 1935 1936 1937
		sum += atomic_read(&cpu_rq(i)->nr_iowait);

	return sum;
}

1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
unsigned long nr_active(void)
{
	unsigned long i, running = 0, uninterruptible = 0;

	for_each_online_cpu(i) {
		running += cpu_rq(i)->nr_running;
		uninterruptible += cpu_rq(i)->nr_uninterruptible;
	}

	if (unlikely((long)uninterruptible < 0))
		uninterruptible = 0;

	return running + uninterruptible;
}

1953
/*
I
Ingo Molnar 已提交
1954 1955
 * Update rq->cpu_load[] statistics. This function is usually called every
 * scheduler tick (TICK_NSEC).
1956
 */
I
Ingo Molnar 已提交
1957
static void update_cpu_load(struct rq *this_rq)
1958
{
I
Ingo Molnar 已提交
1959 1960 1961 1962
	u64 fair_delta64, exec_delta64, idle_delta64, sample_interval64, tmp64;
	unsigned long total_load = this_rq->ls.load.weight;
	unsigned long this_load =  total_load;
	struct load_stat *ls = &this_rq->ls;
I
Ingo Molnar 已提交
1963
	u64 now;
I
Ingo Molnar 已提交
1964 1965
	int i, scale;

I
Ingo Molnar 已提交
1966 1967 1968
	__update_rq_clock(this_rq);
	now = this_rq->clock;

I
Ingo Molnar 已提交
1969 1970 1971 1972 1973
	this_rq->nr_load_updates++;
	if (unlikely(!(sysctl_sched_features & SCHED_FEAT_PRECISE_CPU_LOAD)))
		goto do_avg;

	/* Update delta_fair/delta_exec fields first */
1974
	update_curr_load(this_rq);
I
Ingo Molnar 已提交
1975 1976 1977 1978 1979 1980 1981

	fair_delta64 = ls->delta_fair + 1;
	ls->delta_fair = 0;

	exec_delta64 = ls->delta_exec + 1;
	ls->delta_exec = 0;

1982 1983
	sample_interval64 = this_rq->clock - ls->load_update_last;
	ls->load_update_last = this_rq->clock;
I
Ingo Molnar 已提交
1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010

	if ((s64)sample_interval64 < (s64)TICK_NSEC)
		sample_interval64 = TICK_NSEC;

	if (exec_delta64 > sample_interval64)
		exec_delta64 = sample_interval64;

	idle_delta64 = sample_interval64 - exec_delta64;

	tmp64 = div64_64(SCHED_LOAD_SCALE * exec_delta64, fair_delta64);
	tmp64 = div64_64(tmp64 * exec_delta64, sample_interval64);

	this_load = (unsigned long)tmp64;

do_avg:

	/* Update our load: */
	for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
		unsigned long old_load, new_load;

		/* scale is effectively 1 << i now, and >> i divides by scale */

		old_load = this_rq->cpu_load[i];
		new_load = this_load;

		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
	}
2011 2012
}

I
Ingo Molnar 已提交
2013 2014
#ifdef CONFIG_SMP

L
Linus Torvalds 已提交
2015 2016 2017 2018 2019 2020
/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
2021
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2022 2023 2024
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
2025
	BUG_ON(!irqs_disabled());
L
Linus Torvalds 已提交
2026 2027 2028 2029
	if (rq1 == rq2) {
		spin_lock(&rq1->lock);
		__acquire(rq2->lock);	/* Fake it out ;) */
	} else {
2030
		if (rq1 < rq2) {
L
Linus Torvalds 已提交
2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045
			spin_lock(&rq1->lock);
			spin_lock(&rq2->lock);
		} else {
			spin_lock(&rq2->lock);
			spin_lock(&rq1->lock);
		}
	}
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
2046
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	spin_unlock(&rq1->lock);
	if (rq1 != rq2)
		spin_unlock(&rq2->lock);
	else
		__release(rq2->lock);
}

/*
 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
 */
2060
static void double_lock_balance(struct rq *this_rq, struct rq *busiest)
L
Linus Torvalds 已提交
2061 2062 2063 2064
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
2065 2066 2067 2068 2069
	if (unlikely(!irqs_disabled())) {
		/* printk() doesn't work good under rq->lock */
		spin_unlock(&this_rq->lock);
		BUG_ON(1);
	}
L
Linus Torvalds 已提交
2070
	if (unlikely(!spin_trylock(&busiest->lock))) {
2071
		if (busiest < this_rq) {
L
Linus Torvalds 已提交
2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
			spin_unlock(&this_rq->lock);
			spin_lock(&busiest->lock);
			spin_lock(&this_rq->lock);
		} else
			spin_lock(&busiest->lock);
	}
}

/*
 * If dest_cpu is allowed for this process, migrate the task to it.
 * This is accomplished by forcing the cpu_allowed mask to only
 * allow dest_cpu, which will force the cpu onto dest_cpu.  Then
 * the cpu_allowed mask is restored.
 */
2086
static void sched_migrate_task(struct task_struct *p, int dest_cpu)
L
Linus Torvalds 已提交
2087
{
2088
	struct migration_req req;
L
Linus Torvalds 已提交
2089
	unsigned long flags;
2090
	struct rq *rq;
L
Linus Torvalds 已提交
2091 2092 2093 2094 2095 2096 2097 2098 2099 2100

	rq = task_rq_lock(p, &flags);
	if (!cpu_isset(dest_cpu, p->cpus_allowed)
	    || unlikely(cpu_is_offline(dest_cpu)))
		goto out;

	/* force the process onto the specified CPU */
	if (migrate_task(p, dest_cpu, &req)) {
		/* Need to wait for migration thread (might exit: take ref). */
		struct task_struct *mt = rq->migration_thread;
2101

L
Linus Torvalds 已提交
2102 2103 2104 2105 2106
		get_task_struct(mt);
		task_rq_unlock(rq, &flags);
		wake_up_process(mt);
		put_task_struct(mt);
		wait_for_completion(&req.done);
2107

L
Linus Torvalds 已提交
2108 2109 2110 2111 2112 2113 2114
		return;
	}
out:
	task_rq_unlock(rq, &flags);
}

/*
N
Nick Piggin 已提交
2115 2116
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
L
Linus Torvalds 已提交
2117 2118 2119 2120
 */
void sched_exec(void)
{
	int new_cpu, this_cpu = get_cpu();
N
Nick Piggin 已提交
2121
	new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
L
Linus Torvalds 已提交
2122
	put_cpu();
N
Nick Piggin 已提交
2123 2124
	if (new_cpu != this_cpu)
		sched_migrate_task(current, new_cpu);
L
Linus Torvalds 已提交
2125 2126 2127 2128 2129 2130
}

/*
 * pull_task - move a task from a remote runqueue to the local runqueue.
 * Both runqueues must be locked.
 */
I
Ingo Molnar 已提交
2131 2132
static void pull_task(struct rq *src_rq, struct task_struct *p,
		      struct rq *this_rq, int this_cpu)
L
Linus Torvalds 已提交
2133
{
I
Ingo Molnar 已提交
2134 2135
	update_rq_clock(src_rq);
	deactivate_task(src_rq, p, 0, src_rq->clock);
L
Linus Torvalds 已提交
2136
	set_task_cpu(p, this_cpu);
I
Ingo Molnar 已提交
2137
	activate_task(this_rq, p, 0);
L
Linus Torvalds 已提交
2138 2139 2140 2141
	/*
	 * Note that idle threads have a prio of MAX_PRIO, for this test
	 * to be always true for them.
	 */
I
Ingo Molnar 已提交
2142
	check_preempt_curr(this_rq, p);
L
Linus Torvalds 已提交
2143 2144 2145 2146 2147
}

/*
 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
 */
2148
static
2149
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
I
Ingo Molnar 已提交
2150
		     struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2151
		     int *all_pinned)
L
Linus Torvalds 已提交
2152 2153 2154 2155 2156 2157 2158 2159 2160
{
	/*
	 * We do not migrate tasks that are:
	 * 1) running (obviously), or
	 * 2) cannot be migrated to this CPU due to cpus_allowed, or
	 * 3) are cache-hot on their current CPU.
	 */
	if (!cpu_isset(this_cpu, p->cpus_allowed))
		return 0;
2161 2162 2163 2164
	*all_pinned = 0;

	if (task_running(rq, p))
		return 0;
L
Linus Torvalds 已提交
2165 2166

	/*
I
Ingo Molnar 已提交
2167
	 * Aggressive migration if too many balance attempts have failed:
L
Linus Torvalds 已提交
2168
	 */
I
Ingo Molnar 已提交
2169
	if (sd->nr_balance_failed > sd->cache_nice_tries)
L
Linus Torvalds 已提交
2170 2171 2172 2173 2174
		return 1;

	return 1;
}

I
Ingo Molnar 已提交
2175
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2176
		      unsigned long max_nr_move, unsigned long max_load_move,
I
Ingo Molnar 已提交
2177
		      struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2178
		      int *all_pinned, unsigned long *load_moved,
2179
		      int *this_best_prio, struct rq_iterator *iterator)
L
Linus Torvalds 已提交
2180
{
I
Ingo Molnar 已提交
2181 2182 2183
	int pulled = 0, pinned = 0, skip_for_load;
	struct task_struct *p;
	long rem_load_move = max_load_move;
L
Linus Torvalds 已提交
2184

2185
	if (max_nr_move == 0 || max_load_move == 0)
L
Linus Torvalds 已提交
2186 2187
		goto out;

2188 2189
	pinned = 1;

L
Linus Torvalds 已提交
2190
	/*
I
Ingo Molnar 已提交
2191
	 * Start the load-balancing iterator:
L
Linus Torvalds 已提交
2192
	 */
I
Ingo Molnar 已提交
2193 2194 2195
	p = iterator->start(iterator->arg);
next:
	if (!p)
L
Linus Torvalds 已提交
2196
		goto out;
2197 2198 2199 2200 2201
	/*
	 * To help distribute high priority tasks accross CPUs we don't
	 * skip a task if it will be the highest priority task (i.e. smallest
	 * prio value) on its new queue regardless of its load weight
	 */
I
Ingo Molnar 已提交
2202 2203
	skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
							 SCHED_LOAD_SCALE_FUZZ;
2204
	if ((skip_for_load && p->prio >= *this_best_prio) ||
I
Ingo Molnar 已提交
2205 2206 2207
	    !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2208 2209
	}

I
Ingo Molnar 已提交
2210
	pull_task(busiest, p, this_rq, this_cpu);
L
Linus Torvalds 已提交
2211
	pulled++;
I
Ingo Molnar 已提交
2212
	rem_load_move -= p->se.load.weight;
L
Linus Torvalds 已提交
2213

2214 2215 2216 2217 2218
	/*
	 * We only want to steal up to the prescribed number of tasks
	 * and the prescribed amount of weighted load.
	 */
	if (pulled < max_nr_move && rem_load_move > 0) {
2219 2220
		if (p->prio < *this_best_prio)
			*this_best_prio = p->prio;
I
Ingo Molnar 已提交
2221 2222
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2223 2224 2225 2226 2227 2228 2229 2230
	}
out:
	/*
	 * Right now, this is the only place pull_task() is called,
	 * so we can safely collect pull_task() stats here rather than
	 * inside pull_task().
	 */
	schedstat_add(sd, lb_gained[idle], pulled);
2231 2232 2233

	if (all_pinned)
		*all_pinned = pinned;
I
Ingo Molnar 已提交
2234
	*load_moved = max_load_move - rem_load_move;
L
Linus Torvalds 已提交
2235 2236 2237
	return pulled;
}

I
Ingo Molnar 已提交
2238
/*
P
Peter Williams 已提交
2239 2240 2241
 * move_tasks tries to move up to max_load_move weighted load from busiest to
 * this_rq, as part of a balancing operation within domain "sd".
 * Returns 1 if successful and 0 otherwise.
I
Ingo Molnar 已提交
2242 2243 2244 2245
 *
 * Called with both runqueues locked.
 */
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
P
Peter Williams 已提交
2246
		      unsigned long max_load_move,
I
Ingo Molnar 已提交
2247 2248 2249 2250
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned)
{
	struct sched_class *class = sched_class_highest;
P
Peter Williams 已提交
2251
	unsigned long total_load_moved = 0;
2252
	int this_best_prio = this_rq->curr->prio;
I
Ingo Molnar 已提交
2253 2254

	do {
P
Peter Williams 已提交
2255 2256 2257
		total_load_moved +=
			class->load_balance(this_rq, this_cpu, busiest,
				ULONG_MAX, max_load_move - total_load_moved,
2258
				sd, idle, all_pinned, &this_best_prio);
I
Ingo Molnar 已提交
2259
		class = class->next;
P
Peter Williams 已提交
2260
	} while (class && max_load_move > total_load_moved);
I
Ingo Molnar 已提交
2261

P
Peter Williams 已提交
2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275
	return total_load_moved > 0;
}

/*
 * move_one_task tries to move exactly one task from busiest to this_rq, as
 * part of active balancing operations within "domain".
 * Returns 1 if successful and 0 otherwise.
 *
 * Called with both runqueues locked.
 */
static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
			 struct sched_domain *sd, enum cpu_idle_type idle)
{
	struct sched_class *class;
2276
	int this_best_prio = MAX_PRIO;
P
Peter Williams 已提交
2277 2278 2279

	for (class = sched_class_highest; class; class = class->next)
		if (class->load_balance(this_rq, this_cpu, busiest,
2280 2281
					1, ULONG_MAX, sd, idle, NULL,
					&this_best_prio))
P
Peter Williams 已提交
2282 2283 2284
			return 1;

	return 0;
I
Ingo Molnar 已提交
2285 2286
}

L
Linus Torvalds 已提交
2287 2288
/*
 * find_busiest_group finds and returns the busiest CPU group within the
2289 2290
 * domain. It calculates and returns the amount of weighted load which
 * should be moved to restore balance via the imbalance parameter.
L
Linus Torvalds 已提交
2291 2292 2293
 */
static struct sched_group *
find_busiest_group(struct sched_domain *sd, int this_cpu,
I
Ingo Molnar 已提交
2294 2295
		   unsigned long *imbalance, enum cpu_idle_type idle,
		   int *sd_idle, cpumask_t *cpus, int *balance)
L
Linus Torvalds 已提交
2296 2297 2298
{
	struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
	unsigned long max_load, avg_load, total_load, this_load, total_pwr;
2299
	unsigned long max_pull;
2300 2301
	unsigned long busiest_load_per_task, busiest_nr_running;
	unsigned long this_load_per_task, this_nr_running;
N
Nick Piggin 已提交
2302
	int load_idx;
2303 2304 2305 2306 2307 2308
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
	int power_savings_balance = 1;
	unsigned long leader_nr_running = 0, min_load_per_task = 0;
	unsigned long min_nr_running = ULONG_MAX;
	struct sched_group *group_min = NULL, *group_leader = NULL;
#endif
L
Linus Torvalds 已提交
2309 2310

	max_load = this_load = total_load = total_pwr = 0;
2311 2312
	busiest_load_per_task = busiest_nr_running = 0;
	this_load_per_task = this_nr_running = 0;
I
Ingo Molnar 已提交
2313
	if (idle == CPU_NOT_IDLE)
N
Nick Piggin 已提交
2314
		load_idx = sd->busy_idx;
I
Ingo Molnar 已提交
2315
	else if (idle == CPU_NEWLY_IDLE)
N
Nick Piggin 已提交
2316 2317 2318
		load_idx = sd->newidle_idx;
	else
		load_idx = sd->idle_idx;
L
Linus Torvalds 已提交
2319 2320

	do {
2321
		unsigned long load, group_capacity;
L
Linus Torvalds 已提交
2322 2323
		int local_group;
		int i;
2324
		unsigned int balance_cpu = -1, first_idle_cpu = 0;
2325
		unsigned long sum_nr_running, sum_weighted_load;
L
Linus Torvalds 已提交
2326 2327 2328

		local_group = cpu_isset(this_cpu, group->cpumask);

2329 2330 2331
		if (local_group)
			balance_cpu = first_cpu(group->cpumask);

L
Linus Torvalds 已提交
2332
		/* Tally up the load of all CPUs in the group */
2333
		sum_weighted_load = sum_nr_running = avg_load = 0;
L
Linus Torvalds 已提交
2334 2335

		for_each_cpu_mask(i, group->cpumask) {
2336 2337 2338 2339 2340 2341
			struct rq *rq;

			if (!cpu_isset(i, *cpus))
				continue;

			rq = cpu_rq(i);
2342

2343
			if (*sd_idle && rq->nr_running)
N
Nick Piggin 已提交
2344 2345
				*sd_idle = 0;

L
Linus Torvalds 已提交
2346
			/* Bias balancing toward cpus of our domain */
2347 2348 2349 2350 2351 2352
			if (local_group) {
				if (idle_cpu(i) && !first_idle_cpu) {
					first_idle_cpu = 1;
					balance_cpu = i;
				}

N
Nick Piggin 已提交
2353
				load = target_load(i, load_idx);
2354
			} else
N
Nick Piggin 已提交
2355
				load = source_load(i, load_idx);
L
Linus Torvalds 已提交
2356 2357

			avg_load += load;
2358
			sum_nr_running += rq->nr_running;
I
Ingo Molnar 已提交
2359
			sum_weighted_load += weighted_cpuload(i);
L
Linus Torvalds 已提交
2360 2361
		}

2362 2363 2364
		/*
		 * First idle cpu or the first cpu(busiest) in this sched group
		 * is eligible for doing load balancing at this and above
2365 2366
		 * domains. In the newly idle case, we will allow all the cpu's
		 * to do the newly idle load balance.
2367
		 */
2368 2369
		if (idle != CPU_NEWLY_IDLE && local_group &&
		    balance_cpu != this_cpu && balance) {
2370 2371 2372 2373
			*balance = 0;
			goto ret;
		}

L
Linus Torvalds 已提交
2374
		total_load += avg_load;
2375
		total_pwr += group->__cpu_power;
L
Linus Torvalds 已提交
2376 2377

		/* Adjust by relative CPU power of the group */
2378 2379
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2380

2381
		group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
2382

L
Linus Torvalds 已提交
2383 2384 2385
		if (local_group) {
			this_load = avg_load;
			this = group;
2386 2387 2388
			this_nr_running = sum_nr_running;
			this_load_per_task = sum_weighted_load;
		} else if (avg_load > max_load &&
2389
			   sum_nr_running > group_capacity) {
L
Linus Torvalds 已提交
2390 2391
			max_load = avg_load;
			busiest = group;
2392 2393
			busiest_nr_running = sum_nr_running;
			busiest_load_per_task = sum_weighted_load;
L
Linus Torvalds 已提交
2394
		}
2395 2396 2397 2398 2399 2400

#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
		/*
		 * Busy processors will not participate in power savings
		 * balance.
		 */
I
Ingo Molnar 已提交
2401 2402 2403
		if (idle == CPU_NOT_IDLE ||
				!(sd->flags & SD_POWERSAVINGS_BALANCE))
			goto group_next;
2404 2405 2406 2407 2408 2409 2410 2411 2412

		/*
		 * If the local group is idle or completely loaded
		 * no need to do power savings balance at this domain
		 */
		if (local_group && (this_nr_running >= group_capacity ||
				    !this_nr_running))
			power_savings_balance = 0;

I
Ingo Molnar 已提交
2413
		/*
2414 2415
		 * If a group is already running at full capacity or idle,
		 * don't include that group in power savings calculations
I
Ingo Molnar 已提交
2416 2417
		 */
		if (!power_savings_balance || sum_nr_running >= group_capacity
2418
		    || !sum_nr_running)
I
Ingo Molnar 已提交
2419
			goto group_next;
2420

I
Ingo Molnar 已提交
2421
		/*
2422
		 * Calculate the group which has the least non-idle load.
I
Ingo Molnar 已提交
2423 2424 2425 2426 2427
		 * This is the group from where we need to pick up the load
		 * for saving power
		 */
		if ((sum_nr_running < min_nr_running) ||
		    (sum_nr_running == min_nr_running &&
2428 2429
		     first_cpu(group->cpumask) <
		     first_cpu(group_min->cpumask))) {
I
Ingo Molnar 已提交
2430 2431
			group_min = group;
			min_nr_running = sum_nr_running;
2432 2433
			min_load_per_task = sum_weighted_load /
						sum_nr_running;
I
Ingo Molnar 已提交
2434
		}
2435

I
Ingo Molnar 已提交
2436
		/*
2437
		 * Calculate the group which is almost near its
I
Ingo Molnar 已提交
2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448
		 * capacity but still has some space to pick up some load
		 * from other group and save more power
		 */
		if (sum_nr_running <= group_capacity - 1) {
			if (sum_nr_running > leader_nr_running ||
			    (sum_nr_running == leader_nr_running &&
			     first_cpu(group->cpumask) >
			      first_cpu(group_leader->cpumask))) {
				group_leader = group;
				leader_nr_running = sum_nr_running;
			}
2449
		}
2450 2451
group_next:
#endif
L
Linus Torvalds 已提交
2452 2453 2454
		group = group->next;
	} while (group != sd->groups);

2455
	if (!busiest || this_load >= max_load || busiest_nr_running == 0)
L
Linus Torvalds 已提交
2456 2457 2458 2459 2460 2461 2462 2463
		goto out_balanced;

	avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;

	if (this_load >= avg_load ||
			100*max_load <= sd->imbalance_pct*this_load)
		goto out_balanced;

2464
	busiest_load_per_task /= busiest_nr_running;
L
Linus Torvalds 已提交
2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475
	/*
	 * We're trying to get all the cpus to the average_load, so we don't
	 * want to push ourselves above the average load, nor do we wish to
	 * reduce the max loaded cpu below the average load, as either of these
	 * actions would just result in more rebalancing later, and ping-pong
	 * tasks around. Thus we look for the minimum possible imbalance.
	 * Negative imbalances (*we* are more loaded than anyone else) will
	 * be counted as no imbalance for these purposes -- we can't fix that
	 * by pulling tasks to us.  Be careful of negative numbers as they'll
	 * appear as very large values with unsigned longs.
	 */
2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487
	if (max_load <= busiest_load_per_task)
		goto out_balanced;

	/*
	 * In the presence of smp nice balancing, certain scenarios can have
	 * max load less than avg load(as we skip the groups at or below
	 * its cpu_power, while calculating max_load..)
	 */
	if (max_load < avg_load) {
		*imbalance = 0;
		goto small_imbalance;
	}
2488 2489

	/* Don't want to pull so many tasks that a group would go idle */
2490
	max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
2491

L
Linus Torvalds 已提交
2492
	/* How much load to actually move to equalise the imbalance */
2493 2494
	*imbalance = min(max_pull * busiest->__cpu_power,
				(avg_load - this_load) * this->__cpu_power)
L
Linus Torvalds 已提交
2495 2496
			/ SCHED_LOAD_SCALE;

2497 2498 2499 2500 2501 2502
	/*
	 * if *imbalance is less than the average load per runnable task
	 * there is no gaurantee that any tasks will be moved so we'll have
	 * a think about bumping its value to force at least one task to be
	 * moved
	 */
I
Ingo Molnar 已提交
2503
	if (*imbalance + SCHED_LOAD_SCALE_FUZZ < busiest_load_per_task/2) {
2504
		unsigned long tmp, pwr_now, pwr_move;
2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
		unsigned int imbn;

small_imbalance:
		pwr_move = pwr_now = 0;
		imbn = 2;
		if (this_nr_running) {
			this_load_per_task /= this_nr_running;
			if (busiest_load_per_task > this_load_per_task)
				imbn = 1;
		} else
			this_load_per_task = SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
2516

I
Ingo Molnar 已提交
2517 2518
		if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >=
					busiest_load_per_task * imbn) {
2519
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2520 2521 2522 2523 2524 2525 2526 2527 2528
			return busiest;
		}

		/*
		 * OK, we don't have enough imbalance to justify moving tasks,
		 * however we may be able to increase total CPU power used by
		 * moving them.
		 */

2529 2530 2531 2532
		pwr_now += busiest->__cpu_power *
				min(busiest_load_per_task, max_load);
		pwr_now += this->__cpu_power *
				min(this_load_per_task, this_load);
L
Linus Torvalds 已提交
2533 2534 2535
		pwr_now /= SCHED_LOAD_SCALE;

		/* Amount of load we'd subtract */
2536 2537
		tmp = sg_div_cpu_power(busiest,
				busiest_load_per_task * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2538
		if (max_load > tmp)
2539
			pwr_move += busiest->__cpu_power *
2540
				min(busiest_load_per_task, max_load - tmp);
L
Linus Torvalds 已提交
2541 2542

		/* Amount of load we'd add */
2543
		if (max_load * busiest->__cpu_power <
2544
				busiest_load_per_task * SCHED_LOAD_SCALE)
2545 2546
			tmp = sg_div_cpu_power(this,
					max_load * busiest->__cpu_power);
L
Linus Torvalds 已提交
2547
		else
2548 2549 2550 2551
			tmp = sg_div_cpu_power(this,
				busiest_load_per_task * SCHED_LOAD_SCALE);
		pwr_move += this->__cpu_power *
				min(this_load_per_task, this_load + tmp);
L
Linus Torvalds 已提交
2552 2553 2554 2555 2556 2557
		pwr_move /= SCHED_LOAD_SCALE;

		/* Move if we gain throughput */
		if (pwr_move <= pwr_now)
			goto out_balanced;

2558
		*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2559 2560 2561 2562 2563
	}

	return busiest;

out_balanced:
2564
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
2565
	if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
2566
		goto ret;
L
Linus Torvalds 已提交
2567

2568 2569 2570 2571 2572
	if (this == group_leader && group_leader != group_min) {
		*imbalance = min_load_per_task;
		return group_min;
	}
#endif
2573
ret:
L
Linus Torvalds 已提交
2574 2575 2576 2577 2578 2579 2580
	*imbalance = 0;
	return NULL;
}

/*
 * find_busiest_queue - find the busiest runqueue among the cpus in group.
 */
2581
static struct rq *
I
Ingo Molnar 已提交
2582
find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
2583
		   unsigned long imbalance, cpumask_t *cpus)
L
Linus Torvalds 已提交
2584
{
2585
	struct rq *busiest = NULL, *rq;
2586
	unsigned long max_load = 0;
L
Linus Torvalds 已提交
2587 2588 2589
	int i;

	for_each_cpu_mask(i, group->cpumask) {
I
Ingo Molnar 已提交
2590
		unsigned long wl;
2591 2592 2593 2594

		if (!cpu_isset(i, *cpus))
			continue;

2595
		rq = cpu_rq(i);
I
Ingo Molnar 已提交
2596
		wl = weighted_cpuload(i);
2597

I
Ingo Molnar 已提交
2598
		if (rq->nr_running == 1 && wl > imbalance)
2599
			continue;
L
Linus Torvalds 已提交
2600

I
Ingo Molnar 已提交
2601 2602
		if (wl > max_load) {
			max_load = wl;
2603
			busiest = rq;
L
Linus Torvalds 已提交
2604 2605 2606 2607 2608 2609
		}
	}

	return busiest;
}

2610 2611 2612 2613 2614 2615
/*
 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
 * so long as it is large enough.
 */
#define MAX_PINNED_INTERVAL	512

L
Linus Torvalds 已提交
2616 2617 2618 2619
/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
2620
static int load_balance(int this_cpu, struct rq *this_rq,
I
Ingo Molnar 已提交
2621
			struct sched_domain *sd, enum cpu_idle_type idle,
2622
			int *balance)
L
Linus Torvalds 已提交
2623
{
P
Peter Williams 已提交
2624
	int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
L
Linus Torvalds 已提交
2625 2626
	struct sched_group *group;
	unsigned long imbalance;
2627
	struct rq *busiest;
2628
	cpumask_t cpus = CPU_MASK_ALL;
2629
	unsigned long flags;
N
Nick Piggin 已提交
2630

2631 2632 2633
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
I
Ingo Molnar 已提交
2634
	 * let the state of idle sibling percolate up as CPU_IDLE, instead of
I
Ingo Molnar 已提交
2635
	 * portraying it as CPU_NOT_IDLE.
2636
	 */
I
Ingo Molnar 已提交
2637
	if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
2638
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2639
		sd_idle = 1;
L
Linus Torvalds 已提交
2640 2641 2642

	schedstat_inc(sd, lb_cnt[idle]);

2643 2644
redo:
	group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
2645 2646
				   &cpus, balance);

2647
	if (*balance == 0)
2648 2649
		goto out_balanced;

L
Linus Torvalds 已提交
2650 2651 2652 2653 2654
	if (!group) {
		schedstat_inc(sd, lb_nobusyg[idle]);
		goto out_balanced;
	}

2655
	busiest = find_busiest_queue(group, idle, imbalance, &cpus);
L
Linus Torvalds 已提交
2656 2657 2658 2659 2660
	if (!busiest) {
		schedstat_inc(sd, lb_nobusyq[idle]);
		goto out_balanced;
	}

N
Nick Piggin 已提交
2661
	BUG_ON(busiest == this_rq);
L
Linus Torvalds 已提交
2662 2663 2664

	schedstat_add(sd, lb_imbalance[idle], imbalance);

P
Peter Williams 已提交
2665
	ld_moved = 0;
L
Linus Torvalds 已提交
2666 2667 2668 2669
	if (busiest->nr_running > 1) {
		/*
		 * Attempt to move tasks. If find_busiest_group has found
		 * an imbalance but busiest->nr_running <= 1, the group is
P
Peter Williams 已提交
2670
		 * still unbalanced. ld_moved simply stays zero, so it is
L
Linus Torvalds 已提交
2671 2672
		 * correctly treated as an imbalance.
		 */
2673
		local_irq_save(flags);
N
Nick Piggin 已提交
2674
		double_rq_lock(this_rq, busiest);
P
Peter Williams 已提交
2675
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2676
				      imbalance, sd, idle, &all_pinned);
N
Nick Piggin 已提交
2677
		double_rq_unlock(this_rq, busiest);
2678
		local_irq_restore(flags);
2679

2680 2681 2682
		/*
		 * some other cpu did the load balance for us.
		 */
P
Peter Williams 已提交
2683
		if (ld_moved && this_cpu != smp_processor_id())
2684 2685
			resched_cpu(this_cpu);

2686
		/* All tasks on this runqueue were pinned by CPU affinity */
2687 2688 2689 2690
		if (unlikely(all_pinned)) {
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
2691
			goto out_balanced;
2692
		}
L
Linus Torvalds 已提交
2693
	}
2694

P
Peter Williams 已提交
2695
	if (!ld_moved) {
L
Linus Torvalds 已提交
2696 2697 2698 2699 2700
		schedstat_inc(sd, lb_failed[idle]);
		sd->nr_balance_failed++;

		if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {

2701
			spin_lock_irqsave(&busiest->lock, flags);
2702 2703 2704 2705 2706

			/* don't kick the migration_thread, if the curr
			 * task on busiest cpu can't be moved to this_cpu
			 */
			if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
2707
				spin_unlock_irqrestore(&busiest->lock, flags);
2708 2709 2710 2711
				all_pinned = 1;
				goto out_one_pinned;
			}

L
Linus Torvalds 已提交
2712 2713 2714
			if (!busiest->active_balance) {
				busiest->active_balance = 1;
				busiest->push_cpu = this_cpu;
2715
				active_balance = 1;
L
Linus Torvalds 已提交
2716
			}
2717
			spin_unlock_irqrestore(&busiest->lock, flags);
2718
			if (active_balance)
L
Linus Torvalds 已提交
2719 2720 2721 2722 2723 2724
				wake_up_process(busiest->migration_thread);

			/*
			 * We've kicked active balancing, reset the failure
			 * counter.
			 */
2725
			sd->nr_balance_failed = sd->cache_nice_tries+1;
L
Linus Torvalds 已提交
2726
		}
2727
	} else
L
Linus Torvalds 已提交
2728 2729
		sd->nr_balance_failed = 0;

2730
	if (likely(!active_balance)) {
L
Linus Torvalds 已提交
2731 2732
		/* We were unbalanced, so reset the balancing interval */
		sd->balance_interval = sd->min_interval;
2733 2734 2735 2736 2737 2738 2739 2740 2741
	} else {
		/*
		 * If we've begun active balancing, start to back off. This
		 * case may not be covered by the all_pinned logic if there
		 * is only 1 task on the busy runqueue (because we don't call
		 * move_tasks).
		 */
		if (sd->balance_interval < sd->max_interval)
			sd->balance_interval *= 2;
L
Linus Torvalds 已提交
2742 2743
	}

P
Peter Williams 已提交
2744
	if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2745
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2746
		return -1;
P
Peter Williams 已提交
2747
	return ld_moved;
L
Linus Torvalds 已提交
2748 2749 2750 2751

out_balanced:
	schedstat_inc(sd, lb_balanced[idle]);

2752
	sd->nr_balance_failed = 0;
2753 2754

out_one_pinned:
L
Linus Torvalds 已提交
2755
	/* tune up the balancing interval */
2756 2757
	if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
			(sd->balance_interval < sd->max_interval))
L
Linus Torvalds 已提交
2758 2759
		sd->balance_interval *= 2;

2760
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2761
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2762
		return -1;
L
Linus Torvalds 已提交
2763 2764 2765 2766 2767 2768 2769
	return 0;
}

/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 *
I
Ingo Molnar 已提交
2770
 * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
L
Linus Torvalds 已提交
2771 2772
 * this_rq is locked.
 */
2773
static int
2774
load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
L
Linus Torvalds 已提交
2775 2776
{
	struct sched_group *group;
2777
	struct rq *busiest = NULL;
L
Linus Torvalds 已提交
2778
	unsigned long imbalance;
P
Peter Williams 已提交
2779
	int ld_moved = 0;
N
Nick Piggin 已提交
2780
	int sd_idle = 0;
2781
	int all_pinned = 0;
2782
	cpumask_t cpus = CPU_MASK_ALL;
N
Nick Piggin 已提交
2783

2784 2785 2786 2787
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
	 * let the state of idle sibling percolate up as IDLE, instead of
I
Ingo Molnar 已提交
2788
	 * portraying it as CPU_NOT_IDLE.
2789 2790 2791
	 */
	if (sd->flags & SD_SHARE_CPUPOWER &&
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2792
		sd_idle = 1;
L
Linus Torvalds 已提交
2793

I
Ingo Molnar 已提交
2794
	schedstat_inc(sd, lb_cnt[CPU_NEWLY_IDLE]);
2795
redo:
I
Ingo Molnar 已提交
2796
	group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
2797
				   &sd_idle, &cpus, NULL);
L
Linus Torvalds 已提交
2798
	if (!group) {
I
Ingo Molnar 已提交
2799
		schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
2800
		goto out_balanced;
L
Linus Torvalds 已提交
2801 2802
	}

I
Ingo Molnar 已提交
2803
	busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance,
2804
				&cpus);
N
Nick Piggin 已提交
2805
	if (!busiest) {
I
Ingo Molnar 已提交
2806
		schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
2807
		goto out_balanced;
L
Linus Torvalds 已提交
2808 2809
	}

N
Nick Piggin 已提交
2810 2811
	BUG_ON(busiest == this_rq);

I
Ingo Molnar 已提交
2812
	schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
2813

P
Peter Williams 已提交
2814
	ld_moved = 0;
2815 2816 2817
	if (busiest->nr_running > 1) {
		/* Attempt to move tasks */
		double_lock_balance(this_rq, busiest);
P
Peter Williams 已提交
2818
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2819 2820
					imbalance, sd, CPU_NEWLY_IDLE,
					&all_pinned);
2821
		spin_unlock(&busiest->lock);
2822

2823
		if (unlikely(all_pinned)) {
2824 2825 2826 2827
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
		}
2828 2829
	}

P
Peter Williams 已提交
2830
	if (!ld_moved) {
I
Ingo Molnar 已提交
2831
		schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
2832 2833
		if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
		    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2834 2835
			return -1;
	} else
2836
		sd->nr_balance_failed = 0;
L
Linus Torvalds 已提交
2837

P
Peter Williams 已提交
2838
	return ld_moved;
2839 2840

out_balanced:
I
Ingo Molnar 已提交
2841
	schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
2842
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2843
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2844
		return -1;
2845
	sd->nr_balance_failed = 0;
2846

2847
	return 0;
L
Linus Torvalds 已提交
2848 2849 2850 2851 2852 2853
}

/*
 * idle_balance is called by schedule() if this_cpu is about to become
 * idle. Attempts to pull tasks from other CPUs.
 */
2854
static void idle_balance(int this_cpu, struct rq *this_rq)
L
Linus Torvalds 已提交
2855 2856
{
	struct sched_domain *sd;
I
Ingo Molnar 已提交
2857 2858
	int pulled_task = -1;
	unsigned long next_balance = jiffies + HZ;
L
Linus Torvalds 已提交
2859 2860

	for_each_domain(this_cpu, sd) {
2861 2862 2863 2864 2865 2866
		unsigned long interval;

		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		if (sd->flags & SD_BALANCE_NEWIDLE)
2867
			/* If we've pulled tasks over stop searching: */
2868
			pulled_task = load_balance_newidle(this_cpu,
2869 2870 2871 2872 2873 2874 2875
								this_rq, sd);

		interval = msecs_to_jiffies(sd->balance_interval);
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
		if (pulled_task)
			break;
L
Linus Torvalds 已提交
2876
	}
I
Ingo Molnar 已提交
2877
	if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
2878 2879 2880 2881 2882
		/*
		 * We are going idle. next_balance may be set based on
		 * a busy processor. So reset next_balance.
		 */
		this_rq->next_balance = next_balance;
I
Ingo Molnar 已提交
2883
	}
L
Linus Torvalds 已提交
2884 2885 2886 2887 2888 2889 2890 2891 2892 2893
}

/*
 * active_load_balance is run by migration threads. It pushes running tasks
 * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
 * running on each physical CPU where possible, and avoids physical /
 * logical imbalances.
 *
 * Called with busiest_rq locked.
 */
2894
static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
L
Linus Torvalds 已提交
2895
{
2896
	int target_cpu = busiest_rq->push_cpu;
2897 2898
	struct sched_domain *sd;
	struct rq *target_rq;
2899

2900
	/* Is there any task to move? */
2901 2902 2903 2904
	if (busiest_rq->nr_running <= 1)
		return;

	target_rq = cpu_rq(target_cpu);
L
Linus Torvalds 已提交
2905 2906

	/*
2907 2908 2909
	 * This condition is "impossible", if it occurs
	 * we need to fix it.  Originally reported by
	 * Bjorn Helgaas on a 128-cpu setup.
L
Linus Torvalds 已提交
2910
	 */
2911
	BUG_ON(busiest_rq == target_rq);
L
Linus Torvalds 已提交
2912

2913 2914 2915 2916
	/* move a task from busiest_rq to target_rq */
	double_lock_balance(busiest_rq, target_rq);

	/* Search for an sd spanning us and the target CPU. */
2917
	for_each_domain(target_cpu, sd) {
2918
		if ((sd->flags & SD_LOAD_BALANCE) &&
2919
		    cpu_isset(busiest_cpu, sd->span))
2920
				break;
2921
	}
2922

2923 2924
	if (likely(sd)) {
		schedstat_inc(sd, alb_cnt);
2925

P
Peter Williams 已提交
2926 2927
		if (move_one_task(target_rq, target_cpu, busiest_rq,
				  sd, CPU_IDLE))
2928 2929 2930 2931
			schedstat_inc(sd, alb_pushed);
		else
			schedstat_inc(sd, alb_failed);
	}
2932
	spin_unlock(&target_rq->lock);
L
Linus Torvalds 已提交
2933 2934
}

2935 2936 2937 2938 2939 2940 2941 2942 2943
#ifdef CONFIG_NO_HZ
static struct {
	atomic_t load_balancer;
	cpumask_t  cpu_mask;
} nohz ____cacheline_aligned = {
	.load_balancer = ATOMIC_INIT(-1),
	.cpu_mask = CPU_MASK_NONE,
};

2944
/*
2945 2946 2947 2948 2949 2950 2951 2952 2953 2954
 * This routine will try to nominate the ilb (idle load balancing)
 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
 * load balancing on behalf of all those cpus. If all the cpus in the system
 * go into this tickless mode, then there will be no ilb owner (as there is
 * no need for one) and all the cpus will sleep till the next wakeup event
 * arrives...
 *
 * For the ilb owner, tick is not stopped. And this tick will be used
 * for idle load balancing. ilb owner will still be part of
 * nohz.cpu_mask..
2955
 *
2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011
 * While stopping the tick, this cpu will become the ilb owner if there
 * is no other owner. And will be the owner till that cpu becomes busy
 * or if all cpus in the system stop their ticks at which point
 * there is no need for ilb owner.
 *
 * When the ilb owner becomes busy, it nominates another owner, during the
 * next busy scheduler_tick()
 */
int select_nohz_load_balancer(int stop_tick)
{
	int cpu = smp_processor_id();

	if (stop_tick) {
		cpu_set(cpu, nohz.cpu_mask);
		cpu_rq(cpu)->in_nohz_recently = 1;

		/*
		 * If we are going offline and still the leader, give up!
		 */
		if (cpu_is_offline(cpu) &&
		    atomic_read(&nohz.load_balancer) == cpu) {
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
			return 0;
		}

		/* time for ilb owner also to sleep */
		if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
			if (atomic_read(&nohz.load_balancer) == cpu)
				atomic_set(&nohz.load_balancer, -1);
			return 0;
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/* make me the ilb owner */
			if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
				return 1;
		} else if (atomic_read(&nohz.load_balancer) == cpu)
			return 1;
	} else {
		if (!cpu_isset(cpu, nohz.cpu_mask))
			return 0;

		cpu_clear(cpu, nohz.cpu_mask);

		if (atomic_read(&nohz.load_balancer) == cpu)
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
	}
	return 0;
}
#endif

static DEFINE_SPINLOCK(balancing);

/*
3012 3013 3014 3015 3016
 * It checks each scheduling domain to see if it is due to be balanced,
 * and initiates a balancing operation if so.
 *
 * Balancing parameters are set up in arch_init_sched_domains.
 */
I
Ingo Molnar 已提交
3017
static inline void rebalance_domains(int cpu, enum cpu_idle_type idle)
3018
{
3019 3020
	int balance = 1;
	struct rq *rq = cpu_rq(cpu);
3021 3022
	unsigned long interval;
	struct sched_domain *sd;
3023
	/* Earliest time when we have to do rebalance again */
3024
	unsigned long next_balance = jiffies + 60*HZ;
L
Linus Torvalds 已提交
3025

3026
	for_each_domain(cpu, sd) {
L
Linus Torvalds 已提交
3027 3028 3029 3030
		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		interval = sd->balance_interval;
I
Ingo Molnar 已提交
3031
		if (idle != CPU_IDLE)
L
Linus Torvalds 已提交
3032 3033 3034 3035 3036 3037
			interval *= sd->busy_factor;

		/* scale ms to jiffies */
		interval = msecs_to_jiffies(interval);
		if (unlikely(!interval))
			interval = 1;
I
Ingo Molnar 已提交
3038 3039 3040
		if (interval > HZ*NR_CPUS/10)
			interval = HZ*NR_CPUS/10;

L
Linus Torvalds 已提交
3041

3042 3043 3044 3045 3046
		if (sd->flags & SD_SERIALIZE) {
			if (!spin_trylock(&balancing))
				goto out;
		}

3047
		if (time_after_eq(jiffies, sd->last_balance + interval)) {
3048
			if (load_balance(cpu, rq, sd, idle, &balance)) {
3049 3050
				/*
				 * We've pulled tasks over so either we're no
N
Nick Piggin 已提交
3051 3052 3053
				 * longer idle, or one of our SMT siblings is
				 * not idle.
				 */
I
Ingo Molnar 已提交
3054
				idle = CPU_NOT_IDLE;
L
Linus Torvalds 已提交
3055
			}
3056
			sd->last_balance = jiffies;
L
Linus Torvalds 已提交
3057
		}
3058 3059 3060
		if (sd->flags & SD_SERIALIZE)
			spin_unlock(&balancing);
out:
3061 3062
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
3063 3064 3065 3066 3067 3068 3069 3070

		/*
		 * Stop the load balance at this level. There is another
		 * CPU in our sched group which is doing load balancing more
		 * actively.
		 */
		if (!balance)
			break;
L
Linus Torvalds 已提交
3071
	}
3072 3073 3074 3075 3076 3077 3078 3079 3080 3081
	rq->next_balance = next_balance;
}

/*
 * run_rebalance_domains is triggered when needed from the scheduler tick.
 * In CONFIG_NO_HZ case, the idle load balance owner will do the
 * rebalancing for all the cpus for whom scheduler ticks are stopped.
 */
static void run_rebalance_domains(struct softirq_action *h)
{
I
Ingo Molnar 已提交
3082 3083 3084 3085
	int this_cpu = smp_processor_id();
	struct rq *this_rq = cpu_rq(this_cpu);
	enum cpu_idle_type idle = this_rq->idle_at_tick ?
						CPU_IDLE : CPU_NOT_IDLE;
3086

I
Ingo Molnar 已提交
3087
	rebalance_domains(this_cpu, idle);
3088 3089 3090 3091 3092 3093 3094

#ifdef CONFIG_NO_HZ
	/*
	 * If this cpu is the owner for idle load balancing, then do the
	 * balancing on behalf of the other idle cpus whose ticks are
	 * stopped.
	 */
I
Ingo Molnar 已提交
3095 3096
	if (this_rq->idle_at_tick &&
	    atomic_read(&nohz.load_balancer) == this_cpu) {
3097 3098 3099 3100
		cpumask_t cpus = nohz.cpu_mask;
		struct rq *rq;
		int balance_cpu;

I
Ingo Molnar 已提交
3101
		cpu_clear(this_cpu, cpus);
3102 3103 3104 3105 3106 3107 3108 3109 3110
		for_each_cpu_mask(balance_cpu, cpus) {
			/*
			 * If this cpu gets work to do, stop the load balancing
			 * work being done for other cpus. Next load
			 * balancing owner will pick it up.
			 */
			if (need_resched())
				break;

I
Ingo Molnar 已提交
3111
			rebalance_domains(balance_cpu, SCHED_IDLE);
3112 3113

			rq = cpu_rq(balance_cpu);
I
Ingo Molnar 已提交
3114 3115
			if (time_after(this_rq->next_balance, rq->next_balance))
				this_rq->next_balance = rq->next_balance;
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127
		}
	}
#endif
}

/*
 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
 *
 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
 * idle load balancing owner or decide to stop the periodic load balancing,
 * if the whole system is idle.
 */
I
Ingo Molnar 已提交
3128
static inline void trigger_load_balance(struct rq *rq, int cpu)
3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179
{
#ifdef CONFIG_NO_HZ
	/*
	 * If we were in the nohz mode recently and busy at the current
	 * scheduler tick, then check if we need to nominate new idle
	 * load balancer.
	 */
	if (rq->in_nohz_recently && !rq->idle_at_tick) {
		rq->in_nohz_recently = 0;

		if (atomic_read(&nohz.load_balancer) == cpu) {
			cpu_clear(cpu, nohz.cpu_mask);
			atomic_set(&nohz.load_balancer, -1);
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/*
			 * simple selection for now: Nominate the
			 * first cpu in the nohz list to be the next
			 * ilb owner.
			 *
			 * TBD: Traverse the sched domains and nominate
			 * the nearest cpu in the nohz.cpu_mask.
			 */
			int ilb = first_cpu(nohz.cpu_mask);

			if (ilb != NR_CPUS)
				resched_cpu(ilb);
		}
	}

	/*
	 * If this cpu is idle and doing idle load balancing for all the
	 * cpus with ticks stopped, is it time for that to stop?
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
	    cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
		resched_cpu(cpu);
		return;
	}

	/*
	 * If this cpu is idle and the idle load balancing is done by
	 * someone else, then no need raise the SCHED_SOFTIRQ
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
	    cpu_isset(cpu, nohz.cpu_mask))
		return;
#endif
	if (time_after_eq(jiffies, rq->next_balance))
		raise_softirq(SCHED_SOFTIRQ);
L
Linus Torvalds 已提交
3180
}
I
Ingo Molnar 已提交
3181 3182 3183

#else	/* CONFIG_SMP */

L
Linus Torvalds 已提交
3184 3185 3186
/*
 * on UP we do not need to balance between CPUs:
 */
3187
static inline void idle_balance(int cpu, struct rq *rq)
L
Linus Torvalds 已提交
3188 3189
{
}
I
Ingo Molnar 已提交
3190 3191 3192 3193 3194 3195

/* Avoid "used but not defined" warning on UP */
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
3196
		      int *this_best_prio, struct rq_iterator *iterator)
I
Ingo Molnar 已提交
3197 3198 3199 3200 3201 3202
{
	*load_moved = 0;

	return 0;
}

L
Linus Torvalds 已提交
3203 3204 3205 3206 3207 3208 3209
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
3210 3211
 * Return p->sum_exec_runtime plus any more ns on the sched_clock
 * that have not yet been banked in case the task is currently running.
L
Linus Torvalds 已提交
3212
 */
3213
unsigned long long task_sched_runtime(struct task_struct *p)
L
Linus Torvalds 已提交
3214 3215
{
	unsigned long flags;
3216 3217
	u64 ns, delta_exec;
	struct rq *rq;
3218

3219 3220 3221
	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime;
	if (rq->curr == p) {
I
Ingo Molnar 已提交
3222 3223
		update_rq_clock(rq);
		delta_exec = rq->clock - p->se.exec_start;
3224 3225 3226 3227
		if ((s64)delta_exec > 0)
			ns += delta_exec;
	}
	task_rq_unlock(rq, &flags);
3228

L
Linus Torvalds 已提交
3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262
	return ns;
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in user space since the last update
 */
void account_user_time(struct task_struct *p, cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

	p->utime = cputime_add(p->utime, cputime);

	/* Add user time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (TASK_NICE(p) > 0)
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
	else
		cpustat->user = cputime64_add(cpustat->user, tmp);
}

/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
 */
void account_system_time(struct task_struct *p, int hardirq_offset,
			 cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3263
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292
	cputime64_t tmp;

	p->stime = cputime_add(p->stime, cputime);

	/* Add system time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (hardirq_count() - hardirq_offset)
		cpustat->irq = cputime64_add(cpustat->irq, tmp);
	else if (softirq_count())
		cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
	else if (p != rq->idle)
		cpustat->system = cputime64_add(cpustat->system, tmp);
	else if (atomic_read(&rq->nr_iowait) > 0)
		cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
	else
		cpustat->idle = cputime64_add(cpustat->idle, tmp);
	/* Account for system time used */
	acct_update_integrals(p);
}

/*
 * Account for involuntary wait time.
 * @p: the process from which the cpu time has been stolen
 * @steal: the cpu time spent in involuntary wait
 */
void account_steal_time(struct task_struct *p, cputime_t steal)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp = cputime_to_cputime64(steal);
3293
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304

	if (p == rq->idle) {
		p->stime = cputime_add(p->stime, steal);
		if (atomic_read(&rq->nr_iowait) > 0)
			cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
		else
			cpustat->idle = cputime64_add(cpustat->idle, tmp);
	} else
		cpustat->steal = cputime64_add(cpustat->steal, tmp);
}

3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 *
 * It also gets called by the fork code, when changing the parent's
 * timeslices.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
3316 3317 3318
	struct task_struct *curr = rq->curr;

	spin_lock(&rq->lock);
3319
	update_cpu_load(rq);
I
Ingo Molnar 已提交
3320 3321 3322
	if (curr != rq->idle) /* FIXME: needed? */
		curr->sched_class->task_tick(rq, curr);
	spin_unlock(&rq->lock);
3323

3324
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
3325 3326
	rq->idle_at_tick = idle_cpu(cpu);
	trigger_load_balance(rq, cpu);
3327
#endif
L
Linus Torvalds 已提交
3328 3329 3330 3331 3332 3333 3334 3335 3336
}

#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)

void fastcall add_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3337 3338
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
L
Linus Torvalds 已提交
3339 3340 3341 3342
	preempt_count() += val;
	/*
	 * Spinlock count overflowing soon?
	 */
3343 3344
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
L
Linus Torvalds 已提交
3345 3346 3347 3348 3349 3350 3351 3352
}
EXPORT_SYMBOL(add_preempt_count);

void fastcall sub_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3353 3354
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
		return;
L
Linus Torvalds 已提交
3355 3356 3357
	/*
	 * Is the spinlock portion underflowing?
	 */
3358 3359 3360 3361
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;

L
Linus Torvalds 已提交
3362 3363 3364 3365 3366 3367 3368
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
3369
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
3370
 */
I
Ingo Molnar 已提交
3371
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
3372
{
I
Ingo Molnar 已提交
3373 3374 3375 3376 3377 3378 3379
	printk(KERN_ERR "BUG: scheduling while atomic: %s/0x%08x/%d\n",
		prev->comm, preempt_count(), prev->pid);
	debug_show_held_locks(prev);
	if (irqs_disabled())
		print_irqtrace_events(prev);
	dump_stack();
}
L
Linus Torvalds 已提交
3380

I
Ingo Molnar 已提交
3381 3382 3383 3384 3385
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
3386 3387 3388 3389 3390
	/*
	 * Test if we are atomic.  Since do_exit() needs to call into
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
I
Ingo Molnar 已提交
3391 3392 3393
	if (unlikely(in_atomic_preempt_off()) && unlikely(!prev->exit_state))
		__schedule_bug(prev);

L
Linus Torvalds 已提交
3394 3395
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

I
Ingo Molnar 已提交
3396 3397 3398 3399 3400 3401 3402
	schedstat_inc(this_rq(), sched_cnt);
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
3403
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
3404 3405 3406
{
	struct sched_class *class;
	struct task_struct *p;
L
Linus Torvalds 已提交
3407 3408

	/*
I
Ingo Molnar 已提交
3409 3410
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
3411
	 */
I
Ingo Molnar 已提交
3412
	if (likely(rq->nr_running == rq->cfs.nr_running)) {
3413
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
3414 3415
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
3416 3417
	}

I
Ingo Molnar 已提交
3418 3419
	class = sched_class_highest;
	for ( ; ; ) {
3420
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
3421 3422 3423 3424 3425 3426 3427 3428 3429
		if (p)
			return p;
		/*
		 * Will never be NULL as the idle class always
		 * returns a non-NULL p:
		 */
		class = class->next;
	}
}
L
Linus Torvalds 已提交
3430

I
Ingo Molnar 已提交
3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453
/*
 * schedule() is the main scheduler function.
 */
asmlinkage void __sched schedule(void)
{
	struct task_struct *prev, *next;
	long *switch_count;
	struct rq *rq;
	u64 now;
	int cpu;

need_resched:
	preempt_disable();
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
	rcu_qsctr_inc(cpu);
	prev = rq->curr;
	switch_count = &prev->nivcsw;

	release_kernel_lock(prev);
need_resched_nonpreemptible:

	schedule_debug(prev);
L
Linus Torvalds 已提交
3454 3455

	spin_lock_irq(&rq->lock);
I
Ingo Molnar 已提交
3456
	clear_tsk_need_resched(prev);
I
Ingo Molnar 已提交
3457 3458
	__update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
3459 3460 3461

	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
		if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
I
Ingo Molnar 已提交
3462
				unlikely(signal_pending(prev)))) {
L
Linus Torvalds 已提交
3463
			prev->state = TASK_RUNNING;
I
Ingo Molnar 已提交
3464
		} else {
I
Ingo Molnar 已提交
3465
			deactivate_task(rq, prev, 1, now);
L
Linus Torvalds 已提交
3466
		}
I
Ingo Molnar 已提交
3467
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
3468 3469
	}

I
Ingo Molnar 已提交
3470
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
3471 3472
		idle_balance(cpu, rq);

3473
	prev->sched_class->put_prev_task(rq, prev);
3474
	next = pick_next_task(rq, prev);
L
Linus Torvalds 已提交
3475 3476

	sched_info_switch(prev, next);
I
Ingo Molnar 已提交
3477

L
Linus Torvalds 已提交
3478 3479 3480 3481 3482
	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
3483
		context_switch(rq, prev, next); /* unlocks the rq */
L
Linus Torvalds 已提交
3484 3485 3486
	} else
		spin_unlock_irq(&rq->lock);

I
Ingo Molnar 已提交
3487 3488 3489
	if (unlikely(reacquire_kernel_lock(current) < 0)) {
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
3490
		goto need_resched_nonpreemptible;
I
Ingo Molnar 已提交
3491
	}
L
Linus Torvalds 已提交
3492 3493 3494 3495 3496 3497 3498 3499
	preempt_enable_no_resched();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(schedule);

#ifdef CONFIG_PREEMPT
/*
3500
 * this is the entry point to schedule() from in-kernel preemption
L
Linus Torvalds 已提交
3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514
 * off of preempt_enable.  Kernel preemptions off return from interrupt
 * occur there and call schedule directly.
 */
asmlinkage void __sched preempt_schedule(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
	 * we do not want to preempt the current task.  Just return..
	 */
N
Nick Piggin 已提交
3515
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542
		return;

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	schedule();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(preempt_schedule);

/*
3543
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
3555
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584
	BUG_ON(ti->preempt_count || !irqs_disabled());

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	local_irq_enable();
	schedule();
	local_irq_disable();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}

#endif /* CONFIG_PREEMPT */

I
Ingo Molnar 已提交
3585 3586
int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
			  void *key)
L
Linus Torvalds 已提交
3587
{
3588
	return try_to_wake_up(curr->private, mode, sync);
L
Linus Torvalds 已提交
3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606
}
EXPORT_SYMBOL(default_wake_function);

/*
 * The core wakeup function.  Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up.  If it's an exclusive wakeup (nr_exclusive == small +ve
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
 * started to run but is not in state TASK_RUNNING.  try_to_wake_up() returns
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
			     int nr_exclusive, int sync, void *key)
{
	struct list_head *tmp, *next;

	list_for_each_safe(tmp, next, &q->task_list) {
3607 3608 3609
		wait_queue_t *curr = list_entry(tmp, wait_queue_t, task_list);
		unsigned flags = curr->flags;

L
Linus Torvalds 已提交
3610
		if (curr->func(curr, mode, sync, key) &&
3611
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
3612 3613 3614 3615 3616 3617 3618 3619 3620
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
3621
 * @key: is directly passed to the wakeup function
L
Linus Torvalds 已提交
3622 3623
 */
void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
3624
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
void fastcall __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
{
	__wake_up_common(q, mode, 1, 0, NULL);
}

/**
3643
 * __wake_up_sync - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
 */
I
Ingo Molnar 已提交
3655 3656
void fastcall
__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
L
Linus Torvalds 已提交
3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699
{
	unsigned long flags;
	int sync = 1;

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
		sync = 0;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, sync, NULL);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

void fastcall complete(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 1, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

void fastcall complete_all(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 0, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

void fastcall __sched wait_for_completion(struct completion *x)
{
	might_sleep();
3700

L
Linus Torvalds 已提交
3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818
	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
	spin_unlock_irq(&x->wait.lock);
}
EXPORT_SYMBOL(wait_for_completion);

unsigned long fastcall __sched
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_timeout);

int fastcall __sched wait_for_completion_interruptible(struct completion *x)
{
	int ret = 0;

	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				ret = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);

	return ret;
}
EXPORT_SYMBOL(wait_for_completion_interruptible);

unsigned long fastcall __sched
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				timeout = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);

I
Ingo Molnar 已提交
3819 3820 3821 3822 3823
static inline void
sleep_on_head(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irqsave(&q->lock, *flags);
	__add_wait_queue(q, wait);
L
Linus Torvalds 已提交
3824
	spin_unlock(&q->lock);
I
Ingo Molnar 已提交
3825
}
L
Linus Torvalds 已提交
3826

I
Ingo Molnar 已提交
3827 3828 3829 3830 3831 3832 3833
static inline void
sleep_on_tail(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, wait);
	spin_unlock_irqrestore(&q->lock, *flags);
}
L
Linus Torvalds 已提交
3834

I
Ingo Molnar 已提交
3835
void __sched interruptible_sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3836
{
I
Ingo Molnar 已提交
3837 3838 3839 3840
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3841 3842 3843

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3844
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3845
	schedule();
I
Ingo Molnar 已提交
3846
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3847 3848 3849
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
3850
long __sched
I
Ingo Molnar 已提交
3851
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3852
{
I
Ingo Molnar 已提交
3853 3854 3855 3856
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3857 3858 3859

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3860
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3861
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3862
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3863 3864 3865 3866 3867

	return timeout;
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
3868
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3869
{
I
Ingo Molnar 已提交
3870 3871 3872 3873
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3874 3875 3876

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3877
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3878
	schedule();
I
Ingo Molnar 已提交
3879
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3880 3881 3882
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
3883
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3884
{
I
Ingo Molnar 已提交
3885 3886 3887 3888
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3889 3890 3891

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3892
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3893
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3894
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3895 3896 3897 3898 3899

	return timeout;
}
EXPORT_SYMBOL(sleep_on_timeout);

3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
3912
void rt_mutex_setprio(struct task_struct *p, int prio)
3913 3914
{
	unsigned long flags;
I
Ingo Molnar 已提交
3915
	int oldprio, on_rq;
3916
	struct rq *rq;
I
Ingo Molnar 已提交
3917
	u64 now;
3918 3919 3920 3921

	BUG_ON(prio < 0 || prio > MAX_PRIO);

	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3922 3923
	update_rq_clock(rq);
	now = rq->clock;
3924

3925
	oldprio = p->prio;
I
Ingo Molnar 已提交
3926 3927 3928 3929 3930 3931 3932 3933 3934
	on_rq = p->se.on_rq;
	if (on_rq)
		dequeue_task(rq, p, 0, now);

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

3935 3936
	p->prio = prio;

I
Ingo Molnar 已提交
3937 3938
	if (on_rq) {
		enqueue_task(rq, p, 0, now);
3939 3940
		/*
		 * Reschedule if we are currently running on this runqueue and
3941 3942
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
3943
		 */
3944 3945 3946
		if (task_running(rq, p)) {
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
3947 3948 3949
		} else {
			check_preempt_curr(rq, p);
		}
3950 3951 3952 3953 3954 3955
	}
	task_rq_unlock(rq, &flags);
}

#endif

3956
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
3957
{
I
Ingo Molnar 已提交
3958
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
3959
	unsigned long flags;
3960
	struct rq *rq;
I
Ingo Molnar 已提交
3961
	u64 now;
L
Linus Torvalds 已提交
3962 3963 3964 3965 3966 3967 3968 3969

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3970 3971
	update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
3972 3973 3974 3975
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
3976
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
3977
	 */
3978
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
3979 3980 3981
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
I
Ingo Molnar 已提交
3982 3983 3984 3985
	on_rq = p->se.on_rq;
	if (on_rq) {
		dequeue_task(rq, p, 0, now);
		dec_load(rq, p, now);
3986
	}
L
Linus Torvalds 已提交
3987 3988

	p->static_prio = NICE_TO_PRIO(nice);
3989
	set_load_weight(p);
3990 3991 3992
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
3993

I
Ingo Molnar 已提交
3994 3995 3996
	if (on_rq) {
		enqueue_task(rq, p, 0, now);
		inc_load(rq, p, now);
L
Linus Torvalds 已提交
3997
		/*
3998 3999
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
4000
		 */
4001
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
4002 4003 4004 4005 4006 4007 4008
			resched_task(rq->curr);
	}
out_unlock:
	task_rq_unlock(rq, &flags);
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
4009 4010 4011 4012 4013
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
4014
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
4015
{
4016 4017
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
4018

M
Matt Mackall 已提交
4019 4020 4021 4022
	return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
asmlinkage long sys_nice(int increment)
{
4034
	long nice, retval;
L
Linus Torvalds 已提交
4035 4036 4037 4038 4039 4040

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
4041 4042
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
4043 4044 4045 4046 4047 4048 4049 4050 4051
	if (increment > 40)
		increment = 40;

	nice = PRIO_TO_NICE(current->static_prio) + increment;
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
4052 4053 4054
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
4073
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
4074 4075 4076 4077 4078 4079 4080 4081
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
4082
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100
{
	return TASK_NICE(p);
}
EXPORT_SYMBOL_GPL(task_nice);

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
4101
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
4102 4103 4104 4105 4106 4107 4108 4109
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
4110
static inline struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
4111 4112 4113 4114 4115
{
	return pid ? find_task_by_pid(pid) : current;
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
4116 4117
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
4118
{
I
Ingo Molnar 已提交
4119
	BUG_ON(p->se.on_rq);
4120

L
Linus Torvalds 已提交
4121
	p->policy = policy;
I
Ingo Molnar 已提交
4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133
	switch (p->policy) {
	case SCHED_NORMAL:
	case SCHED_BATCH:
	case SCHED_IDLE:
		p->sched_class = &fair_sched_class;
		break;
	case SCHED_FIFO:
	case SCHED_RR:
		p->sched_class = &rt_sched_class;
		break;
	}

L
Linus Torvalds 已提交
4134
	p->rt_priority = prio;
4135 4136 4137
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
4138
	set_load_weight(p);
L
Linus Torvalds 已提交
4139 4140 4141
}

/**
4142
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
L
Linus Torvalds 已提交
4143 4144 4145
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
4146
 *
4147
 * NOTE that the task may be already dead.
L
Linus Torvalds 已提交
4148
 */
I
Ingo Molnar 已提交
4149 4150
int sched_setscheduler(struct task_struct *p, int policy,
		       struct sched_param *param)
L
Linus Torvalds 已提交
4151
{
I
Ingo Molnar 已提交
4152
	int retval, oldprio, oldpolicy = -1, on_rq;
L
Linus Torvalds 已提交
4153
	unsigned long flags;
4154
	struct rq *rq;
L
Linus Torvalds 已提交
4155

4156 4157
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
4158 4159 4160 4161 4162
recheck:
	/* double check policy once rq lock held */
	if (policy < 0)
		policy = oldpolicy = p->policy;
	else if (policy != SCHED_FIFO && policy != SCHED_RR &&
I
Ingo Molnar 已提交
4163 4164
			policy != SCHED_NORMAL && policy != SCHED_BATCH &&
			policy != SCHED_IDLE)
4165
		return -EINVAL;
L
Linus Torvalds 已提交
4166 4167
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
4168 4169
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
4170 4171
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
4172
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
4173
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
4174
		return -EINVAL;
4175
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
4176 4177
		return -EINVAL;

4178 4179 4180 4181
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
	if (!capable(CAP_SYS_NICE)) {
4182
		if (rt_policy(policy)) {
4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198
			unsigned long rlim_rtprio;

			if (!lock_task_sighand(p, &flags))
				return -ESRCH;
			rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
			unlock_task_sighand(p, &flags);

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
I
Ingo Molnar 已提交
4199 4200 4201 4202 4203 4204
		/*
		 * Like positive nice levels, dont allow tasks to
		 * move out of SCHED_IDLE either:
		 */
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
			return -EPERM;
4205

4206 4207 4208 4209 4210
		/* can't change other user's priorities */
		if ((current->euid != p->euid) &&
		    (current->euid != p->uid))
			return -EPERM;
	}
L
Linus Torvalds 已提交
4211 4212 4213 4214

	retval = security_task_setscheduler(p, policy, param);
	if (retval)
		return retval;
4215 4216 4217 4218 4219
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
	 */
	spin_lock_irqsave(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4220 4221 4222 4223
	/*
	 * To be able to change p->policy safely, the apropriate
	 * runqueue lock must be held.
	 */
4224
	rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
4225 4226 4227
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
4228 4229
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4230 4231
		goto recheck;
	}
I
Ingo Molnar 已提交
4232
	on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
4233 4234 4235 4236
	if (on_rq) {
		update_rq_clock(rq);
		deactivate_task(rq, p, 0, rq->clock);
	}
L
Linus Torvalds 已提交
4237
	oldprio = p->prio;
I
Ingo Molnar 已提交
4238 4239 4240
	__setscheduler(rq, p, policy, param->sched_priority);
	if (on_rq) {
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
4241 4242
		/*
		 * Reschedule if we are currently running on this runqueue and
4243 4244
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
L
Linus Torvalds 已提交
4245
		 */
4246 4247 4248
		if (task_running(rq, p)) {
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
4249 4250 4251
		} else {
			check_preempt_curr(rq, p);
		}
L
Linus Torvalds 已提交
4252
	}
4253 4254 4255
	__task_rq_unlock(rq);
	spin_unlock_irqrestore(&p->pi_lock, flags);

4256 4257
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
4258 4259 4260 4261
	return 0;
}
EXPORT_SYMBOL_GPL(sched_setscheduler);

I
Ingo Molnar 已提交
4262 4263
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
4264 4265 4266
{
	struct sched_param lparam;
	struct task_struct *p;
4267
	int retval;
L
Linus Torvalds 已提交
4268 4269 4270 4271 4272

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
4273 4274 4275

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
4276
	p = find_process_by_pid(pid);
4277 4278 4279
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
4280

L
Linus Torvalds 已提交
4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
				       struct sched_param __user *param)
{
4293 4294 4295 4296
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
asmlinkage long sys_sched_getscheduler(pid_t pid)
{
4316
	struct task_struct *p;
L
Linus Torvalds 已提交
4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343
	int retval = -EINVAL;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
			retval = p->policy;
	}
	read_unlock(&tasklist_lock);

out_nounlock:
	return retval;
}

/**
 * sys_sched_getscheduler - get the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
{
	struct sched_param lp;
4344
	struct task_struct *p;
L
Linus Torvalds 已提交
4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378
	int retval = -EINVAL;

	if (!param || pid < 0)
		goto out_nounlock;

	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
	read_unlock(&tasklist_lock);

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

out_nounlock:
	return retval;

out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

long sched_setaffinity(pid_t pid, cpumask_t new_mask)
{
	cpumask_t cpus_allowed;
4379 4380
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
4381

4382
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4383 4384 4385 4386 4387
	read_lock(&tasklist_lock);

	p = find_process_by_pid(pid);
	if (!p) {
		read_unlock(&tasklist_lock);
4388
		mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404
		return -ESRCH;
	}

	/*
	 * It is not safe to call set_cpus_allowed with the
	 * tasklist_lock held.  We will bump the task_struct's
	 * usage count and then drop tasklist_lock.
	 */
	get_task_struct(p);
	read_unlock(&tasklist_lock);

	retval = -EPERM;
	if ((current->euid != p->euid) && (current->euid != p->uid) &&
			!capable(CAP_SYS_NICE))
		goto out_unlock;

4405 4406 4407 4408
	retval = security_task_setscheduler(p, 0, NULL);
	if (retval)
		goto out_unlock;

L
Linus Torvalds 已提交
4409 4410 4411 4412 4413 4414
	cpus_allowed = cpuset_cpus_allowed(p);
	cpus_and(new_mask, new_mask, cpus_allowed);
	retval = set_cpus_allowed(p, new_mask);

out_unlock:
	put_task_struct(p);
4415
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
			     cpumask_t *new_mask)
{
	if (len < sizeof(cpumask_t)) {
		memset(new_mask, 0, sizeof(cpumask_t));
	} else if (len > sizeof(cpumask_t)) {
		len = sizeof(cpumask_t);
	}
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	cpumask_t new_mask;
	int retval;

	retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask);
	if (retval)
		return retval;

	return sched_setaffinity(pid, new_mask);
}

/*
 * Represents all cpu's present in the system
 * In systems capable of hotplug, this map could dynamically grow
 * as new cpu's are detected in the system via any platform specific
 * method, such as ACPI for e.g.
 */

4456
cpumask_t cpu_present_map __read_mostly;
L
Linus Torvalds 已提交
4457 4458 4459
EXPORT_SYMBOL(cpu_present_map);

#ifndef CONFIG_SMP
4460
cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
4461 4462
EXPORT_SYMBOL(cpu_online_map);

4463
cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
4464
EXPORT_SYMBOL(cpu_possible_map);
L
Linus Torvalds 已提交
4465 4466 4467 4468
#endif

long sched_getaffinity(pid_t pid, cpumask_t *mask)
{
4469
	struct task_struct *p;
L
Linus Torvalds 已提交
4470 4471
	int retval;

4472
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4473 4474 4475 4476 4477 4478 4479
	read_lock(&tasklist_lock);

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

4480 4481 4482 4483
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4484
	cpus_and(*mask, p->cpus_allowed, cpu_online_map);
L
Linus Torvalds 已提交
4485 4486 4487

out_unlock:
	read_unlock(&tasklist_lock);
4488
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4489

4490
	return retval;
L
Linus Torvalds 已提交
4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	int ret;
	cpumask_t mask;

	if (len < sizeof(cpumask_t))
		return -EINVAL;

	ret = sched_getaffinity(pid, &mask);
	if (ret < 0)
		return ret;

	if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t)))
		return -EFAULT;

	return sizeof(cpumask_t);
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
4521 4522
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
4523 4524 4525
 */
asmlinkage long sys_sched_yield(void)
{
4526
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
4527 4528

	schedstat_inc(rq, yld_cnt);
I
Ingo Molnar 已提交
4529
	if (unlikely(rq->nr_running == 1))
L
Linus Torvalds 已提交
4530
		schedstat_inc(rq, yld_act_empty);
I
Ingo Molnar 已提交
4531 4532
	else
		current->sched_class->yield_task(rq, current);
L
Linus Torvalds 已提交
4533 4534 4535 4536 4537 4538

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
4539
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4540 4541 4542 4543 4544 4545 4546 4547
	_raw_spin_unlock(&rq->lock);
	preempt_enable_no_resched();

	schedule();

	return 0;
}

A
Andrew Morton 已提交
4548
static void __cond_resched(void)
L
Linus Torvalds 已提交
4549
{
4550 4551 4552
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
	__might_sleep(__FILE__, __LINE__);
#endif
4553 4554 4555 4556 4557
	/*
	 * The BKS might be reacquired before we have dropped
	 * PREEMPT_ACTIVE, which could trigger a second
	 * cond_resched() call.
	 */
L
Linus Torvalds 已提交
4558 4559 4560 4561 4562 4563 4564 4565 4566
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
	} while (need_resched());
}

int __sched cond_resched(void)
{
4567 4568
	if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
					system_state == SYSTEM_RUNNING) {
L
Linus Torvalds 已提交
4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583
		__cond_resched();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched);

/*
 * cond_resched_lock() - if a reschedule is pending, drop the given lock,
 * call schedule, and on return reacquire the lock.
 *
 * This works OK both with and without CONFIG_PREEMPT.  We do strange low-level
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
I
Ingo Molnar 已提交
4584
int cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
4585
{
J
Jan Kara 已提交
4586 4587
	int ret = 0;

L
Linus Torvalds 已提交
4588 4589 4590
	if (need_lockbreak(lock)) {
		spin_unlock(lock);
		cpu_relax();
J
Jan Kara 已提交
4591
		ret = 1;
L
Linus Torvalds 已提交
4592 4593
		spin_lock(lock);
	}
4594
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4595
		spin_release(&lock->dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4596 4597 4598
		_raw_spin_unlock(lock);
		preempt_enable_no_resched();
		__cond_resched();
J
Jan Kara 已提交
4599
		ret = 1;
L
Linus Torvalds 已提交
4600 4601
		spin_lock(lock);
	}
J
Jan Kara 已提交
4602
	return ret;
L
Linus Torvalds 已提交
4603 4604 4605 4606 4607 4608 4609
}
EXPORT_SYMBOL(cond_resched_lock);

int __sched cond_resched_softirq(void)
{
	BUG_ON(!in_softirq());

4610
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4611
		local_bh_enable();
L
Linus Torvalds 已提交
4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched_softirq);

/**
 * yield - yield the current processor to other threads.
 *
4623
 * This is a shortcut for kernel-space yielding - it marks the
L
Linus Torvalds 已提交
4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641
 * thread runnable and calls sys_sched_yield().
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

/*
 * This task is about to go to sleep on IO.  Increment rq->nr_iowait so
 * that process accounting knows that this is a task in IO wait state.
 *
 * But don't do that if it is a deliberate, throttling IO wait (this task
 * has set its backing_dev_info: the queue against which it should throttle)
 */
void __sched io_schedule(void)
{
4642
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4643

4644
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4645 4646 4647
	atomic_inc(&rq->nr_iowait);
	schedule();
	atomic_dec(&rq->nr_iowait);
4648
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4649 4650 4651 4652 4653
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
4654
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4655 4656
	long ret;

4657
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4658 4659 4660
	atomic_inc(&rq->nr_iowait);
	ret = schedule_timeout(timeout);
	atomic_dec(&rq->nr_iowait);
4661
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_max(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
4682
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4683
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_min(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
4707
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4708
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
asmlinkage
long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
{
4725
	struct task_struct *p;
L
Linus Torvalds 已提交
4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741
	int retval = -EINVAL;
	struct timespec t;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4742
	jiffies_to_timespec(p->policy == SCHED_FIFO ?
I
Ingo Molnar 已提交
4743
				0 : static_prio_timeslice(p->static_prio), &t);
L
Linus Torvalds 已提交
4744 4745 4746 4747 4748 4749 4750 4751 4752
	read_unlock(&tasklist_lock);
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
out_nounlock:
	return retval;
out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

4753
static const char stat_nam[] = "RSDTtZX";
4754 4755

static void show_task(struct task_struct *p)
L
Linus Torvalds 已提交
4756 4757
{
	unsigned long free = 0;
4758
	unsigned state;
L
Linus Torvalds 已提交
4759 4760

	state = p->state ? __ffs(p->state) + 1 : 0;
4761 4762
	printk("%-13.13s %c", p->comm,
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4763
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
4764
	if (state == TASK_RUNNING)
4765
		printk(" running  ");
L
Linus Torvalds 已提交
4766
	else
4767
		printk(" %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4768 4769
#else
	if (state == TASK_RUNNING)
4770
		printk("  running task    ");
L
Linus Torvalds 已提交
4771 4772 4773 4774 4775
	else
		printk(" %016lx ", thread_saved_pc(p));
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
	{
4776
		unsigned long *n = end_of_stack(p);
L
Linus Torvalds 已提交
4777 4778
		while (!*n)
			n++;
4779
		free = (unsigned long)n - (unsigned long)end_of_stack(p);
L
Linus Torvalds 已提交
4780 4781
	}
#endif
4782
	printk("%5lu %5d %6d\n", free, p->pid, p->parent->pid);
L
Linus Torvalds 已提交
4783 4784 4785 4786 4787

	if (state != TASK_RUNNING)
		show_stack(p, NULL);
}

I
Ingo Molnar 已提交
4788
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
4789
{
4790
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
4791

4792 4793 4794
#if BITS_PER_LONG == 32
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
4795
#else
4796 4797
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
4798 4799 4800 4801 4802 4803 4804 4805
#endif
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
		 * console might take alot of time:
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
4806
		if (!state_filter || (p->state & state_filter))
I
Ingo Molnar 已提交
4807
			show_task(p);
L
Linus Torvalds 已提交
4808 4809
	} while_each_thread(g, p);

4810 4811
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
4812 4813 4814
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
L
Linus Torvalds 已提交
4815
	read_unlock(&tasklist_lock);
I
Ingo Molnar 已提交
4816 4817 4818 4819 4820
	/*
	 * Only show locks if all tasks are dumped:
	 */
	if (state_filter == -1)
		debug_show_all_locks();
L
Linus Torvalds 已提交
4821 4822
}

I
Ingo Molnar 已提交
4823 4824
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
4825
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
4826 4827
}

4828 4829 4830 4831 4832 4833 4834 4835
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
4836
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
4837
{
4838
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4839 4840
	unsigned long flags;

I
Ingo Molnar 已提交
4841 4842 4843
	__sched_fork(idle);
	idle->se.exec_start = sched_clock();

4844
	idle->prio = idle->normal_prio = MAX_PRIO;
L
Linus Torvalds 已提交
4845
	idle->cpus_allowed = cpumask_of_cpu(cpu);
I
Ingo Molnar 已提交
4846
	__set_task_cpu(idle, cpu);
L
Linus Torvalds 已提交
4847 4848 4849

	spin_lock_irqsave(&rq->lock, flags);
	rq->curr = rq->idle = idle;
4850 4851 4852
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
	idle->oncpu = 1;
#endif
L
Linus Torvalds 已提交
4853 4854 4855 4856
	spin_unlock_irqrestore(&rq->lock, flags);

	/* Set the preempt count _outside_ the spinlocks! */
#if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_BKL)
A
Al Viro 已提交
4857
	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
L
Linus Torvalds 已提交
4858
#else
A
Al Viro 已提交
4859
	task_thread_info(idle)->preempt_count = 0;
L
Linus Torvalds 已提交
4860
#endif
I
Ingo Molnar 已提交
4861 4862 4863 4864
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875
}

/*
 * In a system that switches off the HZ timer nohz_cpu_mask
 * indicates which cpus entered this state. This is used
 * in the rcu update to wait only for active cpus. For system
 * which do not switch off the HZ timer nohz_cpu_mask should
 * always be CPU_MASK_NONE.
 */
cpumask_t nohz_cpu_mask = CPU_MASK_NONE;

I
Ingo Molnar 已提交
4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887
/*
 * Increase the granularity value when there are more CPUs,
 * because with more CPUs the 'effective latency' as visible
 * to users decreases. But the relationship is not linear,
 * so pick a second-best guess by going with the log2 of the
 * number of CPUs.
 *
 * This idea comes from the SD scheduler of Con Kolivas:
 */
static inline void sched_init_granularity(void)
{
	unsigned int factor = 1 + ilog2(num_online_cpus());
4888
	const unsigned long gran_limit = 100000000;
I
Ingo Molnar 已提交
4889 4890 4891 4892 4893 4894 4895 4896 4897

	sysctl_sched_granularity *= factor;
	if (sysctl_sched_granularity > gran_limit)
		sysctl_sched_granularity = gran_limit;

	sysctl_sched_runtime_limit = sysctl_sched_granularity * 4;
	sysctl_sched_wakeup_granularity = sysctl_sched_granularity / 2;
}

L
Linus Torvalds 已提交
4898 4899 4900 4901
#ifdef CONFIG_SMP
/*
 * This is how migration works:
 *
4902
 * 1) we queue a struct migration_req structure in the source CPU's
L
Linus Torvalds 已提交
4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923
 *    runqueue and wake up that CPU's migration thread.
 * 2) we down() the locked semaphore => thread blocks.
 * 3) migration thread wakes up (implicitly it forces the migrated
 *    thread off the CPU)
 * 4) it gets the migration request and checks whether the migrated
 *    task is still in the wrong runqueue.
 * 5) if it's in the wrong runqueue then the migration thread removes
 *    it and puts it into the right queue.
 * 6) migration thread up()s the semaphore.
 * 7) we wake up and the migration is done.
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
 * task must not exit() & deallocate itself prematurely.  The
 * call is not atomic; no spinlocks may be held.
 */
4924
int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
L
Linus Torvalds 已提交
4925
{
4926
	struct migration_req req;
L
Linus Torvalds 已提交
4927
	unsigned long flags;
4928
	struct rq *rq;
4929
	int ret = 0;
L
Linus Torvalds 已提交
4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951

	rq = task_rq_lock(p, &flags);
	if (!cpus_intersects(new_mask, cpu_online_map)) {
		ret = -EINVAL;
		goto out;
	}

	p->cpus_allowed = new_mask;
	/* Can the task run on the task's current CPU? If so, we're done */
	if (cpu_isset(task_cpu(p), new_mask))
		goto out;

	if (migrate_task(p, any_online_cpu(new_mask), &req)) {
		/* Need help from migration thread: drop lock and wait. */
		task_rq_unlock(rq, &flags);
		wake_up_process(rq->migration_thread);
		wait_for_completion(&req.done);
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
	task_rq_unlock(rq, &flags);
4952

L
Linus Torvalds 已提交
4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964
	return ret;
}
EXPORT_SYMBOL_GPL(set_cpus_allowed);

/*
 * Move (not current) task off this cpu, onto dest cpu.  We're doing
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
4965 4966
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
4967
 */
4968
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
4969
{
4970
	struct rq *rq_dest, *rq_src;
I
Ingo Molnar 已提交
4971
	int ret = 0, on_rq;
L
Linus Torvalds 已提交
4972 4973

	if (unlikely(cpu_is_offline(dest_cpu)))
4974
		return ret;
L
Linus Torvalds 已提交
4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
		goto out;
	/* Affinity changed (again). */
	if (!cpu_isset(dest_cpu, p->cpus_allowed))
		goto out;

I
Ingo Molnar 已提交
4987
	on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
4988 4989 4990 4991
	if (on_rq) {
		update_rq_clock(rq_src);
		deactivate_task(rq_src, p, 0, rq_src->clock);
	}
L
Linus Torvalds 已提交
4992
	set_task_cpu(p, dest_cpu);
I
Ingo Molnar 已提交
4993 4994 4995
	if (on_rq) {
		activate_task(rq_dest, p, 0);
		check_preempt_curr(rq_dest, p);
L
Linus Torvalds 已提交
4996
	}
4997
	ret = 1;
L
Linus Torvalds 已提交
4998 4999
out:
	double_rq_unlock(rq_src, rq_dest);
5000
	return ret;
L
Linus Torvalds 已提交
5001 5002 5003 5004 5005 5006 5007
}

/*
 * migration_thread - this is a highprio system thread that performs
 * thread migration by bumping thread off CPU then 'pushing' onto
 * another runqueue.
 */
I
Ingo Molnar 已提交
5008
static int migration_thread(void *data)
L
Linus Torvalds 已提交
5009 5010
{
	int cpu = (long)data;
5011
	struct rq *rq;
L
Linus Torvalds 已提交
5012 5013 5014 5015 5016 5017

	rq = cpu_rq(cpu);
	BUG_ON(rq->migration_thread != current);

	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
5018
		struct migration_req *req;
L
Linus Torvalds 已提交
5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040
		struct list_head *head;

		spin_lock_irq(&rq->lock);

		if (cpu_is_offline(cpu)) {
			spin_unlock_irq(&rq->lock);
			goto wait_to_die;
		}

		if (rq->active_balance) {
			active_load_balance(rq, cpu);
			rq->active_balance = 0;
		}

		head = &rq->migration_queue;

		if (list_empty(head)) {
			spin_unlock_irq(&rq->lock);
			schedule();
			set_current_state(TASK_INTERRUPTIBLE);
			continue;
		}
5041
		req = list_entry(head->next, struct migration_req, list);
L
Linus Torvalds 已提交
5042 5043
		list_del_init(head->next);

N
Nick Piggin 已提交
5044 5045 5046
		spin_unlock(&rq->lock);
		__migrate_task(req->task, cpu, req->dest_cpu);
		local_irq_enable();
L
Linus Torvalds 已提交
5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064

		complete(&req->done);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
5065 5066 5067 5068
/*
 * Figure out where task on dead CPU should go, use force if neccessary.
 * NOTE: interrupts should be disabled by the caller
 */
5069
static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5070
{
5071
	unsigned long flags;
L
Linus Torvalds 已提交
5072
	cpumask_t mask;
5073 5074
	struct rq *rq;
	int dest_cpu;
L
Linus Torvalds 已提交
5075

5076
restart:
L
Linus Torvalds 已提交
5077 5078
	/* On same node? */
	mask = node_to_cpumask(cpu_to_node(dead_cpu));
5079
	cpus_and(mask, mask, p->cpus_allowed);
L
Linus Torvalds 已提交
5080 5081 5082 5083
	dest_cpu = any_online_cpu(mask);

	/* On any allowed CPU? */
	if (dest_cpu == NR_CPUS)
5084
		dest_cpu = any_online_cpu(p->cpus_allowed);
L
Linus Torvalds 已提交
5085 5086 5087

	/* No more Mr. Nice Guy. */
	if (dest_cpu == NR_CPUS) {
5088 5089 5090
		rq = task_rq_lock(p, &flags);
		cpus_setall(p->cpus_allowed);
		dest_cpu = any_online_cpu(p->cpus_allowed);
5091
		task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
5092 5093 5094 5095 5096 5097

		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
5098
		if (p->mm && printk_ratelimit())
L
Linus Torvalds 已提交
5099 5100
			printk(KERN_INFO "process %d (%s) no "
			       "longer affine to cpu%d\n",
5101
			       p->pid, p->comm, dead_cpu);
L
Linus Torvalds 已提交
5102
	}
5103
	if (!__migrate_task(p, dead_cpu, dest_cpu))
5104
		goto restart;
L
Linus Torvalds 已提交
5105 5106 5107 5108 5109 5110 5111 5112 5113
}

/*
 * While a dead CPU has no uninterruptible tasks queued at this point,
 * it might still have a nonzero ->nr_uninterruptible counter, because
 * for performance reasons the counter is not stricly tracking tasks to
 * their home CPUs. So we just add the counter to another CPU's counter,
 * to keep the global sum constant after CPU-down:
 */
5114
static void migrate_nr_uninterruptible(struct rq *rq_src)
L
Linus Torvalds 已提交
5115
{
5116
	struct rq *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL));
L
Linus Torvalds 已提交
5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129
	unsigned long flags;

	local_irq_save(flags);
	double_rq_lock(rq_src, rq_dest);
	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
	rq_src->nr_uninterruptible = 0;
	double_rq_unlock(rq_src, rq_dest);
	local_irq_restore(flags);
}

/* Run through task list and migrate tasks from the dead cpu. */
static void migrate_live_tasks(int src_cpu)
{
5130
	struct task_struct *p, *t;
L
Linus Torvalds 已提交
5131 5132 5133

	write_lock_irq(&tasklist_lock);

5134 5135
	do_each_thread(t, p) {
		if (p == current)
L
Linus Torvalds 已提交
5136 5137
			continue;

5138 5139 5140
		if (task_cpu(p) == src_cpu)
			move_task_off_dead_cpu(src_cpu, p);
	} while_each_thread(t, p);
L
Linus Torvalds 已提交
5141 5142 5143 5144

	write_unlock_irq(&tasklist_lock);
}

I
Ingo Molnar 已提交
5145 5146
/*
 * Schedules idle task to be the next runnable task on current CPU.
L
Linus Torvalds 已提交
5147
 * It does so by boosting its priority to highest possible and adding it to
5148
 * the _front_ of the runqueue. Used by CPU offline code.
L
Linus Torvalds 已提交
5149 5150 5151
 */
void sched_idle_next(void)
{
5152
	int this_cpu = smp_processor_id();
5153
	struct rq *rq = cpu_rq(this_cpu);
L
Linus Torvalds 已提交
5154 5155 5156 5157
	struct task_struct *p = rq->idle;
	unsigned long flags;

	/* cpu has to be offline */
5158
	BUG_ON(cpu_online(this_cpu));
L
Linus Torvalds 已提交
5159

5160 5161 5162
	/*
	 * Strictly not necessary since rest of the CPUs are stopped by now
	 * and interrupts disabled on the current cpu.
L
Linus Torvalds 已提交
5163 5164 5165
	 */
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
5166
	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
5167 5168

	/* Add idle task to the _front_ of its priority queue: */
I
Ingo Molnar 已提交
5169
	activate_idle_task(p, rq);
L
Linus Torvalds 已提交
5170 5171 5172 5173

	spin_unlock_irqrestore(&rq->lock, flags);
}

5174 5175
/*
 * Ensures that the idle task is using init_mm right before its cpu goes
L
Linus Torvalds 已提交
5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188
 * offline.
 */
void idle_task_exit(void)
{
	struct mm_struct *mm = current->active_mm;

	BUG_ON(cpu_online(smp_processor_id()));

	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
}

5189
/* called under rq->lock with disabled interrupts */
5190
static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5191
{
5192
	struct rq *rq = cpu_rq(dead_cpu);
L
Linus Torvalds 已提交
5193 5194

	/* Must be exiting, otherwise would be on tasklist. */
5195
	BUG_ON(p->exit_state != EXIT_ZOMBIE && p->exit_state != EXIT_DEAD);
L
Linus Torvalds 已提交
5196 5197

	/* Cannot have done final schedule yet: would have vanished. */
5198
	BUG_ON(p->state == TASK_DEAD);
L
Linus Torvalds 已提交
5199

5200
	get_task_struct(p);
L
Linus Torvalds 已提交
5201 5202 5203 5204 5205

	/*
	 * Drop lock around migration; if someone else moves it,
	 * that's OK.  No task can be added to this CPU, so iteration is
	 * fine.
5206
	 * NOTE: interrupts should be left disabled  --dev@
L
Linus Torvalds 已提交
5207
	 */
5208
	spin_unlock(&rq->lock);
5209
	move_task_off_dead_cpu(dead_cpu, p);
5210
	spin_lock(&rq->lock);
L
Linus Torvalds 已提交
5211

5212
	put_task_struct(p);
L
Linus Torvalds 已提交
5213 5214 5215 5216 5217
}

/* release_task() removes task from tasklist, so we won't find dead tasks. */
static void migrate_dead_tasks(unsigned int dead_cpu)
{
5218
	struct rq *rq = cpu_rq(dead_cpu);
I
Ingo Molnar 已提交
5219
	struct task_struct *next;
5220

I
Ingo Molnar 已提交
5221 5222 5223
	for ( ; ; ) {
		if (!rq->nr_running)
			break;
I
Ingo Molnar 已提交
5224
		update_rq_clock(rq);
5225
		next = pick_next_task(rq, rq->curr);
I
Ingo Molnar 已提交
5226 5227 5228
		if (!next)
			break;
		migrate_dead(dead_cpu, next);
5229

L
Linus Torvalds 已提交
5230 5231 5232 5233
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

5234 5235 5236
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
5237 5238 5239 5240
	{
		.procname	= "sched_domain",
		.mode		= 0755,
	},
5241 5242 5243 5244
	{0,},
};

static struct ctl_table sd_ctl_root[] = {
5245 5246 5247 5248 5249
	{
		.procname	= "kernel",
		.mode		= 0755,
		.child		= sd_ctl_dir,
	},
5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264
	{0,},
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
		kmalloc(n * sizeof(struct ctl_table), GFP_KERNEL);

	BUG_ON(!entry);
	memset(entry, 0, n * sizeof(struct ctl_table));

	return entry;
}

static void
5265
set_table_entry(struct ctl_table *entry,
5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280
		const char *procname, void *data, int maxlen,
		mode_t mode, proc_handler *proc_handler)
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
	struct ctl_table *table = sd_alloc_ctl_entry(14);

5281
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
5282
		sizeof(long), 0644, proc_doulongvec_minmax);
5283
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
5284
		sizeof(long), 0644, proc_doulongvec_minmax);
5285
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
5286
		sizeof(int), 0644, proc_dointvec_minmax);
5287
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
5288
		sizeof(int), 0644, proc_dointvec_minmax);
5289
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
5290
		sizeof(int), 0644, proc_dointvec_minmax);
5291
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
5292
		sizeof(int), 0644, proc_dointvec_minmax);
5293
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
5294
		sizeof(int), 0644, proc_dointvec_minmax);
5295
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
5296
		sizeof(int), 0644, proc_dointvec_minmax);
5297
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
5298
		sizeof(int), 0644, proc_dointvec_minmax);
5299
	set_table_entry(&table[10], "cache_nice_tries",
5300 5301
		&sd->cache_nice_tries,
		sizeof(int), 0644, proc_dointvec_minmax);
5302
	set_table_entry(&table[12], "flags", &sd->flags,
5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353
		sizeof(int), 0644, proc_dointvec_minmax);

	return table;
}

static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
		entry->mode = 0755;
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
static void init_sched_domain_sysctl(void)
{
	int i, cpu_num = num_online_cpus();
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

	sd_ctl_dir[0].child = entry;

	for (i = 0; i < cpu_num; i++, entry++) {
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
		entry->mode = 0755;
		entry->child = sd_alloc_ctl_cpu_table(i);
	}
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
#else
static void init_sched_domain_sysctl(void)
{
}
#endif

L
Linus Torvalds 已提交
5354 5355 5356 5357
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
5358 5359
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
5360 5361
{
	struct task_struct *p;
5362
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
5363
	unsigned long flags;
5364
	struct rq *rq;
L
Linus Torvalds 已提交
5365 5366

	switch (action) {
5367 5368 5369 5370
	case CPU_LOCK_ACQUIRE:
		mutex_lock(&sched_hotcpu_mutex);
		break;

L
Linus Torvalds 已提交
5371
	case CPU_UP_PREPARE:
5372
	case CPU_UP_PREPARE_FROZEN:
I
Ingo Molnar 已提交
5373
		p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
L
Linus Torvalds 已提交
5374 5375 5376 5377 5378
		if (IS_ERR(p))
			return NOTIFY_BAD;
		kthread_bind(p, cpu);
		/* Must be high prio: stop_machine expects to yield to it. */
		rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
5379
		__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
L
Linus Torvalds 已提交
5380 5381 5382
		task_rq_unlock(rq, &flags);
		cpu_rq(cpu)->migration_thread = p;
		break;
5383

L
Linus Torvalds 已提交
5384
	case CPU_ONLINE:
5385
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
5386 5387 5388
		/* Strictly unneccessary, as first user will wake it. */
		wake_up_process(cpu_rq(cpu)->migration_thread);
		break;
5389

L
Linus Torvalds 已提交
5390 5391
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
5392
	case CPU_UP_CANCELED_FROZEN:
5393 5394
		if (!cpu_rq(cpu)->migration_thread)
			break;
L
Linus Torvalds 已提交
5395
		/* Unbind it from offline cpu so it can run.  Fall thru. */
5396 5397
		kthread_bind(cpu_rq(cpu)->migration_thread,
			     any_online_cpu(cpu_online_map));
L
Linus Torvalds 已提交
5398 5399 5400
		kthread_stop(cpu_rq(cpu)->migration_thread);
		cpu_rq(cpu)->migration_thread = NULL;
		break;
5401

L
Linus Torvalds 已提交
5402
	case CPU_DEAD:
5403
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
5404 5405 5406 5407 5408 5409
		migrate_live_tasks(cpu);
		rq = cpu_rq(cpu);
		kthread_stop(rq->migration_thread);
		rq->migration_thread = NULL;
		/* Idle task back to normal (off runqueue, low prio) */
		rq = task_rq_lock(rq->idle, &flags);
I
Ingo Molnar 已提交
5410 5411
		update_rq_clock(rq);
		deactivate_task(rq, rq->idle, 0, rq->clock);
L
Linus Torvalds 已提交
5412
		rq->idle->static_prio = MAX_PRIO;
I
Ingo Molnar 已提交
5413 5414
		__setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
		rq->idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
5415 5416 5417 5418 5419 5420
		migrate_dead_tasks(cpu);
		task_rq_unlock(rq, &flags);
		migrate_nr_uninterruptible(rq);
		BUG_ON(rq->nr_running != 0);

		/* No need to migrate the tasks: it was best-effort if
5421
		 * they didn't take sched_hotcpu_mutex.  Just wake up
L
Linus Torvalds 已提交
5422 5423 5424
		 * the requestors. */
		spin_lock_irq(&rq->lock);
		while (!list_empty(&rq->migration_queue)) {
5425 5426
			struct migration_req *req;

L
Linus Torvalds 已提交
5427
			req = list_entry(rq->migration_queue.next,
5428
					 struct migration_req, list);
L
Linus Torvalds 已提交
5429 5430 5431 5432 5433 5434
			list_del_init(&req->list);
			complete(&req->done);
		}
		spin_unlock_irq(&rq->lock);
		break;
#endif
5435 5436 5437
	case CPU_LOCK_RELEASE:
		mutex_unlock(&sched_hotcpu_mutex);
		break;
L
Linus Torvalds 已提交
5438 5439 5440 5441 5442 5443 5444
	}
	return NOTIFY_OK;
}

/* Register at highest priority so that task migration (migrate_all_tasks)
 * happens before everything else.
 */
5445
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
5446 5447 5448 5449 5450 5451 5452
	.notifier_call = migration_call,
	.priority = 10
};

int __init migration_init(void)
{
	void *cpu = (void *)(long)smp_processor_id();
5453
	int err;
5454 5455

	/* Start one for the boot CPU: */
5456 5457
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
5458 5459
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
5460

L
Linus Torvalds 已提交
5461 5462 5463 5464 5465
	return 0;
}
#endif

#ifdef CONFIG_SMP
5466 5467 5468 5469 5470

/* Number of possible processor ids */
int nr_cpu_ids __read_mostly = NR_CPUS;
EXPORT_SYMBOL(nr_cpu_ids);

5471
#undef SCHED_DOMAIN_DEBUG
L
Linus Torvalds 已提交
5472 5473 5474 5475 5476
#ifdef SCHED_DOMAIN_DEBUG
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;

N
Nick Piggin 已提交
5477 5478 5479 5480 5481
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}

L
Linus Torvalds 已提交
5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	do {
		int i;
		char str[NR_CPUS];
		struct sched_group *group = sd->groups;
		cpumask_t groupmask;

		cpumask_scnprintf(str, NR_CPUS, sd->span);
		cpus_clear(groupmask);

		printk(KERN_DEBUG);
		for (i = 0; i < level + 1; i++)
			printk(" ");
		printk("domain %d: ", level);

		if (!(sd->flags & SD_LOAD_BALANCE)) {
			printk("does not load-balance\n");
			if (sd->parent)
5501 5502
				printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
						" has parent");
L
Linus Torvalds 已提交
5503 5504 5505 5506 5507 5508
			break;
		}

		printk("span %s\n", str);

		if (!cpu_isset(cpu, sd->span))
5509 5510
			printk(KERN_ERR "ERROR: domain->span does not contain "
					"CPU%d\n", cpu);
L
Linus Torvalds 已提交
5511
		if (!cpu_isset(cpu, group->cpumask))
5512 5513
			printk(KERN_ERR "ERROR: domain->groups does not contain"
					" CPU%d\n", cpu);
L
Linus Torvalds 已提交
5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525

		printk(KERN_DEBUG);
		for (i = 0; i < level + 2; i++)
			printk(" ");
		printk("groups:");
		do {
			if (!group) {
				printk("\n");
				printk(KERN_ERR "ERROR: group is NULL\n");
				break;
			}

5526
			if (!group->__cpu_power) {
L
Linus Torvalds 已提交
5527
				printk("\n");
5528 5529
				printk(KERN_ERR "ERROR: domain->cpu_power not "
						"set\n");
L
Linus Torvalds 已提交
5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551
			}

			if (!cpus_weight(group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: empty group\n");
			}

			if (cpus_intersects(groupmask, group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: repeated CPUs\n");
			}

			cpus_or(groupmask, groupmask, group->cpumask);

			cpumask_scnprintf(str, NR_CPUS, group->cpumask);
			printk(" %s", str);

			group = group->next;
		} while (group != sd->groups);
		printk("\n");

		if (!cpus_equal(sd->span, groupmask))
5552 5553
			printk(KERN_ERR "ERROR: groups don't span "
					"domain->span\n");
L
Linus Torvalds 已提交
5554 5555 5556

		level++;
		sd = sd->parent;
5557 5558
		if (!sd)
			continue;
L
Linus Torvalds 已提交
5559

5560 5561 5562
		if (!cpus_subset(groupmask, sd->span))
			printk(KERN_ERR "ERROR: parent span is not a superset "
				"of domain->span\n");
L
Linus Torvalds 已提交
5563 5564 5565 5566

	} while (sd);
}
#else
5567
# define sched_domain_debug(sd, cpu) do { } while (0)
L
Linus Torvalds 已提交
5568 5569
#endif

5570
static int sd_degenerate(struct sched_domain *sd)
5571 5572 5573 5574 5575 5576 5577 5578
{
	if (cpus_weight(sd->span) == 1)
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
5579 5580 5581
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
	if (sd->flags & (SD_WAKE_IDLE |
			 SD_WAKE_AFFINE |
			 SD_WAKE_BALANCE))
		return 0;

	return 1;
}

5595 5596
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

	if (!cpus_equal(sd->span, parent->span))
		return 0;

	/* Does parent contain flags not in child? */
	/* WAKE_BALANCE is a subset of WAKE_AFFINE */
	if (cflags & SD_WAKE_AFFINE)
		pflags &= ~SD_WAKE_BALANCE;
	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
5615 5616 5617
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
5618 5619 5620 5621 5622 5623 5624
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

L
Linus Torvalds 已提交
5625 5626 5627 5628
/*
 * Attach the domain 'sd' to 'cpu' as its base domain.  Callers must
 * hold the hotplug lock.
 */
5629
static void cpu_attach_domain(struct sched_domain *sd, int cpu)
L
Linus Torvalds 已提交
5630
{
5631
	struct rq *rq = cpu_rq(cpu);
5632 5633 5634 5635 5636 5637 5638
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
	for (tmp = sd; tmp; tmp = tmp->parent) {
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
5639
		if (sd_parent_degenerate(tmp, parent)) {
5640
			tmp->parent = parent->parent;
5641 5642 5643
			if (parent->parent)
				parent->parent->child = tmp;
		}
5644 5645
	}

5646
	if (sd && sd_degenerate(sd)) {
5647
		sd = sd->parent;
5648 5649 5650
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
5651 5652 5653

	sched_domain_debug(sd, cpu);

N
Nick Piggin 已提交
5654
	rcu_assign_pointer(rq->sd, sd);
L
Linus Torvalds 已提交
5655 5656 5657
}

/* cpus with isolated domains */
5658
static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
L
Linus Torvalds 已提交
5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
	int ints[NR_CPUS], i;

	str = get_options(str, ARRAY_SIZE(ints), ints);
	cpus_clear(cpu_isolated_map);
	for (i = 1; i <= ints[0]; i++)
		if (ints[i] < NR_CPUS)
			cpu_set(ints[i], cpu_isolated_map);
	return 1;
}

__setup ("isolcpus=", isolated_cpu_setup);

/*
5676 5677 5678 5679
 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
 * to a function which identifies what group(along with sched group) a CPU
 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS
 * (due to the fact that we keep track of groups covered with a cpumask_t).
L
Linus Torvalds 已提交
5680 5681 5682 5683 5684
 *
 * init_sched_build_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
 */
5685
static void
5686 5687 5688
init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map,
			int (*group_fn)(int cpu, const cpumask_t *cpu_map,
					struct sched_group **sg))
L
Linus Torvalds 已提交
5689 5690 5691 5692 5693 5694
{
	struct sched_group *first = NULL, *last = NULL;
	cpumask_t covered = CPU_MASK_NONE;
	int i;

	for_each_cpu_mask(i, span) {
5695 5696
		struct sched_group *sg;
		int group = group_fn(i, cpu_map, &sg);
L
Linus Torvalds 已提交
5697 5698 5699 5700 5701 5702
		int j;

		if (cpu_isset(i, covered))
			continue;

		sg->cpumask = CPU_MASK_NONE;
5703
		sg->__cpu_power = 0;
L
Linus Torvalds 已提交
5704 5705

		for_each_cpu_mask(j, span) {
5706
			if (group_fn(j, cpu_map, NULL) != group)
L
Linus Torvalds 已提交
5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720
				continue;

			cpu_set(j, covered);
			cpu_set(j, sg->cpumask);
		}
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
}

5721
#define SD_NODES_PER_DOMAIN 16
L
Linus Torvalds 已提交
5722

5723
#ifdef CONFIG_NUMA
5724

5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776
/**
 * find_next_best_node - find the next node to include in a sched_domain
 * @node: node whose sched_domain we're building
 * @used_nodes: nodes already in the sched_domain
 *
 * Find the next node to include in a given scheduling domain.  Simply
 * finds the closest node not already in the @used_nodes map.
 *
 * Should use nodemask_t.
 */
static int find_next_best_node(int node, unsigned long *used_nodes)
{
	int i, n, val, min_val, best_node = 0;

	min_val = INT_MAX;

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Start at @node */
		n = (node + i) % MAX_NUMNODES;

		if (!nr_cpus_node(n))
			continue;

		/* Skip already used nodes */
		if (test_bit(n, used_nodes))
			continue;

		/* Simple min distance search */
		val = node_distance(node, n);

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

	set_bit(best_node, used_nodes);
	return best_node;
}

/**
 * sched_domain_node_span - get a cpumask for a node's sched_domain
 * @node: node whose cpumask we're constructing
 * @size: number of nodes to include in this span
 *
 * Given a node, construct a good cpumask for its sched_domain to span.  It
 * should be one that prevents unnecessary balancing, but also spreads tasks
 * out optimally.
 */
static cpumask_t sched_domain_node_span(int node)
{
	DECLARE_BITMAP(used_nodes, MAX_NUMNODES);
5777 5778
	cpumask_t span, nodemask;
	int i;
5779 5780 5781 5782 5783 5784 5785 5786 5787 5788

	cpus_clear(span);
	bitmap_zero(used_nodes, MAX_NUMNODES);

	nodemask = node_to_cpumask(node);
	cpus_or(span, span, nodemask);
	set_bit(node, used_nodes);

	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
		int next_node = find_next_best_node(node, used_nodes);
5789

5790 5791 5792 5793 5794 5795 5796 5797
		nodemask = node_to_cpumask(next_node);
		cpus_or(span, span, nodemask);
	}

	return span;
}
#endif

5798
int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
5799

5800
/*
5801
 * SMT sched-domains:
5802
 */
L
Linus Torvalds 已提交
5803 5804
#ifdef CONFIG_SCHED_SMT
static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
5805
static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
5806

5807 5808
static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map,
			    struct sched_group **sg)
L
Linus Torvalds 已提交
5809
{
5810 5811
	if (sg)
		*sg = &per_cpu(sched_group_cpus, cpu);
L
Linus Torvalds 已提交
5812 5813 5814 5815
	return cpu;
}
#endif

5816 5817 5818
/*
 * multi-core sched-domains:
 */
5819 5820
#ifdef CONFIG_SCHED_MC
static DEFINE_PER_CPU(struct sched_domain, core_domains);
5821
static DEFINE_PER_CPU(struct sched_group, sched_group_core);
5822 5823 5824
#endif

#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
5825 5826
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5827
{
5828
	int group;
5829 5830
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5831 5832 5833 5834
	group = first_cpu(mask);
	if (sg)
		*sg = &per_cpu(sched_group_core, group);
	return group;
5835 5836
}
#elif defined(CONFIG_SCHED_MC)
5837 5838
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5839
{
5840 5841
	if (sg)
		*sg = &per_cpu(sched_group_core, cpu);
5842 5843 5844 5845
	return cpu;
}
#endif

L
Linus Torvalds 已提交
5846
static DEFINE_PER_CPU(struct sched_domain, phys_domains);
5847
static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
5848

5849 5850
static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
L
Linus Torvalds 已提交
5851
{
5852
	int group;
5853
#ifdef CONFIG_SCHED_MC
5854
	cpumask_t mask = cpu_coregroup_map(cpu);
5855
	cpus_and(mask, mask, *cpu_map);
5856
	group = first_cpu(mask);
5857
#elif defined(CONFIG_SCHED_SMT)
5858 5859
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5860
	group = first_cpu(mask);
L
Linus Torvalds 已提交
5861
#else
5862
	group = cpu;
L
Linus Torvalds 已提交
5863
#endif
5864 5865 5866
	if (sg)
		*sg = &per_cpu(sched_group_phys, group);
	return group;
L
Linus Torvalds 已提交
5867 5868 5869 5870
}

#ifdef CONFIG_NUMA
/*
5871 5872 5873
 * The init_sched_build_groups can't handle what we want to do with node
 * groups, so roll our own. Now each node has its own list of groups which
 * gets dynamically allocated.
L
Linus Torvalds 已提交
5874
 */
5875
static DEFINE_PER_CPU(struct sched_domain, node_domains);
5876
static struct sched_group **sched_group_nodes_bycpu[NR_CPUS];
L
Linus Torvalds 已提交
5877

5878
static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
5879
static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
5880

5881 5882
static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
				 struct sched_group **sg)
5883
{
5884 5885 5886 5887 5888 5889 5890 5891 5892
	cpumask_t nodemask = node_to_cpumask(cpu_to_node(cpu));
	int group;

	cpus_and(nodemask, nodemask, *cpu_map);
	group = first_cpu(nodemask);

	if (sg)
		*sg = &per_cpu(sched_group_allnodes, group);
	return group;
L
Linus Torvalds 已提交
5893
}
5894

5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914
static void init_numa_sched_groups_power(struct sched_group *group_head)
{
	struct sched_group *sg = group_head;
	int j;

	if (!sg)
		return;
next_sg:
	for_each_cpu_mask(j, sg->cpumask) {
		struct sched_domain *sd;

		sd = &per_cpu(phys_domains, j);
		if (j != first_cpu(sd->groups->cpumask)) {
			/*
			 * Only add "power" once for each
			 * physical package.
			 */
			continue;
		}

5915
		sg_inc_cpu_power(sg, sd->groups->__cpu_power);
5916 5917 5918 5919 5920
	}
	sg = sg->next;
	if (sg != group_head)
		goto next_sg;
}
L
Linus Torvalds 已提交
5921 5922
#endif

5923
#ifdef CONFIG_NUMA
5924 5925 5926
/* Free memory allocated for various sched_group structures */
static void free_sched_groups(const cpumask_t *cpu_map)
{
5927
	int cpu, i;
5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957

	for_each_cpu_mask(cpu, *cpu_map) {
		struct sched_group **sched_group_nodes
			= sched_group_nodes_bycpu[cpu];

		if (!sched_group_nodes)
			continue;

		for (i = 0; i < MAX_NUMNODES; i++) {
			cpumask_t nodemask = node_to_cpumask(i);
			struct sched_group *oldsg, *sg = sched_group_nodes[i];

			cpus_and(nodemask, nodemask, *cpu_map);
			if (cpus_empty(nodemask))
				continue;

			if (sg == NULL)
				continue;
			sg = sg->next;
next_sg:
			oldsg = sg;
			sg = sg->next;
			kfree(oldsg);
			if (oldsg != sched_group_nodes[i])
				goto next_sg;
		}
		kfree(sched_group_nodes);
		sched_group_nodes_bycpu[cpu] = NULL;
	}
}
5958 5959 5960 5961 5962
#else
static void free_sched_groups(const cpumask_t *cpu_map)
{
}
#endif
5963

5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 *
 * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
 * the maximum number of tasks a group can handle in the presence of other idle
 * or lightly loaded groups in the same sched domain.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
	struct sched_domain *child;
	struct sched_group *group;

	WARN_ON(!sd || !sd->groups);

	if (cpu != first_cpu(sd->groups->cpumask))
		return;

	child = sd->child;

5990 5991
	sd->groups->__cpu_power = 0;

5992 5993 5994 5995 5996 5997 5998 5999 6000 6001
	/*
	 * For perf policy, if the groups in child domain share resources
	 * (for example cores sharing some portions of the cache hierarchy
	 * or SMT), then set this domain groups cpu_power such that each group
	 * can handle only one task, when there are other idle groups in the
	 * same sched domain.
	 */
	if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
		       (child->flags &
			(SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
6002
		sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
6003 6004 6005 6006 6007 6008 6009 6010
		return;
	}

	/*
	 * add cpu_power of each child group to this groups cpu_power
	 */
	group = child->groups;
	do {
6011
		sg_inc_cpu_power(sd->groups, group->__cpu_power);
6012 6013 6014 6015
		group = group->next;
	} while (group != child->groups);
}

L
Linus Torvalds 已提交
6016
/*
6017 6018
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
L
Linus Torvalds 已提交
6019
 */
6020
static int build_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6021 6022
{
	int i;
6023 6024
#ifdef CONFIG_NUMA
	struct sched_group **sched_group_nodes = NULL;
6025
	int sd_allnodes = 0;
6026 6027 6028 6029

	/*
	 * Allocate the per-node list of sched groups
	 */
I
Ingo Molnar 已提交
6030
	sched_group_nodes = kzalloc(sizeof(struct sched_group *)*MAX_NUMNODES,
6031
					   GFP_KERNEL);
6032 6033
	if (!sched_group_nodes) {
		printk(KERN_WARNING "Can not alloc sched group node list\n");
6034
		return -ENOMEM;
6035 6036 6037
	}
	sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
#endif
L
Linus Torvalds 已提交
6038 6039

	/*
6040
	 * Set up domains for cpus specified by the cpu_map.
L
Linus Torvalds 已提交
6041
	 */
6042
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6043 6044 6045
		struct sched_domain *sd = NULL, *p;
		cpumask_t nodemask = node_to_cpumask(cpu_to_node(i));

6046
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6047 6048

#ifdef CONFIG_NUMA
I
Ingo Molnar 已提交
6049 6050
		if (cpus_weight(*cpu_map) >
				SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) {
6051 6052 6053
			sd = &per_cpu(allnodes_domains, i);
			*sd = SD_ALLNODES_INIT;
			sd->span = *cpu_map;
6054
			cpu_to_allnodes_group(i, cpu_map, &sd->groups);
6055
			p = sd;
6056
			sd_allnodes = 1;
6057 6058 6059
		} else
			p = NULL;

L
Linus Torvalds 已提交
6060 6061
		sd = &per_cpu(node_domains, i);
		*sd = SD_NODE_INIT;
6062 6063
		sd->span = sched_domain_node_span(cpu_to_node(i));
		sd->parent = p;
6064 6065
		if (p)
			p->child = sd;
6066
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6067 6068 6069 6070 6071 6072 6073
#endif

		p = sd;
		sd = &per_cpu(phys_domains, i);
		*sd = SD_CPU_INIT;
		sd->span = nodemask;
		sd->parent = p;
6074 6075
		if (p)
			p->child = sd;
6076
		cpu_to_phys_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6077

6078 6079 6080 6081 6082 6083 6084
#ifdef CONFIG_SCHED_MC
		p = sd;
		sd = &per_cpu(core_domains, i);
		*sd = SD_MC_INIT;
		sd->span = cpu_coregroup_map(i);
		cpus_and(sd->span, sd->span, *cpu_map);
		sd->parent = p;
6085
		p->child = sd;
6086
		cpu_to_core_group(i, cpu_map, &sd->groups);
6087 6088
#endif

L
Linus Torvalds 已提交
6089 6090 6091 6092 6093
#ifdef CONFIG_SCHED_SMT
		p = sd;
		sd = &per_cpu(cpu_domains, i);
		*sd = SD_SIBLING_INIT;
		sd->span = cpu_sibling_map[i];
6094
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6095
		sd->parent = p;
6096
		p->child = sd;
6097
		cpu_to_cpu_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6098 6099 6100 6101 6102
#endif
	}

#ifdef CONFIG_SCHED_SMT
	/* Set up CPU (sibling) groups */
6103
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6104
		cpumask_t this_sibling_map = cpu_sibling_map[i];
6105
		cpus_and(this_sibling_map, this_sibling_map, *cpu_map);
L
Linus Torvalds 已提交
6106 6107 6108
		if (i != first_cpu(this_sibling_map))
			continue;

I
Ingo Molnar 已提交
6109 6110
		init_sched_build_groups(this_sibling_map, cpu_map,
					&cpu_to_cpu_group);
L
Linus Torvalds 已提交
6111 6112 6113
	}
#endif

6114 6115 6116 6117 6118 6119 6120
#ifdef CONFIG_SCHED_MC
	/* Set up multi-core groups */
	for_each_cpu_mask(i, *cpu_map) {
		cpumask_t this_core_map = cpu_coregroup_map(i);
		cpus_and(this_core_map, this_core_map, *cpu_map);
		if (i != first_cpu(this_core_map))
			continue;
I
Ingo Molnar 已提交
6121 6122
		init_sched_build_groups(this_core_map, cpu_map,
					&cpu_to_core_group);
6123 6124 6125
	}
#endif

L
Linus Torvalds 已提交
6126 6127 6128 6129
	/* Set up physical groups */
	for (i = 0; i < MAX_NUMNODES; i++) {
		cpumask_t nodemask = node_to_cpumask(i);

6130
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6131 6132 6133
		if (cpus_empty(nodemask))
			continue;

6134
		init_sched_build_groups(nodemask, cpu_map, &cpu_to_phys_group);
L
Linus Torvalds 已提交
6135 6136 6137 6138
	}

#ifdef CONFIG_NUMA
	/* Set up node groups */
6139
	if (sd_allnodes)
I
Ingo Molnar 已提交
6140 6141
		init_sched_build_groups(*cpu_map, cpu_map,
					&cpu_to_allnodes_group);
6142 6143 6144 6145 6146 6147 6148 6149 6150 6151

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Set up node groups */
		struct sched_group *sg, *prev;
		cpumask_t nodemask = node_to_cpumask(i);
		cpumask_t domainspan;
		cpumask_t covered = CPU_MASK_NONE;
		int j;

		cpus_and(nodemask, nodemask, *cpu_map);
6152 6153
		if (cpus_empty(nodemask)) {
			sched_group_nodes[i] = NULL;
6154
			continue;
6155
		}
6156 6157 6158 6159

		domainspan = sched_domain_node_span(i);
		cpus_and(domainspan, domainspan, *cpu_map);

6160
		sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);
6161 6162 6163 6164 6165
		if (!sg) {
			printk(KERN_WARNING "Can not alloc domain group for "
				"node %d\n", i);
			goto error;
		}
6166 6167 6168
		sched_group_nodes[i] = sg;
		for_each_cpu_mask(j, nodemask) {
			struct sched_domain *sd;
I
Ingo Molnar 已提交
6169

6170 6171 6172
			sd = &per_cpu(node_domains, j);
			sd->groups = sg;
		}
6173
		sg->__cpu_power = 0;
6174
		sg->cpumask = nodemask;
6175
		sg->next = sg;
6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193
		cpus_or(covered, covered, nodemask);
		prev = sg;

		for (j = 0; j < MAX_NUMNODES; j++) {
			cpumask_t tmp, notcovered;
			int n = (i + j) % MAX_NUMNODES;

			cpus_complement(notcovered, covered);
			cpus_and(tmp, notcovered, *cpu_map);
			cpus_and(tmp, tmp, domainspan);
			if (cpus_empty(tmp))
				break;

			nodemask = node_to_cpumask(n);
			cpus_and(tmp, tmp, nodemask);
			if (cpus_empty(tmp))
				continue;

6194 6195
			sg = kmalloc_node(sizeof(struct sched_group),
					  GFP_KERNEL, i);
6196 6197 6198
			if (!sg) {
				printk(KERN_WARNING
				"Can not alloc domain group for node %d\n", j);
6199
				goto error;
6200
			}
6201
			sg->__cpu_power = 0;
6202
			sg->cpumask = tmp;
6203
			sg->next = prev->next;
6204 6205 6206 6207 6208
			cpus_or(covered, covered, tmp);
			prev->next = sg;
			prev = sg;
		}
	}
L
Linus Torvalds 已提交
6209 6210 6211
#endif

	/* Calculate CPU power for physical packages and nodes */
6212
#ifdef CONFIG_SCHED_SMT
6213
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6214 6215
		struct sched_domain *sd = &per_cpu(cpu_domains, i);

6216
		init_sched_groups_power(i, sd);
6217
	}
L
Linus Torvalds 已提交
6218
#endif
6219
#ifdef CONFIG_SCHED_MC
6220
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6221 6222
		struct sched_domain *sd = &per_cpu(core_domains, i);

6223
		init_sched_groups_power(i, sd);
6224 6225
	}
#endif
6226

6227
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6228 6229
		struct sched_domain *sd = &per_cpu(phys_domains, i);

6230
		init_sched_groups_power(i, sd);
L
Linus Torvalds 已提交
6231 6232
	}

6233
#ifdef CONFIG_NUMA
6234 6235
	for (i = 0; i < MAX_NUMNODES; i++)
		init_numa_sched_groups_power(sched_group_nodes[i]);
6236

6237 6238
	if (sd_allnodes) {
		struct sched_group *sg;
6239

6240
		cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg);
6241 6242
		init_numa_sched_groups_power(sg);
	}
6243 6244
#endif

L
Linus Torvalds 已提交
6245
	/* Attach the domains */
6246
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6247 6248 6249
		struct sched_domain *sd;
#ifdef CONFIG_SCHED_SMT
		sd = &per_cpu(cpu_domains, i);
6250 6251
#elif defined(CONFIG_SCHED_MC)
		sd = &per_cpu(core_domains, i);
L
Linus Torvalds 已提交
6252 6253 6254 6255 6256
#else
		sd = &per_cpu(phys_domains, i);
#endif
		cpu_attach_domain(sd, i);
	}
6257 6258 6259

	return 0;

6260
#ifdef CONFIG_NUMA
6261 6262 6263
error:
	free_sched_groups(cpu_map);
	return -ENOMEM;
6264
#endif
L
Linus Torvalds 已提交
6265
}
6266 6267 6268
/*
 * Set up scheduler domains and groups.  Callers must hold the hotplug lock.
 */
6269
static int arch_init_sched_domains(const cpumask_t *cpu_map)
6270 6271
{
	cpumask_t cpu_default_map;
6272
	int err;
L
Linus Torvalds 已提交
6273

6274 6275 6276 6277 6278 6279 6280
	/*
	 * Setup mask for cpus without special case scheduling requirements.
	 * For now this just excludes isolated cpus, but could be used to
	 * exclude other special cases in the future.
	 */
	cpus_andnot(cpu_default_map, *cpu_map, cpu_isolated_map);

6281 6282 6283
	err = build_sched_domains(&cpu_default_map);

	return err;
6284 6285 6286
}

static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6287
{
6288
	free_sched_groups(cpu_map);
6289
}
L
Linus Torvalds 已提交
6290

6291 6292 6293 6294
/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
6295
static void detach_destroy_domains(const cpumask_t *cpu_map)
6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312
{
	int i;

	for_each_cpu_mask(i, *cpu_map)
		cpu_attach_domain(NULL, i);
	synchronize_sched();
	arch_destroy_sched_domains(cpu_map);
}

/*
 * Partition sched domains as specified by the cpumasks below.
 * This attaches all cpus from the cpumasks to the NULL domain,
 * waits for a RCU quiescent period, recalculates sched
 * domain information and then attaches them back to the
 * correct sched domains
 * Call with hotplug lock held
 */
6313
int partition_sched_domains(cpumask_t *partition1, cpumask_t *partition2)
6314 6315
{
	cpumask_t change_map;
6316
	int err = 0;
6317 6318 6319 6320 6321 6322 6323 6324

	cpus_and(*partition1, *partition1, cpu_online_map);
	cpus_and(*partition2, *partition2, cpu_online_map);
	cpus_or(change_map, *partition1, *partition2);

	/* Detach sched domains from all of the affected cpus */
	detach_destroy_domains(&change_map);
	if (!cpus_empty(*partition1))
6325 6326 6327 6328 6329
		err = build_sched_domains(partition1);
	if (!err && !cpus_empty(*partition2))
		err = build_sched_domains(partition2);

	return err;
6330 6331
}

6332 6333 6334 6335 6336
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
int arch_reinit_sched_domains(void)
{
	int err;

6337
	mutex_lock(&sched_hotcpu_mutex);
6338 6339
	detach_destroy_domains(&cpu_online_map);
	err = arch_init_sched_domains(&cpu_online_map);
6340
	mutex_unlock(&sched_hotcpu_mutex);
6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364

	return err;
}

static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
{
	int ret;

	if (buf[0] != '0' && buf[0] != '1')
		return -EINVAL;

	if (smt)
		sched_smt_power_savings = (buf[0] == '1');
	else
		sched_mc_power_savings = (buf[0] == '1');

	ret = arch_reinit_sched_domains();

	return ret ? ret : count;
}

int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
{
	int err = 0;
6365

6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384
#ifdef CONFIG_SCHED_SMT
	if (smt_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_smt_power_savings.attr);
#endif
#ifdef CONFIG_SCHED_MC
	if (!err && mc_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_mc_power_savings.attr);
#endif
	return err;
}
#endif

#ifdef CONFIG_SCHED_MC
static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_mc_power_savings);
}
6385 6386
static ssize_t sched_mc_power_savings_store(struct sys_device *dev,
					    const char *buf, size_t count)
6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398
{
	return sched_power_savings_store(buf, count, 0);
}
SYSDEV_ATTR(sched_mc_power_savings, 0644, sched_mc_power_savings_show,
	    sched_mc_power_savings_store);
#endif

#ifdef CONFIG_SCHED_SMT
static ssize_t sched_smt_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_smt_power_savings);
}
6399 6400
static ssize_t sched_smt_power_savings_store(struct sys_device *dev,
					     const char *buf, size_t count)
6401 6402 6403 6404 6405 6406 6407
{
	return sched_power_savings_store(buf, count, 1);
}
SYSDEV_ATTR(sched_smt_power_savings, 0644, sched_smt_power_savings_show,
	    sched_smt_power_savings_store);
#endif

L
Linus Torvalds 已提交
6408 6409 6410
/*
 * Force a reinitialization of the sched domains hierarchy.  The domains
 * and groups cannot be updated in place without racing with the balancing
N
Nick Piggin 已提交
6411
 * code, so we temporarily attach all running cpus to the NULL domain
L
Linus Torvalds 已提交
6412 6413 6414 6415 6416 6417 6418
 * which will prevent rebalancing while the sched domains are recalculated.
 */
static int update_sched_domains(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
{
	switch (action) {
	case CPU_UP_PREPARE:
6419
	case CPU_UP_PREPARE_FROZEN:
L
Linus Torvalds 已提交
6420
	case CPU_DOWN_PREPARE:
6421
	case CPU_DOWN_PREPARE_FROZEN:
6422
		detach_destroy_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6423 6424 6425
		return NOTIFY_OK;

	case CPU_UP_CANCELED:
6426
	case CPU_UP_CANCELED_FROZEN:
L
Linus Torvalds 已提交
6427
	case CPU_DOWN_FAILED:
6428
	case CPU_DOWN_FAILED_FROZEN:
L
Linus Torvalds 已提交
6429
	case CPU_ONLINE:
6430
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
6431
	case CPU_DEAD:
6432
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
6433 6434 6435 6436 6437 6438 6439 6440 6441
		/*
		 * Fall through and re-initialise the domains.
		 */
		break;
	default:
		return NOTIFY_DONE;
	}

	/* The hotplug lock is already held by cpu_up/cpu_down */
6442
	arch_init_sched_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6443 6444 6445 6446 6447 6448

	return NOTIFY_OK;
}

void __init sched_init_smp(void)
{
6449 6450
	cpumask_t non_isolated_cpus;

6451
	mutex_lock(&sched_hotcpu_mutex);
6452
	arch_init_sched_domains(&cpu_online_map);
6453
	cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map);
6454 6455
	if (cpus_empty(non_isolated_cpus))
		cpu_set(smp_processor_id(), non_isolated_cpus);
6456
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
6457 6458
	/* XXX: Theoretical race here - CPU may be hotplugged now */
	hotcpu_notifier(update_sched_domains, 0);
6459

6460 6461
	init_sched_domain_sysctl();

6462 6463 6464
	/* Move init over to a non-isolated CPU */
	if (set_cpus_allowed(current, non_isolated_cpus) < 0)
		BUG();
I
Ingo Molnar 已提交
6465
	sched_init_granularity();
L
Linus Torvalds 已提交
6466 6467 6468 6469
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
6470
	sched_init_granularity();
L
Linus Torvalds 已提交
6471 6472 6473 6474 6475 6476 6477
}
#endif /* CONFIG_SMP */

int in_sched_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __sched functions */
	extern char __sched_text_start[], __sched_text_end[];
6478

L
Linus Torvalds 已提交
6479 6480 6481 6482 6483
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

I
Ingo Molnar 已提交
6484 6485 6486 6487 6488 6489 6490 6491 6492
static inline void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
{
	cfs_rq->tasks_timeline = RB_ROOT;
	cfs_rq->fair_clock = 1;
#ifdef CONFIG_FAIR_GROUP_SCHED
	cfs_rq->rq = rq;
#endif
}

L
Linus Torvalds 已提交
6493 6494
void __init sched_init(void)
{
I
Ingo Molnar 已提交
6495
	u64 now = sched_clock();
6496
	int highest_cpu = 0;
I
Ingo Molnar 已提交
6497 6498 6499 6500 6501 6502 6503 6504
	int i, j;

	/*
	 * Link up the scheduling class hierarchy:
	 */
	rt_sched_class.next = &fair_sched_class;
	fair_sched_class.next = &idle_sched_class;
	idle_sched_class.next = NULL;
L
Linus Torvalds 已提交
6505

6506
	for_each_possible_cpu(i) {
I
Ingo Molnar 已提交
6507
		struct rt_prio_array *array;
6508
		struct rq *rq;
L
Linus Torvalds 已提交
6509 6510 6511

		rq = cpu_rq(i);
		spin_lock_init(&rq->lock);
6512
		lockdep_set_class(&rq->lock, &rq->rq_lock_key);
N
Nick Piggin 已提交
6513
		rq->nr_running = 0;
I
Ingo Molnar 已提交
6514 6515 6516 6517 6518 6519 6520 6521
		rq->clock = 1;
		init_cfs_rq(&rq->cfs, rq);
#ifdef CONFIG_FAIR_GROUP_SCHED
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
		list_add(&rq->cfs.leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
#endif
		rq->ls.load_update_last = now;
		rq->ls.load_update_start = now;
L
Linus Torvalds 已提交
6522

I
Ingo Molnar 已提交
6523 6524
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
L
Linus Torvalds 已提交
6525
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
6526
		rq->sd = NULL;
L
Linus Torvalds 已提交
6527
		rq->active_balance = 0;
I
Ingo Molnar 已提交
6528
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
6529
		rq->push_cpu = 0;
6530
		rq->cpu = i;
L
Linus Torvalds 已提交
6531 6532 6533 6534 6535
		rq->migration_thread = NULL;
		INIT_LIST_HEAD(&rq->migration_queue);
#endif
		atomic_set(&rq->nr_iowait, 0);

I
Ingo Molnar 已提交
6536 6537 6538 6539
		array = &rq->rt.active;
		for (j = 0; j < MAX_RT_PRIO; j++) {
			INIT_LIST_HEAD(array->queue + j);
			__clear_bit(j, array->bitmap);
L
Linus Torvalds 已提交
6540
		}
6541
		highest_cpu = i;
I
Ingo Molnar 已提交
6542 6543
		/* delimiter for bitsearch: */
		__set_bit(MAX_RT_PRIO, array->bitmap);
L
Linus Torvalds 已提交
6544 6545
	}

6546
	set_load_weight(&init_task);
6547

6548 6549 6550 6551
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

6552
#ifdef CONFIG_SMP
6553
	nr_cpu_ids = highest_cpu + 1;
6554 6555 6556
	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
#endif

6557 6558 6559 6560
#ifdef CONFIG_RT_MUTEXES
	plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
#endif

L
Linus Torvalds 已提交
6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
I
Ingo Molnar 已提交
6574 6575 6576 6577
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
L
Linus Torvalds 已提交
6578 6579 6580 6581 6582
}

#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
void __might_sleep(char *file, int line)
{
6583
#ifdef in_atomic
L
Linus Torvalds 已提交
6584 6585 6586 6587 6588 6589 6590
	static unsigned long prev_jiffy;	/* ratelimiting */

	if ((in_atomic() || irqs_disabled()) &&
	    system_state == SYSTEM_RUNNING && !oops_in_progress) {
		if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
			return;
		prev_jiffy = jiffies;
6591
		printk(KERN_ERR "BUG: sleeping function called from invalid"
L
Linus Torvalds 已提交
6592 6593 6594
				" context at %s:%d\n", file, line);
		printk("in_atomic():%d, irqs_disabled():%d\n",
			in_atomic(), irqs_disabled());
6595
		debug_show_held_locks(current);
6596 6597
		if (irqs_disabled())
			print_irqtrace_events(current);
L
Linus Torvalds 已提交
6598 6599 6600 6601 6602 6603 6604 6605 6606 6607
		dump_stack();
	}
#endif
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
void normalize_rt_tasks(void)
{
6608
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
6609
	unsigned long flags;
6610
	struct rq *rq;
I
Ingo Molnar 已提交
6611
	int on_rq;
L
Linus Torvalds 已提交
6612 6613

	read_lock_irq(&tasklist_lock);
6614
	do_each_thread(g, p) {
I
Ingo Molnar 已提交
6615 6616
		p->se.fair_key			= 0;
		p->se.wait_runtime		= 0;
I
Ingo Molnar 已提交
6617
		p->se.exec_start		= 0;
I
Ingo Molnar 已提交
6618
		p->se.wait_start_fair		= 0;
I
Ingo Molnar 已提交
6619 6620
		p->se.sleep_start_fair		= 0;
#ifdef CONFIG_SCHEDSTATS
I
Ingo Molnar 已提交
6621 6622 6623
		p->se.wait_start		= 0;
		p->se.sleep_start		= 0;
		p->se.block_start		= 0;
I
Ingo Molnar 已提交
6624
#endif
I
Ingo Molnar 已提交
6625 6626 6627 6628 6629 6630 6631 6632 6633 6634
		task_rq(p)->cfs.fair_clock	= 0;
		task_rq(p)->clock		= 0;

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
6635
			continue;
I
Ingo Molnar 已提交
6636
		}
L
Linus Torvalds 已提交
6637

6638 6639
		spin_lock_irqsave(&p->pi_lock, flags);
		rq = __task_rq_lock(p);
I
Ingo Molnar 已提交
6640 6641 6642 6643 6644 6645 6646
#ifdef CONFIG_SMP
		/*
		 * Do not touch the migration thread:
		 */
		if (p == rq->migration_thread)
			goto out_unlock;
#endif
L
Linus Torvalds 已提交
6647

I
Ingo Molnar 已提交
6648
		on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
6649 6650 6651 6652
		if (on_rq) {
			update_rq_clock(task_rq(p));
			deactivate_task(task_rq(p), p, 0, task_rq(p)->clock);
		}
I
Ingo Molnar 已提交
6653 6654 6655
		__setscheduler(rq, p, SCHED_NORMAL, 0);
		if (on_rq) {
			activate_task(task_rq(p), p, 0);
L
Linus Torvalds 已提交
6656 6657
			resched_task(rq->curr);
		}
I
Ingo Molnar 已提交
6658 6659 6660
#ifdef CONFIG_SMP
 out_unlock:
#endif
6661 6662
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
6663 6664
	} while_each_thread(g, p);

L
Linus Torvalds 已提交
6665 6666 6667 6668
	read_unlock_irq(&tasklist_lock);
}

#endif /* CONFIG_MAGIC_SYSRQ */
6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686

#ifdef CONFIG_IA64
/*
 * These functions are only useful for the IA64 MCA handling.
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6687
struct task_struct *curr_task(int cpu)
6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706
{
	return cpu_curr(cpu);
}

/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
 * are serviced on a separate stack.  It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner.  This function
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6707
void set_curr_task(int cpu, struct task_struct *p)
6708 6709 6710 6711 6712
{
	cpu_curr(cpu) = p;
}

#endif