sched.c 164.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
L
Linus Torvalds 已提交
25 26 27 28 29 30
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
31
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
32 33 34 35
#include <linux/highmem.h>
#include <linux/smp_lock.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
36
#include <linux/capability.h>
L
Linus Torvalds 已提交
37 38
#include <linux/completion.h>
#include <linux/kernel_stat.h>
39
#include <linux/debug_locks.h>
L
Linus Torvalds 已提交
40 41 42
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
43
#include <linux/freezer.h>
44
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
45 46 47 48 49 50 51 52 53 54 55
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
#include <linux/seq_file.h>
56
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
57 58
#include <linux/syscalls.h>
#include <linux/times.h>
59
#include <linux/tsacct_kern.h>
60
#include <linux/kprobes.h>
61
#include <linux/delayacct.h>
62
#include <linux/reciprocal_div.h>
63
#include <linux/unistd.h>
L
Linus Torvalds 已提交
64

65
#include <asm/tlb.h>
L
Linus Torvalds 已提交
66

67 68 69 70 71 72 73 74 75 76
/*
 * Scheduler clock - returns current time in nanosec units.
 * This is default implementation.
 * Architectures and sub-architectures can override this.
 */
unsigned long long __attribute__((weak)) sched_clock(void)
{
	return (unsigned long long)jiffies * (1000000000 / HZ);
}

L
Linus Torvalds 已提交
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
 * Some helpers for converting nanosecond timing to jiffy resolution
 */
#define NS_TO_JIFFIES(TIME)	((TIME) / (1000000000 / HZ))
#define JIFFIES_TO_NS(TIME)	((TIME) * (1000000000 / HZ))

I
Ingo Molnar 已提交
101 102 103
#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

L
Linus Torvalds 已提交
104 105 106 107 108 109 110 111 112
/*
 * These are the 'tuning knobs' of the scheduler:
 *
 * Minimum timeslice is 5 msecs (or 1 jiffy, whichever is larger),
 * default timeslice is 100 msecs, maximum timeslice is 800 msecs.
 * Timeslices get refilled after they expire.
 */
#define MIN_TIMESLICE		max(5 * HZ / 1000, 1)
#define DEF_TIMESLICE		(100 * HZ / 1000)
113

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
#ifdef CONFIG_SMP
/*
 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 */
static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
{
	return reciprocal_divide(load, sg->reciprocal_cpu_power);
}

/*
 * Each time a sched group cpu_power is changed,
 * we must compute its reciprocal value
 */
static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
{
	sg->__cpu_power += val;
	sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
}
#endif

I
Ingo Molnar 已提交
135 136 137
#define SCALE_PRIO(x, prio) \
	max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_TIMESLICE)

138
/*
I
Ingo Molnar 已提交
139
 * static_prio_timeslice() scales user-nice values [ -20 ... 0 ... 19 ]
140 141
 * to time slice values: [800ms ... 100ms ... 5ms]
 */
I
Ingo Molnar 已提交
142
static unsigned int static_prio_timeslice(int static_prio)
143
{
I
Ingo Molnar 已提交
144 145 146 147 148 149 150
	if (static_prio == NICE_TO_PRIO(19))
		return 1;

	if (static_prio < NICE_TO_PRIO(0))
		return SCALE_PRIO(DEF_TIMESLICE * 4, static_prio);
	else
		return SCALE_PRIO(DEF_TIMESLICE, static_prio);
151 152
}

153 154 155 156 157 158 159 160 161 162 163 164
static inline int rt_policy(int policy)
{
	if (unlikely(policy == SCHED_FIFO) || unlikely(policy == SCHED_RR))
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

L
Linus Torvalds 已提交
165
/*
I
Ingo Molnar 已提交
166
 * This is the priority-queue data structure of the RT scheduling class:
L
Linus Torvalds 已提交
167
 */
I
Ingo Molnar 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

struct load_stat {
	struct load_weight load;
	u64 load_update_start, load_update_last;
	unsigned long delta_fair, delta_exec, delta_stat;
};

/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	s64 fair_clock;
	u64 exec_clock;
	s64 wait_runtime;
	u64 sleeper_bonus;
	unsigned long wait_runtime_overruns, wait_runtime_underruns;

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;
	struct rb_node *rb_load_balance_curr;
#ifdef CONFIG_FAIR_GROUP_SCHED
	/* 'curr' points to currently running entity on this cfs_rq.
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
	struct sched_entity *curr;
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

	/* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
	struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */
#endif
};
L
Linus Torvalds 已提交
210

I
Ingo Molnar 已提交
211 212 213 214 215 216 217
/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
	int rt_load_balance_idx;
	struct list_head *rt_load_balance_head, *rt_load_balance_curr;
};

L
Linus Torvalds 已提交
218 219 220 221 222 223 224
/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
225
struct rq {
I
Ingo Molnar 已提交
226
	spinlock_t lock;	/* runqueue lock */
L
Linus Torvalds 已提交
227 228 229 230 231 232

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
I
Ingo Molnar 已提交
233 234
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
235
	unsigned char idle_at_tick;
236 237 238
#ifdef CONFIG_NO_HZ
	unsigned char in_nohz_recently;
#endif
I
Ingo Molnar 已提交
239 240 241 242 243 244 245
	struct load_stat ls;	/* capture load from *all* tasks on this cpu */
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
#ifdef CONFIG_FAIR_GROUP_SCHED
	struct list_head leaf_cfs_rq_list; /* list of leaf cfs_rq on this cpu */
L
Linus Torvalds 已提交
246
#endif
I
Ingo Molnar 已提交
247
	struct rt_rq  rt;
L
Linus Torvalds 已提交
248 249 250 251 252 253 254 255 256

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

257
	struct task_struct *curr, *idle;
258
	unsigned long next_balance;
L
Linus Torvalds 已提交
259
	struct mm_struct *prev_mm;
I
Ingo Molnar 已提交
260 261 262 263 264 265 266

	u64 clock, prev_clock_raw;
	s64 clock_max_delta;

	unsigned int clock_warps, clock_overflows;
	unsigned int clock_unstable_events;

L
Linus Torvalds 已提交
267 268 269 270 271 272 273 274
	atomic_t nr_iowait;

#ifdef CONFIG_SMP
	struct sched_domain *sd;

	/* For active balancing */
	int active_balance;
	int push_cpu;
275
	int cpu;		/* cpu of this runqueue */
L
Linus Torvalds 已提交
276

277
	struct task_struct *migration_thread;
L
Linus Torvalds 已提交
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
	struct list_head migration_queue;
#endif

#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;

	/* sys_sched_yield() stats */
	unsigned long yld_exp_empty;
	unsigned long yld_act_empty;
	unsigned long yld_both_empty;
	unsigned long yld_cnt;

	/* schedule() stats */
	unsigned long sched_switch;
	unsigned long sched_cnt;
	unsigned long sched_goidle;

	/* try_to_wake_up() stats */
	unsigned long ttwu_cnt;
	unsigned long ttwu_local;
#endif
300
	struct lock_class_key rq_lock_key;
L
Linus Torvalds 已提交
301 302
};

303
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
304
static DEFINE_MUTEX(sched_hotcpu_mutex);
L
Linus Torvalds 已提交
305

I
Ingo Molnar 已提交
306 307 308 309 310
static inline void check_preempt_curr(struct rq *rq, struct task_struct *p)
{
	rq->curr->sched_class->check_preempt_curr(rq, p);
}

311 312 313 314 315 316 317 318 319
static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

I
Ingo Molnar 已提交
320
/*
I
Ingo Molnar 已提交
321 322
 * Update the per-runqueue clock, as finegrained as the platform can give
 * us, but without assuming monotonicity, etc.:
I
Ingo Molnar 已提交
323
 */
I
Ingo Molnar 已提交
324
static void __update_rq_clock(struct rq *rq)
I
Ingo Molnar 已提交
325 326 327 328 329 330
{
	u64 prev_raw = rq->prev_clock_raw;
	u64 now = sched_clock();
	s64 delta = now - prev_raw;
	u64 clock = rq->clock;

I
Ingo Molnar 已提交
331 332 333
#ifdef CONFIG_SCHED_DEBUG
	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
#endif
I
Ingo Molnar 已提交
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
	/*
	 * Protect against sched_clock() occasionally going backwards:
	 */
	if (unlikely(delta < 0)) {
		clock++;
		rq->clock_warps++;
	} else {
		/*
		 * Catch too large forward jumps too:
		 */
		if (unlikely(delta > 2*TICK_NSEC)) {
			clock++;
			rq->clock_overflows++;
		} else {
			if (unlikely(delta > rq->clock_max_delta))
				rq->clock_max_delta = delta;
			clock += delta;
		}
	}

	rq->prev_clock_raw = now;
	rq->clock = clock;
I
Ingo Molnar 已提交
356
}
I
Ingo Molnar 已提交
357

I
Ingo Molnar 已提交
358 359 360 361
static void update_rq_clock(struct rq *rq)
{
	if (likely(smp_processor_id() == cpu_of(rq)))
		__update_rq_clock(rq);
I
Ingo Molnar 已提交
362 363
}

N
Nick Piggin 已提交
364 365
/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
366
 * See detach_destroy_domains: synchronize_sched for details.
N
Nick Piggin 已提交
367 368 369 370
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
371 372
#define for_each_domain(cpu, __sd) \
	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
L
Linus Torvalds 已提交
373 374 375 376 377 378

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

379 380 381 382 383 384 385 386
/*
 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
 * clock constructed from sched_clock():
 */
unsigned long long cpu_clock(int cpu)
{
	unsigned long long now;
	unsigned long flags;
I
Ingo Molnar 已提交
387
	struct rq *rq;
388

389
	local_irq_save(flags);
I
Ingo Molnar 已提交
390 391 392
	rq = cpu_rq(cpu);
	update_rq_clock(rq);
	now = rq->clock;
393
	local_irq_restore(flags);
394 395 396 397

	return now;
}

I
Ingo Molnar 已提交
398 399 400 401 402 403 404 405 406 407 408 409
#ifdef CONFIG_FAIR_GROUP_SCHED
/* Change a task's ->cfs_rq if it moves across CPUs */
static inline void set_task_cfs_rq(struct task_struct *p)
{
	p->se.cfs_rq = &task_rq(p)->cfs;
}
#else
static inline void set_task_cfs_rq(struct task_struct *p)
{
}
#endif

L
Linus Torvalds 已提交
410
#ifndef prepare_arch_switch
411 412 413 414 415 416 417
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

#ifndef __ARCH_WANT_UNLOCKED_CTXSW
418
static inline int task_running(struct rq *rq, struct task_struct *p)
419 420 421 422
{
	return rq->curr == p;
}

423
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
424 425 426
{
}

427
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
428
{
429 430 431 432
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
433 434 435 436 437 438 439
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

440 441 442 443
	spin_unlock_irq(&rq->lock);
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
444
static inline int task_running(struct rq *rq, struct task_struct *p)
445 446 447 448 449 450 451 452
{
#ifdef CONFIG_SMP
	return p->oncpu;
#else
	return rq->curr == p;
#endif
}

453
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->oncpu = 1;
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	spin_unlock_irq(&rq->lock);
#else
	spin_unlock(&rq->lock);
#endif
}

470
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
471 472 473 474 475 476 477 478 479 480 481 482
{
#ifdef CONFIG_SMP
	/*
	 * After ->oncpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->oncpu = 0;
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
L
Linus Torvalds 已提交
483
#endif
484 485
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
L
Linus Torvalds 已提交
486

487 488 489 490
/*
 * __task_rq_lock - lock the runqueue a given task resides on.
 * Must be called interrupts disabled.
 */
491
static inline struct rq *__task_rq_lock(struct task_struct *p)
492 493
	__acquires(rq->lock)
{
494
	struct rq *rq;
495 496 497 498 499 500 501 502 503 504 505

repeat_lock_task:
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock(&rq->lock);
		goto repeat_lock_task;
	}
	return rq;
}

L
Linus Torvalds 已提交
506 507 508 509 510
/*
 * task_rq_lock - lock the runqueue a given task resides on and disable
 * interrupts.  Note the ordering: we can safely lookup the task_rq without
 * explicitly disabling preemption.
 */
511
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
512 513
	__acquires(rq->lock)
{
514
	struct rq *rq;
L
Linus Torvalds 已提交
515 516 517 518 519 520 521 522 523 524 525 526

repeat_lock_task:
	local_irq_save(*flags);
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock_irqrestore(&rq->lock, *flags);
		goto repeat_lock_task;
	}
	return rq;
}

527
static inline void __task_rq_unlock(struct rq *rq)
528 529 530 531 532
	__releases(rq->lock)
{
	spin_unlock(&rq->lock);
}

533
static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
L
Linus Torvalds 已提交
534 535 536 537 538 539
	__releases(rq->lock)
{
	spin_unlock_irqrestore(&rq->lock, *flags);
}

/*
540
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
541
 */
542
static inline struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
543 544
	__acquires(rq->lock)
{
545
	struct rq *rq;
L
Linus Torvalds 已提交
546 547 548 549 550 551 552 553

	local_irq_disable();
	rq = this_rq();
	spin_lock(&rq->lock);

	return rq;
}

554 555 556 557 558 559 560 561 562 563 564 565 566 567
/*
 * CPU frequency is/was unstable - start new by setting prev_clock_raw:
 */
void sched_clock_unstable_event(void)
{
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(current, &flags);
	rq->prev_clock_raw = sched_clock();
	rq->clock_unstable_events++;
	task_rq_unlock(rq, &flags);
}

I
Ingo Molnar 已提交
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

static void resched_task(struct task_struct *p)
{
	int cpu;

	assert_spin_locked(&task_rq(p)->lock);

	if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
		return;

	set_tsk_thread_flag(p, TIF_NEED_RESCHED);

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	if (!spin_trylock_irqsave(&rq->lock, flags))
		return;
	resched_task(cpu_curr(cpu));
	spin_unlock_irqrestore(&rq->lock, flags);
}
#else
static inline void resched_task(struct task_struct *p)
{
	assert_spin_locked(&task_rq(p)->lock);
	set_tsk_need_resched(p);
}
#endif

620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
static u64 div64_likely32(u64 divident, unsigned long divisor)
{
#if BITS_PER_LONG == 32
	if (likely(divident <= 0xffffffffULL))
		return (u32)divident / divisor;
	do_div(divident, divisor);

	return divident;
#else
	return divident / divisor;
#endif
}

#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

641
static unsigned long
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

	if (unlikely(!lw->inv_weight))
		lw->inv_weight = WMULT_CONST / lw->weight;

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
	if (unlikely(tmp > WMULT_CONST)) {
		tmp = ((tmp >> WMULT_SHIFT/2) * lw->inv_weight)
				>> (WMULT_SHIFT/2);
	} else {
		tmp = (tmp * lw->inv_weight) >> WMULT_SHIFT;
	}

661
	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
}

static inline unsigned long
calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
{
	return calc_delta_mine(delta_exec, NICE_0_LOAD, lw);
}

static void update_load_add(struct load_weight *lw, unsigned long inc)
{
	lw->weight += inc;
	lw->inv_weight = 0;
}

static void update_load_sub(struct load_weight *lw, unsigned long dec)
{
	lw->weight -= dec;
	lw->inv_weight = 0;
}

682 683 684 685 686 687 688 689 690
/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
 * scheduling class and "nice" value.  For SCHED_NORMAL tasks this is just a
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

I
Ingo Molnar 已提交
691 692 693 694 695 696 697 698 699 700 701
#define WEIGHT_IDLEPRIO		2
#define WMULT_IDLEPRIO		(1 << 31)

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
702 703 704
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
I
Ingo Molnar 已提交
705 706 707 708 709 710 711 712 713
 */
static const int prio_to_weight[40] = {
/* -20 */ 88818, 71054, 56843, 45475, 36380, 29104, 23283, 18626, 14901, 11921,
/* -10 */  9537,  7629,  6103,  4883,  3906,  3125,  2500,  2000,  1600,  1280,
/*   0 */  NICE_0_LOAD /* 1024 */,
/*   1 */          819,   655,   524,   419,   336,   268,   215,   172,   137,
/*  10 */   110,    87,    70,    56,    45,    36,    29,    23,    18,    15,
};

714 715 716 717 718 719 720
/*
 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
 *
 * In cases where the weight does not change often, we can use the
 * precalculated inverse to speed up arithmetics by turning divisions
 * into multiplications:
 */
I
Ingo Molnar 已提交
721
static const u32 prio_to_wmult[40] = {
722 723 724 725 726 727 728 729
/* -20 */     48356,     60446,     75558,     94446,    118058,
/* -15 */    147573,    184467,    230589,    288233,    360285,
/* -10 */    450347,    562979,    703746,    879575,   1099582,
/*  -5 */   1374389,   1717986,   2147483,   2684354,   3355443,
/*   0 */   4194304,   5244160,   6557201,   8196502,  10250518,
/*   5 */  12782640,  16025997,  19976592,  24970740,  31350126,
/*  10 */  39045157,  49367440,  61356675,  76695844,  95443717,
/*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
I
Ingo Molnar 已提交
730
};
731

I
Ingo Molnar 已提交
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);

/*
 * runqueue iterator, to support SMP load-balancing between different
 * scheduling classes, without having to expose their internal data
 * structures to the load-balancing proper:
 */
struct rq_iterator {
	void *arg;
	struct task_struct *(*start)(void *);
	struct task_struct *(*next)(void *);
};

static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
749
		      int *this_best_prio, struct rq_iterator *iterator);
I
Ingo Molnar 已提交
750 751 752 753 754 755 756 757 758 759 760

#include "sched_stats.h"
#include "sched_rt.c"
#include "sched_fair.c"
#include "sched_idletask.c"
#ifdef CONFIG_SCHED_DEBUG
# include "sched_debug.c"
#endif

#define sched_class_highest (&rt_sched_class)

761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
static void __update_curr_load(struct rq *rq, struct load_stat *ls)
{
	if (rq->curr != rq->idle && ls->load.weight) {
		ls->delta_exec += ls->delta_stat;
		ls->delta_fair += calc_delta_fair(ls->delta_stat, &ls->load);
		ls->delta_stat = 0;
	}
}

/*
 * Update delta_exec, delta_fair fields for rq.
 *
 * delta_fair clock advances at a rate inversely proportional to
 * total load (rq->ls.load.weight) on the runqueue, while
 * delta_exec advances at the same rate as wall-clock (provided
 * cpu is not idle).
 *
 * delta_exec / delta_fair is a measure of the (smoothened) load on this
 * runqueue over any given interval. This (smoothened) load is used
 * during load balance.
 *
 * This function is called /before/ updating rq->ls.load
 * and when switching tasks.
 */
785
static void update_curr_load(struct rq *rq)
786 787 788 789 790
{
	struct load_stat *ls = &rq->ls;
	u64 start;

	start = ls->load_update_start;
791 792
	ls->load_update_start = rq->clock;
	ls->delta_stat += rq->clock - start;
793 794 795 796 797 798 799 800
	/*
	 * Stagger updates to ls->delta_fair. Very frequent updates
	 * can be expensive.
	 */
	if (ls->delta_stat >= sysctl_sched_stat_granularity)
		__update_curr_load(rq, ls);
}

801
static inline void inc_load(struct rq *rq, const struct task_struct *p)
802
{
803
	update_curr_load(rq);
804 805 806
	update_load_add(&rq->ls.load, p->se.load.weight);
}

807
static inline void dec_load(struct rq *rq, const struct task_struct *p)
808
{
809
	update_curr_load(rq);
810 811 812
	update_load_sub(&rq->ls.load, p->se.load.weight);
}

813
static void inc_nr_running(struct task_struct *p, struct rq *rq)
814 815
{
	rq->nr_running++;
816
	inc_load(rq, p);
817 818
}

819
static void dec_nr_running(struct task_struct *p, struct rq *rq)
820 821
{
	rq->nr_running--;
822
	dec_load(rq, p);
823 824
}

825 826
static void set_load_weight(struct task_struct *p)
{
I
Ingo Molnar 已提交
827 828 829
	task_rq(p)->cfs.wait_runtime -= p->se.wait_runtime;
	p->se.wait_runtime = 0;

830
	if (task_has_rt_policy(p)) {
I
Ingo Molnar 已提交
831 832 833 834
		p->se.load.weight = prio_to_weight[0] * 2;
		p->se.load.inv_weight = prio_to_wmult[0] >> 1;
		return;
	}
835

I
Ingo Molnar 已提交
836 837 838 839 840 841 842 843
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
		p->se.load.weight = WEIGHT_IDLEPRIO;
		p->se.load.inv_weight = WMULT_IDLEPRIO;
		return;
	}
844

I
Ingo Molnar 已提交
845 846
	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
847 848
}

I
Ingo Molnar 已提交
849 850
static void
enqueue_task(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
851
{
I
Ingo Molnar 已提交
852
	sched_info_queued(p);
853
	p->sched_class->enqueue_task(rq, p, wakeup);
I
Ingo Molnar 已提交
854
	p->se.on_rq = 1;
855 856
}

I
Ingo Molnar 已提交
857 858
static void
dequeue_task(struct rq *rq, struct task_struct *p, int sleep, u64 now)
859
{
860
	p->sched_class->dequeue_task(rq, p, sleep);
I
Ingo Molnar 已提交
861
	p->se.on_rq = 0;
862 863
}

864
/*
I
Ingo Molnar 已提交
865
 * __normal_prio - return the priority that is based on the static prio
866 867 868
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
869
	return p->static_prio;
870 871
}

872 873 874 875 876 877 878
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
879
static inline int normal_prio(struct task_struct *p)
880 881 882
{
	int prio;

883
	if (task_has_rt_policy(p))
884 885 886 887 888 889 890 891 892 893 894 895 896
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
897
static int effective_prio(struct task_struct *p)
898 899 900 901 902 903 904 905 906 907 908 909
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
910
/*
I
Ingo Molnar 已提交
911
 * activate_task - move a task to the runqueue.
L
Linus Torvalds 已提交
912
 */
I
Ingo Molnar 已提交
913
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
L
Linus Torvalds 已提交
914
{
I
Ingo Molnar 已提交
915 916 917 918
	u64 now;

	update_rq_clock(rq);
	now = rq->clock;
919

I
Ingo Molnar 已提交
920 921
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;
L
Linus Torvalds 已提交
922

I
Ingo Molnar 已提交
923
	enqueue_task(rq, p, wakeup, now);
924
	inc_nr_running(p, rq);
L
Linus Torvalds 已提交
925 926 927
}

/*
I
Ingo Molnar 已提交
928
 * activate_idle_task - move idle task to the _front_ of runqueue.
L
Linus Torvalds 已提交
929
 */
I
Ingo Molnar 已提交
930
static inline void activate_idle_task(struct task_struct *p, struct rq *rq)
L
Linus Torvalds 已提交
931
{
I
Ingo Molnar 已提交
932 933 934 935
	u64 now;

	update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
936

I
Ingo Molnar 已提交
937 938
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;
I
Ingo Molnar 已提交
939

I
Ingo Molnar 已提交
940
	enqueue_task(rq, p, 0, now);
941
	inc_nr_running(p, rq);
L
Linus Torvalds 已提交
942 943 944 945 946
}

/*
 * deactivate_task - remove a task from the runqueue.
 */
I
Ingo Molnar 已提交
947 948
static void
deactivate_task(struct rq *rq, struct task_struct *p, int sleep, u64 now)
L
Linus Torvalds 已提交
949
{
I
Ingo Molnar 已提交
950 951 952 953
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible++;

	dequeue_task(rq, p, sleep, now);
954
	dec_nr_running(p, rq);
L
Linus Torvalds 已提交
955 956 957 958 959 960
}

/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
961
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
962 963 964 965
{
	return cpu_curr(task_cpu(p)) == p;
}

966 967 968
/* Used instead of source_load when we know the type == 0 */
unsigned long weighted_cpuload(const int cpu)
{
I
Ingo Molnar 已提交
969 970 971 972 973 974 975 976 977
	return cpu_rq(cpu)->ls.load.weight;
}

static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
#ifdef CONFIG_SMP
	task_thread_info(p)->cpu = cpu;
	set_task_cfs_rq(p);
#endif
978 979
}

L
Linus Torvalds 已提交
980
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
981

I
Ingo Molnar 已提交
982
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
983
{
I
Ingo Molnar 已提交
984 985 986 987 988
	int old_cpu = task_cpu(p);
	struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
	u64 clock_offset, fair_clock_offset;

	clock_offset = old_rq->clock - new_rq->clock;
I
Ingo Molnar 已提交
989 990
	fair_clock_offset = old_rq->cfs.fair_clock - new_rq->cfs.fair_clock;

I
Ingo Molnar 已提交
991 992
	if (p->se.wait_start_fair)
		p->se.wait_start_fair -= fair_clock_offset;
I
Ingo Molnar 已提交
993 994 995 996 997 998
	if (p->se.sleep_start_fair)
		p->se.sleep_start_fair -= fair_clock_offset;

#ifdef CONFIG_SCHEDSTATS
	if (p->se.wait_start)
		p->se.wait_start -= clock_offset;
I
Ingo Molnar 已提交
999 1000 1001 1002
	if (p->se.sleep_start)
		p->se.sleep_start -= clock_offset;
	if (p->se.block_start)
		p->se.block_start -= clock_offset;
I
Ingo Molnar 已提交
1003
#endif
I
Ingo Molnar 已提交
1004 1005

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1006 1007
}

1008
struct migration_req {
L
Linus Torvalds 已提交
1009 1010
	struct list_head list;

1011
	struct task_struct *task;
L
Linus Torvalds 已提交
1012 1013 1014
	int dest_cpu;

	struct completion done;
1015
};
L
Linus Torvalds 已提交
1016 1017 1018 1019 1020

/*
 * The task's runqueue lock must be held.
 * Returns true if you have to wait for migration thread.
 */
1021
static int
1022
migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
L
Linus Torvalds 已提交
1023
{
1024
	struct rq *rq = task_rq(p);
L
Linus Torvalds 已提交
1025 1026 1027 1028 1029

	/*
	 * If the task is not on a runqueue (and not running), then
	 * it is sufficient to simply update the task's cpu field.
	 */
I
Ingo Molnar 已提交
1030
	if (!p->se.on_rq && !task_running(rq, p)) {
L
Linus Torvalds 已提交
1031 1032 1033 1034 1035 1036 1037 1038
		set_task_cpu(p, dest_cpu);
		return 0;
	}

	init_completion(&req->done);
	req->task = p;
	req->dest_cpu = dest_cpu;
	list_add(&req->list, &rq->migration_queue);
1039

L
Linus Torvalds 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
	return 1;
}

/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
1052
void wait_task_inactive(struct task_struct *p)
L
Linus Torvalds 已提交
1053 1054
{
	unsigned long flags;
I
Ingo Molnar 已提交
1055
	int running, on_rq;
1056
	struct rq *rq;
L
Linus Torvalds 已提交
1057 1058

repeat:
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
	/*
	 * We do the initial early heuristics without holding
	 * any task-queue locks at all. We'll only try to get
	 * the runqueue lock when things look like they will
	 * work out!
	 */
	rq = task_rq(p);

	/*
	 * If the task is actively running on another CPU
	 * still, just relax and busy-wait without holding
	 * any locks.
	 *
	 * NOTE! Since we don't hold any locks, it's not
	 * even sure that "rq" stays as the right runqueue!
	 * But we don't care, since "task_running()" will
	 * return false if the runqueue has changed and p
	 * is actually now running somewhere else!
	 */
	while (task_running(rq, p))
		cpu_relax();

	/*
	 * Ok, time to look more closely! We need the rq
	 * lock now, to be *sure*. If we're wrong, we'll
	 * just go back and repeat.
	 */
L
Linus Torvalds 已提交
1086
	rq = task_rq_lock(p, &flags);
1087
	running = task_running(rq, p);
I
Ingo Molnar 已提交
1088
	on_rq = p->se.on_rq;
1089 1090 1091 1092 1093 1094 1095 1096 1097
	task_rq_unlock(rq, &flags);

	/*
	 * Was it really running after all now that we
	 * checked with the proper locks actually held?
	 *
	 * Oops. Go back and try again..
	 */
	if (unlikely(running)) {
L
Linus Torvalds 已提交
1098 1099 1100
		cpu_relax();
		goto repeat;
	}
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110

	/*
	 * It's not enough that it's not actively running,
	 * it must be off the runqueue _entirely_, and not
	 * preempted!
	 *
	 * So if it wa still runnable (but just not actively
	 * running right now), it's preempted, and we should
	 * yield - it could be a while.
	 */
I
Ingo Molnar 已提交
1111
	if (unlikely(on_rq)) {
1112 1113 1114 1115 1116 1117 1118 1119 1120
		yield();
		goto repeat;
	}

	/*
	 * Ahh, all good. It wasn't running, and it wasn't
	 * runnable, which means that it will never become
	 * running in the future either. We're all done!
	 */
L
Linus Torvalds 已提交
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
 * NOTE: this function doesnt have to take the runqueue lock,
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
1136
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}

/*
1148 1149
 * Return a low guess at the load of a migration-source cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1150 1151 1152 1153
 *
 * We want to under-estimate the load of migration sources, to
 * balance conservatively.
 */
N
Nick Piggin 已提交
1154
static inline unsigned long source_load(int cpu, int type)
L
Linus Torvalds 已提交
1155
{
1156
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1157
	unsigned long total = weighted_cpuload(cpu);
1158

1159
	if (type == 0)
I
Ingo Molnar 已提交
1160
		return total;
1161

I
Ingo Molnar 已提交
1162
	return min(rq->cpu_load[type-1], total);
L
Linus Torvalds 已提交
1163 1164 1165
}

/*
1166 1167
 * Return a high guess at the load of a migration-target cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1168
 */
N
Nick Piggin 已提交
1169
static inline unsigned long target_load(int cpu, int type)
L
Linus Torvalds 已提交
1170
{
1171
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1172
	unsigned long total = weighted_cpuload(cpu);
1173

N
Nick Piggin 已提交
1174
	if (type == 0)
I
Ingo Molnar 已提交
1175
		return total;
1176

I
Ingo Molnar 已提交
1177
	return max(rq->cpu_load[type-1], total);
1178 1179 1180 1181 1182 1183 1184
}

/*
 * Return the average load per task on the cpu's run queue
 */
static inline unsigned long cpu_avg_load_per_task(int cpu)
{
1185
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1186
	unsigned long total = weighted_cpuload(cpu);
1187 1188
	unsigned long n = rq->nr_running;

I
Ingo Molnar 已提交
1189
	return n ? total / n : SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
1190 1191
}

N
Nick Piggin 已提交
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
/*
 * find_idlest_group finds and returns the least busy CPU group within the
 * domain.
 */
static struct sched_group *
find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
{
	struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
	unsigned long min_load = ULONG_MAX, this_load = 0;
	int load_idx = sd->forkexec_idx;
	int imbalance = 100 + (sd->imbalance_pct-100)/2;

	do {
		unsigned long load, avg_load;
		int local_group;
		int i;

1209 1210 1211 1212
		/* Skip over this group if it has no CPUs allowed */
		if (!cpus_intersects(group->cpumask, p->cpus_allowed))
			goto nextgroup;

N
Nick Piggin 已提交
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
		local_group = cpu_isset(this_cpu, group->cpumask);

		/* Tally up the load of all CPUs in the group */
		avg_load = 0;

		for_each_cpu_mask(i, group->cpumask) {
			/* Bias balancing toward cpus of our domain */
			if (local_group)
				load = source_load(i, load_idx);
			else
				load = target_load(i, load_idx);

			avg_load += load;
		}

		/* Adjust by relative CPU power of the group */
1229 1230
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
N
Nick Piggin 已提交
1231 1232 1233 1234 1235 1236 1237 1238

		if (local_group) {
			this_load = avg_load;
			this = group;
		} else if (avg_load < min_load) {
			min_load = avg_load;
			idlest = group;
		}
1239
nextgroup:
N
Nick Piggin 已提交
1240 1241 1242 1243 1244 1245 1246 1247 1248
		group = group->next;
	} while (group != sd->groups);

	if (!idlest || 100*this_load < imbalance*min_load)
		return NULL;
	return idlest;
}

/*
1249
 * find_idlest_cpu - find the idlest cpu among the cpus in group.
N
Nick Piggin 已提交
1250
 */
I
Ingo Molnar 已提交
1251 1252
static int
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
N
Nick Piggin 已提交
1253
{
1254
	cpumask_t tmp;
N
Nick Piggin 已提交
1255 1256 1257 1258
	unsigned long load, min_load = ULONG_MAX;
	int idlest = -1;
	int i;

1259 1260 1261 1262
	/* Traverse only the allowed CPUs */
	cpus_and(tmp, group->cpumask, p->cpus_allowed);

	for_each_cpu_mask(i, tmp) {
1263
		load = weighted_cpuload(i);
N
Nick Piggin 已提交
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273

		if (load < min_load || (load == min_load && i == this_cpu)) {
			min_load = load;
			idlest = i;
		}
	}

	return idlest;
}

N
Nick Piggin 已提交
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
/*
 * sched_balance_self: balance the current task (running on cpu) in domains
 * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
 * SD_BALANCE_EXEC.
 *
 * Balance, ie. select the least loaded group.
 *
 * Returns the target CPU number, or the same CPU if no balancing is needed.
 *
 * preempt must be disabled.
 */
static int sched_balance_self(int cpu, int flag)
{
	struct task_struct *t = current;
	struct sched_domain *tmp, *sd = NULL;
N
Nick Piggin 已提交
1289

1290
	for_each_domain(cpu, tmp) {
I
Ingo Molnar 已提交
1291 1292 1293
		/*
		 * If power savings logic is enabled for a domain, stop there.
		 */
1294 1295
		if (tmp->flags & SD_POWERSAVINGS_BALANCE)
			break;
N
Nick Piggin 已提交
1296 1297
		if (tmp->flags & flag)
			sd = tmp;
1298
	}
N
Nick Piggin 已提交
1299 1300 1301 1302

	while (sd) {
		cpumask_t span;
		struct sched_group *group;
1303 1304 1305 1306 1307 1308
		int new_cpu, weight;

		if (!(sd->flags & flag)) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1309 1310 1311

		span = sd->span;
		group = find_idlest_group(sd, t, cpu);
1312 1313 1314 1315
		if (!group) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1316

1317
		new_cpu = find_idlest_cpu(group, t, cpu);
1318 1319 1320 1321 1322
		if (new_cpu == -1 || new_cpu == cpu) {
			/* Now try balancing at a lower domain level of cpu */
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1323

1324
		/* Now try balancing at a lower domain level of new_cpu */
N
Nick Piggin 已提交
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340
		cpu = new_cpu;
		sd = NULL;
		weight = cpus_weight(span);
		for_each_domain(cpu, tmp) {
			if (weight <= cpus_weight(tmp->span))
				break;
			if (tmp->flags & flag)
				sd = tmp;
		}
		/* while loop will break here if sd == NULL */
	}

	return cpu;
}

#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350

/*
 * wake_idle() will wake a task on an idle cpu if task->cpu is
 * not idle and an idle cpu is available.  The span of cpus to
 * search starts with cpus closest then further out as needed,
 * so we always favor a closer, idle cpu.
 *
 * Returns the CPU we should wake onto.
 */
#if defined(ARCH_HAS_SCHED_WAKE_IDLE)
1351
static int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1352 1353 1354 1355 1356
{
	cpumask_t tmp;
	struct sched_domain *sd;
	int i;

1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
	/*
	 * If it is idle, then it is the best cpu to run this task.
	 *
	 * This cpu is also the best, if it has more than one task already.
	 * Siblings must be also busy(in most cases) as they didn't already
	 * pickup the extra load from this cpu and hence we need not check
	 * sibling runqueue info. This will avoid the checks and cache miss
	 * penalities associated with that.
	 */
	if (idle_cpu(cpu) || cpu_rq(cpu)->nr_running > 1)
L
Linus Torvalds 已提交
1367 1368 1369 1370
		return cpu;

	for_each_domain(cpu, sd) {
		if (sd->flags & SD_WAKE_IDLE) {
N
Nick Piggin 已提交
1371
			cpus_and(tmp, sd->span, p->cpus_allowed);
L
Linus Torvalds 已提交
1372 1373 1374 1375
			for_each_cpu_mask(i, tmp) {
				if (idle_cpu(i))
					return i;
			}
I
Ingo Molnar 已提交
1376
		} else {
N
Nick Piggin 已提交
1377
			break;
I
Ingo Molnar 已提交
1378
		}
L
Linus Torvalds 已提交
1379 1380 1381 1382
	}
	return cpu;
}
#else
1383
static inline int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
{
	return cpu;
}
#endif

/***
 * try_to_wake_up - wake up a thread
 * @p: the to-be-woken-up thread
 * @state: the mask of task states that can be woken
 * @sync: do a synchronous wakeup?
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
 * returns failure only if the task is already active.
 */
1403
static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
L
Linus Torvalds 已提交
1404 1405 1406 1407
{
	int cpu, this_cpu, success = 0;
	unsigned long flags;
	long old_state;
1408
	struct rq *rq;
L
Linus Torvalds 已提交
1409
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
1410
	struct sched_domain *sd, *this_sd = NULL;
1411
	unsigned long load, this_load;
L
Linus Torvalds 已提交
1412 1413 1414 1415 1416 1417 1418 1419
	int new_cpu;
#endif

	rq = task_rq_lock(p, &flags);
	old_state = p->state;
	if (!(old_state & state))
		goto out;

I
Ingo Molnar 已提交
1420
	if (p->se.on_rq)
L
Linus Torvalds 已提交
1421 1422 1423 1424 1425 1426 1427 1428 1429
		goto out_running;

	cpu = task_cpu(p);
	this_cpu = smp_processor_id();

#ifdef CONFIG_SMP
	if (unlikely(task_running(rq, p)))
		goto out_activate;

N
Nick Piggin 已提交
1430 1431
	new_cpu = cpu;

L
Linus Torvalds 已提交
1432 1433 1434
	schedstat_inc(rq, ttwu_cnt);
	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
N
Nick Piggin 已提交
1435 1436 1437 1438 1439 1440 1441 1442
		goto out_set_cpu;
	}

	for_each_domain(this_cpu, sd) {
		if (cpu_isset(cpu, sd->span)) {
			schedstat_inc(sd, ttwu_wake_remote);
			this_sd = sd;
			break;
L
Linus Torvalds 已提交
1443 1444 1445
		}
	}

N
Nick Piggin 已提交
1446
	if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed)))
L
Linus Torvalds 已提交
1447 1448 1449
		goto out_set_cpu;

	/*
N
Nick Piggin 已提交
1450
	 * Check for affine wakeup and passive balancing possibilities.
L
Linus Torvalds 已提交
1451
	 */
N
Nick Piggin 已提交
1452 1453 1454
	if (this_sd) {
		int idx = this_sd->wake_idx;
		unsigned int imbalance;
L
Linus Torvalds 已提交
1455

1456 1457
		imbalance = 100 + (this_sd->imbalance_pct - 100) / 2;

N
Nick Piggin 已提交
1458 1459
		load = source_load(cpu, idx);
		this_load = target_load(this_cpu, idx);
L
Linus Torvalds 已提交
1460

N
Nick Piggin 已提交
1461 1462
		new_cpu = this_cpu; /* Wake to this CPU if we can */

1463 1464
		if (this_sd->flags & SD_WAKE_AFFINE) {
			unsigned long tl = this_load;
1465 1466 1467
			unsigned long tl_per_task;

			tl_per_task = cpu_avg_load_per_task(this_cpu);
1468

L
Linus Torvalds 已提交
1469
			/*
1470 1471 1472
			 * If sync wakeup then subtract the (maximum possible)
			 * effect of the currently running task from the load
			 * of the current CPU:
L
Linus Torvalds 已提交
1473
			 */
1474
			if (sync)
I
Ingo Molnar 已提交
1475
				tl -= current->se.load.weight;
1476 1477

			if ((tl <= load &&
1478
				tl + target_load(cpu, idx) <= tl_per_task) ||
I
Ingo Molnar 已提交
1479
			       100*(tl + p->se.load.weight) <= imbalance*load) {
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
				/*
				 * This domain has SD_WAKE_AFFINE and
				 * p is cache cold in this domain, and
				 * there is no bad imbalance.
				 */
				schedstat_inc(this_sd, ttwu_move_affine);
				goto out_set_cpu;
			}
		}

		/*
		 * Start passive balancing when half the imbalance_pct
		 * limit is reached.
		 */
		if (this_sd->flags & SD_WAKE_BALANCE) {
			if (imbalance*this_load <= 100*load) {
				schedstat_inc(this_sd, ttwu_move_balance);
				goto out_set_cpu;
			}
L
Linus Torvalds 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
		}
	}

	new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */
out_set_cpu:
	new_cpu = wake_idle(new_cpu, p);
	if (new_cpu != cpu) {
		set_task_cpu(p, new_cpu);
		task_rq_unlock(rq, &flags);
		/* might preempt at this point */
		rq = task_rq_lock(p, &flags);
		old_state = p->state;
		if (!(old_state & state))
			goto out;
I
Ingo Molnar 已提交
1513
		if (p->se.on_rq)
L
Linus Torvalds 已提交
1514 1515 1516 1517 1518 1519 1520 1521
			goto out_running;

		this_cpu = smp_processor_id();
		cpu = task_cpu(p);
	}

out_activate:
#endif /* CONFIG_SMP */
I
Ingo Molnar 已提交
1522
	activate_task(rq, p, 1);
L
Linus Torvalds 已提交
1523 1524 1525 1526 1527 1528 1529 1530
	/*
	 * Sync wakeups (i.e. those types of wakeups where the waker
	 * has indicated that it will leave the CPU in short order)
	 * don't trigger a preemption, if the woken up task will run on
	 * this cpu. (in this case the 'I will reschedule' promise of
	 * the waker guarantees that the freshly woken up task is going
	 * to be considered on this CPU.)
	 */
I
Ingo Molnar 已提交
1531 1532
	if (!sync || cpu != this_cpu)
		check_preempt_curr(rq, p);
L
Linus Torvalds 已提交
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542
	success = 1;

out_running:
	p->state = TASK_RUNNING;
out:
	task_rq_unlock(rq, &flags);

	return success;
}

1543
int fastcall wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
1544 1545 1546 1547 1548 1549
{
	return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED |
				 TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0);
}
EXPORT_SYMBOL(wake_up_process);

1550
int fastcall wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
1551 1552 1553 1554 1555 1556 1557
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
	p->se.wait_start_fair		= 0;
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
	p->se.delta_exec		= 0;
	p->se.delta_fair_run		= 0;
	p->se.delta_fair_sleep		= 0;
	p->se.wait_runtime		= 0;
I
Ingo Molnar 已提交
1570 1571 1572 1573
	p->se.sleep_start_fair		= 0;

#ifdef CONFIG_SCHEDSTATS
	p->se.wait_start		= 0;
I
Ingo Molnar 已提交
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
	p->se.sum_wait_runtime		= 0;
	p->se.sum_sleep_runtime		= 0;
	p->se.sleep_start		= 0;
	p->se.block_start		= 0;
	p->se.sleep_max			= 0;
	p->se.block_max			= 0;
	p->se.exec_max			= 0;
	p->se.wait_max			= 0;
	p->se.wait_runtime_overruns	= 0;
	p->se.wait_runtime_underruns	= 0;
I
Ingo Molnar 已提交
1584
#endif
N
Nick Piggin 已提交
1585

I
Ingo Molnar 已提交
1586 1587
	INIT_LIST_HEAD(&p->run_list);
	p->se.on_rq = 0;
N
Nick Piggin 已提交
1588

1589 1590 1591 1592
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif

L
Linus Torvalds 已提交
1593 1594 1595 1596 1597 1598 1599
	/*
	 * We mark the process as running here, but have not actually
	 * inserted it onto the runqueue yet. This guarantees that
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
}

/*
 * fork()/clone()-time setup:
 */
void sched_fork(struct task_struct *p, int clone_flags)
{
	int cpu = get_cpu();

	__sched_fork(p);

#ifdef CONFIG_SMP
	cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
#endif
	__set_task_cpu(p, cpu);
1615 1616 1617 1618 1619 1620

	/*
	 * Make sure we do not leak PI boosting priority to the child:
	 */
	p->prio = current->normal_prio;

1621
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
1622
	if (likely(sched_info_on()))
1623
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
1624
#endif
1625
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
1626 1627
	p->oncpu = 0;
#endif
L
Linus Torvalds 已提交
1628
#ifdef CONFIG_PREEMPT
1629
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
1630
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
1631
#endif
N
Nick Piggin 已提交
1632
	put_cpu();
L
Linus Torvalds 已提交
1633 1634
}

I
Ingo Molnar 已提交
1635 1636 1637 1638 1639 1640
/*
 * After fork, child runs first. (default) If set to 0 then
 * parent will (try to) run first.
 */
unsigned int __read_mostly sysctl_sched_child_runs_first = 1;

L
Linus Torvalds 已提交
1641 1642 1643 1644 1645 1646 1647
/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
1648
void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
L
Linus Torvalds 已提交
1649 1650
{
	unsigned long flags;
I
Ingo Molnar 已提交
1651 1652
	struct rq *rq;
	int this_cpu;
I
Ingo Molnar 已提交
1653
	u64 now;
L
Linus Torvalds 已提交
1654 1655

	rq = task_rq_lock(p, &flags);
N
Nick Piggin 已提交
1656
	BUG_ON(p->state != TASK_RUNNING);
I
Ingo Molnar 已提交
1657
	this_cpu = smp_processor_id(); /* parent's CPU */
I
Ingo Molnar 已提交
1658 1659
	update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
1660 1661 1662

	p->prio = effective_prio(p);

I
Ingo Molnar 已提交
1663 1664 1665 1666
	if (!p->sched_class->task_new || !sysctl_sched_child_runs_first ||
			(clone_flags & CLONE_VM) || task_cpu(p) != this_cpu ||
			!current->se.on_rq) {

I
Ingo Molnar 已提交
1667
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
1668 1669
	} else {
		/*
I
Ingo Molnar 已提交
1670 1671
		 * Let the scheduling class do new task startup
		 * management (if any):
L
Linus Torvalds 已提交
1672
		 */
1673
		p->sched_class->task_new(rq, p);
1674
		inc_nr_running(p, rq);
L
Linus Torvalds 已提交
1675
	}
I
Ingo Molnar 已提交
1676 1677
	check_preempt_curr(rq, p);
	task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
1678 1679
}

1680 1681 1682
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
R
Randy Dunlap 已提交
1683 1684
 * preempt_notifier_register - tell me when current is being being preempted & rescheduled
 * @notifier: notifier struct to register
1685 1686 1687 1688 1689 1690 1691 1692 1693
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
1694
 * @notifier: notifier struct to unregister
1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_out(notifier, next);
}

#else

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

#endif

1738 1739 1740
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
1741
 * @prev: the current task that is being switched out
1742 1743 1744 1745 1746 1747 1748 1749 1750
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
1751 1752 1753
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
1754
{
1755
	fire_sched_out_preempt_notifiers(prev, next);
1756 1757 1758 1759
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
1760 1761
/**
 * finish_task_switch - clean up after a task-switch
1762
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
1763 1764
 * @prev: the thread we just switched away from.
 *
1765 1766 1767 1768
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
1769 1770 1771 1772 1773 1774
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
 * so, we finish that here outside of the runqueue lock.  (Doing it
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
1775
static inline void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
1776 1777 1778
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
1779
	long prev_state;
L
Linus Torvalds 已提交
1780 1781 1782 1783 1784

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
1785
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
1786 1787
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
1788
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
1789 1790 1791 1792 1793
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
1794
	prev_state = prev->state;
1795 1796
	finish_arch_switch(prev);
	finish_lock_switch(rq, prev);
1797
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
1798 1799
	if (mm)
		mmdrop(mm);
1800
	if (unlikely(prev_state == TASK_DEAD)) {
1801 1802 1803
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
1804
		 */
1805
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
1806
		put_task_struct(prev);
1807
	}
L
Linus Torvalds 已提交
1808 1809 1810 1811 1812 1813
}

/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
1814
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
1815 1816
	__releases(rq->lock)
{
1817 1818
	struct rq *rq = this_rq();

1819 1820 1821 1822 1823
	finish_task_switch(rq, prev);
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
1824 1825 1826 1827 1828 1829 1830 1831
	if (current->set_child_tid)
		put_user(current->pid, current->set_child_tid);
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
1832
static inline void
1833
context_switch(struct rq *rq, struct task_struct *prev,
1834
	       struct task_struct *next)
L
Linus Torvalds 已提交
1835
{
I
Ingo Molnar 已提交
1836
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
1837

1838
	prepare_task_switch(rq, prev, next);
I
Ingo Molnar 已提交
1839 1840
	mm = next->mm;
	oldmm = prev->active_mm;
1841 1842 1843 1844 1845 1846 1847
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
	arch_enter_lazy_cpu_mode();

I
Ingo Molnar 已提交
1848
	if (unlikely(!mm)) {
L
Linus Torvalds 已提交
1849 1850 1851 1852 1853 1854
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

I
Ingo Molnar 已提交
1855
	if (unlikely(!prev->mm)) {
L
Linus Torvalds 已提交
1856 1857 1858
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
1859 1860 1861 1862 1863 1864 1865
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
1866
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
1867
#endif
L
Linus Torvalds 已提交
1868 1869 1870 1871

	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
1872 1873 1874 1875 1876 1877 1878
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
}

/*
 * nr_running, nr_uninterruptible and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, current number of uninterruptible-sleeping threads, total
 * number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
}

unsigned long nr_uninterruptible(void)
{
	unsigned long i, sum = 0;

1902
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916
		sum += cpu_rq(i)->nr_uninterruptible;

	/*
	 * Since we read the counters lockless, it might be slightly
	 * inaccurate. Do not allow it to go below zero though:
	 */
	if (unlikely((long)sum < 0))
		sum = 0;

	return sum;
}

unsigned long long nr_context_switches(void)
{
1917 1918
	int i;
	unsigned long long sum = 0;
L
Linus Torvalds 已提交
1919

1920
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1921 1922 1923 1924 1925 1926 1927 1928 1929
		sum += cpu_rq(i)->nr_switches;

	return sum;
}

unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;

1930
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1931 1932 1933 1934 1935
		sum += atomic_read(&cpu_rq(i)->nr_iowait);

	return sum;
}

1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950
unsigned long nr_active(void)
{
	unsigned long i, running = 0, uninterruptible = 0;

	for_each_online_cpu(i) {
		running += cpu_rq(i)->nr_running;
		uninterruptible += cpu_rq(i)->nr_uninterruptible;
	}

	if (unlikely((long)uninterruptible < 0))
		uninterruptible = 0;

	return running + uninterruptible;
}

1951
/*
I
Ingo Molnar 已提交
1952 1953
 * Update rq->cpu_load[] statistics. This function is usually called every
 * scheduler tick (TICK_NSEC).
1954
 */
I
Ingo Molnar 已提交
1955
static void update_cpu_load(struct rq *this_rq)
1956
{
I
Ingo Molnar 已提交
1957 1958 1959 1960
	u64 fair_delta64, exec_delta64, idle_delta64, sample_interval64, tmp64;
	unsigned long total_load = this_rq->ls.load.weight;
	unsigned long this_load =  total_load;
	struct load_stat *ls = &this_rq->ls;
I
Ingo Molnar 已提交
1961
	u64 now;
I
Ingo Molnar 已提交
1962 1963
	int i, scale;

I
Ingo Molnar 已提交
1964 1965 1966
	__update_rq_clock(this_rq);
	now = this_rq->clock;

I
Ingo Molnar 已提交
1967 1968 1969 1970 1971
	this_rq->nr_load_updates++;
	if (unlikely(!(sysctl_sched_features & SCHED_FEAT_PRECISE_CPU_LOAD)))
		goto do_avg;

	/* Update delta_fair/delta_exec fields first */
1972
	update_curr_load(this_rq);
I
Ingo Molnar 已提交
1973 1974 1975 1976 1977 1978 1979

	fair_delta64 = ls->delta_fair + 1;
	ls->delta_fair = 0;

	exec_delta64 = ls->delta_exec + 1;
	ls->delta_exec = 0;

1980 1981
	sample_interval64 = this_rq->clock - ls->load_update_last;
	ls->load_update_last = this_rq->clock;
I
Ingo Molnar 已提交
1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008

	if ((s64)sample_interval64 < (s64)TICK_NSEC)
		sample_interval64 = TICK_NSEC;

	if (exec_delta64 > sample_interval64)
		exec_delta64 = sample_interval64;

	idle_delta64 = sample_interval64 - exec_delta64;

	tmp64 = div64_64(SCHED_LOAD_SCALE * exec_delta64, fair_delta64);
	tmp64 = div64_64(tmp64 * exec_delta64, sample_interval64);

	this_load = (unsigned long)tmp64;

do_avg:

	/* Update our load: */
	for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
		unsigned long old_load, new_load;

		/* scale is effectively 1 << i now, and >> i divides by scale */

		old_load = this_rq->cpu_load[i];
		new_load = this_load;

		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
	}
2009 2010
}

I
Ingo Molnar 已提交
2011 2012
#ifdef CONFIG_SMP

L
Linus Torvalds 已提交
2013 2014 2015 2016 2017 2018
/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
2019
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2020 2021 2022
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
2023
	BUG_ON(!irqs_disabled());
L
Linus Torvalds 已提交
2024 2025 2026 2027
	if (rq1 == rq2) {
		spin_lock(&rq1->lock);
		__acquire(rq2->lock);	/* Fake it out ;) */
	} else {
2028
		if (rq1 < rq2) {
L
Linus Torvalds 已提交
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
			spin_lock(&rq1->lock);
			spin_lock(&rq2->lock);
		} else {
			spin_lock(&rq2->lock);
			spin_lock(&rq1->lock);
		}
	}
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
2044
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	spin_unlock(&rq1->lock);
	if (rq1 != rq2)
		spin_unlock(&rq2->lock);
	else
		__release(rq2->lock);
}

/*
 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
 */
2058
static void double_lock_balance(struct rq *this_rq, struct rq *busiest)
L
Linus Torvalds 已提交
2059 2060 2061 2062
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
2063 2064 2065 2066 2067
	if (unlikely(!irqs_disabled())) {
		/* printk() doesn't work good under rq->lock */
		spin_unlock(&this_rq->lock);
		BUG_ON(1);
	}
L
Linus Torvalds 已提交
2068
	if (unlikely(!spin_trylock(&busiest->lock))) {
2069
		if (busiest < this_rq) {
L
Linus Torvalds 已提交
2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083
			spin_unlock(&this_rq->lock);
			spin_lock(&busiest->lock);
			spin_lock(&this_rq->lock);
		} else
			spin_lock(&busiest->lock);
	}
}

/*
 * If dest_cpu is allowed for this process, migrate the task to it.
 * This is accomplished by forcing the cpu_allowed mask to only
 * allow dest_cpu, which will force the cpu onto dest_cpu.  Then
 * the cpu_allowed mask is restored.
 */
2084
static void sched_migrate_task(struct task_struct *p, int dest_cpu)
L
Linus Torvalds 已提交
2085
{
2086
	struct migration_req req;
L
Linus Torvalds 已提交
2087
	unsigned long flags;
2088
	struct rq *rq;
L
Linus Torvalds 已提交
2089 2090 2091 2092 2093 2094 2095 2096 2097 2098

	rq = task_rq_lock(p, &flags);
	if (!cpu_isset(dest_cpu, p->cpus_allowed)
	    || unlikely(cpu_is_offline(dest_cpu)))
		goto out;

	/* force the process onto the specified CPU */
	if (migrate_task(p, dest_cpu, &req)) {
		/* Need to wait for migration thread (might exit: take ref). */
		struct task_struct *mt = rq->migration_thread;
2099

L
Linus Torvalds 已提交
2100 2101 2102 2103 2104
		get_task_struct(mt);
		task_rq_unlock(rq, &flags);
		wake_up_process(mt);
		put_task_struct(mt);
		wait_for_completion(&req.done);
2105

L
Linus Torvalds 已提交
2106 2107 2108 2109 2110 2111 2112
		return;
	}
out:
	task_rq_unlock(rq, &flags);
}

/*
N
Nick Piggin 已提交
2113 2114
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
L
Linus Torvalds 已提交
2115 2116 2117 2118
 */
void sched_exec(void)
{
	int new_cpu, this_cpu = get_cpu();
N
Nick Piggin 已提交
2119
	new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
L
Linus Torvalds 已提交
2120
	put_cpu();
N
Nick Piggin 已提交
2121 2122
	if (new_cpu != this_cpu)
		sched_migrate_task(current, new_cpu);
L
Linus Torvalds 已提交
2123 2124 2125 2126 2127 2128
}

/*
 * pull_task - move a task from a remote runqueue to the local runqueue.
 * Both runqueues must be locked.
 */
I
Ingo Molnar 已提交
2129 2130
static void pull_task(struct rq *src_rq, struct task_struct *p,
		      struct rq *this_rq, int this_cpu)
L
Linus Torvalds 已提交
2131
{
I
Ingo Molnar 已提交
2132 2133
	update_rq_clock(src_rq);
	deactivate_task(src_rq, p, 0, src_rq->clock);
L
Linus Torvalds 已提交
2134
	set_task_cpu(p, this_cpu);
I
Ingo Molnar 已提交
2135
	activate_task(this_rq, p, 0);
L
Linus Torvalds 已提交
2136 2137 2138 2139
	/*
	 * Note that idle threads have a prio of MAX_PRIO, for this test
	 * to be always true for them.
	 */
I
Ingo Molnar 已提交
2140
	check_preempt_curr(this_rq, p);
L
Linus Torvalds 已提交
2141 2142 2143 2144 2145
}

/*
 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
 */
2146
static
2147
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
I
Ingo Molnar 已提交
2148
		     struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2149
		     int *all_pinned)
L
Linus Torvalds 已提交
2150 2151 2152 2153 2154 2155 2156 2157 2158
{
	/*
	 * We do not migrate tasks that are:
	 * 1) running (obviously), or
	 * 2) cannot be migrated to this CPU due to cpus_allowed, or
	 * 3) are cache-hot on their current CPU.
	 */
	if (!cpu_isset(this_cpu, p->cpus_allowed))
		return 0;
2159 2160 2161 2162
	*all_pinned = 0;

	if (task_running(rq, p))
		return 0;
L
Linus Torvalds 已提交
2163 2164

	/*
I
Ingo Molnar 已提交
2165
	 * Aggressive migration if too many balance attempts have failed:
L
Linus Torvalds 已提交
2166
	 */
I
Ingo Molnar 已提交
2167
	if (sd->nr_balance_failed > sd->cache_nice_tries)
L
Linus Torvalds 已提交
2168 2169 2170 2171 2172
		return 1;

	return 1;
}

I
Ingo Molnar 已提交
2173
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2174
		      unsigned long max_nr_move, unsigned long max_load_move,
I
Ingo Molnar 已提交
2175
		      struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2176
		      int *all_pinned, unsigned long *load_moved,
2177
		      int *this_best_prio, struct rq_iterator *iterator)
L
Linus Torvalds 已提交
2178
{
I
Ingo Molnar 已提交
2179 2180 2181
	int pulled = 0, pinned = 0, skip_for_load;
	struct task_struct *p;
	long rem_load_move = max_load_move;
L
Linus Torvalds 已提交
2182

2183
	if (max_nr_move == 0 || max_load_move == 0)
L
Linus Torvalds 已提交
2184 2185
		goto out;

2186 2187
	pinned = 1;

L
Linus Torvalds 已提交
2188
	/*
I
Ingo Molnar 已提交
2189
	 * Start the load-balancing iterator:
L
Linus Torvalds 已提交
2190
	 */
I
Ingo Molnar 已提交
2191 2192 2193
	p = iterator->start(iterator->arg);
next:
	if (!p)
L
Linus Torvalds 已提交
2194
		goto out;
2195 2196 2197 2198 2199
	/*
	 * To help distribute high priority tasks accross CPUs we don't
	 * skip a task if it will be the highest priority task (i.e. smallest
	 * prio value) on its new queue regardless of its load weight
	 */
I
Ingo Molnar 已提交
2200 2201
	skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
							 SCHED_LOAD_SCALE_FUZZ;
2202
	if ((skip_for_load && p->prio >= *this_best_prio) ||
I
Ingo Molnar 已提交
2203 2204 2205
	    !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2206 2207
	}

I
Ingo Molnar 已提交
2208
	pull_task(busiest, p, this_rq, this_cpu);
L
Linus Torvalds 已提交
2209
	pulled++;
I
Ingo Molnar 已提交
2210
	rem_load_move -= p->se.load.weight;
L
Linus Torvalds 已提交
2211

2212 2213 2214 2215 2216
	/*
	 * We only want to steal up to the prescribed number of tasks
	 * and the prescribed amount of weighted load.
	 */
	if (pulled < max_nr_move && rem_load_move > 0) {
2217 2218
		if (p->prio < *this_best_prio)
			*this_best_prio = p->prio;
I
Ingo Molnar 已提交
2219 2220
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2221 2222 2223 2224 2225 2226 2227 2228
	}
out:
	/*
	 * Right now, this is the only place pull_task() is called,
	 * so we can safely collect pull_task() stats here rather than
	 * inside pull_task().
	 */
	schedstat_add(sd, lb_gained[idle], pulled);
2229 2230 2231

	if (all_pinned)
		*all_pinned = pinned;
I
Ingo Molnar 已提交
2232
	*load_moved = max_load_move - rem_load_move;
L
Linus Torvalds 已提交
2233 2234 2235
	return pulled;
}

I
Ingo Molnar 已提交
2236
/*
P
Peter Williams 已提交
2237 2238 2239
 * move_tasks tries to move up to max_load_move weighted load from busiest to
 * this_rq, as part of a balancing operation within domain "sd".
 * Returns 1 if successful and 0 otherwise.
I
Ingo Molnar 已提交
2240 2241 2242 2243
 *
 * Called with both runqueues locked.
 */
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
P
Peter Williams 已提交
2244
		      unsigned long max_load_move,
I
Ingo Molnar 已提交
2245 2246 2247 2248
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned)
{
	struct sched_class *class = sched_class_highest;
P
Peter Williams 已提交
2249
	unsigned long total_load_moved = 0;
2250
	int this_best_prio = this_rq->curr->prio;
I
Ingo Molnar 已提交
2251 2252

	do {
P
Peter Williams 已提交
2253 2254 2255
		total_load_moved +=
			class->load_balance(this_rq, this_cpu, busiest,
				ULONG_MAX, max_load_move - total_load_moved,
2256
				sd, idle, all_pinned, &this_best_prio);
I
Ingo Molnar 已提交
2257
		class = class->next;
P
Peter Williams 已提交
2258
	} while (class && max_load_move > total_load_moved);
I
Ingo Molnar 已提交
2259

P
Peter Williams 已提交
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
	return total_load_moved > 0;
}

/*
 * move_one_task tries to move exactly one task from busiest to this_rq, as
 * part of active balancing operations within "domain".
 * Returns 1 if successful and 0 otherwise.
 *
 * Called with both runqueues locked.
 */
static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
			 struct sched_domain *sd, enum cpu_idle_type idle)
{
	struct sched_class *class;
2274
	int this_best_prio = MAX_PRIO;
P
Peter Williams 已提交
2275 2276 2277

	for (class = sched_class_highest; class; class = class->next)
		if (class->load_balance(this_rq, this_cpu, busiest,
2278 2279
					1, ULONG_MAX, sd, idle, NULL,
					&this_best_prio))
P
Peter Williams 已提交
2280 2281 2282
			return 1;

	return 0;
I
Ingo Molnar 已提交
2283 2284
}

L
Linus Torvalds 已提交
2285 2286
/*
 * find_busiest_group finds and returns the busiest CPU group within the
2287 2288
 * domain. It calculates and returns the amount of weighted load which
 * should be moved to restore balance via the imbalance parameter.
L
Linus Torvalds 已提交
2289 2290 2291
 */
static struct sched_group *
find_busiest_group(struct sched_domain *sd, int this_cpu,
I
Ingo Molnar 已提交
2292 2293
		   unsigned long *imbalance, enum cpu_idle_type idle,
		   int *sd_idle, cpumask_t *cpus, int *balance)
L
Linus Torvalds 已提交
2294 2295 2296
{
	struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
	unsigned long max_load, avg_load, total_load, this_load, total_pwr;
2297
	unsigned long max_pull;
2298 2299
	unsigned long busiest_load_per_task, busiest_nr_running;
	unsigned long this_load_per_task, this_nr_running;
N
Nick Piggin 已提交
2300
	int load_idx;
2301 2302 2303 2304 2305 2306
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
	int power_savings_balance = 1;
	unsigned long leader_nr_running = 0, min_load_per_task = 0;
	unsigned long min_nr_running = ULONG_MAX;
	struct sched_group *group_min = NULL, *group_leader = NULL;
#endif
L
Linus Torvalds 已提交
2307 2308

	max_load = this_load = total_load = total_pwr = 0;
2309 2310
	busiest_load_per_task = busiest_nr_running = 0;
	this_load_per_task = this_nr_running = 0;
I
Ingo Molnar 已提交
2311
	if (idle == CPU_NOT_IDLE)
N
Nick Piggin 已提交
2312
		load_idx = sd->busy_idx;
I
Ingo Molnar 已提交
2313
	else if (idle == CPU_NEWLY_IDLE)
N
Nick Piggin 已提交
2314 2315 2316
		load_idx = sd->newidle_idx;
	else
		load_idx = sd->idle_idx;
L
Linus Torvalds 已提交
2317 2318

	do {
2319
		unsigned long load, group_capacity;
L
Linus Torvalds 已提交
2320 2321
		int local_group;
		int i;
2322
		unsigned int balance_cpu = -1, first_idle_cpu = 0;
2323
		unsigned long sum_nr_running, sum_weighted_load;
L
Linus Torvalds 已提交
2324 2325 2326

		local_group = cpu_isset(this_cpu, group->cpumask);

2327 2328 2329
		if (local_group)
			balance_cpu = first_cpu(group->cpumask);

L
Linus Torvalds 已提交
2330
		/* Tally up the load of all CPUs in the group */
2331
		sum_weighted_load = sum_nr_running = avg_load = 0;
L
Linus Torvalds 已提交
2332 2333

		for_each_cpu_mask(i, group->cpumask) {
2334 2335 2336 2337 2338 2339
			struct rq *rq;

			if (!cpu_isset(i, *cpus))
				continue;

			rq = cpu_rq(i);
2340

2341
			if (*sd_idle && rq->nr_running)
N
Nick Piggin 已提交
2342 2343
				*sd_idle = 0;

L
Linus Torvalds 已提交
2344
			/* Bias balancing toward cpus of our domain */
2345 2346 2347 2348 2349 2350
			if (local_group) {
				if (idle_cpu(i) && !first_idle_cpu) {
					first_idle_cpu = 1;
					balance_cpu = i;
				}

N
Nick Piggin 已提交
2351
				load = target_load(i, load_idx);
2352
			} else
N
Nick Piggin 已提交
2353
				load = source_load(i, load_idx);
L
Linus Torvalds 已提交
2354 2355

			avg_load += load;
2356
			sum_nr_running += rq->nr_running;
I
Ingo Molnar 已提交
2357
			sum_weighted_load += weighted_cpuload(i);
L
Linus Torvalds 已提交
2358 2359
		}

2360 2361 2362
		/*
		 * First idle cpu or the first cpu(busiest) in this sched group
		 * is eligible for doing load balancing at this and above
2363 2364
		 * domains. In the newly idle case, we will allow all the cpu's
		 * to do the newly idle load balance.
2365
		 */
2366 2367
		if (idle != CPU_NEWLY_IDLE && local_group &&
		    balance_cpu != this_cpu && balance) {
2368 2369 2370 2371
			*balance = 0;
			goto ret;
		}

L
Linus Torvalds 已提交
2372
		total_load += avg_load;
2373
		total_pwr += group->__cpu_power;
L
Linus Torvalds 已提交
2374 2375

		/* Adjust by relative CPU power of the group */
2376 2377
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2378

2379
		group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
2380

L
Linus Torvalds 已提交
2381 2382 2383
		if (local_group) {
			this_load = avg_load;
			this = group;
2384 2385 2386
			this_nr_running = sum_nr_running;
			this_load_per_task = sum_weighted_load;
		} else if (avg_load > max_load &&
2387
			   sum_nr_running > group_capacity) {
L
Linus Torvalds 已提交
2388 2389
			max_load = avg_load;
			busiest = group;
2390 2391
			busiest_nr_running = sum_nr_running;
			busiest_load_per_task = sum_weighted_load;
L
Linus Torvalds 已提交
2392
		}
2393 2394 2395 2396 2397 2398

#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
		/*
		 * Busy processors will not participate in power savings
		 * balance.
		 */
I
Ingo Molnar 已提交
2399 2400 2401
		if (idle == CPU_NOT_IDLE ||
				!(sd->flags & SD_POWERSAVINGS_BALANCE))
			goto group_next;
2402 2403 2404 2405 2406 2407 2408 2409 2410

		/*
		 * If the local group is idle or completely loaded
		 * no need to do power savings balance at this domain
		 */
		if (local_group && (this_nr_running >= group_capacity ||
				    !this_nr_running))
			power_savings_balance = 0;

I
Ingo Molnar 已提交
2411
		/*
2412 2413
		 * If a group is already running at full capacity or idle,
		 * don't include that group in power savings calculations
I
Ingo Molnar 已提交
2414 2415
		 */
		if (!power_savings_balance || sum_nr_running >= group_capacity
2416
		    || !sum_nr_running)
I
Ingo Molnar 已提交
2417
			goto group_next;
2418

I
Ingo Molnar 已提交
2419
		/*
2420
		 * Calculate the group which has the least non-idle load.
I
Ingo Molnar 已提交
2421 2422 2423 2424 2425
		 * This is the group from where we need to pick up the load
		 * for saving power
		 */
		if ((sum_nr_running < min_nr_running) ||
		    (sum_nr_running == min_nr_running &&
2426 2427
		     first_cpu(group->cpumask) <
		     first_cpu(group_min->cpumask))) {
I
Ingo Molnar 已提交
2428 2429
			group_min = group;
			min_nr_running = sum_nr_running;
2430 2431
			min_load_per_task = sum_weighted_load /
						sum_nr_running;
I
Ingo Molnar 已提交
2432
		}
2433

I
Ingo Molnar 已提交
2434
		/*
2435
		 * Calculate the group which is almost near its
I
Ingo Molnar 已提交
2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446
		 * capacity but still has some space to pick up some load
		 * from other group and save more power
		 */
		if (sum_nr_running <= group_capacity - 1) {
			if (sum_nr_running > leader_nr_running ||
			    (sum_nr_running == leader_nr_running &&
			     first_cpu(group->cpumask) >
			      first_cpu(group_leader->cpumask))) {
				group_leader = group;
				leader_nr_running = sum_nr_running;
			}
2447
		}
2448 2449
group_next:
#endif
L
Linus Torvalds 已提交
2450 2451 2452
		group = group->next;
	} while (group != sd->groups);

2453
	if (!busiest || this_load >= max_load || busiest_nr_running == 0)
L
Linus Torvalds 已提交
2454 2455 2456 2457 2458 2459 2460 2461
		goto out_balanced;

	avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;

	if (this_load >= avg_load ||
			100*max_load <= sd->imbalance_pct*this_load)
		goto out_balanced;

2462
	busiest_load_per_task /= busiest_nr_running;
L
Linus Torvalds 已提交
2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473
	/*
	 * We're trying to get all the cpus to the average_load, so we don't
	 * want to push ourselves above the average load, nor do we wish to
	 * reduce the max loaded cpu below the average load, as either of these
	 * actions would just result in more rebalancing later, and ping-pong
	 * tasks around. Thus we look for the minimum possible imbalance.
	 * Negative imbalances (*we* are more loaded than anyone else) will
	 * be counted as no imbalance for these purposes -- we can't fix that
	 * by pulling tasks to us.  Be careful of negative numbers as they'll
	 * appear as very large values with unsigned longs.
	 */
2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485
	if (max_load <= busiest_load_per_task)
		goto out_balanced;

	/*
	 * In the presence of smp nice balancing, certain scenarios can have
	 * max load less than avg load(as we skip the groups at or below
	 * its cpu_power, while calculating max_load..)
	 */
	if (max_load < avg_load) {
		*imbalance = 0;
		goto small_imbalance;
	}
2486 2487

	/* Don't want to pull so many tasks that a group would go idle */
2488
	max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
2489

L
Linus Torvalds 已提交
2490
	/* How much load to actually move to equalise the imbalance */
2491 2492
	*imbalance = min(max_pull * busiest->__cpu_power,
				(avg_load - this_load) * this->__cpu_power)
L
Linus Torvalds 已提交
2493 2494
			/ SCHED_LOAD_SCALE;

2495 2496 2497 2498 2499 2500
	/*
	 * if *imbalance is less than the average load per runnable task
	 * there is no gaurantee that any tasks will be moved so we'll have
	 * a think about bumping its value to force at least one task to be
	 * moved
	 */
I
Ingo Molnar 已提交
2501
	if (*imbalance + SCHED_LOAD_SCALE_FUZZ < busiest_load_per_task/2) {
2502
		unsigned long tmp, pwr_now, pwr_move;
2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513
		unsigned int imbn;

small_imbalance:
		pwr_move = pwr_now = 0;
		imbn = 2;
		if (this_nr_running) {
			this_load_per_task /= this_nr_running;
			if (busiest_load_per_task > this_load_per_task)
				imbn = 1;
		} else
			this_load_per_task = SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
2514

I
Ingo Molnar 已提交
2515 2516
		if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >=
					busiest_load_per_task * imbn) {
2517
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2518 2519 2520 2521 2522 2523 2524 2525 2526
			return busiest;
		}

		/*
		 * OK, we don't have enough imbalance to justify moving tasks,
		 * however we may be able to increase total CPU power used by
		 * moving them.
		 */

2527 2528 2529 2530
		pwr_now += busiest->__cpu_power *
				min(busiest_load_per_task, max_load);
		pwr_now += this->__cpu_power *
				min(this_load_per_task, this_load);
L
Linus Torvalds 已提交
2531 2532 2533
		pwr_now /= SCHED_LOAD_SCALE;

		/* Amount of load we'd subtract */
2534 2535
		tmp = sg_div_cpu_power(busiest,
				busiest_load_per_task * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2536
		if (max_load > tmp)
2537
			pwr_move += busiest->__cpu_power *
2538
				min(busiest_load_per_task, max_load - tmp);
L
Linus Torvalds 已提交
2539 2540

		/* Amount of load we'd add */
2541
		if (max_load * busiest->__cpu_power <
2542
				busiest_load_per_task * SCHED_LOAD_SCALE)
2543 2544
			tmp = sg_div_cpu_power(this,
					max_load * busiest->__cpu_power);
L
Linus Torvalds 已提交
2545
		else
2546 2547 2548 2549
			tmp = sg_div_cpu_power(this,
				busiest_load_per_task * SCHED_LOAD_SCALE);
		pwr_move += this->__cpu_power *
				min(this_load_per_task, this_load + tmp);
L
Linus Torvalds 已提交
2550 2551 2552 2553 2554 2555
		pwr_move /= SCHED_LOAD_SCALE;

		/* Move if we gain throughput */
		if (pwr_move <= pwr_now)
			goto out_balanced;

2556
		*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2557 2558 2559 2560 2561
	}

	return busiest;

out_balanced:
2562
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
2563
	if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
2564
		goto ret;
L
Linus Torvalds 已提交
2565

2566 2567 2568 2569 2570
	if (this == group_leader && group_leader != group_min) {
		*imbalance = min_load_per_task;
		return group_min;
	}
#endif
2571
ret:
L
Linus Torvalds 已提交
2572 2573 2574 2575 2576 2577 2578
	*imbalance = 0;
	return NULL;
}

/*
 * find_busiest_queue - find the busiest runqueue among the cpus in group.
 */
2579
static struct rq *
I
Ingo Molnar 已提交
2580
find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
2581
		   unsigned long imbalance, cpumask_t *cpus)
L
Linus Torvalds 已提交
2582
{
2583
	struct rq *busiest = NULL, *rq;
2584
	unsigned long max_load = 0;
L
Linus Torvalds 已提交
2585 2586 2587
	int i;

	for_each_cpu_mask(i, group->cpumask) {
I
Ingo Molnar 已提交
2588
		unsigned long wl;
2589 2590 2591 2592

		if (!cpu_isset(i, *cpus))
			continue;

2593
		rq = cpu_rq(i);
I
Ingo Molnar 已提交
2594
		wl = weighted_cpuload(i);
2595

I
Ingo Molnar 已提交
2596
		if (rq->nr_running == 1 && wl > imbalance)
2597
			continue;
L
Linus Torvalds 已提交
2598

I
Ingo Molnar 已提交
2599 2600
		if (wl > max_load) {
			max_load = wl;
2601
			busiest = rq;
L
Linus Torvalds 已提交
2602 2603 2604 2605 2606 2607
		}
	}

	return busiest;
}

2608 2609 2610 2611 2612 2613
/*
 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
 * so long as it is large enough.
 */
#define MAX_PINNED_INTERVAL	512

L
Linus Torvalds 已提交
2614 2615 2616 2617
/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
2618
static int load_balance(int this_cpu, struct rq *this_rq,
I
Ingo Molnar 已提交
2619
			struct sched_domain *sd, enum cpu_idle_type idle,
2620
			int *balance)
L
Linus Torvalds 已提交
2621
{
P
Peter Williams 已提交
2622
	int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
L
Linus Torvalds 已提交
2623 2624
	struct sched_group *group;
	unsigned long imbalance;
2625
	struct rq *busiest;
2626
	cpumask_t cpus = CPU_MASK_ALL;
2627
	unsigned long flags;
N
Nick Piggin 已提交
2628

2629 2630 2631
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
I
Ingo Molnar 已提交
2632
	 * let the state of idle sibling percolate up as CPU_IDLE, instead of
I
Ingo Molnar 已提交
2633
	 * portraying it as CPU_NOT_IDLE.
2634
	 */
I
Ingo Molnar 已提交
2635
	if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
2636
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2637
		sd_idle = 1;
L
Linus Torvalds 已提交
2638 2639 2640

	schedstat_inc(sd, lb_cnt[idle]);

2641 2642
redo:
	group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
2643 2644
				   &cpus, balance);

2645
	if (*balance == 0)
2646 2647
		goto out_balanced;

L
Linus Torvalds 已提交
2648 2649 2650 2651 2652
	if (!group) {
		schedstat_inc(sd, lb_nobusyg[idle]);
		goto out_balanced;
	}

2653
	busiest = find_busiest_queue(group, idle, imbalance, &cpus);
L
Linus Torvalds 已提交
2654 2655 2656 2657 2658
	if (!busiest) {
		schedstat_inc(sd, lb_nobusyq[idle]);
		goto out_balanced;
	}

N
Nick Piggin 已提交
2659
	BUG_ON(busiest == this_rq);
L
Linus Torvalds 已提交
2660 2661 2662

	schedstat_add(sd, lb_imbalance[idle], imbalance);

P
Peter Williams 已提交
2663
	ld_moved = 0;
L
Linus Torvalds 已提交
2664 2665 2666 2667
	if (busiest->nr_running > 1) {
		/*
		 * Attempt to move tasks. If find_busiest_group has found
		 * an imbalance but busiest->nr_running <= 1, the group is
P
Peter Williams 已提交
2668
		 * still unbalanced. ld_moved simply stays zero, so it is
L
Linus Torvalds 已提交
2669 2670
		 * correctly treated as an imbalance.
		 */
2671
		local_irq_save(flags);
N
Nick Piggin 已提交
2672
		double_rq_lock(this_rq, busiest);
P
Peter Williams 已提交
2673
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2674
				      imbalance, sd, idle, &all_pinned);
N
Nick Piggin 已提交
2675
		double_rq_unlock(this_rq, busiest);
2676
		local_irq_restore(flags);
2677

2678 2679 2680
		/*
		 * some other cpu did the load balance for us.
		 */
P
Peter Williams 已提交
2681
		if (ld_moved && this_cpu != smp_processor_id())
2682 2683
			resched_cpu(this_cpu);

2684
		/* All tasks on this runqueue were pinned by CPU affinity */
2685 2686 2687 2688
		if (unlikely(all_pinned)) {
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
2689
			goto out_balanced;
2690
		}
L
Linus Torvalds 已提交
2691
	}
2692

P
Peter Williams 已提交
2693
	if (!ld_moved) {
L
Linus Torvalds 已提交
2694 2695 2696 2697 2698
		schedstat_inc(sd, lb_failed[idle]);
		sd->nr_balance_failed++;

		if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {

2699
			spin_lock_irqsave(&busiest->lock, flags);
2700 2701 2702 2703 2704

			/* don't kick the migration_thread, if the curr
			 * task on busiest cpu can't be moved to this_cpu
			 */
			if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
2705
				spin_unlock_irqrestore(&busiest->lock, flags);
2706 2707 2708 2709
				all_pinned = 1;
				goto out_one_pinned;
			}

L
Linus Torvalds 已提交
2710 2711 2712
			if (!busiest->active_balance) {
				busiest->active_balance = 1;
				busiest->push_cpu = this_cpu;
2713
				active_balance = 1;
L
Linus Torvalds 已提交
2714
			}
2715
			spin_unlock_irqrestore(&busiest->lock, flags);
2716
			if (active_balance)
L
Linus Torvalds 已提交
2717 2718 2719 2720 2721 2722
				wake_up_process(busiest->migration_thread);

			/*
			 * We've kicked active balancing, reset the failure
			 * counter.
			 */
2723
			sd->nr_balance_failed = sd->cache_nice_tries+1;
L
Linus Torvalds 已提交
2724
		}
2725
	} else
L
Linus Torvalds 已提交
2726 2727
		sd->nr_balance_failed = 0;

2728
	if (likely(!active_balance)) {
L
Linus Torvalds 已提交
2729 2730
		/* We were unbalanced, so reset the balancing interval */
		sd->balance_interval = sd->min_interval;
2731 2732 2733 2734 2735 2736 2737 2738 2739
	} else {
		/*
		 * If we've begun active balancing, start to back off. This
		 * case may not be covered by the all_pinned logic if there
		 * is only 1 task on the busy runqueue (because we don't call
		 * move_tasks).
		 */
		if (sd->balance_interval < sd->max_interval)
			sd->balance_interval *= 2;
L
Linus Torvalds 已提交
2740 2741
	}

P
Peter Williams 已提交
2742
	if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2743
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2744
		return -1;
P
Peter Williams 已提交
2745
	return ld_moved;
L
Linus Torvalds 已提交
2746 2747 2748 2749

out_balanced:
	schedstat_inc(sd, lb_balanced[idle]);

2750
	sd->nr_balance_failed = 0;
2751 2752

out_one_pinned:
L
Linus Torvalds 已提交
2753
	/* tune up the balancing interval */
2754 2755
	if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
			(sd->balance_interval < sd->max_interval))
L
Linus Torvalds 已提交
2756 2757
		sd->balance_interval *= 2;

2758
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2759
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2760
		return -1;
L
Linus Torvalds 已提交
2761 2762 2763 2764 2765 2766 2767
	return 0;
}

/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 *
I
Ingo Molnar 已提交
2768
 * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
L
Linus Torvalds 已提交
2769 2770
 * this_rq is locked.
 */
2771
static int
2772
load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
L
Linus Torvalds 已提交
2773 2774
{
	struct sched_group *group;
2775
	struct rq *busiest = NULL;
L
Linus Torvalds 已提交
2776
	unsigned long imbalance;
P
Peter Williams 已提交
2777
	int ld_moved = 0;
N
Nick Piggin 已提交
2778
	int sd_idle = 0;
2779
	int all_pinned = 0;
2780
	cpumask_t cpus = CPU_MASK_ALL;
N
Nick Piggin 已提交
2781

2782 2783 2784 2785
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
	 * let the state of idle sibling percolate up as IDLE, instead of
I
Ingo Molnar 已提交
2786
	 * portraying it as CPU_NOT_IDLE.
2787 2788 2789
	 */
	if (sd->flags & SD_SHARE_CPUPOWER &&
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2790
		sd_idle = 1;
L
Linus Torvalds 已提交
2791

I
Ingo Molnar 已提交
2792
	schedstat_inc(sd, lb_cnt[CPU_NEWLY_IDLE]);
2793
redo:
I
Ingo Molnar 已提交
2794
	group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
2795
				   &sd_idle, &cpus, NULL);
L
Linus Torvalds 已提交
2796
	if (!group) {
I
Ingo Molnar 已提交
2797
		schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
2798
		goto out_balanced;
L
Linus Torvalds 已提交
2799 2800
	}

I
Ingo Molnar 已提交
2801
	busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance,
2802
				&cpus);
N
Nick Piggin 已提交
2803
	if (!busiest) {
I
Ingo Molnar 已提交
2804
		schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
2805
		goto out_balanced;
L
Linus Torvalds 已提交
2806 2807
	}

N
Nick Piggin 已提交
2808 2809
	BUG_ON(busiest == this_rq);

I
Ingo Molnar 已提交
2810
	schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
2811

P
Peter Williams 已提交
2812
	ld_moved = 0;
2813 2814 2815
	if (busiest->nr_running > 1) {
		/* Attempt to move tasks */
		double_lock_balance(this_rq, busiest);
P
Peter Williams 已提交
2816
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2817 2818
					imbalance, sd, CPU_NEWLY_IDLE,
					&all_pinned);
2819
		spin_unlock(&busiest->lock);
2820

2821
		if (unlikely(all_pinned)) {
2822 2823 2824 2825
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
		}
2826 2827
	}

P
Peter Williams 已提交
2828
	if (!ld_moved) {
I
Ingo Molnar 已提交
2829
		schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
2830 2831
		if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
		    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2832 2833
			return -1;
	} else
2834
		sd->nr_balance_failed = 0;
L
Linus Torvalds 已提交
2835

P
Peter Williams 已提交
2836
	return ld_moved;
2837 2838

out_balanced:
I
Ingo Molnar 已提交
2839
	schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
2840
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2841
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2842
		return -1;
2843
	sd->nr_balance_failed = 0;
2844

2845
	return 0;
L
Linus Torvalds 已提交
2846 2847 2848 2849 2850 2851
}

/*
 * idle_balance is called by schedule() if this_cpu is about to become
 * idle. Attempts to pull tasks from other CPUs.
 */
2852
static void idle_balance(int this_cpu, struct rq *this_rq)
L
Linus Torvalds 已提交
2853 2854
{
	struct sched_domain *sd;
I
Ingo Molnar 已提交
2855 2856
	int pulled_task = -1;
	unsigned long next_balance = jiffies + HZ;
L
Linus Torvalds 已提交
2857 2858

	for_each_domain(this_cpu, sd) {
2859 2860 2861 2862 2863 2864
		unsigned long interval;

		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		if (sd->flags & SD_BALANCE_NEWIDLE)
2865
			/* If we've pulled tasks over stop searching: */
2866
			pulled_task = load_balance_newidle(this_cpu,
2867 2868 2869 2870 2871 2872 2873
								this_rq, sd);

		interval = msecs_to_jiffies(sd->balance_interval);
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
		if (pulled_task)
			break;
L
Linus Torvalds 已提交
2874
	}
I
Ingo Molnar 已提交
2875
	if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
2876 2877 2878 2879 2880
		/*
		 * We are going idle. next_balance may be set based on
		 * a busy processor. So reset next_balance.
		 */
		this_rq->next_balance = next_balance;
I
Ingo Molnar 已提交
2881
	}
L
Linus Torvalds 已提交
2882 2883 2884 2885 2886 2887 2888 2889 2890 2891
}

/*
 * active_load_balance is run by migration threads. It pushes running tasks
 * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
 * running on each physical CPU where possible, and avoids physical /
 * logical imbalances.
 *
 * Called with busiest_rq locked.
 */
2892
static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
L
Linus Torvalds 已提交
2893
{
2894
	int target_cpu = busiest_rq->push_cpu;
2895 2896
	struct sched_domain *sd;
	struct rq *target_rq;
2897

2898
	/* Is there any task to move? */
2899 2900 2901 2902
	if (busiest_rq->nr_running <= 1)
		return;

	target_rq = cpu_rq(target_cpu);
L
Linus Torvalds 已提交
2903 2904

	/*
2905 2906 2907
	 * This condition is "impossible", if it occurs
	 * we need to fix it.  Originally reported by
	 * Bjorn Helgaas on a 128-cpu setup.
L
Linus Torvalds 已提交
2908
	 */
2909
	BUG_ON(busiest_rq == target_rq);
L
Linus Torvalds 已提交
2910

2911 2912 2913 2914
	/* move a task from busiest_rq to target_rq */
	double_lock_balance(busiest_rq, target_rq);

	/* Search for an sd spanning us and the target CPU. */
2915
	for_each_domain(target_cpu, sd) {
2916
		if ((sd->flags & SD_LOAD_BALANCE) &&
2917
		    cpu_isset(busiest_cpu, sd->span))
2918
				break;
2919
	}
2920

2921 2922
	if (likely(sd)) {
		schedstat_inc(sd, alb_cnt);
2923

P
Peter Williams 已提交
2924 2925
		if (move_one_task(target_rq, target_cpu, busiest_rq,
				  sd, CPU_IDLE))
2926 2927 2928 2929
			schedstat_inc(sd, alb_pushed);
		else
			schedstat_inc(sd, alb_failed);
	}
2930
	spin_unlock(&target_rq->lock);
L
Linus Torvalds 已提交
2931 2932
}

2933 2934 2935 2936 2937 2938 2939 2940 2941
#ifdef CONFIG_NO_HZ
static struct {
	atomic_t load_balancer;
	cpumask_t  cpu_mask;
} nohz ____cacheline_aligned = {
	.load_balancer = ATOMIC_INIT(-1),
	.cpu_mask = CPU_MASK_NONE,
};

2942
/*
2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
 * This routine will try to nominate the ilb (idle load balancing)
 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
 * load balancing on behalf of all those cpus. If all the cpus in the system
 * go into this tickless mode, then there will be no ilb owner (as there is
 * no need for one) and all the cpus will sleep till the next wakeup event
 * arrives...
 *
 * For the ilb owner, tick is not stopped. And this tick will be used
 * for idle load balancing. ilb owner will still be part of
 * nohz.cpu_mask..
2953
 *
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
 * While stopping the tick, this cpu will become the ilb owner if there
 * is no other owner. And will be the owner till that cpu becomes busy
 * or if all cpus in the system stop their ticks at which point
 * there is no need for ilb owner.
 *
 * When the ilb owner becomes busy, it nominates another owner, during the
 * next busy scheduler_tick()
 */
int select_nohz_load_balancer(int stop_tick)
{
	int cpu = smp_processor_id();

	if (stop_tick) {
		cpu_set(cpu, nohz.cpu_mask);
		cpu_rq(cpu)->in_nohz_recently = 1;

		/*
		 * If we are going offline and still the leader, give up!
		 */
		if (cpu_is_offline(cpu) &&
		    atomic_read(&nohz.load_balancer) == cpu) {
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
			return 0;
		}

		/* time for ilb owner also to sleep */
		if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
			if (atomic_read(&nohz.load_balancer) == cpu)
				atomic_set(&nohz.load_balancer, -1);
			return 0;
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/* make me the ilb owner */
			if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
				return 1;
		} else if (atomic_read(&nohz.load_balancer) == cpu)
			return 1;
	} else {
		if (!cpu_isset(cpu, nohz.cpu_mask))
			return 0;

		cpu_clear(cpu, nohz.cpu_mask);

		if (atomic_read(&nohz.load_balancer) == cpu)
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
	}
	return 0;
}
#endif

static DEFINE_SPINLOCK(balancing);

/*
3010 3011 3012 3013 3014
 * It checks each scheduling domain to see if it is due to be balanced,
 * and initiates a balancing operation if so.
 *
 * Balancing parameters are set up in arch_init_sched_domains.
 */
I
Ingo Molnar 已提交
3015
static inline void rebalance_domains(int cpu, enum cpu_idle_type idle)
3016
{
3017 3018
	int balance = 1;
	struct rq *rq = cpu_rq(cpu);
3019 3020
	unsigned long interval;
	struct sched_domain *sd;
3021
	/* Earliest time when we have to do rebalance again */
3022
	unsigned long next_balance = jiffies + 60*HZ;
L
Linus Torvalds 已提交
3023

3024
	for_each_domain(cpu, sd) {
L
Linus Torvalds 已提交
3025 3026 3027 3028
		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		interval = sd->balance_interval;
I
Ingo Molnar 已提交
3029
		if (idle != CPU_IDLE)
L
Linus Torvalds 已提交
3030 3031 3032 3033 3034 3035
			interval *= sd->busy_factor;

		/* scale ms to jiffies */
		interval = msecs_to_jiffies(interval);
		if (unlikely(!interval))
			interval = 1;
I
Ingo Molnar 已提交
3036 3037 3038
		if (interval > HZ*NR_CPUS/10)
			interval = HZ*NR_CPUS/10;

L
Linus Torvalds 已提交
3039

3040 3041 3042 3043 3044
		if (sd->flags & SD_SERIALIZE) {
			if (!spin_trylock(&balancing))
				goto out;
		}

3045
		if (time_after_eq(jiffies, sd->last_balance + interval)) {
3046
			if (load_balance(cpu, rq, sd, idle, &balance)) {
3047 3048
				/*
				 * We've pulled tasks over so either we're no
N
Nick Piggin 已提交
3049 3050 3051
				 * longer idle, or one of our SMT siblings is
				 * not idle.
				 */
I
Ingo Molnar 已提交
3052
				idle = CPU_NOT_IDLE;
L
Linus Torvalds 已提交
3053
			}
3054
			sd->last_balance = jiffies;
L
Linus Torvalds 已提交
3055
		}
3056 3057 3058
		if (sd->flags & SD_SERIALIZE)
			spin_unlock(&balancing);
out:
3059 3060
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
3061 3062 3063 3064 3065 3066 3067 3068

		/*
		 * Stop the load balance at this level. There is another
		 * CPU in our sched group which is doing load balancing more
		 * actively.
		 */
		if (!balance)
			break;
L
Linus Torvalds 已提交
3069
	}
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079
	rq->next_balance = next_balance;
}

/*
 * run_rebalance_domains is triggered when needed from the scheduler tick.
 * In CONFIG_NO_HZ case, the idle load balance owner will do the
 * rebalancing for all the cpus for whom scheduler ticks are stopped.
 */
static void run_rebalance_domains(struct softirq_action *h)
{
I
Ingo Molnar 已提交
3080 3081 3082 3083
	int this_cpu = smp_processor_id();
	struct rq *this_rq = cpu_rq(this_cpu);
	enum cpu_idle_type idle = this_rq->idle_at_tick ?
						CPU_IDLE : CPU_NOT_IDLE;
3084

I
Ingo Molnar 已提交
3085
	rebalance_domains(this_cpu, idle);
3086 3087 3088 3089 3090 3091 3092

#ifdef CONFIG_NO_HZ
	/*
	 * If this cpu is the owner for idle load balancing, then do the
	 * balancing on behalf of the other idle cpus whose ticks are
	 * stopped.
	 */
I
Ingo Molnar 已提交
3093 3094
	if (this_rq->idle_at_tick &&
	    atomic_read(&nohz.load_balancer) == this_cpu) {
3095 3096 3097 3098
		cpumask_t cpus = nohz.cpu_mask;
		struct rq *rq;
		int balance_cpu;

I
Ingo Molnar 已提交
3099
		cpu_clear(this_cpu, cpus);
3100 3101 3102 3103 3104 3105 3106 3107 3108
		for_each_cpu_mask(balance_cpu, cpus) {
			/*
			 * If this cpu gets work to do, stop the load balancing
			 * work being done for other cpus. Next load
			 * balancing owner will pick it up.
			 */
			if (need_resched())
				break;

I
Ingo Molnar 已提交
3109
			rebalance_domains(balance_cpu, SCHED_IDLE);
3110 3111

			rq = cpu_rq(balance_cpu);
I
Ingo Molnar 已提交
3112 3113
			if (time_after(this_rq->next_balance, rq->next_balance))
				this_rq->next_balance = rq->next_balance;
3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125
		}
	}
#endif
}

/*
 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
 *
 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
 * idle load balancing owner or decide to stop the periodic load balancing,
 * if the whole system is idle.
 */
I
Ingo Molnar 已提交
3126
static inline void trigger_load_balance(struct rq *rq, int cpu)
3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177
{
#ifdef CONFIG_NO_HZ
	/*
	 * If we were in the nohz mode recently and busy at the current
	 * scheduler tick, then check if we need to nominate new idle
	 * load balancer.
	 */
	if (rq->in_nohz_recently && !rq->idle_at_tick) {
		rq->in_nohz_recently = 0;

		if (atomic_read(&nohz.load_balancer) == cpu) {
			cpu_clear(cpu, nohz.cpu_mask);
			atomic_set(&nohz.load_balancer, -1);
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/*
			 * simple selection for now: Nominate the
			 * first cpu in the nohz list to be the next
			 * ilb owner.
			 *
			 * TBD: Traverse the sched domains and nominate
			 * the nearest cpu in the nohz.cpu_mask.
			 */
			int ilb = first_cpu(nohz.cpu_mask);

			if (ilb != NR_CPUS)
				resched_cpu(ilb);
		}
	}

	/*
	 * If this cpu is idle and doing idle load balancing for all the
	 * cpus with ticks stopped, is it time for that to stop?
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
	    cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
		resched_cpu(cpu);
		return;
	}

	/*
	 * If this cpu is idle and the idle load balancing is done by
	 * someone else, then no need raise the SCHED_SOFTIRQ
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
	    cpu_isset(cpu, nohz.cpu_mask))
		return;
#endif
	if (time_after_eq(jiffies, rq->next_balance))
		raise_softirq(SCHED_SOFTIRQ);
L
Linus Torvalds 已提交
3178
}
I
Ingo Molnar 已提交
3179 3180 3181

#else	/* CONFIG_SMP */

L
Linus Torvalds 已提交
3182 3183 3184
/*
 * on UP we do not need to balance between CPUs:
 */
3185
static inline void idle_balance(int cpu, struct rq *rq)
L
Linus Torvalds 已提交
3186 3187
{
}
I
Ingo Molnar 已提交
3188 3189 3190 3191 3192 3193

/* Avoid "used but not defined" warning on UP */
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
3194
		      int *this_best_prio, struct rq_iterator *iterator)
I
Ingo Molnar 已提交
3195 3196 3197 3198 3199 3200
{
	*load_moved = 0;

	return 0;
}

L
Linus Torvalds 已提交
3201 3202 3203 3204 3205 3206 3207
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
3208 3209
 * Return p->sum_exec_runtime plus any more ns on the sched_clock
 * that have not yet been banked in case the task is currently running.
L
Linus Torvalds 已提交
3210
 */
3211
unsigned long long task_sched_runtime(struct task_struct *p)
L
Linus Torvalds 已提交
3212 3213
{
	unsigned long flags;
3214 3215
	u64 ns, delta_exec;
	struct rq *rq;
3216

3217 3218 3219
	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime;
	if (rq->curr == p) {
I
Ingo Molnar 已提交
3220 3221
		update_rq_clock(rq);
		delta_exec = rq->clock - p->se.exec_start;
3222 3223 3224 3225
		if ((s64)delta_exec > 0)
			ns += delta_exec;
	}
	task_rq_unlock(rq, &flags);
3226

L
Linus Torvalds 已提交
3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260
	return ns;
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in user space since the last update
 */
void account_user_time(struct task_struct *p, cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

	p->utime = cputime_add(p->utime, cputime);

	/* Add user time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (TASK_NICE(p) > 0)
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
	else
		cpustat->user = cputime64_add(cpustat->user, tmp);
}

/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
 */
void account_system_time(struct task_struct *p, int hardirq_offset,
			 cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3261
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
	cputime64_t tmp;

	p->stime = cputime_add(p->stime, cputime);

	/* Add system time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (hardirq_count() - hardirq_offset)
		cpustat->irq = cputime64_add(cpustat->irq, tmp);
	else if (softirq_count())
		cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
	else if (p != rq->idle)
		cpustat->system = cputime64_add(cpustat->system, tmp);
	else if (atomic_read(&rq->nr_iowait) > 0)
		cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
	else
		cpustat->idle = cputime64_add(cpustat->idle, tmp);
	/* Account for system time used */
	acct_update_integrals(p);
}

/*
 * Account for involuntary wait time.
 * @p: the process from which the cpu time has been stolen
 * @steal: the cpu time spent in involuntary wait
 */
void account_steal_time(struct task_struct *p, cputime_t steal)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp = cputime_to_cputime64(steal);
3291
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302

	if (p == rq->idle) {
		p->stime = cputime_add(p->stime, steal);
		if (atomic_read(&rq->nr_iowait) > 0)
			cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
		else
			cpustat->idle = cputime64_add(cpustat->idle, tmp);
	} else
		cpustat->steal = cputime64_add(cpustat->steal, tmp);
}

3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 *
 * It also gets called by the fork code, when changing the parent's
 * timeslices.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
3314 3315 3316
	struct task_struct *curr = rq->curr;

	spin_lock(&rq->lock);
3317
	update_cpu_load(rq);
I
Ingo Molnar 已提交
3318 3319 3320
	if (curr != rq->idle) /* FIXME: needed? */
		curr->sched_class->task_tick(rq, curr);
	spin_unlock(&rq->lock);
3321

3322
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
3323 3324
	rq->idle_at_tick = idle_cpu(cpu);
	trigger_load_balance(rq, cpu);
3325
#endif
L
Linus Torvalds 已提交
3326 3327 3328 3329 3330 3331 3332 3333 3334
}

#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)

void fastcall add_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3335 3336
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
L
Linus Torvalds 已提交
3337 3338 3339 3340
	preempt_count() += val;
	/*
	 * Spinlock count overflowing soon?
	 */
3341 3342
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
L
Linus Torvalds 已提交
3343 3344 3345 3346 3347 3348 3349 3350
}
EXPORT_SYMBOL(add_preempt_count);

void fastcall sub_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3351 3352
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
		return;
L
Linus Torvalds 已提交
3353 3354 3355
	/*
	 * Is the spinlock portion underflowing?
	 */
3356 3357 3358 3359
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;

L
Linus Torvalds 已提交
3360 3361 3362 3363 3364 3365 3366
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
3367
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
3368
 */
I
Ingo Molnar 已提交
3369
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
3370
{
I
Ingo Molnar 已提交
3371 3372 3373 3374 3375 3376 3377
	printk(KERN_ERR "BUG: scheduling while atomic: %s/0x%08x/%d\n",
		prev->comm, preempt_count(), prev->pid);
	debug_show_held_locks(prev);
	if (irqs_disabled())
		print_irqtrace_events(prev);
	dump_stack();
}
L
Linus Torvalds 已提交
3378

I
Ingo Molnar 已提交
3379 3380 3381 3382 3383
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
3384 3385 3386 3387 3388
	/*
	 * Test if we are atomic.  Since do_exit() needs to call into
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
I
Ingo Molnar 已提交
3389 3390 3391
	if (unlikely(in_atomic_preempt_off()) && unlikely(!prev->exit_state))
		__schedule_bug(prev);

L
Linus Torvalds 已提交
3392 3393
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

I
Ingo Molnar 已提交
3394 3395 3396 3397 3398 3399 3400
	schedstat_inc(this_rq(), sched_cnt);
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
3401
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
3402 3403 3404
{
	struct sched_class *class;
	struct task_struct *p;
L
Linus Torvalds 已提交
3405 3406

	/*
I
Ingo Molnar 已提交
3407 3408
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
3409
	 */
I
Ingo Molnar 已提交
3410
	if (likely(rq->nr_running == rq->cfs.nr_running)) {
3411
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
3412 3413
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
3414 3415
	}

I
Ingo Molnar 已提交
3416 3417
	class = sched_class_highest;
	for ( ; ; ) {
3418
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
3419 3420 3421 3422 3423 3424 3425 3426 3427
		if (p)
			return p;
		/*
		 * Will never be NULL as the idle class always
		 * returns a non-NULL p:
		 */
		class = class->next;
	}
}
L
Linus Torvalds 已提交
3428

I
Ingo Molnar 已提交
3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451
/*
 * schedule() is the main scheduler function.
 */
asmlinkage void __sched schedule(void)
{
	struct task_struct *prev, *next;
	long *switch_count;
	struct rq *rq;
	u64 now;
	int cpu;

need_resched:
	preempt_disable();
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
	rcu_qsctr_inc(cpu);
	prev = rq->curr;
	switch_count = &prev->nivcsw;

	release_kernel_lock(prev);
need_resched_nonpreemptible:

	schedule_debug(prev);
L
Linus Torvalds 已提交
3452 3453

	spin_lock_irq(&rq->lock);
I
Ingo Molnar 已提交
3454
	clear_tsk_need_resched(prev);
I
Ingo Molnar 已提交
3455 3456
	__update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
3457 3458 3459

	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
		if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
I
Ingo Molnar 已提交
3460
				unlikely(signal_pending(prev)))) {
L
Linus Torvalds 已提交
3461
			prev->state = TASK_RUNNING;
I
Ingo Molnar 已提交
3462
		} else {
I
Ingo Molnar 已提交
3463
			deactivate_task(rq, prev, 1, now);
L
Linus Torvalds 已提交
3464
		}
I
Ingo Molnar 已提交
3465
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
3466 3467
	}

I
Ingo Molnar 已提交
3468
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
3469 3470
		idle_balance(cpu, rq);

3471
	prev->sched_class->put_prev_task(rq, prev);
3472
	next = pick_next_task(rq, prev);
L
Linus Torvalds 已提交
3473 3474

	sched_info_switch(prev, next);
I
Ingo Molnar 已提交
3475

L
Linus Torvalds 已提交
3476 3477 3478 3479 3480
	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
3481
		context_switch(rq, prev, next); /* unlocks the rq */
L
Linus Torvalds 已提交
3482 3483 3484
	} else
		spin_unlock_irq(&rq->lock);

I
Ingo Molnar 已提交
3485 3486 3487
	if (unlikely(reacquire_kernel_lock(current) < 0)) {
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
3488
		goto need_resched_nonpreemptible;
I
Ingo Molnar 已提交
3489
	}
L
Linus Torvalds 已提交
3490 3491 3492 3493 3494 3495 3496 3497
	preempt_enable_no_resched();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(schedule);

#ifdef CONFIG_PREEMPT
/*
3498
 * this is the entry point to schedule() from in-kernel preemption
L
Linus Torvalds 已提交
3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512
 * off of preempt_enable.  Kernel preemptions off return from interrupt
 * occur there and call schedule directly.
 */
asmlinkage void __sched preempt_schedule(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
	 * we do not want to preempt the current task.  Just return..
	 */
N
Nick Piggin 已提交
3513
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540
		return;

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	schedule();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(preempt_schedule);

/*
3541
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
3553
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582
	BUG_ON(ti->preempt_count || !irqs_disabled());

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	local_irq_enable();
	schedule();
	local_irq_disable();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}

#endif /* CONFIG_PREEMPT */

I
Ingo Molnar 已提交
3583 3584
int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
			  void *key)
L
Linus Torvalds 已提交
3585
{
3586
	return try_to_wake_up(curr->private, mode, sync);
L
Linus Torvalds 已提交
3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604
}
EXPORT_SYMBOL(default_wake_function);

/*
 * The core wakeup function.  Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up.  If it's an exclusive wakeup (nr_exclusive == small +ve
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
 * started to run but is not in state TASK_RUNNING.  try_to_wake_up() returns
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
			     int nr_exclusive, int sync, void *key)
{
	struct list_head *tmp, *next;

	list_for_each_safe(tmp, next, &q->task_list) {
3605 3606 3607
		wait_queue_t *curr = list_entry(tmp, wait_queue_t, task_list);
		unsigned flags = curr->flags;

L
Linus Torvalds 已提交
3608
		if (curr->func(curr, mode, sync, key) &&
3609
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
3610 3611 3612 3613 3614 3615 3616 3617 3618
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
3619
 * @key: is directly passed to the wakeup function
L
Linus Torvalds 已提交
3620 3621
 */
void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
3622
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
void fastcall __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
{
	__wake_up_common(q, mode, 1, 0, NULL);
}

/**
3641
 * __wake_up_sync - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
 */
I
Ingo Molnar 已提交
3653 3654
void fastcall
__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
L
Linus Torvalds 已提交
3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697
{
	unsigned long flags;
	int sync = 1;

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
		sync = 0;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, sync, NULL);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

void fastcall complete(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 1, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

void fastcall complete_all(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 0, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

void fastcall __sched wait_for_completion(struct completion *x)
{
	might_sleep();
3698

L
Linus Torvalds 已提交
3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816
	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
	spin_unlock_irq(&x->wait.lock);
}
EXPORT_SYMBOL(wait_for_completion);

unsigned long fastcall __sched
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_timeout);

int fastcall __sched wait_for_completion_interruptible(struct completion *x)
{
	int ret = 0;

	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				ret = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);

	return ret;
}
EXPORT_SYMBOL(wait_for_completion_interruptible);

unsigned long fastcall __sched
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				timeout = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);

I
Ingo Molnar 已提交
3817 3818 3819 3820 3821
static inline void
sleep_on_head(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irqsave(&q->lock, *flags);
	__add_wait_queue(q, wait);
L
Linus Torvalds 已提交
3822
	spin_unlock(&q->lock);
I
Ingo Molnar 已提交
3823
}
L
Linus Torvalds 已提交
3824

I
Ingo Molnar 已提交
3825 3826 3827 3828 3829 3830 3831
static inline void
sleep_on_tail(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, wait);
	spin_unlock_irqrestore(&q->lock, *flags);
}
L
Linus Torvalds 已提交
3832

I
Ingo Molnar 已提交
3833
void __sched interruptible_sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3834
{
I
Ingo Molnar 已提交
3835 3836 3837 3838
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3839 3840 3841

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3842
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3843
	schedule();
I
Ingo Molnar 已提交
3844
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3845 3846 3847
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
3848
long __sched
I
Ingo Molnar 已提交
3849
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3850
{
I
Ingo Molnar 已提交
3851 3852 3853 3854
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3855 3856 3857

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3858
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3859
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3860
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3861 3862 3863 3864 3865

	return timeout;
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
3866
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3867
{
I
Ingo Molnar 已提交
3868 3869 3870 3871
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3872 3873 3874

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3875
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3876
	schedule();
I
Ingo Molnar 已提交
3877
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3878 3879 3880
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
3881
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3882
{
I
Ingo Molnar 已提交
3883 3884 3885 3886
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3887 3888 3889

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3890
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3891
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3892
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3893 3894 3895 3896 3897

	return timeout;
}
EXPORT_SYMBOL(sleep_on_timeout);

3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
3910
void rt_mutex_setprio(struct task_struct *p, int prio)
3911 3912
{
	unsigned long flags;
I
Ingo Molnar 已提交
3913
	int oldprio, on_rq;
3914
	struct rq *rq;
I
Ingo Molnar 已提交
3915
	u64 now;
3916 3917 3918 3919

	BUG_ON(prio < 0 || prio > MAX_PRIO);

	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3920 3921
	update_rq_clock(rq);
	now = rq->clock;
3922

3923
	oldprio = p->prio;
I
Ingo Molnar 已提交
3924 3925 3926 3927 3928 3929 3930 3931 3932
	on_rq = p->se.on_rq;
	if (on_rq)
		dequeue_task(rq, p, 0, now);

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

3933 3934
	p->prio = prio;

I
Ingo Molnar 已提交
3935 3936
	if (on_rq) {
		enqueue_task(rq, p, 0, now);
3937 3938
		/*
		 * Reschedule if we are currently running on this runqueue and
3939 3940
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
3941
		 */
3942 3943 3944
		if (task_running(rq, p)) {
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
3945 3946 3947
		} else {
			check_preempt_curr(rq, p);
		}
3948 3949 3950 3951 3952 3953
	}
	task_rq_unlock(rq, &flags);
}

#endif

3954
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
3955
{
I
Ingo Molnar 已提交
3956
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
3957
	unsigned long flags;
3958
	struct rq *rq;
I
Ingo Molnar 已提交
3959
	u64 now;
L
Linus Torvalds 已提交
3960 3961 3962 3963 3964 3965 3966 3967

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3968 3969
	update_rq_clock(rq);
	now = rq->clock;
L
Linus Torvalds 已提交
3970 3971 3972 3973
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
3974
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
3975
	 */
3976
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
3977 3978 3979
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
I
Ingo Molnar 已提交
3980 3981 3982
	on_rq = p->se.on_rq;
	if (on_rq) {
		dequeue_task(rq, p, 0, now);
3983
		dec_load(rq, p);
3984
	}
L
Linus Torvalds 已提交
3985 3986

	p->static_prio = NICE_TO_PRIO(nice);
3987
	set_load_weight(p);
3988 3989 3990
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
3991

I
Ingo Molnar 已提交
3992 3993
	if (on_rq) {
		enqueue_task(rq, p, 0, now);
3994
		inc_load(rq, p);
L
Linus Torvalds 已提交
3995
		/*
3996 3997
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
3998
		 */
3999
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
4000 4001 4002 4003 4004 4005 4006
			resched_task(rq->curr);
	}
out_unlock:
	task_rq_unlock(rq, &flags);
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
4007 4008 4009 4010 4011
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
4012
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
4013
{
4014 4015
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
4016

M
Matt Mackall 已提交
4017 4018 4019 4020
	return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
asmlinkage long sys_nice(int increment)
{
4032
	long nice, retval;
L
Linus Torvalds 已提交
4033 4034 4035 4036 4037 4038

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
4039 4040
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
4041 4042 4043 4044 4045 4046 4047 4048 4049
	if (increment > 40)
		increment = 40;

	nice = PRIO_TO_NICE(current->static_prio) + increment;
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
4050 4051 4052
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
4071
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
4072 4073 4074 4075 4076 4077 4078 4079
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
4080
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098
{
	return TASK_NICE(p);
}
EXPORT_SYMBOL_GPL(task_nice);

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
4099
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
4100 4101 4102 4103 4104 4105 4106 4107
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
4108
static inline struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
4109 4110 4111 4112 4113
{
	return pid ? find_task_by_pid(pid) : current;
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
4114 4115
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
4116
{
I
Ingo Molnar 已提交
4117
	BUG_ON(p->se.on_rq);
4118

L
Linus Torvalds 已提交
4119
	p->policy = policy;
I
Ingo Molnar 已提交
4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131
	switch (p->policy) {
	case SCHED_NORMAL:
	case SCHED_BATCH:
	case SCHED_IDLE:
		p->sched_class = &fair_sched_class;
		break;
	case SCHED_FIFO:
	case SCHED_RR:
		p->sched_class = &rt_sched_class;
		break;
	}

L
Linus Torvalds 已提交
4132
	p->rt_priority = prio;
4133 4134 4135
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
4136
	set_load_weight(p);
L
Linus Torvalds 已提交
4137 4138 4139
}

/**
4140
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
L
Linus Torvalds 已提交
4141 4142 4143
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
4144
 *
4145
 * NOTE that the task may be already dead.
L
Linus Torvalds 已提交
4146
 */
I
Ingo Molnar 已提交
4147 4148
int sched_setscheduler(struct task_struct *p, int policy,
		       struct sched_param *param)
L
Linus Torvalds 已提交
4149
{
I
Ingo Molnar 已提交
4150
	int retval, oldprio, oldpolicy = -1, on_rq;
L
Linus Torvalds 已提交
4151
	unsigned long flags;
4152
	struct rq *rq;
L
Linus Torvalds 已提交
4153

4154 4155
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
4156 4157 4158 4159 4160
recheck:
	/* double check policy once rq lock held */
	if (policy < 0)
		policy = oldpolicy = p->policy;
	else if (policy != SCHED_FIFO && policy != SCHED_RR &&
I
Ingo Molnar 已提交
4161 4162
			policy != SCHED_NORMAL && policy != SCHED_BATCH &&
			policy != SCHED_IDLE)
4163
		return -EINVAL;
L
Linus Torvalds 已提交
4164 4165
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
4166 4167
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
4168 4169
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
4170
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
4171
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
4172
		return -EINVAL;
4173
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
4174 4175
		return -EINVAL;

4176 4177 4178 4179
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
	if (!capable(CAP_SYS_NICE)) {
4180
		if (rt_policy(policy)) {
4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196
			unsigned long rlim_rtprio;

			if (!lock_task_sighand(p, &flags))
				return -ESRCH;
			rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
			unlock_task_sighand(p, &flags);

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
I
Ingo Molnar 已提交
4197 4198 4199 4200 4201 4202
		/*
		 * Like positive nice levels, dont allow tasks to
		 * move out of SCHED_IDLE either:
		 */
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
			return -EPERM;
4203

4204 4205 4206 4207 4208
		/* can't change other user's priorities */
		if ((current->euid != p->euid) &&
		    (current->euid != p->uid))
			return -EPERM;
	}
L
Linus Torvalds 已提交
4209 4210 4211 4212

	retval = security_task_setscheduler(p, policy, param);
	if (retval)
		return retval;
4213 4214 4215 4216 4217
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
	 */
	spin_lock_irqsave(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4218 4219 4220 4221
	/*
	 * To be able to change p->policy safely, the apropriate
	 * runqueue lock must be held.
	 */
4222
	rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
4223 4224 4225
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
4226 4227
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4228 4229
		goto recheck;
	}
I
Ingo Molnar 已提交
4230
	on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
4231 4232 4233 4234
	if (on_rq) {
		update_rq_clock(rq);
		deactivate_task(rq, p, 0, rq->clock);
	}
L
Linus Torvalds 已提交
4235
	oldprio = p->prio;
I
Ingo Molnar 已提交
4236 4237 4238
	__setscheduler(rq, p, policy, param->sched_priority);
	if (on_rq) {
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
4239 4240
		/*
		 * Reschedule if we are currently running on this runqueue and
4241 4242
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
L
Linus Torvalds 已提交
4243
		 */
4244 4245 4246
		if (task_running(rq, p)) {
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
4247 4248 4249
		} else {
			check_preempt_curr(rq, p);
		}
L
Linus Torvalds 已提交
4250
	}
4251 4252 4253
	__task_rq_unlock(rq);
	spin_unlock_irqrestore(&p->pi_lock, flags);

4254 4255
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
4256 4257 4258 4259
	return 0;
}
EXPORT_SYMBOL_GPL(sched_setscheduler);

I
Ingo Molnar 已提交
4260 4261
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
4262 4263 4264
{
	struct sched_param lparam;
	struct task_struct *p;
4265
	int retval;
L
Linus Torvalds 已提交
4266 4267 4268 4269 4270

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
4271 4272 4273

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
4274
	p = find_process_by_pid(pid);
4275 4276 4277
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
4278

L
Linus Torvalds 已提交
4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
				       struct sched_param __user *param)
{
4291 4292 4293 4294
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
asmlinkage long sys_sched_getscheduler(pid_t pid)
{
4314
	struct task_struct *p;
L
Linus Torvalds 已提交
4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341
	int retval = -EINVAL;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
			retval = p->policy;
	}
	read_unlock(&tasklist_lock);

out_nounlock:
	return retval;
}

/**
 * sys_sched_getscheduler - get the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
{
	struct sched_param lp;
4342
	struct task_struct *p;
L
Linus Torvalds 已提交
4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376
	int retval = -EINVAL;

	if (!param || pid < 0)
		goto out_nounlock;

	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
	read_unlock(&tasklist_lock);

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

out_nounlock:
	return retval;

out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

long sched_setaffinity(pid_t pid, cpumask_t new_mask)
{
	cpumask_t cpus_allowed;
4377 4378
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
4379

4380
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4381 4382 4383 4384 4385
	read_lock(&tasklist_lock);

	p = find_process_by_pid(pid);
	if (!p) {
		read_unlock(&tasklist_lock);
4386
		mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402
		return -ESRCH;
	}

	/*
	 * It is not safe to call set_cpus_allowed with the
	 * tasklist_lock held.  We will bump the task_struct's
	 * usage count and then drop tasklist_lock.
	 */
	get_task_struct(p);
	read_unlock(&tasklist_lock);

	retval = -EPERM;
	if ((current->euid != p->euid) && (current->euid != p->uid) &&
			!capable(CAP_SYS_NICE))
		goto out_unlock;

4403 4404 4405 4406
	retval = security_task_setscheduler(p, 0, NULL);
	if (retval)
		goto out_unlock;

L
Linus Torvalds 已提交
4407 4408 4409 4410 4411 4412
	cpus_allowed = cpuset_cpus_allowed(p);
	cpus_and(new_mask, new_mask, cpus_allowed);
	retval = set_cpus_allowed(p, new_mask);

out_unlock:
	put_task_struct(p);
4413
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
			     cpumask_t *new_mask)
{
	if (len < sizeof(cpumask_t)) {
		memset(new_mask, 0, sizeof(cpumask_t));
	} else if (len > sizeof(cpumask_t)) {
		len = sizeof(cpumask_t);
	}
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	cpumask_t new_mask;
	int retval;

	retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask);
	if (retval)
		return retval;

	return sched_setaffinity(pid, new_mask);
}

/*
 * Represents all cpu's present in the system
 * In systems capable of hotplug, this map could dynamically grow
 * as new cpu's are detected in the system via any platform specific
 * method, such as ACPI for e.g.
 */

4454
cpumask_t cpu_present_map __read_mostly;
L
Linus Torvalds 已提交
4455 4456 4457
EXPORT_SYMBOL(cpu_present_map);

#ifndef CONFIG_SMP
4458
cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
4459 4460
EXPORT_SYMBOL(cpu_online_map);

4461
cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
4462
EXPORT_SYMBOL(cpu_possible_map);
L
Linus Torvalds 已提交
4463 4464 4465 4466
#endif

long sched_getaffinity(pid_t pid, cpumask_t *mask)
{
4467
	struct task_struct *p;
L
Linus Torvalds 已提交
4468 4469
	int retval;

4470
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4471 4472 4473 4474 4475 4476 4477
	read_lock(&tasklist_lock);

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

4478 4479 4480 4481
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4482
	cpus_and(*mask, p->cpus_allowed, cpu_online_map);
L
Linus Torvalds 已提交
4483 4484 4485

out_unlock:
	read_unlock(&tasklist_lock);
4486
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4487

4488
	return retval;
L
Linus Torvalds 已提交
4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	int ret;
	cpumask_t mask;

	if (len < sizeof(cpumask_t))
		return -EINVAL;

	ret = sched_getaffinity(pid, &mask);
	if (ret < 0)
		return ret;

	if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t)))
		return -EFAULT;

	return sizeof(cpumask_t);
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
4519 4520
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
4521 4522 4523
 */
asmlinkage long sys_sched_yield(void)
{
4524
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
4525 4526

	schedstat_inc(rq, yld_cnt);
I
Ingo Molnar 已提交
4527
	if (unlikely(rq->nr_running == 1))
L
Linus Torvalds 已提交
4528
		schedstat_inc(rq, yld_act_empty);
I
Ingo Molnar 已提交
4529 4530
	else
		current->sched_class->yield_task(rq, current);
L
Linus Torvalds 已提交
4531 4532 4533 4534 4535 4536

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
4537
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4538 4539 4540 4541 4542 4543 4544 4545
	_raw_spin_unlock(&rq->lock);
	preempt_enable_no_resched();

	schedule();

	return 0;
}

A
Andrew Morton 已提交
4546
static void __cond_resched(void)
L
Linus Torvalds 已提交
4547
{
4548 4549 4550
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
	__might_sleep(__FILE__, __LINE__);
#endif
4551 4552 4553 4554 4555
	/*
	 * The BKS might be reacquired before we have dropped
	 * PREEMPT_ACTIVE, which could trigger a second
	 * cond_resched() call.
	 */
L
Linus Torvalds 已提交
4556 4557 4558 4559 4560 4561 4562 4563 4564
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
	} while (need_resched());
}

int __sched cond_resched(void)
{
4565 4566
	if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
					system_state == SYSTEM_RUNNING) {
L
Linus Torvalds 已提交
4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581
		__cond_resched();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched);

/*
 * cond_resched_lock() - if a reschedule is pending, drop the given lock,
 * call schedule, and on return reacquire the lock.
 *
 * This works OK both with and without CONFIG_PREEMPT.  We do strange low-level
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
I
Ingo Molnar 已提交
4582
int cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
4583
{
J
Jan Kara 已提交
4584 4585
	int ret = 0;

L
Linus Torvalds 已提交
4586 4587 4588
	if (need_lockbreak(lock)) {
		spin_unlock(lock);
		cpu_relax();
J
Jan Kara 已提交
4589
		ret = 1;
L
Linus Torvalds 已提交
4590 4591
		spin_lock(lock);
	}
4592
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4593
		spin_release(&lock->dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4594 4595 4596
		_raw_spin_unlock(lock);
		preempt_enable_no_resched();
		__cond_resched();
J
Jan Kara 已提交
4597
		ret = 1;
L
Linus Torvalds 已提交
4598 4599
		spin_lock(lock);
	}
J
Jan Kara 已提交
4600
	return ret;
L
Linus Torvalds 已提交
4601 4602 4603 4604 4605 4606 4607
}
EXPORT_SYMBOL(cond_resched_lock);

int __sched cond_resched_softirq(void)
{
	BUG_ON(!in_softirq());

4608
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4609
		local_bh_enable();
L
Linus Torvalds 已提交
4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched_softirq);

/**
 * yield - yield the current processor to other threads.
 *
4621
 * This is a shortcut for kernel-space yielding - it marks the
L
Linus Torvalds 已提交
4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639
 * thread runnable and calls sys_sched_yield().
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

/*
 * This task is about to go to sleep on IO.  Increment rq->nr_iowait so
 * that process accounting knows that this is a task in IO wait state.
 *
 * But don't do that if it is a deliberate, throttling IO wait (this task
 * has set its backing_dev_info: the queue against which it should throttle)
 */
void __sched io_schedule(void)
{
4640
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4641

4642
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4643 4644 4645
	atomic_inc(&rq->nr_iowait);
	schedule();
	atomic_dec(&rq->nr_iowait);
4646
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4647 4648 4649 4650 4651
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
4652
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4653 4654
	long ret;

4655
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4656 4657 4658
	atomic_inc(&rq->nr_iowait);
	ret = schedule_timeout(timeout);
	atomic_dec(&rq->nr_iowait);
4659
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_max(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
4680
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4681
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_min(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
4705
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4706
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
asmlinkage
long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
{
4723
	struct task_struct *p;
L
Linus Torvalds 已提交
4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739
	int retval = -EINVAL;
	struct timespec t;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4740
	jiffies_to_timespec(p->policy == SCHED_FIFO ?
I
Ingo Molnar 已提交
4741
				0 : static_prio_timeslice(p->static_prio), &t);
L
Linus Torvalds 已提交
4742 4743 4744 4745 4746 4747 4748 4749 4750
	read_unlock(&tasklist_lock);
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
out_nounlock:
	return retval;
out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

4751
static const char stat_nam[] = "RSDTtZX";
4752 4753

static void show_task(struct task_struct *p)
L
Linus Torvalds 已提交
4754 4755
{
	unsigned long free = 0;
4756
	unsigned state;
L
Linus Torvalds 已提交
4757 4758

	state = p->state ? __ffs(p->state) + 1 : 0;
4759 4760
	printk("%-13.13s %c", p->comm,
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4761
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
4762
	if (state == TASK_RUNNING)
4763
		printk(" running  ");
L
Linus Torvalds 已提交
4764
	else
4765
		printk(" %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4766 4767
#else
	if (state == TASK_RUNNING)
4768
		printk("  running task    ");
L
Linus Torvalds 已提交
4769 4770 4771 4772 4773
	else
		printk(" %016lx ", thread_saved_pc(p));
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
	{
4774
		unsigned long *n = end_of_stack(p);
L
Linus Torvalds 已提交
4775 4776
		while (!*n)
			n++;
4777
		free = (unsigned long)n - (unsigned long)end_of_stack(p);
L
Linus Torvalds 已提交
4778 4779
	}
#endif
4780
	printk("%5lu %5d %6d\n", free, p->pid, p->parent->pid);
L
Linus Torvalds 已提交
4781 4782 4783 4784 4785

	if (state != TASK_RUNNING)
		show_stack(p, NULL);
}

I
Ingo Molnar 已提交
4786
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
4787
{
4788
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
4789

4790 4791 4792
#if BITS_PER_LONG == 32
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
4793
#else
4794 4795
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
4796 4797 4798 4799 4800 4801 4802 4803
#endif
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
		 * console might take alot of time:
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
4804
		if (!state_filter || (p->state & state_filter))
I
Ingo Molnar 已提交
4805
			show_task(p);
L
Linus Torvalds 已提交
4806 4807
	} while_each_thread(g, p);

4808 4809
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
4810 4811 4812
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
L
Linus Torvalds 已提交
4813
	read_unlock(&tasklist_lock);
I
Ingo Molnar 已提交
4814 4815 4816 4817 4818
	/*
	 * Only show locks if all tasks are dumped:
	 */
	if (state_filter == -1)
		debug_show_all_locks();
L
Linus Torvalds 已提交
4819 4820
}

I
Ingo Molnar 已提交
4821 4822
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
4823
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
4824 4825
}

4826 4827 4828 4829 4830 4831 4832 4833
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
4834
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
4835
{
4836
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4837 4838
	unsigned long flags;

I
Ingo Molnar 已提交
4839 4840 4841
	__sched_fork(idle);
	idle->se.exec_start = sched_clock();

4842
	idle->prio = idle->normal_prio = MAX_PRIO;
L
Linus Torvalds 已提交
4843
	idle->cpus_allowed = cpumask_of_cpu(cpu);
I
Ingo Molnar 已提交
4844
	__set_task_cpu(idle, cpu);
L
Linus Torvalds 已提交
4845 4846 4847

	spin_lock_irqsave(&rq->lock, flags);
	rq->curr = rq->idle = idle;
4848 4849 4850
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
	idle->oncpu = 1;
#endif
L
Linus Torvalds 已提交
4851 4852 4853 4854
	spin_unlock_irqrestore(&rq->lock, flags);

	/* Set the preempt count _outside_ the spinlocks! */
#if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_BKL)
A
Al Viro 已提交
4855
	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
L
Linus Torvalds 已提交
4856
#else
A
Al Viro 已提交
4857
	task_thread_info(idle)->preempt_count = 0;
L
Linus Torvalds 已提交
4858
#endif
I
Ingo Molnar 已提交
4859 4860 4861 4862
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873
}

/*
 * In a system that switches off the HZ timer nohz_cpu_mask
 * indicates which cpus entered this state. This is used
 * in the rcu update to wait only for active cpus. For system
 * which do not switch off the HZ timer nohz_cpu_mask should
 * always be CPU_MASK_NONE.
 */
cpumask_t nohz_cpu_mask = CPU_MASK_NONE;

I
Ingo Molnar 已提交
4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885
/*
 * Increase the granularity value when there are more CPUs,
 * because with more CPUs the 'effective latency' as visible
 * to users decreases. But the relationship is not linear,
 * so pick a second-best guess by going with the log2 of the
 * number of CPUs.
 *
 * This idea comes from the SD scheduler of Con Kolivas:
 */
static inline void sched_init_granularity(void)
{
	unsigned int factor = 1 + ilog2(num_online_cpus());
4886
	const unsigned long gran_limit = 100000000;
I
Ingo Molnar 已提交
4887 4888 4889 4890 4891 4892 4893 4894 4895

	sysctl_sched_granularity *= factor;
	if (sysctl_sched_granularity > gran_limit)
		sysctl_sched_granularity = gran_limit;

	sysctl_sched_runtime_limit = sysctl_sched_granularity * 4;
	sysctl_sched_wakeup_granularity = sysctl_sched_granularity / 2;
}

L
Linus Torvalds 已提交
4896 4897 4898 4899
#ifdef CONFIG_SMP
/*
 * This is how migration works:
 *
4900
 * 1) we queue a struct migration_req structure in the source CPU's
L
Linus Torvalds 已提交
4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921
 *    runqueue and wake up that CPU's migration thread.
 * 2) we down() the locked semaphore => thread blocks.
 * 3) migration thread wakes up (implicitly it forces the migrated
 *    thread off the CPU)
 * 4) it gets the migration request and checks whether the migrated
 *    task is still in the wrong runqueue.
 * 5) if it's in the wrong runqueue then the migration thread removes
 *    it and puts it into the right queue.
 * 6) migration thread up()s the semaphore.
 * 7) we wake up and the migration is done.
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
 * task must not exit() & deallocate itself prematurely.  The
 * call is not atomic; no spinlocks may be held.
 */
4922
int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
L
Linus Torvalds 已提交
4923
{
4924
	struct migration_req req;
L
Linus Torvalds 已提交
4925
	unsigned long flags;
4926
	struct rq *rq;
4927
	int ret = 0;
L
Linus Torvalds 已提交
4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949

	rq = task_rq_lock(p, &flags);
	if (!cpus_intersects(new_mask, cpu_online_map)) {
		ret = -EINVAL;
		goto out;
	}

	p->cpus_allowed = new_mask;
	/* Can the task run on the task's current CPU? If so, we're done */
	if (cpu_isset(task_cpu(p), new_mask))
		goto out;

	if (migrate_task(p, any_online_cpu(new_mask), &req)) {
		/* Need help from migration thread: drop lock and wait. */
		task_rq_unlock(rq, &flags);
		wake_up_process(rq->migration_thread);
		wait_for_completion(&req.done);
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
	task_rq_unlock(rq, &flags);
4950

L
Linus Torvalds 已提交
4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962
	return ret;
}
EXPORT_SYMBOL_GPL(set_cpus_allowed);

/*
 * Move (not current) task off this cpu, onto dest cpu.  We're doing
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
4963 4964
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
4965
 */
4966
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
4967
{
4968
	struct rq *rq_dest, *rq_src;
I
Ingo Molnar 已提交
4969
	int ret = 0, on_rq;
L
Linus Torvalds 已提交
4970 4971

	if (unlikely(cpu_is_offline(dest_cpu)))
4972
		return ret;
L
Linus Torvalds 已提交
4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
		goto out;
	/* Affinity changed (again). */
	if (!cpu_isset(dest_cpu, p->cpus_allowed))
		goto out;

I
Ingo Molnar 已提交
4985
	on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
4986 4987 4988 4989
	if (on_rq) {
		update_rq_clock(rq_src);
		deactivate_task(rq_src, p, 0, rq_src->clock);
	}
L
Linus Torvalds 已提交
4990
	set_task_cpu(p, dest_cpu);
I
Ingo Molnar 已提交
4991 4992 4993
	if (on_rq) {
		activate_task(rq_dest, p, 0);
		check_preempt_curr(rq_dest, p);
L
Linus Torvalds 已提交
4994
	}
4995
	ret = 1;
L
Linus Torvalds 已提交
4996 4997
out:
	double_rq_unlock(rq_src, rq_dest);
4998
	return ret;
L
Linus Torvalds 已提交
4999 5000 5001 5002 5003 5004 5005
}

/*
 * migration_thread - this is a highprio system thread that performs
 * thread migration by bumping thread off CPU then 'pushing' onto
 * another runqueue.
 */
I
Ingo Molnar 已提交
5006
static int migration_thread(void *data)
L
Linus Torvalds 已提交
5007 5008
{
	int cpu = (long)data;
5009
	struct rq *rq;
L
Linus Torvalds 已提交
5010 5011 5012 5013 5014 5015

	rq = cpu_rq(cpu);
	BUG_ON(rq->migration_thread != current);

	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
5016
		struct migration_req *req;
L
Linus Torvalds 已提交
5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038
		struct list_head *head;

		spin_lock_irq(&rq->lock);

		if (cpu_is_offline(cpu)) {
			spin_unlock_irq(&rq->lock);
			goto wait_to_die;
		}

		if (rq->active_balance) {
			active_load_balance(rq, cpu);
			rq->active_balance = 0;
		}

		head = &rq->migration_queue;

		if (list_empty(head)) {
			spin_unlock_irq(&rq->lock);
			schedule();
			set_current_state(TASK_INTERRUPTIBLE);
			continue;
		}
5039
		req = list_entry(head->next, struct migration_req, list);
L
Linus Torvalds 已提交
5040 5041
		list_del_init(head->next);

N
Nick Piggin 已提交
5042 5043 5044
		spin_unlock(&rq->lock);
		__migrate_task(req->task, cpu, req->dest_cpu);
		local_irq_enable();
L
Linus Torvalds 已提交
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062

		complete(&req->done);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
5063 5064 5065 5066
/*
 * Figure out where task on dead CPU should go, use force if neccessary.
 * NOTE: interrupts should be disabled by the caller
 */
5067
static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5068
{
5069
	unsigned long flags;
L
Linus Torvalds 已提交
5070
	cpumask_t mask;
5071 5072
	struct rq *rq;
	int dest_cpu;
L
Linus Torvalds 已提交
5073

5074
restart:
L
Linus Torvalds 已提交
5075 5076
	/* On same node? */
	mask = node_to_cpumask(cpu_to_node(dead_cpu));
5077
	cpus_and(mask, mask, p->cpus_allowed);
L
Linus Torvalds 已提交
5078 5079 5080 5081
	dest_cpu = any_online_cpu(mask);

	/* On any allowed CPU? */
	if (dest_cpu == NR_CPUS)
5082
		dest_cpu = any_online_cpu(p->cpus_allowed);
L
Linus Torvalds 已提交
5083 5084 5085

	/* No more Mr. Nice Guy. */
	if (dest_cpu == NR_CPUS) {
5086 5087 5088
		rq = task_rq_lock(p, &flags);
		cpus_setall(p->cpus_allowed);
		dest_cpu = any_online_cpu(p->cpus_allowed);
5089
		task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
5090 5091 5092 5093 5094 5095

		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
5096
		if (p->mm && printk_ratelimit())
L
Linus Torvalds 已提交
5097 5098
			printk(KERN_INFO "process %d (%s) no "
			       "longer affine to cpu%d\n",
5099
			       p->pid, p->comm, dead_cpu);
L
Linus Torvalds 已提交
5100
	}
5101
	if (!__migrate_task(p, dead_cpu, dest_cpu))
5102
		goto restart;
L
Linus Torvalds 已提交
5103 5104 5105 5106 5107 5108 5109 5110 5111
}

/*
 * While a dead CPU has no uninterruptible tasks queued at this point,
 * it might still have a nonzero ->nr_uninterruptible counter, because
 * for performance reasons the counter is not stricly tracking tasks to
 * their home CPUs. So we just add the counter to another CPU's counter,
 * to keep the global sum constant after CPU-down:
 */
5112
static void migrate_nr_uninterruptible(struct rq *rq_src)
L
Linus Torvalds 已提交
5113
{
5114
	struct rq *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL));
L
Linus Torvalds 已提交
5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127
	unsigned long flags;

	local_irq_save(flags);
	double_rq_lock(rq_src, rq_dest);
	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
	rq_src->nr_uninterruptible = 0;
	double_rq_unlock(rq_src, rq_dest);
	local_irq_restore(flags);
}

/* Run through task list and migrate tasks from the dead cpu. */
static void migrate_live_tasks(int src_cpu)
{
5128
	struct task_struct *p, *t;
L
Linus Torvalds 已提交
5129 5130 5131

	write_lock_irq(&tasklist_lock);

5132 5133
	do_each_thread(t, p) {
		if (p == current)
L
Linus Torvalds 已提交
5134 5135
			continue;

5136 5137 5138
		if (task_cpu(p) == src_cpu)
			move_task_off_dead_cpu(src_cpu, p);
	} while_each_thread(t, p);
L
Linus Torvalds 已提交
5139 5140 5141 5142

	write_unlock_irq(&tasklist_lock);
}

I
Ingo Molnar 已提交
5143 5144
/*
 * Schedules idle task to be the next runnable task on current CPU.
L
Linus Torvalds 已提交
5145
 * It does so by boosting its priority to highest possible and adding it to
5146
 * the _front_ of the runqueue. Used by CPU offline code.
L
Linus Torvalds 已提交
5147 5148 5149
 */
void sched_idle_next(void)
{
5150
	int this_cpu = smp_processor_id();
5151
	struct rq *rq = cpu_rq(this_cpu);
L
Linus Torvalds 已提交
5152 5153 5154 5155
	struct task_struct *p = rq->idle;
	unsigned long flags;

	/* cpu has to be offline */
5156
	BUG_ON(cpu_online(this_cpu));
L
Linus Torvalds 已提交
5157

5158 5159 5160
	/*
	 * Strictly not necessary since rest of the CPUs are stopped by now
	 * and interrupts disabled on the current cpu.
L
Linus Torvalds 已提交
5161 5162 5163
	 */
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
5164
	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
5165 5166

	/* Add idle task to the _front_ of its priority queue: */
I
Ingo Molnar 已提交
5167
	activate_idle_task(p, rq);
L
Linus Torvalds 已提交
5168 5169 5170 5171

	spin_unlock_irqrestore(&rq->lock, flags);
}

5172 5173
/*
 * Ensures that the idle task is using init_mm right before its cpu goes
L
Linus Torvalds 已提交
5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186
 * offline.
 */
void idle_task_exit(void)
{
	struct mm_struct *mm = current->active_mm;

	BUG_ON(cpu_online(smp_processor_id()));

	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
}

5187
/* called under rq->lock with disabled interrupts */
5188
static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5189
{
5190
	struct rq *rq = cpu_rq(dead_cpu);
L
Linus Torvalds 已提交
5191 5192

	/* Must be exiting, otherwise would be on tasklist. */
5193
	BUG_ON(p->exit_state != EXIT_ZOMBIE && p->exit_state != EXIT_DEAD);
L
Linus Torvalds 已提交
5194 5195

	/* Cannot have done final schedule yet: would have vanished. */
5196
	BUG_ON(p->state == TASK_DEAD);
L
Linus Torvalds 已提交
5197

5198
	get_task_struct(p);
L
Linus Torvalds 已提交
5199 5200 5201 5202 5203

	/*
	 * Drop lock around migration; if someone else moves it,
	 * that's OK.  No task can be added to this CPU, so iteration is
	 * fine.
5204
	 * NOTE: interrupts should be left disabled  --dev@
L
Linus Torvalds 已提交
5205
	 */
5206
	spin_unlock(&rq->lock);
5207
	move_task_off_dead_cpu(dead_cpu, p);
5208
	spin_lock(&rq->lock);
L
Linus Torvalds 已提交
5209

5210
	put_task_struct(p);
L
Linus Torvalds 已提交
5211 5212 5213 5214 5215
}

/* release_task() removes task from tasklist, so we won't find dead tasks. */
static void migrate_dead_tasks(unsigned int dead_cpu)
{
5216
	struct rq *rq = cpu_rq(dead_cpu);
I
Ingo Molnar 已提交
5217
	struct task_struct *next;
5218

I
Ingo Molnar 已提交
5219 5220 5221
	for ( ; ; ) {
		if (!rq->nr_running)
			break;
I
Ingo Molnar 已提交
5222
		update_rq_clock(rq);
5223
		next = pick_next_task(rq, rq->curr);
I
Ingo Molnar 已提交
5224 5225 5226
		if (!next)
			break;
		migrate_dead(dead_cpu, next);
5227

L
Linus Torvalds 已提交
5228 5229 5230 5231
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

5232 5233 5234
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
5235 5236 5237 5238
	{
		.procname	= "sched_domain",
		.mode		= 0755,
	},
5239 5240 5241 5242
	{0,},
};

static struct ctl_table sd_ctl_root[] = {
5243 5244 5245 5246 5247
	{
		.procname	= "kernel",
		.mode		= 0755,
		.child		= sd_ctl_dir,
	},
5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262
	{0,},
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
		kmalloc(n * sizeof(struct ctl_table), GFP_KERNEL);

	BUG_ON(!entry);
	memset(entry, 0, n * sizeof(struct ctl_table));

	return entry;
}

static void
5263
set_table_entry(struct ctl_table *entry,
5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278
		const char *procname, void *data, int maxlen,
		mode_t mode, proc_handler *proc_handler)
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
	struct ctl_table *table = sd_alloc_ctl_entry(14);

5279
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
5280
		sizeof(long), 0644, proc_doulongvec_minmax);
5281
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
5282
		sizeof(long), 0644, proc_doulongvec_minmax);
5283
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
5284
		sizeof(int), 0644, proc_dointvec_minmax);
5285
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
5286
		sizeof(int), 0644, proc_dointvec_minmax);
5287
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
5288
		sizeof(int), 0644, proc_dointvec_minmax);
5289
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
5290
		sizeof(int), 0644, proc_dointvec_minmax);
5291
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
5292
		sizeof(int), 0644, proc_dointvec_minmax);
5293
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
5294
		sizeof(int), 0644, proc_dointvec_minmax);
5295
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
5296
		sizeof(int), 0644, proc_dointvec_minmax);
5297
	set_table_entry(&table[10], "cache_nice_tries",
5298 5299
		&sd->cache_nice_tries,
		sizeof(int), 0644, proc_dointvec_minmax);
5300
	set_table_entry(&table[12], "flags", &sd->flags,
5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351
		sizeof(int), 0644, proc_dointvec_minmax);

	return table;
}

static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
		entry->mode = 0755;
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
static void init_sched_domain_sysctl(void)
{
	int i, cpu_num = num_online_cpus();
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

	sd_ctl_dir[0].child = entry;

	for (i = 0; i < cpu_num; i++, entry++) {
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
		entry->mode = 0755;
		entry->child = sd_alloc_ctl_cpu_table(i);
	}
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
#else
static void init_sched_domain_sysctl(void)
{
}
#endif

L
Linus Torvalds 已提交
5352 5353 5354 5355
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
5356 5357
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
5358 5359
{
	struct task_struct *p;
5360
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
5361
	unsigned long flags;
5362
	struct rq *rq;
L
Linus Torvalds 已提交
5363 5364

	switch (action) {
5365 5366 5367 5368
	case CPU_LOCK_ACQUIRE:
		mutex_lock(&sched_hotcpu_mutex);
		break;

L
Linus Torvalds 已提交
5369
	case CPU_UP_PREPARE:
5370
	case CPU_UP_PREPARE_FROZEN:
I
Ingo Molnar 已提交
5371
		p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
L
Linus Torvalds 已提交
5372 5373 5374 5375 5376
		if (IS_ERR(p))
			return NOTIFY_BAD;
		kthread_bind(p, cpu);
		/* Must be high prio: stop_machine expects to yield to it. */
		rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
5377
		__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
L
Linus Torvalds 已提交
5378 5379 5380
		task_rq_unlock(rq, &flags);
		cpu_rq(cpu)->migration_thread = p;
		break;
5381

L
Linus Torvalds 已提交
5382
	case CPU_ONLINE:
5383
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
5384 5385 5386
		/* Strictly unneccessary, as first user will wake it. */
		wake_up_process(cpu_rq(cpu)->migration_thread);
		break;
5387

L
Linus Torvalds 已提交
5388 5389
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
5390
	case CPU_UP_CANCELED_FROZEN:
5391 5392
		if (!cpu_rq(cpu)->migration_thread)
			break;
L
Linus Torvalds 已提交
5393
		/* Unbind it from offline cpu so it can run.  Fall thru. */
5394 5395
		kthread_bind(cpu_rq(cpu)->migration_thread,
			     any_online_cpu(cpu_online_map));
L
Linus Torvalds 已提交
5396 5397 5398
		kthread_stop(cpu_rq(cpu)->migration_thread);
		cpu_rq(cpu)->migration_thread = NULL;
		break;
5399

L
Linus Torvalds 已提交
5400
	case CPU_DEAD:
5401
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
5402 5403 5404 5405 5406 5407
		migrate_live_tasks(cpu);
		rq = cpu_rq(cpu);
		kthread_stop(rq->migration_thread);
		rq->migration_thread = NULL;
		/* Idle task back to normal (off runqueue, low prio) */
		rq = task_rq_lock(rq->idle, &flags);
I
Ingo Molnar 已提交
5408 5409
		update_rq_clock(rq);
		deactivate_task(rq, rq->idle, 0, rq->clock);
L
Linus Torvalds 已提交
5410
		rq->idle->static_prio = MAX_PRIO;
I
Ingo Molnar 已提交
5411 5412
		__setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
		rq->idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
5413 5414 5415 5416 5417 5418
		migrate_dead_tasks(cpu);
		task_rq_unlock(rq, &flags);
		migrate_nr_uninterruptible(rq);
		BUG_ON(rq->nr_running != 0);

		/* No need to migrate the tasks: it was best-effort if
5419
		 * they didn't take sched_hotcpu_mutex.  Just wake up
L
Linus Torvalds 已提交
5420 5421 5422
		 * the requestors. */
		spin_lock_irq(&rq->lock);
		while (!list_empty(&rq->migration_queue)) {
5423 5424
			struct migration_req *req;

L
Linus Torvalds 已提交
5425
			req = list_entry(rq->migration_queue.next,
5426
					 struct migration_req, list);
L
Linus Torvalds 已提交
5427 5428 5429 5430 5431 5432
			list_del_init(&req->list);
			complete(&req->done);
		}
		spin_unlock_irq(&rq->lock);
		break;
#endif
5433 5434 5435
	case CPU_LOCK_RELEASE:
		mutex_unlock(&sched_hotcpu_mutex);
		break;
L
Linus Torvalds 已提交
5436 5437 5438 5439 5440 5441 5442
	}
	return NOTIFY_OK;
}

/* Register at highest priority so that task migration (migrate_all_tasks)
 * happens before everything else.
 */
5443
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
5444 5445 5446 5447 5448 5449 5450
	.notifier_call = migration_call,
	.priority = 10
};

int __init migration_init(void)
{
	void *cpu = (void *)(long)smp_processor_id();
5451
	int err;
5452 5453

	/* Start one for the boot CPU: */
5454 5455
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
5456 5457
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
5458

L
Linus Torvalds 已提交
5459 5460 5461 5462 5463
	return 0;
}
#endif

#ifdef CONFIG_SMP
5464 5465 5466 5467 5468

/* Number of possible processor ids */
int nr_cpu_ids __read_mostly = NR_CPUS;
EXPORT_SYMBOL(nr_cpu_ids);

5469
#undef SCHED_DOMAIN_DEBUG
L
Linus Torvalds 已提交
5470 5471 5472 5473 5474
#ifdef SCHED_DOMAIN_DEBUG
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;

N
Nick Piggin 已提交
5475 5476 5477 5478 5479
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}

L
Linus Torvalds 已提交
5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	do {
		int i;
		char str[NR_CPUS];
		struct sched_group *group = sd->groups;
		cpumask_t groupmask;

		cpumask_scnprintf(str, NR_CPUS, sd->span);
		cpus_clear(groupmask);

		printk(KERN_DEBUG);
		for (i = 0; i < level + 1; i++)
			printk(" ");
		printk("domain %d: ", level);

		if (!(sd->flags & SD_LOAD_BALANCE)) {
			printk("does not load-balance\n");
			if (sd->parent)
5499 5500
				printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
						" has parent");
L
Linus Torvalds 已提交
5501 5502 5503 5504 5505 5506
			break;
		}

		printk("span %s\n", str);

		if (!cpu_isset(cpu, sd->span))
5507 5508
			printk(KERN_ERR "ERROR: domain->span does not contain "
					"CPU%d\n", cpu);
L
Linus Torvalds 已提交
5509
		if (!cpu_isset(cpu, group->cpumask))
5510 5511
			printk(KERN_ERR "ERROR: domain->groups does not contain"
					" CPU%d\n", cpu);
L
Linus Torvalds 已提交
5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523

		printk(KERN_DEBUG);
		for (i = 0; i < level + 2; i++)
			printk(" ");
		printk("groups:");
		do {
			if (!group) {
				printk("\n");
				printk(KERN_ERR "ERROR: group is NULL\n");
				break;
			}

5524
			if (!group->__cpu_power) {
L
Linus Torvalds 已提交
5525
				printk("\n");
5526 5527
				printk(KERN_ERR "ERROR: domain->cpu_power not "
						"set\n");
L
Linus Torvalds 已提交
5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549
			}

			if (!cpus_weight(group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: empty group\n");
			}

			if (cpus_intersects(groupmask, group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: repeated CPUs\n");
			}

			cpus_or(groupmask, groupmask, group->cpumask);

			cpumask_scnprintf(str, NR_CPUS, group->cpumask);
			printk(" %s", str);

			group = group->next;
		} while (group != sd->groups);
		printk("\n");

		if (!cpus_equal(sd->span, groupmask))
5550 5551
			printk(KERN_ERR "ERROR: groups don't span "
					"domain->span\n");
L
Linus Torvalds 已提交
5552 5553 5554

		level++;
		sd = sd->parent;
5555 5556
		if (!sd)
			continue;
L
Linus Torvalds 已提交
5557

5558 5559 5560
		if (!cpus_subset(groupmask, sd->span))
			printk(KERN_ERR "ERROR: parent span is not a superset "
				"of domain->span\n");
L
Linus Torvalds 已提交
5561 5562 5563 5564

	} while (sd);
}
#else
5565
# define sched_domain_debug(sd, cpu) do { } while (0)
L
Linus Torvalds 已提交
5566 5567
#endif

5568
static int sd_degenerate(struct sched_domain *sd)
5569 5570 5571 5572 5573 5574 5575 5576
{
	if (cpus_weight(sd->span) == 1)
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
5577 5578 5579
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
	if (sd->flags & (SD_WAKE_IDLE |
			 SD_WAKE_AFFINE |
			 SD_WAKE_BALANCE))
		return 0;

	return 1;
}

5593 5594
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

	if (!cpus_equal(sd->span, parent->span))
		return 0;

	/* Does parent contain flags not in child? */
	/* WAKE_BALANCE is a subset of WAKE_AFFINE */
	if (cflags & SD_WAKE_AFFINE)
		pflags &= ~SD_WAKE_BALANCE;
	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
5613 5614 5615
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
5616 5617 5618 5619 5620 5621 5622
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

L
Linus Torvalds 已提交
5623 5624 5625 5626
/*
 * Attach the domain 'sd' to 'cpu' as its base domain.  Callers must
 * hold the hotplug lock.
 */
5627
static void cpu_attach_domain(struct sched_domain *sd, int cpu)
L
Linus Torvalds 已提交
5628
{
5629
	struct rq *rq = cpu_rq(cpu);
5630 5631 5632 5633 5634 5635 5636
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
	for (tmp = sd; tmp; tmp = tmp->parent) {
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
5637
		if (sd_parent_degenerate(tmp, parent)) {
5638
			tmp->parent = parent->parent;
5639 5640 5641
			if (parent->parent)
				parent->parent->child = tmp;
		}
5642 5643
	}

5644
	if (sd && sd_degenerate(sd)) {
5645
		sd = sd->parent;
5646 5647 5648
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
5649 5650 5651

	sched_domain_debug(sd, cpu);

N
Nick Piggin 已提交
5652
	rcu_assign_pointer(rq->sd, sd);
L
Linus Torvalds 已提交
5653 5654 5655
}

/* cpus with isolated domains */
5656
static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
L
Linus Torvalds 已提交
5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
	int ints[NR_CPUS], i;

	str = get_options(str, ARRAY_SIZE(ints), ints);
	cpus_clear(cpu_isolated_map);
	for (i = 1; i <= ints[0]; i++)
		if (ints[i] < NR_CPUS)
			cpu_set(ints[i], cpu_isolated_map);
	return 1;
}

__setup ("isolcpus=", isolated_cpu_setup);

/*
5674 5675 5676 5677
 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
 * to a function which identifies what group(along with sched group) a CPU
 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS
 * (due to the fact that we keep track of groups covered with a cpumask_t).
L
Linus Torvalds 已提交
5678 5679 5680 5681 5682
 *
 * init_sched_build_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
 */
5683
static void
5684 5685 5686
init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map,
			int (*group_fn)(int cpu, const cpumask_t *cpu_map,
					struct sched_group **sg))
L
Linus Torvalds 已提交
5687 5688 5689 5690 5691 5692
{
	struct sched_group *first = NULL, *last = NULL;
	cpumask_t covered = CPU_MASK_NONE;
	int i;

	for_each_cpu_mask(i, span) {
5693 5694
		struct sched_group *sg;
		int group = group_fn(i, cpu_map, &sg);
L
Linus Torvalds 已提交
5695 5696 5697 5698 5699 5700
		int j;

		if (cpu_isset(i, covered))
			continue;

		sg->cpumask = CPU_MASK_NONE;
5701
		sg->__cpu_power = 0;
L
Linus Torvalds 已提交
5702 5703

		for_each_cpu_mask(j, span) {
5704
			if (group_fn(j, cpu_map, NULL) != group)
L
Linus Torvalds 已提交
5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718
				continue;

			cpu_set(j, covered);
			cpu_set(j, sg->cpumask);
		}
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
}

5719
#define SD_NODES_PER_DOMAIN 16
L
Linus Torvalds 已提交
5720

5721
#ifdef CONFIG_NUMA
5722

5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774
/**
 * find_next_best_node - find the next node to include in a sched_domain
 * @node: node whose sched_domain we're building
 * @used_nodes: nodes already in the sched_domain
 *
 * Find the next node to include in a given scheduling domain.  Simply
 * finds the closest node not already in the @used_nodes map.
 *
 * Should use nodemask_t.
 */
static int find_next_best_node(int node, unsigned long *used_nodes)
{
	int i, n, val, min_val, best_node = 0;

	min_val = INT_MAX;

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Start at @node */
		n = (node + i) % MAX_NUMNODES;

		if (!nr_cpus_node(n))
			continue;

		/* Skip already used nodes */
		if (test_bit(n, used_nodes))
			continue;

		/* Simple min distance search */
		val = node_distance(node, n);

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

	set_bit(best_node, used_nodes);
	return best_node;
}

/**
 * sched_domain_node_span - get a cpumask for a node's sched_domain
 * @node: node whose cpumask we're constructing
 * @size: number of nodes to include in this span
 *
 * Given a node, construct a good cpumask for its sched_domain to span.  It
 * should be one that prevents unnecessary balancing, but also spreads tasks
 * out optimally.
 */
static cpumask_t sched_domain_node_span(int node)
{
	DECLARE_BITMAP(used_nodes, MAX_NUMNODES);
5775 5776
	cpumask_t span, nodemask;
	int i;
5777 5778 5779 5780 5781 5782 5783 5784 5785 5786

	cpus_clear(span);
	bitmap_zero(used_nodes, MAX_NUMNODES);

	nodemask = node_to_cpumask(node);
	cpus_or(span, span, nodemask);
	set_bit(node, used_nodes);

	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
		int next_node = find_next_best_node(node, used_nodes);
5787

5788 5789 5790 5791 5792 5793 5794 5795
		nodemask = node_to_cpumask(next_node);
		cpus_or(span, span, nodemask);
	}

	return span;
}
#endif

5796
int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
5797

5798
/*
5799
 * SMT sched-domains:
5800
 */
L
Linus Torvalds 已提交
5801 5802
#ifdef CONFIG_SCHED_SMT
static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
5803
static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
5804

5805 5806
static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map,
			    struct sched_group **sg)
L
Linus Torvalds 已提交
5807
{
5808 5809
	if (sg)
		*sg = &per_cpu(sched_group_cpus, cpu);
L
Linus Torvalds 已提交
5810 5811 5812 5813
	return cpu;
}
#endif

5814 5815 5816
/*
 * multi-core sched-domains:
 */
5817 5818
#ifdef CONFIG_SCHED_MC
static DEFINE_PER_CPU(struct sched_domain, core_domains);
5819
static DEFINE_PER_CPU(struct sched_group, sched_group_core);
5820 5821 5822
#endif

#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
5823 5824
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5825
{
5826
	int group;
5827 5828
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5829 5830 5831 5832
	group = first_cpu(mask);
	if (sg)
		*sg = &per_cpu(sched_group_core, group);
	return group;
5833 5834
}
#elif defined(CONFIG_SCHED_MC)
5835 5836
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5837
{
5838 5839
	if (sg)
		*sg = &per_cpu(sched_group_core, cpu);
5840 5841 5842 5843
	return cpu;
}
#endif

L
Linus Torvalds 已提交
5844
static DEFINE_PER_CPU(struct sched_domain, phys_domains);
5845
static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
5846

5847 5848
static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
L
Linus Torvalds 已提交
5849
{
5850
	int group;
5851
#ifdef CONFIG_SCHED_MC
5852
	cpumask_t mask = cpu_coregroup_map(cpu);
5853
	cpus_and(mask, mask, *cpu_map);
5854
	group = first_cpu(mask);
5855
#elif defined(CONFIG_SCHED_SMT)
5856 5857
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5858
	group = first_cpu(mask);
L
Linus Torvalds 已提交
5859
#else
5860
	group = cpu;
L
Linus Torvalds 已提交
5861
#endif
5862 5863 5864
	if (sg)
		*sg = &per_cpu(sched_group_phys, group);
	return group;
L
Linus Torvalds 已提交
5865 5866 5867 5868
}

#ifdef CONFIG_NUMA
/*
5869 5870 5871
 * The init_sched_build_groups can't handle what we want to do with node
 * groups, so roll our own. Now each node has its own list of groups which
 * gets dynamically allocated.
L
Linus Torvalds 已提交
5872
 */
5873
static DEFINE_PER_CPU(struct sched_domain, node_domains);
5874
static struct sched_group **sched_group_nodes_bycpu[NR_CPUS];
L
Linus Torvalds 已提交
5875

5876
static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
5877
static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
5878

5879 5880
static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
				 struct sched_group **sg)
5881
{
5882 5883 5884 5885 5886 5887 5888 5889 5890
	cpumask_t nodemask = node_to_cpumask(cpu_to_node(cpu));
	int group;

	cpus_and(nodemask, nodemask, *cpu_map);
	group = first_cpu(nodemask);

	if (sg)
		*sg = &per_cpu(sched_group_allnodes, group);
	return group;
L
Linus Torvalds 已提交
5891
}
5892

5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912
static void init_numa_sched_groups_power(struct sched_group *group_head)
{
	struct sched_group *sg = group_head;
	int j;

	if (!sg)
		return;
next_sg:
	for_each_cpu_mask(j, sg->cpumask) {
		struct sched_domain *sd;

		sd = &per_cpu(phys_domains, j);
		if (j != first_cpu(sd->groups->cpumask)) {
			/*
			 * Only add "power" once for each
			 * physical package.
			 */
			continue;
		}

5913
		sg_inc_cpu_power(sg, sd->groups->__cpu_power);
5914 5915 5916 5917 5918
	}
	sg = sg->next;
	if (sg != group_head)
		goto next_sg;
}
L
Linus Torvalds 已提交
5919 5920
#endif

5921
#ifdef CONFIG_NUMA
5922 5923 5924
/* Free memory allocated for various sched_group structures */
static void free_sched_groups(const cpumask_t *cpu_map)
{
5925
	int cpu, i;
5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955

	for_each_cpu_mask(cpu, *cpu_map) {
		struct sched_group **sched_group_nodes
			= sched_group_nodes_bycpu[cpu];

		if (!sched_group_nodes)
			continue;

		for (i = 0; i < MAX_NUMNODES; i++) {
			cpumask_t nodemask = node_to_cpumask(i);
			struct sched_group *oldsg, *sg = sched_group_nodes[i];

			cpus_and(nodemask, nodemask, *cpu_map);
			if (cpus_empty(nodemask))
				continue;

			if (sg == NULL)
				continue;
			sg = sg->next;
next_sg:
			oldsg = sg;
			sg = sg->next;
			kfree(oldsg);
			if (oldsg != sched_group_nodes[i])
				goto next_sg;
		}
		kfree(sched_group_nodes);
		sched_group_nodes_bycpu[cpu] = NULL;
	}
}
5956 5957 5958 5959 5960
#else
static void free_sched_groups(const cpumask_t *cpu_map)
{
}
#endif
5961

5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 *
 * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
 * the maximum number of tasks a group can handle in the presence of other idle
 * or lightly loaded groups in the same sched domain.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
	struct sched_domain *child;
	struct sched_group *group;

	WARN_ON(!sd || !sd->groups);

	if (cpu != first_cpu(sd->groups->cpumask))
		return;

	child = sd->child;

5988 5989
	sd->groups->__cpu_power = 0;

5990 5991 5992 5993 5994 5995 5996 5997 5998 5999
	/*
	 * For perf policy, if the groups in child domain share resources
	 * (for example cores sharing some portions of the cache hierarchy
	 * or SMT), then set this domain groups cpu_power such that each group
	 * can handle only one task, when there are other idle groups in the
	 * same sched domain.
	 */
	if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
		       (child->flags &
			(SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
6000
		sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
6001 6002 6003 6004 6005 6006 6007 6008
		return;
	}

	/*
	 * add cpu_power of each child group to this groups cpu_power
	 */
	group = child->groups;
	do {
6009
		sg_inc_cpu_power(sd->groups, group->__cpu_power);
6010 6011 6012 6013
		group = group->next;
	} while (group != child->groups);
}

L
Linus Torvalds 已提交
6014
/*
6015 6016
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
L
Linus Torvalds 已提交
6017
 */
6018
static int build_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6019 6020
{
	int i;
6021 6022
#ifdef CONFIG_NUMA
	struct sched_group **sched_group_nodes = NULL;
6023
	int sd_allnodes = 0;
6024 6025 6026 6027

	/*
	 * Allocate the per-node list of sched groups
	 */
I
Ingo Molnar 已提交
6028
	sched_group_nodes = kzalloc(sizeof(struct sched_group *)*MAX_NUMNODES,
6029
					   GFP_KERNEL);
6030 6031
	if (!sched_group_nodes) {
		printk(KERN_WARNING "Can not alloc sched group node list\n");
6032
		return -ENOMEM;
6033 6034 6035
	}
	sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
#endif
L
Linus Torvalds 已提交
6036 6037

	/*
6038
	 * Set up domains for cpus specified by the cpu_map.
L
Linus Torvalds 已提交
6039
	 */
6040
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6041 6042 6043
		struct sched_domain *sd = NULL, *p;
		cpumask_t nodemask = node_to_cpumask(cpu_to_node(i));

6044
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6045 6046

#ifdef CONFIG_NUMA
I
Ingo Molnar 已提交
6047 6048
		if (cpus_weight(*cpu_map) >
				SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) {
6049 6050 6051
			sd = &per_cpu(allnodes_domains, i);
			*sd = SD_ALLNODES_INIT;
			sd->span = *cpu_map;
6052
			cpu_to_allnodes_group(i, cpu_map, &sd->groups);
6053
			p = sd;
6054
			sd_allnodes = 1;
6055 6056 6057
		} else
			p = NULL;

L
Linus Torvalds 已提交
6058 6059
		sd = &per_cpu(node_domains, i);
		*sd = SD_NODE_INIT;
6060 6061
		sd->span = sched_domain_node_span(cpu_to_node(i));
		sd->parent = p;
6062 6063
		if (p)
			p->child = sd;
6064
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6065 6066 6067 6068 6069 6070 6071
#endif

		p = sd;
		sd = &per_cpu(phys_domains, i);
		*sd = SD_CPU_INIT;
		sd->span = nodemask;
		sd->parent = p;
6072 6073
		if (p)
			p->child = sd;
6074
		cpu_to_phys_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6075

6076 6077 6078 6079 6080 6081 6082
#ifdef CONFIG_SCHED_MC
		p = sd;
		sd = &per_cpu(core_domains, i);
		*sd = SD_MC_INIT;
		sd->span = cpu_coregroup_map(i);
		cpus_and(sd->span, sd->span, *cpu_map);
		sd->parent = p;
6083
		p->child = sd;
6084
		cpu_to_core_group(i, cpu_map, &sd->groups);
6085 6086
#endif

L
Linus Torvalds 已提交
6087 6088 6089 6090 6091
#ifdef CONFIG_SCHED_SMT
		p = sd;
		sd = &per_cpu(cpu_domains, i);
		*sd = SD_SIBLING_INIT;
		sd->span = cpu_sibling_map[i];
6092
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6093
		sd->parent = p;
6094
		p->child = sd;
6095
		cpu_to_cpu_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6096 6097 6098 6099 6100
#endif
	}

#ifdef CONFIG_SCHED_SMT
	/* Set up CPU (sibling) groups */
6101
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6102
		cpumask_t this_sibling_map = cpu_sibling_map[i];
6103
		cpus_and(this_sibling_map, this_sibling_map, *cpu_map);
L
Linus Torvalds 已提交
6104 6105 6106
		if (i != first_cpu(this_sibling_map))
			continue;

I
Ingo Molnar 已提交
6107 6108
		init_sched_build_groups(this_sibling_map, cpu_map,
					&cpu_to_cpu_group);
L
Linus Torvalds 已提交
6109 6110 6111
	}
#endif

6112 6113 6114 6115 6116 6117 6118
#ifdef CONFIG_SCHED_MC
	/* Set up multi-core groups */
	for_each_cpu_mask(i, *cpu_map) {
		cpumask_t this_core_map = cpu_coregroup_map(i);
		cpus_and(this_core_map, this_core_map, *cpu_map);
		if (i != first_cpu(this_core_map))
			continue;
I
Ingo Molnar 已提交
6119 6120
		init_sched_build_groups(this_core_map, cpu_map,
					&cpu_to_core_group);
6121 6122 6123
	}
#endif

L
Linus Torvalds 已提交
6124 6125 6126 6127
	/* Set up physical groups */
	for (i = 0; i < MAX_NUMNODES; i++) {
		cpumask_t nodemask = node_to_cpumask(i);

6128
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6129 6130 6131
		if (cpus_empty(nodemask))
			continue;

6132
		init_sched_build_groups(nodemask, cpu_map, &cpu_to_phys_group);
L
Linus Torvalds 已提交
6133 6134 6135 6136
	}

#ifdef CONFIG_NUMA
	/* Set up node groups */
6137
	if (sd_allnodes)
I
Ingo Molnar 已提交
6138 6139
		init_sched_build_groups(*cpu_map, cpu_map,
					&cpu_to_allnodes_group);
6140 6141 6142 6143 6144 6145 6146 6147 6148 6149

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Set up node groups */
		struct sched_group *sg, *prev;
		cpumask_t nodemask = node_to_cpumask(i);
		cpumask_t domainspan;
		cpumask_t covered = CPU_MASK_NONE;
		int j;

		cpus_and(nodemask, nodemask, *cpu_map);
6150 6151
		if (cpus_empty(nodemask)) {
			sched_group_nodes[i] = NULL;
6152
			continue;
6153
		}
6154 6155 6156 6157

		domainspan = sched_domain_node_span(i);
		cpus_and(domainspan, domainspan, *cpu_map);

6158
		sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);
6159 6160 6161 6162 6163
		if (!sg) {
			printk(KERN_WARNING "Can not alloc domain group for "
				"node %d\n", i);
			goto error;
		}
6164 6165 6166
		sched_group_nodes[i] = sg;
		for_each_cpu_mask(j, nodemask) {
			struct sched_domain *sd;
I
Ingo Molnar 已提交
6167

6168 6169 6170
			sd = &per_cpu(node_domains, j);
			sd->groups = sg;
		}
6171
		sg->__cpu_power = 0;
6172
		sg->cpumask = nodemask;
6173
		sg->next = sg;
6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191
		cpus_or(covered, covered, nodemask);
		prev = sg;

		for (j = 0; j < MAX_NUMNODES; j++) {
			cpumask_t tmp, notcovered;
			int n = (i + j) % MAX_NUMNODES;

			cpus_complement(notcovered, covered);
			cpus_and(tmp, notcovered, *cpu_map);
			cpus_and(tmp, tmp, domainspan);
			if (cpus_empty(tmp))
				break;

			nodemask = node_to_cpumask(n);
			cpus_and(tmp, tmp, nodemask);
			if (cpus_empty(tmp))
				continue;

6192 6193
			sg = kmalloc_node(sizeof(struct sched_group),
					  GFP_KERNEL, i);
6194 6195 6196
			if (!sg) {
				printk(KERN_WARNING
				"Can not alloc domain group for node %d\n", j);
6197
				goto error;
6198
			}
6199
			sg->__cpu_power = 0;
6200
			sg->cpumask = tmp;
6201
			sg->next = prev->next;
6202 6203 6204 6205 6206
			cpus_or(covered, covered, tmp);
			prev->next = sg;
			prev = sg;
		}
	}
L
Linus Torvalds 已提交
6207 6208 6209
#endif

	/* Calculate CPU power for physical packages and nodes */
6210
#ifdef CONFIG_SCHED_SMT
6211
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6212 6213
		struct sched_domain *sd = &per_cpu(cpu_domains, i);

6214
		init_sched_groups_power(i, sd);
6215
	}
L
Linus Torvalds 已提交
6216
#endif
6217
#ifdef CONFIG_SCHED_MC
6218
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6219 6220
		struct sched_domain *sd = &per_cpu(core_domains, i);

6221
		init_sched_groups_power(i, sd);
6222 6223
	}
#endif
6224

6225
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6226 6227
		struct sched_domain *sd = &per_cpu(phys_domains, i);

6228
		init_sched_groups_power(i, sd);
L
Linus Torvalds 已提交
6229 6230
	}

6231
#ifdef CONFIG_NUMA
6232 6233
	for (i = 0; i < MAX_NUMNODES; i++)
		init_numa_sched_groups_power(sched_group_nodes[i]);
6234

6235 6236
	if (sd_allnodes) {
		struct sched_group *sg;
6237

6238
		cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg);
6239 6240
		init_numa_sched_groups_power(sg);
	}
6241 6242
#endif

L
Linus Torvalds 已提交
6243
	/* Attach the domains */
6244
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6245 6246 6247
		struct sched_domain *sd;
#ifdef CONFIG_SCHED_SMT
		sd = &per_cpu(cpu_domains, i);
6248 6249
#elif defined(CONFIG_SCHED_MC)
		sd = &per_cpu(core_domains, i);
L
Linus Torvalds 已提交
6250 6251 6252 6253 6254
#else
		sd = &per_cpu(phys_domains, i);
#endif
		cpu_attach_domain(sd, i);
	}
6255 6256 6257

	return 0;

6258
#ifdef CONFIG_NUMA
6259 6260 6261
error:
	free_sched_groups(cpu_map);
	return -ENOMEM;
6262
#endif
L
Linus Torvalds 已提交
6263
}
6264 6265 6266
/*
 * Set up scheduler domains and groups.  Callers must hold the hotplug lock.
 */
6267
static int arch_init_sched_domains(const cpumask_t *cpu_map)
6268 6269
{
	cpumask_t cpu_default_map;
6270
	int err;
L
Linus Torvalds 已提交
6271

6272 6273 6274 6275 6276 6277 6278
	/*
	 * Setup mask for cpus without special case scheduling requirements.
	 * For now this just excludes isolated cpus, but could be used to
	 * exclude other special cases in the future.
	 */
	cpus_andnot(cpu_default_map, *cpu_map, cpu_isolated_map);

6279 6280 6281
	err = build_sched_domains(&cpu_default_map);

	return err;
6282 6283 6284
}

static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6285
{
6286
	free_sched_groups(cpu_map);
6287
}
L
Linus Torvalds 已提交
6288

6289 6290 6291 6292
/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
6293
static void detach_destroy_domains(const cpumask_t *cpu_map)
6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310
{
	int i;

	for_each_cpu_mask(i, *cpu_map)
		cpu_attach_domain(NULL, i);
	synchronize_sched();
	arch_destroy_sched_domains(cpu_map);
}

/*
 * Partition sched domains as specified by the cpumasks below.
 * This attaches all cpus from the cpumasks to the NULL domain,
 * waits for a RCU quiescent period, recalculates sched
 * domain information and then attaches them back to the
 * correct sched domains
 * Call with hotplug lock held
 */
6311
int partition_sched_domains(cpumask_t *partition1, cpumask_t *partition2)
6312 6313
{
	cpumask_t change_map;
6314
	int err = 0;
6315 6316 6317 6318 6319 6320 6321 6322

	cpus_and(*partition1, *partition1, cpu_online_map);
	cpus_and(*partition2, *partition2, cpu_online_map);
	cpus_or(change_map, *partition1, *partition2);

	/* Detach sched domains from all of the affected cpus */
	detach_destroy_domains(&change_map);
	if (!cpus_empty(*partition1))
6323 6324 6325 6326 6327
		err = build_sched_domains(partition1);
	if (!err && !cpus_empty(*partition2))
		err = build_sched_domains(partition2);

	return err;
6328 6329
}

6330 6331 6332 6333 6334
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
int arch_reinit_sched_domains(void)
{
	int err;

6335
	mutex_lock(&sched_hotcpu_mutex);
6336 6337
	detach_destroy_domains(&cpu_online_map);
	err = arch_init_sched_domains(&cpu_online_map);
6338
	mutex_unlock(&sched_hotcpu_mutex);
6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362

	return err;
}

static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
{
	int ret;

	if (buf[0] != '0' && buf[0] != '1')
		return -EINVAL;

	if (smt)
		sched_smt_power_savings = (buf[0] == '1');
	else
		sched_mc_power_savings = (buf[0] == '1');

	ret = arch_reinit_sched_domains();

	return ret ? ret : count;
}

int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
{
	int err = 0;
6363

6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382
#ifdef CONFIG_SCHED_SMT
	if (smt_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_smt_power_savings.attr);
#endif
#ifdef CONFIG_SCHED_MC
	if (!err && mc_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_mc_power_savings.attr);
#endif
	return err;
}
#endif

#ifdef CONFIG_SCHED_MC
static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_mc_power_savings);
}
6383 6384
static ssize_t sched_mc_power_savings_store(struct sys_device *dev,
					    const char *buf, size_t count)
6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396
{
	return sched_power_savings_store(buf, count, 0);
}
SYSDEV_ATTR(sched_mc_power_savings, 0644, sched_mc_power_savings_show,
	    sched_mc_power_savings_store);
#endif

#ifdef CONFIG_SCHED_SMT
static ssize_t sched_smt_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_smt_power_savings);
}
6397 6398
static ssize_t sched_smt_power_savings_store(struct sys_device *dev,
					     const char *buf, size_t count)
6399 6400 6401 6402 6403 6404 6405
{
	return sched_power_savings_store(buf, count, 1);
}
SYSDEV_ATTR(sched_smt_power_savings, 0644, sched_smt_power_savings_show,
	    sched_smt_power_savings_store);
#endif

L
Linus Torvalds 已提交
6406 6407 6408
/*
 * Force a reinitialization of the sched domains hierarchy.  The domains
 * and groups cannot be updated in place without racing with the balancing
N
Nick Piggin 已提交
6409
 * code, so we temporarily attach all running cpus to the NULL domain
L
Linus Torvalds 已提交
6410 6411 6412 6413 6414 6415 6416
 * which will prevent rebalancing while the sched domains are recalculated.
 */
static int update_sched_domains(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
{
	switch (action) {
	case CPU_UP_PREPARE:
6417
	case CPU_UP_PREPARE_FROZEN:
L
Linus Torvalds 已提交
6418
	case CPU_DOWN_PREPARE:
6419
	case CPU_DOWN_PREPARE_FROZEN:
6420
		detach_destroy_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6421 6422 6423
		return NOTIFY_OK;

	case CPU_UP_CANCELED:
6424
	case CPU_UP_CANCELED_FROZEN:
L
Linus Torvalds 已提交
6425
	case CPU_DOWN_FAILED:
6426
	case CPU_DOWN_FAILED_FROZEN:
L
Linus Torvalds 已提交
6427
	case CPU_ONLINE:
6428
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
6429
	case CPU_DEAD:
6430
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
6431 6432 6433 6434 6435 6436 6437 6438 6439
		/*
		 * Fall through and re-initialise the domains.
		 */
		break;
	default:
		return NOTIFY_DONE;
	}

	/* The hotplug lock is already held by cpu_up/cpu_down */
6440
	arch_init_sched_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6441 6442 6443 6444 6445 6446

	return NOTIFY_OK;
}

void __init sched_init_smp(void)
{
6447 6448
	cpumask_t non_isolated_cpus;

6449
	mutex_lock(&sched_hotcpu_mutex);
6450
	arch_init_sched_domains(&cpu_online_map);
6451
	cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map);
6452 6453
	if (cpus_empty(non_isolated_cpus))
		cpu_set(smp_processor_id(), non_isolated_cpus);
6454
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
6455 6456
	/* XXX: Theoretical race here - CPU may be hotplugged now */
	hotcpu_notifier(update_sched_domains, 0);
6457

6458 6459
	init_sched_domain_sysctl();

6460 6461 6462
	/* Move init over to a non-isolated CPU */
	if (set_cpus_allowed(current, non_isolated_cpus) < 0)
		BUG();
I
Ingo Molnar 已提交
6463
	sched_init_granularity();
L
Linus Torvalds 已提交
6464 6465 6466 6467
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
6468
	sched_init_granularity();
L
Linus Torvalds 已提交
6469 6470 6471 6472 6473 6474 6475
}
#endif /* CONFIG_SMP */

int in_sched_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __sched functions */
	extern char __sched_text_start[], __sched_text_end[];
6476

L
Linus Torvalds 已提交
6477 6478 6479 6480 6481
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

I
Ingo Molnar 已提交
6482 6483 6484 6485 6486 6487 6488 6489 6490
static inline void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
{
	cfs_rq->tasks_timeline = RB_ROOT;
	cfs_rq->fair_clock = 1;
#ifdef CONFIG_FAIR_GROUP_SCHED
	cfs_rq->rq = rq;
#endif
}

L
Linus Torvalds 已提交
6491 6492
void __init sched_init(void)
{
I
Ingo Molnar 已提交
6493
	u64 now = sched_clock();
6494
	int highest_cpu = 0;
I
Ingo Molnar 已提交
6495 6496 6497 6498 6499 6500 6501 6502
	int i, j;

	/*
	 * Link up the scheduling class hierarchy:
	 */
	rt_sched_class.next = &fair_sched_class;
	fair_sched_class.next = &idle_sched_class;
	idle_sched_class.next = NULL;
L
Linus Torvalds 已提交
6503

6504
	for_each_possible_cpu(i) {
I
Ingo Molnar 已提交
6505
		struct rt_prio_array *array;
6506
		struct rq *rq;
L
Linus Torvalds 已提交
6507 6508 6509

		rq = cpu_rq(i);
		spin_lock_init(&rq->lock);
6510
		lockdep_set_class(&rq->lock, &rq->rq_lock_key);
N
Nick Piggin 已提交
6511
		rq->nr_running = 0;
I
Ingo Molnar 已提交
6512 6513 6514 6515 6516 6517 6518 6519
		rq->clock = 1;
		init_cfs_rq(&rq->cfs, rq);
#ifdef CONFIG_FAIR_GROUP_SCHED
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
		list_add(&rq->cfs.leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
#endif
		rq->ls.load_update_last = now;
		rq->ls.load_update_start = now;
L
Linus Torvalds 已提交
6520

I
Ingo Molnar 已提交
6521 6522
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
L
Linus Torvalds 已提交
6523
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
6524
		rq->sd = NULL;
L
Linus Torvalds 已提交
6525
		rq->active_balance = 0;
I
Ingo Molnar 已提交
6526
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
6527
		rq->push_cpu = 0;
6528
		rq->cpu = i;
L
Linus Torvalds 已提交
6529 6530 6531 6532 6533
		rq->migration_thread = NULL;
		INIT_LIST_HEAD(&rq->migration_queue);
#endif
		atomic_set(&rq->nr_iowait, 0);

I
Ingo Molnar 已提交
6534 6535 6536 6537
		array = &rq->rt.active;
		for (j = 0; j < MAX_RT_PRIO; j++) {
			INIT_LIST_HEAD(array->queue + j);
			__clear_bit(j, array->bitmap);
L
Linus Torvalds 已提交
6538
		}
6539
		highest_cpu = i;
I
Ingo Molnar 已提交
6540 6541
		/* delimiter for bitsearch: */
		__set_bit(MAX_RT_PRIO, array->bitmap);
L
Linus Torvalds 已提交
6542 6543
	}

6544
	set_load_weight(&init_task);
6545

6546 6547 6548 6549
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

6550
#ifdef CONFIG_SMP
6551
	nr_cpu_ids = highest_cpu + 1;
6552 6553 6554
	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
#endif

6555 6556 6557 6558
#ifdef CONFIG_RT_MUTEXES
	plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
#endif

L
Linus Torvalds 已提交
6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
I
Ingo Molnar 已提交
6572 6573 6574 6575
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
L
Linus Torvalds 已提交
6576 6577 6578 6579 6580
}

#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
void __might_sleep(char *file, int line)
{
6581
#ifdef in_atomic
L
Linus Torvalds 已提交
6582 6583 6584 6585 6586 6587 6588
	static unsigned long prev_jiffy;	/* ratelimiting */

	if ((in_atomic() || irqs_disabled()) &&
	    system_state == SYSTEM_RUNNING && !oops_in_progress) {
		if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
			return;
		prev_jiffy = jiffies;
6589
		printk(KERN_ERR "BUG: sleeping function called from invalid"
L
Linus Torvalds 已提交
6590 6591 6592
				" context at %s:%d\n", file, line);
		printk("in_atomic():%d, irqs_disabled():%d\n",
			in_atomic(), irqs_disabled());
6593
		debug_show_held_locks(current);
6594 6595
		if (irqs_disabled())
			print_irqtrace_events(current);
L
Linus Torvalds 已提交
6596 6597 6598 6599 6600 6601 6602 6603 6604 6605
		dump_stack();
	}
#endif
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
void normalize_rt_tasks(void)
{
6606
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
6607
	unsigned long flags;
6608
	struct rq *rq;
I
Ingo Molnar 已提交
6609
	int on_rq;
L
Linus Torvalds 已提交
6610 6611

	read_lock_irq(&tasklist_lock);
6612
	do_each_thread(g, p) {
I
Ingo Molnar 已提交
6613 6614
		p->se.fair_key			= 0;
		p->se.wait_runtime		= 0;
I
Ingo Molnar 已提交
6615
		p->se.exec_start		= 0;
I
Ingo Molnar 已提交
6616
		p->se.wait_start_fair		= 0;
I
Ingo Molnar 已提交
6617 6618
		p->se.sleep_start_fair		= 0;
#ifdef CONFIG_SCHEDSTATS
I
Ingo Molnar 已提交
6619 6620 6621
		p->se.wait_start		= 0;
		p->se.sleep_start		= 0;
		p->se.block_start		= 0;
I
Ingo Molnar 已提交
6622
#endif
I
Ingo Molnar 已提交
6623 6624 6625 6626 6627 6628 6629 6630 6631 6632
		task_rq(p)->cfs.fair_clock	= 0;
		task_rq(p)->clock		= 0;

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
6633
			continue;
I
Ingo Molnar 已提交
6634
		}
L
Linus Torvalds 已提交
6635

6636 6637
		spin_lock_irqsave(&p->pi_lock, flags);
		rq = __task_rq_lock(p);
I
Ingo Molnar 已提交
6638 6639 6640 6641 6642 6643 6644
#ifdef CONFIG_SMP
		/*
		 * Do not touch the migration thread:
		 */
		if (p == rq->migration_thread)
			goto out_unlock;
#endif
L
Linus Torvalds 已提交
6645

I
Ingo Molnar 已提交
6646
		on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
6647 6648 6649 6650
		if (on_rq) {
			update_rq_clock(task_rq(p));
			deactivate_task(task_rq(p), p, 0, task_rq(p)->clock);
		}
I
Ingo Molnar 已提交
6651 6652 6653
		__setscheduler(rq, p, SCHED_NORMAL, 0);
		if (on_rq) {
			activate_task(task_rq(p), p, 0);
L
Linus Torvalds 已提交
6654 6655
			resched_task(rq->curr);
		}
I
Ingo Molnar 已提交
6656 6657 6658
#ifdef CONFIG_SMP
 out_unlock:
#endif
6659 6660
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
6661 6662
	} while_each_thread(g, p);

L
Linus Torvalds 已提交
6663 6664 6665 6666
	read_unlock_irq(&tasklist_lock);
}

#endif /* CONFIG_MAGIC_SYSRQ */
6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684

#ifdef CONFIG_IA64
/*
 * These functions are only useful for the IA64 MCA handling.
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6685
struct task_struct *curr_task(int cpu)
6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704
{
	return cpu_curr(cpu);
}

/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
 * are serviced on a separate stack.  It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner.  This function
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6705
void set_curr_task(int cpu, struct task_struct *p)
6706 6707 6708 6709 6710
{
	cpu_curr(cpu) = p;
}

#endif