sched.c 165.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
L
Linus Torvalds 已提交
25 26 27 28 29 30
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
31
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
32 33 34 35
#include <linux/highmem.h>
#include <linux/smp_lock.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
36
#include <linux/capability.h>
L
Linus Torvalds 已提交
37 38
#include <linux/completion.h>
#include <linux/kernel_stat.h>
39
#include <linux/debug_locks.h>
L
Linus Torvalds 已提交
40 41 42
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
43
#include <linux/freezer.h>
44
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
45 46 47 48 49 50 51 52 53 54 55
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
#include <linux/seq_file.h>
56
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
57 58
#include <linux/syscalls.h>
#include <linux/times.h>
59
#include <linux/tsacct_kern.h>
60
#include <linux/kprobes.h>
61
#include <linux/delayacct.h>
62
#include <linux/reciprocal_div.h>
63
#include <linux/unistd.h>
L
Linus Torvalds 已提交
64

65
#include <asm/tlb.h>
L
Linus Torvalds 已提交
66

67 68 69 70 71 72 73 74 75 76
/*
 * Scheduler clock - returns current time in nanosec units.
 * This is default implementation.
 * Architectures and sub-architectures can override this.
 */
unsigned long long __attribute__((weak)) sched_clock(void)
{
	return (unsigned long long)jiffies * (1000000000 / HZ);
}

L
Linus Torvalds 已提交
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
 * Some helpers for converting nanosecond timing to jiffy resolution
 */
#define NS_TO_JIFFIES(TIME)	((TIME) / (1000000000 / HZ))
#define JIFFIES_TO_NS(TIME)	((TIME) * (1000000000 / HZ))

I
Ingo Molnar 已提交
101 102 103
#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

L
Linus Torvalds 已提交
104 105 106 107 108 109 110 111 112
/*
 * These are the 'tuning knobs' of the scheduler:
 *
 * Minimum timeslice is 5 msecs (or 1 jiffy, whichever is larger),
 * default timeslice is 100 msecs, maximum timeslice is 800 msecs.
 * Timeslices get refilled after they expire.
 */
#define MIN_TIMESLICE		max(5 * HZ / 1000, 1)
#define DEF_TIMESLICE		(100 * HZ / 1000)
113

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
#ifdef CONFIG_SMP
/*
 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 */
static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
{
	return reciprocal_divide(load, sg->reciprocal_cpu_power);
}

/*
 * Each time a sched group cpu_power is changed,
 * we must compute its reciprocal value
 */
static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
{
	sg->__cpu_power += val;
	sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
}
#endif

I
Ingo Molnar 已提交
135 136 137
#define SCALE_PRIO(x, prio) \
	max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_TIMESLICE)

138
/*
I
Ingo Molnar 已提交
139
 * static_prio_timeslice() scales user-nice values [ -20 ... 0 ... 19 ]
140 141
 * to time slice values: [800ms ... 100ms ... 5ms]
 */
I
Ingo Molnar 已提交
142
static unsigned int static_prio_timeslice(int static_prio)
143
{
I
Ingo Molnar 已提交
144 145 146 147 148 149 150
	if (static_prio == NICE_TO_PRIO(19))
		return 1;

	if (static_prio < NICE_TO_PRIO(0))
		return SCALE_PRIO(DEF_TIMESLICE * 4, static_prio);
	else
		return SCALE_PRIO(DEF_TIMESLICE, static_prio);
151 152
}

153 154 155 156 157 158 159 160 161 162 163 164
static inline int rt_policy(int policy)
{
	if (unlikely(policy == SCHED_FIFO) || unlikely(policy == SCHED_RR))
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

L
Linus Torvalds 已提交
165
/*
I
Ingo Molnar 已提交
166
 * This is the priority-queue data structure of the RT scheduling class:
L
Linus Torvalds 已提交
167
 */
I
Ingo Molnar 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

struct load_stat {
	struct load_weight load;
	u64 load_update_start, load_update_last;
	unsigned long delta_fair, delta_exec, delta_stat;
};

/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	s64 fair_clock;
	u64 exec_clock;
	s64 wait_runtime;
	u64 sleeper_bonus;
	unsigned long wait_runtime_overruns, wait_runtime_underruns;

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;
	struct rb_node *rb_load_balance_curr;
#ifdef CONFIG_FAIR_GROUP_SCHED
	/* 'curr' points to currently running entity on this cfs_rq.
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
	struct sched_entity *curr;
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

	/* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
	struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */
#endif
};
L
Linus Torvalds 已提交
210

I
Ingo Molnar 已提交
211 212 213 214 215 216 217
/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
	int rt_load_balance_idx;
	struct list_head *rt_load_balance_head, *rt_load_balance_curr;
};

L
Linus Torvalds 已提交
218 219 220 221 222 223 224
/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
225
struct rq {
I
Ingo Molnar 已提交
226
	spinlock_t lock;	/* runqueue lock */
L
Linus Torvalds 已提交
227 228 229 230 231 232

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
I
Ingo Molnar 已提交
233 234
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
235
	unsigned char idle_at_tick;
236 237 238
#ifdef CONFIG_NO_HZ
	unsigned char in_nohz_recently;
#endif
I
Ingo Molnar 已提交
239 240 241 242 243 244 245
	struct load_stat ls;	/* capture load from *all* tasks on this cpu */
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
#ifdef CONFIG_FAIR_GROUP_SCHED
	struct list_head leaf_cfs_rq_list; /* list of leaf cfs_rq on this cpu */
L
Linus Torvalds 已提交
246
#endif
I
Ingo Molnar 已提交
247
	struct rt_rq  rt;
L
Linus Torvalds 已提交
248 249 250 251 252 253 254 255 256

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

257
	struct task_struct *curr, *idle;
258
	unsigned long next_balance;
L
Linus Torvalds 已提交
259
	struct mm_struct *prev_mm;
I
Ingo Molnar 已提交
260 261 262 263 264 265

	u64 clock, prev_clock_raw;
	s64 clock_max_delta;

	unsigned int clock_warps, clock_overflows;
	unsigned int clock_unstable_events;
266
	u64 tick_timestamp;
I
Ingo Molnar 已提交
267

L
Linus Torvalds 已提交
268 269 270 271 272 273 274 275
	atomic_t nr_iowait;

#ifdef CONFIG_SMP
	struct sched_domain *sd;

	/* For active balancing */
	int active_balance;
	int push_cpu;
276
	int cpu;		/* cpu of this runqueue */
L
Linus Torvalds 已提交
277

278
	struct task_struct *migration_thread;
L
Linus Torvalds 已提交
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
	struct list_head migration_queue;
#endif

#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;

	/* sys_sched_yield() stats */
	unsigned long yld_exp_empty;
	unsigned long yld_act_empty;
	unsigned long yld_both_empty;
	unsigned long yld_cnt;

	/* schedule() stats */
	unsigned long sched_switch;
	unsigned long sched_cnt;
	unsigned long sched_goidle;

	/* try_to_wake_up() stats */
	unsigned long ttwu_cnt;
	unsigned long ttwu_local;
#endif
301
	struct lock_class_key rq_lock_key;
L
Linus Torvalds 已提交
302 303
};

304
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
305
static DEFINE_MUTEX(sched_hotcpu_mutex);
L
Linus Torvalds 已提交
306

I
Ingo Molnar 已提交
307 308 309 310 311
static inline void check_preempt_curr(struct rq *rq, struct task_struct *p)
{
	rq->curr->sched_class->check_preempt_curr(rq, p);
}

312 313 314 315 316 317 318 319 320
static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

I
Ingo Molnar 已提交
321
/*
I
Ingo Molnar 已提交
322 323
 * Update the per-runqueue clock, as finegrained as the platform can give
 * us, but without assuming monotonicity, etc.:
I
Ingo Molnar 已提交
324
 */
I
Ingo Molnar 已提交
325
static void __update_rq_clock(struct rq *rq)
I
Ingo Molnar 已提交
326 327 328 329 330 331
{
	u64 prev_raw = rq->prev_clock_raw;
	u64 now = sched_clock();
	s64 delta = now - prev_raw;
	u64 clock = rq->clock;

I
Ingo Molnar 已提交
332 333 334
#ifdef CONFIG_SCHED_DEBUG
	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
#endif
I
Ingo Molnar 已提交
335 336 337 338 339 340 341 342 343 344
	/*
	 * Protect against sched_clock() occasionally going backwards:
	 */
	if (unlikely(delta < 0)) {
		clock++;
		rq->clock_warps++;
	} else {
		/*
		 * Catch too large forward jumps too:
		 */
345 346 347 348 349
		if (unlikely(clock + delta > rq->tick_timestamp + TICK_NSEC)) {
			if (clock < rq->tick_timestamp + TICK_NSEC)
				clock = rq->tick_timestamp + TICK_NSEC;
			else
				clock++;
I
Ingo Molnar 已提交
350 351 352 353 354 355 356 357 358 359
			rq->clock_overflows++;
		} else {
			if (unlikely(delta > rq->clock_max_delta))
				rq->clock_max_delta = delta;
			clock += delta;
		}
	}

	rq->prev_clock_raw = now;
	rq->clock = clock;
I
Ingo Molnar 已提交
360
}
I
Ingo Molnar 已提交
361

I
Ingo Molnar 已提交
362 363 364 365
static void update_rq_clock(struct rq *rq)
{
	if (likely(smp_processor_id() == cpu_of(rq)))
		__update_rq_clock(rq);
I
Ingo Molnar 已提交
366 367
}

N
Nick Piggin 已提交
368 369
/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
370
 * See detach_destroy_domains: synchronize_sched for details.
N
Nick Piggin 已提交
371 372 373 374
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
375 376
#define for_each_domain(cpu, __sd) \
	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
L
Linus Torvalds 已提交
377 378 379 380 381 382

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

383 384 385 386 387 388 389 390
/*
 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
 * clock constructed from sched_clock():
 */
unsigned long long cpu_clock(int cpu)
{
	unsigned long long now;
	unsigned long flags;
I
Ingo Molnar 已提交
391
	struct rq *rq;
392

393
	local_irq_save(flags);
I
Ingo Molnar 已提交
394 395 396
	rq = cpu_rq(cpu);
	update_rq_clock(rq);
	now = rq->clock;
397
	local_irq_restore(flags);
398 399 400 401

	return now;
}

I
Ingo Molnar 已提交
402 403 404 405 406 407 408 409 410 411 412 413
#ifdef CONFIG_FAIR_GROUP_SCHED
/* Change a task's ->cfs_rq if it moves across CPUs */
static inline void set_task_cfs_rq(struct task_struct *p)
{
	p->se.cfs_rq = &task_rq(p)->cfs;
}
#else
static inline void set_task_cfs_rq(struct task_struct *p)
{
}
#endif

L
Linus Torvalds 已提交
414
#ifndef prepare_arch_switch
415 416 417 418 419 420 421
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

#ifndef __ARCH_WANT_UNLOCKED_CTXSW
422
static inline int task_running(struct rq *rq, struct task_struct *p)
423 424 425 426
{
	return rq->curr == p;
}

427
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
428 429 430
{
}

431
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
432
{
433 434 435 436
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
437 438 439 440 441 442 443
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

444 445 446 447
	spin_unlock_irq(&rq->lock);
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
448
static inline int task_running(struct rq *rq, struct task_struct *p)
449 450 451 452 453 454 455 456
{
#ifdef CONFIG_SMP
	return p->oncpu;
#else
	return rq->curr == p;
#endif
}

457
static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->oncpu = 1;
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	spin_unlock_irq(&rq->lock);
#else
	spin_unlock(&rq->lock);
#endif
}

474
static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
475 476 477 478 479 480 481 482 483 484 485 486
{
#ifdef CONFIG_SMP
	/*
	 * After ->oncpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->oncpu = 0;
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
L
Linus Torvalds 已提交
487
#endif
488 489
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
L
Linus Torvalds 已提交
490

491 492 493 494
/*
 * __task_rq_lock - lock the runqueue a given task resides on.
 * Must be called interrupts disabled.
 */
495
static inline struct rq *__task_rq_lock(struct task_struct *p)
496 497
	__acquires(rq->lock)
{
498
	struct rq *rq;
499 500 501 502 503 504 505 506 507 508 509

repeat_lock_task:
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock(&rq->lock);
		goto repeat_lock_task;
	}
	return rq;
}

L
Linus Torvalds 已提交
510 511 512 513 514
/*
 * task_rq_lock - lock the runqueue a given task resides on and disable
 * interrupts.  Note the ordering: we can safely lookup the task_rq without
 * explicitly disabling preemption.
 */
515
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
516 517
	__acquires(rq->lock)
{
518
	struct rq *rq;
L
Linus Torvalds 已提交
519 520 521 522 523 524 525 526 527 528 529 530

repeat_lock_task:
	local_irq_save(*flags);
	rq = task_rq(p);
	spin_lock(&rq->lock);
	if (unlikely(rq != task_rq(p))) {
		spin_unlock_irqrestore(&rq->lock, *flags);
		goto repeat_lock_task;
	}
	return rq;
}

531
static inline void __task_rq_unlock(struct rq *rq)
532 533 534 535 536
	__releases(rq->lock)
{
	spin_unlock(&rq->lock);
}

537
static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
L
Linus Torvalds 已提交
538 539 540 541 542 543
	__releases(rq->lock)
{
	spin_unlock_irqrestore(&rq->lock, *flags);
}

/*
544
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
545
 */
546
static inline struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
547 548
	__acquires(rq->lock)
{
549
	struct rq *rq;
L
Linus Torvalds 已提交
550 551 552 553 554 555 556 557

	local_irq_disable();
	rq = this_rq();
	spin_lock(&rq->lock);

	return rq;
}

558 559 560 561 562 563 564 565 566 567 568 569 570 571
/*
 * CPU frequency is/was unstable - start new by setting prev_clock_raw:
 */
void sched_clock_unstable_event(void)
{
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(current, &flags);
	rq->prev_clock_raw = sched_clock();
	rq->clock_unstable_events++;
	task_rq_unlock(rq, &flags);
}

I
Ingo Molnar 已提交
572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

static void resched_task(struct task_struct *p)
{
	int cpu;

	assert_spin_locked(&task_rq(p)->lock);

	if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
		return;

	set_tsk_thread_flag(p, TIF_NEED_RESCHED);

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	if (!spin_trylock_irqsave(&rq->lock, flags))
		return;
	resched_task(cpu_curr(cpu));
	spin_unlock_irqrestore(&rq->lock, flags);
}
#else
static inline void resched_task(struct task_struct *p)
{
	assert_spin_locked(&task_rq(p)->lock);
	set_tsk_need_resched(p);
}
#endif

624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
static u64 div64_likely32(u64 divident, unsigned long divisor)
{
#if BITS_PER_LONG == 32
	if (likely(divident <= 0xffffffffULL))
		return (u32)divident / divisor;
	do_div(divident, divisor);

	return divident;
#else
	return divident / divisor;
#endif
}

#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

I
Ingo Molnar 已提交
645 646 647 648 649
/*
 * Shift right and round:
 */
#define RSR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))

650
static unsigned long
651 652 653 654 655 656
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

	if (unlikely(!lw->inv_weight))
I
Ingo Molnar 已提交
657
		lw->inv_weight = (WMULT_CONST - lw->weight/2) / lw->weight + 1;
658 659 660 661 662

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
I
Ingo Molnar 已提交
663 664 665 666 667
	if (unlikely(tmp > WMULT_CONST))
		tmp = RSR(RSR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
			WMULT_SHIFT/2);
	else
		tmp = RSR(tmp * lw->inv_weight, WMULT_SHIFT);
668

669
	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
}

static inline unsigned long
calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
{
	return calc_delta_mine(delta_exec, NICE_0_LOAD, lw);
}

static void update_load_add(struct load_weight *lw, unsigned long inc)
{
	lw->weight += inc;
	lw->inv_weight = 0;
}

static void update_load_sub(struct load_weight *lw, unsigned long dec)
{
	lw->weight -= dec;
	lw->inv_weight = 0;
}

690 691 692 693 694 695 696 697 698
/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
 * scheduling class and "nice" value.  For SCHED_NORMAL tasks this is just a
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

I
Ingo Molnar 已提交
699 700 701 702 703 704 705 706 707 708 709
#define WEIGHT_IDLEPRIO		2
#define WMULT_IDLEPRIO		(1 << 31)

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
710 711 712
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
I
Ingo Molnar 已提交
713 714
 */
static const int prio_to_weight[40] = {
715 716 717 718 719 720 721 722
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,
 /*  10 */       110,        87,        70,        56,        45,
 /*  15 */        36,        29,        23,        18,        15,
I
Ingo Molnar 已提交
723 724
};

725 726 727 728 729 730 731
/*
 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
 *
 * In cases where the weight does not change often, we can use the
 * precalculated inverse to speed up arithmetics by turning divisions
 * into multiplications:
 */
I
Ingo Molnar 已提交
732
static const u32 prio_to_wmult[40] = {
733 734 735 736 737 738 739 740
 /* -20 */     48388,     59856,     76040,     92818,    118348,
 /* -15 */    147320,    184698,    229616,    287308,    360437,
 /* -10 */    449829,    563644,    704093,    875809,   1099582,
 /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
 /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
 /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
 /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
 /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
I
Ingo Molnar 已提交
741
};
742

I
Ingo Molnar 已提交
743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);

/*
 * runqueue iterator, to support SMP load-balancing between different
 * scheduling classes, without having to expose their internal data
 * structures to the load-balancing proper:
 */
struct rq_iterator {
	void *arg;
	struct task_struct *(*start)(void *);
	struct task_struct *(*next)(void *);
};

static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
760
		      int *this_best_prio, struct rq_iterator *iterator);
I
Ingo Molnar 已提交
761 762 763 764 765 766 767 768 769 770 771

#include "sched_stats.h"
#include "sched_rt.c"
#include "sched_fair.c"
#include "sched_idletask.c"
#ifdef CONFIG_SCHED_DEBUG
# include "sched_debug.c"
#endif

#define sched_class_highest (&rt_sched_class)

772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795
static void __update_curr_load(struct rq *rq, struct load_stat *ls)
{
	if (rq->curr != rq->idle && ls->load.weight) {
		ls->delta_exec += ls->delta_stat;
		ls->delta_fair += calc_delta_fair(ls->delta_stat, &ls->load);
		ls->delta_stat = 0;
	}
}

/*
 * Update delta_exec, delta_fair fields for rq.
 *
 * delta_fair clock advances at a rate inversely proportional to
 * total load (rq->ls.load.weight) on the runqueue, while
 * delta_exec advances at the same rate as wall-clock (provided
 * cpu is not idle).
 *
 * delta_exec / delta_fair is a measure of the (smoothened) load on this
 * runqueue over any given interval. This (smoothened) load is used
 * during load balance.
 *
 * This function is called /before/ updating rq->ls.load
 * and when switching tasks.
 */
796
static void update_curr_load(struct rq *rq)
797 798 799 800 801
{
	struct load_stat *ls = &rq->ls;
	u64 start;

	start = ls->load_update_start;
802 803
	ls->load_update_start = rq->clock;
	ls->delta_stat += rq->clock - start;
804 805 806 807 808 809 810 811
	/*
	 * Stagger updates to ls->delta_fair. Very frequent updates
	 * can be expensive.
	 */
	if (ls->delta_stat >= sysctl_sched_stat_granularity)
		__update_curr_load(rq, ls);
}

812
static inline void inc_load(struct rq *rq, const struct task_struct *p)
813
{
814
	update_curr_load(rq);
815 816 817
	update_load_add(&rq->ls.load, p->se.load.weight);
}

818
static inline void dec_load(struct rq *rq, const struct task_struct *p)
819
{
820
	update_curr_load(rq);
821 822 823
	update_load_sub(&rq->ls.load, p->se.load.weight);
}

824
static void inc_nr_running(struct task_struct *p, struct rq *rq)
825 826
{
	rq->nr_running++;
827
	inc_load(rq, p);
828 829
}

830
static void dec_nr_running(struct task_struct *p, struct rq *rq)
831 832
{
	rq->nr_running--;
833
	dec_load(rq, p);
834 835
}

836 837
static void set_load_weight(struct task_struct *p)
{
I
Ingo Molnar 已提交
838 839 840
	task_rq(p)->cfs.wait_runtime -= p->se.wait_runtime;
	p->se.wait_runtime = 0;

841
	if (task_has_rt_policy(p)) {
I
Ingo Molnar 已提交
842 843 844 845
		p->se.load.weight = prio_to_weight[0] * 2;
		p->se.load.inv_weight = prio_to_wmult[0] >> 1;
		return;
	}
846

I
Ingo Molnar 已提交
847 848 849 850 851 852 853 854
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
		p->se.load.weight = WEIGHT_IDLEPRIO;
		p->se.load.inv_weight = WMULT_IDLEPRIO;
		return;
	}
855

I
Ingo Molnar 已提交
856 857
	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
858 859
}

860
static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
861
{
I
Ingo Molnar 已提交
862
	sched_info_queued(p);
863
	p->sched_class->enqueue_task(rq, p, wakeup);
I
Ingo Molnar 已提交
864
	p->se.on_rq = 1;
865 866
}

867
static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
868
{
869
	p->sched_class->dequeue_task(rq, p, sleep);
I
Ingo Molnar 已提交
870
	p->se.on_rq = 0;
871 872
}

873
/*
I
Ingo Molnar 已提交
874
 * __normal_prio - return the priority that is based on the static prio
875 876 877
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
878
	return p->static_prio;
879 880
}

881 882 883 884 885 886 887
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
888
static inline int normal_prio(struct task_struct *p)
889 890 891
{
	int prio;

892
	if (task_has_rt_policy(p))
893 894 895 896 897 898 899 900 901 902 903 904 905
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
906
static int effective_prio(struct task_struct *p)
907 908 909 910 911 912 913 914 915 916 917 918
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
919
/*
I
Ingo Molnar 已提交
920
 * activate_task - move a task to the runqueue.
L
Linus Torvalds 已提交
921
 */
I
Ingo Molnar 已提交
922
static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
L
Linus Torvalds 已提交
923
{
I
Ingo Molnar 已提交
924 925
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;
L
Linus Torvalds 已提交
926

927
	enqueue_task(rq, p, wakeup);
928
	inc_nr_running(p, rq);
L
Linus Torvalds 已提交
929 930 931
}

/*
I
Ingo Molnar 已提交
932
 * activate_idle_task - move idle task to the _front_ of runqueue.
L
Linus Torvalds 已提交
933
 */
I
Ingo Molnar 已提交
934
static inline void activate_idle_task(struct task_struct *p, struct rq *rq)
L
Linus Torvalds 已提交
935
{
I
Ingo Molnar 已提交
936
	update_rq_clock(rq);
L
Linus Torvalds 已提交
937

I
Ingo Molnar 已提交
938 939
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible--;
I
Ingo Molnar 已提交
940

941
	enqueue_task(rq, p, 0);
942
	inc_nr_running(p, rq);
L
Linus Torvalds 已提交
943 944 945 946 947
}

/*
 * deactivate_task - remove a task from the runqueue.
 */
948
static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
L
Linus Torvalds 已提交
949
{
I
Ingo Molnar 已提交
950 951 952
	if (p->state == TASK_UNINTERRUPTIBLE)
		rq->nr_uninterruptible++;

953
	dequeue_task(rq, p, sleep);
954
	dec_nr_running(p, rq);
L
Linus Torvalds 已提交
955 956 957 958 959 960
}

/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
961
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
962 963 964 965
{
	return cpu_curr(task_cpu(p)) == p;
}

966 967 968
/* Used instead of source_load when we know the type == 0 */
unsigned long weighted_cpuload(const int cpu)
{
I
Ingo Molnar 已提交
969 970 971 972 973 974 975 976 977
	return cpu_rq(cpu)->ls.load.weight;
}

static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
#ifdef CONFIG_SMP
	task_thread_info(p)->cpu = cpu;
	set_task_cfs_rq(p);
#endif
978 979
}

L
Linus Torvalds 已提交
980
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
981

I
Ingo Molnar 已提交
982
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
983
{
I
Ingo Molnar 已提交
984 985 986 987 988
	int old_cpu = task_cpu(p);
	struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
	u64 clock_offset, fair_clock_offset;

	clock_offset = old_rq->clock - new_rq->clock;
I
Ingo Molnar 已提交
989 990
	fair_clock_offset = old_rq->cfs.fair_clock - new_rq->cfs.fair_clock;

I
Ingo Molnar 已提交
991 992
	if (p->se.wait_start_fair)
		p->se.wait_start_fair -= fair_clock_offset;
I
Ingo Molnar 已提交
993 994 995 996 997 998
	if (p->se.sleep_start_fair)
		p->se.sleep_start_fair -= fair_clock_offset;

#ifdef CONFIG_SCHEDSTATS
	if (p->se.wait_start)
		p->se.wait_start -= clock_offset;
I
Ingo Molnar 已提交
999 1000 1001 1002
	if (p->se.sleep_start)
		p->se.sleep_start -= clock_offset;
	if (p->se.block_start)
		p->se.block_start -= clock_offset;
I
Ingo Molnar 已提交
1003
#endif
I
Ingo Molnar 已提交
1004 1005

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1006 1007
}

1008
struct migration_req {
L
Linus Torvalds 已提交
1009 1010
	struct list_head list;

1011
	struct task_struct *task;
L
Linus Torvalds 已提交
1012 1013 1014
	int dest_cpu;

	struct completion done;
1015
};
L
Linus Torvalds 已提交
1016 1017 1018 1019 1020

/*
 * The task's runqueue lock must be held.
 * Returns true if you have to wait for migration thread.
 */
1021
static int
1022
migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
L
Linus Torvalds 已提交
1023
{
1024
	struct rq *rq = task_rq(p);
L
Linus Torvalds 已提交
1025 1026 1027 1028 1029

	/*
	 * If the task is not on a runqueue (and not running), then
	 * it is sufficient to simply update the task's cpu field.
	 */
I
Ingo Molnar 已提交
1030
	if (!p->se.on_rq && !task_running(rq, p)) {
L
Linus Torvalds 已提交
1031 1032 1033 1034 1035 1036 1037 1038
		set_task_cpu(p, dest_cpu);
		return 0;
	}

	init_completion(&req->done);
	req->task = p;
	req->dest_cpu = dest_cpu;
	list_add(&req->list, &rq->migration_queue);
1039

L
Linus Torvalds 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
	return 1;
}

/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
1052
void wait_task_inactive(struct task_struct *p)
L
Linus Torvalds 已提交
1053 1054
{
	unsigned long flags;
I
Ingo Molnar 已提交
1055
	int running, on_rq;
1056
	struct rq *rq;
L
Linus Torvalds 已提交
1057 1058

repeat:
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
	/*
	 * We do the initial early heuristics without holding
	 * any task-queue locks at all. We'll only try to get
	 * the runqueue lock when things look like they will
	 * work out!
	 */
	rq = task_rq(p);

	/*
	 * If the task is actively running on another CPU
	 * still, just relax and busy-wait without holding
	 * any locks.
	 *
	 * NOTE! Since we don't hold any locks, it's not
	 * even sure that "rq" stays as the right runqueue!
	 * But we don't care, since "task_running()" will
	 * return false if the runqueue has changed and p
	 * is actually now running somewhere else!
	 */
	while (task_running(rq, p))
		cpu_relax();

	/*
	 * Ok, time to look more closely! We need the rq
	 * lock now, to be *sure*. If we're wrong, we'll
	 * just go back and repeat.
	 */
L
Linus Torvalds 已提交
1086
	rq = task_rq_lock(p, &flags);
1087
	running = task_running(rq, p);
I
Ingo Molnar 已提交
1088
	on_rq = p->se.on_rq;
1089 1090 1091 1092 1093 1094 1095 1096 1097
	task_rq_unlock(rq, &flags);

	/*
	 * Was it really running after all now that we
	 * checked with the proper locks actually held?
	 *
	 * Oops. Go back and try again..
	 */
	if (unlikely(running)) {
L
Linus Torvalds 已提交
1098 1099 1100
		cpu_relax();
		goto repeat;
	}
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110

	/*
	 * It's not enough that it's not actively running,
	 * it must be off the runqueue _entirely_, and not
	 * preempted!
	 *
	 * So if it wa still runnable (but just not actively
	 * running right now), it's preempted, and we should
	 * yield - it could be a while.
	 */
I
Ingo Molnar 已提交
1111
	if (unlikely(on_rq)) {
1112 1113 1114 1115 1116 1117 1118 1119 1120
		yield();
		goto repeat;
	}

	/*
	 * Ahh, all good. It wasn't running, and it wasn't
	 * runnable, which means that it will never become
	 * running in the future either. We're all done!
	 */
L
Linus Torvalds 已提交
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
 * NOTE: this function doesnt have to take the runqueue lock,
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
1136
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}

/*
1148 1149
 * Return a low guess at the load of a migration-source cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1150 1151 1152 1153
 *
 * We want to under-estimate the load of migration sources, to
 * balance conservatively.
 */
N
Nick Piggin 已提交
1154
static inline unsigned long source_load(int cpu, int type)
L
Linus Torvalds 已提交
1155
{
1156
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1157
	unsigned long total = weighted_cpuload(cpu);
1158

1159
	if (type == 0)
I
Ingo Molnar 已提交
1160
		return total;
1161

I
Ingo Molnar 已提交
1162
	return min(rq->cpu_load[type-1], total);
L
Linus Torvalds 已提交
1163 1164 1165
}

/*
1166 1167
 * Return a high guess at the load of a migration-target cpu weighted
 * according to the scheduling class and "nice" value.
L
Linus Torvalds 已提交
1168
 */
N
Nick Piggin 已提交
1169
static inline unsigned long target_load(int cpu, int type)
L
Linus Torvalds 已提交
1170
{
1171
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1172
	unsigned long total = weighted_cpuload(cpu);
1173

N
Nick Piggin 已提交
1174
	if (type == 0)
I
Ingo Molnar 已提交
1175
		return total;
1176

I
Ingo Molnar 已提交
1177
	return max(rq->cpu_load[type-1], total);
1178 1179 1180 1181 1182 1183 1184
}

/*
 * Return the average load per task on the cpu's run queue
 */
static inline unsigned long cpu_avg_load_per_task(int cpu)
{
1185
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
1186
	unsigned long total = weighted_cpuload(cpu);
1187 1188
	unsigned long n = rq->nr_running;

I
Ingo Molnar 已提交
1189
	return n ? total / n : SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
1190 1191
}

N
Nick Piggin 已提交
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
/*
 * find_idlest_group finds and returns the least busy CPU group within the
 * domain.
 */
static struct sched_group *
find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
{
	struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
	unsigned long min_load = ULONG_MAX, this_load = 0;
	int load_idx = sd->forkexec_idx;
	int imbalance = 100 + (sd->imbalance_pct-100)/2;

	do {
		unsigned long load, avg_load;
		int local_group;
		int i;

1209 1210 1211 1212
		/* Skip over this group if it has no CPUs allowed */
		if (!cpus_intersects(group->cpumask, p->cpus_allowed))
			goto nextgroup;

N
Nick Piggin 已提交
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
		local_group = cpu_isset(this_cpu, group->cpumask);

		/* Tally up the load of all CPUs in the group */
		avg_load = 0;

		for_each_cpu_mask(i, group->cpumask) {
			/* Bias balancing toward cpus of our domain */
			if (local_group)
				load = source_load(i, load_idx);
			else
				load = target_load(i, load_idx);

			avg_load += load;
		}

		/* Adjust by relative CPU power of the group */
1229 1230
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
N
Nick Piggin 已提交
1231 1232 1233 1234 1235 1236 1237 1238

		if (local_group) {
			this_load = avg_load;
			this = group;
		} else if (avg_load < min_load) {
			min_load = avg_load;
			idlest = group;
		}
1239
nextgroup:
N
Nick Piggin 已提交
1240 1241 1242 1243 1244 1245 1246 1247 1248
		group = group->next;
	} while (group != sd->groups);

	if (!idlest || 100*this_load < imbalance*min_load)
		return NULL;
	return idlest;
}

/*
1249
 * find_idlest_cpu - find the idlest cpu among the cpus in group.
N
Nick Piggin 已提交
1250
 */
I
Ingo Molnar 已提交
1251 1252
static int
find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
N
Nick Piggin 已提交
1253
{
1254
	cpumask_t tmp;
N
Nick Piggin 已提交
1255 1256 1257 1258
	unsigned long load, min_load = ULONG_MAX;
	int idlest = -1;
	int i;

1259 1260 1261 1262
	/* Traverse only the allowed CPUs */
	cpus_and(tmp, group->cpumask, p->cpus_allowed);

	for_each_cpu_mask(i, tmp) {
1263
		load = weighted_cpuload(i);
N
Nick Piggin 已提交
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273

		if (load < min_load || (load == min_load && i == this_cpu)) {
			min_load = load;
			idlest = i;
		}
	}

	return idlest;
}

N
Nick Piggin 已提交
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
/*
 * sched_balance_self: balance the current task (running on cpu) in domains
 * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
 * SD_BALANCE_EXEC.
 *
 * Balance, ie. select the least loaded group.
 *
 * Returns the target CPU number, or the same CPU if no balancing is needed.
 *
 * preempt must be disabled.
 */
static int sched_balance_self(int cpu, int flag)
{
	struct task_struct *t = current;
	struct sched_domain *tmp, *sd = NULL;
N
Nick Piggin 已提交
1289

1290
	for_each_domain(cpu, tmp) {
I
Ingo Molnar 已提交
1291 1292 1293
		/*
		 * If power savings logic is enabled for a domain, stop there.
		 */
1294 1295
		if (tmp->flags & SD_POWERSAVINGS_BALANCE)
			break;
N
Nick Piggin 已提交
1296 1297
		if (tmp->flags & flag)
			sd = tmp;
1298
	}
N
Nick Piggin 已提交
1299 1300 1301 1302

	while (sd) {
		cpumask_t span;
		struct sched_group *group;
1303 1304 1305 1306 1307 1308
		int new_cpu, weight;

		if (!(sd->flags & flag)) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1309 1310 1311

		span = sd->span;
		group = find_idlest_group(sd, t, cpu);
1312 1313 1314 1315
		if (!group) {
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1316

1317
		new_cpu = find_idlest_cpu(group, t, cpu);
1318 1319 1320 1321 1322
		if (new_cpu == -1 || new_cpu == cpu) {
			/* Now try balancing at a lower domain level of cpu */
			sd = sd->child;
			continue;
		}
N
Nick Piggin 已提交
1323

1324
		/* Now try balancing at a lower domain level of new_cpu */
N
Nick Piggin 已提交
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340
		cpu = new_cpu;
		sd = NULL;
		weight = cpus_weight(span);
		for_each_domain(cpu, tmp) {
			if (weight <= cpus_weight(tmp->span))
				break;
			if (tmp->flags & flag)
				sd = tmp;
		}
		/* while loop will break here if sd == NULL */
	}

	return cpu;
}

#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350

/*
 * wake_idle() will wake a task on an idle cpu if task->cpu is
 * not idle and an idle cpu is available.  The span of cpus to
 * search starts with cpus closest then further out as needed,
 * so we always favor a closer, idle cpu.
 *
 * Returns the CPU we should wake onto.
 */
#if defined(ARCH_HAS_SCHED_WAKE_IDLE)
1351
static int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1352 1353 1354 1355 1356
{
	cpumask_t tmp;
	struct sched_domain *sd;
	int i;

1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
	/*
	 * If it is idle, then it is the best cpu to run this task.
	 *
	 * This cpu is also the best, if it has more than one task already.
	 * Siblings must be also busy(in most cases) as they didn't already
	 * pickup the extra load from this cpu and hence we need not check
	 * sibling runqueue info. This will avoid the checks and cache miss
	 * penalities associated with that.
	 */
	if (idle_cpu(cpu) || cpu_rq(cpu)->nr_running > 1)
L
Linus Torvalds 已提交
1367 1368 1369 1370
		return cpu;

	for_each_domain(cpu, sd) {
		if (sd->flags & SD_WAKE_IDLE) {
N
Nick Piggin 已提交
1371
			cpus_and(tmp, sd->span, p->cpus_allowed);
L
Linus Torvalds 已提交
1372 1373 1374 1375
			for_each_cpu_mask(i, tmp) {
				if (idle_cpu(i))
					return i;
			}
I
Ingo Molnar 已提交
1376
		} else {
N
Nick Piggin 已提交
1377
			break;
I
Ingo Molnar 已提交
1378
		}
L
Linus Torvalds 已提交
1379 1380 1381 1382
	}
	return cpu;
}
#else
1383
static inline int wake_idle(int cpu, struct task_struct *p)
L
Linus Torvalds 已提交
1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
{
	return cpu;
}
#endif

/***
 * try_to_wake_up - wake up a thread
 * @p: the to-be-woken-up thread
 * @state: the mask of task states that can be woken
 * @sync: do a synchronous wakeup?
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
 * returns failure only if the task is already active.
 */
1403
static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
L
Linus Torvalds 已提交
1404 1405 1406 1407
{
	int cpu, this_cpu, success = 0;
	unsigned long flags;
	long old_state;
1408
	struct rq *rq;
L
Linus Torvalds 已提交
1409
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
1410
	struct sched_domain *sd, *this_sd = NULL;
1411
	unsigned long load, this_load;
L
Linus Torvalds 已提交
1412 1413 1414 1415 1416 1417 1418 1419
	int new_cpu;
#endif

	rq = task_rq_lock(p, &flags);
	old_state = p->state;
	if (!(old_state & state))
		goto out;

I
Ingo Molnar 已提交
1420
	if (p->se.on_rq)
L
Linus Torvalds 已提交
1421 1422 1423 1424 1425 1426 1427 1428 1429
		goto out_running;

	cpu = task_cpu(p);
	this_cpu = smp_processor_id();

#ifdef CONFIG_SMP
	if (unlikely(task_running(rq, p)))
		goto out_activate;

N
Nick Piggin 已提交
1430 1431
	new_cpu = cpu;

L
Linus Torvalds 已提交
1432 1433 1434
	schedstat_inc(rq, ttwu_cnt);
	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
N
Nick Piggin 已提交
1435 1436 1437 1438 1439 1440 1441 1442
		goto out_set_cpu;
	}

	for_each_domain(this_cpu, sd) {
		if (cpu_isset(cpu, sd->span)) {
			schedstat_inc(sd, ttwu_wake_remote);
			this_sd = sd;
			break;
L
Linus Torvalds 已提交
1443 1444 1445
		}
	}

N
Nick Piggin 已提交
1446
	if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed)))
L
Linus Torvalds 已提交
1447 1448 1449
		goto out_set_cpu;

	/*
N
Nick Piggin 已提交
1450
	 * Check for affine wakeup and passive balancing possibilities.
L
Linus Torvalds 已提交
1451
	 */
N
Nick Piggin 已提交
1452 1453 1454
	if (this_sd) {
		int idx = this_sd->wake_idx;
		unsigned int imbalance;
L
Linus Torvalds 已提交
1455

1456 1457
		imbalance = 100 + (this_sd->imbalance_pct - 100) / 2;

N
Nick Piggin 已提交
1458 1459
		load = source_load(cpu, idx);
		this_load = target_load(this_cpu, idx);
L
Linus Torvalds 已提交
1460

N
Nick Piggin 已提交
1461 1462
		new_cpu = this_cpu; /* Wake to this CPU if we can */

1463 1464
		if (this_sd->flags & SD_WAKE_AFFINE) {
			unsigned long tl = this_load;
1465 1466 1467
			unsigned long tl_per_task;

			tl_per_task = cpu_avg_load_per_task(this_cpu);
1468

L
Linus Torvalds 已提交
1469
			/*
1470 1471 1472
			 * If sync wakeup then subtract the (maximum possible)
			 * effect of the currently running task from the load
			 * of the current CPU:
L
Linus Torvalds 已提交
1473
			 */
1474
			if (sync)
I
Ingo Molnar 已提交
1475
				tl -= current->se.load.weight;
1476 1477

			if ((tl <= load &&
1478
				tl + target_load(cpu, idx) <= tl_per_task) ||
I
Ingo Molnar 已提交
1479
			       100*(tl + p->se.load.weight) <= imbalance*load) {
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
				/*
				 * This domain has SD_WAKE_AFFINE and
				 * p is cache cold in this domain, and
				 * there is no bad imbalance.
				 */
				schedstat_inc(this_sd, ttwu_move_affine);
				goto out_set_cpu;
			}
		}

		/*
		 * Start passive balancing when half the imbalance_pct
		 * limit is reached.
		 */
		if (this_sd->flags & SD_WAKE_BALANCE) {
			if (imbalance*this_load <= 100*load) {
				schedstat_inc(this_sd, ttwu_move_balance);
				goto out_set_cpu;
			}
L
Linus Torvalds 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
		}
	}

	new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */
out_set_cpu:
	new_cpu = wake_idle(new_cpu, p);
	if (new_cpu != cpu) {
		set_task_cpu(p, new_cpu);
		task_rq_unlock(rq, &flags);
		/* might preempt at this point */
		rq = task_rq_lock(p, &flags);
		old_state = p->state;
		if (!(old_state & state))
			goto out;
I
Ingo Molnar 已提交
1513
		if (p->se.on_rq)
L
Linus Torvalds 已提交
1514 1515 1516 1517 1518 1519 1520 1521
			goto out_running;

		this_cpu = smp_processor_id();
		cpu = task_cpu(p);
	}

out_activate:
#endif /* CONFIG_SMP */
I
Ingo Molnar 已提交
1522
	update_rq_clock(rq);
I
Ingo Molnar 已提交
1523
	activate_task(rq, p, 1);
L
Linus Torvalds 已提交
1524 1525 1526 1527 1528 1529 1530 1531
	/*
	 * Sync wakeups (i.e. those types of wakeups where the waker
	 * has indicated that it will leave the CPU in short order)
	 * don't trigger a preemption, if the woken up task will run on
	 * this cpu. (in this case the 'I will reschedule' promise of
	 * the waker guarantees that the freshly woken up task is going
	 * to be considered on this CPU.)
	 */
I
Ingo Molnar 已提交
1532 1533
	if (!sync || cpu != this_cpu)
		check_preempt_curr(rq, p);
L
Linus Torvalds 已提交
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
	success = 1;

out_running:
	p->state = TASK_RUNNING;
out:
	task_rq_unlock(rq, &flags);

	return success;
}

1544
int fastcall wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
1545 1546 1547 1548 1549 1550
{
	return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED |
				 TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0);
}
EXPORT_SYMBOL(wake_up_process);

1551
int fastcall wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
1552 1553 1554 1555 1556 1557 1558
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
	p->se.wait_start_fair		= 0;
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
	p->se.delta_exec		= 0;
	p->se.delta_fair_run		= 0;
	p->se.delta_fair_sleep		= 0;
	p->se.wait_runtime		= 0;
I
Ingo Molnar 已提交
1571 1572 1573 1574
	p->se.sleep_start_fair		= 0;

#ifdef CONFIG_SCHEDSTATS
	p->se.wait_start		= 0;
I
Ingo Molnar 已提交
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
	p->se.sum_wait_runtime		= 0;
	p->se.sum_sleep_runtime		= 0;
	p->se.sleep_start		= 0;
	p->se.block_start		= 0;
	p->se.sleep_max			= 0;
	p->se.block_max			= 0;
	p->se.exec_max			= 0;
	p->se.wait_max			= 0;
	p->se.wait_runtime_overruns	= 0;
	p->se.wait_runtime_underruns	= 0;
I
Ingo Molnar 已提交
1585
#endif
N
Nick Piggin 已提交
1586

I
Ingo Molnar 已提交
1587 1588
	INIT_LIST_HEAD(&p->run_list);
	p->se.on_rq = 0;
N
Nick Piggin 已提交
1589

1590 1591 1592 1593
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif

L
Linus Torvalds 已提交
1594 1595 1596 1597 1598 1599 1600
	/*
	 * We mark the process as running here, but have not actually
	 * inserted it onto the runqueue yet. This guarantees that
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
}

/*
 * fork()/clone()-time setup:
 */
void sched_fork(struct task_struct *p, int clone_flags)
{
	int cpu = get_cpu();

	__sched_fork(p);

#ifdef CONFIG_SMP
	cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
#endif
	__set_task_cpu(p, cpu);
1616 1617 1618 1619 1620 1621

	/*
	 * Make sure we do not leak PI boosting priority to the child:
	 */
	p->prio = current->normal_prio;

1622
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
1623
	if (likely(sched_info_on()))
1624
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
1625
#endif
1626
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
1627 1628
	p->oncpu = 0;
#endif
L
Linus Torvalds 已提交
1629
#ifdef CONFIG_PREEMPT
1630
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
1631
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
1632
#endif
N
Nick Piggin 已提交
1633
	put_cpu();
L
Linus Torvalds 已提交
1634 1635
}

I
Ingo Molnar 已提交
1636 1637 1638 1639 1640 1641
/*
 * After fork, child runs first. (default) If set to 0 then
 * parent will (try to) run first.
 */
unsigned int __read_mostly sysctl_sched_child_runs_first = 1;

L
Linus Torvalds 已提交
1642 1643 1644 1645 1646 1647 1648
/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
1649
void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
L
Linus Torvalds 已提交
1650 1651
{
	unsigned long flags;
I
Ingo Molnar 已提交
1652 1653
	struct rq *rq;
	int this_cpu;
L
Linus Torvalds 已提交
1654 1655

	rq = task_rq_lock(p, &flags);
N
Nick Piggin 已提交
1656
	BUG_ON(p->state != TASK_RUNNING);
I
Ingo Molnar 已提交
1657
	this_cpu = smp_processor_id(); /* parent's CPU */
I
Ingo Molnar 已提交
1658
	update_rq_clock(rq);
L
Linus Torvalds 已提交
1659 1660 1661

	p->prio = effective_prio(p);

I
Ingo Molnar 已提交
1662 1663 1664 1665
	if (!p->sched_class->task_new || !sysctl_sched_child_runs_first ||
			(clone_flags & CLONE_VM) || task_cpu(p) != this_cpu ||
			!current->se.on_rq) {

I
Ingo Molnar 已提交
1666
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
1667 1668
	} else {
		/*
I
Ingo Molnar 已提交
1669 1670
		 * Let the scheduling class do new task startup
		 * management (if any):
L
Linus Torvalds 已提交
1671
		 */
1672
		p->sched_class->task_new(rq, p);
1673
		inc_nr_running(p, rq);
L
Linus Torvalds 已提交
1674
	}
I
Ingo Molnar 已提交
1675 1676
	check_preempt_curr(rq, p);
	task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
1677 1678
}

1679 1680 1681
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
R
Randy Dunlap 已提交
1682 1683
 * preempt_notifier_register - tell me when current is being being preempted & rescheduled
 * @notifier: notifier struct to register
1684 1685 1686 1687 1688 1689 1690 1691 1692
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
1693
 * @notifier: notifier struct to unregister
1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;
	struct hlist_node *node;

	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
		notifier->ops->sched_out(notifier, next);
}

#else

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

#endif

1737 1738 1739
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
1740
 * @prev: the current task that is being switched out
1741 1742 1743 1744 1745 1746 1747 1748 1749
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
1750 1751 1752
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
1753
{
1754
	fire_sched_out_preempt_notifiers(prev, next);
1755 1756 1757 1758
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
1759 1760
/**
 * finish_task_switch - clean up after a task-switch
1761
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
1762 1763
 * @prev: the thread we just switched away from.
 *
1764 1765 1766 1767
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
1768 1769 1770 1771 1772 1773
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
 * so, we finish that here outside of the runqueue lock.  (Doing it
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
1774
static inline void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
1775 1776 1777
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
1778
	long prev_state;
L
Linus Torvalds 已提交
1779 1780 1781 1782 1783

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
1784
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
1785 1786
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
1787
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
1788 1789 1790 1791 1792
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
1793
	prev_state = prev->state;
1794 1795
	finish_arch_switch(prev);
	finish_lock_switch(rq, prev);
1796
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
1797 1798
	if (mm)
		mmdrop(mm);
1799
	if (unlikely(prev_state == TASK_DEAD)) {
1800 1801 1802
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
1803
		 */
1804
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
1805
		put_task_struct(prev);
1806
	}
L
Linus Torvalds 已提交
1807 1808 1809 1810 1811 1812
}

/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
1813
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
1814 1815
	__releases(rq->lock)
{
1816 1817
	struct rq *rq = this_rq();

1818 1819 1820 1821 1822
	finish_task_switch(rq, prev);
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
1823 1824 1825 1826 1827 1828 1829 1830
	if (current->set_child_tid)
		put_user(current->pid, current->set_child_tid);
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
1831
static inline void
1832
context_switch(struct rq *rq, struct task_struct *prev,
1833
	       struct task_struct *next)
L
Linus Torvalds 已提交
1834
{
I
Ingo Molnar 已提交
1835
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
1836

1837
	prepare_task_switch(rq, prev, next);
I
Ingo Molnar 已提交
1838 1839
	mm = next->mm;
	oldmm = prev->active_mm;
1840 1841 1842 1843 1844 1845 1846
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
	arch_enter_lazy_cpu_mode();

I
Ingo Molnar 已提交
1847
	if (unlikely(!mm)) {
L
Linus Torvalds 已提交
1848 1849 1850 1851 1852 1853
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

I
Ingo Molnar 已提交
1854
	if (unlikely(!prev->mm)) {
L
Linus Torvalds 已提交
1855 1856 1857
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
1858 1859 1860 1861 1862 1863 1864
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
1865
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
1866
#endif
L
Linus Torvalds 已提交
1867 1868 1869 1870

	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
1871 1872 1873 1874 1875 1876 1877
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900
}

/*
 * nr_running, nr_uninterruptible and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, current number of uninterruptible-sleeping threads, total
 * number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
}

unsigned long nr_uninterruptible(void)
{
	unsigned long i, sum = 0;

1901
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915
		sum += cpu_rq(i)->nr_uninterruptible;

	/*
	 * Since we read the counters lockless, it might be slightly
	 * inaccurate. Do not allow it to go below zero though:
	 */
	if (unlikely((long)sum < 0))
		sum = 0;

	return sum;
}

unsigned long long nr_context_switches(void)
{
1916 1917
	int i;
	unsigned long long sum = 0;
L
Linus Torvalds 已提交
1918

1919
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1920 1921 1922 1923 1924 1925 1926 1927 1928
		sum += cpu_rq(i)->nr_switches;

	return sum;
}

unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;

1929
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
1930 1931 1932 1933 1934
		sum += atomic_read(&cpu_rq(i)->nr_iowait);

	return sum;
}

1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949
unsigned long nr_active(void)
{
	unsigned long i, running = 0, uninterruptible = 0;

	for_each_online_cpu(i) {
		running += cpu_rq(i)->nr_running;
		uninterruptible += cpu_rq(i)->nr_uninterruptible;
	}

	if (unlikely((long)uninterruptible < 0))
		uninterruptible = 0;

	return running + uninterruptible;
}

1950
/*
I
Ingo Molnar 已提交
1951 1952
 * Update rq->cpu_load[] statistics. This function is usually called every
 * scheduler tick (TICK_NSEC).
1953
 */
I
Ingo Molnar 已提交
1954
static void update_cpu_load(struct rq *this_rq)
1955
{
I
Ingo Molnar 已提交
1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966
	u64 fair_delta64, exec_delta64, idle_delta64, sample_interval64, tmp64;
	unsigned long total_load = this_rq->ls.load.weight;
	unsigned long this_load =  total_load;
	struct load_stat *ls = &this_rq->ls;
	int i, scale;

	this_rq->nr_load_updates++;
	if (unlikely(!(sysctl_sched_features & SCHED_FEAT_PRECISE_CPU_LOAD)))
		goto do_avg;

	/* Update delta_fair/delta_exec fields first */
1967
	update_curr_load(this_rq);
I
Ingo Molnar 已提交
1968 1969 1970 1971 1972 1973 1974

	fair_delta64 = ls->delta_fair + 1;
	ls->delta_fair = 0;

	exec_delta64 = ls->delta_exec + 1;
	ls->delta_exec = 0;

1975 1976
	sample_interval64 = this_rq->clock - ls->load_update_last;
	ls->load_update_last = this_rq->clock;
I
Ingo Molnar 已提交
1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003

	if ((s64)sample_interval64 < (s64)TICK_NSEC)
		sample_interval64 = TICK_NSEC;

	if (exec_delta64 > sample_interval64)
		exec_delta64 = sample_interval64;

	idle_delta64 = sample_interval64 - exec_delta64;

	tmp64 = div64_64(SCHED_LOAD_SCALE * exec_delta64, fair_delta64);
	tmp64 = div64_64(tmp64 * exec_delta64, sample_interval64);

	this_load = (unsigned long)tmp64;

do_avg:

	/* Update our load: */
	for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
		unsigned long old_load, new_load;

		/* scale is effectively 1 << i now, and >> i divides by scale */

		old_load = this_rq->cpu_load[i];
		new_load = this_load;

		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
	}
2004 2005
}

I
Ingo Molnar 已提交
2006 2007
#ifdef CONFIG_SMP

L
Linus Torvalds 已提交
2008 2009 2010 2011 2012 2013
/*
 * double_rq_lock - safely lock two runqueues
 *
 * Note this does not disable interrupts like task_rq_lock,
 * you need to do so manually before calling.
 */
2014
static void double_rq_lock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2015 2016 2017
	__acquires(rq1->lock)
	__acquires(rq2->lock)
{
2018
	BUG_ON(!irqs_disabled());
L
Linus Torvalds 已提交
2019 2020 2021 2022
	if (rq1 == rq2) {
		spin_lock(&rq1->lock);
		__acquire(rq2->lock);	/* Fake it out ;) */
	} else {
2023
		if (rq1 < rq2) {
L
Linus Torvalds 已提交
2024 2025 2026 2027 2028 2029 2030
			spin_lock(&rq1->lock);
			spin_lock(&rq2->lock);
		} else {
			spin_lock(&rq2->lock);
			spin_lock(&rq1->lock);
		}
	}
2031 2032
	update_rq_clock(rq1);
	update_rq_clock(rq2);
L
Linus Torvalds 已提交
2033 2034 2035 2036 2037 2038 2039 2040
}

/*
 * double_rq_unlock - safely unlock two runqueues
 *
 * Note this does not restore interrupts like task_rq_unlock,
 * you need to do so manually after calling.
 */
2041
static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
L
Linus Torvalds 已提交
2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054
	__releases(rq1->lock)
	__releases(rq2->lock)
{
	spin_unlock(&rq1->lock);
	if (rq1 != rq2)
		spin_unlock(&rq2->lock);
	else
		__release(rq2->lock);
}

/*
 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
 */
2055
static void double_lock_balance(struct rq *this_rq, struct rq *busiest)
L
Linus Torvalds 已提交
2056 2057 2058 2059
	__releases(this_rq->lock)
	__acquires(busiest->lock)
	__acquires(this_rq->lock)
{
2060 2061 2062 2063 2064
	if (unlikely(!irqs_disabled())) {
		/* printk() doesn't work good under rq->lock */
		spin_unlock(&this_rq->lock);
		BUG_ON(1);
	}
L
Linus Torvalds 已提交
2065
	if (unlikely(!spin_trylock(&busiest->lock))) {
2066
		if (busiest < this_rq) {
L
Linus Torvalds 已提交
2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080
			spin_unlock(&this_rq->lock);
			spin_lock(&busiest->lock);
			spin_lock(&this_rq->lock);
		} else
			spin_lock(&busiest->lock);
	}
}

/*
 * If dest_cpu is allowed for this process, migrate the task to it.
 * This is accomplished by forcing the cpu_allowed mask to only
 * allow dest_cpu, which will force the cpu onto dest_cpu.  Then
 * the cpu_allowed mask is restored.
 */
2081
static void sched_migrate_task(struct task_struct *p, int dest_cpu)
L
Linus Torvalds 已提交
2082
{
2083
	struct migration_req req;
L
Linus Torvalds 已提交
2084
	unsigned long flags;
2085
	struct rq *rq;
L
Linus Torvalds 已提交
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095

	rq = task_rq_lock(p, &flags);
	if (!cpu_isset(dest_cpu, p->cpus_allowed)
	    || unlikely(cpu_is_offline(dest_cpu)))
		goto out;

	/* force the process onto the specified CPU */
	if (migrate_task(p, dest_cpu, &req)) {
		/* Need to wait for migration thread (might exit: take ref). */
		struct task_struct *mt = rq->migration_thread;
2096

L
Linus Torvalds 已提交
2097 2098 2099 2100 2101
		get_task_struct(mt);
		task_rq_unlock(rq, &flags);
		wake_up_process(mt);
		put_task_struct(mt);
		wait_for_completion(&req.done);
2102

L
Linus Torvalds 已提交
2103 2104 2105 2106 2107 2108 2109
		return;
	}
out:
	task_rq_unlock(rq, &flags);
}

/*
N
Nick Piggin 已提交
2110 2111
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
L
Linus Torvalds 已提交
2112 2113 2114 2115
 */
void sched_exec(void)
{
	int new_cpu, this_cpu = get_cpu();
N
Nick Piggin 已提交
2116
	new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
L
Linus Torvalds 已提交
2117
	put_cpu();
N
Nick Piggin 已提交
2118 2119
	if (new_cpu != this_cpu)
		sched_migrate_task(current, new_cpu);
L
Linus Torvalds 已提交
2120 2121 2122 2123 2124 2125
}

/*
 * pull_task - move a task from a remote runqueue to the local runqueue.
 * Both runqueues must be locked.
 */
I
Ingo Molnar 已提交
2126 2127
static void pull_task(struct rq *src_rq, struct task_struct *p,
		      struct rq *this_rq, int this_cpu)
L
Linus Torvalds 已提交
2128
{
2129
	deactivate_task(src_rq, p, 0);
L
Linus Torvalds 已提交
2130
	set_task_cpu(p, this_cpu);
I
Ingo Molnar 已提交
2131
	activate_task(this_rq, p, 0);
L
Linus Torvalds 已提交
2132 2133 2134 2135
	/*
	 * Note that idle threads have a prio of MAX_PRIO, for this test
	 * to be always true for them.
	 */
I
Ingo Molnar 已提交
2136
	check_preempt_curr(this_rq, p);
L
Linus Torvalds 已提交
2137 2138 2139 2140 2141
}

/*
 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
 */
2142
static
2143
int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
I
Ingo Molnar 已提交
2144
		     struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2145
		     int *all_pinned)
L
Linus Torvalds 已提交
2146 2147 2148 2149 2150 2151 2152 2153 2154
{
	/*
	 * We do not migrate tasks that are:
	 * 1) running (obviously), or
	 * 2) cannot be migrated to this CPU due to cpus_allowed, or
	 * 3) are cache-hot on their current CPU.
	 */
	if (!cpu_isset(this_cpu, p->cpus_allowed))
		return 0;
2155 2156 2157 2158
	*all_pinned = 0;

	if (task_running(rq, p))
		return 0;
L
Linus Torvalds 已提交
2159 2160

	/*
I
Ingo Molnar 已提交
2161
	 * Aggressive migration if too many balance attempts have failed:
L
Linus Torvalds 已提交
2162
	 */
I
Ingo Molnar 已提交
2163
	if (sd->nr_balance_failed > sd->cache_nice_tries)
L
Linus Torvalds 已提交
2164 2165 2166 2167 2168
		return 1;

	return 1;
}

I
Ingo Molnar 已提交
2169
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2170
		      unsigned long max_nr_move, unsigned long max_load_move,
I
Ingo Molnar 已提交
2171
		      struct sched_domain *sd, enum cpu_idle_type idle,
I
Ingo Molnar 已提交
2172
		      int *all_pinned, unsigned long *load_moved,
2173
		      int *this_best_prio, struct rq_iterator *iterator)
L
Linus Torvalds 已提交
2174
{
I
Ingo Molnar 已提交
2175 2176 2177
	int pulled = 0, pinned = 0, skip_for_load;
	struct task_struct *p;
	long rem_load_move = max_load_move;
L
Linus Torvalds 已提交
2178

2179
	if (max_nr_move == 0 || max_load_move == 0)
L
Linus Torvalds 已提交
2180 2181
		goto out;

2182 2183
	pinned = 1;

L
Linus Torvalds 已提交
2184
	/*
I
Ingo Molnar 已提交
2185
	 * Start the load-balancing iterator:
L
Linus Torvalds 已提交
2186
	 */
I
Ingo Molnar 已提交
2187 2188 2189
	p = iterator->start(iterator->arg);
next:
	if (!p)
L
Linus Torvalds 已提交
2190
		goto out;
2191 2192 2193 2194 2195
	/*
	 * To help distribute high priority tasks accross CPUs we don't
	 * skip a task if it will be the highest priority task (i.e. smallest
	 * prio value) on its new queue regardless of its load weight
	 */
I
Ingo Molnar 已提交
2196 2197
	skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
							 SCHED_LOAD_SCALE_FUZZ;
2198
	if ((skip_for_load && p->prio >= *this_best_prio) ||
I
Ingo Molnar 已提交
2199 2200 2201
	    !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2202 2203
	}

I
Ingo Molnar 已提交
2204
	pull_task(busiest, p, this_rq, this_cpu);
L
Linus Torvalds 已提交
2205
	pulled++;
I
Ingo Molnar 已提交
2206
	rem_load_move -= p->se.load.weight;
L
Linus Torvalds 已提交
2207

2208 2209 2210 2211 2212
	/*
	 * We only want to steal up to the prescribed number of tasks
	 * and the prescribed amount of weighted load.
	 */
	if (pulled < max_nr_move && rem_load_move > 0) {
2213 2214
		if (p->prio < *this_best_prio)
			*this_best_prio = p->prio;
I
Ingo Molnar 已提交
2215 2216
		p = iterator->next(iterator->arg);
		goto next;
L
Linus Torvalds 已提交
2217 2218 2219 2220 2221 2222 2223 2224
	}
out:
	/*
	 * Right now, this is the only place pull_task() is called,
	 * so we can safely collect pull_task() stats here rather than
	 * inside pull_task().
	 */
	schedstat_add(sd, lb_gained[idle], pulled);
2225 2226 2227

	if (all_pinned)
		*all_pinned = pinned;
I
Ingo Molnar 已提交
2228
	*load_moved = max_load_move - rem_load_move;
L
Linus Torvalds 已提交
2229 2230 2231
	return pulled;
}

I
Ingo Molnar 已提交
2232
/*
P
Peter Williams 已提交
2233 2234 2235
 * move_tasks tries to move up to max_load_move weighted load from busiest to
 * this_rq, as part of a balancing operation within domain "sd".
 * Returns 1 if successful and 0 otherwise.
I
Ingo Molnar 已提交
2236 2237 2238 2239
 *
 * Called with both runqueues locked.
 */
static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
P
Peter Williams 已提交
2240
		      unsigned long max_load_move,
I
Ingo Molnar 已提交
2241 2242 2243 2244
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned)
{
	struct sched_class *class = sched_class_highest;
P
Peter Williams 已提交
2245
	unsigned long total_load_moved = 0;
2246
	int this_best_prio = this_rq->curr->prio;
I
Ingo Molnar 已提交
2247 2248

	do {
P
Peter Williams 已提交
2249 2250 2251
		total_load_moved +=
			class->load_balance(this_rq, this_cpu, busiest,
				ULONG_MAX, max_load_move - total_load_moved,
2252
				sd, idle, all_pinned, &this_best_prio);
I
Ingo Molnar 已提交
2253
		class = class->next;
P
Peter Williams 已提交
2254
	} while (class && max_load_move > total_load_moved);
I
Ingo Molnar 已提交
2255

P
Peter Williams 已提交
2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269
	return total_load_moved > 0;
}

/*
 * move_one_task tries to move exactly one task from busiest to this_rq, as
 * part of active balancing operations within "domain".
 * Returns 1 if successful and 0 otherwise.
 *
 * Called with both runqueues locked.
 */
static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
			 struct sched_domain *sd, enum cpu_idle_type idle)
{
	struct sched_class *class;
2270
	int this_best_prio = MAX_PRIO;
P
Peter Williams 已提交
2271 2272 2273

	for (class = sched_class_highest; class; class = class->next)
		if (class->load_balance(this_rq, this_cpu, busiest,
2274 2275
					1, ULONG_MAX, sd, idle, NULL,
					&this_best_prio))
P
Peter Williams 已提交
2276 2277 2278
			return 1;

	return 0;
I
Ingo Molnar 已提交
2279 2280
}

L
Linus Torvalds 已提交
2281 2282
/*
 * find_busiest_group finds and returns the busiest CPU group within the
2283 2284
 * domain. It calculates and returns the amount of weighted load which
 * should be moved to restore balance via the imbalance parameter.
L
Linus Torvalds 已提交
2285 2286 2287
 */
static struct sched_group *
find_busiest_group(struct sched_domain *sd, int this_cpu,
I
Ingo Molnar 已提交
2288 2289
		   unsigned long *imbalance, enum cpu_idle_type idle,
		   int *sd_idle, cpumask_t *cpus, int *balance)
L
Linus Torvalds 已提交
2290 2291 2292
{
	struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
	unsigned long max_load, avg_load, total_load, this_load, total_pwr;
2293
	unsigned long max_pull;
2294 2295
	unsigned long busiest_load_per_task, busiest_nr_running;
	unsigned long this_load_per_task, this_nr_running;
N
Nick Piggin 已提交
2296
	int load_idx;
2297 2298 2299 2300 2301 2302
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
	int power_savings_balance = 1;
	unsigned long leader_nr_running = 0, min_load_per_task = 0;
	unsigned long min_nr_running = ULONG_MAX;
	struct sched_group *group_min = NULL, *group_leader = NULL;
#endif
L
Linus Torvalds 已提交
2303 2304

	max_load = this_load = total_load = total_pwr = 0;
2305 2306
	busiest_load_per_task = busiest_nr_running = 0;
	this_load_per_task = this_nr_running = 0;
I
Ingo Molnar 已提交
2307
	if (idle == CPU_NOT_IDLE)
N
Nick Piggin 已提交
2308
		load_idx = sd->busy_idx;
I
Ingo Molnar 已提交
2309
	else if (idle == CPU_NEWLY_IDLE)
N
Nick Piggin 已提交
2310 2311 2312
		load_idx = sd->newidle_idx;
	else
		load_idx = sd->idle_idx;
L
Linus Torvalds 已提交
2313 2314

	do {
2315
		unsigned long load, group_capacity;
L
Linus Torvalds 已提交
2316 2317
		int local_group;
		int i;
2318
		unsigned int balance_cpu = -1, first_idle_cpu = 0;
2319
		unsigned long sum_nr_running, sum_weighted_load;
L
Linus Torvalds 已提交
2320 2321 2322

		local_group = cpu_isset(this_cpu, group->cpumask);

2323 2324 2325
		if (local_group)
			balance_cpu = first_cpu(group->cpumask);

L
Linus Torvalds 已提交
2326
		/* Tally up the load of all CPUs in the group */
2327
		sum_weighted_load = sum_nr_running = avg_load = 0;
L
Linus Torvalds 已提交
2328 2329

		for_each_cpu_mask(i, group->cpumask) {
2330 2331 2332 2333 2334 2335
			struct rq *rq;

			if (!cpu_isset(i, *cpus))
				continue;

			rq = cpu_rq(i);
2336

2337
			if (*sd_idle && rq->nr_running)
N
Nick Piggin 已提交
2338 2339
				*sd_idle = 0;

L
Linus Torvalds 已提交
2340
			/* Bias balancing toward cpus of our domain */
2341 2342 2343 2344 2345 2346
			if (local_group) {
				if (idle_cpu(i) && !first_idle_cpu) {
					first_idle_cpu = 1;
					balance_cpu = i;
				}

N
Nick Piggin 已提交
2347
				load = target_load(i, load_idx);
2348
			} else
N
Nick Piggin 已提交
2349
				load = source_load(i, load_idx);
L
Linus Torvalds 已提交
2350 2351

			avg_load += load;
2352
			sum_nr_running += rq->nr_running;
I
Ingo Molnar 已提交
2353
			sum_weighted_load += weighted_cpuload(i);
L
Linus Torvalds 已提交
2354 2355
		}

2356 2357 2358
		/*
		 * First idle cpu or the first cpu(busiest) in this sched group
		 * is eligible for doing load balancing at this and above
2359 2360
		 * domains. In the newly idle case, we will allow all the cpu's
		 * to do the newly idle load balance.
2361
		 */
2362 2363
		if (idle != CPU_NEWLY_IDLE && local_group &&
		    balance_cpu != this_cpu && balance) {
2364 2365 2366 2367
			*balance = 0;
			goto ret;
		}

L
Linus Torvalds 已提交
2368
		total_load += avg_load;
2369
		total_pwr += group->__cpu_power;
L
Linus Torvalds 已提交
2370 2371

		/* Adjust by relative CPU power of the group */
2372 2373
		avg_load = sg_div_cpu_power(group,
				avg_load * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2374

2375
		group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
2376

L
Linus Torvalds 已提交
2377 2378 2379
		if (local_group) {
			this_load = avg_load;
			this = group;
2380 2381 2382
			this_nr_running = sum_nr_running;
			this_load_per_task = sum_weighted_load;
		} else if (avg_load > max_load &&
2383
			   sum_nr_running > group_capacity) {
L
Linus Torvalds 已提交
2384 2385
			max_load = avg_load;
			busiest = group;
2386 2387
			busiest_nr_running = sum_nr_running;
			busiest_load_per_task = sum_weighted_load;
L
Linus Torvalds 已提交
2388
		}
2389 2390 2391 2392 2393 2394

#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
		/*
		 * Busy processors will not participate in power savings
		 * balance.
		 */
I
Ingo Molnar 已提交
2395 2396 2397
		if (idle == CPU_NOT_IDLE ||
				!(sd->flags & SD_POWERSAVINGS_BALANCE))
			goto group_next;
2398 2399 2400 2401 2402 2403 2404 2405 2406

		/*
		 * If the local group is idle or completely loaded
		 * no need to do power savings balance at this domain
		 */
		if (local_group && (this_nr_running >= group_capacity ||
				    !this_nr_running))
			power_savings_balance = 0;

I
Ingo Molnar 已提交
2407
		/*
2408 2409
		 * If a group is already running at full capacity or idle,
		 * don't include that group in power savings calculations
I
Ingo Molnar 已提交
2410 2411
		 */
		if (!power_savings_balance || sum_nr_running >= group_capacity
2412
		    || !sum_nr_running)
I
Ingo Molnar 已提交
2413
			goto group_next;
2414

I
Ingo Molnar 已提交
2415
		/*
2416
		 * Calculate the group which has the least non-idle load.
I
Ingo Molnar 已提交
2417 2418 2419 2420 2421
		 * This is the group from where we need to pick up the load
		 * for saving power
		 */
		if ((sum_nr_running < min_nr_running) ||
		    (sum_nr_running == min_nr_running &&
2422 2423
		     first_cpu(group->cpumask) <
		     first_cpu(group_min->cpumask))) {
I
Ingo Molnar 已提交
2424 2425
			group_min = group;
			min_nr_running = sum_nr_running;
2426 2427
			min_load_per_task = sum_weighted_load /
						sum_nr_running;
I
Ingo Molnar 已提交
2428
		}
2429

I
Ingo Molnar 已提交
2430
		/*
2431
		 * Calculate the group which is almost near its
I
Ingo Molnar 已提交
2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
		 * capacity but still has some space to pick up some load
		 * from other group and save more power
		 */
		if (sum_nr_running <= group_capacity - 1) {
			if (sum_nr_running > leader_nr_running ||
			    (sum_nr_running == leader_nr_running &&
			     first_cpu(group->cpumask) >
			      first_cpu(group_leader->cpumask))) {
				group_leader = group;
				leader_nr_running = sum_nr_running;
			}
2443
		}
2444 2445
group_next:
#endif
L
Linus Torvalds 已提交
2446 2447 2448
		group = group->next;
	} while (group != sd->groups);

2449
	if (!busiest || this_load >= max_load || busiest_nr_running == 0)
L
Linus Torvalds 已提交
2450 2451 2452 2453 2454 2455 2456 2457
		goto out_balanced;

	avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;

	if (this_load >= avg_load ||
			100*max_load <= sd->imbalance_pct*this_load)
		goto out_balanced;

2458
	busiest_load_per_task /= busiest_nr_running;
L
Linus Torvalds 已提交
2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
	/*
	 * We're trying to get all the cpus to the average_load, so we don't
	 * want to push ourselves above the average load, nor do we wish to
	 * reduce the max loaded cpu below the average load, as either of these
	 * actions would just result in more rebalancing later, and ping-pong
	 * tasks around. Thus we look for the minimum possible imbalance.
	 * Negative imbalances (*we* are more loaded than anyone else) will
	 * be counted as no imbalance for these purposes -- we can't fix that
	 * by pulling tasks to us.  Be careful of negative numbers as they'll
	 * appear as very large values with unsigned longs.
	 */
2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
	if (max_load <= busiest_load_per_task)
		goto out_balanced;

	/*
	 * In the presence of smp nice balancing, certain scenarios can have
	 * max load less than avg load(as we skip the groups at or below
	 * its cpu_power, while calculating max_load..)
	 */
	if (max_load < avg_load) {
		*imbalance = 0;
		goto small_imbalance;
	}
2482 2483

	/* Don't want to pull so many tasks that a group would go idle */
2484
	max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
2485

L
Linus Torvalds 已提交
2486
	/* How much load to actually move to equalise the imbalance */
2487 2488
	*imbalance = min(max_pull * busiest->__cpu_power,
				(avg_load - this_load) * this->__cpu_power)
L
Linus Torvalds 已提交
2489 2490
			/ SCHED_LOAD_SCALE;

2491 2492 2493 2494 2495 2496
	/*
	 * if *imbalance is less than the average load per runnable task
	 * there is no gaurantee that any tasks will be moved so we'll have
	 * a think about bumping its value to force at least one task to be
	 * moved
	 */
I
Ingo Molnar 已提交
2497
	if (*imbalance + SCHED_LOAD_SCALE_FUZZ < busiest_load_per_task/2) {
2498
		unsigned long tmp, pwr_now, pwr_move;
2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509
		unsigned int imbn;

small_imbalance:
		pwr_move = pwr_now = 0;
		imbn = 2;
		if (this_nr_running) {
			this_load_per_task /= this_nr_running;
			if (busiest_load_per_task > this_load_per_task)
				imbn = 1;
		} else
			this_load_per_task = SCHED_LOAD_SCALE;
L
Linus Torvalds 已提交
2510

I
Ingo Molnar 已提交
2511 2512
		if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >=
					busiest_load_per_task * imbn) {
2513
			*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2514 2515 2516 2517 2518 2519 2520 2521 2522
			return busiest;
		}

		/*
		 * OK, we don't have enough imbalance to justify moving tasks,
		 * however we may be able to increase total CPU power used by
		 * moving them.
		 */

2523 2524 2525 2526
		pwr_now += busiest->__cpu_power *
				min(busiest_load_per_task, max_load);
		pwr_now += this->__cpu_power *
				min(this_load_per_task, this_load);
L
Linus Torvalds 已提交
2527 2528 2529
		pwr_now /= SCHED_LOAD_SCALE;

		/* Amount of load we'd subtract */
2530 2531
		tmp = sg_div_cpu_power(busiest,
				busiest_load_per_task * SCHED_LOAD_SCALE);
L
Linus Torvalds 已提交
2532
		if (max_load > tmp)
2533
			pwr_move += busiest->__cpu_power *
2534
				min(busiest_load_per_task, max_load - tmp);
L
Linus Torvalds 已提交
2535 2536

		/* Amount of load we'd add */
2537
		if (max_load * busiest->__cpu_power <
2538
				busiest_load_per_task * SCHED_LOAD_SCALE)
2539 2540
			tmp = sg_div_cpu_power(this,
					max_load * busiest->__cpu_power);
L
Linus Torvalds 已提交
2541
		else
2542 2543 2544 2545
			tmp = sg_div_cpu_power(this,
				busiest_load_per_task * SCHED_LOAD_SCALE);
		pwr_move += this->__cpu_power *
				min(this_load_per_task, this_load + tmp);
L
Linus Torvalds 已提交
2546 2547 2548 2549 2550 2551
		pwr_move /= SCHED_LOAD_SCALE;

		/* Move if we gain throughput */
		if (pwr_move <= pwr_now)
			goto out_balanced;

2552
		*imbalance = busiest_load_per_task;
L
Linus Torvalds 已提交
2553 2554 2555 2556 2557
	}

	return busiest;

out_balanced:
2558
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
I
Ingo Molnar 已提交
2559
	if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
2560
		goto ret;
L
Linus Torvalds 已提交
2561

2562 2563 2564 2565 2566
	if (this == group_leader && group_leader != group_min) {
		*imbalance = min_load_per_task;
		return group_min;
	}
#endif
2567
ret:
L
Linus Torvalds 已提交
2568 2569 2570 2571 2572 2573 2574
	*imbalance = 0;
	return NULL;
}

/*
 * find_busiest_queue - find the busiest runqueue among the cpus in group.
 */
2575
static struct rq *
I
Ingo Molnar 已提交
2576
find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
2577
		   unsigned long imbalance, cpumask_t *cpus)
L
Linus Torvalds 已提交
2578
{
2579
	struct rq *busiest = NULL, *rq;
2580
	unsigned long max_load = 0;
L
Linus Torvalds 已提交
2581 2582 2583
	int i;

	for_each_cpu_mask(i, group->cpumask) {
I
Ingo Molnar 已提交
2584
		unsigned long wl;
2585 2586 2587 2588

		if (!cpu_isset(i, *cpus))
			continue;

2589
		rq = cpu_rq(i);
I
Ingo Molnar 已提交
2590
		wl = weighted_cpuload(i);
2591

I
Ingo Molnar 已提交
2592
		if (rq->nr_running == 1 && wl > imbalance)
2593
			continue;
L
Linus Torvalds 已提交
2594

I
Ingo Molnar 已提交
2595 2596
		if (wl > max_load) {
			max_load = wl;
2597
			busiest = rq;
L
Linus Torvalds 已提交
2598 2599 2600 2601 2602 2603
		}
	}

	return busiest;
}

2604 2605 2606 2607 2608 2609
/*
 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
 * so long as it is large enough.
 */
#define MAX_PINNED_INTERVAL	512

L
Linus Torvalds 已提交
2610 2611 2612 2613
/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 */
2614
static int load_balance(int this_cpu, struct rq *this_rq,
I
Ingo Molnar 已提交
2615
			struct sched_domain *sd, enum cpu_idle_type idle,
2616
			int *balance)
L
Linus Torvalds 已提交
2617
{
P
Peter Williams 已提交
2618
	int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
L
Linus Torvalds 已提交
2619 2620
	struct sched_group *group;
	unsigned long imbalance;
2621
	struct rq *busiest;
2622
	cpumask_t cpus = CPU_MASK_ALL;
2623
	unsigned long flags;
N
Nick Piggin 已提交
2624

2625 2626 2627
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
I
Ingo Molnar 已提交
2628
	 * let the state of idle sibling percolate up as CPU_IDLE, instead of
I
Ingo Molnar 已提交
2629
	 * portraying it as CPU_NOT_IDLE.
2630
	 */
I
Ingo Molnar 已提交
2631
	if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
2632
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2633
		sd_idle = 1;
L
Linus Torvalds 已提交
2634 2635 2636

	schedstat_inc(sd, lb_cnt[idle]);

2637 2638
redo:
	group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
2639 2640
				   &cpus, balance);

2641
	if (*balance == 0)
2642 2643
		goto out_balanced;

L
Linus Torvalds 已提交
2644 2645 2646 2647 2648
	if (!group) {
		schedstat_inc(sd, lb_nobusyg[idle]);
		goto out_balanced;
	}

2649
	busiest = find_busiest_queue(group, idle, imbalance, &cpus);
L
Linus Torvalds 已提交
2650 2651 2652 2653 2654
	if (!busiest) {
		schedstat_inc(sd, lb_nobusyq[idle]);
		goto out_balanced;
	}

N
Nick Piggin 已提交
2655
	BUG_ON(busiest == this_rq);
L
Linus Torvalds 已提交
2656 2657 2658

	schedstat_add(sd, lb_imbalance[idle], imbalance);

P
Peter Williams 已提交
2659
	ld_moved = 0;
L
Linus Torvalds 已提交
2660 2661 2662 2663
	if (busiest->nr_running > 1) {
		/*
		 * Attempt to move tasks. If find_busiest_group has found
		 * an imbalance but busiest->nr_running <= 1, the group is
P
Peter Williams 已提交
2664
		 * still unbalanced. ld_moved simply stays zero, so it is
L
Linus Torvalds 已提交
2665 2666
		 * correctly treated as an imbalance.
		 */
2667
		local_irq_save(flags);
N
Nick Piggin 已提交
2668
		double_rq_lock(this_rq, busiest);
P
Peter Williams 已提交
2669
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2670
				      imbalance, sd, idle, &all_pinned);
N
Nick Piggin 已提交
2671
		double_rq_unlock(this_rq, busiest);
2672
		local_irq_restore(flags);
2673

2674 2675 2676
		/*
		 * some other cpu did the load balance for us.
		 */
P
Peter Williams 已提交
2677
		if (ld_moved && this_cpu != smp_processor_id())
2678 2679
			resched_cpu(this_cpu);

2680
		/* All tasks on this runqueue were pinned by CPU affinity */
2681 2682 2683 2684
		if (unlikely(all_pinned)) {
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
2685
			goto out_balanced;
2686
		}
L
Linus Torvalds 已提交
2687
	}
2688

P
Peter Williams 已提交
2689
	if (!ld_moved) {
L
Linus Torvalds 已提交
2690 2691 2692 2693 2694
		schedstat_inc(sd, lb_failed[idle]);
		sd->nr_balance_failed++;

		if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {

2695
			spin_lock_irqsave(&busiest->lock, flags);
2696 2697 2698 2699 2700

			/* don't kick the migration_thread, if the curr
			 * task on busiest cpu can't be moved to this_cpu
			 */
			if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
2701
				spin_unlock_irqrestore(&busiest->lock, flags);
2702 2703 2704 2705
				all_pinned = 1;
				goto out_one_pinned;
			}

L
Linus Torvalds 已提交
2706 2707 2708
			if (!busiest->active_balance) {
				busiest->active_balance = 1;
				busiest->push_cpu = this_cpu;
2709
				active_balance = 1;
L
Linus Torvalds 已提交
2710
			}
2711
			spin_unlock_irqrestore(&busiest->lock, flags);
2712
			if (active_balance)
L
Linus Torvalds 已提交
2713 2714 2715 2716 2717 2718
				wake_up_process(busiest->migration_thread);

			/*
			 * We've kicked active balancing, reset the failure
			 * counter.
			 */
2719
			sd->nr_balance_failed = sd->cache_nice_tries+1;
L
Linus Torvalds 已提交
2720
		}
2721
	} else
L
Linus Torvalds 已提交
2722 2723
		sd->nr_balance_failed = 0;

2724
	if (likely(!active_balance)) {
L
Linus Torvalds 已提交
2725 2726
		/* We were unbalanced, so reset the balancing interval */
		sd->balance_interval = sd->min_interval;
2727 2728 2729 2730 2731 2732 2733 2734 2735
	} else {
		/*
		 * If we've begun active balancing, start to back off. This
		 * case may not be covered by the all_pinned logic if there
		 * is only 1 task on the busy runqueue (because we don't call
		 * move_tasks).
		 */
		if (sd->balance_interval < sd->max_interval)
			sd->balance_interval *= 2;
L
Linus Torvalds 已提交
2736 2737
	}

P
Peter Williams 已提交
2738
	if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2739
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2740
		return -1;
P
Peter Williams 已提交
2741
	return ld_moved;
L
Linus Torvalds 已提交
2742 2743 2744 2745

out_balanced:
	schedstat_inc(sd, lb_balanced[idle]);

2746
	sd->nr_balance_failed = 0;
2747 2748

out_one_pinned:
L
Linus Torvalds 已提交
2749
	/* tune up the balancing interval */
2750 2751
	if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
			(sd->balance_interval < sd->max_interval))
L
Linus Torvalds 已提交
2752 2753
		sd->balance_interval *= 2;

2754
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2755
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2756
		return -1;
L
Linus Torvalds 已提交
2757 2758 2759 2760 2761 2762 2763
	return 0;
}

/*
 * Check this_cpu to ensure it is balanced within domain. Attempt to move
 * tasks if there is an imbalance.
 *
I
Ingo Molnar 已提交
2764
 * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
L
Linus Torvalds 已提交
2765 2766
 * this_rq is locked.
 */
2767
static int
2768
load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
L
Linus Torvalds 已提交
2769 2770
{
	struct sched_group *group;
2771
	struct rq *busiest = NULL;
L
Linus Torvalds 已提交
2772
	unsigned long imbalance;
P
Peter Williams 已提交
2773
	int ld_moved = 0;
N
Nick Piggin 已提交
2774
	int sd_idle = 0;
2775
	int all_pinned = 0;
2776
	cpumask_t cpus = CPU_MASK_ALL;
N
Nick Piggin 已提交
2777

2778 2779 2780 2781
	/*
	 * When power savings policy is enabled for the parent domain, idle
	 * sibling can pick up load irrespective of busy siblings. In this case,
	 * let the state of idle sibling percolate up as IDLE, instead of
I
Ingo Molnar 已提交
2782
	 * portraying it as CPU_NOT_IDLE.
2783 2784 2785
	 */
	if (sd->flags & SD_SHARE_CPUPOWER &&
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2786
		sd_idle = 1;
L
Linus Torvalds 已提交
2787

I
Ingo Molnar 已提交
2788
	schedstat_inc(sd, lb_cnt[CPU_NEWLY_IDLE]);
2789
redo:
I
Ingo Molnar 已提交
2790
	group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
2791
				   &sd_idle, &cpus, NULL);
L
Linus Torvalds 已提交
2792
	if (!group) {
I
Ingo Molnar 已提交
2793
		schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
2794
		goto out_balanced;
L
Linus Torvalds 已提交
2795 2796
	}

I
Ingo Molnar 已提交
2797
	busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance,
2798
				&cpus);
N
Nick Piggin 已提交
2799
	if (!busiest) {
I
Ingo Molnar 已提交
2800
		schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
2801
		goto out_balanced;
L
Linus Torvalds 已提交
2802 2803
	}

N
Nick Piggin 已提交
2804 2805
	BUG_ON(busiest == this_rq);

I
Ingo Molnar 已提交
2806
	schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
2807

P
Peter Williams 已提交
2808
	ld_moved = 0;
2809 2810 2811
	if (busiest->nr_running > 1) {
		/* Attempt to move tasks */
		double_lock_balance(this_rq, busiest);
2812 2813
		/* this_rq->clock is already updated */
		update_rq_clock(busiest);
P
Peter Williams 已提交
2814
		ld_moved = move_tasks(this_rq, this_cpu, busiest,
2815 2816
					imbalance, sd, CPU_NEWLY_IDLE,
					&all_pinned);
2817
		spin_unlock(&busiest->lock);
2818

2819
		if (unlikely(all_pinned)) {
2820 2821 2822 2823
			cpu_clear(cpu_of(busiest), cpus);
			if (!cpus_empty(cpus))
				goto redo;
		}
2824 2825
	}

P
Peter Williams 已提交
2826
	if (!ld_moved) {
I
Ingo Molnar 已提交
2827
		schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
2828 2829
		if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
		    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2830 2831
			return -1;
	} else
2832
		sd->nr_balance_failed = 0;
L
Linus Torvalds 已提交
2833

P
Peter Williams 已提交
2834
	return ld_moved;
2835 2836

out_balanced:
I
Ingo Molnar 已提交
2837
	schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
2838
	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2839
	    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
N
Nick Piggin 已提交
2840
		return -1;
2841
	sd->nr_balance_failed = 0;
2842

2843
	return 0;
L
Linus Torvalds 已提交
2844 2845 2846 2847 2848 2849
}

/*
 * idle_balance is called by schedule() if this_cpu is about to become
 * idle. Attempts to pull tasks from other CPUs.
 */
2850
static void idle_balance(int this_cpu, struct rq *this_rq)
L
Linus Torvalds 已提交
2851 2852
{
	struct sched_domain *sd;
I
Ingo Molnar 已提交
2853 2854
	int pulled_task = -1;
	unsigned long next_balance = jiffies + HZ;
L
Linus Torvalds 已提交
2855 2856

	for_each_domain(this_cpu, sd) {
2857 2858 2859 2860 2861 2862
		unsigned long interval;

		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		if (sd->flags & SD_BALANCE_NEWIDLE)
2863
			/* If we've pulled tasks over stop searching: */
2864
			pulled_task = load_balance_newidle(this_cpu,
2865 2866 2867 2868 2869 2870 2871
								this_rq, sd);

		interval = msecs_to_jiffies(sd->balance_interval);
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
		if (pulled_task)
			break;
L
Linus Torvalds 已提交
2872
	}
I
Ingo Molnar 已提交
2873
	if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
2874 2875 2876 2877 2878
		/*
		 * We are going idle. next_balance may be set based on
		 * a busy processor. So reset next_balance.
		 */
		this_rq->next_balance = next_balance;
I
Ingo Molnar 已提交
2879
	}
L
Linus Torvalds 已提交
2880 2881 2882 2883 2884 2885 2886 2887 2888 2889
}

/*
 * active_load_balance is run by migration threads. It pushes running tasks
 * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
 * running on each physical CPU where possible, and avoids physical /
 * logical imbalances.
 *
 * Called with busiest_rq locked.
 */
2890
static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
L
Linus Torvalds 已提交
2891
{
2892
	int target_cpu = busiest_rq->push_cpu;
2893 2894
	struct sched_domain *sd;
	struct rq *target_rq;
2895

2896
	/* Is there any task to move? */
2897 2898 2899 2900
	if (busiest_rq->nr_running <= 1)
		return;

	target_rq = cpu_rq(target_cpu);
L
Linus Torvalds 已提交
2901 2902

	/*
2903 2904 2905
	 * This condition is "impossible", if it occurs
	 * we need to fix it.  Originally reported by
	 * Bjorn Helgaas on a 128-cpu setup.
L
Linus Torvalds 已提交
2906
	 */
2907
	BUG_ON(busiest_rq == target_rq);
L
Linus Torvalds 已提交
2908

2909 2910
	/* move a task from busiest_rq to target_rq */
	double_lock_balance(busiest_rq, target_rq);
2911 2912
	update_rq_clock(busiest_rq);
	update_rq_clock(target_rq);
2913 2914

	/* Search for an sd spanning us and the target CPU. */
2915
	for_each_domain(target_cpu, sd) {
2916
		if ((sd->flags & SD_LOAD_BALANCE) &&
2917
		    cpu_isset(busiest_cpu, sd->span))
2918
				break;
2919
	}
2920

2921 2922
	if (likely(sd)) {
		schedstat_inc(sd, alb_cnt);
2923

P
Peter Williams 已提交
2924 2925
		if (move_one_task(target_rq, target_cpu, busiest_rq,
				  sd, CPU_IDLE))
2926 2927 2928 2929
			schedstat_inc(sd, alb_pushed);
		else
			schedstat_inc(sd, alb_failed);
	}
2930
	spin_unlock(&target_rq->lock);
L
Linus Torvalds 已提交
2931 2932
}

2933 2934 2935 2936 2937 2938 2939 2940 2941
#ifdef CONFIG_NO_HZ
static struct {
	atomic_t load_balancer;
	cpumask_t  cpu_mask;
} nohz ____cacheline_aligned = {
	.load_balancer = ATOMIC_INIT(-1),
	.cpu_mask = CPU_MASK_NONE,
};

2942
/*
2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
 * This routine will try to nominate the ilb (idle load balancing)
 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
 * load balancing on behalf of all those cpus. If all the cpus in the system
 * go into this tickless mode, then there will be no ilb owner (as there is
 * no need for one) and all the cpus will sleep till the next wakeup event
 * arrives...
 *
 * For the ilb owner, tick is not stopped. And this tick will be used
 * for idle load balancing. ilb owner will still be part of
 * nohz.cpu_mask..
2953
 *
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
 * While stopping the tick, this cpu will become the ilb owner if there
 * is no other owner. And will be the owner till that cpu becomes busy
 * or if all cpus in the system stop their ticks at which point
 * there is no need for ilb owner.
 *
 * When the ilb owner becomes busy, it nominates another owner, during the
 * next busy scheduler_tick()
 */
int select_nohz_load_balancer(int stop_tick)
{
	int cpu = smp_processor_id();

	if (stop_tick) {
		cpu_set(cpu, nohz.cpu_mask);
		cpu_rq(cpu)->in_nohz_recently = 1;

		/*
		 * If we are going offline and still the leader, give up!
		 */
		if (cpu_is_offline(cpu) &&
		    atomic_read(&nohz.load_balancer) == cpu) {
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
			return 0;
		}

		/* time for ilb owner also to sleep */
		if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
			if (atomic_read(&nohz.load_balancer) == cpu)
				atomic_set(&nohz.load_balancer, -1);
			return 0;
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/* make me the ilb owner */
			if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
				return 1;
		} else if (atomic_read(&nohz.load_balancer) == cpu)
			return 1;
	} else {
		if (!cpu_isset(cpu, nohz.cpu_mask))
			return 0;

		cpu_clear(cpu, nohz.cpu_mask);

		if (atomic_read(&nohz.load_balancer) == cpu)
			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
				BUG();
	}
	return 0;
}
#endif

static DEFINE_SPINLOCK(balancing);

/*
3010 3011 3012 3013 3014
 * It checks each scheduling domain to see if it is due to be balanced,
 * and initiates a balancing operation if so.
 *
 * Balancing parameters are set up in arch_init_sched_domains.
 */
I
Ingo Molnar 已提交
3015
static inline void rebalance_domains(int cpu, enum cpu_idle_type idle)
3016
{
3017 3018
	int balance = 1;
	struct rq *rq = cpu_rq(cpu);
3019 3020
	unsigned long interval;
	struct sched_domain *sd;
3021
	/* Earliest time when we have to do rebalance again */
3022
	unsigned long next_balance = jiffies + 60*HZ;
L
Linus Torvalds 已提交
3023

3024
	for_each_domain(cpu, sd) {
L
Linus Torvalds 已提交
3025 3026 3027 3028
		if (!(sd->flags & SD_LOAD_BALANCE))
			continue;

		interval = sd->balance_interval;
I
Ingo Molnar 已提交
3029
		if (idle != CPU_IDLE)
L
Linus Torvalds 已提交
3030 3031 3032 3033 3034 3035
			interval *= sd->busy_factor;

		/* scale ms to jiffies */
		interval = msecs_to_jiffies(interval);
		if (unlikely(!interval))
			interval = 1;
I
Ingo Molnar 已提交
3036 3037 3038
		if (interval > HZ*NR_CPUS/10)
			interval = HZ*NR_CPUS/10;

L
Linus Torvalds 已提交
3039

3040 3041 3042 3043 3044
		if (sd->flags & SD_SERIALIZE) {
			if (!spin_trylock(&balancing))
				goto out;
		}

3045
		if (time_after_eq(jiffies, sd->last_balance + interval)) {
3046
			if (load_balance(cpu, rq, sd, idle, &balance)) {
3047 3048
				/*
				 * We've pulled tasks over so either we're no
N
Nick Piggin 已提交
3049 3050 3051
				 * longer idle, or one of our SMT siblings is
				 * not idle.
				 */
I
Ingo Molnar 已提交
3052
				idle = CPU_NOT_IDLE;
L
Linus Torvalds 已提交
3053
			}
3054
			sd->last_balance = jiffies;
L
Linus Torvalds 已提交
3055
		}
3056 3057 3058
		if (sd->flags & SD_SERIALIZE)
			spin_unlock(&balancing);
out:
3059 3060
		if (time_after(next_balance, sd->last_balance + interval))
			next_balance = sd->last_balance + interval;
3061 3062 3063 3064 3065 3066 3067 3068

		/*
		 * Stop the load balance at this level. There is another
		 * CPU in our sched group which is doing load balancing more
		 * actively.
		 */
		if (!balance)
			break;
L
Linus Torvalds 已提交
3069
	}
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079
	rq->next_balance = next_balance;
}

/*
 * run_rebalance_domains is triggered when needed from the scheduler tick.
 * In CONFIG_NO_HZ case, the idle load balance owner will do the
 * rebalancing for all the cpus for whom scheduler ticks are stopped.
 */
static void run_rebalance_domains(struct softirq_action *h)
{
I
Ingo Molnar 已提交
3080 3081 3082 3083
	int this_cpu = smp_processor_id();
	struct rq *this_rq = cpu_rq(this_cpu);
	enum cpu_idle_type idle = this_rq->idle_at_tick ?
						CPU_IDLE : CPU_NOT_IDLE;
3084

I
Ingo Molnar 已提交
3085
	rebalance_domains(this_cpu, idle);
3086 3087 3088 3089 3090 3091 3092

#ifdef CONFIG_NO_HZ
	/*
	 * If this cpu is the owner for idle load balancing, then do the
	 * balancing on behalf of the other idle cpus whose ticks are
	 * stopped.
	 */
I
Ingo Molnar 已提交
3093 3094
	if (this_rq->idle_at_tick &&
	    atomic_read(&nohz.load_balancer) == this_cpu) {
3095 3096 3097 3098
		cpumask_t cpus = nohz.cpu_mask;
		struct rq *rq;
		int balance_cpu;

I
Ingo Molnar 已提交
3099
		cpu_clear(this_cpu, cpus);
3100 3101 3102 3103 3104 3105 3106 3107 3108
		for_each_cpu_mask(balance_cpu, cpus) {
			/*
			 * If this cpu gets work to do, stop the load balancing
			 * work being done for other cpus. Next load
			 * balancing owner will pick it up.
			 */
			if (need_resched())
				break;

3109
			rebalance_domains(balance_cpu, CPU_IDLE);
3110 3111

			rq = cpu_rq(balance_cpu);
I
Ingo Molnar 已提交
3112 3113
			if (time_after(this_rq->next_balance, rq->next_balance))
				this_rq->next_balance = rq->next_balance;
3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125
		}
	}
#endif
}

/*
 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
 *
 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
 * idle load balancing owner or decide to stop the periodic load balancing,
 * if the whole system is idle.
 */
I
Ingo Molnar 已提交
3126
static inline void trigger_load_balance(struct rq *rq, int cpu)
3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177
{
#ifdef CONFIG_NO_HZ
	/*
	 * If we were in the nohz mode recently and busy at the current
	 * scheduler tick, then check if we need to nominate new idle
	 * load balancer.
	 */
	if (rq->in_nohz_recently && !rq->idle_at_tick) {
		rq->in_nohz_recently = 0;

		if (atomic_read(&nohz.load_balancer) == cpu) {
			cpu_clear(cpu, nohz.cpu_mask);
			atomic_set(&nohz.load_balancer, -1);
		}

		if (atomic_read(&nohz.load_balancer) == -1) {
			/*
			 * simple selection for now: Nominate the
			 * first cpu in the nohz list to be the next
			 * ilb owner.
			 *
			 * TBD: Traverse the sched domains and nominate
			 * the nearest cpu in the nohz.cpu_mask.
			 */
			int ilb = first_cpu(nohz.cpu_mask);

			if (ilb != NR_CPUS)
				resched_cpu(ilb);
		}
	}

	/*
	 * If this cpu is idle and doing idle load balancing for all the
	 * cpus with ticks stopped, is it time for that to stop?
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
	    cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
		resched_cpu(cpu);
		return;
	}

	/*
	 * If this cpu is idle and the idle load balancing is done by
	 * someone else, then no need raise the SCHED_SOFTIRQ
	 */
	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
	    cpu_isset(cpu, nohz.cpu_mask))
		return;
#endif
	if (time_after_eq(jiffies, rq->next_balance))
		raise_softirq(SCHED_SOFTIRQ);
L
Linus Torvalds 已提交
3178
}
I
Ingo Molnar 已提交
3179 3180 3181

#else	/* CONFIG_SMP */

L
Linus Torvalds 已提交
3182 3183 3184
/*
 * on UP we do not need to balance between CPUs:
 */
3185
static inline void idle_balance(int cpu, struct rq *rq)
L
Linus Torvalds 已提交
3186 3187
{
}
I
Ingo Molnar 已提交
3188 3189 3190 3191 3192 3193

/* Avoid "used but not defined" warning on UP */
static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
		      unsigned long max_nr_move, unsigned long max_load_move,
		      struct sched_domain *sd, enum cpu_idle_type idle,
		      int *all_pinned, unsigned long *load_moved,
3194
		      int *this_best_prio, struct rq_iterator *iterator)
I
Ingo Molnar 已提交
3195 3196 3197 3198 3199 3200
{
	*load_moved = 0;

	return 0;
}

L
Linus Torvalds 已提交
3201 3202 3203 3204 3205 3206 3207
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);

EXPORT_PER_CPU_SYMBOL(kstat);

/*
3208 3209
 * Return p->sum_exec_runtime plus any more ns on the sched_clock
 * that have not yet been banked in case the task is currently running.
L
Linus Torvalds 已提交
3210
 */
3211
unsigned long long task_sched_runtime(struct task_struct *p)
L
Linus Torvalds 已提交
3212 3213
{
	unsigned long flags;
3214 3215
	u64 ns, delta_exec;
	struct rq *rq;
3216

3217 3218 3219
	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime;
	if (rq->curr == p) {
I
Ingo Molnar 已提交
3220 3221
		update_rq_clock(rq);
		delta_exec = rq->clock - p->se.exec_start;
3222 3223 3224 3225
		if ((s64)delta_exec > 0)
			ns += delta_exec;
	}
	task_rq_unlock(rq, &flags);
3226

L
Linus Torvalds 已提交
3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260
	return ns;
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in user space since the last update
 */
void account_user_time(struct task_struct *p, cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp;

	p->utime = cputime_add(p->utime, cputime);

	/* Add user time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (TASK_NICE(p) > 0)
		cpustat->nice = cputime64_add(cpustat->nice, tmp);
	else
		cpustat->user = cputime64_add(cpustat->user, tmp);
}

/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
 */
void account_system_time(struct task_struct *p, int hardirq_offset,
			 cputime_t cputime)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3261
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
	cputime64_t tmp;

	p->stime = cputime_add(p->stime, cputime);

	/* Add system time to cpustat. */
	tmp = cputime_to_cputime64(cputime);
	if (hardirq_count() - hardirq_offset)
		cpustat->irq = cputime64_add(cpustat->irq, tmp);
	else if (softirq_count())
		cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
	else if (p != rq->idle)
		cpustat->system = cputime64_add(cpustat->system, tmp);
	else if (atomic_read(&rq->nr_iowait) > 0)
		cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
	else
		cpustat->idle = cputime64_add(cpustat->idle, tmp);
	/* Account for system time used */
	acct_update_integrals(p);
}

/*
 * Account for involuntary wait time.
 * @p: the process from which the cpu time has been stolen
 * @steal: the cpu time spent in involuntary wait
 */
void account_steal_time(struct task_struct *p, cputime_t steal)
{
	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
	cputime64_t tmp = cputime_to_cputime64(steal);
3291
	struct rq *rq = this_rq();
L
Linus Torvalds 已提交
3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302

	if (p == rq->idle) {
		p->stime = cputime_add(p->stime, steal);
		if (atomic_read(&rq->nr_iowait) > 0)
			cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
		else
			cpustat->idle = cputime64_add(cpustat->idle, tmp);
	} else
		cpustat->steal = cputime64_add(cpustat->steal, tmp);
}

3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 *
 * It also gets called by the fork code, when changing the parent's
 * timeslices.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
3314
	struct task_struct *curr = rq->curr;
3315
	u64 next_tick = rq->tick_timestamp + TICK_NSEC;
I
Ingo Molnar 已提交
3316 3317

	spin_lock(&rq->lock);
3318
	__update_rq_clock(rq);
3319 3320 3321 3322 3323 3324
	/*
	 * Let rq->clock advance by at least TICK_NSEC:
	 */
	if (unlikely(rq->clock < next_tick))
		rq->clock = next_tick;
	rq->tick_timestamp = rq->clock;
3325
	update_cpu_load(rq);
I
Ingo Molnar 已提交
3326 3327 3328
	if (curr != rq->idle) /* FIXME: needed? */
		curr->sched_class->task_tick(rq, curr);
	spin_unlock(&rq->lock);
3329

3330
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
3331 3332
	rq->idle_at_tick = idle_cpu(cpu);
	trigger_load_balance(rq, cpu);
3333
#endif
L
Linus Torvalds 已提交
3334 3335 3336 3337 3338 3339 3340 3341 3342
}

#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)

void fastcall add_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3343 3344
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
L
Linus Torvalds 已提交
3345 3346 3347 3348
	preempt_count() += val;
	/*
	 * Spinlock count overflowing soon?
	 */
3349 3350
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
L
Linus Torvalds 已提交
3351 3352 3353 3354 3355 3356 3357 3358
}
EXPORT_SYMBOL(add_preempt_count);

void fastcall sub_preempt_count(int val)
{
	/*
	 * Underflow?
	 */
3359 3360
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
		return;
L
Linus Torvalds 已提交
3361 3362 3363
	/*
	 * Is the spinlock portion underflowing?
	 */
3364 3365 3366 3367
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;

L
Linus Torvalds 已提交
3368 3369 3370 3371 3372 3373 3374
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
3375
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
3376
 */
I
Ingo Molnar 已提交
3377
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
3378
{
I
Ingo Molnar 已提交
3379 3380 3381 3382 3383 3384 3385
	printk(KERN_ERR "BUG: scheduling while atomic: %s/0x%08x/%d\n",
		prev->comm, preempt_count(), prev->pid);
	debug_show_held_locks(prev);
	if (irqs_disabled())
		print_irqtrace_events(prev);
	dump_stack();
}
L
Linus Torvalds 已提交
3386

I
Ingo Molnar 已提交
3387 3388 3389 3390 3391
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
3392 3393 3394 3395 3396
	/*
	 * Test if we are atomic.  Since do_exit() needs to call into
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
I
Ingo Molnar 已提交
3397 3398 3399
	if (unlikely(in_atomic_preempt_off()) && unlikely(!prev->exit_state))
		__schedule_bug(prev);

L
Linus Torvalds 已提交
3400 3401
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

I
Ingo Molnar 已提交
3402 3403 3404 3405 3406 3407 3408
	schedstat_inc(this_rq(), sched_cnt);
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
3409
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
3410 3411 3412
{
	struct sched_class *class;
	struct task_struct *p;
L
Linus Torvalds 已提交
3413 3414

	/*
I
Ingo Molnar 已提交
3415 3416
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
3417
	 */
I
Ingo Molnar 已提交
3418
	if (likely(rq->nr_running == rq->cfs.nr_running)) {
3419
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
3420 3421
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
3422 3423
	}

I
Ingo Molnar 已提交
3424 3425
	class = sched_class_highest;
	for ( ; ; ) {
3426
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
3427 3428 3429 3430 3431 3432 3433 3434 3435
		if (p)
			return p;
		/*
		 * Will never be NULL as the idle class always
		 * returns a non-NULL p:
		 */
		class = class->next;
	}
}
L
Linus Torvalds 已提交
3436

I
Ingo Molnar 已提交
3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458
/*
 * schedule() is the main scheduler function.
 */
asmlinkage void __sched schedule(void)
{
	struct task_struct *prev, *next;
	long *switch_count;
	struct rq *rq;
	int cpu;

need_resched:
	preempt_disable();
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
	rcu_qsctr_inc(cpu);
	prev = rq->curr;
	switch_count = &prev->nivcsw;

	release_kernel_lock(prev);
need_resched_nonpreemptible:

	schedule_debug(prev);
L
Linus Torvalds 已提交
3459 3460

	spin_lock_irq(&rq->lock);
I
Ingo Molnar 已提交
3461
	clear_tsk_need_resched(prev);
I
Ingo Molnar 已提交
3462
	__update_rq_clock(rq);
L
Linus Torvalds 已提交
3463 3464 3465

	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
		if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
I
Ingo Molnar 已提交
3466
				unlikely(signal_pending(prev)))) {
L
Linus Torvalds 已提交
3467
			prev->state = TASK_RUNNING;
I
Ingo Molnar 已提交
3468
		} else {
3469
			deactivate_task(rq, prev, 1);
L
Linus Torvalds 已提交
3470
		}
I
Ingo Molnar 已提交
3471
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
3472 3473
	}

I
Ingo Molnar 已提交
3474
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
3475 3476
		idle_balance(cpu, rq);

3477
	prev->sched_class->put_prev_task(rq, prev);
3478
	next = pick_next_task(rq, prev);
L
Linus Torvalds 已提交
3479 3480

	sched_info_switch(prev, next);
I
Ingo Molnar 已提交
3481

L
Linus Torvalds 已提交
3482 3483 3484 3485 3486
	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
3487
		context_switch(rq, prev, next); /* unlocks the rq */
L
Linus Torvalds 已提交
3488 3489 3490
	} else
		spin_unlock_irq(&rq->lock);

I
Ingo Molnar 已提交
3491 3492 3493
	if (unlikely(reacquire_kernel_lock(current) < 0)) {
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
3494
		goto need_resched_nonpreemptible;
I
Ingo Molnar 已提交
3495
	}
L
Linus Torvalds 已提交
3496 3497 3498 3499 3500 3501 3502 3503
	preempt_enable_no_resched();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(schedule);

#ifdef CONFIG_PREEMPT
/*
3504
 * this is the entry point to schedule() from in-kernel preemption
L
Linus Torvalds 已提交
3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518
 * off of preempt_enable.  Kernel preemptions off return from interrupt
 * occur there and call schedule directly.
 */
asmlinkage void __sched preempt_schedule(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
	 * we do not want to preempt the current task.  Just return..
	 */
N
Nick Piggin 已提交
3519
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546
		return;

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	schedule();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}
EXPORT_SYMBOL(preempt_schedule);

/*
3547
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
#ifdef CONFIG_PREEMPT_BKL
	struct task_struct *task = current;
	int saved_lock_depth;
#endif
3559
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588
	BUG_ON(ti->preempt_count || !irqs_disabled());

need_resched:
	add_preempt_count(PREEMPT_ACTIVE);
	/*
	 * We keep the big kernel semaphore locked, but we
	 * clear ->lock_depth so that schedule() doesnt
	 * auto-release the semaphore:
	 */
#ifdef CONFIG_PREEMPT_BKL
	saved_lock_depth = task->lock_depth;
	task->lock_depth = -1;
#endif
	local_irq_enable();
	schedule();
	local_irq_disable();
#ifdef CONFIG_PREEMPT_BKL
	task->lock_depth = saved_lock_depth;
#endif
	sub_preempt_count(PREEMPT_ACTIVE);

	/* we could miss a preemption opportunity between schedule and now */
	barrier();
	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
		goto need_resched;
}

#endif /* CONFIG_PREEMPT */

I
Ingo Molnar 已提交
3589 3590
int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
			  void *key)
L
Linus Torvalds 已提交
3591
{
3592
	return try_to_wake_up(curr->private, mode, sync);
L
Linus Torvalds 已提交
3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610
}
EXPORT_SYMBOL(default_wake_function);

/*
 * The core wakeup function.  Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up.  If it's an exclusive wakeup (nr_exclusive == small +ve
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
 * started to run but is not in state TASK_RUNNING.  try_to_wake_up() returns
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
			     int nr_exclusive, int sync, void *key)
{
	struct list_head *tmp, *next;

	list_for_each_safe(tmp, next, &q->task_list) {
3611 3612 3613
		wait_queue_t *curr = list_entry(tmp, wait_queue_t, task_list);
		unsigned flags = curr->flags;

L
Linus Torvalds 已提交
3614
		if (curr->func(curr, mode, sync, key) &&
3615
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
3616 3617 3618 3619 3620 3621 3622 3623 3624
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
3625
 * @key: is directly passed to the wakeup function
L
Linus Torvalds 已提交
3626 3627
 */
void fastcall __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
3628
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
void fastcall __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
{
	__wake_up_common(q, mode, 1, 0, NULL);
}

/**
3647
 * __wake_up_sync - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
 */
I
Ingo Molnar 已提交
3659 3660
void fastcall
__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
L
Linus Torvalds 已提交
3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703
{
	unsigned long flags;
	int sync = 1;

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
		sync = 0;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, sync, NULL);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

void fastcall complete(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 1, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

void fastcall complete_all(struct completion *x)
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
	__wake_up_common(&x->wait, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE,
			 0, 0, NULL);
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

void fastcall __sched wait_for_completion(struct completion *x)
{
	might_sleep();
3704

L
Linus Torvalds 已提交
3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
	spin_unlock_irq(&x->wait.lock);
}
EXPORT_SYMBOL(wait_for_completion);

unsigned long fastcall __sched
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			__set_current_state(TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_timeout);

int fastcall __sched wait_for_completion_interruptible(struct completion *x)
{
	int ret = 0;

	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				ret = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			schedule();
			spin_lock_irq(&x->wait.lock);
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);

	return ret;
}
EXPORT_SYMBOL(wait_for_completion_interruptible);

unsigned long fastcall __sched
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

		wait.flags |= WQ_FLAG_EXCLUSIVE;
		__add_wait_queue_tail(&x->wait, &wait);
		do {
			if (signal_pending(current)) {
				timeout = -ERESTARTSYS;
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
			__set_current_state(TASK_INTERRUPTIBLE);
			spin_unlock_irq(&x->wait.lock);
			timeout = schedule_timeout(timeout);
			spin_lock_irq(&x->wait.lock);
			if (!timeout) {
				__remove_wait_queue(&x->wait, &wait);
				goto out;
			}
		} while (!x->done);
		__remove_wait_queue(&x->wait, &wait);
	}
	x->done--;
out:
	spin_unlock_irq(&x->wait.lock);
	return timeout;
}
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);

I
Ingo Molnar 已提交
3823 3824 3825 3826 3827
static inline void
sleep_on_head(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irqsave(&q->lock, *flags);
	__add_wait_queue(q, wait);
L
Linus Torvalds 已提交
3828
	spin_unlock(&q->lock);
I
Ingo Molnar 已提交
3829
}
L
Linus Torvalds 已提交
3830

I
Ingo Molnar 已提交
3831 3832 3833 3834 3835 3836 3837
static inline void
sleep_on_tail(wait_queue_head_t *q, wait_queue_t *wait, unsigned long *flags)
{
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, wait);
	spin_unlock_irqrestore(&q->lock, *flags);
}
L
Linus Torvalds 已提交
3838

I
Ingo Molnar 已提交
3839
void __sched interruptible_sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3840
{
I
Ingo Molnar 已提交
3841 3842 3843 3844
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3845 3846 3847

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3848
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3849
	schedule();
I
Ingo Molnar 已提交
3850
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3851 3852 3853
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
3854
long __sched
I
Ingo Molnar 已提交
3855
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3856
{
I
Ingo Molnar 已提交
3857 3858 3859 3860
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3861 3862 3863

	current->state = TASK_INTERRUPTIBLE;

I
Ingo Molnar 已提交
3864
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3865
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3866
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3867 3868 3869 3870 3871

	return timeout;
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
3872
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3873
{
I
Ingo Molnar 已提交
3874 3875 3876 3877
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3878 3879 3880

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3881
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3882
	schedule();
I
Ingo Molnar 已提交
3883
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3884 3885 3886
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
3887
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3888
{
I
Ingo Molnar 已提交
3889 3890 3891 3892
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3893 3894 3895

	current->state = TASK_UNINTERRUPTIBLE;

I
Ingo Molnar 已提交
3896
	sleep_on_head(q, &wait, &flags);
L
Linus Torvalds 已提交
3897
	timeout = schedule_timeout(timeout);
I
Ingo Molnar 已提交
3898
	sleep_on_tail(q, &wait, &flags);
L
Linus Torvalds 已提交
3899 3900 3901 3902 3903

	return timeout;
}
EXPORT_SYMBOL(sleep_on_timeout);

3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
3916
void rt_mutex_setprio(struct task_struct *p, int prio)
3917 3918
{
	unsigned long flags;
I
Ingo Molnar 已提交
3919
	int oldprio, on_rq;
3920
	struct rq *rq;
3921 3922 3923 3924

	BUG_ON(prio < 0 || prio > MAX_PRIO);

	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3925
	update_rq_clock(rq);
3926

3927
	oldprio = p->prio;
I
Ingo Molnar 已提交
3928 3929
	on_rq = p->se.on_rq;
	if (on_rq)
3930
		dequeue_task(rq, p, 0);
I
Ingo Molnar 已提交
3931 3932 3933 3934 3935 3936

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

3937 3938
	p->prio = prio;

I
Ingo Molnar 已提交
3939
	if (on_rq) {
3940
		enqueue_task(rq, p, 0);
3941 3942
		/*
		 * Reschedule if we are currently running on this runqueue and
3943 3944
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
3945
		 */
3946 3947 3948
		if (task_running(rq, p)) {
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
3949 3950 3951
		} else {
			check_preempt_curr(rq, p);
		}
3952 3953 3954 3955 3956 3957
	}
	task_rq_unlock(rq, &flags);
}

#endif

3958
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
3959
{
I
Ingo Molnar 已提交
3960
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
3961
	unsigned long flags;
3962
	struct rq *rq;
L
Linus Torvalds 已提交
3963 3964 3965 3966 3967 3968 3969 3970

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
3971
	update_rq_clock(rq);
L
Linus Torvalds 已提交
3972 3973 3974 3975
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
3976
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
3977
	 */
3978
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
3979 3980 3981
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
I
Ingo Molnar 已提交
3982 3983
	on_rq = p->se.on_rq;
	if (on_rq) {
3984
		dequeue_task(rq, p, 0);
3985
		dec_load(rq, p);
3986
	}
L
Linus Torvalds 已提交
3987 3988

	p->static_prio = NICE_TO_PRIO(nice);
3989
	set_load_weight(p);
3990 3991 3992
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
3993

I
Ingo Molnar 已提交
3994
	if (on_rq) {
3995
		enqueue_task(rq, p, 0);
3996
		inc_load(rq, p);
L
Linus Torvalds 已提交
3997
		/*
3998 3999
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
4000
		 */
4001
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
4002 4003 4004 4005 4006 4007 4008
			resched_task(rq->curr);
	}
out_unlock:
	task_rq_unlock(rq, &flags);
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
4009 4010 4011 4012 4013
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
4014
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
4015
{
4016 4017
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
4018

M
Matt Mackall 已提交
4019 4020 4021 4022
	return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
asmlinkage long sys_nice(int increment)
{
4034
	long nice, retval;
L
Linus Torvalds 已提交
4035 4036 4037 4038 4039 4040

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
4041 4042
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
4043 4044 4045 4046 4047 4048 4049 4050 4051
	if (increment > 40)
		increment = 40;

	nice = PRIO_TO_NICE(current->static_prio) + increment;
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
4052 4053 4054
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
4073
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
4074 4075 4076 4077 4078 4079 4080 4081
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
4082
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100
{
	return TASK_NICE(p);
}
EXPORT_SYMBOL_GPL(task_nice);

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
4101
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
4102 4103 4104 4105 4106 4107 4108 4109
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
4110
static inline struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
4111 4112 4113 4114 4115
{
	return pid ? find_task_by_pid(pid) : current;
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
4116 4117
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
4118
{
I
Ingo Molnar 已提交
4119
	BUG_ON(p->se.on_rq);
4120

L
Linus Torvalds 已提交
4121
	p->policy = policy;
I
Ingo Molnar 已提交
4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133
	switch (p->policy) {
	case SCHED_NORMAL:
	case SCHED_BATCH:
	case SCHED_IDLE:
		p->sched_class = &fair_sched_class;
		break;
	case SCHED_FIFO:
	case SCHED_RR:
		p->sched_class = &rt_sched_class;
		break;
	}

L
Linus Torvalds 已提交
4134
	p->rt_priority = prio;
4135 4136 4137
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
4138
	set_load_weight(p);
L
Linus Torvalds 已提交
4139 4140 4141
}

/**
4142
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
L
Linus Torvalds 已提交
4143 4144 4145
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
4146
 *
4147
 * NOTE that the task may be already dead.
L
Linus Torvalds 已提交
4148
 */
I
Ingo Molnar 已提交
4149 4150
int sched_setscheduler(struct task_struct *p, int policy,
		       struct sched_param *param)
L
Linus Torvalds 已提交
4151
{
I
Ingo Molnar 已提交
4152
	int retval, oldprio, oldpolicy = -1, on_rq;
L
Linus Torvalds 已提交
4153
	unsigned long flags;
4154
	struct rq *rq;
L
Linus Torvalds 已提交
4155

4156 4157
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
4158 4159 4160 4161 4162
recheck:
	/* double check policy once rq lock held */
	if (policy < 0)
		policy = oldpolicy = p->policy;
	else if (policy != SCHED_FIFO && policy != SCHED_RR &&
I
Ingo Molnar 已提交
4163 4164
			policy != SCHED_NORMAL && policy != SCHED_BATCH &&
			policy != SCHED_IDLE)
4165
		return -EINVAL;
L
Linus Torvalds 已提交
4166 4167
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
4168 4169
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
4170 4171
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
4172
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
4173
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
4174
		return -EINVAL;
4175
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
4176 4177
		return -EINVAL;

4178 4179 4180 4181
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
	if (!capable(CAP_SYS_NICE)) {
4182
		if (rt_policy(policy)) {
4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198
			unsigned long rlim_rtprio;

			if (!lock_task_sighand(p, &flags))
				return -ESRCH;
			rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
			unlock_task_sighand(p, &flags);

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
I
Ingo Molnar 已提交
4199 4200 4201 4202 4203 4204
		/*
		 * Like positive nice levels, dont allow tasks to
		 * move out of SCHED_IDLE either:
		 */
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
			return -EPERM;
4205

4206 4207 4208 4209 4210
		/* can't change other user's priorities */
		if ((current->euid != p->euid) &&
		    (current->euid != p->uid))
			return -EPERM;
	}
L
Linus Torvalds 已提交
4211 4212 4213 4214

	retval = security_task_setscheduler(p, policy, param);
	if (retval)
		return retval;
4215 4216 4217 4218 4219
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
	 */
	spin_lock_irqsave(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4220 4221 4222 4223
	/*
	 * To be able to change p->policy safely, the apropriate
	 * runqueue lock must be held.
	 */
4224
	rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
4225 4226 4227
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
4228 4229
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4230 4231
		goto recheck;
	}
I
Ingo Molnar 已提交
4232
	update_rq_clock(rq);
I
Ingo Molnar 已提交
4233
	on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
4234
	if (on_rq)
4235
		deactivate_task(rq, p, 0);
L
Linus Torvalds 已提交
4236
	oldprio = p->prio;
I
Ingo Molnar 已提交
4237 4238 4239
	__setscheduler(rq, p, policy, param->sched_priority);
	if (on_rq) {
		activate_task(rq, p, 0);
L
Linus Torvalds 已提交
4240 4241
		/*
		 * Reschedule if we are currently running on this runqueue and
4242 4243
		 * our priority decreased, or if we are not currently running on
		 * this runqueue and our priority is higher than the current's
L
Linus Torvalds 已提交
4244
		 */
4245 4246 4247
		if (task_running(rq, p)) {
			if (p->prio > oldprio)
				resched_task(rq->curr);
I
Ingo Molnar 已提交
4248 4249 4250
		} else {
			check_preempt_curr(rq, p);
		}
L
Linus Torvalds 已提交
4251
	}
4252 4253 4254
	__task_rq_unlock(rq);
	spin_unlock_irqrestore(&p->pi_lock, flags);

4255 4256
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
4257 4258 4259 4260
	return 0;
}
EXPORT_SYMBOL_GPL(sched_setscheduler);

I
Ingo Molnar 已提交
4261 4262
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
4263 4264 4265
{
	struct sched_param lparam;
	struct task_struct *p;
4266
	int retval;
L
Linus Torvalds 已提交
4267 4268 4269 4270 4271

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
4272 4273 4274

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
4275
	p = find_process_by_pid(pid);
4276 4277 4278
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
4279

L
Linus Torvalds 已提交
4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
				       struct sched_param __user *param)
{
4292 4293 4294 4295
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
asmlinkage long sys_sched_getscheduler(pid_t pid)
{
4315
	struct task_struct *p;
L
Linus Torvalds 已提交
4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342
	int retval = -EINVAL;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
			retval = p->policy;
	}
	read_unlock(&tasklist_lock);

out_nounlock:
	return retval;
}

/**
 * sys_sched_getscheduler - get the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
{
	struct sched_param lp;
4343
	struct task_struct *p;
L
Linus Torvalds 已提交
4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377
	int retval = -EINVAL;

	if (!param || pid < 0)
		goto out_nounlock;

	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
	read_unlock(&tasklist_lock);

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

out_nounlock:
	return retval;

out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

long sched_setaffinity(pid_t pid, cpumask_t new_mask)
{
	cpumask_t cpus_allowed;
4378 4379
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
4380

4381
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4382 4383 4384 4385 4386
	read_lock(&tasklist_lock);

	p = find_process_by_pid(pid);
	if (!p) {
		read_unlock(&tasklist_lock);
4387
		mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403
		return -ESRCH;
	}

	/*
	 * It is not safe to call set_cpus_allowed with the
	 * tasklist_lock held.  We will bump the task_struct's
	 * usage count and then drop tasklist_lock.
	 */
	get_task_struct(p);
	read_unlock(&tasklist_lock);

	retval = -EPERM;
	if ((current->euid != p->euid) && (current->euid != p->uid) &&
			!capable(CAP_SYS_NICE))
		goto out_unlock;

4404 4405 4406 4407
	retval = security_task_setscheduler(p, 0, NULL);
	if (retval)
		goto out_unlock;

L
Linus Torvalds 已提交
4408 4409 4410 4411 4412 4413
	cpus_allowed = cpuset_cpus_allowed(p);
	cpus_and(new_mask, new_mask, cpus_allowed);
	retval = set_cpus_allowed(p, new_mask);

out_unlock:
	put_task_struct(p);
4414
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
			     cpumask_t *new_mask)
{
	if (len < sizeof(cpumask_t)) {
		memset(new_mask, 0, sizeof(cpumask_t));
	} else if (len > sizeof(cpumask_t)) {
		len = sizeof(cpumask_t);
	}
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	cpumask_t new_mask;
	int retval;

	retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask);
	if (retval)
		return retval;

	return sched_setaffinity(pid, new_mask);
}

/*
 * Represents all cpu's present in the system
 * In systems capable of hotplug, this map could dynamically grow
 * as new cpu's are detected in the system via any platform specific
 * method, such as ACPI for e.g.
 */

4455
cpumask_t cpu_present_map __read_mostly;
L
Linus Torvalds 已提交
4456 4457 4458
EXPORT_SYMBOL(cpu_present_map);

#ifndef CONFIG_SMP
4459
cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
4460 4461
EXPORT_SYMBOL(cpu_online_map);

4462
cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
4463
EXPORT_SYMBOL(cpu_possible_map);
L
Linus Torvalds 已提交
4464 4465 4466 4467
#endif

long sched_getaffinity(pid_t pid, cpumask_t *mask)
{
4468
	struct task_struct *p;
L
Linus Torvalds 已提交
4469 4470
	int retval;

4471
	mutex_lock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4472 4473 4474 4475 4476 4477 4478
	read_lock(&tasklist_lock);

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

4479 4480 4481 4482
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4483
	cpus_and(*mask, p->cpus_allowed, cpu_online_map);
L
Linus Torvalds 已提交
4484 4485 4486

out_unlock:
	read_unlock(&tasklist_lock);
4487
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
4488

4489
	return retval;
L
Linus Torvalds 已提交
4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
				      unsigned long __user *user_mask_ptr)
{
	int ret;
	cpumask_t mask;

	if (len < sizeof(cpumask_t))
		return -EINVAL;

	ret = sched_getaffinity(pid, &mask);
	if (ret < 0)
		return ret;

	if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t)))
		return -EFAULT;

	return sizeof(cpumask_t);
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
4520 4521
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
4522 4523 4524
 */
asmlinkage long sys_sched_yield(void)
{
4525
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
4526 4527

	schedstat_inc(rq, yld_cnt);
I
Ingo Molnar 已提交
4528
	if (unlikely(rq->nr_running == 1))
L
Linus Torvalds 已提交
4529
		schedstat_inc(rq, yld_act_empty);
I
Ingo Molnar 已提交
4530 4531
	else
		current->sched_class->yield_task(rq, current);
L
Linus Torvalds 已提交
4532 4533 4534 4535 4536 4537

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
4538
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4539 4540 4541 4542 4543 4544 4545 4546
	_raw_spin_unlock(&rq->lock);
	preempt_enable_no_resched();

	schedule();

	return 0;
}

A
Andrew Morton 已提交
4547
static void __cond_resched(void)
L
Linus Torvalds 已提交
4548
{
4549 4550 4551
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
	__might_sleep(__FILE__, __LINE__);
#endif
4552 4553 4554 4555 4556
	/*
	 * The BKS might be reacquired before we have dropped
	 * PREEMPT_ACTIVE, which could trigger a second
	 * cond_resched() call.
	 */
L
Linus Torvalds 已提交
4557 4558 4559 4560 4561 4562 4563 4564 4565
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		schedule();
		sub_preempt_count(PREEMPT_ACTIVE);
	} while (need_resched());
}

int __sched cond_resched(void)
{
4566 4567
	if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
					system_state == SYSTEM_RUNNING) {
L
Linus Torvalds 已提交
4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582
		__cond_resched();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched);

/*
 * cond_resched_lock() - if a reschedule is pending, drop the given lock,
 * call schedule, and on return reacquire the lock.
 *
 * This works OK both with and without CONFIG_PREEMPT.  We do strange low-level
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
I
Ingo Molnar 已提交
4583
int cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
4584
{
J
Jan Kara 已提交
4585 4586
	int ret = 0;

L
Linus Torvalds 已提交
4587 4588 4589
	if (need_lockbreak(lock)) {
		spin_unlock(lock);
		cpu_relax();
J
Jan Kara 已提交
4590
		ret = 1;
L
Linus Torvalds 已提交
4591 4592
		spin_lock(lock);
	}
4593
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4594
		spin_release(&lock->dep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
4595 4596 4597
		_raw_spin_unlock(lock);
		preempt_enable_no_resched();
		__cond_resched();
J
Jan Kara 已提交
4598
		ret = 1;
L
Linus Torvalds 已提交
4599 4600
		spin_lock(lock);
	}
J
Jan Kara 已提交
4601
	return ret;
L
Linus Torvalds 已提交
4602 4603 4604 4605 4606 4607 4608
}
EXPORT_SYMBOL(cond_resched_lock);

int __sched cond_resched_softirq(void)
{
	BUG_ON(!in_softirq());

4609
	if (need_resched() && system_state == SYSTEM_RUNNING) {
4610
		local_bh_enable();
L
Linus Torvalds 已提交
4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL(cond_resched_softirq);

/**
 * yield - yield the current processor to other threads.
 *
4622
 * This is a shortcut for kernel-space yielding - it marks the
L
Linus Torvalds 已提交
4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640
 * thread runnable and calls sys_sched_yield().
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

/*
 * This task is about to go to sleep on IO.  Increment rq->nr_iowait so
 * that process accounting knows that this is a task in IO wait state.
 *
 * But don't do that if it is a deliberate, throttling IO wait (this task
 * has set its backing_dev_info: the queue against which it should throttle)
 */
void __sched io_schedule(void)
{
4641
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4642

4643
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4644 4645 4646
	atomic_inc(&rq->nr_iowait);
	schedule();
	atomic_dec(&rq->nr_iowait);
4647
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4648 4649 4650 4651 4652
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
4653
	struct rq *rq = &__raw_get_cpu_var(runqueues);
L
Linus Torvalds 已提交
4654 4655
	long ret;

4656
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4657 4658 4659
	atomic_inc(&rq->nr_iowait);
	ret = schedule_timeout(timeout);
	atomic_dec(&rq->nr_iowait);
4660
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_max(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
4681
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4682
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
asmlinkage long sys_sched_get_priority_min(int policy)
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
4706
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4707
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
asmlinkage
long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
{
4724
	struct task_struct *p;
L
Linus Torvalds 已提交
4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740
	int retval = -EINVAL;
	struct timespec t;

	if (pid < 0)
		goto out_nounlock;

	retval = -ESRCH;
	read_lock(&tasklist_lock);
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4741
	jiffies_to_timespec(p->policy == SCHED_FIFO ?
I
Ingo Molnar 已提交
4742
				0 : static_prio_timeslice(p->static_prio), &t);
L
Linus Torvalds 已提交
4743 4744 4745 4746 4747 4748 4749 4750 4751
	read_unlock(&tasklist_lock);
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
out_nounlock:
	return retval;
out_unlock:
	read_unlock(&tasklist_lock);
	return retval;
}

4752
static const char stat_nam[] = "RSDTtZX";
4753 4754

static void show_task(struct task_struct *p)
L
Linus Torvalds 已提交
4755 4756
{
	unsigned long free = 0;
4757
	unsigned state;
L
Linus Torvalds 已提交
4758 4759

	state = p->state ? __ffs(p->state) + 1 : 0;
4760 4761
	printk("%-13.13s %c", p->comm,
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4762
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
4763
	if (state == TASK_RUNNING)
4764
		printk(" running  ");
L
Linus Torvalds 已提交
4765
	else
4766
		printk(" %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4767 4768
#else
	if (state == TASK_RUNNING)
4769
		printk("  running task    ");
L
Linus Torvalds 已提交
4770 4771 4772 4773 4774
	else
		printk(" %016lx ", thread_saved_pc(p));
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
	{
4775
		unsigned long *n = end_of_stack(p);
L
Linus Torvalds 已提交
4776 4777
		while (!*n)
			n++;
4778
		free = (unsigned long)n - (unsigned long)end_of_stack(p);
L
Linus Torvalds 已提交
4779 4780
	}
#endif
4781
	printk("%5lu %5d %6d\n", free, p->pid, p->parent->pid);
L
Linus Torvalds 已提交
4782 4783 4784 4785 4786

	if (state != TASK_RUNNING)
		show_stack(p, NULL);
}

I
Ingo Molnar 已提交
4787
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
4788
{
4789
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
4790

4791 4792 4793
#if BITS_PER_LONG == 32
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
4794
#else
4795 4796
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
4797 4798 4799 4800 4801 4802 4803 4804
#endif
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
		 * console might take alot of time:
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
4805
		if (!state_filter || (p->state & state_filter))
I
Ingo Molnar 已提交
4806
			show_task(p);
L
Linus Torvalds 已提交
4807 4808
	} while_each_thread(g, p);

4809 4810
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
4811 4812 4813
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
L
Linus Torvalds 已提交
4814
	read_unlock(&tasklist_lock);
I
Ingo Molnar 已提交
4815 4816 4817 4818 4819
	/*
	 * Only show locks if all tasks are dumped:
	 */
	if (state_filter == -1)
		debug_show_all_locks();
L
Linus Torvalds 已提交
4820 4821
}

I
Ingo Molnar 已提交
4822 4823
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
4824
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
4825 4826
}

4827 4828 4829 4830 4831 4832 4833 4834
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
4835
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
4836
{
4837
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4838 4839
	unsigned long flags;

I
Ingo Molnar 已提交
4840 4841 4842
	__sched_fork(idle);
	idle->se.exec_start = sched_clock();

4843
	idle->prio = idle->normal_prio = MAX_PRIO;
L
Linus Torvalds 已提交
4844
	idle->cpus_allowed = cpumask_of_cpu(cpu);
I
Ingo Molnar 已提交
4845
	__set_task_cpu(idle, cpu);
L
Linus Torvalds 已提交
4846 4847 4848

	spin_lock_irqsave(&rq->lock, flags);
	rq->curr = rq->idle = idle;
4849 4850 4851
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
	idle->oncpu = 1;
#endif
L
Linus Torvalds 已提交
4852 4853 4854 4855
	spin_unlock_irqrestore(&rq->lock, flags);

	/* Set the preempt count _outside_ the spinlocks! */
#if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_BKL)
A
Al Viro 已提交
4856
	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
L
Linus Torvalds 已提交
4857
#else
A
Al Viro 已提交
4858
	task_thread_info(idle)->preempt_count = 0;
L
Linus Torvalds 已提交
4859
#endif
I
Ingo Molnar 已提交
4860 4861 4862 4863
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874
}

/*
 * In a system that switches off the HZ timer nohz_cpu_mask
 * indicates which cpus entered this state. This is used
 * in the rcu update to wait only for active cpus. For system
 * which do not switch off the HZ timer nohz_cpu_mask should
 * always be CPU_MASK_NONE.
 */
cpumask_t nohz_cpu_mask = CPU_MASK_NONE;

I
Ingo Molnar 已提交
4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886
/*
 * Increase the granularity value when there are more CPUs,
 * because with more CPUs the 'effective latency' as visible
 * to users decreases. But the relationship is not linear,
 * so pick a second-best guess by going with the log2 of the
 * number of CPUs.
 *
 * This idea comes from the SD scheduler of Con Kolivas:
 */
static inline void sched_init_granularity(void)
{
	unsigned int factor = 1 + ilog2(num_online_cpus());
4887
	const unsigned long gran_limit = 100000000;
I
Ingo Molnar 已提交
4888 4889 4890 4891 4892 4893 4894 4895 4896

	sysctl_sched_granularity *= factor;
	if (sysctl_sched_granularity > gran_limit)
		sysctl_sched_granularity = gran_limit;

	sysctl_sched_runtime_limit = sysctl_sched_granularity * 4;
	sysctl_sched_wakeup_granularity = sysctl_sched_granularity / 2;
}

L
Linus Torvalds 已提交
4897 4898 4899 4900
#ifdef CONFIG_SMP
/*
 * This is how migration works:
 *
4901
 * 1) we queue a struct migration_req structure in the source CPU's
L
Linus Torvalds 已提交
4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922
 *    runqueue and wake up that CPU's migration thread.
 * 2) we down() the locked semaphore => thread blocks.
 * 3) migration thread wakes up (implicitly it forces the migrated
 *    thread off the CPU)
 * 4) it gets the migration request and checks whether the migrated
 *    task is still in the wrong runqueue.
 * 5) if it's in the wrong runqueue then the migration thread removes
 *    it and puts it into the right queue.
 * 6) migration thread up()s the semaphore.
 * 7) we wake up and the migration is done.
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
 * task must not exit() & deallocate itself prematurely.  The
 * call is not atomic; no spinlocks may be held.
 */
4923
int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
L
Linus Torvalds 已提交
4924
{
4925
	struct migration_req req;
L
Linus Torvalds 已提交
4926
	unsigned long flags;
4927
	struct rq *rq;
4928
	int ret = 0;
L
Linus Torvalds 已提交
4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950

	rq = task_rq_lock(p, &flags);
	if (!cpus_intersects(new_mask, cpu_online_map)) {
		ret = -EINVAL;
		goto out;
	}

	p->cpus_allowed = new_mask;
	/* Can the task run on the task's current CPU? If so, we're done */
	if (cpu_isset(task_cpu(p), new_mask))
		goto out;

	if (migrate_task(p, any_online_cpu(new_mask), &req)) {
		/* Need help from migration thread: drop lock and wait. */
		task_rq_unlock(rq, &flags);
		wake_up_process(rq->migration_thread);
		wait_for_completion(&req.done);
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
	task_rq_unlock(rq, &flags);
4951

L
Linus Torvalds 已提交
4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963
	return ret;
}
EXPORT_SYMBOL_GPL(set_cpus_allowed);

/*
 * Move (not current) task off this cpu, onto dest cpu.  We're doing
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
4964 4965
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
4966
 */
4967
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
4968
{
4969
	struct rq *rq_dest, *rq_src;
I
Ingo Molnar 已提交
4970
	int ret = 0, on_rq;
L
Linus Torvalds 已提交
4971 4972

	if (unlikely(cpu_is_offline(dest_cpu)))
4973
		return ret;
L
Linus Torvalds 已提交
4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
		goto out;
	/* Affinity changed (again). */
	if (!cpu_isset(dest_cpu, p->cpus_allowed))
		goto out;

I
Ingo Molnar 已提交
4986
	on_rq = p->se.on_rq;
4987
	if (on_rq)
4988
		deactivate_task(rq_src, p, 0);
4989

L
Linus Torvalds 已提交
4990
	set_task_cpu(p, dest_cpu);
I
Ingo Molnar 已提交
4991 4992 4993
	if (on_rq) {
		activate_task(rq_dest, p, 0);
		check_preempt_curr(rq_dest, p);
L
Linus Torvalds 已提交
4994
	}
4995
	ret = 1;
L
Linus Torvalds 已提交
4996 4997
out:
	double_rq_unlock(rq_src, rq_dest);
4998
	return ret;
L
Linus Torvalds 已提交
4999 5000 5001 5002 5003 5004 5005
}

/*
 * migration_thread - this is a highprio system thread that performs
 * thread migration by bumping thread off CPU then 'pushing' onto
 * another runqueue.
 */
I
Ingo Molnar 已提交
5006
static int migration_thread(void *data)
L
Linus Torvalds 已提交
5007 5008
{
	int cpu = (long)data;
5009
	struct rq *rq;
L
Linus Torvalds 已提交
5010 5011 5012 5013 5014 5015

	rq = cpu_rq(cpu);
	BUG_ON(rq->migration_thread != current);

	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
5016
		struct migration_req *req;
L
Linus Torvalds 已提交
5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038
		struct list_head *head;

		spin_lock_irq(&rq->lock);

		if (cpu_is_offline(cpu)) {
			spin_unlock_irq(&rq->lock);
			goto wait_to_die;
		}

		if (rq->active_balance) {
			active_load_balance(rq, cpu);
			rq->active_balance = 0;
		}

		head = &rq->migration_queue;

		if (list_empty(head)) {
			spin_unlock_irq(&rq->lock);
			schedule();
			set_current_state(TASK_INTERRUPTIBLE);
			continue;
		}
5039
		req = list_entry(head->next, struct migration_req, list);
L
Linus Torvalds 已提交
5040 5041
		list_del_init(head->next);

N
Nick Piggin 已提交
5042 5043 5044
		spin_unlock(&rq->lock);
		__migrate_task(req->task, cpu, req->dest_cpu);
		local_irq_enable();
L
Linus Torvalds 已提交
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062

		complete(&req->done);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
5063 5064 5065 5066
/*
 * Figure out where task on dead CPU should go, use force if neccessary.
 * NOTE: interrupts should be disabled by the caller
 */
5067
static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5068
{
5069
	unsigned long flags;
L
Linus Torvalds 已提交
5070
	cpumask_t mask;
5071 5072
	struct rq *rq;
	int dest_cpu;
L
Linus Torvalds 已提交
5073

5074
restart:
L
Linus Torvalds 已提交
5075 5076
	/* On same node? */
	mask = node_to_cpumask(cpu_to_node(dead_cpu));
5077
	cpus_and(mask, mask, p->cpus_allowed);
L
Linus Torvalds 已提交
5078 5079 5080 5081
	dest_cpu = any_online_cpu(mask);

	/* On any allowed CPU? */
	if (dest_cpu == NR_CPUS)
5082
		dest_cpu = any_online_cpu(p->cpus_allowed);
L
Linus Torvalds 已提交
5083 5084 5085

	/* No more Mr. Nice Guy. */
	if (dest_cpu == NR_CPUS) {
5086 5087 5088
		rq = task_rq_lock(p, &flags);
		cpus_setall(p->cpus_allowed);
		dest_cpu = any_online_cpu(p->cpus_allowed);
5089
		task_rq_unlock(rq, &flags);
L
Linus Torvalds 已提交
5090 5091 5092 5093 5094 5095

		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
5096
		if (p->mm && printk_ratelimit())
L
Linus Torvalds 已提交
5097 5098
			printk(KERN_INFO "process %d (%s) no "
			       "longer affine to cpu%d\n",
5099
			       p->pid, p->comm, dead_cpu);
L
Linus Torvalds 已提交
5100
	}
5101
	if (!__migrate_task(p, dead_cpu, dest_cpu))
5102
		goto restart;
L
Linus Torvalds 已提交
5103 5104 5105 5106 5107 5108 5109 5110 5111
}

/*
 * While a dead CPU has no uninterruptible tasks queued at this point,
 * it might still have a nonzero ->nr_uninterruptible counter, because
 * for performance reasons the counter is not stricly tracking tasks to
 * their home CPUs. So we just add the counter to another CPU's counter,
 * to keep the global sum constant after CPU-down:
 */
5112
static void migrate_nr_uninterruptible(struct rq *rq_src)
L
Linus Torvalds 已提交
5113
{
5114
	struct rq *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL));
L
Linus Torvalds 已提交
5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127
	unsigned long flags;

	local_irq_save(flags);
	double_rq_lock(rq_src, rq_dest);
	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
	rq_src->nr_uninterruptible = 0;
	double_rq_unlock(rq_src, rq_dest);
	local_irq_restore(flags);
}

/* Run through task list and migrate tasks from the dead cpu. */
static void migrate_live_tasks(int src_cpu)
{
5128
	struct task_struct *p, *t;
L
Linus Torvalds 已提交
5129 5130 5131

	write_lock_irq(&tasklist_lock);

5132 5133
	do_each_thread(t, p) {
		if (p == current)
L
Linus Torvalds 已提交
5134 5135
			continue;

5136 5137 5138
		if (task_cpu(p) == src_cpu)
			move_task_off_dead_cpu(src_cpu, p);
	} while_each_thread(t, p);
L
Linus Torvalds 已提交
5139 5140 5141 5142

	write_unlock_irq(&tasklist_lock);
}

I
Ingo Molnar 已提交
5143 5144
/*
 * Schedules idle task to be the next runnable task on current CPU.
L
Linus Torvalds 已提交
5145
 * It does so by boosting its priority to highest possible and adding it to
5146
 * the _front_ of the runqueue. Used by CPU offline code.
L
Linus Torvalds 已提交
5147 5148 5149
 */
void sched_idle_next(void)
{
5150
	int this_cpu = smp_processor_id();
5151
	struct rq *rq = cpu_rq(this_cpu);
L
Linus Torvalds 已提交
5152 5153 5154 5155
	struct task_struct *p = rq->idle;
	unsigned long flags;

	/* cpu has to be offline */
5156
	BUG_ON(cpu_online(this_cpu));
L
Linus Torvalds 已提交
5157

5158 5159 5160
	/*
	 * Strictly not necessary since rest of the CPUs are stopped by now
	 * and interrupts disabled on the current cpu.
L
Linus Torvalds 已提交
5161 5162 5163
	 */
	spin_lock_irqsave(&rq->lock, flags);

I
Ingo Molnar 已提交
5164
	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
5165 5166

	/* Add idle task to the _front_ of its priority queue: */
I
Ingo Molnar 已提交
5167
	activate_idle_task(p, rq);
L
Linus Torvalds 已提交
5168 5169 5170 5171

	spin_unlock_irqrestore(&rq->lock, flags);
}

5172 5173
/*
 * Ensures that the idle task is using init_mm right before its cpu goes
L
Linus Torvalds 已提交
5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186
 * offline.
 */
void idle_task_exit(void)
{
	struct mm_struct *mm = current->active_mm;

	BUG_ON(cpu_online(smp_processor_id()));

	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
}

5187
/* called under rq->lock with disabled interrupts */
5188
static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
L
Linus Torvalds 已提交
5189
{
5190
	struct rq *rq = cpu_rq(dead_cpu);
L
Linus Torvalds 已提交
5191 5192

	/* Must be exiting, otherwise would be on tasklist. */
5193
	BUG_ON(p->exit_state != EXIT_ZOMBIE && p->exit_state != EXIT_DEAD);
L
Linus Torvalds 已提交
5194 5195

	/* Cannot have done final schedule yet: would have vanished. */
5196
	BUG_ON(p->state == TASK_DEAD);
L
Linus Torvalds 已提交
5197

5198
	get_task_struct(p);
L
Linus Torvalds 已提交
5199 5200 5201 5202 5203

	/*
	 * Drop lock around migration; if someone else moves it,
	 * that's OK.  No task can be added to this CPU, so iteration is
	 * fine.
5204
	 * NOTE: interrupts should be left disabled  --dev@
L
Linus Torvalds 已提交
5205
	 */
5206
	spin_unlock(&rq->lock);
5207
	move_task_off_dead_cpu(dead_cpu, p);
5208
	spin_lock(&rq->lock);
L
Linus Torvalds 已提交
5209

5210
	put_task_struct(p);
L
Linus Torvalds 已提交
5211 5212 5213 5214 5215
}

/* release_task() removes task from tasklist, so we won't find dead tasks. */
static void migrate_dead_tasks(unsigned int dead_cpu)
{
5216
	struct rq *rq = cpu_rq(dead_cpu);
I
Ingo Molnar 已提交
5217
	struct task_struct *next;
5218

I
Ingo Molnar 已提交
5219 5220 5221
	for ( ; ; ) {
		if (!rq->nr_running)
			break;
I
Ingo Molnar 已提交
5222
		update_rq_clock(rq);
5223
		next = pick_next_task(rq, rq->curr);
I
Ingo Molnar 已提交
5224 5225 5226
		if (!next)
			break;
		migrate_dead(dead_cpu, next);
5227

L
Linus Torvalds 已提交
5228 5229 5230 5231
	}
}
#endif /* CONFIG_HOTPLUG_CPU */

5232 5233 5234
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
5235 5236 5237 5238
	{
		.procname	= "sched_domain",
		.mode		= 0755,
	},
5239 5240 5241 5242
	{0,},
};

static struct ctl_table sd_ctl_root[] = {
5243 5244 5245 5246 5247
	{
		.procname	= "kernel",
		.mode		= 0755,
		.child		= sd_ctl_dir,
	},
5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262
	{0,},
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
		kmalloc(n * sizeof(struct ctl_table), GFP_KERNEL);

	BUG_ON(!entry);
	memset(entry, 0, n * sizeof(struct ctl_table));

	return entry;
}

static void
5263
set_table_entry(struct ctl_table *entry,
5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278
		const char *procname, void *data, int maxlen,
		mode_t mode, proc_handler *proc_handler)
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
	struct ctl_table *table = sd_alloc_ctl_entry(14);

5279
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
5280
		sizeof(long), 0644, proc_doulongvec_minmax);
5281
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
5282
		sizeof(long), 0644, proc_doulongvec_minmax);
5283
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
5284
		sizeof(int), 0644, proc_dointvec_minmax);
5285
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
5286
		sizeof(int), 0644, proc_dointvec_minmax);
5287
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
5288
		sizeof(int), 0644, proc_dointvec_minmax);
5289
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
5290
		sizeof(int), 0644, proc_dointvec_minmax);
5291
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
5292
		sizeof(int), 0644, proc_dointvec_minmax);
5293
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
5294
		sizeof(int), 0644, proc_dointvec_minmax);
5295
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
5296
		sizeof(int), 0644, proc_dointvec_minmax);
5297
	set_table_entry(&table[10], "cache_nice_tries",
5298 5299
		&sd->cache_nice_tries,
		sizeof(int), 0644, proc_dointvec_minmax);
5300
	set_table_entry(&table[12], "flags", &sd->flags,
5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351
		sizeof(int), 0644, proc_dointvec_minmax);

	return table;
}

static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
		entry->mode = 0755;
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
static void init_sched_domain_sysctl(void)
{
	int i, cpu_num = num_online_cpus();
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

	sd_ctl_dir[0].child = entry;

	for (i = 0; i < cpu_num; i++, entry++) {
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
		entry->mode = 0755;
		entry->child = sd_alloc_ctl_cpu_table(i);
	}
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
#else
static void init_sched_domain_sysctl(void)
{
}
#endif

L
Linus Torvalds 已提交
5352 5353 5354 5355
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
5356 5357
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
5358 5359
{
	struct task_struct *p;
5360
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
5361
	unsigned long flags;
5362
	struct rq *rq;
L
Linus Torvalds 已提交
5363 5364

	switch (action) {
5365 5366 5367 5368
	case CPU_LOCK_ACQUIRE:
		mutex_lock(&sched_hotcpu_mutex);
		break;

L
Linus Torvalds 已提交
5369
	case CPU_UP_PREPARE:
5370
	case CPU_UP_PREPARE_FROZEN:
I
Ingo Molnar 已提交
5371
		p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
L
Linus Torvalds 已提交
5372 5373 5374 5375 5376
		if (IS_ERR(p))
			return NOTIFY_BAD;
		kthread_bind(p, cpu);
		/* Must be high prio: stop_machine expects to yield to it. */
		rq = task_rq_lock(p, &flags);
I
Ingo Molnar 已提交
5377
		__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
L
Linus Torvalds 已提交
5378 5379 5380
		task_rq_unlock(rq, &flags);
		cpu_rq(cpu)->migration_thread = p;
		break;
5381

L
Linus Torvalds 已提交
5382
	case CPU_ONLINE:
5383
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
5384 5385 5386
		/* Strictly unneccessary, as first user will wake it. */
		wake_up_process(cpu_rq(cpu)->migration_thread);
		break;
5387

L
Linus Torvalds 已提交
5388 5389
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
5390
	case CPU_UP_CANCELED_FROZEN:
5391 5392
		if (!cpu_rq(cpu)->migration_thread)
			break;
L
Linus Torvalds 已提交
5393
		/* Unbind it from offline cpu so it can run.  Fall thru. */
5394 5395
		kthread_bind(cpu_rq(cpu)->migration_thread,
			     any_online_cpu(cpu_online_map));
L
Linus Torvalds 已提交
5396 5397 5398
		kthread_stop(cpu_rq(cpu)->migration_thread);
		cpu_rq(cpu)->migration_thread = NULL;
		break;
5399

L
Linus Torvalds 已提交
5400
	case CPU_DEAD:
5401
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
5402 5403 5404 5405 5406 5407
		migrate_live_tasks(cpu);
		rq = cpu_rq(cpu);
		kthread_stop(rq->migration_thread);
		rq->migration_thread = NULL;
		/* Idle task back to normal (off runqueue, low prio) */
		rq = task_rq_lock(rq->idle, &flags);
I
Ingo Molnar 已提交
5408
		update_rq_clock(rq);
5409
		deactivate_task(rq, rq->idle, 0);
L
Linus Torvalds 已提交
5410
		rq->idle->static_prio = MAX_PRIO;
I
Ingo Molnar 已提交
5411 5412
		__setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
		rq->idle->sched_class = &idle_sched_class;
L
Linus Torvalds 已提交
5413 5414 5415 5416 5417 5418
		migrate_dead_tasks(cpu);
		task_rq_unlock(rq, &flags);
		migrate_nr_uninterruptible(rq);
		BUG_ON(rq->nr_running != 0);

		/* No need to migrate the tasks: it was best-effort if
5419
		 * they didn't take sched_hotcpu_mutex.  Just wake up
L
Linus Torvalds 已提交
5420 5421 5422
		 * the requestors. */
		spin_lock_irq(&rq->lock);
		while (!list_empty(&rq->migration_queue)) {
5423 5424
			struct migration_req *req;

L
Linus Torvalds 已提交
5425
			req = list_entry(rq->migration_queue.next,
5426
					 struct migration_req, list);
L
Linus Torvalds 已提交
5427 5428 5429 5430 5431 5432
			list_del_init(&req->list);
			complete(&req->done);
		}
		spin_unlock_irq(&rq->lock);
		break;
#endif
5433 5434 5435
	case CPU_LOCK_RELEASE:
		mutex_unlock(&sched_hotcpu_mutex);
		break;
L
Linus Torvalds 已提交
5436 5437 5438 5439 5440 5441 5442
	}
	return NOTIFY_OK;
}

/* Register at highest priority so that task migration (migrate_all_tasks)
 * happens before everything else.
 */
5443
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
5444 5445 5446 5447 5448 5449 5450
	.notifier_call = migration_call,
	.priority = 10
};

int __init migration_init(void)
{
	void *cpu = (void *)(long)smp_processor_id();
5451
	int err;
5452 5453

	/* Start one for the boot CPU: */
5454 5455
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
5456 5457
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
5458

L
Linus Torvalds 已提交
5459 5460 5461 5462 5463
	return 0;
}
#endif

#ifdef CONFIG_SMP
5464 5465 5466 5467 5468

/* Number of possible processor ids */
int nr_cpu_ids __read_mostly = NR_CPUS;
EXPORT_SYMBOL(nr_cpu_ids);

5469
#undef SCHED_DOMAIN_DEBUG
L
Linus Torvalds 已提交
5470 5471 5472 5473 5474
#ifdef SCHED_DOMAIN_DEBUG
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;

N
Nick Piggin 已提交
5475 5476 5477 5478 5479
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}

L
Linus Torvalds 已提交
5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	do {
		int i;
		char str[NR_CPUS];
		struct sched_group *group = sd->groups;
		cpumask_t groupmask;

		cpumask_scnprintf(str, NR_CPUS, sd->span);
		cpus_clear(groupmask);

		printk(KERN_DEBUG);
		for (i = 0; i < level + 1; i++)
			printk(" ");
		printk("domain %d: ", level);

		if (!(sd->flags & SD_LOAD_BALANCE)) {
			printk("does not load-balance\n");
			if (sd->parent)
5499 5500
				printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
						" has parent");
L
Linus Torvalds 已提交
5501 5502 5503 5504 5505 5506
			break;
		}

		printk("span %s\n", str);

		if (!cpu_isset(cpu, sd->span))
5507 5508
			printk(KERN_ERR "ERROR: domain->span does not contain "
					"CPU%d\n", cpu);
L
Linus Torvalds 已提交
5509
		if (!cpu_isset(cpu, group->cpumask))
5510 5511
			printk(KERN_ERR "ERROR: domain->groups does not contain"
					" CPU%d\n", cpu);
L
Linus Torvalds 已提交
5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523

		printk(KERN_DEBUG);
		for (i = 0; i < level + 2; i++)
			printk(" ");
		printk("groups:");
		do {
			if (!group) {
				printk("\n");
				printk(KERN_ERR "ERROR: group is NULL\n");
				break;
			}

5524
			if (!group->__cpu_power) {
L
Linus Torvalds 已提交
5525
				printk("\n");
5526 5527
				printk(KERN_ERR "ERROR: domain->cpu_power not "
						"set\n");
L
Linus Torvalds 已提交
5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549
			}

			if (!cpus_weight(group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: empty group\n");
			}

			if (cpus_intersects(groupmask, group->cpumask)) {
				printk("\n");
				printk(KERN_ERR "ERROR: repeated CPUs\n");
			}

			cpus_or(groupmask, groupmask, group->cpumask);

			cpumask_scnprintf(str, NR_CPUS, group->cpumask);
			printk(" %s", str);

			group = group->next;
		} while (group != sd->groups);
		printk("\n");

		if (!cpus_equal(sd->span, groupmask))
5550 5551
			printk(KERN_ERR "ERROR: groups don't span "
					"domain->span\n");
L
Linus Torvalds 已提交
5552 5553 5554

		level++;
		sd = sd->parent;
5555 5556
		if (!sd)
			continue;
L
Linus Torvalds 已提交
5557

5558 5559 5560
		if (!cpus_subset(groupmask, sd->span))
			printk(KERN_ERR "ERROR: parent span is not a superset "
				"of domain->span\n");
L
Linus Torvalds 已提交
5561 5562 5563 5564

	} while (sd);
}
#else
5565
# define sched_domain_debug(sd, cpu) do { } while (0)
L
Linus Torvalds 已提交
5566 5567
#endif

5568
static int sd_degenerate(struct sched_domain *sd)
5569 5570 5571 5572 5573 5574 5575 5576
{
	if (cpus_weight(sd->span) == 1)
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
5577 5578 5579
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
	if (sd->flags & (SD_WAKE_IDLE |
			 SD_WAKE_AFFINE |
			 SD_WAKE_BALANCE))
		return 0;

	return 1;
}

5593 5594
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

	if (!cpus_equal(sd->span, parent->span))
		return 0;

	/* Does parent contain flags not in child? */
	/* WAKE_BALANCE is a subset of WAKE_AFFINE */
	if (cflags & SD_WAKE_AFFINE)
		pflags &= ~SD_WAKE_BALANCE;
	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
5613 5614 5615
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
5616 5617 5618 5619 5620 5621 5622
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

L
Linus Torvalds 已提交
5623 5624 5625 5626
/*
 * Attach the domain 'sd' to 'cpu' as its base domain.  Callers must
 * hold the hotplug lock.
 */
5627
static void cpu_attach_domain(struct sched_domain *sd, int cpu)
L
Linus Torvalds 已提交
5628
{
5629
	struct rq *rq = cpu_rq(cpu);
5630 5631 5632 5633 5634 5635 5636
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
	for (tmp = sd; tmp; tmp = tmp->parent) {
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
5637
		if (sd_parent_degenerate(tmp, parent)) {
5638
			tmp->parent = parent->parent;
5639 5640 5641
			if (parent->parent)
				parent->parent->child = tmp;
		}
5642 5643
	}

5644
	if (sd && sd_degenerate(sd)) {
5645
		sd = sd->parent;
5646 5647 5648
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
5649 5650 5651

	sched_domain_debug(sd, cpu);

N
Nick Piggin 已提交
5652
	rcu_assign_pointer(rq->sd, sd);
L
Linus Torvalds 已提交
5653 5654 5655
}

/* cpus with isolated domains */
5656
static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
L
Linus Torvalds 已提交
5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
	int ints[NR_CPUS], i;

	str = get_options(str, ARRAY_SIZE(ints), ints);
	cpus_clear(cpu_isolated_map);
	for (i = 1; i <= ints[0]; i++)
		if (ints[i] < NR_CPUS)
			cpu_set(ints[i], cpu_isolated_map);
	return 1;
}

__setup ("isolcpus=", isolated_cpu_setup);

/*
5674 5675 5676 5677
 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
 * to a function which identifies what group(along with sched group) a CPU
 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS
 * (due to the fact that we keep track of groups covered with a cpumask_t).
L
Linus Torvalds 已提交
5678 5679 5680 5681 5682
 *
 * init_sched_build_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
 */
5683
static void
5684 5685 5686
init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map,
			int (*group_fn)(int cpu, const cpumask_t *cpu_map,
					struct sched_group **sg))
L
Linus Torvalds 已提交
5687 5688 5689 5690 5691 5692
{
	struct sched_group *first = NULL, *last = NULL;
	cpumask_t covered = CPU_MASK_NONE;
	int i;

	for_each_cpu_mask(i, span) {
5693 5694
		struct sched_group *sg;
		int group = group_fn(i, cpu_map, &sg);
L
Linus Torvalds 已提交
5695 5696 5697 5698 5699 5700
		int j;

		if (cpu_isset(i, covered))
			continue;

		sg->cpumask = CPU_MASK_NONE;
5701
		sg->__cpu_power = 0;
L
Linus Torvalds 已提交
5702 5703

		for_each_cpu_mask(j, span) {
5704
			if (group_fn(j, cpu_map, NULL) != group)
L
Linus Torvalds 已提交
5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718
				continue;

			cpu_set(j, covered);
			cpu_set(j, sg->cpumask);
		}
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
}

5719
#define SD_NODES_PER_DOMAIN 16
L
Linus Torvalds 已提交
5720

5721
#ifdef CONFIG_NUMA
5722

5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774
/**
 * find_next_best_node - find the next node to include in a sched_domain
 * @node: node whose sched_domain we're building
 * @used_nodes: nodes already in the sched_domain
 *
 * Find the next node to include in a given scheduling domain.  Simply
 * finds the closest node not already in the @used_nodes map.
 *
 * Should use nodemask_t.
 */
static int find_next_best_node(int node, unsigned long *used_nodes)
{
	int i, n, val, min_val, best_node = 0;

	min_val = INT_MAX;

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Start at @node */
		n = (node + i) % MAX_NUMNODES;

		if (!nr_cpus_node(n))
			continue;

		/* Skip already used nodes */
		if (test_bit(n, used_nodes))
			continue;

		/* Simple min distance search */
		val = node_distance(node, n);

		if (val < min_val) {
			min_val = val;
			best_node = n;
		}
	}

	set_bit(best_node, used_nodes);
	return best_node;
}

/**
 * sched_domain_node_span - get a cpumask for a node's sched_domain
 * @node: node whose cpumask we're constructing
 * @size: number of nodes to include in this span
 *
 * Given a node, construct a good cpumask for its sched_domain to span.  It
 * should be one that prevents unnecessary balancing, but also spreads tasks
 * out optimally.
 */
static cpumask_t sched_domain_node_span(int node)
{
	DECLARE_BITMAP(used_nodes, MAX_NUMNODES);
5775 5776
	cpumask_t span, nodemask;
	int i;
5777 5778 5779 5780 5781 5782 5783 5784 5785 5786

	cpus_clear(span);
	bitmap_zero(used_nodes, MAX_NUMNODES);

	nodemask = node_to_cpumask(node);
	cpus_or(span, span, nodemask);
	set_bit(node, used_nodes);

	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
		int next_node = find_next_best_node(node, used_nodes);
5787

5788 5789 5790 5791 5792 5793 5794 5795
		nodemask = node_to_cpumask(next_node);
		cpus_or(span, span, nodemask);
	}

	return span;
}
#endif

5796
int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
5797

5798
/*
5799
 * SMT sched-domains:
5800
 */
L
Linus Torvalds 已提交
5801 5802
#ifdef CONFIG_SCHED_SMT
static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
5803
static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
5804

5805 5806
static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map,
			    struct sched_group **sg)
L
Linus Torvalds 已提交
5807
{
5808 5809
	if (sg)
		*sg = &per_cpu(sched_group_cpus, cpu);
L
Linus Torvalds 已提交
5810 5811 5812 5813
	return cpu;
}
#endif

5814 5815 5816
/*
 * multi-core sched-domains:
 */
5817 5818
#ifdef CONFIG_SCHED_MC
static DEFINE_PER_CPU(struct sched_domain, core_domains);
5819
static DEFINE_PER_CPU(struct sched_group, sched_group_core);
5820 5821 5822
#endif

#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
5823 5824
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5825
{
5826
	int group;
5827 5828
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5829 5830 5831 5832
	group = first_cpu(mask);
	if (sg)
		*sg = &per_cpu(sched_group_core, group);
	return group;
5833 5834
}
#elif defined(CONFIG_SCHED_MC)
5835 5836
static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
5837
{
5838 5839
	if (sg)
		*sg = &per_cpu(sched_group_core, cpu);
5840 5841 5842 5843
	return cpu;
}
#endif

L
Linus Torvalds 已提交
5844
static DEFINE_PER_CPU(struct sched_domain, phys_domains);
5845
static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
5846

5847 5848
static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map,
			     struct sched_group **sg)
L
Linus Torvalds 已提交
5849
{
5850
	int group;
5851
#ifdef CONFIG_SCHED_MC
5852
	cpumask_t mask = cpu_coregroup_map(cpu);
5853
	cpus_and(mask, mask, *cpu_map);
5854
	group = first_cpu(mask);
5855
#elif defined(CONFIG_SCHED_SMT)
5856 5857
	cpumask_t mask = cpu_sibling_map[cpu];
	cpus_and(mask, mask, *cpu_map);
5858
	group = first_cpu(mask);
L
Linus Torvalds 已提交
5859
#else
5860
	group = cpu;
L
Linus Torvalds 已提交
5861
#endif
5862 5863 5864
	if (sg)
		*sg = &per_cpu(sched_group_phys, group);
	return group;
L
Linus Torvalds 已提交
5865 5866 5867 5868
}

#ifdef CONFIG_NUMA
/*
5869 5870 5871
 * The init_sched_build_groups can't handle what we want to do with node
 * groups, so roll our own. Now each node has its own list of groups which
 * gets dynamically allocated.
L
Linus Torvalds 已提交
5872
 */
5873
static DEFINE_PER_CPU(struct sched_domain, node_domains);
5874
static struct sched_group **sched_group_nodes_bycpu[NR_CPUS];
L
Linus Torvalds 已提交
5875

5876
static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
5877
static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
5878

5879 5880
static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
				 struct sched_group **sg)
5881
{
5882 5883 5884 5885 5886 5887 5888 5889 5890
	cpumask_t nodemask = node_to_cpumask(cpu_to_node(cpu));
	int group;

	cpus_and(nodemask, nodemask, *cpu_map);
	group = first_cpu(nodemask);

	if (sg)
		*sg = &per_cpu(sched_group_allnodes, group);
	return group;
L
Linus Torvalds 已提交
5891
}
5892

5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912
static void init_numa_sched_groups_power(struct sched_group *group_head)
{
	struct sched_group *sg = group_head;
	int j;

	if (!sg)
		return;
next_sg:
	for_each_cpu_mask(j, sg->cpumask) {
		struct sched_domain *sd;

		sd = &per_cpu(phys_domains, j);
		if (j != first_cpu(sd->groups->cpumask)) {
			/*
			 * Only add "power" once for each
			 * physical package.
			 */
			continue;
		}

5913
		sg_inc_cpu_power(sg, sd->groups->__cpu_power);
5914 5915 5916 5917 5918
	}
	sg = sg->next;
	if (sg != group_head)
		goto next_sg;
}
L
Linus Torvalds 已提交
5919 5920
#endif

5921
#ifdef CONFIG_NUMA
5922 5923 5924
/* Free memory allocated for various sched_group structures */
static void free_sched_groups(const cpumask_t *cpu_map)
{
5925
	int cpu, i;
5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955

	for_each_cpu_mask(cpu, *cpu_map) {
		struct sched_group **sched_group_nodes
			= sched_group_nodes_bycpu[cpu];

		if (!sched_group_nodes)
			continue;

		for (i = 0; i < MAX_NUMNODES; i++) {
			cpumask_t nodemask = node_to_cpumask(i);
			struct sched_group *oldsg, *sg = sched_group_nodes[i];

			cpus_and(nodemask, nodemask, *cpu_map);
			if (cpus_empty(nodemask))
				continue;

			if (sg == NULL)
				continue;
			sg = sg->next;
next_sg:
			oldsg = sg;
			sg = sg->next;
			kfree(oldsg);
			if (oldsg != sched_group_nodes[i])
				goto next_sg;
		}
		kfree(sched_group_nodes);
		sched_group_nodes_bycpu[cpu] = NULL;
	}
}
5956 5957 5958 5959 5960
#else
static void free_sched_groups(const cpumask_t *cpu_map)
{
}
#endif
5961

5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 *
 * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
 * the maximum number of tasks a group can handle in the presence of other idle
 * or lightly loaded groups in the same sched domain.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
	struct sched_domain *child;
	struct sched_group *group;

	WARN_ON(!sd || !sd->groups);

	if (cpu != first_cpu(sd->groups->cpumask))
		return;

	child = sd->child;

5988 5989
	sd->groups->__cpu_power = 0;

5990 5991 5992 5993 5994 5995 5996 5997 5998 5999
	/*
	 * For perf policy, if the groups in child domain share resources
	 * (for example cores sharing some portions of the cache hierarchy
	 * or SMT), then set this domain groups cpu_power such that each group
	 * can handle only one task, when there are other idle groups in the
	 * same sched domain.
	 */
	if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
		       (child->flags &
			(SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
6000
		sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
6001 6002 6003 6004 6005 6006 6007 6008
		return;
	}

	/*
	 * add cpu_power of each child group to this groups cpu_power
	 */
	group = child->groups;
	do {
6009
		sg_inc_cpu_power(sd->groups, group->__cpu_power);
6010 6011 6012 6013
		group = group->next;
	} while (group != child->groups);
}

L
Linus Torvalds 已提交
6014
/*
6015 6016
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
L
Linus Torvalds 已提交
6017
 */
6018
static int build_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6019 6020
{
	int i;
6021 6022
#ifdef CONFIG_NUMA
	struct sched_group **sched_group_nodes = NULL;
6023
	int sd_allnodes = 0;
6024 6025 6026 6027

	/*
	 * Allocate the per-node list of sched groups
	 */
I
Ingo Molnar 已提交
6028
	sched_group_nodes = kzalloc(sizeof(struct sched_group *)*MAX_NUMNODES,
6029
					   GFP_KERNEL);
6030 6031
	if (!sched_group_nodes) {
		printk(KERN_WARNING "Can not alloc sched group node list\n");
6032
		return -ENOMEM;
6033 6034 6035
	}
	sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
#endif
L
Linus Torvalds 已提交
6036 6037

	/*
6038
	 * Set up domains for cpus specified by the cpu_map.
L
Linus Torvalds 已提交
6039
	 */
6040
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6041 6042 6043
		struct sched_domain *sd = NULL, *p;
		cpumask_t nodemask = node_to_cpumask(cpu_to_node(i));

6044
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6045 6046

#ifdef CONFIG_NUMA
I
Ingo Molnar 已提交
6047 6048
		if (cpus_weight(*cpu_map) >
				SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) {
6049 6050 6051
			sd = &per_cpu(allnodes_domains, i);
			*sd = SD_ALLNODES_INIT;
			sd->span = *cpu_map;
6052
			cpu_to_allnodes_group(i, cpu_map, &sd->groups);
6053
			p = sd;
6054
			sd_allnodes = 1;
6055 6056 6057
		} else
			p = NULL;

L
Linus Torvalds 已提交
6058 6059
		sd = &per_cpu(node_domains, i);
		*sd = SD_NODE_INIT;
6060 6061
		sd->span = sched_domain_node_span(cpu_to_node(i));
		sd->parent = p;
6062 6063
		if (p)
			p->child = sd;
6064
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6065 6066 6067 6068 6069 6070 6071
#endif

		p = sd;
		sd = &per_cpu(phys_domains, i);
		*sd = SD_CPU_INIT;
		sd->span = nodemask;
		sd->parent = p;
6072 6073
		if (p)
			p->child = sd;
6074
		cpu_to_phys_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6075

6076 6077 6078 6079 6080 6081 6082
#ifdef CONFIG_SCHED_MC
		p = sd;
		sd = &per_cpu(core_domains, i);
		*sd = SD_MC_INIT;
		sd->span = cpu_coregroup_map(i);
		cpus_and(sd->span, sd->span, *cpu_map);
		sd->parent = p;
6083
		p->child = sd;
6084
		cpu_to_core_group(i, cpu_map, &sd->groups);
6085 6086
#endif

L
Linus Torvalds 已提交
6087 6088 6089 6090 6091
#ifdef CONFIG_SCHED_SMT
		p = sd;
		sd = &per_cpu(cpu_domains, i);
		*sd = SD_SIBLING_INIT;
		sd->span = cpu_sibling_map[i];
6092
		cpus_and(sd->span, sd->span, *cpu_map);
L
Linus Torvalds 已提交
6093
		sd->parent = p;
6094
		p->child = sd;
6095
		cpu_to_cpu_group(i, cpu_map, &sd->groups);
L
Linus Torvalds 已提交
6096 6097 6098 6099 6100
#endif
	}

#ifdef CONFIG_SCHED_SMT
	/* Set up CPU (sibling) groups */
6101
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6102
		cpumask_t this_sibling_map = cpu_sibling_map[i];
6103
		cpus_and(this_sibling_map, this_sibling_map, *cpu_map);
L
Linus Torvalds 已提交
6104 6105 6106
		if (i != first_cpu(this_sibling_map))
			continue;

I
Ingo Molnar 已提交
6107 6108
		init_sched_build_groups(this_sibling_map, cpu_map,
					&cpu_to_cpu_group);
L
Linus Torvalds 已提交
6109 6110 6111
	}
#endif

6112 6113 6114 6115 6116 6117 6118
#ifdef CONFIG_SCHED_MC
	/* Set up multi-core groups */
	for_each_cpu_mask(i, *cpu_map) {
		cpumask_t this_core_map = cpu_coregroup_map(i);
		cpus_and(this_core_map, this_core_map, *cpu_map);
		if (i != first_cpu(this_core_map))
			continue;
I
Ingo Molnar 已提交
6119 6120
		init_sched_build_groups(this_core_map, cpu_map,
					&cpu_to_core_group);
6121 6122 6123
	}
#endif

L
Linus Torvalds 已提交
6124 6125 6126 6127
	/* Set up physical groups */
	for (i = 0; i < MAX_NUMNODES; i++) {
		cpumask_t nodemask = node_to_cpumask(i);

6128
		cpus_and(nodemask, nodemask, *cpu_map);
L
Linus Torvalds 已提交
6129 6130 6131
		if (cpus_empty(nodemask))
			continue;

6132
		init_sched_build_groups(nodemask, cpu_map, &cpu_to_phys_group);
L
Linus Torvalds 已提交
6133 6134 6135 6136
	}

#ifdef CONFIG_NUMA
	/* Set up node groups */
6137
	if (sd_allnodes)
I
Ingo Molnar 已提交
6138 6139
		init_sched_build_groups(*cpu_map, cpu_map,
					&cpu_to_allnodes_group);
6140 6141 6142 6143 6144 6145 6146 6147 6148 6149

	for (i = 0; i < MAX_NUMNODES; i++) {
		/* Set up node groups */
		struct sched_group *sg, *prev;
		cpumask_t nodemask = node_to_cpumask(i);
		cpumask_t domainspan;
		cpumask_t covered = CPU_MASK_NONE;
		int j;

		cpus_and(nodemask, nodemask, *cpu_map);
6150 6151
		if (cpus_empty(nodemask)) {
			sched_group_nodes[i] = NULL;
6152
			continue;
6153
		}
6154 6155 6156 6157

		domainspan = sched_domain_node_span(i);
		cpus_and(domainspan, domainspan, *cpu_map);

6158
		sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);
6159 6160 6161 6162 6163
		if (!sg) {
			printk(KERN_WARNING "Can not alloc domain group for "
				"node %d\n", i);
			goto error;
		}
6164 6165 6166
		sched_group_nodes[i] = sg;
		for_each_cpu_mask(j, nodemask) {
			struct sched_domain *sd;
I
Ingo Molnar 已提交
6167

6168 6169 6170
			sd = &per_cpu(node_domains, j);
			sd->groups = sg;
		}
6171
		sg->__cpu_power = 0;
6172
		sg->cpumask = nodemask;
6173
		sg->next = sg;
6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191
		cpus_or(covered, covered, nodemask);
		prev = sg;

		for (j = 0; j < MAX_NUMNODES; j++) {
			cpumask_t tmp, notcovered;
			int n = (i + j) % MAX_NUMNODES;

			cpus_complement(notcovered, covered);
			cpus_and(tmp, notcovered, *cpu_map);
			cpus_and(tmp, tmp, domainspan);
			if (cpus_empty(tmp))
				break;

			nodemask = node_to_cpumask(n);
			cpus_and(tmp, tmp, nodemask);
			if (cpus_empty(tmp))
				continue;

6192 6193
			sg = kmalloc_node(sizeof(struct sched_group),
					  GFP_KERNEL, i);
6194 6195 6196
			if (!sg) {
				printk(KERN_WARNING
				"Can not alloc domain group for node %d\n", j);
6197
				goto error;
6198
			}
6199
			sg->__cpu_power = 0;
6200
			sg->cpumask = tmp;
6201
			sg->next = prev->next;
6202 6203 6204 6205 6206
			cpus_or(covered, covered, tmp);
			prev->next = sg;
			prev = sg;
		}
	}
L
Linus Torvalds 已提交
6207 6208 6209
#endif

	/* Calculate CPU power for physical packages and nodes */
6210
#ifdef CONFIG_SCHED_SMT
6211
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6212 6213
		struct sched_domain *sd = &per_cpu(cpu_domains, i);

6214
		init_sched_groups_power(i, sd);
6215
	}
L
Linus Torvalds 已提交
6216
#endif
6217
#ifdef CONFIG_SCHED_MC
6218
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6219 6220
		struct sched_domain *sd = &per_cpu(core_domains, i);

6221
		init_sched_groups_power(i, sd);
6222 6223
	}
#endif
6224

6225
	for_each_cpu_mask(i, *cpu_map) {
I
Ingo Molnar 已提交
6226 6227
		struct sched_domain *sd = &per_cpu(phys_domains, i);

6228
		init_sched_groups_power(i, sd);
L
Linus Torvalds 已提交
6229 6230
	}

6231
#ifdef CONFIG_NUMA
6232 6233
	for (i = 0; i < MAX_NUMNODES; i++)
		init_numa_sched_groups_power(sched_group_nodes[i]);
6234

6235 6236
	if (sd_allnodes) {
		struct sched_group *sg;
6237

6238
		cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg);
6239 6240
		init_numa_sched_groups_power(sg);
	}
6241 6242
#endif

L
Linus Torvalds 已提交
6243
	/* Attach the domains */
6244
	for_each_cpu_mask(i, *cpu_map) {
L
Linus Torvalds 已提交
6245 6246 6247
		struct sched_domain *sd;
#ifdef CONFIG_SCHED_SMT
		sd = &per_cpu(cpu_domains, i);
6248 6249
#elif defined(CONFIG_SCHED_MC)
		sd = &per_cpu(core_domains, i);
L
Linus Torvalds 已提交
6250 6251 6252 6253 6254
#else
		sd = &per_cpu(phys_domains, i);
#endif
		cpu_attach_domain(sd, i);
	}
6255 6256 6257

	return 0;

6258
#ifdef CONFIG_NUMA
6259 6260 6261
error:
	free_sched_groups(cpu_map);
	return -ENOMEM;
6262
#endif
L
Linus Torvalds 已提交
6263
}
6264 6265 6266
/*
 * Set up scheduler domains and groups.  Callers must hold the hotplug lock.
 */
6267
static int arch_init_sched_domains(const cpumask_t *cpu_map)
6268 6269
{
	cpumask_t cpu_default_map;
6270
	int err;
L
Linus Torvalds 已提交
6271

6272 6273 6274 6275 6276 6277 6278
	/*
	 * Setup mask for cpus without special case scheduling requirements.
	 * For now this just excludes isolated cpus, but could be used to
	 * exclude other special cases in the future.
	 */
	cpus_andnot(cpu_default_map, *cpu_map, cpu_isolated_map);

6279 6280 6281
	err = build_sched_domains(&cpu_default_map);

	return err;
6282 6283 6284
}

static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
L
Linus Torvalds 已提交
6285
{
6286
	free_sched_groups(cpu_map);
6287
}
L
Linus Torvalds 已提交
6288

6289 6290 6291 6292
/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
6293
static void detach_destroy_domains(const cpumask_t *cpu_map)
6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310
{
	int i;

	for_each_cpu_mask(i, *cpu_map)
		cpu_attach_domain(NULL, i);
	synchronize_sched();
	arch_destroy_sched_domains(cpu_map);
}

/*
 * Partition sched domains as specified by the cpumasks below.
 * This attaches all cpus from the cpumasks to the NULL domain,
 * waits for a RCU quiescent period, recalculates sched
 * domain information and then attaches them back to the
 * correct sched domains
 * Call with hotplug lock held
 */
6311
int partition_sched_domains(cpumask_t *partition1, cpumask_t *partition2)
6312 6313
{
	cpumask_t change_map;
6314
	int err = 0;
6315 6316 6317 6318 6319 6320 6321 6322

	cpus_and(*partition1, *partition1, cpu_online_map);
	cpus_and(*partition2, *partition2, cpu_online_map);
	cpus_or(change_map, *partition1, *partition2);

	/* Detach sched domains from all of the affected cpus */
	detach_destroy_domains(&change_map);
	if (!cpus_empty(*partition1))
6323 6324 6325 6326 6327
		err = build_sched_domains(partition1);
	if (!err && !cpus_empty(*partition2))
		err = build_sched_domains(partition2);

	return err;
6328 6329
}

6330
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
A
Adrian Bunk 已提交
6331
static int arch_reinit_sched_domains(void)
6332 6333 6334
{
	int err;

6335
	mutex_lock(&sched_hotcpu_mutex);
6336 6337
	detach_destroy_domains(&cpu_online_map);
	err = arch_init_sched_domains(&cpu_online_map);
6338
	mutex_unlock(&sched_hotcpu_mutex);
6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364

	return err;
}

static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
{
	int ret;

	if (buf[0] != '0' && buf[0] != '1')
		return -EINVAL;

	if (smt)
		sched_smt_power_savings = (buf[0] == '1');
	else
		sched_mc_power_savings = (buf[0] == '1');

	ret = arch_reinit_sched_domains();

	return ret ? ret : count;
}

#ifdef CONFIG_SCHED_MC
static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_mc_power_savings);
}
6365 6366
static ssize_t sched_mc_power_savings_store(struct sys_device *dev,
					    const char *buf, size_t count)
6367 6368 6369
{
	return sched_power_savings_store(buf, count, 0);
}
A
Adrian Bunk 已提交
6370 6371
static SYSDEV_ATTR(sched_mc_power_savings, 0644, sched_mc_power_savings_show,
		   sched_mc_power_savings_store);
6372 6373 6374 6375 6376 6377 6378
#endif

#ifdef CONFIG_SCHED_SMT
static ssize_t sched_smt_power_savings_show(struct sys_device *dev, char *page)
{
	return sprintf(page, "%u\n", sched_smt_power_savings);
}
6379 6380
static ssize_t sched_smt_power_savings_store(struct sys_device *dev,
					     const char *buf, size_t count)
6381 6382 6383
{
	return sched_power_savings_store(buf, count, 1);
}
A
Adrian Bunk 已提交
6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403
static SYSDEV_ATTR(sched_smt_power_savings, 0644, sched_smt_power_savings_show,
		   sched_smt_power_savings_store);
#endif

int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
{
	int err = 0;

#ifdef CONFIG_SCHED_SMT
	if (smt_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_smt_power_savings.attr);
#endif
#ifdef CONFIG_SCHED_MC
	if (!err && mc_capable())
		err = sysfs_create_file(&cls->kset.kobj,
					&attr_sched_mc_power_savings.attr);
#endif
	return err;
}
6404 6405
#endif

L
Linus Torvalds 已提交
6406 6407 6408
/*
 * Force a reinitialization of the sched domains hierarchy.  The domains
 * and groups cannot be updated in place without racing with the balancing
N
Nick Piggin 已提交
6409
 * code, so we temporarily attach all running cpus to the NULL domain
L
Linus Torvalds 已提交
6410 6411 6412 6413 6414 6415 6416
 * which will prevent rebalancing while the sched domains are recalculated.
 */
static int update_sched_domains(struct notifier_block *nfb,
				unsigned long action, void *hcpu)
{
	switch (action) {
	case CPU_UP_PREPARE:
6417
	case CPU_UP_PREPARE_FROZEN:
L
Linus Torvalds 已提交
6418
	case CPU_DOWN_PREPARE:
6419
	case CPU_DOWN_PREPARE_FROZEN:
6420
		detach_destroy_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6421 6422 6423
		return NOTIFY_OK;

	case CPU_UP_CANCELED:
6424
	case CPU_UP_CANCELED_FROZEN:
L
Linus Torvalds 已提交
6425
	case CPU_DOWN_FAILED:
6426
	case CPU_DOWN_FAILED_FROZEN:
L
Linus Torvalds 已提交
6427
	case CPU_ONLINE:
6428
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
6429
	case CPU_DEAD:
6430
	case CPU_DEAD_FROZEN:
L
Linus Torvalds 已提交
6431 6432 6433 6434 6435 6436 6437 6438 6439
		/*
		 * Fall through and re-initialise the domains.
		 */
		break;
	default:
		return NOTIFY_DONE;
	}

	/* The hotplug lock is already held by cpu_up/cpu_down */
6440
	arch_init_sched_domains(&cpu_online_map);
L
Linus Torvalds 已提交
6441 6442 6443 6444 6445 6446

	return NOTIFY_OK;
}

void __init sched_init_smp(void)
{
6447 6448
	cpumask_t non_isolated_cpus;

6449
	mutex_lock(&sched_hotcpu_mutex);
6450
	arch_init_sched_domains(&cpu_online_map);
6451
	cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map);
6452 6453
	if (cpus_empty(non_isolated_cpus))
		cpu_set(smp_processor_id(), non_isolated_cpus);
6454
	mutex_unlock(&sched_hotcpu_mutex);
L
Linus Torvalds 已提交
6455 6456
	/* XXX: Theoretical race here - CPU may be hotplugged now */
	hotcpu_notifier(update_sched_domains, 0);
6457

6458 6459
	init_sched_domain_sysctl();

6460 6461 6462
	/* Move init over to a non-isolated CPU */
	if (set_cpus_allowed(current, non_isolated_cpus) < 0)
		BUG();
I
Ingo Molnar 已提交
6463
	sched_init_granularity();
L
Linus Torvalds 已提交
6464 6465 6466 6467
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
6468
	sched_init_granularity();
L
Linus Torvalds 已提交
6469 6470 6471 6472 6473 6474 6475
}
#endif /* CONFIG_SMP */

int in_sched_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __sched functions */
	extern char __sched_text_start[], __sched_text_end[];
6476

L
Linus Torvalds 已提交
6477 6478 6479 6480 6481
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

I
Ingo Molnar 已提交
6482 6483 6484 6485 6486 6487 6488 6489 6490
static inline void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
{
	cfs_rq->tasks_timeline = RB_ROOT;
	cfs_rq->fair_clock = 1;
#ifdef CONFIG_FAIR_GROUP_SCHED
	cfs_rq->rq = rq;
#endif
}

L
Linus Torvalds 已提交
6491 6492
void __init sched_init(void)
{
I
Ingo Molnar 已提交
6493
	u64 now = sched_clock();
6494
	int highest_cpu = 0;
I
Ingo Molnar 已提交
6495 6496 6497 6498 6499 6500 6501 6502
	int i, j;

	/*
	 * Link up the scheduling class hierarchy:
	 */
	rt_sched_class.next = &fair_sched_class;
	fair_sched_class.next = &idle_sched_class;
	idle_sched_class.next = NULL;
L
Linus Torvalds 已提交
6503

6504
	for_each_possible_cpu(i) {
I
Ingo Molnar 已提交
6505
		struct rt_prio_array *array;
6506
		struct rq *rq;
L
Linus Torvalds 已提交
6507 6508 6509

		rq = cpu_rq(i);
		spin_lock_init(&rq->lock);
6510
		lockdep_set_class(&rq->lock, &rq->rq_lock_key);
N
Nick Piggin 已提交
6511
		rq->nr_running = 0;
I
Ingo Molnar 已提交
6512 6513 6514 6515 6516 6517 6518 6519
		rq->clock = 1;
		init_cfs_rq(&rq->cfs, rq);
#ifdef CONFIG_FAIR_GROUP_SCHED
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
		list_add(&rq->cfs.leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
#endif
		rq->ls.load_update_last = now;
		rq->ls.load_update_start = now;
L
Linus Torvalds 已提交
6520

I
Ingo Molnar 已提交
6521 6522
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
L
Linus Torvalds 已提交
6523
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
6524
		rq->sd = NULL;
L
Linus Torvalds 已提交
6525
		rq->active_balance = 0;
I
Ingo Molnar 已提交
6526
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
6527
		rq->push_cpu = 0;
6528
		rq->cpu = i;
L
Linus Torvalds 已提交
6529 6530 6531 6532 6533
		rq->migration_thread = NULL;
		INIT_LIST_HEAD(&rq->migration_queue);
#endif
		atomic_set(&rq->nr_iowait, 0);

I
Ingo Molnar 已提交
6534 6535 6536 6537
		array = &rq->rt.active;
		for (j = 0; j < MAX_RT_PRIO; j++) {
			INIT_LIST_HEAD(array->queue + j);
			__clear_bit(j, array->bitmap);
L
Linus Torvalds 已提交
6538
		}
6539
		highest_cpu = i;
I
Ingo Molnar 已提交
6540 6541
		/* delimiter for bitsearch: */
		__set_bit(MAX_RT_PRIO, array->bitmap);
L
Linus Torvalds 已提交
6542 6543
	}

6544
	set_load_weight(&init_task);
6545

6546 6547 6548 6549
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

6550
#ifdef CONFIG_SMP
6551
	nr_cpu_ids = highest_cpu + 1;
6552 6553 6554
	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
#endif

6555 6556 6557 6558
#ifdef CONFIG_RT_MUTEXES
	plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
#endif

L
Linus Torvalds 已提交
6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
I
Ingo Molnar 已提交
6572 6573 6574 6575
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
L
Linus Torvalds 已提交
6576 6577 6578 6579 6580
}

#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
void __might_sleep(char *file, int line)
{
6581
#ifdef in_atomic
L
Linus Torvalds 已提交
6582 6583 6584 6585 6586 6587 6588
	static unsigned long prev_jiffy;	/* ratelimiting */

	if ((in_atomic() || irqs_disabled()) &&
	    system_state == SYSTEM_RUNNING && !oops_in_progress) {
		if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
			return;
		prev_jiffy = jiffies;
6589
		printk(KERN_ERR "BUG: sleeping function called from invalid"
L
Linus Torvalds 已提交
6590 6591 6592
				" context at %s:%d\n", file, line);
		printk("in_atomic():%d, irqs_disabled():%d\n",
			in_atomic(), irqs_disabled());
6593
		debug_show_held_locks(current);
6594 6595
		if (irqs_disabled())
			print_irqtrace_events(current);
L
Linus Torvalds 已提交
6596 6597 6598 6599 6600 6601 6602 6603 6604 6605
		dump_stack();
	}
#endif
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
void normalize_rt_tasks(void)
{
6606
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
6607
	unsigned long flags;
6608
	struct rq *rq;
I
Ingo Molnar 已提交
6609
	int on_rq;
L
Linus Torvalds 已提交
6610 6611

	read_lock_irq(&tasklist_lock);
6612
	do_each_thread(g, p) {
I
Ingo Molnar 已提交
6613 6614
		p->se.fair_key			= 0;
		p->se.wait_runtime		= 0;
I
Ingo Molnar 已提交
6615
		p->se.exec_start		= 0;
I
Ingo Molnar 已提交
6616
		p->se.wait_start_fair		= 0;
I
Ingo Molnar 已提交
6617 6618
		p->se.sleep_start_fair		= 0;
#ifdef CONFIG_SCHEDSTATS
I
Ingo Molnar 已提交
6619 6620 6621
		p->se.wait_start		= 0;
		p->se.sleep_start		= 0;
		p->se.block_start		= 0;
I
Ingo Molnar 已提交
6622
#endif
I
Ingo Molnar 已提交
6623 6624 6625 6626 6627 6628 6629 6630 6631 6632
		task_rq(p)->cfs.fair_clock	= 0;
		task_rq(p)->clock		= 0;

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
6633
			continue;
I
Ingo Molnar 已提交
6634
		}
L
Linus Torvalds 已提交
6635

6636 6637
		spin_lock_irqsave(&p->pi_lock, flags);
		rq = __task_rq_lock(p);
I
Ingo Molnar 已提交
6638 6639 6640 6641 6642 6643 6644
#ifdef CONFIG_SMP
		/*
		 * Do not touch the migration thread:
		 */
		if (p == rq->migration_thread)
			goto out_unlock;
#endif
L
Linus Torvalds 已提交
6645

I
Ingo Molnar 已提交
6646
		update_rq_clock(rq);
I
Ingo Molnar 已提交
6647
		on_rq = p->se.on_rq;
I
Ingo Molnar 已提交
6648 6649
		if (on_rq)
			deactivate_task(rq, p, 0);
I
Ingo Molnar 已提交
6650 6651
		__setscheduler(rq, p, SCHED_NORMAL, 0);
		if (on_rq) {
I
Ingo Molnar 已提交
6652
			activate_task(rq, p, 0);
L
Linus Torvalds 已提交
6653 6654
			resched_task(rq->curr);
		}
I
Ingo Molnar 已提交
6655 6656 6657
#ifdef CONFIG_SMP
 out_unlock:
#endif
6658 6659
		__task_rq_unlock(rq);
		spin_unlock_irqrestore(&p->pi_lock, flags);
6660 6661
	} while_each_thread(g, p);

L
Linus Torvalds 已提交
6662 6663 6664 6665
	read_unlock_irq(&tasklist_lock);
}

#endif /* CONFIG_MAGIC_SYSRQ */
6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683

#ifdef CONFIG_IA64
/*
 * These functions are only useful for the IA64 MCA handling.
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6684
struct task_struct *curr_task(int cpu)
6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703
{
	return cpu_curr(cpu);
}

/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
 * are serviced on a separate stack.  It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner.  This function
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
6704
void set_curr_task(int cpu, struct task_struct *p)
6705 6706 6707 6708 6709
{
	cpu_curr(cpu) = p;
}

#endif