core.c 199.5 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  kernel/sched/core.c
L
Linus Torvalds 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
25 26
 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 *              Thomas Gleixner, Mike Kravetz
L
Linus Torvalds 已提交
27 28 29 30 31 32
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
33
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
34 35 36
#include <linux/highmem.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
37
#include <linux/capability.h>
L
Linus Torvalds 已提交
38 39
#include <linux/completion.h>
#include <linux/kernel_stat.h>
40
#include <linux/debug_locks.h>
41
#include <linux/perf_event.h>
L
Linus Torvalds 已提交
42 43 44
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
45
#include <linux/freezer.h>
46
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
47 48
#include <linux/blkdev.h>
#include <linux/delay.h>
49
#include <linux/pid_namespace.h>
L
Linus Torvalds 已提交
50 51 52 53 54 55 56
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
57
#include <linux/proc_fs.h>
L
Linus Torvalds 已提交
58
#include <linux/seq_file.h>
59
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
60 61
#include <linux/syscalls.h>
#include <linux/times.h>
62
#include <linux/tsacct_kern.h>
63
#include <linux/kprobes.h>
64
#include <linux/delayacct.h>
65
#include <linux/unistd.h>
J
Jens Axboe 已提交
66
#include <linux/pagemap.h>
P
Peter Zijlstra 已提交
67
#include <linux/hrtimer.h>
R
Reynes Philippe 已提交
68
#include <linux/tick.h>
P
Peter Zijlstra 已提交
69 70
#include <linux/debugfs.h>
#include <linux/ctype.h>
71
#include <linux/ftrace.h>
72
#include <linux/slab.h>
73
#include <linux/init_task.h>
A
Al Viro 已提交
74
#include <linux/binfmts.h>
75
#include <linux/context_tracking.h>
L
Linus Torvalds 已提交
76

77
#include <asm/switch_to.h>
78
#include <asm/tlb.h>
79
#include <asm/irq_regs.h>
80
#include <asm/mutex.h>
G
Glauber Costa 已提交
81 82 83
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#endif
L
Linus Torvalds 已提交
84

85
#include "sched.h"
86
#include "../workqueue_internal.h"
87
#include "../smpboot.h"
88

89
#define CREATE_TRACE_POINTS
90
#include <trace/events/sched.h>
91

92
void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
93
{
94 95
	unsigned long delta;
	ktime_t soft, hard, now;
96

97 98 99 100 101 102
	for (;;) {
		if (hrtimer_active(period_timer))
			break;

		now = hrtimer_cb_get_time(period_timer);
		hrtimer_forward(period_timer, now, period);
103

104 105 106 107 108 109 110 111
		soft = hrtimer_get_softexpires(period_timer);
		hard = hrtimer_get_expires(period_timer);
		delta = ktime_to_ns(ktime_sub(hard, soft));
		__hrtimer_start_range_ns(period_timer, soft, delta,
					 HRTIMER_MODE_ABS_PINNED, 0);
	}
}

112 113
DEFINE_MUTEX(sched_domains_mutex);
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
114

115
static void update_rq_clock_task(struct rq *rq, s64 delta);
116

117
void update_rq_clock(struct rq *rq)
118
{
119
	s64 delta;
120

121
	if (rq->skip_clock_update > 0)
122
		return;
123

124 125 126
	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
	rq->clock += delta;
	update_rq_clock_task(rq, delta);
127 128
}

I
Ingo Molnar 已提交
129 130 131
/*
 * Debugging: various feature bits
 */
P
Peter Zijlstra 已提交
132 133 134 135

#define SCHED_FEAT(name, enabled)	\
	(1UL << __SCHED_FEAT_##name) * enabled |

I
Ingo Molnar 已提交
136
const_debug unsigned int sysctl_sched_features =
137
#include "features.h"
P
Peter Zijlstra 已提交
138 139 140 141 142 143 144 145
	0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG
#define SCHED_FEAT(name, enabled)	\
	#name ,

146
static const char * const sched_feat_names[] = {
147
#include "features.h"
P
Peter Zijlstra 已提交
148 149 150 151
};

#undef SCHED_FEAT

L
Li Zefan 已提交
152
static int sched_feat_show(struct seq_file *m, void *v)
P
Peter Zijlstra 已提交
153 154 155
{
	int i;

156
	for (i = 0; i < __SCHED_FEAT_NR; i++) {
L
Li Zefan 已提交
157 158 159
		if (!(sysctl_sched_features & (1UL << i)))
			seq_puts(m, "NO_");
		seq_printf(m, "%s ", sched_feat_names[i]);
P
Peter Zijlstra 已提交
160
	}
L
Li Zefan 已提交
161
	seq_puts(m, "\n");
P
Peter Zijlstra 已提交
162

L
Li Zefan 已提交
163
	return 0;
P
Peter Zijlstra 已提交
164 165
}

166 167
#ifdef HAVE_JUMP_LABEL

168 169
#define jump_label_key__true  STATIC_KEY_INIT_TRUE
#define jump_label_key__false STATIC_KEY_INIT_FALSE
170 171 172 173

#define SCHED_FEAT(name, enabled)	\
	jump_label_key__##enabled ,

174
struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
175 176 177 178 179 180 181
#include "features.h"
};

#undef SCHED_FEAT

static void sched_feat_disable(int i)
{
182 183
	if (static_key_enabled(&sched_feat_keys[i]))
		static_key_slow_dec(&sched_feat_keys[i]);
184 185 186 187
}

static void sched_feat_enable(int i)
{
188 189
	if (!static_key_enabled(&sched_feat_keys[i]))
		static_key_slow_inc(&sched_feat_keys[i]);
190 191 192 193 194 195
}
#else
static void sched_feat_disable(int i) { };
static void sched_feat_enable(int i) { };
#endif /* HAVE_JUMP_LABEL */

196
static int sched_feat_set(char *cmp)
P
Peter Zijlstra 已提交
197 198
{
	int i;
199
	int neg = 0;
P
Peter Zijlstra 已提交
200

H
Hillf Danton 已提交
201
	if (strncmp(cmp, "NO_", 3) == 0) {
P
Peter Zijlstra 已提交
202 203 204 205
		neg = 1;
		cmp += 3;
	}

206
	for (i = 0; i < __SCHED_FEAT_NR; i++) {
207
		if (strcmp(cmp, sched_feat_names[i]) == 0) {
208
			if (neg) {
P
Peter Zijlstra 已提交
209
				sysctl_sched_features &= ~(1UL << i);
210 211
				sched_feat_disable(i);
			} else {
P
Peter Zijlstra 已提交
212
				sysctl_sched_features |= (1UL << i);
213 214
				sched_feat_enable(i);
			}
P
Peter Zijlstra 已提交
215 216 217 218
			break;
		}
	}

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
	return i;
}

static ssize_t
sched_feat_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
	char buf[64];
	char *cmp;
	int i;

	if (cnt > 63)
		cnt = 63;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;
	cmp = strstrip(buf);

	i = sched_feat_set(cmp);
240
	if (i == __SCHED_FEAT_NR)
P
Peter Zijlstra 已提交
241 242
		return -EINVAL;

243
	*ppos += cnt;
P
Peter Zijlstra 已提交
244 245 246 247

	return cnt;
}

L
Li Zefan 已提交
248 249 250 251 252
static int sched_feat_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_feat_show, NULL);
}

253
static const struct file_operations sched_feat_fops = {
L
Li Zefan 已提交
254 255 256 257 258
	.open		= sched_feat_open,
	.write		= sched_feat_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
P
Peter Zijlstra 已提交
259 260 261 262 263 264 265 266 267 268
};

static __init int sched_init_debug(void)
{
	debugfs_create_file("sched_features", 0644, NULL, NULL,
			&sched_feat_fops);

	return 0;
}
late_initcall(sched_init_debug);
269
#endif /* CONFIG_SCHED_DEBUG */
I
Ingo Molnar 已提交
270

271 272 273 274 275 276
/*
 * Number of tasks to iterate in a single balance run.
 * Limited because this is done with IRQs disabled.
 */
const_debug unsigned int sysctl_sched_nr_migrate = 32;

277 278 279 280 281 282 283 284
/*
 * period over which we average the RT time consumption, measured
 * in ms.
 *
 * default: 1s
 */
const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;

P
Peter Zijlstra 已提交
285
/*
P
Peter Zijlstra 已提交
286
 * period over which we measure -rt task cpu usage in us.
P
Peter Zijlstra 已提交
287 288
 * default: 1s
 */
P
Peter Zijlstra 已提交
289
unsigned int sysctl_sched_rt_period = 1000000;
P
Peter Zijlstra 已提交
290

291
__read_mostly int scheduler_running;
292

P
Peter Zijlstra 已提交
293 294 295 296 297
/*
 * part of the period that we allow rt tasks to run in us.
 * default: 0.95s
 */
int sysctl_sched_rt_runtime = 950000;
P
Peter Zijlstra 已提交
298 299


L
Linus Torvalds 已提交
300

301
/*
302
 * __task_rq_lock - lock the rq @p resides on.
303
 */
304
static inline struct rq *__task_rq_lock(struct task_struct *p)
305 306
	__acquires(rq->lock)
{
307 308
	struct rq *rq;

309 310
	lockdep_assert_held(&p->pi_lock);

311
	for (;;) {
312
		rq = task_rq(p);
313
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
314
		if (likely(rq == task_rq(p)))
315
			return rq;
316
		raw_spin_unlock(&rq->lock);
317 318 319
	}
}

L
Linus Torvalds 已提交
320
/*
321
 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
L
Linus Torvalds 已提交
322
 */
323
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
324
	__acquires(p->pi_lock)
L
Linus Torvalds 已提交
325 326
	__acquires(rq->lock)
{
327
	struct rq *rq;
L
Linus Torvalds 已提交
328

329
	for (;;) {
330
		raw_spin_lock_irqsave(&p->pi_lock, *flags);
331
		rq = task_rq(p);
332
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
333
		if (likely(rq == task_rq(p)))
334
			return rq;
335 336
		raw_spin_unlock(&rq->lock);
		raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
337 338 339
	}
}

A
Alexey Dobriyan 已提交
340
static void __task_rq_unlock(struct rq *rq)
341 342
	__releases(rq->lock)
{
343
	raw_spin_unlock(&rq->lock);
344 345
}

346 347
static inline void
task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
348
	__releases(rq->lock)
349
	__releases(p->pi_lock)
L
Linus Torvalds 已提交
350
{
351 352
	raw_spin_unlock(&rq->lock);
	raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
353 354 355
}

/*
356
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
357
 */
A
Alexey Dobriyan 已提交
358
static struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
359 360
	__acquires(rq->lock)
{
361
	struct rq *rq;
L
Linus Torvalds 已提交
362 363 364

	local_irq_disable();
	rq = this_rq();
365
	raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
366 367 368 369

	return rq;
}

P
Peter Zijlstra 已提交
370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
#ifdef CONFIG_SCHED_HRTICK
/*
 * Use HR-timers to deliver accurate preemption points.
 *
 * Its all a bit involved since we cannot program an hrt while holding the
 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
 * reschedule event.
 *
 * When we get rescheduled we reprogram the hrtick_timer outside of the
 * rq->lock.
 */

static void hrtick_clear(struct rq *rq)
{
	if (hrtimer_active(&rq->hrtick_timer))
		hrtimer_cancel(&rq->hrtick_timer);
}

/*
 * High-resolution timer tick.
 * Runs from hardirq context with interrupts disabled.
 */
static enum hrtimer_restart hrtick(struct hrtimer *timer)
{
	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

398
	raw_spin_lock(&rq->lock);
399
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
400
	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
401
	raw_spin_unlock(&rq->lock);
P
Peter Zijlstra 已提交
402 403 404 405

	return HRTIMER_NORESTART;
}

406
#ifdef CONFIG_SMP
407 408 409 410
/*
 * called from hardirq (IPI) context
 */
static void __hrtick_start(void *arg)
411
{
412
	struct rq *rq = arg;
413

414
	raw_spin_lock(&rq->lock);
415 416
	hrtimer_restart(&rq->hrtick_timer);
	rq->hrtick_csd_pending = 0;
417
	raw_spin_unlock(&rq->lock);
418 419
}

420 421 422 423 424
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
425
void hrtick_start(struct rq *rq, u64 delay)
426
{
427 428
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
429

430
	hrtimer_set_expires(timer, time);
431 432 433 434

	if (rq == this_rq()) {
		hrtimer_restart(timer);
	} else if (!rq->hrtick_csd_pending) {
435
		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
436 437
		rq->hrtick_csd_pending = 1;
	}
438 439 440 441 442 443 444 445 446 447 448 449 450 451
}

static int
hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
	int cpu = (int)(long)hcpu;

	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
452
		hrtick_clear(cpu_rq(cpu));
453 454 455 456 457 458
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

459
static __init void init_hrtick(void)
460 461 462
{
	hotcpu_notifier(hotplug_hrtick, 0);
}
463 464 465 466 467 468
#else
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
469
void hrtick_start(struct rq *rq, u64 delay)
470
{
471
	__hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
472
			HRTIMER_MODE_REL_PINNED, 0);
473
}
474

A
Andrew Morton 已提交
475
static inline void init_hrtick(void)
P
Peter Zijlstra 已提交
476 477
{
}
478
#endif /* CONFIG_SMP */
P
Peter Zijlstra 已提交
479

480
static void init_rq_hrtick(struct rq *rq)
P
Peter Zijlstra 已提交
481
{
482 483
#ifdef CONFIG_SMP
	rq->hrtick_csd_pending = 0;
P
Peter Zijlstra 已提交
484

485 486 487 488
	rq->hrtick_csd.flags = 0;
	rq->hrtick_csd.func = __hrtick_start;
	rq->hrtick_csd.info = rq;
#endif
P
Peter Zijlstra 已提交
489

490 491
	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rq->hrtick_timer.function = hrtick;
P
Peter Zijlstra 已提交
492
}
A
Andrew Morton 已提交
493
#else	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
494 495 496 497 498 499 500 501
static inline void hrtick_clear(struct rq *rq)
{
}

static inline void init_rq_hrtick(struct rq *rq)
{
}

502 503 504
static inline void init_hrtick(void)
{
}
A
Andrew Morton 已提交
505
#endif	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
506

I
Ingo Molnar 已提交
507 508 509 510 511 512 513 514 515 516
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
A
Al Viro 已提交
517
#define tsk_is_polling(t) 0
I
Ingo Molnar 已提交
518 519
#endif

520
void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
521 522 523
{
	int cpu;

524
	assert_raw_spin_locked(&task_rq(p)->lock);
I
Ingo Molnar 已提交
525

526
	if (test_tsk_need_resched(p))
I
Ingo Molnar 已提交
527 528
		return;

529
	set_tsk_need_resched(p);
I
Ingo Molnar 已提交
530 531 532 533 534 535 536 537 538 539 540

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

541
void resched_cpu(int cpu)
I
Ingo Molnar 已提交
542 543 544 545
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

546
	if (!raw_spin_trylock_irqsave(&rq->lock, flags))
I
Ingo Molnar 已提交
547 548
		return;
	resched_task(cpu_curr(cpu));
549
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
550
}
551

552
#ifdef CONFIG_NO_HZ_COMMON
553 554 555 556 557 558 559 560 561 562 563 564 565 566
/*
 * In the semi idle case, use the nearest busy cpu for migrating timers
 * from an idle cpu.  This is good for power-savings.
 *
 * We don't do similar optimization for completely idle system, as
 * selecting an idle cpu will add more delays to the timers than intended
 * (as that cpu's timer base may not be uptodate wrt jiffies etc).
 */
int get_nohz_timer_target(void)
{
	int cpu = smp_processor_id();
	int i;
	struct sched_domain *sd;

567
	rcu_read_lock();
568
	for_each_domain(cpu, sd) {
569 570 571 572 573 574
		for_each_cpu(i, sched_domain_span(sd)) {
			if (!idle_cpu(i)) {
				cpu = i;
				goto unlock;
			}
		}
575
	}
576 577
unlock:
	rcu_read_unlock();
578 579
	return cpu;
}
580 581 582 583 584 585 586 587 588 589
/*
 * When add_timer_on() enqueues a timer into the timer wheel of an
 * idle CPU then this timer might expire before the next timer event
 * which is scheduled to wake up that CPU. In case of a completely
 * idle system the next event might even be infinite time into the
 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 * leaves the inner idle loop so the newly added timer is taken into
 * account when the CPU goes back to idle and evaluates the timer
 * wheel for the next timer event.
 */
590
static void wake_up_idle_cpu(int cpu)
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
{
	struct rq *rq = cpu_rq(cpu);

	if (cpu == smp_processor_id())
		return;

	/*
	 * This is safe, as this function is called with the timer
	 * wheel base lock of (cpu) held. When the CPU is on the way
	 * to idle and has not yet set rq->curr to idle then it will
	 * be serialized on the timer wheel base lock and take the new
	 * timer into account automatically.
	 */
	if (rq->curr != rq->idle)
		return;
606 607

	/*
608 609 610
	 * We can set TIF_RESCHED on the idle task of the other CPU
	 * lockless. The worst case is that the other CPU runs the
	 * idle task through an additional NOOP schedule()
611
	 */
612
	set_tsk_need_resched(rq->idle);
613

614 615 616 617
	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(rq->idle))
		smp_send_reschedule(cpu);
618 619
}

620
static bool wake_up_full_nohz_cpu(int cpu)
621
{
622
	if (tick_nohz_full_cpu(cpu)) {
623 624 625 626 627 628 629 630 631 632 633
		if (cpu != smp_processor_id() ||
		    tick_nohz_tick_stopped())
			smp_send_reschedule(cpu);
		return true;
	}

	return false;
}

void wake_up_nohz_cpu(int cpu)
{
634
	if (!wake_up_full_nohz_cpu(cpu))
635 636 637
		wake_up_idle_cpu(cpu);
}

638
static inline bool got_nohz_idle_kick(void)
639
{
640 641
	int cpu = smp_processor_id();
	return idle_cpu(cpu) && test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
642 643
}

644
#else /* CONFIG_NO_HZ_COMMON */
645

646
static inline bool got_nohz_idle_kick(void)
P
Peter Zijlstra 已提交
647
{
648
	return false;
P
Peter Zijlstra 已提交
649 650
}

651
#endif /* CONFIG_NO_HZ_COMMON */
652

653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
#ifdef CONFIG_NO_HZ_FULL
bool sched_can_stop_tick(void)
{
       struct rq *rq;

       rq = this_rq();

       /* Make sure rq->nr_running update is visible after the IPI */
       smp_rmb();

       /* More than one running task need preemption */
       if (rq->nr_running > 1)
               return false;

       return true;
}
#endif /* CONFIG_NO_HZ_FULL */

671
void sched_avg_update(struct rq *rq)
672
{
673 674 675
	s64 period = sched_avg_period();

	while ((s64)(rq->clock - rq->age_stamp) > period) {
676 677 678 679 680 681
		/*
		 * Inline assembly required to prevent the compiler
		 * optimising this loop into a divmod call.
		 * See __iter_div_u64_rem() for another example of this.
		 */
		asm("" : "+rm" (rq->age_stamp));
682 683 684
		rq->age_stamp += period;
		rq->rt_avg /= 2;
	}
685 686
}

687
#else /* !CONFIG_SMP */
688
void resched_task(struct task_struct *p)
689
{
690
	assert_raw_spin_locked(&task_rq(p)->lock);
691
	set_tsk_need_resched(p);
692
}
693
#endif /* CONFIG_SMP */
694

695 696
#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
697
/*
698 699 700 701
 * Iterate task_group tree rooted at *from, calling @down when first entering a
 * node and @up when leaving it for the final time.
 *
 * Caller must hold rcu_lock or sufficient equivalent.
702
 */
703
int walk_tg_tree_from(struct task_group *from,
704
			     tg_visitor down, tg_visitor up, void *data)
705 706
{
	struct task_group *parent, *child;
P
Peter Zijlstra 已提交
707
	int ret;
708

709 710
	parent = from;

711
down:
P
Peter Zijlstra 已提交
712 713
	ret = (*down)(parent, data);
	if (ret)
714
		goto out;
715 716 717 718 719 720 721
	list_for_each_entry_rcu(child, &parent->children, siblings) {
		parent = child;
		goto down;

up:
		continue;
	}
P
Peter Zijlstra 已提交
722
	ret = (*up)(parent, data);
723 724
	if (ret || parent == from)
		goto out;
725 726 727 728 729

	child = parent;
	parent = parent->parent;
	if (parent)
		goto up;
730
out:
P
Peter Zijlstra 已提交
731
	return ret;
732 733
}

734
int tg_nop(struct task_group *tg, void *data)
P
Peter Zijlstra 已提交
735
{
736
	return 0;
P
Peter Zijlstra 已提交
737
}
738 739
#endif

740 741
static void set_load_weight(struct task_struct *p)
{
N
Nikhil Rao 已提交
742 743 744
	int prio = p->static_prio - MAX_RT_PRIO;
	struct load_weight *load = &p->se.load;

I
Ingo Molnar 已提交
745 746 747 748
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
749
		load->weight = scale_load(WEIGHT_IDLEPRIO);
N
Nikhil Rao 已提交
750
		load->inv_weight = WMULT_IDLEPRIO;
I
Ingo Molnar 已提交
751 752
		return;
	}
753

754
	load->weight = scale_load(prio_to_weight[prio]);
N
Nikhil Rao 已提交
755
	load->inv_weight = prio_to_wmult[prio];
756 757
}

758
static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
759
{
760
	update_rq_clock(rq);
I
Ingo Molnar 已提交
761
	sched_info_queued(p);
762
	p->sched_class->enqueue_task(rq, p, flags);
763 764
}

765
static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
766
{
767
	update_rq_clock(rq);
768
	sched_info_dequeued(p);
769
	p->sched_class->dequeue_task(rq, p, flags);
770 771
}

772
void activate_task(struct rq *rq, struct task_struct *p, int flags)
773 774 775 776
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible--;

777
	enqueue_task(rq, p, flags);
778 779
}

780
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
781 782 783 784
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible++;

785
	dequeue_task(rq, p, flags);
786 787
}

788
static void update_rq_clock_task(struct rq *rq, s64 delta)
789
{
790 791 792 793 794 795 796 797
/*
 * In theory, the compile should just see 0 here, and optimize out the call
 * to sched_rt_avg_update. But I don't trust it...
 */
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
	s64 steal = 0, irq_delta = 0;
#endif
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
798
	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819

	/*
	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
	 * this case when a previous update_rq_clock() happened inside a
	 * {soft,}irq region.
	 *
	 * When this happens, we stop ->clock_task and only update the
	 * prev_irq_time stamp to account for the part that fit, so that a next
	 * update will consume the rest. This ensures ->clock_task is
	 * monotonic.
	 *
	 * It does however cause some slight miss-attribution of {soft,}irq
	 * time, a more accurate solution would be to update the irq_time using
	 * the current rq->clock timestamp, except that would require using
	 * atomic ops.
	 */
	if (irq_delta > delta)
		irq_delta = delta;

	rq->prev_irq_time += irq_delta;
	delta -= irq_delta;
820 821
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
822
	if (static_key_false((&paravirt_steal_rq_enabled))) {
823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
		u64 st;

		steal = paravirt_steal_clock(cpu_of(rq));
		steal -= rq->prev_steal_time_rq;

		if (unlikely(steal > delta))
			steal = delta;

		st = steal_ticks(steal);
		steal = st * TICK_NSEC;

		rq->prev_steal_time_rq += steal;

		delta -= steal;
	}
#endif

840 841
	rq->clock_task += delta;

842 843 844 845
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
	if ((irq_delta + steal) && sched_feat(NONTASK_POWER))
		sched_rt_avg_update(rq, irq_delta + steal);
#endif
846 847
}

848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877
void sched_set_stop_task(int cpu, struct task_struct *stop)
{
	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	struct task_struct *old_stop = cpu_rq(cpu)->stop;

	if (stop) {
		/*
		 * Make it appear like a SCHED_FIFO task, its something
		 * userspace knows about and won't get confused about.
		 *
		 * Also, it will make PI more or less work without too
		 * much confusion -- but then, stop work should not
		 * rely on PI working anyway.
		 */
		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);

		stop->sched_class = &stop_sched_class;
	}

	cpu_rq(cpu)->stop = stop;

	if (old_stop) {
		/*
		 * Reset it back to a normal scheduling class so that
		 * it can die in pieces.
		 */
		old_stop->sched_class = &rt_sched_class;
	}
}

878
/*
I
Ingo Molnar 已提交
879
 * __normal_prio - return the priority that is based on the static prio
880 881 882
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
883
	return p->static_prio;
884 885
}

886 887 888 889 890 891 892
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
893
static inline int normal_prio(struct task_struct *p)
894 895 896
{
	int prio;

897
	if (task_has_rt_policy(p))
898 899 900 901 902 903 904 905 906 907 908 909 910
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
911
static int effective_prio(struct task_struct *p)
912 913 914 915 916 917 918 919 920 921 922 923
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
924 925 926 927
/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
 */
928
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
929 930 931 932
{
	return cpu_curr(task_cpu(p)) == p;
}

933 934
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
				       const struct sched_class *prev_class,
P
Peter Zijlstra 已提交
935
				       int oldprio)
936 937 938
{
	if (prev_class != p->sched_class) {
		if (prev_class->switched_from)
P
Peter Zijlstra 已提交
939 940 941 942
			prev_class->switched_from(rq, p);
		p->sched_class->switched_to(rq, p);
	} else if (oldprio != p->prio)
		p->sched_class->prio_changed(rq, p, oldprio);
943 944
}

945
void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
{
	const struct sched_class *class;

	if (p->sched_class == rq->curr->sched_class) {
		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
	} else {
		for_each_class(class) {
			if (class == rq->curr->sched_class)
				break;
			if (class == p->sched_class) {
				resched_task(rq->curr);
				break;
			}
		}
	}

	/*
	 * A queue event has occurred, and we're going to schedule.  In
	 * this case, we can save a useless back to back clock update.
	 */
P
Peter Zijlstra 已提交
966
	if (rq->curr->on_rq && test_tsk_need_resched(rq->curr))
967 968 969
		rq->skip_clock_update = 1;
}

970 971 972 973 974 975 976
static ATOMIC_NOTIFIER_HEAD(task_migration_notifier);

void register_task_migration_notifier(struct notifier_block *n)
{
	atomic_notifier_chain_register(&task_migration_notifier, n);
}

L
Linus Torvalds 已提交
977
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
978
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
979
{
980 981 982 983 984
#ifdef CONFIG_SCHED_DEBUG
	/*
	 * We should never call set_task_cpu() on a blocked task,
	 * ttwu() will sort out the placement.
	 */
P
Peter Zijlstra 已提交
985 986
	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
			!(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
987 988

#ifdef CONFIG_LOCKDEP
989 990 991 992 993
	/*
	 * The caller should hold either p->pi_lock or rq->lock, when changing
	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
	 *
	 * sched_move_task() holds both and thus holding either pins the cgroup,
P
Peter Zijlstra 已提交
994
	 * see task_group().
995 996 997 998
	 *
	 * Furthermore, all task_rq users should acquire both locks, see
	 * task_rq_lock().
	 */
999 1000 1001
	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
				      lockdep_is_held(&task_rq(p)->lock)));
#endif
1002 1003
#endif

1004
	trace_sched_migrate_task(p, new_cpu);
1005

1006
	if (task_cpu(p) != new_cpu) {
1007 1008
		struct task_migration_notifier tmn;

1009 1010
		if (p->sched_class->migrate_task_rq)
			p->sched_class->migrate_task_rq(p, new_cpu);
1011
		p->se.nr_migrations++;
1012
		perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
1013 1014 1015 1016 1017 1018

		tmn.task = p;
		tmn.from_cpu = task_cpu(p);
		tmn.to_cpu = new_cpu;

		atomic_notifier_call_chain(&task_migration_notifier, 0, &tmn);
1019
	}
I
Ingo Molnar 已提交
1020 1021

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1022 1023
}

1024
struct migration_arg {
1025
	struct task_struct *task;
L
Linus Torvalds 已提交
1026
	int dest_cpu;
1027
};
L
Linus Torvalds 已提交
1028

1029 1030
static int migration_cpu_stop(void *data);

L
Linus Torvalds 已提交
1031 1032 1033
/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
R
Roland McGrath 已提交
1034 1035 1036 1037 1038 1039 1040
 * If @match_state is nonzero, it's the @p->state value just checked and
 * not expected to change.  If it changes, i.e. @p might have woken up,
 * then return zero.  When we succeed in waiting for @p to be off its CPU,
 * we return a positive number (its total switch count).  If a second call
 * a short while later returns the same number, the caller can be sure that
 * @p has remained unscheduled the whole time.
 *
L
Linus Torvalds 已提交
1041 1042 1043 1044 1045 1046
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
R
Roland McGrath 已提交
1047
unsigned long wait_task_inactive(struct task_struct *p, long match_state)
L
Linus Torvalds 已提交
1048 1049
{
	unsigned long flags;
I
Ingo Molnar 已提交
1050
	int running, on_rq;
R
Roland McGrath 已提交
1051
	unsigned long ncsw;
1052
	struct rq *rq;
L
Linus Torvalds 已提交
1053

1054 1055 1056 1057 1058 1059 1060 1061
	for (;;) {
		/*
		 * We do the initial early heuristics without holding
		 * any task-queue locks at all. We'll only try to get
		 * the runqueue lock when things look like they will
		 * work out!
		 */
		rq = task_rq(p);
1062

1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
		/*
		 * If the task is actively running on another CPU
		 * still, just relax and busy-wait without holding
		 * any locks.
		 *
		 * NOTE! Since we don't hold any locks, it's not
		 * even sure that "rq" stays as the right runqueue!
		 * But we don't care, since "task_running()" will
		 * return false if the runqueue has changed and p
		 * is actually now running somewhere else!
		 */
R
Roland McGrath 已提交
1074 1075 1076
		while (task_running(rq, p)) {
			if (match_state && unlikely(p->state != match_state))
				return 0;
1077
			cpu_relax();
R
Roland McGrath 已提交
1078
		}
1079

1080 1081 1082 1083 1084 1085
		/*
		 * Ok, time to look more closely! We need the rq
		 * lock now, to be *sure*. If we're wrong, we'll
		 * just go back and repeat.
		 */
		rq = task_rq_lock(p, &flags);
1086
		trace_sched_wait_task(p);
1087
		running = task_running(rq, p);
P
Peter Zijlstra 已提交
1088
		on_rq = p->on_rq;
R
Roland McGrath 已提交
1089
		ncsw = 0;
1090
		if (!match_state || p->state == match_state)
1091
			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1092
		task_rq_unlock(rq, p, &flags);
1093

R
Roland McGrath 已提交
1094 1095 1096 1097 1098 1099
		/*
		 * If it changed from the expected state, bail out now.
		 */
		if (unlikely(!ncsw))
			break;

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
		/*
		 * Was it really running after all now that we
		 * checked with the proper locks actually held?
		 *
		 * Oops. Go back and try again..
		 */
		if (unlikely(running)) {
			cpu_relax();
			continue;
		}
1110

1111 1112 1113 1114 1115
		/*
		 * It's not enough that it's not actively running,
		 * it must be off the runqueue _entirely_, and not
		 * preempted!
		 *
1116
		 * So if it was still runnable (but just not actively
1117 1118 1119 1120
		 * running right now), it's preempted, and we should
		 * yield - it could be a while.
		 */
		if (unlikely(on_rq)) {
1121 1122 1123 1124
			ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);

			set_current_state(TASK_UNINTERRUPTIBLE);
			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1125 1126
			continue;
		}
1127

1128 1129 1130 1131 1132 1133 1134
		/*
		 * Ahh, all good. It wasn't running, and it wasn't
		 * runnable, which means that it will never become
		 * running in the future either. We're all done!
		 */
		break;
	}
R
Roland McGrath 已提交
1135 1136

	return ncsw;
L
Linus Torvalds 已提交
1137 1138 1139 1140 1141 1142 1143 1144 1145
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
L
Lucas De Marchi 已提交
1146
 * NOTE: this function doesn't have to take the runqueue lock,
L
Linus Torvalds 已提交
1147 1148 1149 1150 1151
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
1152
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
1153 1154 1155 1156 1157 1158 1159 1160 1161
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}
R
Rusty Russell 已提交
1162
EXPORT_SYMBOL_GPL(kick_process);
N
Nick Piggin 已提交
1163
#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
1164

1165
#ifdef CONFIG_SMP
1166
/*
1167
 * ->cpus_allowed is protected by both rq->lock and p->pi_lock
1168
 */
1169 1170
static int select_fallback_rq(int cpu, struct task_struct *p)
{
1171 1172
	int nid = cpu_to_node(cpu);
	const struct cpumask *nodemask = NULL;
1173 1174
	enum { cpuset, possible, fail } state = cpuset;
	int dest_cpu;
1175

1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
	/*
	 * If the node that the cpu is on has been offlined, cpu_to_node()
	 * will return -1. There is no cpu on the node, and we should
	 * select the cpu on the other node.
	 */
	if (nid != -1) {
		nodemask = cpumask_of_node(nid);

		/* Look for allowed, online CPU in same node. */
		for_each_cpu(dest_cpu, nodemask) {
			if (!cpu_online(dest_cpu))
				continue;
			if (!cpu_active(dest_cpu))
				continue;
			if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
				return dest_cpu;
		}
1193
	}
1194

1195 1196
	for (;;) {
		/* Any allowed, online CPU? */
1197
		for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
1198 1199 1200 1201 1202 1203
			if (!cpu_online(dest_cpu))
				continue;
			if (!cpu_active(dest_cpu))
				continue;
			goto out;
		}
1204

1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
		switch (state) {
		case cpuset:
			/* No more Mr. Nice Guy. */
			cpuset_cpus_allowed_fallback(p);
			state = possible;
			break;

		case possible:
			do_set_cpus_allowed(p, cpu_possible_mask);
			state = fail;
			break;

		case fail:
			BUG();
			break;
		}
	}

out:
	if (state != cpuset) {
		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
		if (p->mm && printk_ratelimit()) {
			printk_sched("process %d (%s) no longer affine to cpu%d\n",
					task_pid_nr(p), p->comm, cpu);
		}
1234 1235 1236 1237 1238
	}

	return dest_cpu;
}

1239
/*
1240
 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
1241
 */
1242
static inline
1243
int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
1244
{
1245
	int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256

	/*
	 * In order not to call set_task_cpu() on a blocking task we need
	 * to rely on ttwu() to place the task on a valid ->cpus_allowed
	 * cpu.
	 *
	 * Since this is common to all placement strategies, this lives here.
	 *
	 * [ this allows ->select_task() to simply return task_cpu(p) and
	 *   not worry about this generic constraint ]
	 */
1257
	if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
P
Peter Zijlstra 已提交
1258
		     !cpu_online(cpu)))
1259
		cpu = select_fallback_rq(task_cpu(p), p);
1260 1261

	return cpu;
1262
}
1263 1264 1265 1266 1267 1268

static void update_avg(u64 *avg, u64 sample)
{
	s64 diff = sample - *avg;
	*avg += diff >> 3;
}
1269 1270
#endif

P
Peter Zijlstra 已提交
1271
static void
1272
ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
T
Tejun Heo 已提交
1273
{
P
Peter Zijlstra 已提交
1274
#ifdef CONFIG_SCHEDSTATS
1275 1276
	struct rq *rq = this_rq();

P
Peter Zijlstra 已提交
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
#ifdef CONFIG_SMP
	int this_cpu = smp_processor_id();

	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
		schedstat_inc(p, se.statistics.nr_wakeups_local);
	} else {
		struct sched_domain *sd;

		schedstat_inc(p, se.statistics.nr_wakeups_remote);
1287
		rcu_read_lock();
P
Peter Zijlstra 已提交
1288 1289 1290 1291 1292 1293
		for_each_domain(this_cpu, sd) {
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
				schedstat_inc(sd, ttwu_wake_remote);
				break;
			}
		}
1294
		rcu_read_unlock();
P
Peter Zijlstra 已提交
1295
	}
1296 1297 1298 1299

	if (wake_flags & WF_MIGRATED)
		schedstat_inc(p, se.statistics.nr_wakeups_migrate);

P
Peter Zijlstra 已提交
1300 1301 1302
#endif /* CONFIG_SMP */

	schedstat_inc(rq, ttwu_count);
T
Tejun Heo 已提交
1303
	schedstat_inc(p, se.statistics.nr_wakeups);
P
Peter Zijlstra 已提交
1304 1305

	if (wake_flags & WF_SYNC)
T
Tejun Heo 已提交
1306
		schedstat_inc(p, se.statistics.nr_wakeups_sync);
P
Peter Zijlstra 已提交
1307 1308 1309 1310 1311 1312

#endif /* CONFIG_SCHEDSTATS */
}

static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
{
T
Tejun Heo 已提交
1313
	activate_task(rq, p, en_flags);
P
Peter Zijlstra 已提交
1314
	p->on_rq = 1;
1315 1316 1317 1318

	/* if a worker is waking up, notify workqueue */
	if (p->flags & PF_WQ_WORKER)
		wq_worker_waking_up(p, cpu_of(rq));
T
Tejun Heo 已提交
1319 1320
}

1321 1322 1323
/*
 * Mark the task runnable and perform wakeup-preemption.
 */
1324
static void
1325
ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
T
Tejun Heo 已提交
1326 1327
{
	check_preempt_curr(rq, p, wake_flags);
1328
	trace_sched_wakeup(p, true);
T
Tejun Heo 已提交
1329 1330 1331 1332 1333 1334

	p->state = TASK_RUNNING;
#ifdef CONFIG_SMP
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);

1335
	if (rq->idle_stamp) {
T
Tejun Heo 已提交
1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
		u64 delta = rq->clock - rq->idle_stamp;
		u64 max = 2*sysctl_sched_migration_cost;

		if (delta > max)
			rq->avg_idle = max;
		else
			update_avg(&rq->avg_idle, delta);
		rq->idle_stamp = 0;
	}
#endif
}

1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
static void
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
{
#ifdef CONFIG_SMP
	if (p->sched_contributes_to_load)
		rq->nr_uninterruptible--;
#endif

	ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
	ttwu_do_wakeup(rq, p, wake_flags);
}

/*
 * Called in case the task @p isn't fully descheduled from its runqueue,
 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
 * since all we need to do is flip p->state to TASK_RUNNING, since
 * the task is still ->on_rq.
 */
static int ttwu_remote(struct task_struct *p, int wake_flags)
{
	struct rq *rq;
	int ret = 0;

	rq = __task_rq_lock(p);
	if (p->on_rq) {
		ttwu_do_wakeup(rq, p, wake_flags);
		ret = 1;
	}
	__task_rq_unlock(rq);

	return ret;
}

1381
#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
1382
static void sched_ttwu_pending(void)
1383 1384
{
	struct rq *rq = this_rq();
P
Peter Zijlstra 已提交
1385 1386
	struct llist_node *llist = llist_del_all(&rq->wake_list);
	struct task_struct *p;
1387 1388 1389

	raw_spin_lock(&rq->lock);

P
Peter Zijlstra 已提交
1390 1391 1392
	while (llist) {
		p = llist_entry(llist, struct task_struct, wake_entry);
		llist = llist_next(llist);
1393 1394 1395 1396 1397 1398 1399 1400
		ttwu_do_activate(rq, p, 0);
	}

	raw_spin_unlock(&rq->lock);
}

void scheduler_ipi(void)
{
1401 1402
	if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()
	    && !tick_nohz_full_cpu(smp_processor_id()))
1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418
		return;

	/*
	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
	 * traditionally all their work was done from the interrupt return
	 * path. Now that we actually do some work, we need to make sure
	 * we do call them.
	 *
	 * Some archs already do call them, luckily irq_enter/exit nest
	 * properly.
	 *
	 * Arguably we should visit all archs and update all handlers,
	 * however a fair share of IPIs are still resched only so this would
	 * somewhat pessimize the simple resched case.
	 */
	irq_enter();
1419
	tick_nohz_full_check();
P
Peter Zijlstra 已提交
1420
	sched_ttwu_pending();
1421 1422 1423 1424

	/*
	 * Check if someone kicked us for doing the nohz idle load balance.
	 */
1425 1426
	if (unlikely(got_nohz_idle_kick() && !need_resched())) {
		this_rq()->idle_balance = 1;
1427
		raise_softirq_irqoff(SCHED_SOFTIRQ);
1428
	}
1429
	irq_exit();
1430 1431 1432 1433
}

static void ttwu_queue_remote(struct task_struct *p, int cpu)
{
P
Peter Zijlstra 已提交
1434
	if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list))
1435 1436
		smp_send_reschedule(cpu);
}
1437

1438
bool cpus_share_cache(int this_cpu, int that_cpu)
1439 1440 1441
{
	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
}
1442
#endif /* CONFIG_SMP */
1443

1444 1445 1446 1447
static void ttwu_queue(struct task_struct *p, int cpu)
{
	struct rq *rq = cpu_rq(cpu);

1448
#if defined(CONFIG_SMP)
1449
	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
1450
		sched_clock_cpu(cpu); /* sync clocks x-cpu */
1451 1452 1453 1454 1455
		ttwu_queue_remote(p, cpu);
		return;
	}
#endif

1456 1457 1458
	raw_spin_lock(&rq->lock);
	ttwu_do_activate(rq, p, 0);
	raw_spin_unlock(&rq->lock);
T
Tejun Heo 已提交
1459 1460 1461
}

/**
L
Linus Torvalds 已提交
1462
 * try_to_wake_up - wake up a thread
T
Tejun Heo 已提交
1463
 * @p: the thread to be awakened
L
Linus Torvalds 已提交
1464
 * @state: the mask of task states that can be woken
T
Tejun Heo 已提交
1465
 * @wake_flags: wake modifier flags (WF_*)
L
Linus Torvalds 已提交
1466 1467 1468 1469 1470 1471 1472
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
T
Tejun Heo 已提交
1473 1474
 * Returns %true if @p was woken up, %false if it was already running
 * or @state didn't match @p's state.
L
Linus Torvalds 已提交
1475
 */
1476 1477
static int
try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
L
Linus Torvalds 已提交
1478 1479
{
	unsigned long flags;
1480
	int cpu, success = 0;
P
Peter Zijlstra 已提交
1481

1482
	smp_wmb();
1483
	raw_spin_lock_irqsave(&p->pi_lock, flags);
P
Peter Zijlstra 已提交
1484
	if (!(p->state & state))
L
Linus Torvalds 已提交
1485 1486
		goto out;

1487
	success = 1; /* we're going to change ->state */
L
Linus Torvalds 已提交
1488 1489
	cpu = task_cpu(p);

1490 1491
	if (p->on_rq && ttwu_remote(p, wake_flags))
		goto stat;
L
Linus Torvalds 已提交
1492 1493

#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
1494
	/*
1495 1496
	 * If the owning (remote) cpu is still in the middle of schedule() with
	 * this task as prev, wait until its done referencing the task.
P
Peter Zijlstra 已提交
1497
	 */
1498
	while (p->on_cpu)
1499
		cpu_relax();
1500
	/*
1501
	 * Pairs with the smp_wmb() in finish_lock_switch().
1502
	 */
1503
	smp_rmb();
L
Linus Torvalds 已提交
1504

1505
	p->sched_contributes_to_load = !!task_contributes_to_load(p);
P
Peter Zijlstra 已提交
1506
	p->state = TASK_WAKING;
1507

1508
	if (p->sched_class->task_waking)
1509
		p->sched_class->task_waking(p);
1510

1511
	cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
1512 1513
	if (task_cpu(p) != cpu) {
		wake_flags |= WF_MIGRATED;
1514
		set_task_cpu(p, cpu);
1515
	}
L
Linus Torvalds 已提交
1516 1517
#endif /* CONFIG_SMP */

1518 1519
	ttwu_queue(p, cpu);
stat:
1520
	ttwu_stat(p, cpu, wake_flags);
L
Linus Torvalds 已提交
1521
out:
1522
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
1523 1524 1525 1526

	return success;
}

T
Tejun Heo 已提交
1527 1528 1529 1530
/**
 * try_to_wake_up_local - try to wake up a local task with rq lock held
 * @p: the thread to be awakened
 *
1531
 * Put @p on the run-queue if it's not already there. The caller must
T
Tejun Heo 已提交
1532
 * ensure that this_rq() is locked, @p is bound to this_rq() and not
1533
 * the current task.
T
Tejun Heo 已提交
1534 1535 1536 1537 1538 1539 1540 1541 1542
 */
static void try_to_wake_up_local(struct task_struct *p)
{
	struct rq *rq = task_rq(p);

	BUG_ON(rq != this_rq());
	BUG_ON(p == current);
	lockdep_assert_held(&rq->lock);

1543 1544 1545 1546 1547 1548
	if (!raw_spin_trylock(&p->pi_lock)) {
		raw_spin_unlock(&rq->lock);
		raw_spin_lock(&p->pi_lock);
		raw_spin_lock(&rq->lock);
	}

T
Tejun Heo 已提交
1549
	if (!(p->state & TASK_NORMAL))
1550
		goto out;
T
Tejun Heo 已提交
1551

P
Peter Zijlstra 已提交
1552
	if (!p->on_rq)
P
Peter Zijlstra 已提交
1553 1554
		ttwu_activate(rq, p, ENQUEUE_WAKEUP);

1555
	ttwu_do_wakeup(rq, p, 0);
1556
	ttwu_stat(p, smp_processor_id(), 0);
1557 1558
out:
	raw_spin_unlock(&p->pi_lock);
T
Tejun Heo 已提交
1559 1560
}

1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
/**
 * wake_up_process - Wake up a specific process
 * @p: The process to be woken up.
 *
 * Attempt to wake up the nominated process and move it to the set of runnable
 * processes.  Returns 1 if the process was woken up, 0 if it was already
 * running.
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
 */
1572
int wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
1573
{
1574 1575
	WARN_ON(task_is_stopped_or_traced(p));
	return try_to_wake_up(p, TASK_NORMAL, 0);
L
Linus Torvalds 已提交
1576 1577 1578
}
EXPORT_SYMBOL(wake_up_process);

1579
int wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
1580 1581 1582 1583 1584 1585 1586
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
1587 1588 1589 1590 1591
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
static void __sched_fork(struct task_struct *p)
{
P
Peter Zijlstra 已提交
1592 1593 1594
	p->on_rq			= 0;

	p->se.on_rq			= 0;
I
Ingo Molnar 已提交
1595 1596
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
1597
	p->se.prev_sum_exec_runtime	= 0;
1598
	p->se.nr_migrations		= 0;
P
Peter Zijlstra 已提交
1599
	p->se.vruntime			= 0;
P
Peter Zijlstra 已提交
1600
	INIT_LIST_HEAD(&p->se.group_node);
I
Ingo Molnar 已提交
1601

1602 1603 1604 1605 1606 1607
/*
 * Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be
 * removed when useful for applications beyond shares distribution (e.g.
 * load-balance).
 */
#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)
1608 1609 1610
	p->se.avg.runnable_avg_period = 0;
	p->se.avg.runnable_avg_sum = 0;
#endif
I
Ingo Molnar 已提交
1611
#ifdef CONFIG_SCHEDSTATS
1612
	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
I
Ingo Molnar 已提交
1613
#endif
N
Nick Piggin 已提交
1614

P
Peter Zijlstra 已提交
1615
	INIT_LIST_HEAD(&p->rt.run_list);
N
Nick Piggin 已提交
1616

1617 1618 1619
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif
1620 1621 1622 1623

#ifdef CONFIG_NUMA_BALANCING
	if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
		p->mm->numa_next_scan = jiffies;
1624
		p->mm->numa_next_reset = jiffies;
1625 1626 1627 1628 1629 1630
		p->mm->numa_scan_seq = 0;
	}

	p->node_stamp = 0ULL;
	p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
	p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq - 1 : 0;
1631
	p->numa_scan_period = sysctl_numa_balancing_scan_delay;
1632 1633
	p->numa_work.next = &p->numa_work;
#endif /* CONFIG_NUMA_BALANCING */
I
Ingo Molnar 已提交
1634 1635
}

1636
#ifdef CONFIG_NUMA_BALANCING
1637
#ifdef CONFIG_SCHED_DEBUG
1638 1639 1640 1641 1642 1643 1644
void set_numabalancing_state(bool enabled)
{
	if (enabled)
		sched_feat_set("NUMA");
	else
		sched_feat_set("NO_NUMA");
}
1645 1646 1647 1648 1649 1650
#else
__read_mostly bool numabalancing_enabled;

void set_numabalancing_state(bool enabled)
{
	numabalancing_enabled = enabled;
I
Ingo Molnar 已提交
1651
}
1652
#endif /* CONFIG_SCHED_DEBUG */
1653
#endif /* CONFIG_NUMA_BALANCING */
I
Ingo Molnar 已提交
1654 1655 1656 1657

/*
 * fork()/clone()-time setup:
 */
1658
void sched_fork(struct task_struct *p)
I
Ingo Molnar 已提交
1659
{
1660
	unsigned long flags;
I
Ingo Molnar 已提交
1661 1662 1663
	int cpu = get_cpu();

	__sched_fork(p);
1664
	/*
1665
	 * We mark the process as running here. This guarantees that
1666 1667 1668
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
1669
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
1670

1671 1672 1673 1674 1675
	/*
	 * Make sure we do not leak PI boosting priority to the child.
	 */
	p->prio = current->normal_prio;

1676 1677 1678 1679
	/*
	 * Revert to default priority/policy on fork if requested.
	 */
	if (unlikely(p->sched_reset_on_fork)) {
1680
		if (task_has_rt_policy(p)) {
1681
			p->policy = SCHED_NORMAL;
1682
			p->static_prio = NICE_TO_PRIO(0);
1683 1684 1685 1686 1687 1688
			p->rt_priority = 0;
		} else if (PRIO_TO_NICE(p->static_prio) < 0)
			p->static_prio = NICE_TO_PRIO(0);

		p->prio = p->normal_prio = __normal_prio(p);
		set_load_weight(p);
1689

1690 1691 1692 1693 1694 1695
		/*
		 * We don't need the reset flag anymore after the fork. It has
		 * fulfilled its duty:
		 */
		p->sched_reset_on_fork = 0;
	}
1696

H
Hiroshi Shimamoto 已提交
1697 1698
	if (!rt_prio(p->prio))
		p->sched_class = &fair_sched_class;
1699

P
Peter Zijlstra 已提交
1700 1701 1702
	if (p->sched_class->task_fork)
		p->sched_class->task_fork(p);

1703 1704 1705 1706 1707 1708 1709
	/*
	 * The child is not yet in the pid-hash so no cgroup attach races,
	 * and the cgroup is pinned to this child due to cgroup_fork()
	 * is ran before sched_fork().
	 *
	 * Silence PROVE_RCU.
	 */
1710
	raw_spin_lock_irqsave(&p->pi_lock, flags);
1711
	set_task_cpu(p, cpu);
1712
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1713

1714
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
1715
	if (likely(sched_info_on()))
1716
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
1717
#endif
P
Peter Zijlstra 已提交
1718 1719
#if defined(CONFIG_SMP)
	p->on_cpu = 0;
1720
#endif
1721
#ifdef CONFIG_PREEMPT_COUNT
1722
	/* Want to start with kernel preemption disabled. */
A
Al Viro 已提交
1723
	task_thread_info(p)->preempt_count = 1;
L
Linus Torvalds 已提交
1724
#endif
1725
#ifdef CONFIG_SMP
1726
	plist_node_init(&p->pushable_tasks, MAX_PRIO);
1727
#endif
1728

N
Nick Piggin 已提交
1729
	put_cpu();
L
Linus Torvalds 已提交
1730 1731 1732 1733 1734 1735 1736 1737 1738
}

/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
1739
void wake_up_new_task(struct task_struct *p)
L
Linus Torvalds 已提交
1740 1741
{
	unsigned long flags;
I
Ingo Molnar 已提交
1742
	struct rq *rq;
1743

1744
	raw_spin_lock_irqsave(&p->pi_lock, flags);
1745 1746 1747 1748 1749 1750
#ifdef CONFIG_SMP
	/*
	 * Fork balancing, do it here and not earlier because:
	 *  - cpus_allowed can change in the fork path
	 *  - any previously selected cpu might disappear through hotplug
	 */
1751
	set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0));
1752 1753
#endif

1754
	rq = __task_rq_lock(p);
P
Peter Zijlstra 已提交
1755
	activate_task(rq, p, 0);
P
Peter Zijlstra 已提交
1756
	p->on_rq = 1;
1757
	trace_sched_wakeup_new(p, true);
P
Peter Zijlstra 已提交
1758
	check_preempt_curr(rq, p, WF_FORK);
1759
#ifdef CONFIG_SMP
1760 1761
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);
1762
#endif
1763
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
1764 1765
}

1766 1767 1768
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
1769
 * preempt_notifier_register - tell me when current is being preempted & rescheduled
R
Randy Dunlap 已提交
1770
 * @notifier: notifier struct to register
1771 1772 1773 1774 1775 1776 1777 1778 1779
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
1780
 * @notifier: notifier struct to unregister
1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;

1794
	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
1795 1796 1797 1798 1799 1800 1801 1802 1803
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;

1804
	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
1805 1806 1807
		notifier->ops->sched_out(notifier, next);
}

1808
#else /* !CONFIG_PREEMPT_NOTIFIERS */
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

1820
#endif /* CONFIG_PREEMPT_NOTIFIERS */
1821

1822 1823 1824
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
1825
 * @prev: the current task that is being switched out
1826 1827 1828 1829 1830 1831 1832 1833 1834
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
1835 1836 1837
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
1838
{
1839
	trace_sched_switch(prev, next);
1840 1841
	sched_info_switch(prev, next);
	perf_event_task_sched_out(prev, next);
1842
	fire_sched_out_preempt_notifiers(prev, next);
1843 1844 1845 1846
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
1847 1848
/**
 * finish_task_switch - clean up after a task-switch
1849
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
1850 1851
 * @prev: the thread we just switched away from.
 *
1852 1853 1854 1855
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
1856 1857
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
I
Ingo Molnar 已提交
1858
 * so, we finish that here outside of the runqueue lock. (Doing it
L
Linus Torvalds 已提交
1859 1860 1861
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
A
Alexey Dobriyan 已提交
1862
static void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
1863 1864 1865
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
1866
	long prev_state;
L
Linus Torvalds 已提交
1867 1868 1869 1870 1871

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
1872
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
1873 1874
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
1875
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
1876 1877 1878 1879 1880
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
1881
	prev_state = prev->state;
1882
	vtime_task_switch(prev);
1883
	finish_arch_switch(prev);
1884
	perf_event_task_sched_in(prev, current);
1885
	finish_lock_switch(rq, prev);
1886
	finish_arch_post_lock_switch();
S
Steven Rostedt 已提交
1887

1888
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
1889 1890
	if (mm)
		mmdrop(mm);
1891
	if (unlikely(prev_state == TASK_DEAD)) {
1892 1893 1894
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
1895
		 */
1896
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
1897
		put_task_struct(prev);
1898
	}
1899 1900

	tick_nohz_task_switch(current);
L
Linus Torvalds 已提交
1901 1902
}

1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917
#ifdef CONFIG_SMP

/* assumes rq->lock is held */
static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
{
	if (prev->sched_class->pre_schedule)
		prev->sched_class->pre_schedule(rq, prev);
}

/* rq->lock is NOT held, but preemption is disabled */
static inline void post_schedule(struct rq *rq)
{
	if (rq->post_schedule) {
		unsigned long flags;

1918
		raw_spin_lock_irqsave(&rq->lock, flags);
1919 1920
		if (rq->curr->sched_class->post_schedule)
			rq->curr->sched_class->post_schedule(rq);
1921
		raw_spin_unlock_irqrestore(&rq->lock, flags);
1922 1923 1924 1925 1926 1927

		rq->post_schedule = 0;
	}
}

#else
1928

1929 1930 1931 1932 1933 1934
static inline void pre_schedule(struct rq *rq, struct task_struct *p)
{
}

static inline void post_schedule(struct rq *rq)
{
L
Linus Torvalds 已提交
1935 1936
}

1937 1938
#endif

L
Linus Torvalds 已提交
1939 1940 1941 1942
/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
1943
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
1944 1945
	__releases(rq->lock)
{
1946 1947
	struct rq *rq = this_rq();

1948
	finish_task_switch(rq, prev);
1949

1950 1951 1952 1953 1954
	/*
	 * FIXME: do we need to worry about rq being invalidated by the
	 * task_switch?
	 */
	post_schedule(rq);
1955

1956 1957 1958 1959
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
1960
	if (current->set_child_tid)
1961
		put_user(task_pid_vnr(current), current->set_child_tid);
L
Linus Torvalds 已提交
1962 1963 1964 1965 1966 1967
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
1968
static inline void
1969
context_switch(struct rq *rq, struct task_struct *prev,
1970
	       struct task_struct *next)
L
Linus Torvalds 已提交
1971
{
I
Ingo Molnar 已提交
1972
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
1973

1974
	prepare_task_switch(rq, prev, next);
1975

I
Ingo Molnar 已提交
1976 1977
	mm = next->mm;
	oldmm = prev->active_mm;
1978 1979 1980 1981 1982
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
1983
	arch_start_context_switch(prev);
1984

1985
	if (!mm) {
L
Linus Torvalds 已提交
1986 1987 1988 1989 1990 1991
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

1992
	if (!prev->mm) {
L
Linus Torvalds 已提交
1993 1994 1995
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
1996 1997 1998 1999 2000 2001 2002
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2003
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2004
#endif
L
Linus Torvalds 已提交
2005

2006
	context_tracking_task_switch(prev, next);
L
Linus Torvalds 已提交
2007 2008 2009
	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
2010 2011 2012 2013 2014 2015 2016
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
2017 2018 2019
}

/*
2020
 * nr_running and nr_context_switches:
L
Linus Torvalds 已提交
2021 2022
 *
 * externally visible scheduler statistics: current number of runnable
2023
 * threads, total number of context switches performed since bootup.
L
Linus Torvalds 已提交
2024 2025 2026 2027 2028 2029 2030 2031 2032
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
2033
}
L
Linus Torvalds 已提交
2034 2035

unsigned long long nr_context_switches(void)
2036
{
2037 2038
	int i;
	unsigned long long sum = 0;
2039

2040
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2041
		sum += cpu_rq(i)->nr_switches;
2042

L
Linus Torvalds 已提交
2043 2044
	return sum;
}
2045

L
Linus Torvalds 已提交
2046 2047 2048
unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;
2049

2050
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2051
		sum += atomic_read(&cpu_rq(i)->nr_iowait);
2052

L
Linus Torvalds 已提交
2053 2054
	return sum;
}
2055

2056
unsigned long nr_iowait_cpu(int cpu)
2057
{
2058
	struct rq *this = cpu_rq(cpu);
2059 2060
	return atomic_read(&this->nr_iowait);
}
2061

2062 2063 2064 2065 2066
unsigned long this_cpu_load(void)
{
	struct rq *this = this_rq();
	return this->cpu_load[0];
}
2067

2068

2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115
/*
 * Global load-average calculations
 *
 * We take a distributed and async approach to calculating the global load-avg
 * in order to minimize overhead.
 *
 * The global load average is an exponentially decaying average of nr_running +
 * nr_uninterruptible.
 *
 * Once every LOAD_FREQ:
 *
 *   nr_active = 0;
 *   for_each_possible_cpu(cpu)
 *   	nr_active += cpu_of(cpu)->nr_running + cpu_of(cpu)->nr_uninterruptible;
 *
 *   avenrun[n] = avenrun[0] * exp_n + nr_active * (1 - exp_n)
 *
 * Due to a number of reasons the above turns in the mess below:
 *
 *  - for_each_possible_cpu() is prohibitively expensive on machines with
 *    serious number of cpus, therefore we need to take a distributed approach
 *    to calculating nr_active.
 *
 *        \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0
 *                      = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) }
 *
 *    So assuming nr_active := 0 when we start out -- true per definition, we
 *    can simply take per-cpu deltas and fold those into a global accumulate
 *    to obtain the same result. See calc_load_fold_active().
 *
 *    Furthermore, in order to avoid synchronizing all per-cpu delta folding
 *    across the machine, we assume 10 ticks is sufficient time for every
 *    cpu to have completed this task.
 *
 *    This places an upper-bound on the IRQ-off latency of the machine. Then
 *    again, being late doesn't loose the delta, just wrecks the sample.
 *
 *  - cpu_rq()->nr_uninterruptible isn't accurately tracked per-cpu because
 *    this would add another cross-cpu cacheline miss and atomic operation
 *    to the wakeup path. Instead we increment on whatever cpu the task ran
 *    when it went into uninterruptible state and decrement on whatever cpu
 *    did the wakeup. This means that only the sum of nr_uninterruptible over
 *    all cpus yields the correct result.
 *
 *  This covers the NO_HZ=n code, for extra head-aches, see the comment below.
 */

2116 2117 2118 2119
/* Variables and functions for calc_load */
static atomic_long_t calc_load_tasks;
static unsigned long calc_load_update;
unsigned long avenrun[3];
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
EXPORT_SYMBOL(avenrun); /* should be removed */

/**
 * get_avenrun - get the load average array
 * @loads:	pointer to dest load array
 * @offset:	offset to add
 * @shift:	shift count to shift the result left
 *
 * These values are estimates at best, so no need for locking.
 */
void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
{
	loads[0] = (avenrun[0] + offset) << shift;
	loads[1] = (avenrun[1] + offset) << shift;
	loads[2] = (avenrun[2] + offset) << shift;
}
2136

2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
static long calc_load_fold_active(struct rq *this_rq)
{
	long nr_active, delta = 0;

	nr_active = this_rq->nr_running;
	nr_active += (long) this_rq->nr_uninterruptible;

	if (nr_active != this_rq->calc_load_active) {
		delta = nr_active - this_rq->calc_load_active;
		this_rq->calc_load_active = nr_active;
	}

	return delta;
}

2152 2153 2154
/*
 * a1 = a0 * e + a * (1 - e)
 */
2155 2156 2157 2158 2159 2160 2161 2162 2163
static unsigned long
calc_load(unsigned long load, unsigned long exp, unsigned long active)
{
	load *= exp;
	load += active * (FIXED_1 - exp);
	load += 1UL << (FSHIFT - 1);
	return load >> FSHIFT;
}

2164
#ifdef CONFIG_NO_HZ_COMMON
2165
/*
2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203
 * Handle NO_HZ for the global load-average.
 *
 * Since the above described distributed algorithm to compute the global
 * load-average relies on per-cpu sampling from the tick, it is affected by
 * NO_HZ.
 *
 * The basic idea is to fold the nr_active delta into a global idle-delta upon
 * entering NO_HZ state such that we can include this as an 'extra' cpu delta
 * when we read the global state.
 *
 * Obviously reality has to ruin such a delightfully simple scheme:
 *
 *  - When we go NO_HZ idle during the window, we can negate our sample
 *    contribution, causing under-accounting.
 *
 *    We avoid this by keeping two idle-delta counters and flipping them
 *    when the window starts, thus separating old and new NO_HZ load.
 *
 *    The only trick is the slight shift in index flip for read vs write.
 *
 *        0s            5s            10s           15s
 *          +10           +10           +10           +10
 *        |-|-----------|-|-----------|-|-----------|-|
 *    r:0 0 1           1 0           0 1           1 0
 *    w:0 1 1           0 0           1 1           0 0
 *
 *    This ensures we'll fold the old idle contribution in this window while
 *    accumlating the new one.
 *
 *  - When we wake up from NO_HZ idle during the window, we push up our
 *    contribution, since we effectively move our sample point to a known
 *    busy state.
 *
 *    This is solved by pushing the window forward, and thus skipping the
 *    sample, for this cpu (effectively using the idle-delta for this cpu which
 *    was in effect at the time the window opened). This also solves the issue
 *    of having to deal with a cpu having been in NOHZ idle for multiple
 *    LOAD_FREQ intervals.
2204 2205 2206
 *
 * When making the ILB scale, we should try to pull this in as well.
 */
2207 2208
static atomic_long_t calc_load_idle[2];
static int calc_load_idx;
2209

2210
static inline int calc_load_write_idx(void)
2211
{
2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
	int idx = calc_load_idx;

	/*
	 * See calc_global_nohz(), if we observe the new index, we also
	 * need to observe the new update time.
	 */
	smp_rmb();

	/*
	 * If the folding window started, make sure we start writing in the
	 * next idle-delta.
	 */
	if (!time_before(jiffies, calc_load_update))
		idx++;

	return idx & 1;
}

static inline int calc_load_read_idx(void)
{
	return calc_load_idx & 1;
}

void calc_load_enter_idle(void)
{
	struct rq *this_rq = this_rq();
2238 2239
	long delta;

2240 2241 2242 2243
	/*
	 * We're going into NOHZ mode, if there's any pending delta, fold it
	 * into the pending idle delta.
	 */
2244
	delta = calc_load_fold_active(this_rq);
2245 2246 2247 2248
	if (delta) {
		int idx = calc_load_write_idx();
		atomic_long_add(delta, &calc_load_idle[idx]);
	}
2249 2250
}

2251
void calc_load_exit_idle(void)
2252
{
2253 2254 2255 2256 2257 2258 2259
	struct rq *this_rq = this_rq();

	/*
	 * If we're still before the sample window, we're done.
	 */
	if (time_before(jiffies, this_rq->calc_load_update))
		return;
2260 2261

	/*
2262 2263 2264
	 * We woke inside or after the sample window, this means we're already
	 * accounted through the nohz accounting, so skip the entire deal and
	 * sync up for the next window.
2265
	 */
2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277
	this_rq->calc_load_update = calc_load_update;
	if (time_before(jiffies, this_rq->calc_load_update + 10))
		this_rq->calc_load_update += LOAD_FREQ;
}

static long calc_load_fold_idle(void)
{
	int idx = calc_load_read_idx();
	long delta = 0;

	if (atomic_long_read(&calc_load_idle[idx]))
		delta = atomic_long_xchg(&calc_load_idle[idx], 0);
2278 2279 2280

	return delta;
}
2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358

/**
 * fixed_power_int - compute: x^n, in O(log n) time
 *
 * @x:         base of the power
 * @frac_bits: fractional bits of @x
 * @n:         power to raise @x to.
 *
 * By exploiting the relation between the definition of the natural power
 * function: x^n := x*x*...*x (x multiplied by itself for n times), and
 * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i,
 * (where: n_i \elem {0, 1}, the binary vector representing n),
 * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is
 * of course trivially computable in O(log_2 n), the length of our binary
 * vector.
 */
static unsigned long
fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n)
{
	unsigned long result = 1UL << frac_bits;

	if (n) for (;;) {
		if (n & 1) {
			result *= x;
			result += 1UL << (frac_bits - 1);
			result >>= frac_bits;
		}
		n >>= 1;
		if (!n)
			break;
		x *= x;
		x += 1UL << (frac_bits - 1);
		x >>= frac_bits;
	}

	return result;
}

/*
 * a1 = a0 * e + a * (1 - e)
 *
 * a2 = a1 * e + a * (1 - e)
 *    = (a0 * e + a * (1 - e)) * e + a * (1 - e)
 *    = a0 * e^2 + a * (1 - e) * (1 + e)
 *
 * a3 = a2 * e + a * (1 - e)
 *    = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e)
 *    = a0 * e^3 + a * (1 - e) * (1 + e + e^2)
 *
 *  ...
 *
 * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1]
 *    = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e)
 *    = a0 * e^n + a * (1 - e^n)
 *
 * [1] application of the geometric series:
 *
 *              n         1 - x^(n+1)
 *     S_n := \Sum x^i = -------------
 *             i=0          1 - x
 */
static unsigned long
calc_load_n(unsigned long load, unsigned long exp,
	    unsigned long active, unsigned int n)
{

	return calc_load(load, fixed_power_int(exp, FSHIFT, n), active);
}

/*
 * NO_HZ can leave us missing all per-cpu ticks calling
 * calc_load_account_active(), but since an idle CPU folds its delta into
 * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold
 * in the pending idle delta if our idle period crossed a load cycle boundary.
 *
 * Once we've updated the global active value, we need to apply the exponential
 * weights adjusted to the number of cycles missed.
 */
2359
static void calc_global_nohz(void)
2360 2361 2362
{
	long delta, active, n;

2363 2364 2365 2366 2367 2368
	if (!time_before(jiffies, calc_load_update + 10)) {
		/*
		 * Catch-up, fold however many we are behind still
		 */
		delta = jiffies - calc_load_update - 10;
		n = 1 + (delta / LOAD_FREQ);
2369

2370 2371
		active = atomic_long_read(&calc_load_tasks);
		active = active > 0 ? active * FIXED_1 : 0;
2372

2373 2374 2375
		avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n);
		avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n);
		avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n);
2376

2377 2378
		calc_load_update += n * LOAD_FREQ;
	}
2379

2380 2381 2382 2383 2384 2385 2386 2387 2388
	/*
	 * Flip the idle index...
	 *
	 * Make sure we first write the new time then flip the index, so that
	 * calc_load_write_idx() will see the new time when it reads the new
	 * index, this avoids a double flip messing things up.
	 */
	smp_wmb();
	calc_load_idx++;
2389
}
2390
#else /* !CONFIG_NO_HZ_COMMON */
2391

2392 2393
static inline long calc_load_fold_idle(void) { return 0; }
static inline void calc_global_nohz(void) { }
2394

2395
#endif /* CONFIG_NO_HZ_COMMON */
2396 2397

/*
2398 2399
 * calc_load - update the avenrun load estimates 10 ticks after the
 * CPUs have updated calc_load_tasks.
2400
 */
2401
void calc_global_load(unsigned long ticks)
2402
{
2403
	long active, delta;
L
Linus Torvalds 已提交
2404

2405
	if (time_before(jiffies, calc_load_update + 10))
2406
		return;
L
Linus Torvalds 已提交
2407

2408 2409 2410 2411 2412 2413 2414
	/*
	 * Fold the 'old' idle-delta to include all NO_HZ cpus.
	 */
	delta = calc_load_fold_idle();
	if (delta)
		atomic_long_add(delta, &calc_load_tasks);

2415 2416
	active = atomic_long_read(&calc_load_tasks);
	active = active > 0 ? active * FIXED_1 : 0;
L
Linus Torvalds 已提交
2417

2418 2419 2420
	avenrun[0] = calc_load(avenrun[0], EXP_1, active);
	avenrun[1] = calc_load(avenrun[1], EXP_5, active);
	avenrun[2] = calc_load(avenrun[2], EXP_15, active);
I
Ingo Molnar 已提交
2421

2422
	calc_load_update += LOAD_FREQ;
2423 2424

	/*
2425
	 * In case we idled for multiple LOAD_FREQ intervals, catch up in bulk.
2426 2427
	 */
	calc_global_nohz();
2428
}
L
Linus Torvalds 已提交
2429

2430
/*
2431 2432
 * Called from update_cpu_load() to periodically update this CPU's
 * active count.
2433 2434 2435
 */
static void calc_load_account_active(struct rq *this_rq)
{
2436
	long delta;
2437

2438 2439
	if (time_before(jiffies, this_rq->calc_load_update))
		return;
2440

2441 2442
	delta  = calc_load_fold_active(this_rq);
	if (delta)
2443
		atomic_long_add(delta, &calc_load_tasks);
2444 2445

	this_rq->calc_load_update += LOAD_FREQ;
2446 2447
}

2448 2449 2450 2451
/*
 * End of global load-average stuff
 */

2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
/*
 * The exact cpuload at various idx values, calculated at every tick would be
 * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load
 *
 * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called
 * on nth tick when cpu may be busy, then we have:
 * load = ((2^idx - 1) / 2^idx)^(n-1) * load
 * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load
 *
 * decay_load_missed() below does efficient calculation of
 * load = ((2^idx - 1) / 2^idx)^(n-1) * load
 * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load
 *
 * The calculation is approximated on a 128 point scale.
 * degrade_zero_ticks is the number of ticks after which load at any
 * particular idx is approximated to be zero.
 * degrade_factor is a precomputed table, a row for each load idx.
 * Each column corresponds to degradation factor for a power of two ticks,
 * based on 128 point scale.
 * Example:
 * row 2, col 3 (=12) says that the degradation at load idx 2 after
 * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8).
 *
 * With this power of 2 load factors, we can degrade the load n times
 * by looking at 1 bits in n and doing as many mult/shift instead of
 * n mult/shifts needed by the exact degradation.
 */
#define DEGRADE_SHIFT		7
static const unsigned char
		degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
static const unsigned char
		degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
					{0, 0, 0, 0, 0, 0, 0, 0},
					{64, 32, 8, 0, 0, 0, 0, 0},
					{96, 72, 40, 12, 1, 0, 0},
					{112, 98, 75, 43, 15, 1, 0},
					{120, 112, 98, 76, 45, 16, 2} };

/*
 * Update cpu_load for any missed ticks, due to tickless idle. The backlog
 * would be when CPU is idle and so we just decay the old load without
 * adding any new load.
 */
static unsigned long
decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
{
	int j = 0;

	if (!missed_updates)
		return load;

	if (missed_updates >= degrade_zero_ticks[idx])
		return 0;

	if (idx == 1)
		return load >> missed_updates;

	while (missed_updates) {
		if (missed_updates % 2)
			load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;

		missed_updates >>= 1;
		j++;
	}
	return load;
}

2519
/*
I
Ingo Molnar 已提交
2520
 * Update rq->cpu_load[] statistics. This function is usually called every
2521 2522
 * scheduler tick (TICK_NSEC). With tickless idle this will not be called
 * every tick. We fix it up based on jiffies.
2523
 */
2524 2525
static void __update_cpu_load(struct rq *this_rq, unsigned long this_load,
			      unsigned long pending_updates)
2526
{
I
Ingo Molnar 已提交
2527
	int i, scale;
2528

I
Ingo Molnar 已提交
2529
	this_rq->nr_load_updates++;
2530

I
Ingo Molnar 已提交
2531
	/* Update our load: */
2532 2533
	this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
	for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
I
Ingo Molnar 已提交
2534
		unsigned long old_load, new_load;
2535

I
Ingo Molnar 已提交
2536
		/* scale is effectively 1 << i now, and >> i divides by scale */
2537

I
Ingo Molnar 已提交
2538
		old_load = this_rq->cpu_load[i];
2539
		old_load = decay_load_missed(old_load, pending_updates - 1, i);
I
Ingo Molnar 已提交
2540
		new_load = this_load;
I
Ingo Molnar 已提交
2541 2542 2543 2544 2545 2546
		/*
		 * Round up the averaging division if load is increasing. This
		 * prevents us from getting stuck on 9 if the load is 10, for
		 * example.
		 */
		if (new_load > old_load)
2547 2548 2549
			new_load += scale - 1;

		this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
I
Ingo Molnar 已提交
2550
	}
2551 2552

	sched_avg_update(this_rq);
2553 2554
}

2555
#ifdef CONFIG_NO_HZ_COMMON
2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568
/*
 * There is no sane way to deal with nohz on smp when using jiffies because the
 * cpu doing the jiffies update might drift wrt the cpu doing the jiffy reading
 * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}.
 *
 * Therefore we cannot use the delta approach from the regular tick since that
 * would seriously skew the load calculation. However we'll make do for those
 * updates happening while idle (nohz_idle_balance) or coming out of idle
 * (tick_nohz_idle_exit).
 *
 * This means we might still be one tick off for nohz periods.
 */

2569 2570 2571 2572 2573 2574
/*
 * Called from nohz_idle_balance() to update the load ratings before doing the
 * idle balance.
 */
void update_idle_cpu_load(struct rq *this_rq)
{
2575
	unsigned long curr_jiffies = ACCESS_ONCE(jiffies);
2576 2577 2578 2579
	unsigned long load = this_rq->load.weight;
	unsigned long pending_updates;

	/*
2580
	 * bail if there's load or we're actually up-to-date.
2581 2582 2583 2584 2585 2586 2587 2588 2589 2590
	 */
	if (load || curr_jiffies == this_rq->last_load_update_tick)
		return;

	pending_updates = curr_jiffies - this_rq->last_load_update_tick;
	this_rq->last_load_update_tick = curr_jiffies;

	__update_cpu_load(this_rq, load, pending_updates);
}

2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614
/*
 * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed.
 */
void update_cpu_load_nohz(void)
{
	struct rq *this_rq = this_rq();
	unsigned long curr_jiffies = ACCESS_ONCE(jiffies);
	unsigned long pending_updates;

	if (curr_jiffies == this_rq->last_load_update_tick)
		return;

	raw_spin_lock(&this_rq->lock);
	pending_updates = curr_jiffies - this_rq->last_load_update_tick;
	if (pending_updates) {
		this_rq->last_load_update_tick = curr_jiffies;
		/*
		 * We were idle, this means load 0, the current load might be
		 * !0 due to remote wakeups and the sort.
		 */
		__update_cpu_load(this_rq, 0, pending_updates);
	}
	raw_spin_unlock(&this_rq->lock);
}
2615
#endif /* CONFIG_NO_HZ_COMMON */
2616

2617 2618 2619
/*
 * Called from scheduler_tick()
 */
2620 2621
static void update_cpu_load_active(struct rq *this_rq)
{
2622
	/*
2623
	 * See the mess around update_idle_cpu_load() / update_cpu_load_nohz().
2624 2625 2626
	 */
	this_rq->last_load_update_tick = jiffies;
	__update_cpu_load(this_rq, this_rq->load.weight, 1);
2627

2628
	calc_load_account_active(this_rq);
2629 2630
}

I
Ingo Molnar 已提交
2631
#ifdef CONFIG_SMP
2632

2633
/*
P
Peter Zijlstra 已提交
2634 2635
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
2636
 */
P
Peter Zijlstra 已提交
2637
void sched_exec(void)
2638
{
P
Peter Zijlstra 已提交
2639
	struct task_struct *p = current;
L
Linus Torvalds 已提交
2640
	unsigned long flags;
2641
	int dest_cpu;
2642

2643
	raw_spin_lock_irqsave(&p->pi_lock, flags);
2644
	dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0);
2645 2646
	if (dest_cpu == smp_processor_id())
		goto unlock;
P
Peter Zijlstra 已提交
2647

2648
	if (likely(cpu_active(dest_cpu))) {
2649
		struct migration_arg arg = { p, dest_cpu };
2650

2651 2652
		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
2653 2654
		return;
	}
2655
unlock:
2656
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
2657
}
I
Ingo Molnar 已提交
2658

L
Linus Torvalds 已提交
2659 2660 2661
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);
2662
DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
L
Linus Torvalds 已提交
2663 2664

EXPORT_PER_CPU_SYMBOL(kstat);
2665
EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
L
Linus Torvalds 已提交
2666 2667

/*
2668
 * Return any ns on the sched_clock that have not yet been accounted in
2669
 * @p in case that task is currently running.
2670 2671
 *
 * Called with task_rq_lock() held on @rq.
L
Linus Torvalds 已提交
2672
 */
2673 2674 2675 2676 2677 2678
static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
{
	u64 ns = 0;

	if (task_current(rq, p)) {
		update_rq_clock(rq);
2679
		ns = rq->clock_task - p->se.exec_start;
2680 2681 2682 2683 2684 2685 2686
		if ((s64)ns < 0)
			ns = 0;
	}

	return ns;
}

2687
unsigned long long task_delta_exec(struct task_struct *p)
L
Linus Torvalds 已提交
2688 2689
{
	unsigned long flags;
2690
	struct rq *rq;
2691
	u64 ns = 0;
2692

2693
	rq = task_rq_lock(p, &flags);
2694
	ns = do_task_delta_exec(p, rq);
2695
	task_rq_unlock(rq, p, &flags);
2696

2697 2698
	return ns;
}
2699

2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712
/*
 * Return accounted runtime for the task.
 * In case the task is currently running, return the runtime plus current's
 * pending runtime that have not been accounted yet.
 */
unsigned long long task_sched_runtime(struct task_struct *p)
{
	unsigned long flags;
	struct rq *rq;
	u64 ns = 0;

	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
2713
	task_rq_unlock(rq, p, &flags);
2714 2715 2716

	return ns;
}
2717

2718 2719 2720 2721 2722 2723 2724 2725
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
2726
	struct task_struct *curr = rq->curr;
2727 2728

	sched_clock_tick();
I
Ingo Molnar 已提交
2729

2730
	raw_spin_lock(&rq->lock);
2731
	update_rq_clock(rq);
2732
	update_cpu_load_active(rq);
P
Peter Zijlstra 已提交
2733
	curr->sched_class->task_tick(rq, curr, 0);
2734
	raw_spin_unlock(&rq->lock);
2735

2736
	perf_event_task_tick();
2737

2738
#ifdef CONFIG_SMP
2739
	rq->idle_balance = idle_cpu(cpu);
I
Ingo Molnar 已提交
2740
	trigger_load_balance(rq, cpu);
2741
#endif
L
Linus Torvalds 已提交
2742 2743
}

2744
notrace unsigned long get_parent_ip(unsigned long addr)
2745 2746 2747 2748 2749 2750 2751 2752
{
	if (in_lock_functions(addr)) {
		addr = CALLER_ADDR2;
		if (in_lock_functions(addr))
			addr = CALLER_ADDR3;
	}
	return addr;
}
L
Linus Torvalds 已提交
2753

2754 2755 2756
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
				defined(CONFIG_PREEMPT_TRACER))

2757
void __kprobes add_preempt_count(int val)
L
Linus Torvalds 已提交
2758
{
2759
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2760 2761 2762
	/*
	 * Underflow?
	 */
2763 2764
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
2765
#endif
L
Linus Torvalds 已提交
2766
	preempt_count() += val;
2767
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2768 2769 2770
	/*
	 * Spinlock count overflowing soon?
	 */
2771 2772
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
2773 2774 2775
#endif
	if (preempt_count() == val)
		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
2776 2777 2778
}
EXPORT_SYMBOL(add_preempt_count);

2779
void __kprobes sub_preempt_count(int val)
L
Linus Torvalds 已提交
2780
{
2781
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2782 2783 2784
	/*
	 * Underflow?
	 */
2785
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
2786
		return;
L
Linus Torvalds 已提交
2787 2788 2789
	/*
	 * Is the spinlock portion underflowing?
	 */
2790 2791 2792
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;
2793
#endif
2794

2795 2796
	if (preempt_count() == val)
		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
2797 2798 2799 2800 2801 2802 2803
	preempt_count() -= val;
}
EXPORT_SYMBOL(sub_preempt_count);

#endif

/*
I
Ingo Molnar 已提交
2804
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
2805
 */
I
Ingo Molnar 已提交
2806
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
2807
{
2808 2809 2810
	if (oops_in_progress)
		return;

P
Peter Zijlstra 已提交
2811 2812
	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
		prev->comm, prev->pid, preempt_count());
2813

I
Ingo Molnar 已提交
2814
	debug_show_held_locks(prev);
2815
	print_modules();
I
Ingo Molnar 已提交
2816 2817
	if (irqs_disabled())
		print_irqtrace_events(prev);
2818
	dump_stack();
2819
	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
I
Ingo Molnar 已提交
2820
}
L
Linus Torvalds 已提交
2821

I
Ingo Molnar 已提交
2822 2823 2824 2825 2826
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
2827
	/*
I
Ingo Molnar 已提交
2828
	 * Test if we are atomic. Since do_exit() needs to call into
L
Linus Torvalds 已提交
2829 2830 2831
	 * schedule() atomically, we ignore that path for now.
	 * Otherwise, whine if we are scheduling when we should not be.
	 */
2832
	if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
I
Ingo Molnar 已提交
2833
		__schedule_bug(prev);
2834
	rcu_sleep_check();
I
Ingo Molnar 已提交
2835

L
Linus Torvalds 已提交
2836 2837
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

2838
	schedstat_inc(this_rq(), sched_count);
I
Ingo Molnar 已提交
2839 2840
}

P
Peter Zijlstra 已提交
2841
static void put_prev_task(struct rq *rq, struct task_struct *prev)
M
Mike Galbraith 已提交
2842
{
2843
	if (prev->on_rq || rq->skip_clock_update < 0)
2844
		update_rq_clock(rq);
P
Peter Zijlstra 已提交
2845
	prev->sched_class->put_prev_task(rq, prev);
M
Mike Galbraith 已提交
2846 2847
}

I
Ingo Molnar 已提交
2848 2849 2850 2851
/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
2852
pick_next_task(struct rq *rq)
I
Ingo Molnar 已提交
2853
{
2854
	const struct sched_class *class;
I
Ingo Molnar 已提交
2855
	struct task_struct *p;
L
Linus Torvalds 已提交
2856 2857

	/*
I
Ingo Molnar 已提交
2858 2859
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
2860
	 */
2861
	if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
2862
		p = fair_sched_class.pick_next_task(rq);
I
Ingo Molnar 已提交
2863 2864
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
2865 2866
	}

2867
	for_each_class(class) {
2868
		p = class->pick_next_task(rq);
I
Ingo Molnar 已提交
2869 2870 2871
		if (p)
			return p;
	}
2872 2873

	BUG(); /* the idle class will always have a runnable task */
I
Ingo Molnar 已提交
2874
}
L
Linus Torvalds 已提交
2875

I
Ingo Molnar 已提交
2876
/*
2877
 * __schedule() is the main scheduler function.
2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
 *
 * The main means of driving the scheduler and thus entering this function are:
 *
 *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
 *
 *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
 *      paths. For example, see arch/x86/entry_64.S.
 *
 *      To drive preemption between tasks, the scheduler sets the flag in timer
 *      interrupt handler scheduler_tick().
 *
 *   3. Wakeups don't really cause entry into schedule(). They add a
 *      task to the run-queue and that's it.
 *
 *      Now, if the new task added to the run-queue preempts the current
 *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
 *      called on the nearest possible occasion:
 *
 *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
 *
 *         - in syscall or exception context, at the next outmost
 *           preempt_enable(). (this might be as soon as the wake_up()'s
 *           spin_unlock()!)
 *
 *         - in IRQ context, return from interrupt-handler to
 *           preemptible context
 *
 *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
 *         then at the next:
 *
 *          - cond_resched() call
 *          - explicit schedule() call
 *          - return from syscall or exception to user-space
 *          - return from interrupt-handler to user-space
I
Ingo Molnar 已提交
2912
 */
2913
static void __sched __schedule(void)
I
Ingo Molnar 已提交
2914 2915
{
	struct task_struct *prev, *next;
2916
	unsigned long *switch_count;
I
Ingo Molnar 已提交
2917
	struct rq *rq;
2918
	int cpu;
I
Ingo Molnar 已提交
2919

2920 2921
need_resched:
	preempt_disable();
I
Ingo Molnar 已提交
2922 2923
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
2924
	rcu_note_context_switch(cpu);
I
Ingo Molnar 已提交
2925 2926 2927
	prev = rq->curr;

	schedule_debug(prev);
L
Linus Torvalds 已提交
2928

2929
	if (sched_feat(HRTICK))
M
Mike Galbraith 已提交
2930
		hrtick_clear(rq);
P
Peter Zijlstra 已提交
2931

2932
	raw_spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
2933

2934
	switch_count = &prev->nivcsw;
L
Linus Torvalds 已提交
2935
	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
T
Tejun Heo 已提交
2936
		if (unlikely(signal_pending_state(prev->state, prev))) {
L
Linus Torvalds 已提交
2937
			prev->state = TASK_RUNNING;
T
Tejun Heo 已提交
2938
		} else {
2939 2940 2941
			deactivate_task(rq, prev, DEQUEUE_SLEEP);
			prev->on_rq = 0;

T
Tejun Heo 已提交
2942
			/*
2943 2944 2945
			 * If a worker went to sleep, notify and ask workqueue
			 * whether it wants to wake up a task to maintain
			 * concurrency.
T
Tejun Heo 已提交
2946 2947 2948 2949 2950 2951 2952 2953 2954
			 */
			if (prev->flags & PF_WQ_WORKER) {
				struct task_struct *to_wakeup;

				to_wakeup = wq_worker_sleeping(prev, cpu);
				if (to_wakeup)
					try_to_wake_up_local(to_wakeup);
			}
		}
I
Ingo Molnar 已提交
2955
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
2956 2957
	}

2958
	pre_schedule(rq, prev);
2959

I
Ingo Molnar 已提交
2960
	if (unlikely(!rq->nr_running))
L
Linus Torvalds 已提交
2961 2962
		idle_balance(cpu, rq);

M
Mike Galbraith 已提交
2963
	put_prev_task(rq, prev);
2964
	next = pick_next_task(rq);
2965 2966
	clear_tsk_need_resched(prev);
	rq->skip_clock_update = 0;
L
Linus Torvalds 已提交
2967 2968 2969 2970 2971 2972

	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
2973
		context_switch(rq, prev, next); /* unlocks the rq */
P
Peter Zijlstra 已提交
2974
		/*
2975 2976 2977 2978
		 * The context switch have flipped the stack from under us
		 * and restored the local variables which were saved when
		 * this task called schedule() in the past. prev == current
		 * is still correct, but it can be moved to another cpu/rq.
P
Peter Zijlstra 已提交
2979 2980 2981
		 */
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
2982
	} else
2983
		raw_spin_unlock_irq(&rq->lock);
L
Linus Torvalds 已提交
2984

2985
	post_schedule(rq);
L
Linus Torvalds 已提交
2986

2987
	sched_preempt_enable_no_resched();
2988
	if (need_resched())
L
Linus Torvalds 已提交
2989 2990
		goto need_resched;
}
2991

2992 2993
static inline void sched_submit_work(struct task_struct *tsk)
{
2994
	if (!tsk->state || tsk_is_pi_blocked(tsk))
2995 2996 2997 2998 2999 3000 3001 3002 3003
		return;
	/*
	 * If we are going to sleep and we have plugged IO queued,
	 * make sure to submit it to avoid deadlocks.
	 */
	if (blk_needs_flush_plug(tsk))
		blk_schedule_flush_plug(tsk);
}

S
Simon Kirby 已提交
3004
asmlinkage void __sched schedule(void)
3005
{
3006 3007 3008
	struct task_struct *tsk = current;

	sched_submit_work(tsk);
3009 3010
	__schedule();
}
L
Linus Torvalds 已提交
3011 3012
EXPORT_SYMBOL(schedule);

3013
#ifdef CONFIG_CONTEXT_TRACKING
3014 3015 3016 3017 3018 3019 3020 3021
asmlinkage void __sched schedule_user(void)
{
	/*
	 * If we come here after a random call to set_need_resched(),
	 * or we have been woken up remotely but the IPI has not yet arrived,
	 * we haven't yet exited the RCU idle mode. Do it here manually until
	 * we find a better solution.
	 */
3022
	user_exit();
3023
	schedule();
3024
	user_enter();
3025 3026 3027
}
#endif

3028 3029 3030 3031 3032 3033 3034
/**
 * schedule_preempt_disabled - called with preemption disabled
 *
 * Returns with preemption disabled. Note: preempt_count must be 1
 */
void __sched schedule_preempt_disabled(void)
{
3035
	sched_preempt_enable_no_resched();
3036 3037 3038 3039
	schedule();
	preempt_disable();
}

3040
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
3041

3042 3043 3044
static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
{
	if (lock->owner != owner)
3045
		return false;
3046 3047

	/*
3048 3049 3050 3051
	 * Ensure we emit the owner->on_cpu, dereference _after_ checking
	 * lock->owner still matches owner, if that fails, owner might
	 * point to free()d memory, if it still matches, the rcu_read_lock()
	 * ensures the memory stays valid.
3052
	 */
3053
	barrier();
3054

3055
	return owner->on_cpu;
3056
}
3057

3058 3059 3060 3061 3062 3063 3064 3065
/*
 * Look out! "owner" is an entirely speculative pointer
 * access and not reliable.
 */
int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
{
	if (!sched_feat(OWNER_SPIN))
		return 0;
3066

3067
	rcu_read_lock();
3068 3069
	while (owner_running(lock, owner)) {
		if (need_resched())
3070
			break;
3071

3072
		arch_mutex_cpu_relax();
3073
	}
3074
	rcu_read_unlock();
3075

3076
	/*
3077 3078 3079
	 * We break out the loop above on need_resched() and when the
	 * owner changed, which is a sign for heavy contention. Return
	 * success only when lock->owner is NULL.
3080
	 */
3081
	return lock->owner == NULL;
3082 3083 3084
}
#endif

L
Linus Torvalds 已提交
3085 3086
#ifdef CONFIG_PREEMPT
/*
3087
 * this is the entry point to schedule() from in-kernel preemption
I
Ingo Molnar 已提交
3088
 * off of preempt_enable. Kernel preemptions off return from interrupt
L
Linus Torvalds 已提交
3089 3090
 * occur there and call schedule directly.
 */
3091
asmlinkage void __sched notrace preempt_schedule(void)
L
Linus Torvalds 已提交
3092 3093
{
	struct thread_info *ti = current_thread_info();
3094

L
Linus Torvalds 已提交
3095 3096
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
I
Ingo Molnar 已提交
3097
	 * we do not want to preempt the current task. Just return..
L
Linus Torvalds 已提交
3098
	 */
N
Nick Piggin 已提交
3099
	if (likely(ti->preempt_count || irqs_disabled()))
L
Linus Torvalds 已提交
3100 3101
		return;

3102
	do {
3103
		add_preempt_count_notrace(PREEMPT_ACTIVE);
3104
		__schedule();
3105
		sub_preempt_count_notrace(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
3106

3107 3108 3109 3110 3111
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
3112
	} while (need_resched());
L
Linus Torvalds 已提交
3113 3114 3115 3116
}
EXPORT_SYMBOL(preempt_schedule);

/*
3117
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
3118 3119 3120 3121 3122 3123 3124
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
	struct thread_info *ti = current_thread_info();
3125
	enum ctx_state prev_state;
3126

3127
	/* Catch callers which need to be fixed */
L
Linus Torvalds 已提交
3128 3129
	BUG_ON(ti->preempt_count || !irqs_disabled());

3130 3131
	prev_state = exception_enter();

3132 3133 3134
	do {
		add_preempt_count(PREEMPT_ACTIVE);
		local_irq_enable();
3135
		__schedule();
3136 3137
		local_irq_disable();
		sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
3138

3139 3140 3141 3142 3143
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
3144
	} while (need_resched());
3145 3146

	exception_exit(prev_state);
L
Linus Torvalds 已提交
3147 3148 3149 3150
}

#endif /* CONFIG_PREEMPT */

P
Peter Zijlstra 已提交
3151
int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
I
Ingo Molnar 已提交
3152
			  void *key)
L
Linus Torvalds 已提交
3153
{
P
Peter Zijlstra 已提交
3154
	return try_to_wake_up(curr->private, mode, wake_flags);
L
Linus Torvalds 已提交
3155 3156 3157 3158
}
EXPORT_SYMBOL(default_wake_function);

/*
I
Ingo Molnar 已提交
3159 3160
 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
L
Linus Torvalds 已提交
3161 3162 3163
 * number) then we wake all the non-exclusive tasks and one exclusive task.
 *
 * There are circumstances in which we can try to wake a task which has already
I
Ingo Molnar 已提交
3164
 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
L
Linus Torvalds 已提交
3165 3166
 * zero in this (rare) case, and we handle it by continuing to scan the queue.
 */
3167
static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
P
Peter Zijlstra 已提交
3168
			int nr_exclusive, int wake_flags, void *key)
L
Linus Torvalds 已提交
3169
{
3170
	wait_queue_t *curr, *next;
L
Linus Torvalds 已提交
3171

3172
	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
3173 3174
		unsigned flags = curr->flags;

P
Peter Zijlstra 已提交
3175
		if (curr->func(curr, mode, wake_flags, key) &&
3176
				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
L
Linus Torvalds 已提交
3177 3178 3179 3180 3181 3182 3183 3184 3185
			break;
	}
}

/**
 * __wake_up - wake up threads blocked on a waitqueue.
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
3186
 * @key: is directly passed to the wakeup function
3187 3188 3189
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
L
Linus Torvalds 已提交
3190
 */
3191
void __wake_up(wait_queue_head_t *q, unsigned int mode,
I
Ingo Molnar 已提交
3192
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__wake_up_common(q, mode, nr_exclusive, 0, key);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL(__wake_up);

/*
 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
 */
3205
void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr)
L
Linus Torvalds 已提交
3206
{
3207
	__wake_up_common(q, mode, nr, 0, NULL);
L
Linus Torvalds 已提交
3208
}
3209
EXPORT_SYMBOL_GPL(__wake_up_locked);
L
Linus Torvalds 已提交
3210

3211 3212 3213 3214
void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
{
	__wake_up_common(q, mode, 1, 0, key);
}
3215
EXPORT_SYMBOL_GPL(__wake_up_locked_key);
3216

L
Linus Torvalds 已提交
3217
/**
3218
 * __wake_up_sync_key - wake up threads blocked on a waitqueue.
L
Linus Torvalds 已提交
3219 3220 3221
 * @q: the waitqueue
 * @mode: which threads
 * @nr_exclusive: how many wake-one or wake-many threads to wake up
3222
 * @key: opaque value to be passed to wakeup targets
L
Linus Torvalds 已提交
3223 3224 3225 3226 3227 3228 3229
 *
 * The sync wakeup differs that the waker knows that it will schedule
 * away soon, so while the target thread will be woken up, it will not
 * be migrated to another CPU - ie. the two threads are 'synchronized'
 * with each other. This can prevent needless bouncing between CPUs.
 *
 * On UP it can prevent extra preemption.
3230 3231 3232
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
L
Linus Torvalds 已提交
3233
 */
3234 3235
void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
			int nr_exclusive, void *key)
L
Linus Torvalds 已提交
3236 3237
{
	unsigned long flags;
P
Peter Zijlstra 已提交
3238
	int wake_flags = WF_SYNC;
L
Linus Torvalds 已提交
3239 3240 3241 3242 3243

	if (unlikely(!q))
		return;

	if (unlikely(!nr_exclusive))
P
Peter Zijlstra 已提交
3244
		wake_flags = 0;
L
Linus Torvalds 已提交
3245 3246

	spin_lock_irqsave(&q->lock, flags);
P
Peter Zijlstra 已提交
3247
	__wake_up_common(q, mode, nr_exclusive, wake_flags, key);
L
Linus Torvalds 已提交
3248 3249
	spin_unlock_irqrestore(&q->lock, flags);
}
3250 3251 3252 3253 3254 3255 3256 3257 3258
EXPORT_SYMBOL_GPL(__wake_up_sync_key);

/*
 * __wake_up_sync - see __wake_up_sync_key()
 */
void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
{
	__wake_up_sync_key(q, mode, nr_exclusive, NULL);
}
L
Linus Torvalds 已提交
3259 3260
EXPORT_SYMBOL_GPL(__wake_up_sync);	/* For internal use only */

3261 3262 3263 3264 3265 3266 3267 3268
/**
 * complete: - signals a single thread waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up a single thread waiting on this completion. Threads will be
 * awakened in the same order in which they were queued.
 *
 * See also complete_all(), wait_for_completion() and related routines.
3269 3270 3271
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
3272
 */
3273
void complete(struct completion *x)
L
Linus Torvalds 已提交
3274 3275 3276 3277 3278
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
3279
	__wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
L
Linus Torvalds 已提交
3280 3281 3282 3283
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete);

3284 3285 3286 3287 3288
/**
 * complete_all: - signals all threads waiting on this completion
 * @x:  holds the state of this particular completion
 *
 * This will wake up all threads waiting on this particular completion event.
3289 3290 3291
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
3292
 */
3293
void complete_all(struct completion *x)
L
Linus Torvalds 已提交
3294 3295 3296 3297 3298
{
	unsigned long flags;

	spin_lock_irqsave(&x->wait.lock, flags);
	x->done += UINT_MAX/2;
3299
	__wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
L
Linus Torvalds 已提交
3300 3301 3302 3303
	spin_unlock_irqrestore(&x->wait.lock, flags);
}
EXPORT_SYMBOL(complete_all);

3304
static inline long __sched
3305 3306
do_wait_for_common(struct completion *x,
		   long (*action)(long), long timeout, int state)
L
Linus Torvalds 已提交
3307 3308 3309 3310
{
	if (!x->done) {
		DECLARE_WAITQUEUE(wait, current);

C
Changli Gao 已提交
3311
		__add_wait_queue_tail_exclusive(&x->wait, &wait);
L
Linus Torvalds 已提交
3312
		do {
3313
			if (signal_pending_state(state, current)) {
3314 3315
				timeout = -ERESTARTSYS;
				break;
3316 3317
			}
			__set_current_state(state);
L
Linus Torvalds 已提交
3318
			spin_unlock_irq(&x->wait.lock);
3319
			timeout = action(timeout);
L
Linus Torvalds 已提交
3320
			spin_lock_irq(&x->wait.lock);
3321
		} while (!x->done && timeout);
L
Linus Torvalds 已提交
3322
		__remove_wait_queue(&x->wait, &wait);
3323 3324
		if (!x->done)
			return timeout;
L
Linus Torvalds 已提交
3325 3326
	}
	x->done--;
3327
	return timeout ?: 1;
L
Linus Torvalds 已提交
3328 3329
}

3330 3331 3332
static inline long __sched
__wait_for_common(struct completion *x,
		  long (*action)(long), long timeout, int state)
L
Linus Torvalds 已提交
3333 3334 3335 3336
{
	might_sleep();

	spin_lock_irq(&x->wait.lock);
3337
	timeout = do_wait_for_common(x, action, timeout, state);
L
Linus Torvalds 已提交
3338
	spin_unlock_irq(&x->wait.lock);
3339 3340
	return timeout;
}
L
Linus Torvalds 已提交
3341

3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353
static long __sched
wait_for_common(struct completion *x, long timeout, int state)
{
	return __wait_for_common(x, schedule_timeout, timeout, state);
}

static long __sched
wait_for_common_io(struct completion *x, long timeout, int state)
{
	return __wait_for_common(x, io_schedule_timeout, timeout, state);
}

3354 3355 3356 3357 3358 3359 3360 3361 3362 3363
/**
 * wait_for_completion: - waits for completion of a task
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It is NOT
 * interruptible and there is no timeout.
 *
 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
 * and interrupt capability. Also see complete().
 */
3364
void __sched wait_for_completion(struct completion *x)
3365 3366
{
	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
3367
}
3368
EXPORT_SYMBOL(wait_for_completion);
L
Linus Torvalds 已提交
3369

3370 3371 3372 3373 3374 3375 3376 3377
/**
 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. The timeout is in jiffies. It is not
 * interruptible.
3378 3379 3380
 *
 * The return value is 0 if timed out, and positive (at least 1, or number of
 * jiffies left till timeout) if completed.
3381
 */
3382
unsigned long __sched
3383
wait_for_completion_timeout(struct completion *x, unsigned long timeout)
L
Linus Torvalds 已提交
3384
{
3385
	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
3386
}
3387
EXPORT_SYMBOL(wait_for_completion_timeout);
L
Linus Torvalds 已提交
3388

3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421
/**
 * wait_for_completion_io: - waits for completion of a task
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It is NOT
 * interruptible and there is no timeout. The caller is accounted as waiting
 * for IO.
 */
void __sched wait_for_completion_io(struct completion *x)
{
	wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(wait_for_completion_io);

/**
 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. The timeout is in jiffies. It is not
 * interruptible. The caller is accounted as waiting for IO.
 *
 * The return value is 0 if timed out, and positive (at least 1, or number of
 * jiffies left till timeout) if completed.
 */
unsigned long __sched
wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
{
	return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(wait_for_completion_io_timeout);

3422 3423 3424 3425 3426 3427
/**
 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
 * @x:  holds the state of this particular completion
 *
 * This waits for completion of a specific task to be signaled. It is
 * interruptible.
3428 3429
 *
 * The return value is -ERESTARTSYS if interrupted, 0 if completed.
3430
 */
3431
int __sched wait_for_completion_interruptible(struct completion *x)
I
Ingo Molnar 已提交
3432
{
3433 3434 3435 3436
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
I
Ingo Molnar 已提交
3437
}
3438
EXPORT_SYMBOL(wait_for_completion_interruptible);
L
Linus Torvalds 已提交
3439

3440 3441 3442 3443 3444 3445 3446
/**
 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be signaled or for a
 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
3447 3448 3449
 *
 * The return value is -ERESTARTSYS if interrupted, 0 if timed out,
 * positive (at least 1, or number of jiffies left till timeout) if completed.
3450
 */
3451
long __sched
3452 3453
wait_for_completion_interruptible_timeout(struct completion *x,
					  unsigned long timeout)
I
Ingo Molnar 已提交
3454
{
3455
	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
I
Ingo Molnar 已提交
3456
}
3457
EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
L
Linus Torvalds 已提交
3458

3459 3460 3461 3462 3463 3464
/**
 * wait_for_completion_killable: - waits for completion of a task (killable)
 * @x:  holds the state of this particular completion
 *
 * This waits to be signaled for completion of a specific task. It can be
 * interrupted by a kill signal.
3465 3466
 *
 * The return value is -ERESTARTSYS if interrupted, 0 if completed.
3467
 */
M
Matthew Wilcox 已提交
3468 3469 3470 3471 3472 3473 3474 3475 3476
int __sched wait_for_completion_killable(struct completion *x)
{
	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
	if (t == -ERESTARTSYS)
		return t;
	return 0;
}
EXPORT_SYMBOL(wait_for_completion_killable);

3477 3478 3479 3480 3481 3482 3483 3484
/**
 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
 * @x:  holds the state of this particular completion
 * @timeout:  timeout value in jiffies
 *
 * This waits for either a completion of a specific task to be
 * signaled or for a specified timeout to expire. It can be
 * interrupted by a kill signal. The timeout is in jiffies.
3485 3486 3487
 *
 * The return value is -ERESTARTSYS if interrupted, 0 if timed out,
 * positive (at least 1, or number of jiffies left till timeout) if completed.
3488
 */
3489
long __sched
3490 3491 3492 3493 3494 3495 3496
wait_for_completion_killable_timeout(struct completion *x,
				     unsigned long timeout)
{
	return wait_for_common(x, timeout, TASK_KILLABLE);
}
EXPORT_SYMBOL(wait_for_completion_killable_timeout);

3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510
/**
 *	try_wait_for_completion - try to decrement a completion without blocking
 *	@x:	completion structure
 *
 *	Returns: 0 if a decrement cannot be done without blocking
 *		 1 if a decrement succeeded.
 *
 *	If a completion is being used as a counting completion,
 *	attempt to decrement the counter without blocking. This
 *	enables us to avoid waiting if the resource the completion
 *	is protecting is not available.
 */
bool try_wait_for_completion(struct completion *x)
{
3511
	unsigned long flags;
3512 3513
	int ret = 1;

3514
	spin_lock_irqsave(&x->wait.lock, flags);
3515 3516 3517 3518
	if (!x->done)
		ret = 0;
	else
		x->done--;
3519
	spin_unlock_irqrestore(&x->wait.lock, flags);
3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533
	return ret;
}
EXPORT_SYMBOL(try_wait_for_completion);

/**
 *	completion_done - Test to see if a completion has any waiters
 *	@x:	completion structure
 *
 *	Returns: 0 if there are waiters (wait_for_completion() in progress)
 *		 1 if there are no waiters.
 *
 */
bool completion_done(struct completion *x)
{
3534
	unsigned long flags;
3535 3536
	int ret = 1;

3537
	spin_lock_irqsave(&x->wait.lock, flags);
3538 3539
	if (!x->done)
		ret = 0;
3540
	spin_unlock_irqrestore(&x->wait.lock, flags);
3541 3542 3543 3544
	return ret;
}
EXPORT_SYMBOL(completion_done);

3545 3546
static long __sched
sleep_on_common(wait_queue_head_t *q, int state, long timeout)
L
Linus Torvalds 已提交
3547
{
I
Ingo Molnar 已提交
3548 3549 3550 3551
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
3552

3553
	__set_current_state(state);
L
Linus Torvalds 已提交
3554

3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568
	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, &wait);
	spin_unlock(&q->lock);
	timeout = schedule_timeout(timeout);
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, &wait);
	spin_unlock_irqrestore(&q->lock, flags);

	return timeout;
}

void __sched interruptible_sleep_on(wait_queue_head_t *q)
{
	sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
3569 3570 3571
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
3572
long __sched
I
Ingo Molnar 已提交
3573
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3574
{
3575
	return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
3576 3577 3578
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
3579
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
3580
{
3581
	sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
3582 3583 3584
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
3585
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
3586
{
3587
	return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
3588 3589 3590
}
EXPORT_SYMBOL(sleep_on_timeout);

3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
3603
void rt_mutex_setprio(struct task_struct *p, int prio)
3604
{
3605
	int oldprio, on_rq, running;
3606
	struct rq *rq;
3607
	const struct sched_class *prev_class;
3608 3609 3610

	BUG_ON(prio < 0 || prio > MAX_PRIO);

3611
	rq = __task_rq_lock(p);
3612

3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630
	/*
	 * Idle task boosting is a nono in general. There is one
	 * exception, when PREEMPT_RT and NOHZ is active:
	 *
	 * The idle task calls get_next_timer_interrupt() and holds
	 * the timer wheel base->lock on the CPU and another CPU wants
	 * to access the timer (probably to cancel it). We can safely
	 * ignore the boosting request, as the idle CPU runs this code
	 * with interrupts disabled and will complete the lock
	 * protected section without being interrupted. So there is no
	 * real need to boost.
	 */
	if (unlikely(p == rq->idle)) {
		WARN_ON(p != rq->curr);
		WARN_ON(p->pi_blocked_on);
		goto out_unlock;
	}

3631
	trace_sched_pi_setprio(p, prio);
3632
	oldprio = p->prio;
3633
	prev_class = p->sched_class;
P
Peter Zijlstra 已提交
3634
	on_rq = p->on_rq;
3635
	running = task_current(rq, p);
3636
	if (on_rq)
3637
		dequeue_task(rq, p, 0);
3638 3639
	if (running)
		p->sched_class->put_prev_task(rq, p);
I
Ingo Molnar 已提交
3640 3641 3642 3643 3644 3645

	if (rt_prio(prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;

3646 3647
	p->prio = prio;

3648 3649
	if (running)
		p->sched_class->set_curr_task(rq);
P
Peter Zijlstra 已提交
3650
	if (on_rq)
3651
		enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);
3652

P
Peter Zijlstra 已提交
3653
	check_class_changed(rq, p, prev_class, oldprio);
3654
out_unlock:
3655
	__task_rq_unlock(rq);
3656 3657
}
#endif
3658
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
3659
{
I
Ingo Molnar 已提交
3660
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
3661
	unsigned long flags;
3662
	struct rq *rq;
L
Linus Torvalds 已提交
3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674

	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
I
Ingo Molnar 已提交
3675
	 * SCHED_FIFO/SCHED_RR:
L
Linus Torvalds 已提交
3676
	 */
3677
	if (task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
3678 3679 3680
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
P
Peter Zijlstra 已提交
3681
	on_rq = p->on_rq;
3682
	if (on_rq)
3683
		dequeue_task(rq, p, 0);
L
Linus Torvalds 已提交
3684 3685

	p->static_prio = NICE_TO_PRIO(nice);
3686
	set_load_weight(p);
3687 3688 3689
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
3690

I
Ingo Molnar 已提交
3691
	if (on_rq) {
3692
		enqueue_task(rq, p, 0);
L
Linus Torvalds 已提交
3693
		/*
3694 3695
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
3696
		 */
3697
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
3698 3699 3700
			resched_task(rq->curr);
	}
out_unlock:
3701
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
3702 3703 3704
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
3705 3706 3707 3708 3709
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
3710
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
3711
{
3712 3713
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
3714

3715
	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
M
Matt Mackall 已提交
3716 3717 3718
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
3719 3720 3721 3722 3723 3724 3725 3726 3727
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
3728
SYSCALL_DEFINE1(nice, int, increment)
L
Linus Torvalds 已提交
3729
{
3730
	long nice, retval;
L
Linus Torvalds 已提交
3731 3732 3733 3734 3735 3736

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
3737 3738
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
3739 3740 3741
	if (increment > 40)
		increment = 40;

3742
	nice = TASK_NICE(current) + increment;
L
Linus Torvalds 已提交
3743 3744 3745 3746 3747
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
3748 3749 3750
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
 * This is the priority value as seen by users in /proc.
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
3769
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
3770 3771 3772 3773 3774 3775 3776 3777
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * task_nice - return the nice value of a given task.
 * @p: the task in question.
 */
3778
int task_nice(const struct task_struct *p)
L
Linus Torvalds 已提交
3779 3780 3781
{
	return TASK_NICE(p);
}
P
Pavel Roskin 已提交
3782
EXPORT_SYMBOL(task_nice);
L
Linus Torvalds 已提交
3783 3784 3785 3786 3787 3788 3789

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
 */
int idle_cpu(int cpu)
{
T
Thomas Gleixner 已提交
3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803
	struct rq *rq = cpu_rq(cpu);

	if (rq->curr != rq->idle)
		return 0;

	if (rq->nr_running)
		return 0;

#ifdef CONFIG_SMP
	if (!llist_empty(&rq->wake_list))
		return 0;
#endif

	return 1;
L
Linus Torvalds 已提交
3804 3805 3806 3807 3808 3809
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
 */
3810
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
3811 3812 3813 3814 3815 3816 3817 3818
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
 */
A
Alexey Dobriyan 已提交
3819
static struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
3820
{
3821
	return pid ? find_task_by_vpid(pid) : current;
L
Linus Torvalds 已提交
3822 3823 3824
}

/* Actually do priority change: must hold rq lock. */
I
Ingo Molnar 已提交
3825 3826
static void
__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
L
Linus Torvalds 已提交
3827 3828 3829
{
	p->policy = policy;
	p->rt_priority = prio;
3830 3831 3832
	p->normal_prio = normal_prio(p);
	/* we are holding p->pi_lock already */
	p->prio = rt_mutex_getprio(p);
3833 3834 3835 3836
	if (rt_prio(p->prio))
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;
3837
	set_load_weight(p);
L
Linus Torvalds 已提交
3838 3839
}

3840 3841 3842 3843 3844 3845 3846 3847 3848 3849
/*
 * check the target process has a UID that matches the current process's
 */
static bool check_same_owner(struct task_struct *p)
{
	const struct cred *cred = current_cred(), *pcred;
	bool match;

	rcu_read_lock();
	pcred = __task_cred(p);
3850 3851
	match = (uid_eq(cred->euid, pcred->euid) ||
		 uid_eq(cred->euid, pcred->uid));
3852 3853 3854 3855
	rcu_read_unlock();
	return match;
}

3856
static int __sched_setscheduler(struct task_struct *p, int policy,
3857
				const struct sched_param *param, bool user)
L
Linus Torvalds 已提交
3858
{
3859
	int retval, oldprio, oldpolicy = -1, on_rq, running;
L
Linus Torvalds 已提交
3860
	unsigned long flags;
3861
	const struct sched_class *prev_class;
3862
	struct rq *rq;
3863
	int reset_on_fork;
L
Linus Torvalds 已提交
3864

3865 3866
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
3867 3868
recheck:
	/* double check policy once rq lock held */
3869 3870
	if (policy < 0) {
		reset_on_fork = p->sched_reset_on_fork;
L
Linus Torvalds 已提交
3871
		policy = oldpolicy = p->policy;
3872 3873 3874 3875 3876 3877 3878 3879 3880 3881
	} else {
		reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
		policy &= ~SCHED_RESET_ON_FORK;

		if (policy != SCHED_FIFO && policy != SCHED_RR &&
				policy != SCHED_NORMAL && policy != SCHED_BATCH &&
				policy != SCHED_IDLE)
			return -EINVAL;
	}

L
Linus Torvalds 已提交
3882 3883
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
3884 3885
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
3886 3887
	 */
	if (param->sched_priority < 0 ||
I
Ingo Molnar 已提交
3888
	    (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
3889
	    (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
3890
		return -EINVAL;
3891
	if (rt_policy(policy) != (param->sched_priority != 0))
L
Linus Torvalds 已提交
3892 3893
		return -EINVAL;

3894 3895 3896
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
3897
	if (user && !capable(CAP_SYS_NICE)) {
3898
		if (rt_policy(policy)) {
3899 3900
			unsigned long rlim_rtprio =
					task_rlimit(p, RLIMIT_RTPRIO);
3901 3902 3903 3904 3905 3906 3907 3908 3909 3910

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
			if (param->sched_priority > p->rt_priority &&
			    param->sched_priority > rlim_rtprio)
				return -EPERM;
		}
3911

I
Ingo Molnar 已提交
3912
		/*
3913 3914
		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
I
Ingo Molnar 已提交
3915
		 */
3916 3917 3918 3919
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
			if (!can_nice(p, TASK_NICE(p)))
				return -EPERM;
		}
3920

3921
		/* can't change other user's priorities */
3922
		if (!check_same_owner(p))
3923
			return -EPERM;
3924 3925 3926 3927

		/* Normal users shall not reset the sched_reset_on_fork flag */
		if (p->sched_reset_on_fork && !reset_on_fork)
			return -EPERM;
3928
	}
L
Linus Torvalds 已提交
3929

3930
	if (user) {
3931
		retval = security_task_setscheduler(p);
3932 3933 3934 3935
		if (retval)
			return retval;
	}

3936 3937 3938
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
3939
	 *
L
Lucas De Marchi 已提交
3940
	 * To be able to change p->policy safely, the appropriate
L
Linus Torvalds 已提交
3941 3942
	 * runqueue lock must be held.
	 */
3943
	rq = task_rq_lock(p, &flags);
3944

3945 3946 3947 3948
	/*
	 * Changing the policy of the stop threads its a very bad idea
	 */
	if (p == rq->stop) {
3949
		task_rq_unlock(rq, p, &flags);
3950 3951 3952
		return -EINVAL;
	}

3953 3954 3955 3956 3957
	/*
	 * If not changing anything there's no need to proceed further:
	 */
	if (unlikely(policy == p->policy && (!rt_policy(policy) ||
			param->sched_priority == p->rt_priority))) {
3958
		task_rq_unlock(rq, p, &flags);
3959 3960 3961
		return 0;
	}

3962 3963 3964 3965 3966 3967 3968
#ifdef CONFIG_RT_GROUP_SCHED
	if (user) {
		/*
		 * Do not allow realtime tasks into groups that have no runtime
		 * assigned.
		 */
		if (rt_bandwidth_enabled() && rt_policy(policy) &&
3969 3970
				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
				!task_group_is_autogroup(task_group(p))) {
3971
			task_rq_unlock(rq, p, &flags);
3972 3973 3974 3975 3976
			return -EPERM;
		}
	}
#endif

L
Linus Torvalds 已提交
3977 3978 3979
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
3980
		task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
3981 3982
		goto recheck;
	}
P
Peter Zijlstra 已提交
3983
	on_rq = p->on_rq;
3984
	running = task_current(rq, p);
3985
	if (on_rq)
3986
		dequeue_task(rq, p, 0);
3987 3988
	if (running)
		p->sched_class->put_prev_task(rq, p);
3989

3990 3991
	p->sched_reset_on_fork = reset_on_fork;

L
Linus Torvalds 已提交
3992
	oldprio = p->prio;
3993
	prev_class = p->sched_class;
I
Ingo Molnar 已提交
3994
	__setscheduler(rq, p, policy, param->sched_priority);
3995

3996 3997
	if (running)
		p->sched_class->set_curr_task(rq);
P
Peter Zijlstra 已提交
3998
	if (on_rq)
3999
		enqueue_task(rq, p, 0);
4000

P
Peter Zijlstra 已提交
4001
	check_class_changed(rq, p, prev_class, oldprio);
4002
	task_rq_unlock(rq, p, &flags);
4003

4004 4005
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
4006 4007
	return 0;
}
4008 4009 4010 4011 4012 4013 4014 4015 4016 4017

/**
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * NOTE that the task may be already dead.
 */
int sched_setscheduler(struct task_struct *p, int policy,
4018
		       const struct sched_param *param)
4019 4020 4021
{
	return __sched_setscheduler(p, policy, param, true);
}
L
Linus Torvalds 已提交
4022 4023
EXPORT_SYMBOL_GPL(sched_setscheduler);

4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035
/**
 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * Just like sched_setscheduler, only don't bother checking if the
 * current context has permission.  For example, this is needed in
 * stop_machine(): we create temporary high priority worker threads,
 * but our caller might not have that capability.
 */
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
4036
			       const struct sched_param *param)
4037 4038 4039 4040
{
	return __sched_setscheduler(p, policy, param, false);
}

I
Ingo Molnar 已提交
4041 4042
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
4043 4044 4045
{
	struct sched_param lparam;
	struct task_struct *p;
4046
	int retval;
L
Linus Torvalds 已提交
4047 4048 4049 4050 4051

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
4052 4053 4054

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
4055
	p = find_process_by_pid(pid);
4056 4057 4058
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
4059

L
Linus Torvalds 已提交
4060 4061 4062 4063 4064 4065 4066 4067 4068
	return retval;
}

/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 */
4069 4070
SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
		struct sched_param __user *, param)
L
Linus Torvalds 已提交
4071
{
4072 4073 4074 4075
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
4076 4077 4078 4079 4080 4081 4082 4083
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
 */
4084
SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
4085 4086 4087 4088 4089 4090 4091 4092
{
	return do_sched_setscheduler(pid, -1, param);
}

/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
 */
4093
SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
L
Linus Torvalds 已提交
4094
{
4095
	struct task_struct *p;
4096
	int retval;
L
Linus Torvalds 已提交
4097 4098

	if (pid < 0)
4099
		return -EINVAL;
L
Linus Torvalds 已提交
4100 4101

	retval = -ESRCH;
4102
	rcu_read_lock();
L
Linus Torvalds 已提交
4103 4104 4105 4106
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
4107 4108
			retval = p->policy
				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
L
Linus Torvalds 已提交
4109
	}
4110
	rcu_read_unlock();
L
Linus Torvalds 已提交
4111 4112 4113 4114
	return retval;
}

/**
4115
 * sys_sched_getparam - get the RT priority of a thread
L
Linus Torvalds 已提交
4116 4117 4118
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
 */
4119
SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
4120 4121
{
	struct sched_param lp;
4122
	struct task_struct *p;
4123
	int retval;
L
Linus Torvalds 已提交
4124 4125

	if (!param || pid < 0)
4126
		return -EINVAL;
L
Linus Torvalds 已提交
4127

4128
	rcu_read_lock();
L
Linus Torvalds 已提交
4129 4130 4131 4132 4133 4134 4135 4136 4137 4138
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	lp.sched_priority = p->rt_priority;
4139
	rcu_read_unlock();
L
Linus Torvalds 已提交
4140 4141 4142 4143 4144 4145 4146 4147 4148

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

	return retval;

out_unlock:
4149
	rcu_read_unlock();
L
Linus Torvalds 已提交
4150 4151 4152
	return retval;
}

4153
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
L
Linus Torvalds 已提交
4154
{
4155
	cpumask_var_t cpus_allowed, new_mask;
4156 4157
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
4158

4159
	get_online_cpus();
4160
	rcu_read_lock();
L
Linus Torvalds 已提交
4161 4162 4163

	p = find_process_by_pid(pid);
	if (!p) {
4164
		rcu_read_unlock();
4165
		put_online_cpus();
L
Linus Torvalds 已提交
4166 4167 4168
		return -ESRCH;
	}

4169
	/* Prevent p going away */
L
Linus Torvalds 已提交
4170
	get_task_struct(p);
4171
	rcu_read_unlock();
L
Linus Torvalds 已提交
4172

4173 4174 4175 4176 4177 4178 4179 4180
	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_put_task;
	}
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_free_cpus_allowed;
	}
L
Linus Torvalds 已提交
4181
	retval = -EPERM;
E
Eric W. Biederman 已提交
4182 4183 4184 4185 4186 4187 4188 4189
	if (!check_same_owner(p)) {
		rcu_read_lock();
		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
			rcu_read_unlock();
			goto out_unlock;
		}
		rcu_read_unlock();
	}
L
Linus Torvalds 已提交
4190

4191
	retval = security_task_setscheduler(p);
4192 4193 4194
	if (retval)
		goto out_unlock;

4195 4196
	cpuset_cpus_allowed(p, cpus_allowed);
	cpumask_and(new_mask, in_mask, cpus_allowed);
P
Peter Zijlstra 已提交
4197
again:
4198
	retval = set_cpus_allowed_ptr(p, new_mask);
L
Linus Torvalds 已提交
4199

P
Paul Menage 已提交
4200
	if (!retval) {
4201 4202
		cpuset_cpus_allowed(p, cpus_allowed);
		if (!cpumask_subset(new_mask, cpus_allowed)) {
P
Paul Menage 已提交
4203 4204 4205 4206 4207
			/*
			 * We must have raced with a concurrent cpuset
			 * update. Just reset the cpus_allowed to the
			 * cpuset's cpus_allowed
			 */
4208
			cpumask_copy(new_mask, cpus_allowed);
P
Paul Menage 已提交
4209 4210 4211
			goto again;
		}
	}
L
Linus Torvalds 已提交
4212
out_unlock:
4213 4214 4215 4216
	free_cpumask_var(new_mask);
out_free_cpus_allowed:
	free_cpumask_var(cpus_allowed);
out_put_task:
L
Linus Torvalds 已提交
4217
	put_task_struct(p);
4218
	put_online_cpus();
L
Linus Torvalds 已提交
4219 4220 4221 4222
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
4223
			     struct cpumask *new_mask)
L
Linus Torvalds 已提交
4224
{
4225 4226 4227 4228 4229
	if (len < cpumask_size())
		cpumask_clear(new_mask);
	else if (len > cpumask_size())
		len = cpumask_size();

L
Linus Torvalds 已提交
4230 4231 4232 4233 4234 4235 4236 4237 4238
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
 */
4239 4240
SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
4241
{
4242
	cpumask_var_t new_mask;
L
Linus Torvalds 已提交
4243 4244
	int retval;

4245 4246
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
4247

4248 4249 4250 4251 4252
	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
	if (retval == 0)
		retval = sched_setaffinity(pid, new_mask);
	free_cpumask_var(new_mask);
	return retval;
L
Linus Torvalds 已提交
4253 4254
}

4255
long sched_getaffinity(pid_t pid, struct cpumask *mask)
L
Linus Torvalds 已提交
4256
{
4257
	struct task_struct *p;
4258
	unsigned long flags;
L
Linus Torvalds 已提交
4259 4260
	int retval;

4261
	get_online_cpus();
4262
	rcu_read_lock();
L
Linus Torvalds 已提交
4263 4264 4265 4266 4267 4268

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

4269 4270 4271 4272
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4273
	raw_spin_lock_irqsave(&p->pi_lock, flags);
4274
	cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
4275
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4276 4277

out_unlock:
4278
	rcu_read_unlock();
4279
	put_online_cpus();
L
Linus Torvalds 已提交
4280

4281
	return retval;
L
Linus Torvalds 已提交
4282 4283 4284 4285 4286 4287 4288 4289
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
 */
4290 4291
SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
4292 4293
{
	int ret;
4294
	cpumask_var_t mask;
L
Linus Torvalds 已提交
4295

A
Anton Blanchard 已提交
4296
	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
4297 4298
		return -EINVAL;
	if (len & (sizeof(unsigned long)-1))
L
Linus Torvalds 已提交
4299 4300
		return -EINVAL;

4301 4302
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
4303

4304 4305
	ret = sched_getaffinity(pid, mask);
	if (ret == 0) {
4306
		size_t retlen = min_t(size_t, len, cpumask_size());
4307 4308

		if (copy_to_user(user_mask_ptr, mask, retlen))
4309 4310
			ret = -EFAULT;
		else
4311
			ret = retlen;
4312 4313
	}
	free_cpumask_var(mask);
L
Linus Torvalds 已提交
4314

4315
	return ret;
L
Linus Torvalds 已提交
4316 4317 4318 4319 4320
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
4321 4322
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
L
Linus Torvalds 已提交
4323
 */
4324
SYSCALL_DEFINE0(sched_yield)
L
Linus Torvalds 已提交
4325
{
4326
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
4327

4328
	schedstat_inc(rq, yld_count);
4329
	current->sched_class->yield_task(rq);
L
Linus Torvalds 已提交
4330 4331 4332 4333 4334 4335

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
4336
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
4337
	do_raw_spin_unlock(&rq->lock);
4338
	sched_preempt_enable_no_resched();
L
Linus Torvalds 已提交
4339 4340 4341 4342 4343 4344

	schedule();

	return 0;
}

P
Peter Zijlstra 已提交
4345 4346 4347 4348 4349
static inline int should_resched(void)
{
	return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
}

A
Andrew Morton 已提交
4350
static void __cond_resched(void)
L
Linus Torvalds 已提交
4351
{
4352
	add_preempt_count(PREEMPT_ACTIVE);
4353
	__schedule();
4354
	sub_preempt_count(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4355 4356
}

4357
int __sched _cond_resched(void)
L
Linus Torvalds 已提交
4358
{
P
Peter Zijlstra 已提交
4359
	if (should_resched()) {
L
Linus Torvalds 已提交
4360 4361 4362 4363 4364
		__cond_resched();
		return 1;
	}
	return 0;
}
4365
EXPORT_SYMBOL(_cond_resched);
L
Linus Torvalds 已提交
4366 4367

/*
4368
 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
L
Linus Torvalds 已提交
4369 4370
 * call schedule, and on return reacquire the lock.
 *
I
Ingo Molnar 已提交
4371
 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
L
Linus Torvalds 已提交
4372 4373 4374
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
4375
int __cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
4376
{
P
Peter Zijlstra 已提交
4377
	int resched = should_resched();
J
Jan Kara 已提交
4378 4379
	int ret = 0;

4380 4381
	lockdep_assert_held(lock);

N
Nick Piggin 已提交
4382
	if (spin_needbreak(lock) || resched) {
L
Linus Torvalds 已提交
4383
		spin_unlock(lock);
P
Peter Zijlstra 已提交
4384
		if (resched)
N
Nick Piggin 已提交
4385 4386 4387
			__cond_resched();
		else
			cpu_relax();
J
Jan Kara 已提交
4388
		ret = 1;
L
Linus Torvalds 已提交
4389 4390
		spin_lock(lock);
	}
J
Jan Kara 已提交
4391
	return ret;
L
Linus Torvalds 已提交
4392
}
4393
EXPORT_SYMBOL(__cond_resched_lock);
L
Linus Torvalds 已提交
4394

4395
int __sched __cond_resched_softirq(void)
L
Linus Torvalds 已提交
4396 4397 4398
{
	BUG_ON(!in_softirq());

P
Peter Zijlstra 已提交
4399
	if (should_resched()) {
4400
		local_bh_enable();
L
Linus Torvalds 已提交
4401 4402 4403 4404 4405 4406
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
4407
EXPORT_SYMBOL(__cond_resched_softirq);
L
Linus Torvalds 已提交
4408 4409 4410 4411

/**
 * yield - yield the current processor to other threads.
 *
P
Peter Zijlstra 已提交
4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429
 * Do not ever use this function, there's a 99% chance you're doing it wrong.
 *
 * The scheduler is at all times free to pick the calling task as the most
 * eligible task to run, if removing the yield() call from your code breaks
 * it, its already broken.
 *
 * Typical broken usage is:
 *
 * while (!event)
 * 	yield();
 *
 * where one assumes that yield() will let 'the other' process run that will
 * make event true. If the current task is a SCHED_FIFO task that will never
 * happen. Never use yield() as a progress guarantee!!
 *
 * If you want to use yield() to wait for something, use wait_event().
 * If you want to use yield() to be 'nice' for others, use cond_resched().
 * If you still want to use yield(), do not!
L
Linus Torvalds 已提交
4430 4431 4432 4433 4434 4435 4436 4437
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

4438 4439 4440 4441
/**
 * yield_to - yield the current processor to another thread in
 * your thread group, or accelerate that thread toward the
 * processor it's on.
R
Randy Dunlap 已提交
4442 4443
 * @p: target task
 * @preempt: whether task preemption is allowed or not
4444 4445 4446 4447
 *
 * It's the caller's job to ensure that the target task struct
 * can't go away on us before we can do any checks.
 *
4448 4449 4450 4451
 * Returns:
 *	true (>0) if we indeed boosted the target task.
 *	false (0) if we failed to boost the target.
 *	-ESRCH if there's no task to yield to.
4452 4453 4454 4455 4456 4457
 */
bool __sched yield_to(struct task_struct *p, bool preempt)
{
	struct task_struct *curr = current;
	struct rq *rq, *p_rq;
	unsigned long flags;
4458
	int yielded = 0;
4459 4460 4461 4462 4463 4464

	local_irq_save(flags);
	rq = this_rq();

again:
	p_rq = task_rq(p);
4465 4466 4467 4468 4469 4470 4471 4472 4473
	/*
	 * If we're the only runnable task on the rq and target rq also
	 * has only one task, there's absolutely no point in yielding.
	 */
	if (rq->nr_running == 1 && p_rq->nr_running == 1) {
		yielded = -ESRCH;
		goto out_irq;
	}

4474 4475 4476 4477 4478 4479 4480
	double_rq_lock(rq, p_rq);
	while (task_rq(p) != p_rq) {
		double_rq_unlock(rq, p_rq);
		goto again;
	}

	if (!curr->sched_class->yield_to_task)
4481
		goto out_unlock;
4482 4483

	if (curr->sched_class != p->sched_class)
4484
		goto out_unlock;
4485 4486

	if (task_running(p_rq, p) || p->state)
4487
		goto out_unlock;
4488 4489

	yielded = curr->sched_class->yield_to_task(rq, p, preempt);
4490
	if (yielded) {
4491
		schedstat_inc(rq, yld_count);
4492 4493 4494 4495 4496 4497 4498
		/*
		 * Make p's CPU reschedule; pick_next_entity takes care of
		 * fairness.
		 */
		if (preempt && rq != p_rq)
			resched_task(p_rq->curr);
	}
4499

4500
out_unlock:
4501
	double_rq_unlock(rq, p_rq);
4502
out_irq:
4503 4504
	local_irq_restore(flags);

4505
	if (yielded > 0)
4506 4507 4508 4509 4510 4511
		schedule();

	return yielded;
}
EXPORT_SYMBOL_GPL(yield_to);

L
Linus Torvalds 已提交
4512
/*
I
Ingo Molnar 已提交
4513
 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
L
Linus Torvalds 已提交
4514 4515 4516 4517
 * that process accounting knows that this is a task in IO wait state.
 */
void __sched io_schedule(void)
{
4518
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
4519

4520
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4521
	atomic_inc(&rq->nr_iowait);
4522
	blk_flush_plug(current);
4523
	current->in_iowait = 1;
L
Linus Torvalds 已提交
4524
	schedule();
4525
	current->in_iowait = 0;
L
Linus Torvalds 已提交
4526
	atomic_dec(&rq->nr_iowait);
4527
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4528 4529 4530 4531 4532
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
4533
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
4534 4535
	long ret;

4536
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4537
	atomic_inc(&rq->nr_iowait);
4538
	blk_flush_plug(current);
4539
	current->in_iowait = 1;
L
Linus Torvalds 已提交
4540
	ret = schedule_timeout(timeout);
4541
	current->in_iowait = 0;
L
Linus Torvalds 已提交
4542
	atomic_dec(&rq->nr_iowait);
4543
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4544 4545 4546 4547 4548 4549 4550 4551 4552 4553
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the maximum rt_priority that can be used
 * by a given scheduling class.
 */
4554
SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
L
Linus Torvalds 已提交
4555 4556 4557 4558 4559 4560 4561 4562 4563
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
	case SCHED_NORMAL:
4564
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4565
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
 * this syscall returns the minimum rt_priority that can be used
 * by a given scheduling class.
 */
4579
SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
L
Linus Torvalds 已提交
4580 4581 4582 4583 4584 4585 4586 4587 4588
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
	case SCHED_NORMAL:
4589
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4590
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
 */
4604
SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
4605
		struct timespec __user *, interval)
L
Linus Torvalds 已提交
4606
{
4607
	struct task_struct *p;
D
Dmitry Adamushko 已提交
4608
	unsigned int time_slice;
4609 4610
	unsigned long flags;
	struct rq *rq;
4611
	int retval;
L
Linus Torvalds 已提交
4612 4613 4614
	struct timespec t;

	if (pid < 0)
4615
		return -EINVAL;
L
Linus Torvalds 已提交
4616 4617

	retval = -ESRCH;
4618
	rcu_read_lock();
L
Linus Torvalds 已提交
4619 4620 4621 4622 4623 4624 4625 4626
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4627 4628
	rq = task_rq_lock(p, &flags);
	time_slice = p->sched_class->get_rr_interval(rq, p);
4629
	task_rq_unlock(rq, p, &flags);
D
Dmitry Adamushko 已提交
4630

4631
	rcu_read_unlock();
D
Dmitry Adamushko 已提交
4632
	jiffies_to_timespec(time_slice, &t);
L
Linus Torvalds 已提交
4633 4634
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
	return retval;
4635

L
Linus Torvalds 已提交
4636
out_unlock:
4637
	rcu_read_unlock();
L
Linus Torvalds 已提交
4638 4639 4640
	return retval;
}

4641
static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
4642

4643
void sched_show_task(struct task_struct *p)
L
Linus Torvalds 已提交
4644 4645
{
	unsigned long free = 0;
4646
	int ppid;
4647
	unsigned state;
L
Linus Torvalds 已提交
4648 4649

	state = p->state ? __ffs(p->state) + 1 : 0;
4650
	printk(KERN_INFO "%-15.15s %c", p->comm,
4651
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4652
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
4653
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
4654
		printk(KERN_CONT " running  ");
L
Linus Torvalds 已提交
4655
	else
P
Peter Zijlstra 已提交
4656
		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4657 4658
#else
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
4659
		printk(KERN_CONT "  running task    ");
L
Linus Torvalds 已提交
4660
	else
P
Peter Zijlstra 已提交
4661
		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4662 4663
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
4664
	free = stack_not_used(p);
L
Linus Torvalds 已提交
4665
#endif
4666 4667 4668
	rcu_read_lock();
	ppid = task_pid_nr(rcu_dereference(p->real_parent));
	rcu_read_unlock();
P
Peter Zijlstra 已提交
4669
	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
4670
		task_pid_nr(p), ppid,
4671
		(unsigned long)task_thread_info(p)->flags);
L
Linus Torvalds 已提交
4672

4673
	show_stack(p, NULL);
L
Linus Torvalds 已提交
4674 4675
}

I
Ingo Molnar 已提交
4676
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
4677
{
4678
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
4679

4680
#if BITS_PER_LONG == 32
P
Peter Zijlstra 已提交
4681 4682
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
4683
#else
P
Peter Zijlstra 已提交
4684 4685
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
4686
#endif
4687
	rcu_read_lock();
L
Linus Torvalds 已提交
4688 4689 4690
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
L
Lucas De Marchi 已提交
4691
		 * console might take a lot of time:
L
Linus Torvalds 已提交
4692 4693
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
4694
		if (!state_filter || (p->state & state_filter))
4695
			sched_show_task(p);
L
Linus Torvalds 已提交
4696 4697
	} while_each_thread(g, p);

4698 4699
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
4700 4701 4702
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
4703
	rcu_read_unlock();
I
Ingo Molnar 已提交
4704 4705 4706
	/*
	 * Only show locks if all tasks are dumped:
	 */
4707
	if (!state_filter)
I
Ingo Molnar 已提交
4708
		debug_show_all_locks();
L
Linus Torvalds 已提交
4709 4710
}

I
Ingo Molnar 已提交
4711 4712
void __cpuinit init_idle_bootup_task(struct task_struct *idle)
{
I
Ingo Molnar 已提交
4713
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
4714 4715
}

4716 4717 4718 4719 4720 4721 4722 4723
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
4724
void __cpuinit init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
4725
{
4726
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4727 4728
	unsigned long flags;

4729
	raw_spin_lock_irqsave(&rq->lock, flags);
4730

I
Ingo Molnar 已提交
4731
	__sched_fork(idle);
4732
	idle->state = TASK_RUNNING;
I
Ingo Molnar 已提交
4733 4734
	idle->se.exec_start = sched_clock();

4735
	do_set_cpus_allowed(idle, cpumask_of(cpu));
4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746
	/*
	 * We're having a chicken and egg problem, even though we are
	 * holding rq->lock, the cpu isn't yet set to this cpu so the
	 * lockdep check in task_group() will fail.
	 *
	 * Similar case to sched_fork(). / Alternatively we could
	 * use task_rq_lock() here and obtain the other rq->lock.
	 *
	 * Silence PROVE_RCU
	 */
	rcu_read_lock();
I
Ingo Molnar 已提交
4747
	__set_task_cpu(idle, cpu);
4748
	rcu_read_unlock();
L
Linus Torvalds 已提交
4749 4750

	rq->curr = rq->idle = idle;
P
Peter Zijlstra 已提交
4751 4752
#if defined(CONFIG_SMP)
	idle->on_cpu = 1;
4753
#endif
4754
	raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
4755 4756

	/* Set the preempt count _outside_ the spinlocks! */
A
Al Viro 已提交
4757
	task_thread_info(idle)->preempt_count = 0;
4758

I
Ingo Molnar 已提交
4759 4760 4761 4762
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
4763
	ftrace_graph_init_idle_task(idle, cpu);
4764
	vtime_init_idle(idle);
4765 4766 4767
#if defined(CONFIG_SMP)
	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
#endif
I
Ingo Molnar 已提交
4768 4769
}

L
Linus Torvalds 已提交
4770
#ifdef CONFIG_SMP
4771 4772 4773 4774
void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
	if (p->sched_class && p->sched_class->set_cpus_allowed)
		p->sched_class->set_cpus_allowed(p, new_mask);
4775 4776

	cpumask_copy(&p->cpus_allowed, new_mask);
4777
	p->nr_cpus_allowed = cpumask_weight(new_mask);
4778 4779
}

L
Linus Torvalds 已提交
4780 4781 4782
/*
 * This is how migration works:
 *
4783 4784 4785 4786 4787 4788
 * 1) we invoke migration_cpu_stop() on the target CPU using
 *    stop_one_cpu().
 * 2) stopper starts to run (implicitly forcing the migrated thread
 *    off the CPU)
 * 3) it checks whether the migrated task is still in the wrong runqueue.
 * 4) if it's in the wrong runqueue then the migration thread removes
L
Linus Torvalds 已提交
4789
 *    it and puts it into the right queue.
4790 4791
 * 5) stopper completes and stop_one_cpu() returns and the migration
 *    is done.
L
Linus Torvalds 已提交
4792 4793 4794 4795 4796 4797 4798 4799
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
I
Ingo Molnar 已提交
4800
 * task must not exit() & deallocate itself prematurely. The
L
Linus Torvalds 已提交
4801 4802
 * call is not atomic; no spinlocks may be held.
 */
4803
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
L
Linus Torvalds 已提交
4804 4805
{
	unsigned long flags;
4806
	struct rq *rq;
4807
	unsigned int dest_cpu;
4808
	int ret = 0;
L
Linus Torvalds 已提交
4809 4810

	rq = task_rq_lock(p, &flags);
4811

4812 4813 4814
	if (cpumask_equal(&p->cpus_allowed, new_mask))
		goto out;

4815
	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
L
Linus Torvalds 已提交
4816 4817 4818 4819
		ret = -EINVAL;
		goto out;
	}

4820
	if (unlikely((p->flags & PF_THREAD_BOUND) && p != current)) {
4821 4822 4823 4824
		ret = -EINVAL;
		goto out;
	}

4825
	do_set_cpus_allowed(p, new_mask);
4826

L
Linus Torvalds 已提交
4827
	/* Can the task run on the task's current CPU? If so, we're done */
4828
	if (cpumask_test_cpu(task_cpu(p), new_mask))
L
Linus Torvalds 已提交
4829 4830
		goto out;

4831
	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
4832
	if (p->on_rq) {
4833
		struct migration_arg arg = { p, dest_cpu };
L
Linus Torvalds 已提交
4834
		/* Need help from migration thread: drop lock and wait. */
4835
		task_rq_unlock(rq, p, &flags);
4836
		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
4837 4838 4839 4840
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
4841
	task_rq_unlock(rq, p, &flags);
4842

L
Linus Torvalds 已提交
4843 4844
	return ret;
}
4845
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
L
Linus Torvalds 已提交
4846 4847

/*
I
Ingo Molnar 已提交
4848
 * Move (not current) task off this cpu, onto dest cpu. We're doing
L
Linus Torvalds 已提交
4849 4850 4851 4852 4853 4854
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
4855 4856
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
4857
 */
4858
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
4859
{
4860
	struct rq *rq_dest, *rq_src;
4861
	int ret = 0;
L
Linus Torvalds 已提交
4862

4863
	if (unlikely(!cpu_active(dest_cpu)))
4864
		return ret;
L
Linus Torvalds 已提交
4865 4866 4867 4868

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

4869
	raw_spin_lock(&p->pi_lock);
L
Linus Torvalds 已提交
4870 4871 4872
	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
L
Linus Torvalds 已提交
4873
		goto done;
L
Linus Torvalds 已提交
4874
	/* Affinity changed (again). */
4875
	if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
L
Linus Torvalds 已提交
4876
		goto fail;
L
Linus Torvalds 已提交
4877

4878 4879 4880 4881
	/*
	 * If we're not on a rq, the next wake-up will ensure we're
	 * placed properly.
	 */
P
Peter Zijlstra 已提交
4882
	if (p->on_rq) {
4883
		dequeue_task(rq_src, p, 0);
4884
		set_task_cpu(p, dest_cpu);
4885
		enqueue_task(rq_dest, p, 0);
4886
		check_preempt_curr(rq_dest, p, 0);
L
Linus Torvalds 已提交
4887
	}
L
Linus Torvalds 已提交
4888
done:
4889
	ret = 1;
L
Linus Torvalds 已提交
4890
fail:
L
Linus Torvalds 已提交
4891
	double_rq_unlock(rq_src, rq_dest);
4892
	raw_spin_unlock(&p->pi_lock);
4893
	return ret;
L
Linus Torvalds 已提交
4894 4895 4896
}

/*
4897 4898 4899
 * migration_cpu_stop - this will be executed by a highprio stopper thread
 * and performs thread migration by bumping thread off CPU then
 * 'pushing' onto another runqueue.
L
Linus Torvalds 已提交
4900
 */
4901
static int migration_cpu_stop(void *data)
L
Linus Torvalds 已提交
4902
{
4903
	struct migration_arg *arg = data;
4904

4905 4906 4907 4908
	/*
	 * The original target cpu might have gone down and we might
	 * be on another cpu but it doesn't matter.
	 */
4909
	local_irq_disable();
4910
	__migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
4911
	local_irq_enable();
L
Linus Torvalds 已提交
4912
	return 0;
4913 4914
}

L
Linus Torvalds 已提交
4915
#ifdef CONFIG_HOTPLUG_CPU
4916

4917
/*
4918 4919
 * Ensures that the idle task is using init_mm right before its cpu goes
 * offline.
4920
 */
4921
void idle_task_exit(void)
L
Linus Torvalds 已提交
4922
{
4923
	struct mm_struct *mm = current->active_mm;
4924

4925
	BUG_ON(cpu_online(smp_processor_id()));
4926

4927 4928 4929
	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
L
Linus Torvalds 已提交
4930 4931 4932
}

/*
4933 4934 4935 4936 4937
 * Since this CPU is going 'away' for a while, fold any nr_active delta
 * we might have. Assumes we're called after migrate_tasks() so that the
 * nr_active count is stable.
 *
 * Also see the comment "Global load-average calculations".
L
Linus Torvalds 已提交
4938
 */
4939
static void calc_load_migrate(struct rq *rq)
L
Linus Torvalds 已提交
4940
{
4941 4942 4943
	long delta = calc_load_fold_active(rq);
	if (delta)
		atomic_long_add(delta, &calc_load_tasks);
L
Linus Torvalds 已提交
4944 4945
}

4946
/*
4947 4948 4949 4950 4951 4952
 * Migrate all tasks from the rq, sleeping tasks will be migrated by
 * try_to_wake_up()->select_task_rq().
 *
 * Called with rq->lock held even though we'er in stop_machine() and
 * there's no concurrency possible, we hold the required locks anyway
 * because of lock validation efforts.
L
Linus Torvalds 已提交
4953
 */
4954
static void migrate_tasks(unsigned int dead_cpu)
L
Linus Torvalds 已提交
4955
{
4956
	struct rq *rq = cpu_rq(dead_cpu);
4957 4958
	struct task_struct *next, *stop = rq->stop;
	int dest_cpu;
L
Linus Torvalds 已提交
4959 4960

	/*
4961 4962 4963 4964 4965 4966 4967
	 * Fudge the rq selection such that the below task selection loop
	 * doesn't get stuck on the currently eligible stop task.
	 *
	 * We're currently inside stop_machine() and the rq is either stuck
	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
	 * either way we should never end up calling schedule() until we're
	 * done here.
L
Linus Torvalds 已提交
4968
	 */
4969
	rq->stop = NULL;
4970

I
Ingo Molnar 已提交
4971
	for ( ; ; ) {
4972 4973 4974 4975 4976
		/*
		 * There's this thread running, bail when that's the only
		 * remaining thread.
		 */
		if (rq->nr_running == 1)
I
Ingo Molnar 已提交
4977
			break;
4978

4979
		next = pick_next_task(rq);
4980
		BUG_ON(!next);
D
Dmitry Adamushko 已提交
4981
		next->sched_class->put_prev_task(rq, next);
4982

4983 4984 4985 4986 4987 4988 4989
		/* Find suitable destination for @next, with force if needed. */
		dest_cpu = select_fallback_rq(dead_cpu, next);
		raw_spin_unlock(&rq->lock);

		__migrate_task(next, dead_cpu, dest_cpu);

		raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
4990
	}
4991

4992
	rq->stop = stop;
4993
}
4994

L
Linus Torvalds 已提交
4995 4996
#endif /* CONFIG_HOTPLUG_CPU */

4997 4998 4999
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
5000 5001
	{
		.procname	= "sched_domain",
5002
		.mode		= 0555,
5003
	},
5004
	{}
5005 5006 5007
};

static struct ctl_table sd_ctl_root[] = {
5008 5009
	{
		.procname	= "kernel",
5010
		.mode		= 0555,
5011 5012
		.child		= sd_ctl_dir,
	},
5013
	{}
5014 5015 5016 5017 5018
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
5019
		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
5020 5021 5022 5023

	return entry;
}

5024 5025
static void sd_free_ctl_entry(struct ctl_table **tablep)
{
5026
	struct ctl_table *entry;
5027

5028 5029 5030
	/*
	 * In the intermediate directories, both the child directory and
	 * procname are dynamically allocated and could fail but the mode
I
Ingo Molnar 已提交
5031
	 * will always be set. In the lowest directory the names are
5032 5033 5034
	 * static strings and all have proc handlers.
	 */
	for (entry = *tablep; entry->mode; entry++) {
5035 5036
		if (entry->child)
			sd_free_ctl_entry(&entry->child);
5037 5038 5039
		if (entry->proc_handler == NULL)
			kfree(entry->procname);
	}
5040 5041 5042 5043 5044

	kfree(*tablep);
	*tablep = NULL;
}

5045 5046 5047
static int min_load_idx = 0;
static int max_load_idx = CPU_LOAD_IDX_MAX;

5048
static void
5049
set_table_entry(struct ctl_table *entry,
5050
		const char *procname, void *data, int maxlen,
5051 5052
		umode_t mode, proc_handler *proc_handler,
		bool load_idx)
5053 5054 5055 5056 5057 5058
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
5059 5060 5061 5062 5063

	if (load_idx) {
		entry->extra1 = &min_load_idx;
		entry->extra2 = &max_load_idx;
	}
5064 5065 5066 5067 5068
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
5069
	struct ctl_table *table = sd_alloc_ctl_entry(13);
5070

5071 5072 5073
	if (table == NULL)
		return NULL;

5074
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
5075
		sizeof(long), 0644, proc_doulongvec_minmax, false);
5076
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
5077
		sizeof(long), 0644, proc_doulongvec_minmax, false);
5078
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
5079
		sizeof(int), 0644, proc_dointvec_minmax, true);
5080
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
5081
		sizeof(int), 0644, proc_dointvec_minmax, true);
5082
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
5083
		sizeof(int), 0644, proc_dointvec_minmax, true);
5084
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
5085
		sizeof(int), 0644, proc_dointvec_minmax, true);
5086
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
5087
		sizeof(int), 0644, proc_dointvec_minmax, true);
5088
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
5089
		sizeof(int), 0644, proc_dointvec_minmax, false);
5090
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
5091
		sizeof(int), 0644, proc_dointvec_minmax, false);
5092
	set_table_entry(&table[9], "cache_nice_tries",
5093
		&sd->cache_nice_tries,
5094
		sizeof(int), 0644, proc_dointvec_minmax, false);
5095
	set_table_entry(&table[10], "flags", &sd->flags,
5096
		sizeof(int), 0644, proc_dointvec_minmax, false);
5097
	set_table_entry(&table[11], "name", sd->name,
5098
		CORENAME_MAX_SIZE, 0444, proc_dostring, false);
5099
	/* &table[12] is terminator */
5100 5101 5102 5103

	return table;
}

5104
static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
5105 5106 5107 5108 5109 5110 5111 5112 5113
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);
5114 5115
	if (table == NULL)
		return NULL;
5116 5117 5118 5119 5120

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
5121
		entry->mode = 0555;
5122 5123 5124 5125 5126 5127 5128 5129
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
5130
static void register_sched_domain_sysctl(void)
5131
{
5132
	int i, cpu_num = num_possible_cpus();
5133 5134 5135
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

5136 5137 5138
	WARN_ON(sd_ctl_dir[0].child);
	sd_ctl_dir[0].child = entry;

5139 5140 5141
	if (entry == NULL)
		return;

5142
	for_each_possible_cpu(i) {
5143 5144
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
5145
		entry->mode = 0555;
5146
		entry->child = sd_alloc_ctl_cpu_table(i);
5147
		entry++;
5148
	}
5149 5150

	WARN_ON(sd_sysctl_header);
5151 5152
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
5153

5154
/* may be called multiple times per register */
5155 5156
static void unregister_sched_domain_sysctl(void)
{
5157 5158
	if (sd_sysctl_header)
		unregister_sysctl_table(sd_sysctl_header);
5159
	sd_sysctl_header = NULL;
5160 5161
	if (sd_ctl_dir[0].child)
		sd_free_ctl_entry(&sd_ctl_dir[0].child);
5162
}
5163
#else
5164 5165 5166 5167
static void register_sched_domain_sysctl(void)
{
}
static void unregister_sched_domain_sysctl(void)
5168 5169 5170 5171
{
}
#endif

5172 5173 5174 5175 5176
static void set_rq_online(struct rq *rq)
{
	if (!rq->online) {
		const struct sched_class *class;

5177
		cpumask_set_cpu(rq->cpu, rq->rd->online);
5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196
		rq->online = 1;

		for_each_class(class) {
			if (class->rq_online)
				class->rq_online(rq);
		}
	}
}

static void set_rq_offline(struct rq *rq)
{
	if (rq->online) {
		const struct sched_class *class;

		for_each_class(class) {
			if (class->rq_offline)
				class->rq_offline(rq);
		}

5197
		cpumask_clear_cpu(rq->cpu, rq->rd->online);
5198 5199 5200 5201
		rq->online = 0;
	}
}

L
Linus Torvalds 已提交
5202 5203 5204 5205
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
5206 5207
static int __cpuinit
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
5208
{
5209
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
5210
	unsigned long flags;
5211
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
5212

5213
	switch (action & ~CPU_TASKS_FROZEN) {
5214

L
Linus Torvalds 已提交
5215
	case CPU_UP_PREPARE:
5216
		rq->calc_load_update = calc_load_update;
L
Linus Torvalds 已提交
5217
		break;
5218

L
Linus Torvalds 已提交
5219
	case CPU_ONLINE:
5220
		/* Update our root-domain */
5221
		raw_spin_lock_irqsave(&rq->lock, flags);
5222
		if (rq->rd) {
5223
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
5224 5225

			set_rq_online(rq);
5226
		}
5227
		raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
5228
		break;
5229

L
Linus Torvalds 已提交
5230
#ifdef CONFIG_HOTPLUG_CPU
5231
	case CPU_DYING:
5232
		sched_ttwu_pending();
G
Gregory Haskins 已提交
5233
		/* Update our root-domain */
5234
		raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
5235
		if (rq->rd) {
5236
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
5237
			set_rq_offline(rq);
G
Gregory Haskins 已提交
5238
		}
5239 5240
		migrate_tasks(cpu);
		BUG_ON(rq->nr_running != 1); /* the migration thread */
5241
		raw_spin_unlock_irqrestore(&rq->lock, flags);
5242
		break;
5243

5244
	case CPU_DEAD:
5245
		calc_load_migrate(rq);
G
Gregory Haskins 已提交
5246
		break;
L
Linus Torvalds 已提交
5247 5248
#endif
	}
5249 5250 5251

	update_max_interval();

L
Linus Torvalds 已提交
5252 5253 5254
	return NOTIFY_OK;
}

5255 5256 5257
/*
 * Register at high priority so that task migration (migrate_all_tasks)
 * happens before everything else.  This has to be lower priority than
5258
 * the notifier in the perf_event subsystem, though.
L
Linus Torvalds 已提交
5259
 */
5260
static struct notifier_block __cpuinitdata migration_notifier = {
L
Linus Torvalds 已提交
5261
	.notifier_call = migration_call,
5262
	.priority = CPU_PRI_MIGRATION,
L
Linus Torvalds 已提交
5263 5264
};

5265 5266 5267 5268
static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
				      unsigned long action, void *hcpu)
{
	switch (action & ~CPU_TASKS_FROZEN) {
5269
	case CPU_STARTING:
5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289
	case CPU_DOWN_FAILED:
		set_cpu_active((long)hcpu, true);
		return NOTIFY_OK;
	default:
		return NOTIFY_DONE;
	}
}

static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb,
					unsigned long action, void *hcpu)
{
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_DOWN_PREPARE:
		set_cpu_active((long)hcpu, false);
		return NOTIFY_OK;
	default:
		return NOTIFY_DONE;
	}
}

5290
static int __init migration_init(void)
L
Linus Torvalds 已提交
5291 5292
{
	void *cpu = (void *)(long)smp_processor_id();
5293
	int err;
5294

5295
	/* Initialize migration for the boot CPU */
5296 5297
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
5298 5299
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
5300

5301 5302 5303 5304
	/* Register cpu active notifiers */
	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);

5305
	return 0;
L
Linus Torvalds 已提交
5306
}
5307
early_initcall(migration_init);
L
Linus Torvalds 已提交
5308 5309 5310
#endif

#ifdef CONFIG_SMP
5311

5312 5313
static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */

5314
#ifdef CONFIG_SCHED_DEBUG
I
Ingo Molnar 已提交
5315

5316
static __read_mostly int sched_debug_enabled;
5317

5318
static int __init sched_debug_setup(char *str)
5319
{
5320
	sched_debug_enabled = 1;
5321 5322 5323

	return 0;
}
5324 5325 5326 5327 5328 5329
early_param("sched_debug", sched_debug_setup);

static inline bool sched_debug(void)
{
	return sched_debug_enabled;
}
5330

5331
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5332
				  struct cpumask *groupmask)
L
Linus Torvalds 已提交
5333
{
I
Ingo Molnar 已提交
5334
	struct sched_group *group = sd->groups;
5335
	char str[256];
L
Linus Torvalds 已提交
5336

R
Rusty Russell 已提交
5337
	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
5338
	cpumask_clear(groupmask);
I
Ingo Molnar 已提交
5339 5340 5341 5342

	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);

	if (!(sd->flags & SD_LOAD_BALANCE)) {
P
Peter Zijlstra 已提交
5343
		printk("does not load-balance\n");
I
Ingo Molnar 已提交
5344
		if (sd->parent)
P
Peter Zijlstra 已提交
5345 5346
			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
					" has parent");
I
Ingo Molnar 已提交
5347
		return -1;
N
Nick Piggin 已提交
5348 5349
	}

P
Peter Zijlstra 已提交
5350
	printk(KERN_CONT "span %s level %s\n", str, sd->name);
I
Ingo Molnar 已提交
5351

5352
	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
P
Peter Zijlstra 已提交
5353 5354
		printk(KERN_ERR "ERROR: domain->span does not contain "
				"CPU%d\n", cpu);
I
Ingo Molnar 已提交
5355
	}
5356
	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5357 5358
		printk(KERN_ERR "ERROR: domain->groups does not contain"
				" CPU%d\n", cpu);
I
Ingo Molnar 已提交
5359
	}
L
Linus Torvalds 已提交
5360

I
Ingo Molnar 已提交
5361
	printk(KERN_DEBUG "%*s groups:", level + 1, "");
L
Linus Torvalds 已提交
5362
	do {
I
Ingo Molnar 已提交
5363
		if (!group) {
P
Peter Zijlstra 已提交
5364 5365
			printk("\n");
			printk(KERN_ERR "ERROR: group is NULL\n");
L
Linus Torvalds 已提交
5366 5367 5368
			break;
		}

5369 5370 5371 5372 5373 5374
		/*
		 * Even though we initialize ->power to something semi-sane,
		 * we leave power_orig unset. This allows us to detect if
		 * domain iteration is still funny without causing /0 traps.
		 */
		if (!group->sgp->power_orig) {
P
Peter Zijlstra 已提交
5375 5376 5377
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: domain->cpu_power not "
					"set\n");
I
Ingo Molnar 已提交
5378 5379
			break;
		}
L
Linus Torvalds 已提交
5380

5381
		if (!cpumask_weight(sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5382 5383
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: empty group\n");
I
Ingo Molnar 已提交
5384 5385
			break;
		}
L
Linus Torvalds 已提交
5386

5387 5388
		if (!(sd->flags & SD_OVERLAP) &&
		    cpumask_intersects(groupmask, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5389 5390
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: repeated CPUs\n");
I
Ingo Molnar 已提交
5391 5392
			break;
		}
L
Linus Torvalds 已提交
5393

5394
		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
L
Linus Torvalds 已提交
5395

R
Rusty Russell 已提交
5396
		cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
5397

P
Peter Zijlstra 已提交
5398
		printk(KERN_CONT " %s", str);
5399
		if (group->sgp->power != SCHED_POWER_SCALE) {
P
Peter Zijlstra 已提交
5400
			printk(KERN_CONT " (cpu_power = %d)",
5401
				group->sgp->power);
5402
		}
L
Linus Torvalds 已提交
5403

I
Ingo Molnar 已提交
5404 5405
		group = group->next;
	} while (group != sd->groups);
P
Peter Zijlstra 已提交
5406
	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
5407

5408
	if (!cpumask_equal(sched_domain_span(sd), groupmask))
P
Peter Zijlstra 已提交
5409
		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
L
Linus Torvalds 已提交
5410

5411 5412
	if (sd->parent &&
	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
P
Peter Zijlstra 已提交
5413 5414
		printk(KERN_ERR "ERROR: parent span is not a superset "
			"of domain->span\n");
I
Ingo Molnar 已提交
5415 5416
	return 0;
}
L
Linus Torvalds 已提交
5417

I
Ingo Molnar 已提交
5418 5419 5420
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;
L
Linus Torvalds 已提交
5421

5422
	if (!sched_debug_enabled)
5423 5424
		return;

I
Ingo Molnar 已提交
5425 5426 5427 5428
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}
L
Linus Torvalds 已提交
5429

I
Ingo Molnar 已提交
5430 5431 5432
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	for (;;) {
5433
		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
I
Ingo Molnar 已提交
5434
			break;
L
Linus Torvalds 已提交
5435 5436
		level++;
		sd = sd->parent;
5437
		if (!sd)
I
Ingo Molnar 已提交
5438 5439
			break;
	}
L
Linus Torvalds 已提交
5440
}
5441
#else /* !CONFIG_SCHED_DEBUG */
5442
# define sched_domain_debug(sd, cpu) do { } while (0)
5443 5444 5445 5446
static inline bool sched_debug(void)
{
	return false;
}
5447
#endif /* CONFIG_SCHED_DEBUG */
L
Linus Torvalds 已提交
5448

5449
static int sd_degenerate(struct sched_domain *sd)
5450
{
5451
	if (cpumask_weight(sched_domain_span(sd)) == 1)
5452 5453 5454 5455 5456 5457
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
5458 5459 5460
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
5461 5462 5463 5464 5465
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
5466
	if (sd->flags & (SD_WAKE_AFFINE))
5467 5468 5469 5470 5471
		return 0;

	return 1;
}

5472 5473
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5474 5475 5476 5477 5478 5479
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

5480
	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
5481 5482 5483 5484 5485 5486 5487
		return 0;

	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
5488 5489 5490
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
				SD_SHARE_PKG_RESOURCES);
5491 5492
		if (nr_node_ids == 1)
			pflags &= ~SD_SERIALIZE;
5493 5494 5495 5496 5497 5498 5499
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

5500
static void free_rootdomain(struct rcu_head *rcu)
5501
{
5502
	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
5503

5504
	cpupri_cleanup(&rd->cpupri);
5505 5506 5507 5508 5509 5510
	free_cpumask_var(rd->rto_mask);
	free_cpumask_var(rd->online);
	free_cpumask_var(rd->span);
	kfree(rd);
}

G
Gregory Haskins 已提交
5511 5512
static void rq_attach_root(struct rq *rq, struct root_domain *rd)
{
I
Ingo Molnar 已提交
5513
	struct root_domain *old_rd = NULL;
G
Gregory Haskins 已提交
5514 5515
	unsigned long flags;

5516
	raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
5517 5518

	if (rq->rd) {
I
Ingo Molnar 已提交
5519
		old_rd = rq->rd;
G
Gregory Haskins 已提交
5520

5521
		if (cpumask_test_cpu(rq->cpu, old_rd->online))
5522
			set_rq_offline(rq);
G
Gregory Haskins 已提交
5523

5524
		cpumask_clear_cpu(rq->cpu, old_rd->span);
5525

I
Ingo Molnar 已提交
5526 5527 5528 5529 5530 5531 5532
		/*
		 * If we dont want to free the old_rt yet then
		 * set old_rd to NULL to skip the freeing later
		 * in this function:
		 */
		if (!atomic_dec_and_test(&old_rd->refcount))
			old_rd = NULL;
G
Gregory Haskins 已提交
5533 5534 5535 5536 5537
	}

	atomic_inc(&rd->refcount);
	rq->rd = rd;

5538
	cpumask_set_cpu(rq->cpu, rd->span);
5539
	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
5540
		set_rq_online(rq);
G
Gregory Haskins 已提交
5541

5542
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
5543 5544

	if (old_rd)
5545
		call_rcu_sched(&old_rd->rcu, free_rootdomain);
G
Gregory Haskins 已提交
5546 5547
}

5548
static int init_rootdomain(struct root_domain *rd)
G
Gregory Haskins 已提交
5549 5550 5551
{
	memset(rd, 0, sizeof(*rd));

5552
	if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
5553
		goto out;
5554
	if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
5555
		goto free_span;
5556
	if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
5557
		goto free_online;
5558

5559
	if (cpupri_init(&rd->cpupri) != 0)
5560
		goto free_rto_mask;
5561
	return 0;
5562

5563 5564
free_rto_mask:
	free_cpumask_var(rd->rto_mask);
5565 5566 5567 5568
free_online:
	free_cpumask_var(rd->online);
free_span:
	free_cpumask_var(rd->span);
5569
out:
5570
	return -ENOMEM;
G
Gregory Haskins 已提交
5571 5572
}

5573 5574 5575 5576 5577 5578
/*
 * By default the system creates a single root-domain with all cpus as
 * members (mimicking the global state we have today).
 */
struct root_domain def_root_domain;

G
Gregory Haskins 已提交
5579 5580
static void init_defrootdomain(void)
{
5581
	init_rootdomain(&def_root_domain);
5582

G
Gregory Haskins 已提交
5583 5584 5585
	atomic_set(&def_root_domain.refcount, 1);
}

5586
static struct root_domain *alloc_rootdomain(void)
G
Gregory Haskins 已提交
5587 5588 5589 5590 5591 5592 5593
{
	struct root_domain *rd;

	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
	if (!rd)
		return NULL;

5594
	if (init_rootdomain(rd) != 0) {
5595 5596 5597
		kfree(rd);
		return NULL;
	}
G
Gregory Haskins 已提交
5598 5599 5600 5601

	return rd;
}

5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620
static void free_sched_groups(struct sched_group *sg, int free_sgp)
{
	struct sched_group *tmp, *first;

	if (!sg)
		return;

	first = sg;
	do {
		tmp = sg->next;

		if (free_sgp && atomic_dec_and_test(&sg->sgp->ref))
			kfree(sg->sgp);

		kfree(sg);
		sg = tmp;
	} while (sg != first);
}

5621 5622 5623
static void free_sched_domain(struct rcu_head *rcu)
{
	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
5624 5625 5626 5627 5628 5629 5630 5631

	/*
	 * If its an overlapping domain it has private groups, iterate and
	 * nuke them all.
	 */
	if (sd->flags & SD_OVERLAP) {
		free_sched_groups(sd->groups, 1);
	} else if (atomic_dec_and_test(&sd->groups->ref)) {
5632
		kfree(sd->groups->sgp);
5633
		kfree(sd->groups);
5634
	}
5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648
	kfree(sd);
}

static void destroy_sched_domain(struct sched_domain *sd, int cpu)
{
	call_rcu(&sd->rcu, free_sched_domain);
}

static void destroy_sched_domains(struct sched_domain *sd, int cpu)
{
	for (; sd; sd = sd->parent)
		destroy_sched_domain(sd, cpu);
}

5649 5650 5651 5652 5653 5654 5655
/*
 * Keep a special pointer to the highest sched_domain that has
 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
 * allows us to avoid some pointer chasing select_idle_sibling().
 *
 * Also keep a unique ID per domain (we use the first cpu number in
 * the cpumask of the domain), this allows us to quickly tell if
5656
 * two cpus are in the same cache domain, see cpus_share_cache().
5657 5658 5659 5660 5661 5662 5663 5664 5665 5666
 */
DEFINE_PER_CPU(struct sched_domain *, sd_llc);
DEFINE_PER_CPU(int, sd_llc_id);

static void update_top_cache_domain(int cpu)
{
	struct sched_domain *sd;
	int id = cpu;

	sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
5667
	if (sd)
5668 5669 5670 5671 5672 5673
		id = cpumask_first(sched_domain_span(sd));

	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
	per_cpu(sd_llc_id, cpu) = id;
}

L
Linus Torvalds 已提交
5674
/*
I
Ingo Molnar 已提交
5675
 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
L
Linus Torvalds 已提交
5676 5677
 * hold the hotplug lock.
 */
I
Ingo Molnar 已提交
5678 5679
static void
cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
L
Linus Torvalds 已提交
5680
{
5681
	struct rq *rq = cpu_rq(cpu);
5682 5683 5684
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
5685
	for (tmp = sd; tmp; ) {
5686 5687 5688
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
5689

5690
		if (sd_parent_degenerate(tmp, parent)) {
5691
			tmp->parent = parent->parent;
5692 5693
			if (parent->parent)
				parent->parent->child = tmp;
5694
			destroy_sched_domain(parent, cpu);
5695 5696
		} else
			tmp = tmp->parent;
5697 5698
	}

5699
	if (sd && sd_degenerate(sd)) {
5700
		tmp = sd;
5701
		sd = sd->parent;
5702
		destroy_sched_domain(tmp, cpu);
5703 5704 5705
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
5706

5707
	sched_domain_debug(sd, cpu);
L
Linus Torvalds 已提交
5708

G
Gregory Haskins 已提交
5709
	rq_attach_root(rq, rd);
5710
	tmp = rq->sd;
N
Nick Piggin 已提交
5711
	rcu_assign_pointer(rq->sd, sd);
5712
	destroy_sched_domains(tmp, cpu);
5713 5714

	update_top_cache_domain(cpu);
L
Linus Torvalds 已提交
5715 5716 5717
}

/* cpus with isolated domains */
5718
static cpumask_var_t cpu_isolated_map;
L
Linus Torvalds 已提交
5719 5720 5721 5722

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
R
Rusty Russell 已提交
5723
	alloc_bootmem_cpumask_var(&cpu_isolated_map);
R
Rusty Russell 已提交
5724
	cpulist_parse(str, cpu_isolated_map);
L
Linus Torvalds 已提交
5725 5726 5727
	return 1;
}

I
Ingo Molnar 已提交
5728
__setup("isolcpus=", isolated_cpu_setup);
L
Linus Torvalds 已提交
5729

5730 5731 5732 5733 5734
static const struct cpumask *cpu_cpu_mask(int cpu)
{
	return cpumask_of_node(cpu_to_node(cpu));
}

5735 5736 5737
struct sd_data {
	struct sched_domain **__percpu sd;
	struct sched_group **__percpu sg;
5738
	struct sched_group_power **__percpu sgp;
5739 5740
};

5741
struct s_data {
5742
	struct sched_domain ** __percpu sd;
5743 5744 5745
	struct root_domain	*rd;
};

5746 5747
enum s_alloc {
	sa_rootdomain,
5748
	sa_sd,
5749
	sa_sd_storage,
5750 5751 5752
	sa_none,
};

5753 5754 5755
struct sched_domain_topology_level;

typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu);
5756 5757
typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);

5758 5759
#define SDTL_OVERLAP	0x01

5760
struct sched_domain_topology_level {
5761 5762
	sched_domain_init_f init;
	sched_domain_mask_f mask;
5763
	int		    flags;
5764
	int		    numa_level;
5765
	struct sd_data      data;
5766 5767
};

P
Peter Zijlstra 已提交
5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805
/*
 * Build an iteration mask that can exclude certain CPUs from the upwards
 * domain traversal.
 *
 * Asymmetric node setups can result in situations where the domain tree is of
 * unequal depth, make sure to skip domains that already cover the entire
 * range.
 *
 * In that case build_sched_domains() will have terminated the iteration early
 * and our sibling sd spans will be empty. Domains should always include the
 * cpu they're built on, so check that.
 *
 */
static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
{
	const struct cpumask *span = sched_domain_span(sd);
	struct sd_data *sdd = sd->private;
	struct sched_domain *sibling;
	int i;

	for_each_cpu(i, span) {
		sibling = *per_cpu_ptr(sdd->sd, i);
		if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
			continue;

		cpumask_set_cpu(i, sched_group_mask(sg));
	}
}

/*
 * Return the canonical balance cpu for this group, this is the first cpu
 * of this group that's also in the iteration mask.
 */
int group_balance_cpu(struct sched_group *sg)
{
	return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
}

5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823
static int
build_overlap_sched_groups(struct sched_domain *sd, int cpu)
{
	struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
	const struct cpumask *span = sched_domain_span(sd);
	struct cpumask *covered = sched_domains_tmpmask;
	struct sd_data *sdd = sd->private;
	struct sched_domain *child;
	int i;

	cpumask_clear(covered);

	for_each_cpu(i, span) {
		struct cpumask *sg_span;

		if (cpumask_test_cpu(i, covered))
			continue;

P
Peter Zijlstra 已提交
5824 5825 5826 5827 5828 5829
		child = *per_cpu_ptr(sdd->sd, i);

		/* See the comment near build_group_mask(). */
		if (!cpumask_test_cpu(i, sched_domain_span(child)))
			continue;

5830
		sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
5831
				GFP_KERNEL, cpu_to_node(cpu));
5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844

		if (!sg)
			goto fail;

		sg_span = sched_group_cpus(sg);
		if (child->child) {
			child = child->child;
			cpumask_copy(sg_span, sched_domain_span(child));
		} else
			cpumask_set_cpu(i, sg_span);

		cpumask_or(covered, covered, sg_span);

P
Peter Zijlstra 已提交
5845
		sg->sgp = *per_cpu_ptr(sdd->sgp, i);
P
Peter Zijlstra 已提交
5846 5847 5848
		if (atomic_inc_return(&sg->sgp->ref) == 1)
			build_group_mask(sd, sg);

5849 5850 5851 5852 5853 5854
		/*
		 * Initialize sgp->power such that even if we mess up the
		 * domains and no possible iteration will get us here, we won't
		 * die on a /0 trap.
		 */
		sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span);
5855

P
Peter Zijlstra 已提交
5856 5857 5858 5859 5860
		/*
		 * Make sure the first group of this domain contains the
		 * canonical balance cpu. Otherwise the sched_domain iteration
		 * breaks. See update_sg_lb_stats().
		 */
P
Peter Zijlstra 已提交
5861
		if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
P
Peter Zijlstra 已提交
5862
		    group_balance_cpu(sg) == cpu)
5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881
			groups = sg;

		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
		last->next = first;
	}
	sd->groups = groups;

	return 0;

fail:
	free_sched_groups(first, 0);

	return -ENOMEM;
}

5882
static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
L
Linus Torvalds 已提交
5883
{
5884 5885
	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
	struct sched_domain *child = sd->child;
L
Linus Torvalds 已提交
5886

5887 5888
	if (child)
		cpu = cpumask_first(sched_domain_span(child));
5889

5890
	if (sg) {
5891
		*sg = *per_cpu_ptr(sdd->sg, cpu);
5892
		(*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu);
5893
		atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */
5894
	}
5895 5896

	return cpu;
5897 5898
}

5899
/*
5900 5901 5902
 * build_sched_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
5903 5904
 *
 * Assumes the sched_domain tree is fully constructed
5905
 */
5906 5907
static int
build_sched_groups(struct sched_domain *sd, int cpu)
L
Linus Torvalds 已提交
5908
{
5909 5910 5911
	struct sched_group *first = NULL, *last = NULL;
	struct sd_data *sdd = sd->private;
	const struct cpumask *span = sched_domain_span(sd);
5912
	struct cpumask *covered;
5913
	int i;
5914

5915 5916 5917 5918 5919 5920
	get_group(cpu, sdd, &sd->groups);
	atomic_inc(&sd->groups->ref);

	if (cpu != cpumask_first(sched_domain_span(sd)))
		return 0;

5921 5922 5923
	lockdep_assert_held(&sched_domains_mutex);
	covered = sched_domains_tmpmask;

5924
	cpumask_clear(covered);
5925

5926 5927 5928 5929
	for_each_cpu(i, span) {
		struct sched_group *sg;
		int group = get_group(i, sdd, &sg);
		int j;
5930

5931 5932
		if (cpumask_test_cpu(i, covered))
			continue;
5933

5934
		cpumask_clear(sched_group_cpus(sg));
5935
		sg->sgp->power = 0;
P
Peter Zijlstra 已提交
5936
		cpumask_setall(sched_group_mask(sg));
5937

5938 5939 5940
		for_each_cpu(j, span) {
			if (get_group(j, sdd, NULL) != group)
				continue;
5941

5942 5943 5944
			cpumask_set_cpu(j, covered);
			cpumask_set_cpu(j, sched_group_cpus(sg));
		}
5945

5946 5947 5948 5949 5950 5951 5952
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
5953 5954

	return 0;
5955
}
5956

5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
5969
	struct sched_group *sg = sd->groups;
5970

5971 5972 5973 5974 5975 5976
	WARN_ON(!sd || !sg);

	do {
		sg->group_weight = cpumask_weight(sched_group_cpus(sg));
		sg = sg->next;
	} while (sg != sd->groups);
5977

P
Peter Zijlstra 已提交
5978
	if (cpu != group_balance_cpu(sg))
5979
		return;
5980

5981
	update_group_power(sd, cpu);
5982
	atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight);
5983 5984
}

5985 5986 5987
int __weak arch_sd_sibling_asym_packing(void)
{
       return 0*SD_ASYM_PACKING;
5988 5989
}

5990 5991 5992 5993 5994
/*
 * Initializers for schedule domains
 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
 */

5995 5996 5997 5998 5999 6000
#ifdef CONFIG_SCHED_DEBUG
# define SD_INIT_NAME(sd, type)		sd->name = #type
#else
# define SD_INIT_NAME(sd, type)		do { } while (0)
#endif

6001 6002 6003 6004 6005 6006 6007 6008 6009
#define SD_INIT_FUNC(type)						\
static noinline struct sched_domain *					\
sd_init_##type(struct sched_domain_topology_level *tl, int cpu) 	\
{									\
	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);	\
	*sd = SD_##type##_INIT;						\
	SD_INIT_NAME(sd, type);						\
	sd->private = &tl->data;					\
	return sd;							\
6010 6011 6012 6013 6014 6015 6016 6017 6018
}

SD_INIT_FUNC(CPU)
#ifdef CONFIG_SCHED_SMT
 SD_INIT_FUNC(SIBLING)
#endif
#ifdef CONFIG_SCHED_MC
 SD_INIT_FUNC(MC)
#endif
6019 6020 6021
#ifdef CONFIG_SCHED_BOOK
 SD_INIT_FUNC(BOOK)
#endif
6022

6023
static int default_relax_domain_level = -1;
6024
int sched_domain_level_max;
6025 6026 6027

static int __init setup_relax_domain_level(char *str)
{
6028 6029
	if (kstrtoint(str, 0, &default_relax_domain_level))
		pr_warn("Unable to set relax_domain_level\n");
6030

6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048
	return 1;
}
__setup("relax_domain_level=", setup_relax_domain_level);

static void set_domain_attribute(struct sched_domain *sd,
				 struct sched_domain_attr *attr)
{
	int request;

	if (!attr || attr->relax_domain_level < 0) {
		if (default_relax_domain_level < 0)
			return;
		else
			request = default_relax_domain_level;
	} else
		request = attr->relax_domain_level;
	if (request < sd->level) {
		/* turn off idle balance on this domain */
6049
		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
6050 6051
	} else {
		/* turn on idle balance on this domain */
6052
		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
6053 6054 6055
	}
}

6056 6057 6058
static void __sdt_free(const struct cpumask *cpu_map);
static int __sdt_alloc(const struct cpumask *cpu_map);

6059 6060 6061 6062 6063
static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
				 const struct cpumask *cpu_map)
{
	switch (what) {
	case sa_rootdomain:
6064 6065
		if (!atomic_read(&d->rd->refcount))
			free_rootdomain(&d->rd->rcu); /* fall through */
6066 6067
	case sa_sd:
		free_percpu(d->sd); /* fall through */
6068
	case sa_sd_storage:
6069
		__sdt_free(cpu_map); /* fall through */
6070 6071 6072 6073
	case sa_none:
		break;
	}
}
6074

6075 6076 6077
static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
						   const struct cpumask *cpu_map)
{
6078 6079
	memset(d, 0, sizeof(*d));

6080 6081
	if (__sdt_alloc(cpu_map))
		return sa_sd_storage;
6082 6083 6084
	d->sd = alloc_percpu(struct sched_domain *);
	if (!d->sd)
		return sa_sd_storage;
6085
	d->rd = alloc_rootdomain();
6086
	if (!d->rd)
6087
		return sa_sd;
6088 6089
	return sa_rootdomain;
}
G
Gregory Haskins 已提交
6090

6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102
/*
 * NULL the sd_data elements we've used to build the sched_domain and
 * sched_group structure so that the subsequent __free_domain_allocs()
 * will not free the data we're using.
 */
static void claim_allocations(int cpu, struct sched_domain *sd)
{
	struct sd_data *sdd = sd->private;

	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
	*per_cpu_ptr(sdd->sd, cpu) = NULL;

6103
	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
6104
		*per_cpu_ptr(sdd->sg, cpu) = NULL;
6105 6106

	if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref))
6107
		*per_cpu_ptr(sdd->sgp, cpu) = NULL;
6108 6109
}

6110 6111
#ifdef CONFIG_SCHED_SMT
static const struct cpumask *cpu_smt_mask(int cpu)
6112
{
6113
	return topology_thread_cpumask(cpu);
6114
}
6115
#endif
6116

6117 6118 6119
/*
 * Topology list, bottom-up.
 */
6120
static struct sched_domain_topology_level default_topology[] = {
6121 6122
#ifdef CONFIG_SCHED_SMT
	{ sd_init_SIBLING, cpu_smt_mask, },
6123
#endif
6124
#ifdef CONFIG_SCHED_MC
6125
	{ sd_init_MC, cpu_coregroup_mask, },
6126
#endif
6127 6128 6129 6130
#ifdef CONFIG_SCHED_BOOK
	{ sd_init_BOOK, cpu_book_mask, },
#endif
	{ sd_init_CPU, cpu_cpu_mask, },
6131 6132 6133 6134 6135
	{ NULL, },
};

static struct sched_domain_topology_level *sched_domain_topology = default_topology;

6136 6137 6138 6139 6140 6141 6142 6143 6144
#ifdef CONFIG_NUMA

static int sched_domains_numa_levels;
static int *sched_domains_numa_distance;
static struct cpumask ***sched_domains_numa_masks;
static int sched_domains_curr_level;

static inline int sd_local_flags(int level)
{
6145
	if (sched_domains_numa_distance[level] > RECLAIM_DISTANCE)
6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162
		return 0;

	return SD_BALANCE_EXEC | SD_BALANCE_FORK | SD_WAKE_AFFINE;
}

static struct sched_domain *
sd_numa_init(struct sched_domain_topology_level *tl, int cpu)
{
	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
	int level = tl->numa_level;
	int sd_weight = cpumask_weight(
			sched_domains_numa_masks[level][cpu_to_node(cpu)]);

	*sd = (struct sched_domain){
		.min_interval		= sd_weight,
		.max_interval		= 2*sd_weight,
		.busy_factor		= 32,
6163
		.imbalance_pct		= 125,
6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201
		.cache_nice_tries	= 2,
		.busy_idx		= 3,
		.idle_idx		= 2,
		.newidle_idx		= 0,
		.wake_idx		= 0,
		.forkexec_idx		= 0,

		.flags			= 1*SD_LOAD_BALANCE
					| 1*SD_BALANCE_NEWIDLE
					| 0*SD_BALANCE_EXEC
					| 0*SD_BALANCE_FORK
					| 0*SD_BALANCE_WAKE
					| 0*SD_WAKE_AFFINE
					| 0*SD_SHARE_CPUPOWER
					| 0*SD_SHARE_PKG_RESOURCES
					| 1*SD_SERIALIZE
					| 0*SD_PREFER_SIBLING
					| sd_local_flags(level)
					,
		.last_balance		= jiffies,
		.balance_interval	= sd_weight,
	};
	SD_INIT_NAME(sd, NUMA);
	sd->private = &tl->data;

	/*
	 * Ugly hack to pass state to sd_numa_mask()...
	 */
	sched_domains_curr_level = tl->numa_level;

	return sd;
}

static const struct cpumask *sd_numa_mask(int cpu)
{
	return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
}

6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237
static void sched_numa_warn(const char *str)
{
	static int done = false;
	int i,j;

	if (done)
		return;

	done = true;

	printk(KERN_WARNING "ERROR: %s\n\n", str);

	for (i = 0; i < nr_node_ids; i++) {
		printk(KERN_WARNING "  ");
		for (j = 0; j < nr_node_ids; j++)
			printk(KERN_CONT "%02d ", node_distance(i,j));
		printk(KERN_CONT "\n");
	}
	printk(KERN_WARNING "\n");
}

static bool find_numa_distance(int distance)
{
	int i;

	if (distance == node_distance(0, 0))
		return true;

	for (i = 0; i < sched_domains_numa_levels; i++) {
		if (sched_domains_numa_distance[i] == distance)
			return true;
	}

	return false;
}

6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258
static void sched_init_numa(void)
{
	int next_distance, curr_distance = node_distance(0, 0);
	struct sched_domain_topology_level *tl;
	int level = 0;
	int i, j, k;

	sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
	if (!sched_domains_numa_distance)
		return;

	/*
	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
	 * unique distances in the node_distance() table.
	 *
	 * Assumes node_distance(0,j) includes all distances in
	 * node_distance(i,j) in order to avoid cubic time.
	 */
	next_distance = curr_distance;
	for (i = 0; i < nr_node_ids; i++) {
		for (j = 0; j < nr_node_ids; j++) {
6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282
			for (k = 0; k < nr_node_ids; k++) {
				int distance = node_distance(i, k);

				if (distance > curr_distance &&
				    (distance < next_distance ||
				     next_distance == curr_distance))
					next_distance = distance;

				/*
				 * While not a strong assumption it would be nice to know
				 * about cases where if node A is connected to B, B is not
				 * equally connected to A.
				 */
				if (sched_debug() && node_distance(k, i) != distance)
					sched_numa_warn("Node-distance not symmetric");

				if (sched_debug() && i && !find_numa_distance(distance))
					sched_numa_warn("Node-0 not representative");
			}
			if (next_distance != curr_distance) {
				sched_domains_numa_distance[level++] = next_distance;
				sched_domains_numa_levels = level;
				curr_distance = next_distance;
			} else break;
6283
		}
6284 6285 6286 6287 6288 6289

		/*
		 * In case of sched_debug() we verify the above assumption.
		 */
		if (!sched_debug())
			break;
6290 6291 6292 6293 6294 6295 6296 6297 6298
	}
	/*
	 * 'level' contains the number of unique distances, excluding the
	 * identity distance node_distance(i,i).
	 *
	 * The sched_domains_nume_distance[] array includes the actual distance
	 * numbers.
	 */

6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309
	/*
	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
	 * the array will contain less then 'level' members. This could be
	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
	 * in other functions.
	 *
	 * We reset it to 'level' at the end of this function.
	 */
	sched_domains_numa_levels = 0;

6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324
	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
	if (!sched_domains_numa_masks)
		return;

	/*
	 * Now for each level, construct a mask per node which contains all
	 * cpus of nodes that are that many hops away from us.
	 */
	for (i = 0; i < level; i++) {
		sched_domains_numa_masks[i] =
			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
		if (!sched_domains_numa_masks[i])
			return;

		for (j = 0; j < nr_node_ids; j++) {
6325
			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
6326 6327 6328 6329 6330 6331
			if (!mask)
				return;

			sched_domains_numa_masks[i][j] = mask;

			for (k = 0; k < nr_node_ids; k++) {
6332
				if (node_distance(j, k) > sched_domains_numa_distance[i])
6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363
					continue;

				cpumask_or(mask, mask, cpumask_of_node(k));
			}
		}
	}

	tl = kzalloc((ARRAY_SIZE(default_topology) + level) *
			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
	if (!tl)
		return;

	/*
	 * Copy the default topology bits..
	 */
	for (i = 0; default_topology[i].init; i++)
		tl[i] = default_topology[i];

	/*
	 * .. and append 'j' levels of NUMA goodness.
	 */
	for (j = 0; j < level; i++, j++) {
		tl[i] = (struct sched_domain_topology_level){
			.init = sd_numa_init,
			.mask = sd_numa_mask,
			.flags = SDTL_OVERLAP,
			.numa_level = j,
		};
	}

	sched_domain_topology = tl;
6364 6365

	sched_domains_numa_levels = level;
6366
}
6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413

static void sched_domains_numa_masks_set(int cpu)
{
	int i, j;
	int node = cpu_to_node(cpu);

	for (i = 0; i < sched_domains_numa_levels; i++) {
		for (j = 0; j < nr_node_ids; j++) {
			if (node_distance(j, node) <= sched_domains_numa_distance[i])
				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
		}
	}
}

static void sched_domains_numa_masks_clear(int cpu)
{
	int i, j;
	for (i = 0; i < sched_domains_numa_levels; i++) {
		for (j = 0; j < nr_node_ids; j++)
			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
	}
}

/*
 * Update sched_domains_numa_masks[level][node] array when new cpus
 * are onlined.
 */
static int sched_domains_numa_masks_update(struct notifier_block *nfb,
					   unsigned long action,
					   void *hcpu)
{
	int cpu = (long)hcpu;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_ONLINE:
		sched_domains_numa_masks_set(cpu);
		break;

	case CPU_DEAD:
		sched_domains_numa_masks_clear(cpu);
		break;

	default:
		return NOTIFY_DONE;
	}

	return NOTIFY_OK;
6414 6415 6416 6417 6418
}
#else
static inline void sched_init_numa(void)
{
}
6419 6420 6421 6422 6423 6424 6425

static int sched_domains_numa_masks_update(struct notifier_block *nfb,
					   unsigned long action,
					   void *hcpu)
{
	return 0;
}
6426 6427
#endif /* CONFIG_NUMA */

6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443
static int __sdt_alloc(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

	for (tl = sched_domain_topology; tl->init; tl++) {
		struct sd_data *sdd = &tl->data;

		sdd->sd = alloc_percpu(struct sched_domain *);
		if (!sdd->sd)
			return -ENOMEM;

		sdd->sg = alloc_percpu(struct sched_group *);
		if (!sdd->sg)
			return -ENOMEM;

6444 6445 6446 6447
		sdd->sgp = alloc_percpu(struct sched_group_power *);
		if (!sdd->sgp)
			return -ENOMEM;

6448 6449 6450
		for_each_cpu(j, cpu_map) {
			struct sched_domain *sd;
			struct sched_group *sg;
6451
			struct sched_group_power *sgp;
6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464

		       	sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sd)
				return -ENOMEM;

			*per_cpu_ptr(sdd->sd, j) = sd;

			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sg)
				return -ENOMEM;

6465 6466
			sg->next = sg;

6467
			*per_cpu_ptr(sdd->sg, j) = sg;
6468

P
Peter Zijlstra 已提交
6469
			sgp = kzalloc_node(sizeof(struct sched_group_power) + cpumask_size(),
6470 6471 6472 6473 6474
					GFP_KERNEL, cpu_to_node(j));
			if (!sgp)
				return -ENOMEM;

			*per_cpu_ptr(sdd->sgp, j) = sgp;
6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489
		}
	}

	return 0;
}

static void __sdt_free(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

	for (tl = sched_domain_topology; tl->init; tl++) {
		struct sd_data *sdd = &tl->data;

		for_each_cpu(j, cpu_map) {
6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502
			struct sched_domain *sd;

			if (sdd->sd) {
				sd = *per_cpu_ptr(sdd->sd, j);
				if (sd && (sd->flags & SD_OVERLAP))
					free_sched_groups(sd->groups, 0);
				kfree(*per_cpu_ptr(sdd->sd, j));
			}

			if (sdd->sg)
				kfree(*per_cpu_ptr(sdd->sg, j));
			if (sdd->sgp)
				kfree(*per_cpu_ptr(sdd->sgp, j));
6503 6504
		}
		free_percpu(sdd->sd);
6505
		sdd->sd = NULL;
6506
		free_percpu(sdd->sg);
6507
		sdd->sg = NULL;
6508
		free_percpu(sdd->sgp);
6509
		sdd->sgp = NULL;
6510 6511 6512
	}
}

6513 6514
struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
		struct s_data *d, const struct cpumask *cpu_map,
6515
		struct sched_domain_attr *attr, struct sched_domain *child,
6516 6517
		int cpu)
{
6518
	struct sched_domain *sd = tl->init(tl, cpu);
6519
	if (!sd)
6520
		return child;
6521 6522

	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
6523 6524 6525
	if (child) {
		sd->level = child->level + 1;
		sched_domain_level_max = max(sched_domain_level_max, sd->level);
6526
		child->parent = sd;
6527
	}
6528
	sd->child = child;
6529
	set_domain_attribute(sd, attr);
6530 6531 6532 6533

	return sd;
}

6534 6535 6536 6537
/*
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
 */
6538 6539
static int build_sched_domains(const struct cpumask *cpu_map,
			       struct sched_domain_attr *attr)
6540 6541
{
	enum s_alloc alloc_state = sa_none;
6542
	struct sched_domain *sd;
6543
	struct s_data d;
6544
	int i, ret = -ENOMEM;
6545

6546 6547 6548
	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
	if (alloc_state != sa_rootdomain)
		goto error;
6549

6550
	/* Set up domains for cpus specified by the cpu_map. */
6551
	for_each_cpu(i, cpu_map) {
6552 6553
		struct sched_domain_topology_level *tl;

6554
		sd = NULL;
6555
		for (tl = sched_domain_topology; tl->init; tl++) {
6556
			sd = build_sched_domain(tl, &d, cpu_map, attr, sd, i);
6557 6558
			if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
				sd->flags |= SD_OVERLAP;
6559 6560
			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
				break;
6561
		}
6562

6563 6564 6565
		while (sd->child)
			sd = sd->child;

6566
		*per_cpu_ptr(d.sd, i) = sd;
6567 6568 6569 6570 6571 6572
	}

	/* Build the groups for the domains */
	for_each_cpu(i, cpu_map) {
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			sd->span_weight = cpumask_weight(sched_domain_span(sd));
6573 6574 6575 6576 6577 6578 6579
			if (sd->flags & SD_OVERLAP) {
				if (build_overlap_sched_groups(sd, i))
					goto error;
			} else {
				if (build_sched_groups(sd, i))
					goto error;
			}
6580
		}
6581
	}
6582

L
Linus Torvalds 已提交
6583
	/* Calculate CPU power for physical packages and nodes */
6584 6585 6586
	for (i = nr_cpumask_bits-1; i >= 0; i--) {
		if (!cpumask_test_cpu(i, cpu_map))
			continue;
6587

6588 6589
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			claim_allocations(i, sd);
6590
			init_sched_groups_power(i, sd);
6591
		}
6592
	}
6593

L
Linus Torvalds 已提交
6594
	/* Attach the domains */
6595
	rcu_read_lock();
6596
	for_each_cpu(i, cpu_map) {
6597
		sd = *per_cpu_ptr(d.sd, i);
6598
		cpu_attach_domain(sd, d.rd, i);
L
Linus Torvalds 已提交
6599
	}
6600
	rcu_read_unlock();
6601

6602
	ret = 0;
6603
error:
6604
	__free_domain_allocs(&d, alloc_state, cpu_map);
6605
	return ret;
L
Linus Torvalds 已提交
6606
}
P
Paul Jackson 已提交
6607

6608
static cpumask_var_t *doms_cur;	/* current sched domains */
P
Paul Jackson 已提交
6609
static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
I
Ingo Molnar 已提交
6610 6611
static struct sched_domain_attr *dattr_cur;
				/* attribues of custom domains in 'doms_cur' */
P
Paul Jackson 已提交
6612 6613 6614

/*
 * Special case: If a kmalloc of a doms_cur partition (array of
6615 6616
 * cpumask) fails, then fallback to a single sched domain,
 * as determined by the single cpumask fallback_doms.
P
Paul Jackson 已提交
6617
 */
6618
static cpumask_var_t fallback_doms;
P
Paul Jackson 已提交
6619

6620 6621 6622 6623 6624 6625
/*
 * arch_update_cpu_topology lets virtualized architectures update the
 * cpu core maps. It is supposed to return 1 if the topology changed
 * or 0 if it stayed the same.
 */
int __attribute__((weak)) arch_update_cpu_topology(void)
6626
{
6627
	return 0;
6628 6629
}

6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654
cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
{
	int i;
	cpumask_var_t *doms;

	doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
	if (!doms)
		return NULL;
	for (i = 0; i < ndoms; i++) {
		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
			free_sched_domains(doms, i);
			return NULL;
		}
	}
	return doms;
}

void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
{
	unsigned int i;
	for (i = 0; i < ndoms; i++)
		free_cpumask_var(doms[i]);
	kfree(doms);
}

6655
/*
I
Ingo Molnar 已提交
6656
 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
P
Paul Jackson 已提交
6657 6658
 * For now this just excludes isolated cpus, but could be used to
 * exclude other special cases in the future.
6659
 */
6660
static int init_sched_domains(const struct cpumask *cpu_map)
6661
{
6662 6663
	int err;

6664
	arch_update_cpu_topology();
P
Paul Jackson 已提交
6665
	ndoms_cur = 1;
6666
	doms_cur = alloc_sched_domains(ndoms_cur);
P
Paul Jackson 已提交
6667
	if (!doms_cur)
6668 6669
		doms_cur = &fallback_doms;
	cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
6670
	err = build_sched_domains(doms_cur[0], NULL);
6671
	register_sched_domain_sysctl();
6672 6673

	return err;
6674 6675 6676 6677 6678 6679
}

/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
6680
static void detach_destroy_domains(const struct cpumask *cpu_map)
6681 6682 6683
{
	int i;

6684
	rcu_read_lock();
6685
	for_each_cpu(i, cpu_map)
G
Gregory Haskins 已提交
6686
		cpu_attach_domain(NULL, &def_root_domain, i);
6687
	rcu_read_unlock();
6688 6689
}

6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705
/* handle null as "default" */
static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
			struct sched_domain_attr *new, int idx_new)
{
	struct sched_domain_attr tmp;

	/* fast path */
	if (!new && !cur)
		return 1;

	tmp = SD_ATTR_INIT;
	return !memcmp(cur ? (cur + idx_cur) : &tmp,
			new ? (new + idx_new) : &tmp,
			sizeof(struct sched_domain_attr));
}

P
Paul Jackson 已提交
6706 6707
/*
 * Partition sched domains as specified by the 'ndoms_new'
I
Ingo Molnar 已提交
6708
 * cpumasks in the array doms_new[] of cpumasks. This compares
P
Paul Jackson 已提交
6709 6710 6711
 * doms_new[] to the current sched domain partitioning, doms_cur[].
 * It destroys each deleted domain and builds each new domain.
 *
6712
 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
I
Ingo Molnar 已提交
6713 6714 6715
 * The masks don't intersect (don't overlap.) We should setup one
 * sched domain for each mask. CPUs not in any of the cpumasks will
 * not be load balanced. If the same cpumask appears both in the
P
Paul Jackson 已提交
6716 6717 6718
 * current 'doms_cur' domains and in the new 'doms_new', we can leave
 * it as it is.
 *
6719 6720 6721 6722 6723 6724
 * The passed in 'doms_new' should be allocated using
 * alloc_sched_domains.  This routine takes ownership of it and will
 * free_sched_domains it when done with it. If the caller failed the
 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
 * and partition_sched_domains() will fallback to the single partition
 * 'fallback_doms', it also forces the domains to be rebuilt.
P
Paul Jackson 已提交
6725
 *
6726
 * If doms_new == NULL it will be replaced with cpu_online_mask.
6727 6728
 * ndoms_new == 0 is a special case for destroying existing domains,
 * and it will not create the default domain.
6729
 *
P
Paul Jackson 已提交
6730 6731
 * Call with hotplug lock held
 */
6732
void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
6733
			     struct sched_domain_attr *dattr_new)
P
Paul Jackson 已提交
6734
{
6735
	int i, j, n;
6736
	int new_topology;
P
Paul Jackson 已提交
6737

6738
	mutex_lock(&sched_domains_mutex);
6739

6740 6741 6742
	/* always unregister in case we don't destroy any domains */
	unregister_sched_domain_sysctl();

6743 6744 6745
	/* Let architecture update cpu core mappings. */
	new_topology = arch_update_cpu_topology();

6746
	n = doms_new ? ndoms_new : 0;
P
Paul Jackson 已提交
6747 6748 6749

	/* Destroy deleted domains */
	for (i = 0; i < ndoms_cur; i++) {
6750
		for (j = 0; j < n && !new_topology; j++) {
6751
			if (cpumask_equal(doms_cur[i], doms_new[j])
6752
			    && dattrs_equal(dattr_cur, i, dattr_new, j))
P
Paul Jackson 已提交
6753 6754 6755
				goto match1;
		}
		/* no match - a current sched domain not in new doms_new[] */
6756
		detach_destroy_domains(doms_cur[i]);
P
Paul Jackson 已提交
6757 6758 6759 6760
match1:
		;
	}

6761 6762
	if (doms_new == NULL) {
		ndoms_cur = 0;
6763
		doms_new = &fallback_doms;
6764
		cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
6765
		WARN_ON_ONCE(dattr_new);
6766 6767
	}

P
Paul Jackson 已提交
6768 6769
	/* Build new domains */
	for (i = 0; i < ndoms_new; i++) {
6770
		for (j = 0; j < ndoms_cur && !new_topology; j++) {
6771
			if (cpumask_equal(doms_new[i], doms_cur[j])
6772
			    && dattrs_equal(dattr_new, i, dattr_cur, j))
P
Paul Jackson 已提交
6773 6774 6775
				goto match2;
		}
		/* no match - add a new doms_new */
6776
		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
P
Paul Jackson 已提交
6777 6778 6779 6780 6781
match2:
		;
	}

	/* Remember the new sched domains */
6782 6783
	if (doms_cur != &fallback_doms)
		free_sched_domains(doms_cur, ndoms_cur);
6784
	kfree(dattr_cur);	/* kfree(NULL) is safe */
P
Paul Jackson 已提交
6785
	doms_cur = doms_new;
6786
	dattr_cur = dattr_new;
P
Paul Jackson 已提交
6787
	ndoms_cur = ndoms_new;
6788 6789

	register_sched_domain_sysctl();
6790

6791
	mutex_unlock(&sched_domains_mutex);
P
Paul Jackson 已提交
6792 6793
}

6794 6795
static int num_cpus_frozen;	/* used to mark begin/end of suspend/resume */

L
Linus Torvalds 已提交
6796
/*
6797 6798 6799
 * Update cpusets according to cpu_active mask.  If cpusets are
 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
 * around partition_sched_domains().
6800 6801 6802
 *
 * If we come here as part of a suspend/resume, don't touch cpusets because we
 * want to restore it back to its original state upon resume anyway.
L
Linus Torvalds 已提交
6803
 */
6804 6805
static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
			     void *hcpu)
6806
{
6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828
	switch (action) {
	case CPU_ONLINE_FROZEN:
	case CPU_DOWN_FAILED_FROZEN:

		/*
		 * num_cpus_frozen tracks how many CPUs are involved in suspend
		 * resume sequence. As long as this is not the last online
		 * operation in the resume sequence, just build a single sched
		 * domain, ignoring cpusets.
		 */
		num_cpus_frozen--;
		if (likely(num_cpus_frozen)) {
			partition_sched_domains(1, NULL, NULL);
			break;
		}

		/*
		 * This is the last CPU online operation. So fall through and
		 * restore the original sched domains by considering the
		 * cpuset configurations.
		 */

6829
	case CPU_ONLINE:
6830
	case CPU_DOWN_FAILED:
6831
		cpuset_update_active_cpus(true);
6832
		break;
6833 6834 6835
	default:
		return NOTIFY_DONE;
	}
6836
	return NOTIFY_OK;
6837
}
6838

6839 6840
static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
			       void *hcpu)
6841
{
6842
	switch (action) {
6843
	case CPU_DOWN_PREPARE:
6844
		cpuset_update_active_cpus(false);
6845 6846 6847 6848 6849
		break;
	case CPU_DOWN_PREPARE_FROZEN:
		num_cpus_frozen++;
		partition_sched_domains(1, NULL, NULL);
		break;
6850 6851 6852
	default:
		return NOTIFY_DONE;
	}
6853
	return NOTIFY_OK;
6854 6855
}

L
Linus Torvalds 已提交
6856 6857
void __init sched_init_smp(void)
{
6858 6859 6860
	cpumask_var_t non_isolated_cpus;

	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
6861
	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
6862

6863 6864
	sched_init_numa();

6865
	get_online_cpus();
6866
	mutex_lock(&sched_domains_mutex);
6867
	init_sched_domains(cpu_active_mask);
6868 6869 6870
	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
	if (cpumask_empty(non_isolated_cpus))
		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
6871
	mutex_unlock(&sched_domains_mutex);
6872
	put_online_cpus();
6873

6874
	hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
6875 6876
	hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
	hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
6877 6878 6879 6880

	/* RT runtime code needs to handle some hotplug events */
	hotcpu_notifier(update_runtime, 0);

6881
	init_hrtick();
6882 6883

	/* Move init over to a non-isolated CPU */
6884
	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
6885
		BUG();
I
Ingo Molnar 已提交
6886
	sched_init_granularity();
6887
	free_cpumask_var(non_isolated_cpus);
6888

6889
	init_sched_rt_class();
L
Linus Torvalds 已提交
6890 6891 6892 6893
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
6894
	sched_init_granularity();
L
Linus Torvalds 已提交
6895 6896 6897
}
#endif /* CONFIG_SMP */

6898 6899
const_debug unsigned int sysctl_timer_migration = 1;

L
Linus Torvalds 已提交
6900 6901 6902 6903 6904 6905 6906
int in_sched_functions(unsigned long addr)
{
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

6907
#ifdef CONFIG_CGROUP_SCHED
6908 6909 6910 6911
/*
 * Default task group.
 * Every task in system belongs to this group at bootup.
 */
6912
struct task_group root_task_group;
6913
LIST_HEAD(task_groups);
6914
#endif
P
Peter Zijlstra 已提交
6915

6916
DECLARE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
P
Peter Zijlstra 已提交
6917

L
Linus Torvalds 已提交
6918 6919
void __init sched_init(void)
{
I
Ingo Molnar 已提交
6920
	int i, j;
6921 6922 6923 6924 6925 6926 6927
	unsigned long alloc_size = 0, ptr;

#ifdef CONFIG_FAIR_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6928
#endif
6929
#ifdef CONFIG_CPUMASK_OFFSTACK
6930
	alloc_size += num_possible_cpus() * cpumask_size();
6931 6932
#endif
	if (alloc_size) {
6933
		ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
6934 6935

#ifdef CONFIG_FAIR_GROUP_SCHED
6936
		root_task_group.se = (struct sched_entity **)ptr;
6937 6938
		ptr += nr_cpu_ids * sizeof(void **);

6939
		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
6940
		ptr += nr_cpu_ids * sizeof(void **);
6941

6942
#endif /* CONFIG_FAIR_GROUP_SCHED */
6943
#ifdef CONFIG_RT_GROUP_SCHED
6944
		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
6945 6946
		ptr += nr_cpu_ids * sizeof(void **);

6947
		root_task_group.rt_rq = (struct rt_rq **)ptr;
6948 6949
		ptr += nr_cpu_ids * sizeof(void **);

6950
#endif /* CONFIG_RT_GROUP_SCHED */
6951 6952 6953 6954 6955 6956
#ifdef CONFIG_CPUMASK_OFFSTACK
		for_each_possible_cpu(i) {
			per_cpu(load_balance_tmpmask, i) = (void *)ptr;
			ptr += cpumask_size();
		}
#endif /* CONFIG_CPUMASK_OFFSTACK */
6957
	}
I
Ingo Molnar 已提交
6958

G
Gregory Haskins 已提交
6959 6960 6961 6962
#ifdef CONFIG_SMP
	init_defrootdomain();
#endif

6963 6964 6965 6966
	init_rt_bandwidth(&def_rt_bandwidth,
			global_rt_period(), global_rt_runtime());

#ifdef CONFIG_RT_GROUP_SCHED
6967
	init_rt_bandwidth(&root_task_group.rt_bandwidth,
6968
			global_rt_period(), global_rt_runtime());
6969
#endif /* CONFIG_RT_GROUP_SCHED */
6970

D
Dhaval Giani 已提交
6971
#ifdef CONFIG_CGROUP_SCHED
6972 6973
	list_add(&root_task_group.list, &task_groups);
	INIT_LIST_HEAD(&root_task_group.children);
6974
	INIT_LIST_HEAD(&root_task_group.siblings);
6975
	autogroup_init(&init_task);
6976

D
Dhaval Giani 已提交
6977
#endif /* CONFIG_CGROUP_SCHED */
P
Peter Zijlstra 已提交
6978

6979 6980 6981 6982 6983 6984
#ifdef CONFIG_CGROUP_CPUACCT
	root_cpuacct.cpustat = &kernel_cpustat;
	root_cpuacct.cpuusage = alloc_percpu(u64);
	/* Too early, not expected to fail */
	BUG_ON(!root_cpuacct.cpuusage);
#endif
6985
	for_each_possible_cpu(i) {
6986
		struct rq *rq;
L
Linus Torvalds 已提交
6987 6988

		rq = cpu_rq(i);
6989
		raw_spin_lock_init(&rq->lock);
N
Nick Piggin 已提交
6990
		rq->nr_running = 0;
6991 6992
		rq->calc_load_active = 0;
		rq->calc_load_update = jiffies + LOAD_FREQ;
6993
		init_cfs_rq(&rq->cfs);
P
Peter Zijlstra 已提交
6994
		init_rt_rq(&rq->rt, rq);
I
Ingo Molnar 已提交
6995
#ifdef CONFIG_FAIR_GROUP_SCHED
6996
		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
P
Peter Zijlstra 已提交
6997
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
D
Dhaval Giani 已提交
6998
		/*
6999
		 * How much cpu bandwidth does root_task_group get?
D
Dhaval Giani 已提交
7000 7001 7002 7003
		 *
		 * In case of task-groups formed thr' the cgroup filesystem, it
		 * gets 100% of the cpu resources in the system. This overall
		 * system cpu resource is divided among the tasks of
7004
		 * root_task_group and its child task-groups in a fair manner,
D
Dhaval Giani 已提交
7005 7006 7007
		 * based on each entity's (task or task-group's) weight
		 * (se->load.weight).
		 *
7008
		 * In other words, if root_task_group has 10 tasks of weight
D
Dhaval Giani 已提交
7009 7010 7011
		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
		 * then A0's share of the cpu resource is:
		 *
7012
		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
D
Dhaval Giani 已提交
7013
		 *
7014 7015
		 * We achieve this by letting root_task_group's tasks sit
		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
D
Dhaval Giani 已提交
7016
		 */
7017
		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
7018
		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
D
Dhaval Giani 已提交
7019 7020 7021
#endif /* CONFIG_FAIR_GROUP_SCHED */

		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
7022
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
7023
		INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
7024
		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
I
Ingo Molnar 已提交
7025
#endif
L
Linus Torvalds 已提交
7026

I
Ingo Molnar 已提交
7027 7028
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
7029 7030 7031

		rq->last_load_update_tick = jiffies;

L
Linus Torvalds 已提交
7032
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
7033
		rq->sd = NULL;
G
Gregory Haskins 已提交
7034
		rq->rd = NULL;
7035
		rq->cpu_power = SCHED_POWER_SCALE;
7036
		rq->post_schedule = 0;
L
Linus Torvalds 已提交
7037
		rq->active_balance = 0;
I
Ingo Molnar 已提交
7038
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
7039
		rq->push_cpu = 0;
7040
		rq->cpu = i;
7041
		rq->online = 0;
7042 7043
		rq->idle_stamp = 0;
		rq->avg_idle = 2*sysctl_sched_migration_cost;
7044 7045 7046

		INIT_LIST_HEAD(&rq->cfs_tasks);

7047
		rq_attach_root(rq, &def_root_domain);
7048
#ifdef CONFIG_NO_HZ_COMMON
7049
		rq->nohz_flags = 0;
7050
#endif
L
Linus Torvalds 已提交
7051
#endif
P
Peter Zijlstra 已提交
7052
		init_rq_hrtick(rq);
L
Linus Torvalds 已提交
7053 7054 7055
		atomic_set(&rq->nr_iowait, 0);
	}

7056
	set_load_weight(&init_task);
7057

7058 7059 7060 7061
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

7062
#ifdef CONFIG_RT_MUTEXES
7063
	plist_head_init(&init_task.pi_waiters);
7064 7065
#endif

L
Linus Torvalds 已提交
7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
7079 7080 7081

	calc_load_update = jiffies + LOAD_FREQ;

I
Ingo Molnar 已提交
7082 7083 7084 7085
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
7086

7087
#ifdef CONFIG_SMP
7088
	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
R
Rusty Russell 已提交
7089 7090 7091
	/* May be allocated at isolcpus cmdline parse time */
	if (cpu_isolated_map == NULL)
		zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
7092
	idle_thread_set_boot_cpu();
7093 7094
#endif
	init_sched_fair_class();
7095

7096
	scheduler_running = 1;
L
Linus Torvalds 已提交
7097 7098
}

7099
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
7100 7101
static inline int preempt_count_equals(int preempt_offset)
{
7102
	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
7103

A
Arnd Bergmann 已提交
7104
	return (nested == preempt_offset);
7105 7106
}

7107
void __might_sleep(const char *file, int line, int preempt_offset)
L
Linus Torvalds 已提交
7108 7109 7110
{
	static unsigned long prev_jiffy;	/* ratelimiting */

7111
	rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
7112 7113
	if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
	    system_state != SYSTEM_RUNNING || oops_in_progress)
I
Ingo Molnar 已提交
7114 7115 7116 7117 7118
		return;
	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
		return;
	prev_jiffy = jiffies;

P
Peter Zijlstra 已提交
7119 7120 7121 7122 7123 7124 7125
	printk(KERN_ERR
		"BUG: sleeping function called from invalid context at %s:%d\n",
			file, line);
	printk(KERN_ERR
		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
			in_atomic(), irqs_disabled(),
			current->pid, current->comm);
I
Ingo Molnar 已提交
7126 7127 7128 7129 7130

	debug_show_held_locks(current);
	if (irqs_disabled())
		print_irqtrace_events(current);
	dump_stack();
L
Linus Torvalds 已提交
7131 7132 7133 7134 7135
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
7136 7137
static void normalize_task(struct rq *rq, struct task_struct *p)
{
P
Peter Zijlstra 已提交
7138 7139
	const struct sched_class *prev_class = p->sched_class;
	int old_prio = p->prio;
7140
	int on_rq;
7141

P
Peter Zijlstra 已提交
7142
	on_rq = p->on_rq;
7143
	if (on_rq)
7144
		dequeue_task(rq, p, 0);
7145 7146
	__setscheduler(rq, p, SCHED_NORMAL, 0);
	if (on_rq) {
7147
		enqueue_task(rq, p, 0);
7148 7149
		resched_task(rq->curr);
	}
P
Peter Zijlstra 已提交
7150 7151

	check_class_changed(rq, p, prev_class, old_prio);
7152 7153
}

L
Linus Torvalds 已提交
7154 7155
void normalize_rt_tasks(void)
{
7156
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
7157
	unsigned long flags;
7158
	struct rq *rq;
L
Linus Torvalds 已提交
7159

7160
	read_lock_irqsave(&tasklist_lock, flags);
7161
	do_each_thread(g, p) {
7162 7163 7164 7165 7166 7167
		/*
		 * Only normalize user tasks:
		 */
		if (!p->mm)
			continue;

I
Ingo Molnar 已提交
7168 7169
		p->se.exec_start		= 0;
#ifdef CONFIG_SCHEDSTATS
7170 7171 7172
		p->se.statistics.wait_start	= 0;
		p->se.statistics.sleep_start	= 0;
		p->se.statistics.block_start	= 0;
I
Ingo Molnar 已提交
7173
#endif
I
Ingo Molnar 已提交
7174 7175 7176 7177 7178 7179 7180 7181

		if (!rt_task(p)) {
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
			if (TASK_NICE(p) < 0 && p->mm)
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
7182
			continue;
I
Ingo Molnar 已提交
7183
		}
L
Linus Torvalds 已提交
7184

7185
		raw_spin_lock(&p->pi_lock);
7186
		rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
7187

7188
		normalize_task(rq, p);
7189

7190
		__task_rq_unlock(rq);
7191
		raw_spin_unlock(&p->pi_lock);
7192 7193
	} while_each_thread(g, p);

7194
	read_unlock_irqrestore(&tasklist_lock, flags);
L
Linus Torvalds 已提交
7195 7196 7197
}

#endif /* CONFIG_MAGIC_SYSRQ */
7198

7199
#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
7200
/*
7201
 * These functions are only useful for the IA64 MCA handling, or kdb.
7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
7216
struct task_struct *curr_task(int cpu)
7217 7218 7219 7220
{
	return cpu_curr(cpu);
}

7221 7222 7223
#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */

#ifdef CONFIG_IA64
7224 7225 7226 7227 7228 7229
/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
I
Ingo Molnar 已提交
7230 7231
 * are serviced on a separate stack. It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner. This function
7232 7233 7234 7235 7236 7237 7238
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
7239
void set_curr_task(int cpu, struct task_struct *p)
7240 7241 7242 7243 7244
{
	cpu_curr(cpu) = p;
}

#endif
S
Srivatsa Vaddagiri 已提交
7245

D
Dhaval Giani 已提交
7246
#ifdef CONFIG_CGROUP_SCHED
7247 7248 7249
/* task_group_lock serializes the addition/removal of task groups */
static DEFINE_SPINLOCK(task_group_lock);

7250 7251 7252 7253
static void free_sched_group(struct task_group *tg)
{
	free_fair_sched_group(tg);
	free_rt_sched_group(tg);
7254
	autogroup_free(tg);
7255 7256 7257 7258
	kfree(tg);
}

/* allocate runqueue etc for a new task group */
7259
struct task_group *sched_create_group(struct task_group *parent)
7260 7261 7262 7263 7264 7265 7266
{
	struct task_group *tg;

	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
	if (!tg)
		return ERR_PTR(-ENOMEM);

7267
	if (!alloc_fair_sched_group(tg, parent))
7268 7269
		goto err;

7270
	if (!alloc_rt_sched_group(tg, parent))
7271 7272
		goto err;

7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283
	return tg;

err:
	free_sched_group(tg);
	return ERR_PTR(-ENOMEM);
}

void sched_online_group(struct task_group *tg, struct task_group *parent)
{
	unsigned long flags;

7284
	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
7285
	list_add_rcu(&tg->list, &task_groups);
P
Peter Zijlstra 已提交
7286 7287 7288 7289 7290

	WARN_ON(!parent); /* root should already exist */

	tg->parent = parent;
	INIT_LIST_HEAD(&tg->children);
7291
	list_add_rcu(&tg->siblings, &parent->children);
7292
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
7293 7294
}

7295
/* rcu callback to free various structures associated with a task group */
P
Peter Zijlstra 已提交
7296
static void free_sched_group_rcu(struct rcu_head *rhp)
S
Srivatsa Vaddagiri 已提交
7297 7298
{
	/* now it should be safe to free those cfs_rqs */
P
Peter Zijlstra 已提交
7299
	free_sched_group(container_of(rhp, struct task_group, rcu));
S
Srivatsa Vaddagiri 已提交
7300 7301
}

7302
/* Destroy runqueue etc associated with a task group */
7303
void sched_destroy_group(struct task_group *tg)
7304 7305 7306 7307 7308 7309
{
	/* wait for possible concurrent references to cfs_rqs complete */
	call_rcu(&tg->rcu, free_sched_group_rcu);
}

void sched_offline_group(struct task_group *tg)
S
Srivatsa Vaddagiri 已提交
7310
{
7311
	unsigned long flags;
7312
	int i;
S
Srivatsa Vaddagiri 已提交
7313

7314 7315
	/* end participation in shares distribution */
	for_each_possible_cpu(i)
7316
		unregister_fair_sched_group(tg, i);
7317 7318

	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
7319
	list_del_rcu(&tg->list);
P
Peter Zijlstra 已提交
7320
	list_del_rcu(&tg->siblings);
7321
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
7322 7323
}

7324
/* change task's runqueue when it moves between groups.
I
Ingo Molnar 已提交
7325 7326 7327
 *	The caller of this function should have put the task in its new group
 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
 *	reflect its new group.
7328 7329
 */
void sched_move_task(struct task_struct *tsk)
S
Srivatsa Vaddagiri 已提交
7330
{
P
Peter Zijlstra 已提交
7331
	struct task_group *tg;
S
Srivatsa Vaddagiri 已提交
7332 7333 7334 7335 7336 7337
	int on_rq, running;
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

7338
	running = task_current(rq, tsk);
P
Peter Zijlstra 已提交
7339
	on_rq = tsk->on_rq;
S
Srivatsa Vaddagiri 已提交
7340

7341
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
7342
		dequeue_task(rq, tsk, 0);
7343 7344
	if (unlikely(running))
		tsk->sched_class->put_prev_task(rq, tsk);
S
Srivatsa Vaddagiri 已提交
7345

P
Peter Zijlstra 已提交
7346 7347 7348 7349 7350 7351
	tg = container_of(task_subsys_state_check(tsk, cpu_cgroup_subsys_id,
				lockdep_is_held(&tsk->sighand->siglock)),
			  struct task_group, css);
	tg = autogroup_task_group(tsk, tg);
	tsk->sched_task_group = tg;

P
Peter Zijlstra 已提交
7352
#ifdef CONFIG_FAIR_GROUP_SCHED
7353 7354 7355
	if (tsk->sched_class->task_move_group)
		tsk->sched_class->task_move_group(tsk, on_rq);
	else
P
Peter Zijlstra 已提交
7356
#endif
7357
		set_task_rq(tsk, task_cpu(tsk));
P
Peter Zijlstra 已提交
7358

7359 7360 7361
	if (unlikely(running))
		tsk->sched_class->set_curr_task(rq);
	if (on_rq)
7362
		enqueue_task(rq, tsk, 0);
S
Srivatsa Vaddagiri 已提交
7363

7364
	task_rq_unlock(rq, tsk, &flags);
S
Srivatsa Vaddagiri 已提交
7365
}
D
Dhaval Giani 已提交
7366
#endif /* CONFIG_CGROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
7367

7368
#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_CFS_BANDWIDTH)
P
Peter Zijlstra 已提交
7369 7370 7371
static unsigned long to_ratio(u64 period, u64 runtime)
{
	if (runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
7372
		return 1ULL << 20;
P
Peter Zijlstra 已提交
7373

P
Peter Zijlstra 已提交
7374
	return div64_u64(runtime << 20, period);
P
Peter Zijlstra 已提交
7375
}
7376 7377 7378 7379 7380 7381 7382
#endif

#ifdef CONFIG_RT_GROUP_SCHED
/*
 * Ensure that the real time constraints are schedulable.
 */
static DEFINE_MUTEX(rt_constraints_mutex);
P
Peter Zijlstra 已提交
7383

P
Peter Zijlstra 已提交
7384 7385
/* Must be called with tasklist_lock held */
static inline int tg_has_rt_tasks(struct task_group *tg)
7386
{
P
Peter Zijlstra 已提交
7387
	struct task_struct *g, *p;
7388

P
Peter Zijlstra 已提交
7389
	do_each_thread(g, p) {
7390
		if (rt_task(p) && task_rq(p)->rt.tg == tg)
P
Peter Zijlstra 已提交
7391 7392
			return 1;
	} while_each_thread(g, p);
7393

P
Peter Zijlstra 已提交
7394 7395
	return 0;
}
7396

P
Peter Zijlstra 已提交
7397 7398 7399 7400 7401
struct rt_schedulable_data {
	struct task_group *tg;
	u64 rt_period;
	u64 rt_runtime;
};
7402

7403
static int tg_rt_schedulable(struct task_group *tg, void *data)
P
Peter Zijlstra 已提交
7404 7405 7406 7407 7408
{
	struct rt_schedulable_data *d = data;
	struct task_group *child;
	unsigned long total, sum = 0;
	u64 period, runtime;
7409

P
Peter Zijlstra 已提交
7410 7411
	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	runtime = tg->rt_bandwidth.rt_runtime;
7412

P
Peter Zijlstra 已提交
7413 7414 7415
	if (tg == d->tg) {
		period = d->rt_period;
		runtime = d->rt_runtime;
7416 7417
	}

7418 7419 7420 7421 7422
	/*
	 * Cannot have more runtime than the period.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
P
Peter Zijlstra 已提交
7423

7424 7425 7426
	/*
	 * Ensure we don't starve existing RT tasks.
	 */
P
Peter Zijlstra 已提交
7427 7428
	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
		return -EBUSY;
P
Peter Zijlstra 已提交
7429

P
Peter Zijlstra 已提交
7430
	total = to_ratio(period, runtime);
P
Peter Zijlstra 已提交
7431

7432 7433 7434 7435 7436
	/*
	 * Nobody can have more than the global setting allows.
	 */
	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
		return -EINVAL;
P
Peter Zijlstra 已提交
7437

7438 7439 7440
	/*
	 * The sum of our children's runtime should not exceed our own.
	 */
P
Peter Zijlstra 已提交
7441 7442 7443
	list_for_each_entry_rcu(child, &tg->children, siblings) {
		period = ktime_to_ns(child->rt_bandwidth.rt_period);
		runtime = child->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
7444

P
Peter Zijlstra 已提交
7445 7446 7447 7448
		if (child == d->tg) {
			period = d->rt_period;
			runtime = d->rt_runtime;
		}
P
Peter Zijlstra 已提交
7449

P
Peter Zijlstra 已提交
7450
		sum += to_ratio(period, runtime);
P
Peter Zijlstra 已提交
7451
	}
P
Peter Zijlstra 已提交
7452

P
Peter Zijlstra 已提交
7453 7454 7455 7456
	if (sum > total)
		return -EINVAL;

	return 0;
P
Peter Zijlstra 已提交
7457 7458
}

P
Peter Zijlstra 已提交
7459
static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
7460
{
7461 7462
	int ret;

P
Peter Zijlstra 已提交
7463 7464 7465 7466 7467 7468
	struct rt_schedulable_data data = {
		.tg = tg,
		.rt_period = period,
		.rt_runtime = runtime,
	};

7469 7470 7471 7472 7473
	rcu_read_lock();
	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
	rcu_read_unlock();

	return ret;
7474 7475
}

7476
static int tg_set_rt_bandwidth(struct task_group *tg,
7477
		u64 rt_period, u64 rt_runtime)
P
Peter Zijlstra 已提交
7478
{
P
Peter Zijlstra 已提交
7479
	int i, err = 0;
P
Peter Zijlstra 已提交
7480 7481

	mutex_lock(&rt_constraints_mutex);
7482
	read_lock(&tasklist_lock);
P
Peter Zijlstra 已提交
7483 7484
	err = __rt_schedulable(tg, rt_period, rt_runtime);
	if (err)
P
Peter Zijlstra 已提交
7485
		goto unlock;
P
Peter Zijlstra 已提交
7486

7487
	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
7488 7489
	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
	tg->rt_bandwidth.rt_runtime = rt_runtime;
P
Peter Zijlstra 已提交
7490 7491 7492 7493

	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = tg->rt_rq[i];

7494
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7495
		rt_rq->rt_runtime = rt_runtime;
7496
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7497
	}
7498
	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
P
Peter Zijlstra 已提交
7499
unlock:
7500
	read_unlock(&tasklist_lock);
P
Peter Zijlstra 已提交
7501 7502 7503
	mutex_unlock(&rt_constraints_mutex);

	return err;
P
Peter Zijlstra 已提交
7504 7505
}

7506
static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
7507 7508 7509 7510 7511 7512 7513 7514
{
	u64 rt_runtime, rt_period;

	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
	if (rt_runtime_us < 0)
		rt_runtime = RUNTIME_INF;

7515
	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7516 7517
}

7518
static long sched_group_rt_runtime(struct task_group *tg)
P
Peter Zijlstra 已提交
7519 7520 7521
{
	u64 rt_runtime_us;

7522
	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
7523 7524
		return -1;

7525
	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
7526 7527 7528
	do_div(rt_runtime_us, NSEC_PER_USEC);
	return rt_runtime_us;
}
7529

7530
static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
7531 7532 7533 7534 7535 7536
{
	u64 rt_runtime, rt_period;

	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
	rt_runtime = tg->rt_bandwidth.rt_runtime;

7537 7538 7539
	if (rt_period == 0)
		return -EINVAL;

7540
	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7541 7542
}

7543
static long sched_group_rt_period(struct task_group *tg)
7544 7545 7546 7547 7548 7549 7550 7551 7552 7553
{
	u64 rt_period_us;

	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
	do_div(rt_period_us, NSEC_PER_USEC);
	return rt_period_us;
}

static int sched_rt_global_constraints(void)
{
7554
	u64 runtime, period;
7555 7556
	int ret = 0;

7557 7558 7559
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

7560 7561 7562 7563 7564 7565 7566 7567
	runtime = global_rt_runtime();
	period = global_rt_period();

	/*
	 * Sanity check on the sysctl variables.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
7568

7569
	mutex_lock(&rt_constraints_mutex);
P
Peter Zijlstra 已提交
7570
	read_lock(&tasklist_lock);
7571
	ret = __rt_schedulable(NULL, 0, 0);
P
Peter Zijlstra 已提交
7572
	read_unlock(&tasklist_lock);
7573 7574 7575 7576
	mutex_unlock(&rt_constraints_mutex);

	return ret;
}
7577

7578
static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
7579 7580 7581 7582 7583 7584 7585 7586
{
	/* Don't accept realtime tasks when there is no way for them to run */
	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
		return 0;

	return 1;
}

7587
#else /* !CONFIG_RT_GROUP_SCHED */
7588 7589
static int sched_rt_global_constraints(void)
{
P
Peter Zijlstra 已提交
7590 7591 7592
	unsigned long flags;
	int i;

7593 7594 7595
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

7596 7597 7598 7599 7600 7601 7602
	/*
	 * There's always some RT tasks in the root group
	 * -- migration, kstopmachine etc..
	 */
	if (sysctl_sched_rt_runtime == 0)
		return -EBUSY;

7603
	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
7604 7605 7606
	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = &cpu_rq(i)->rt;

7607
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7608
		rt_rq->rt_runtime = global_rt_runtime();
7609
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7610
	}
7611
	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
7612

7613 7614
	return 0;
}
7615
#endif /* CONFIG_RT_GROUP_SCHED */
7616

7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635
int sched_rr_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
	/* make sure that internally we keep jiffies */
	/* also, writing zero resets timeslice to default */
	if (!ret && write) {
		sched_rr_timeslice = sched_rr_timeslice <= 0 ?
			RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
	}
	mutex_unlock(&mutex);
	return ret;
}

7636
int sched_rt_handler(struct ctl_table *table, int write,
7637
		void __user *buffer, size_t *lenp,
7638 7639 7640 7641 7642 7643 7644 7645 7646 7647
		loff_t *ppos)
{
	int ret;
	int old_period, old_runtime;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	old_period = sysctl_sched_rt_period;
	old_runtime = sysctl_sched_rt_runtime;

7648
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664

	if (!ret && write) {
		ret = sched_rt_global_constraints();
		if (ret) {
			sysctl_sched_rt_period = old_period;
			sysctl_sched_rt_runtime = old_runtime;
		} else {
			def_rt_bandwidth.rt_runtime = global_rt_runtime();
			def_rt_bandwidth.rt_period =
				ns_to_ktime(global_rt_period());
		}
	}
	mutex_unlock(&mutex);

	return ret;
}
7665

7666
#ifdef CONFIG_CGROUP_SCHED
7667 7668

/* return corresponding task_group object of a cgroup */
7669
static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
7670
{
7671 7672
	return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
			    struct task_group, css);
7673 7674
}

7675
static struct cgroup_subsys_state *cpu_cgroup_css_alloc(struct cgroup *cgrp)
7676
{
7677
	struct task_group *tg, *parent;
7678

7679
	if (!cgrp->parent) {
7680
		/* This is early initialization for the top cgroup */
7681
		return &root_task_group.css;
7682 7683
	}

7684 7685
	parent = cgroup_tg(cgrp->parent);
	tg = sched_create_group(parent);
7686 7687 7688 7689 7690 7691
	if (IS_ERR(tg))
		return ERR_PTR(-ENOMEM);

	return &tg->css;
}

7692 7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704
static int cpu_cgroup_css_online(struct cgroup *cgrp)
{
	struct task_group *tg = cgroup_tg(cgrp);
	struct task_group *parent;

	if (!cgrp->parent)
		return 0;

	parent = cgroup_tg(cgrp->parent);
	sched_online_group(tg, parent);
	return 0;
}

7705
static void cpu_cgroup_css_free(struct cgroup *cgrp)
7706
{
7707
	struct task_group *tg = cgroup_tg(cgrp);
7708 7709 7710 7711

	sched_destroy_group(tg);
}

7712 7713 7714 7715 7716 7717 7718
static void cpu_cgroup_css_offline(struct cgroup *cgrp)
{
	struct task_group *tg = cgroup_tg(cgrp);

	sched_offline_group(tg);
}

7719
static int cpu_cgroup_can_attach(struct cgroup *cgrp,
7720
				 struct cgroup_taskset *tset)
7721
{
7722 7723 7724
	struct task_struct *task;

	cgroup_taskset_for_each(task, cgrp, tset) {
7725
#ifdef CONFIG_RT_GROUP_SCHED
7726 7727
		if (!sched_rt_can_attach(cgroup_tg(cgrp), task))
			return -EINVAL;
7728
#else
7729 7730 7731
		/* We don't support RT-tasks being in separate groups */
		if (task->sched_class != &fair_sched_class)
			return -EINVAL;
7732
#endif
7733
	}
7734 7735
	return 0;
}
7736

7737
static void cpu_cgroup_attach(struct cgroup *cgrp,
7738
			      struct cgroup_taskset *tset)
7739
{
7740 7741 7742 7743
	struct task_struct *task;

	cgroup_taskset_for_each(task, cgrp, tset)
		sched_move_task(task);
7744 7745
}

7746
static void
7747 7748
cpu_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
		struct task_struct *task)
7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760
{
	/*
	 * cgroup_exit() is called in the copy_process() failure path.
	 * Ignore this case since the task hasn't ran yet, this avoids
	 * trying to poke a half freed task state from generic code.
	 */
	if (!(task->flags & PF_EXITING))
		return;

	sched_move_task(task);
}

7761
#ifdef CONFIG_FAIR_GROUP_SCHED
7762
static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
7763
				u64 shareval)
7764
{
7765
	return sched_group_set_shares(cgroup_tg(cgrp), scale_load(shareval));
7766 7767
}

7768
static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
7769
{
7770
	struct task_group *tg = cgroup_tg(cgrp);
7771

7772
	return (u64) scale_load_down(tg->shares);
7773
}
7774 7775

#ifdef CONFIG_CFS_BANDWIDTH
7776 7777
static DEFINE_MUTEX(cfs_constraints_mutex);

7778 7779 7780
const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */

7781 7782
static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);

7783 7784
static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
{
7785
	int i, ret = 0, runtime_enabled, runtime_was_enabled;
7786
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806

	if (tg == &root_task_group)
		return -EINVAL;

	/*
	 * Ensure we have at some amount of bandwidth every period.  This is
	 * to prevent reaching a state of large arrears when throttled via
	 * entity_tick() resulting in prolonged exit starvation.
	 */
	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
		return -EINVAL;

	/*
	 * Likewise, bound things on the otherside by preventing insane quota
	 * periods.  This also allows us to normalize in computing quota
	 * feasibility.
	 */
	if (period > max_cfs_quota_period)
		return -EINVAL;

7807 7808 7809 7810 7811
	mutex_lock(&cfs_constraints_mutex);
	ret = __cfs_schedulable(tg, period, quota);
	if (ret)
		goto out_unlock;

7812
	runtime_enabled = quota != RUNTIME_INF;
7813 7814
	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
	account_cfs_bandwidth_used(runtime_enabled, runtime_was_enabled);
7815 7816 7817
	raw_spin_lock_irq(&cfs_b->lock);
	cfs_b->period = ns_to_ktime(period);
	cfs_b->quota = quota;
7818

P
Paul Turner 已提交
7819
	__refill_cfs_bandwidth_runtime(cfs_b);
7820 7821 7822 7823 7824 7825
	/* restart the period timer (if active) to handle new period expiry */
	if (runtime_enabled && cfs_b->timer_active) {
		/* force a reprogram */
		cfs_b->timer_active = 0;
		__start_cfs_bandwidth(cfs_b);
	}
7826 7827 7828 7829
	raw_spin_unlock_irq(&cfs_b->lock);

	for_each_possible_cpu(i) {
		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
7830
		struct rq *rq = cfs_rq->rq;
7831 7832

		raw_spin_lock_irq(&rq->lock);
7833
		cfs_rq->runtime_enabled = runtime_enabled;
7834
		cfs_rq->runtime_remaining = 0;
7835

7836
		if (cfs_rq->throttled)
7837
			unthrottle_cfs_rq(cfs_rq);
7838 7839
		raw_spin_unlock_irq(&rq->lock);
	}
7840 7841
out_unlock:
	mutex_unlock(&cfs_constraints_mutex);
7842

7843
	return ret;
7844 7845 7846 7847 7848 7849
}

int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
{
	u64 quota, period;

7850
	period = ktime_to_ns(tg->cfs_bandwidth.period);
7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862
	if (cfs_quota_us < 0)
		quota = RUNTIME_INF;
	else
		quota = (u64)cfs_quota_us * NSEC_PER_USEC;

	return tg_set_cfs_bandwidth(tg, period, quota);
}

long tg_get_cfs_quota(struct task_group *tg)
{
	u64 quota_us;

7863
	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
7864 7865
		return -1;

7866
	quota_us = tg->cfs_bandwidth.quota;
7867 7868 7869 7870 7871 7872 7873 7874 7875 7876
	do_div(quota_us, NSEC_PER_USEC);

	return quota_us;
}

int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
{
	u64 quota, period;

	period = (u64)cfs_period_us * NSEC_PER_USEC;
7877
	quota = tg->cfs_bandwidth.quota;
7878 7879 7880 7881 7882 7883 7884 7885

	return tg_set_cfs_bandwidth(tg, period, quota);
}

long tg_get_cfs_period(struct task_group *tg)
{
	u64 cfs_period_us;

7886
	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
7887 7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913
	do_div(cfs_period_us, NSEC_PER_USEC);

	return cfs_period_us;
}

static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft)
{
	return tg_get_cfs_quota(cgroup_tg(cgrp));
}

static int cpu_cfs_quota_write_s64(struct cgroup *cgrp, struct cftype *cftype,
				s64 cfs_quota_us)
{
	return tg_set_cfs_quota(cgroup_tg(cgrp), cfs_quota_us);
}

static u64 cpu_cfs_period_read_u64(struct cgroup *cgrp, struct cftype *cft)
{
	return tg_get_cfs_period(cgroup_tg(cgrp));
}

static int cpu_cfs_period_write_u64(struct cgroup *cgrp, struct cftype *cftype,
				u64 cfs_period_us)
{
	return tg_set_cfs_period(cgroup_tg(cgrp), cfs_period_us);
}

7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945
struct cfs_schedulable_data {
	struct task_group *tg;
	u64 period, quota;
};

/*
 * normalize group quota/period to be quota/max_period
 * note: units are usecs
 */
static u64 normalize_cfs_quota(struct task_group *tg,
			       struct cfs_schedulable_data *d)
{
	u64 quota, period;

	if (tg == d->tg) {
		period = d->period;
		quota = d->quota;
	} else {
		period = tg_get_cfs_period(tg);
		quota = tg_get_cfs_quota(tg);
	}

	/* note: these should typically be equivalent */
	if (quota == RUNTIME_INF || quota == -1)
		return RUNTIME_INF;

	return to_ratio(period, quota);
}

static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
{
	struct cfs_schedulable_data *d = data;
7946
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7947 7948 7949 7950 7951
	s64 quota = 0, parent_quota = -1;

	if (!tg->parent) {
		quota = RUNTIME_INF;
	} else {
7952
		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972

		quota = normalize_cfs_quota(tg, d);
		parent_quota = parent_b->hierarchal_quota;

		/*
		 * ensure max(child_quota) <= parent_quota, inherit when no
		 * limit is set
		 */
		if (quota == RUNTIME_INF)
			quota = parent_quota;
		else if (parent_quota != RUNTIME_INF && quota > parent_quota)
			return -EINVAL;
	}
	cfs_b->hierarchal_quota = quota;

	return 0;
}

static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
{
7973
	int ret;
7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984
	struct cfs_schedulable_data data = {
		.tg = tg,
		.period = period,
		.quota = quota,
	};

	if (quota != RUNTIME_INF) {
		do_div(data.period, NSEC_PER_USEC);
		do_div(data.quota, NSEC_PER_USEC);
	}

7985 7986 7987 7988 7989
	rcu_read_lock();
	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
	rcu_read_unlock();

	return ret;
7990
}
7991 7992 7993 7994 7995

static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft,
		struct cgroup_map_cb *cb)
{
	struct task_group *tg = cgroup_tg(cgrp);
7996
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7997 7998 7999 8000 8001 8002 8003

	cb->fill(cb, "nr_periods", cfs_b->nr_periods);
	cb->fill(cb, "nr_throttled", cfs_b->nr_throttled);
	cb->fill(cb, "throttled_time", cfs_b->throttled_time);

	return 0;
}
8004
#endif /* CONFIG_CFS_BANDWIDTH */
8005
#endif /* CONFIG_FAIR_GROUP_SCHED */
8006

8007
#ifdef CONFIG_RT_GROUP_SCHED
M
Mirco Tischler 已提交
8008
static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
8009
				s64 val)
P
Peter Zijlstra 已提交
8010
{
8011
	return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
P
Peter Zijlstra 已提交
8012 8013
}

8014
static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
P
Peter Zijlstra 已提交
8015
{
8016
	return sched_group_rt_runtime(cgroup_tg(cgrp));
P
Peter Zijlstra 已提交
8017
}
8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028

static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
		u64 rt_period_us)
{
	return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
}

static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
{
	return sched_group_rt_period(cgroup_tg(cgrp));
}
8029
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
8030

8031
static struct cftype cpu_files[] = {
8032
#ifdef CONFIG_FAIR_GROUP_SCHED
8033 8034
	{
		.name = "shares",
8035 8036
		.read_u64 = cpu_shares_read_u64,
		.write_u64 = cpu_shares_write_u64,
8037
	},
8038
#endif
8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049
#ifdef CONFIG_CFS_BANDWIDTH
	{
		.name = "cfs_quota_us",
		.read_s64 = cpu_cfs_quota_read_s64,
		.write_s64 = cpu_cfs_quota_write_s64,
	},
	{
		.name = "cfs_period_us",
		.read_u64 = cpu_cfs_period_read_u64,
		.write_u64 = cpu_cfs_period_write_u64,
	},
8050 8051 8052 8053
	{
		.name = "stat",
		.read_map = cpu_stats_show,
	},
8054
#endif
8055
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8056
	{
P
Peter Zijlstra 已提交
8057
		.name = "rt_runtime_us",
8058 8059
		.read_s64 = cpu_rt_runtime_read,
		.write_s64 = cpu_rt_runtime_write,
P
Peter Zijlstra 已提交
8060
	},
8061 8062
	{
		.name = "rt_period_us",
8063 8064
		.read_u64 = cpu_rt_period_read_uint,
		.write_u64 = cpu_rt_period_write_uint,
8065
	},
8066
#endif
8067
	{ }	/* terminate */
8068 8069 8070
};

struct cgroup_subsys cpu_cgroup_subsys = {
I
Ingo Molnar 已提交
8071
	.name		= "cpu",
8072 8073
	.css_alloc	= cpu_cgroup_css_alloc,
	.css_free	= cpu_cgroup_css_free,
8074 8075
	.css_online	= cpu_cgroup_css_online,
	.css_offline	= cpu_cgroup_css_offline,
8076 8077
	.can_attach	= cpu_cgroup_can_attach,
	.attach		= cpu_cgroup_attach,
8078
	.exit		= cpu_cgroup_exit,
I
Ingo Molnar 已提交
8079
	.subsys_id	= cpu_cgroup_subsys_id,
8080
	.base_cftypes	= cpu_files,
8081 8082 8083
	.early_init	= 1,
};

8084
#endif	/* CONFIG_CGROUP_SCHED */
8085 8086 8087 8088 8089 8090 8091 8092 8093 8094

#ifdef CONFIG_CGROUP_CPUACCT

/*
 * CPU accounting code for task groups.
 *
 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
 * (balbir@in.ibm.com).
 */

8095 8096
struct cpuacct root_cpuacct;

8097
/* create a new cpu accounting group */
8098
static struct cgroup_subsys_state *cpuacct_css_alloc(struct cgroup *cgrp)
8099
{
8100
	struct cpuacct *ca;
8101

8102 8103 8104 8105
	if (!cgrp->parent)
		return &root_cpuacct.css;

	ca = kzalloc(sizeof(*ca), GFP_KERNEL);
8106
	if (!ca)
8107
		goto out;
8108 8109

	ca->cpuusage = alloc_percpu(u64);
8110 8111 8112
	if (!ca->cpuusage)
		goto out_free_ca;

8113 8114 8115
	ca->cpustat = alloc_percpu(struct kernel_cpustat);
	if (!ca->cpustat)
		goto out_free_cpuusage;
8116

8117
	return &ca->css;
8118

8119
out_free_cpuusage:
8120 8121 8122 8123 8124
	free_percpu(ca->cpuusage);
out_free_ca:
	kfree(ca);
out:
	return ERR_PTR(-ENOMEM);
8125 8126 8127
}

/* destroy an existing cpu accounting group */
8128
static void cpuacct_css_free(struct cgroup *cgrp)
8129
{
8130
	struct cpuacct *ca = cgroup_ca(cgrp);
8131

8132
	free_percpu(ca->cpustat);
8133 8134 8135 8136
	free_percpu(ca->cpuusage);
	kfree(ca);
}

8137 8138
static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
{
8139
	u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
8140 8141 8142 8143 8144 8145
	u64 data;

#ifndef CONFIG_64BIT
	/*
	 * Take rq->lock to make 64-bit read safe on 32-bit platforms.
	 */
8146
	raw_spin_lock_irq(&cpu_rq(cpu)->lock);
8147
	data = *cpuusage;
8148
	raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
8149 8150 8151 8152 8153 8154 8155 8156 8157
#else
	data = *cpuusage;
#endif

	return data;
}

static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
{
8158
	u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
8159 8160 8161 8162 8163

#ifndef CONFIG_64BIT
	/*
	 * Take rq->lock to make 64-bit write safe on 32-bit platforms.
	 */
8164
	raw_spin_lock_irq(&cpu_rq(cpu)->lock);
8165
	*cpuusage = val;
8166
	raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
8167 8168 8169 8170 8171
#else
	*cpuusage = val;
#endif
}

8172
/* return total cpu usage (in nanoseconds) of a group */
8173
static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
8174
{
8175
	struct cpuacct *ca = cgroup_ca(cgrp);
8176 8177 8178
	u64 totalcpuusage = 0;
	int i;

8179 8180
	for_each_present_cpu(i)
		totalcpuusage += cpuacct_cpuusage_read(ca, i);
8181 8182 8183 8184

	return totalcpuusage;
}

8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196
static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
								u64 reset)
{
	struct cpuacct *ca = cgroup_ca(cgrp);
	int err = 0;
	int i;

	if (reset) {
		err = -EINVAL;
		goto out;
	}

8197 8198
	for_each_present_cpu(i)
		cpuacct_cpuusage_write(ca, i, 0);
8199 8200 8201 8202 8203

out:
	return err;
}

8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218
static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
				   struct seq_file *m)
{
	struct cpuacct *ca = cgroup_ca(cgroup);
	u64 percpu;
	int i;

	for_each_present_cpu(i) {
		percpu = cpuacct_cpuusage_read(ca, i);
		seq_printf(m, "%llu ", (unsigned long long) percpu);
	}
	seq_printf(m, "\n");
	return 0;
}

8219 8220 8221 8222 8223 8224
static const char *cpuacct_stat_desc[] = {
	[CPUACCT_STAT_USER] = "user",
	[CPUACCT_STAT_SYSTEM] = "system",
};

static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
8225
			      struct cgroup_map_cb *cb)
8226 8227
{
	struct cpuacct *ca = cgroup_ca(cgrp);
8228 8229
	int cpu;
	s64 val = 0;
8230

8231 8232 8233 8234
	for_each_online_cpu(cpu) {
		struct kernel_cpustat *kcpustat = per_cpu_ptr(ca->cpustat, cpu);
		val += kcpustat->cpustat[CPUTIME_USER];
		val += kcpustat->cpustat[CPUTIME_NICE];
8235
	}
8236 8237
	val = cputime64_to_clock_t(val);
	cb->fill(cb, cpuacct_stat_desc[CPUACCT_STAT_USER], val);
8238

8239 8240 8241 8242 8243 8244
	val = 0;
	for_each_online_cpu(cpu) {
		struct kernel_cpustat *kcpustat = per_cpu_ptr(ca->cpustat, cpu);
		val += kcpustat->cpustat[CPUTIME_SYSTEM];
		val += kcpustat->cpustat[CPUTIME_IRQ];
		val += kcpustat->cpustat[CPUTIME_SOFTIRQ];
8245
	}
8246 8247 8248 8249

	val = cputime64_to_clock_t(val);
	cb->fill(cb, cpuacct_stat_desc[CPUACCT_STAT_SYSTEM], val);

8250 8251 8252
	return 0;
}

8253 8254 8255
static struct cftype files[] = {
	{
		.name = "usage",
8256 8257
		.read_u64 = cpuusage_read,
		.write_u64 = cpuusage_write,
8258
	},
8259 8260 8261 8262
	{
		.name = "usage_percpu",
		.read_seq_string = cpuacct_percpu_seq_read,
	},
8263 8264 8265 8266
	{
		.name = "stat",
		.read_map = cpuacct_stats_show,
	},
8267
	{ }	/* terminate */
8268 8269 8270 8271 8272 8273 8274
};

/*
 * charge this task's execution time to its accounting group.
 *
 * called with rq->lock held.
 */
8275
void cpuacct_charge(struct task_struct *tsk, u64 cputime)
8276 8277
{
	struct cpuacct *ca;
8278
	int cpu;
8279

L
Li Zefan 已提交
8280
	if (unlikely(!cpuacct_subsys.active))
8281 8282
		return;

8283
	cpu = task_cpu(tsk);
8284 8285 8286

	rcu_read_lock();

8287 8288
	ca = task_ca(tsk);

8289
	for (; ca; ca = parent_ca(ca)) {
8290
		u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
8291 8292
		*cpuusage += cputime;
	}
8293 8294

	rcu_read_unlock();
8295 8296 8297 8298
}

struct cgroup_subsys cpuacct_subsys = {
	.name = "cpuacct",
8299 8300
	.css_alloc = cpuacct_css_alloc,
	.css_free = cpuacct_css_free,
8301
	.subsys_id = cpuacct_subsys_id,
8302
	.base_cftypes = files,
8303 8304
};
#endif	/* CONFIG_CGROUP_CPUACCT */
8305 8306 8307 8308 8309 8310

void dump_cpu_task(int cpu)
{
	pr_info("Task dump for CPU %d:\n", cpu);
	sched_show_task(cpu_curr(cpu));
}