core.c 186.9 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  kernel/sched/core.c
L
Linus Torvalds 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
25 26
 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 *              Thomas Gleixner, Mike Kravetz
L
Linus Torvalds 已提交
27 28 29 30 31 32
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
33
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
34 35 36
#include <linux/highmem.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
37
#include <linux/capability.h>
L
Linus Torvalds 已提交
38 39
#include <linux/completion.h>
#include <linux/kernel_stat.h>
40
#include <linux/debug_locks.h>
41
#include <linux/perf_event.h>
L
Linus Torvalds 已提交
42 43 44
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
45
#include <linux/freezer.h>
46
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
47 48
#include <linux/blkdev.h>
#include <linux/delay.h>
49
#include <linux/pid_namespace.h>
L
Linus Torvalds 已提交
50 51 52 53 54 55 56
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
57
#include <linux/proc_fs.h>
L
Linus Torvalds 已提交
58
#include <linux/seq_file.h>
59
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
60 61
#include <linux/syscalls.h>
#include <linux/times.h>
62
#include <linux/tsacct_kern.h>
63
#include <linux/kprobes.h>
64
#include <linux/delayacct.h>
65
#include <linux/unistd.h>
J
Jens Axboe 已提交
66
#include <linux/pagemap.h>
P
Peter Zijlstra 已提交
67
#include <linux/hrtimer.h>
R
Reynes Philippe 已提交
68
#include <linux/tick.h>
P
Peter Zijlstra 已提交
69 70
#include <linux/debugfs.h>
#include <linux/ctype.h>
71
#include <linux/ftrace.h>
72
#include <linux/slab.h>
73
#include <linux/init_task.h>
A
Al Viro 已提交
74
#include <linux/binfmts.h>
75
#include <linux/context_tracking.h>
L
Linus Torvalds 已提交
76

77
#include <asm/switch_to.h>
78
#include <asm/tlb.h>
79
#include <asm/irq_regs.h>
80
#include <asm/mutex.h>
G
Glauber Costa 已提交
81 82 83
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#endif
L
Linus Torvalds 已提交
84

85
#include "sched.h"
86
#include "../workqueue_internal.h"
87
#include "../smpboot.h"
88

89
#define CREATE_TRACE_POINTS
90
#include <trace/events/sched.h>
91

92
void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
93
{
94 95
	unsigned long delta;
	ktime_t soft, hard, now;
96

97 98 99 100 101 102
	for (;;) {
		if (hrtimer_active(period_timer))
			break;

		now = hrtimer_cb_get_time(period_timer);
		hrtimer_forward(period_timer, now, period);
103

104 105 106 107 108 109 110 111
		soft = hrtimer_get_softexpires(period_timer);
		hard = hrtimer_get_expires(period_timer);
		delta = ktime_to_ns(ktime_sub(hard, soft));
		__hrtimer_start_range_ns(period_timer, soft, delta,
					 HRTIMER_MODE_ABS_PINNED, 0);
	}
}

112 113
DEFINE_MUTEX(sched_domains_mutex);
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
114

115
static void update_rq_clock_task(struct rq *rq, s64 delta);
116

117
void update_rq_clock(struct rq *rq)
118
{
119
	s64 delta;
120

121
	if (rq->skip_clock_update > 0)
122
		return;
123

124 125 126
	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
	rq->clock += delta;
	update_rq_clock_task(rq, delta);
127 128
}

I
Ingo Molnar 已提交
129 130 131
/*
 * Debugging: various feature bits
 */
P
Peter Zijlstra 已提交
132 133 134 135

#define SCHED_FEAT(name, enabled)	\
	(1UL << __SCHED_FEAT_##name) * enabled |

I
Ingo Molnar 已提交
136
const_debug unsigned int sysctl_sched_features =
137
#include "features.h"
P
Peter Zijlstra 已提交
138 139 140 141 142 143 144 145
	0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG
#define SCHED_FEAT(name, enabled)	\
	#name ,

146
static const char * const sched_feat_names[] = {
147
#include "features.h"
P
Peter Zijlstra 已提交
148 149 150 151
};

#undef SCHED_FEAT

L
Li Zefan 已提交
152
static int sched_feat_show(struct seq_file *m, void *v)
P
Peter Zijlstra 已提交
153 154 155
{
	int i;

156
	for (i = 0; i < __SCHED_FEAT_NR; i++) {
L
Li Zefan 已提交
157 158 159
		if (!(sysctl_sched_features & (1UL << i)))
			seq_puts(m, "NO_");
		seq_printf(m, "%s ", sched_feat_names[i]);
P
Peter Zijlstra 已提交
160
	}
L
Li Zefan 已提交
161
	seq_puts(m, "\n");
P
Peter Zijlstra 已提交
162

L
Li Zefan 已提交
163
	return 0;
P
Peter Zijlstra 已提交
164 165
}

166 167
#ifdef HAVE_JUMP_LABEL

168 169
#define jump_label_key__true  STATIC_KEY_INIT_TRUE
#define jump_label_key__false STATIC_KEY_INIT_FALSE
170 171 172 173

#define SCHED_FEAT(name, enabled)	\
	jump_label_key__##enabled ,

174
struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
175 176 177 178 179 180 181
#include "features.h"
};

#undef SCHED_FEAT

static void sched_feat_disable(int i)
{
182 183
	if (static_key_enabled(&sched_feat_keys[i]))
		static_key_slow_dec(&sched_feat_keys[i]);
184 185 186 187
}

static void sched_feat_enable(int i)
{
188 189
	if (!static_key_enabled(&sched_feat_keys[i]))
		static_key_slow_inc(&sched_feat_keys[i]);
190 191 192 193 194 195
}
#else
static void sched_feat_disable(int i) { };
static void sched_feat_enable(int i) { };
#endif /* HAVE_JUMP_LABEL */

196
static int sched_feat_set(char *cmp)
P
Peter Zijlstra 已提交
197 198
{
	int i;
199
	int neg = 0;
P
Peter Zijlstra 已提交
200

H
Hillf Danton 已提交
201
	if (strncmp(cmp, "NO_", 3) == 0) {
P
Peter Zijlstra 已提交
202 203 204 205
		neg = 1;
		cmp += 3;
	}

206
	for (i = 0; i < __SCHED_FEAT_NR; i++) {
207
		if (strcmp(cmp, sched_feat_names[i]) == 0) {
208
			if (neg) {
P
Peter Zijlstra 已提交
209
				sysctl_sched_features &= ~(1UL << i);
210 211
				sched_feat_disable(i);
			} else {
P
Peter Zijlstra 已提交
212
				sysctl_sched_features |= (1UL << i);
213 214
				sched_feat_enable(i);
			}
P
Peter Zijlstra 已提交
215 216 217 218
			break;
		}
	}

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
	return i;
}

static ssize_t
sched_feat_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
	char buf[64];
	char *cmp;
	int i;

	if (cnt > 63)
		cnt = 63;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;
	cmp = strstrip(buf);

	i = sched_feat_set(cmp);
240
	if (i == __SCHED_FEAT_NR)
P
Peter Zijlstra 已提交
241 242
		return -EINVAL;

243
	*ppos += cnt;
P
Peter Zijlstra 已提交
244 245 246 247

	return cnt;
}

L
Li Zefan 已提交
248 249 250 251 252
static int sched_feat_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_feat_show, NULL);
}

253
static const struct file_operations sched_feat_fops = {
L
Li Zefan 已提交
254 255 256 257 258
	.open		= sched_feat_open,
	.write		= sched_feat_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
P
Peter Zijlstra 已提交
259 260 261 262 263 264 265 266 267 268
};

static __init int sched_init_debug(void)
{
	debugfs_create_file("sched_features", 0644, NULL, NULL,
			&sched_feat_fops);

	return 0;
}
late_initcall(sched_init_debug);
269
#endif /* CONFIG_SCHED_DEBUG */
I
Ingo Molnar 已提交
270

271 272 273 274 275 276
/*
 * Number of tasks to iterate in a single balance run.
 * Limited because this is done with IRQs disabled.
 */
const_debug unsigned int sysctl_sched_nr_migrate = 32;

277 278 279 280 281 282 283 284
/*
 * period over which we average the RT time consumption, measured
 * in ms.
 *
 * default: 1s
 */
const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;

P
Peter Zijlstra 已提交
285
/*
P
Peter Zijlstra 已提交
286
 * period over which we measure -rt task cpu usage in us.
P
Peter Zijlstra 已提交
287 288
 * default: 1s
 */
P
Peter Zijlstra 已提交
289
unsigned int sysctl_sched_rt_period = 1000000;
P
Peter Zijlstra 已提交
290

291
__read_mostly int scheduler_running;
292

P
Peter Zijlstra 已提交
293 294 295 296 297
/*
 * part of the period that we allow rt tasks to run in us.
 * default: 0.95s
 */
int sysctl_sched_rt_runtime = 950000;
P
Peter Zijlstra 已提交
298

299
/*
300
 * __task_rq_lock - lock the rq @p resides on.
301
 */
302
static inline struct rq *__task_rq_lock(struct task_struct *p)
303 304
	__acquires(rq->lock)
{
305 306
	struct rq *rq;

307 308
	lockdep_assert_held(&p->pi_lock);

309
	for (;;) {
310
		rq = task_rq(p);
311
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
312
		if (likely(rq == task_rq(p)))
313
			return rq;
314
		raw_spin_unlock(&rq->lock);
315 316 317
	}
}

L
Linus Torvalds 已提交
318
/*
319
 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
L
Linus Torvalds 已提交
320
 */
321
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
322
	__acquires(p->pi_lock)
L
Linus Torvalds 已提交
323 324
	__acquires(rq->lock)
{
325
	struct rq *rq;
L
Linus Torvalds 已提交
326

327
	for (;;) {
328
		raw_spin_lock_irqsave(&p->pi_lock, *flags);
329
		rq = task_rq(p);
330
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
331
		if (likely(rq == task_rq(p)))
332
			return rq;
333 334
		raw_spin_unlock(&rq->lock);
		raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
335 336 337
	}
}

A
Alexey Dobriyan 已提交
338
static void __task_rq_unlock(struct rq *rq)
339 340
	__releases(rq->lock)
{
341
	raw_spin_unlock(&rq->lock);
342 343
}

344 345
static inline void
task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
346
	__releases(rq->lock)
347
	__releases(p->pi_lock)
L
Linus Torvalds 已提交
348
{
349 350
	raw_spin_unlock(&rq->lock);
	raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
351 352 353
}

/*
354
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
355
 */
A
Alexey Dobriyan 已提交
356
static struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
357 358
	__acquires(rq->lock)
{
359
	struct rq *rq;
L
Linus Torvalds 已提交
360 361 362

	local_irq_disable();
	rq = this_rq();
363
	raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
364 365 366 367

	return rq;
}

P
Peter Zijlstra 已提交
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
#ifdef CONFIG_SCHED_HRTICK
/*
 * Use HR-timers to deliver accurate preemption points.
 */

static void hrtick_clear(struct rq *rq)
{
	if (hrtimer_active(&rq->hrtick_timer))
		hrtimer_cancel(&rq->hrtick_timer);
}

/*
 * High-resolution timer tick.
 * Runs from hardirq context with interrupts disabled.
 */
static enum hrtimer_restart hrtick(struct hrtimer *timer)
{
	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

389
	raw_spin_lock(&rq->lock);
390
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
391
	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
392
	raw_spin_unlock(&rq->lock);
P
Peter Zijlstra 已提交
393 394 395 396

	return HRTIMER_NORESTART;
}

397
#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
398 399 400 401 402 403 404 405 406

static int __hrtick_restart(struct rq *rq)
{
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = hrtimer_get_softexpires(timer);

	return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0);
}

407 408 409 410
/*
 * called from hardirq (IPI) context
 */
static void __hrtick_start(void *arg)
411
{
412
	struct rq *rq = arg;
413

414
	raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
415
	__hrtick_restart(rq);
416
	rq->hrtick_csd_pending = 0;
417
	raw_spin_unlock(&rq->lock);
418 419
}

420 421 422 423 424
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
425
void hrtick_start(struct rq *rq, u64 delay)
426
{
427 428
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
429

430
	hrtimer_set_expires(timer, time);
431 432

	if (rq == this_rq()) {
P
Peter Zijlstra 已提交
433
		__hrtick_restart(rq);
434
	} else if (!rq->hrtick_csd_pending) {
435
		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
436 437
		rq->hrtick_csd_pending = 1;
	}
438 439 440 441 442 443 444 445 446 447 448 449 450 451
}

static int
hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
	int cpu = (int)(long)hcpu;

	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
452
		hrtick_clear(cpu_rq(cpu));
453 454 455 456 457 458
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

459
static __init void init_hrtick(void)
460 461 462
{
	hotcpu_notifier(hotplug_hrtick, 0);
}
463 464 465 466 467 468
#else
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
469
void hrtick_start(struct rq *rq, u64 delay)
470
{
471
	__hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
472
			HRTIMER_MODE_REL_PINNED, 0);
473
}
474

A
Andrew Morton 已提交
475
static inline void init_hrtick(void)
P
Peter Zijlstra 已提交
476 477
{
}
478
#endif /* CONFIG_SMP */
P
Peter Zijlstra 已提交
479

480
static void init_rq_hrtick(struct rq *rq)
P
Peter Zijlstra 已提交
481
{
482 483
#ifdef CONFIG_SMP
	rq->hrtick_csd_pending = 0;
P
Peter Zijlstra 已提交
484

485 486 487 488
	rq->hrtick_csd.flags = 0;
	rq->hrtick_csd.func = __hrtick_start;
	rq->hrtick_csd.info = rq;
#endif
P
Peter Zijlstra 已提交
489

490 491
	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rq->hrtick_timer.function = hrtick;
P
Peter Zijlstra 已提交
492
}
A
Andrew Morton 已提交
493
#else	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
494 495 496 497 498 499 500 501
static inline void hrtick_clear(struct rq *rq)
{
}

static inline void init_rq_hrtick(struct rq *rq)
{
}

502 503 504
static inline void init_hrtick(void)
{
}
A
Andrew Morton 已提交
505
#endif	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
506

I
Ingo Molnar 已提交
507 508 509 510 511 512 513
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
514
void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
515 516 517
{
	int cpu;

518
	lockdep_assert_held(&task_rq(p)->lock);
I
Ingo Molnar 已提交
519

520
	if (test_tsk_need_resched(p))
I
Ingo Molnar 已提交
521 522
		return;

523
	set_tsk_need_resched(p);
I
Ingo Molnar 已提交
524 525

	cpu = task_cpu(p);
526 527
	if (cpu == smp_processor_id()) {
		set_preempt_need_resched();
I
Ingo Molnar 已提交
528
		return;
529
	}
I
Ingo Molnar 已提交
530 531 532 533 534 535 536

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

537
void resched_cpu(int cpu)
I
Ingo Molnar 已提交
538 539 540 541
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

542
	if (!raw_spin_trylock_irqsave(&rq->lock, flags))
I
Ingo Molnar 已提交
543 544
		return;
	resched_task(cpu_curr(cpu));
545
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
546
}
547

548
#ifdef CONFIG_SMP
549
#ifdef CONFIG_NO_HZ_COMMON
550 551 552 553 554 555 556 557 558 559 560 561 562 563
/*
 * In the semi idle case, use the nearest busy cpu for migrating timers
 * from an idle cpu.  This is good for power-savings.
 *
 * We don't do similar optimization for completely idle system, as
 * selecting an idle cpu will add more delays to the timers than intended
 * (as that cpu's timer base may not be uptodate wrt jiffies etc).
 */
int get_nohz_timer_target(void)
{
	int cpu = smp_processor_id();
	int i;
	struct sched_domain *sd;

564
	rcu_read_lock();
565
	for_each_domain(cpu, sd) {
566 567 568 569 570 571
		for_each_cpu(i, sched_domain_span(sd)) {
			if (!idle_cpu(i)) {
				cpu = i;
				goto unlock;
			}
		}
572
	}
573 574
unlock:
	rcu_read_unlock();
575 576
	return cpu;
}
577 578 579 580 581 582 583 584 585 586
/*
 * When add_timer_on() enqueues a timer into the timer wheel of an
 * idle CPU then this timer might expire before the next timer event
 * which is scheduled to wake up that CPU. In case of a completely
 * idle system the next event might even be infinite time into the
 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 * leaves the inner idle loop so the newly added timer is taken into
 * account when the CPU goes back to idle and evaluates the timer
 * wheel for the next timer event.
 */
587
static void wake_up_idle_cpu(int cpu)
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
{
	struct rq *rq = cpu_rq(cpu);

	if (cpu == smp_processor_id())
		return;

	/*
	 * This is safe, as this function is called with the timer
	 * wheel base lock of (cpu) held. When the CPU is on the way
	 * to idle and has not yet set rq->curr to idle then it will
	 * be serialized on the timer wheel base lock and take the new
	 * timer into account automatically.
	 */
	if (rq->curr != rq->idle)
		return;
603 604

	/*
605 606 607
	 * We can set TIF_RESCHED on the idle task of the other CPU
	 * lockless. The worst case is that the other CPU runs the
	 * idle task through an additional NOOP schedule()
608
	 */
609
	set_tsk_need_resched(rq->idle);
610

611 612 613 614
	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(rq->idle))
		smp_send_reschedule(cpu);
615 616
}

617
static bool wake_up_full_nohz_cpu(int cpu)
618
{
619
	if (tick_nohz_full_cpu(cpu)) {
620 621 622 623 624 625 626 627 628 629 630
		if (cpu != smp_processor_id() ||
		    tick_nohz_tick_stopped())
			smp_send_reschedule(cpu);
		return true;
	}

	return false;
}

void wake_up_nohz_cpu(int cpu)
{
631
	if (!wake_up_full_nohz_cpu(cpu))
632 633 634
		wake_up_idle_cpu(cpu);
}

635
static inline bool got_nohz_idle_kick(void)
636
{
637
	int cpu = smp_processor_id();
638 639 640 641 642 643 644 645 646 647 648 649 650

	if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
		return false;

	if (idle_cpu(cpu) && !need_resched())
		return true;

	/*
	 * We can't run Idle Load Balance on this CPU for this time so we
	 * cancel it and clear NOHZ_BALANCE_KICK
	 */
	clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
	return false;
651 652
}

653
#else /* CONFIG_NO_HZ_COMMON */
654

655
static inline bool got_nohz_idle_kick(void)
P
Peter Zijlstra 已提交
656
{
657
	return false;
P
Peter Zijlstra 已提交
658 659
}

660
#endif /* CONFIG_NO_HZ_COMMON */
661

662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
#ifdef CONFIG_NO_HZ_FULL
bool sched_can_stop_tick(void)
{
       struct rq *rq;

       rq = this_rq();

       /* Make sure rq->nr_running update is visible after the IPI */
       smp_rmb();

       /* More than one running task need preemption */
       if (rq->nr_running > 1)
               return false;

       return true;
}
#endif /* CONFIG_NO_HZ_FULL */
679

680
void sched_avg_update(struct rq *rq)
681
{
682 683
	s64 period = sched_avg_period();

684
	while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
685 686 687 688 689 690
		/*
		 * Inline assembly required to prevent the compiler
		 * optimising this loop into a divmod call.
		 * See __iter_div_u64_rem() for another example of this.
		 */
		asm("" : "+rm" (rq->age_stamp));
691 692 693
		rq->age_stamp += period;
		rq->rt_avg /= 2;
	}
694 695
}

696
#endif /* CONFIG_SMP */
697

698 699
#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
700
/*
701 702 703 704
 * Iterate task_group tree rooted at *from, calling @down when first entering a
 * node and @up when leaving it for the final time.
 *
 * Caller must hold rcu_lock or sufficient equivalent.
705
 */
706
int walk_tg_tree_from(struct task_group *from,
707
			     tg_visitor down, tg_visitor up, void *data)
708 709
{
	struct task_group *parent, *child;
P
Peter Zijlstra 已提交
710
	int ret;
711

712 713
	parent = from;

714
down:
P
Peter Zijlstra 已提交
715 716
	ret = (*down)(parent, data);
	if (ret)
717
		goto out;
718 719 720 721 722 723 724
	list_for_each_entry_rcu(child, &parent->children, siblings) {
		parent = child;
		goto down;

up:
		continue;
	}
P
Peter Zijlstra 已提交
725
	ret = (*up)(parent, data);
726 727
	if (ret || parent == from)
		goto out;
728 729 730 731 732

	child = parent;
	parent = parent->parent;
	if (parent)
		goto up;
733
out:
P
Peter Zijlstra 已提交
734
	return ret;
735 736
}

737
int tg_nop(struct task_group *tg, void *data)
P
Peter Zijlstra 已提交
738
{
739
	return 0;
P
Peter Zijlstra 已提交
740
}
741 742
#endif

743 744
static void set_load_weight(struct task_struct *p)
{
N
Nikhil Rao 已提交
745 746 747
	int prio = p->static_prio - MAX_RT_PRIO;
	struct load_weight *load = &p->se.load;

I
Ingo Molnar 已提交
748 749 750 751
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
752
		load->weight = scale_load(WEIGHT_IDLEPRIO);
N
Nikhil Rao 已提交
753
		load->inv_weight = WMULT_IDLEPRIO;
I
Ingo Molnar 已提交
754 755
		return;
	}
756

757
	load->weight = scale_load(prio_to_weight[prio]);
N
Nikhil Rao 已提交
758
	load->inv_weight = prio_to_wmult[prio];
759 760
}

761
static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
762
{
763
	update_rq_clock(rq);
764
	sched_info_queued(rq, p);
765
	p->sched_class->enqueue_task(rq, p, flags);
766 767
}

768
static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
769
{
770
	update_rq_clock(rq);
771
	sched_info_dequeued(rq, p);
772
	p->sched_class->dequeue_task(rq, p, flags);
773 774
}

775
void activate_task(struct rq *rq, struct task_struct *p, int flags)
776 777 778 779
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible--;

780
	enqueue_task(rq, p, flags);
781 782
}

783
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
784 785 786 787
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible++;

788
	dequeue_task(rq, p, flags);
789 790
}

791
static void update_rq_clock_task(struct rq *rq, s64 delta)
792
{
793 794 795 796 797 798 799 800
/*
 * In theory, the compile should just see 0 here, and optimize out the call
 * to sched_rt_avg_update. But I don't trust it...
 */
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
	s64 steal = 0, irq_delta = 0;
#endif
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
801
	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822

	/*
	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
	 * this case when a previous update_rq_clock() happened inside a
	 * {soft,}irq region.
	 *
	 * When this happens, we stop ->clock_task and only update the
	 * prev_irq_time stamp to account for the part that fit, so that a next
	 * update will consume the rest. This ensures ->clock_task is
	 * monotonic.
	 *
	 * It does however cause some slight miss-attribution of {soft,}irq
	 * time, a more accurate solution would be to update the irq_time using
	 * the current rq->clock timestamp, except that would require using
	 * atomic ops.
	 */
	if (irq_delta > delta)
		irq_delta = delta;

	rq->prev_irq_time += irq_delta;
	delta -= irq_delta;
823 824
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
825
	if (static_key_false((&paravirt_steal_rq_enabled))) {
826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
		u64 st;

		steal = paravirt_steal_clock(cpu_of(rq));
		steal -= rq->prev_steal_time_rq;

		if (unlikely(steal > delta))
			steal = delta;

		st = steal_ticks(steal);
		steal = st * TICK_NSEC;

		rq->prev_steal_time_rq += steal;

		delta -= steal;
	}
#endif

843 844
	rq->clock_task += delta;

845 846 847 848
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
	if ((irq_delta + steal) && sched_feat(NONTASK_POWER))
		sched_rt_avg_update(rq, irq_delta + steal);
#endif
849 850
}

851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880
void sched_set_stop_task(int cpu, struct task_struct *stop)
{
	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	struct task_struct *old_stop = cpu_rq(cpu)->stop;

	if (stop) {
		/*
		 * Make it appear like a SCHED_FIFO task, its something
		 * userspace knows about and won't get confused about.
		 *
		 * Also, it will make PI more or less work without too
		 * much confusion -- but then, stop work should not
		 * rely on PI working anyway.
		 */
		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);

		stop->sched_class = &stop_sched_class;
	}

	cpu_rq(cpu)->stop = stop;

	if (old_stop) {
		/*
		 * Reset it back to a normal scheduling class so that
		 * it can die in pieces.
		 */
		old_stop->sched_class = &rt_sched_class;
	}
}

881
/*
I
Ingo Molnar 已提交
882
 * __normal_prio - return the priority that is based on the static prio
883 884 885
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
886
	return p->static_prio;
887 888
}

889 890 891 892 893 894 895
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
896
static inline int normal_prio(struct task_struct *p)
897 898 899
{
	int prio;

900 901 902
	if (task_has_dl_policy(p))
		prio = MAX_DL_PRIO-1;
	else if (task_has_rt_policy(p))
903 904 905 906 907 908 909 910 911 912 913 914 915
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
916
static int effective_prio(struct task_struct *p)
917 918 919 920 921 922 923 924 925 926 927 928
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
929 930 931
/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
932 933
 *
 * Return: 1 if the task is currently executing. 0 otherwise.
L
Linus Torvalds 已提交
934
 */
935
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
936 937 938 939
{
	return cpu_curr(task_cpu(p)) == p;
}

940 941
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
				       const struct sched_class *prev_class,
P
Peter Zijlstra 已提交
942
				       int oldprio)
943 944 945
{
	if (prev_class != p->sched_class) {
		if (prev_class->switched_from)
P
Peter Zijlstra 已提交
946 947
			prev_class->switched_from(rq, p);
		p->sched_class->switched_to(rq, p);
948
	} else if (oldprio != p->prio || dl_task(p))
P
Peter Zijlstra 已提交
949
		p->sched_class->prio_changed(rq, p, oldprio);
950 951
}

952
void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972
{
	const struct sched_class *class;

	if (p->sched_class == rq->curr->sched_class) {
		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
	} else {
		for_each_class(class) {
			if (class == rq->curr->sched_class)
				break;
			if (class == p->sched_class) {
				resched_task(rq->curr);
				break;
			}
		}
	}

	/*
	 * A queue event has occurred, and we're going to schedule.  In
	 * this case, we can save a useless back to back clock update.
	 */
P
Peter Zijlstra 已提交
973
	if (rq->curr->on_rq && test_tsk_need_resched(rq->curr))
974 975 976
		rq->skip_clock_update = 1;
}

L
Linus Torvalds 已提交
977
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
978
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
979
{
980 981 982 983 984
#ifdef CONFIG_SCHED_DEBUG
	/*
	 * We should never call set_task_cpu() on a blocked task,
	 * ttwu() will sort out the placement.
	 */
P
Peter Zijlstra 已提交
985
	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
986
			!(task_preempt_count(p) & PREEMPT_ACTIVE));
987 988

#ifdef CONFIG_LOCKDEP
989 990 991 992 993
	/*
	 * The caller should hold either p->pi_lock or rq->lock, when changing
	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
	 *
	 * sched_move_task() holds both and thus holding either pins the cgroup,
P
Peter Zijlstra 已提交
994
	 * see task_group().
995 996 997 998
	 *
	 * Furthermore, all task_rq users should acquire both locks, see
	 * task_rq_lock().
	 */
999 1000 1001
	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
				      lockdep_is_held(&task_rq(p)->lock)));
#endif
1002 1003
#endif

1004
	trace_sched_migrate_task(p, new_cpu);
1005

1006
	if (task_cpu(p) != new_cpu) {
1007 1008
		if (p->sched_class->migrate_task_rq)
			p->sched_class->migrate_task_rq(p, new_cpu);
1009
		p->se.nr_migrations++;
1010
		perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
1011
	}
I
Ingo Molnar 已提交
1012 1013

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1014 1015
}

1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
static void __migrate_swap_task(struct task_struct *p, int cpu)
{
	if (p->on_rq) {
		struct rq *src_rq, *dst_rq;

		src_rq = task_rq(p);
		dst_rq = cpu_rq(cpu);

		deactivate_task(src_rq, p, 0);
		set_task_cpu(p, cpu);
		activate_task(dst_rq, p, 0);
		check_preempt_curr(dst_rq, p, 0);
	} else {
		/*
		 * Task isn't running anymore; make it appear like we migrated
		 * it before it went to sleep. This means on wakeup we make the
		 * previous cpu our targer instead of where it really is.
		 */
		p->wake_cpu = cpu;
	}
}

struct migration_swap_arg {
	struct task_struct *src_task, *dst_task;
	int src_cpu, dst_cpu;
};

static int migrate_swap_stop(void *data)
{
	struct migration_swap_arg *arg = data;
	struct rq *src_rq, *dst_rq;
	int ret = -EAGAIN;

	src_rq = cpu_rq(arg->src_cpu);
	dst_rq = cpu_rq(arg->dst_cpu);

1052 1053
	double_raw_lock(&arg->src_task->pi_lock,
			&arg->dst_task->pi_lock);
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
	double_rq_lock(src_rq, dst_rq);
	if (task_cpu(arg->dst_task) != arg->dst_cpu)
		goto unlock;

	if (task_cpu(arg->src_task) != arg->src_cpu)
		goto unlock;

	if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
		goto unlock;

	if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
		goto unlock;

	__migrate_swap_task(arg->src_task, arg->dst_cpu);
	__migrate_swap_task(arg->dst_task, arg->src_cpu);

	ret = 0;

unlock:
	double_rq_unlock(src_rq, dst_rq);
1074 1075
	raw_spin_unlock(&arg->dst_task->pi_lock);
	raw_spin_unlock(&arg->src_task->pi_lock);
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097

	return ret;
}

/*
 * Cross migrate two tasks
 */
int migrate_swap(struct task_struct *cur, struct task_struct *p)
{
	struct migration_swap_arg arg;
	int ret = -EINVAL;

	arg = (struct migration_swap_arg){
		.src_task = cur,
		.src_cpu = task_cpu(cur),
		.dst_task = p,
		.dst_cpu = task_cpu(p),
	};

	if (arg.src_cpu == arg.dst_cpu)
		goto out;

1098 1099 1100 1101
	/*
	 * These three tests are all lockless; this is OK since all of them
	 * will be re-checked with proper locks held further down the line.
	 */
1102 1103 1104 1105 1106 1107 1108 1109 1110
	if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
		goto out;

	if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
		goto out;

	if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
		goto out;

1111
	trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
1112 1113 1114 1115 1116 1117
	ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);

out:
	return ret;
}

1118
struct migration_arg {
1119
	struct task_struct *task;
L
Linus Torvalds 已提交
1120
	int dest_cpu;
1121
};
L
Linus Torvalds 已提交
1122

1123 1124
static int migration_cpu_stop(void *data);

L
Linus Torvalds 已提交
1125 1126 1127
/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
R
Roland McGrath 已提交
1128 1129 1130 1131 1132 1133 1134
 * If @match_state is nonzero, it's the @p->state value just checked and
 * not expected to change.  If it changes, i.e. @p might have woken up,
 * then return zero.  When we succeed in waiting for @p to be off its CPU,
 * we return a positive number (its total switch count).  If a second call
 * a short while later returns the same number, the caller can be sure that
 * @p has remained unscheduled the whole time.
 *
L
Linus Torvalds 已提交
1135 1136 1137 1138 1139 1140
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
R
Roland McGrath 已提交
1141
unsigned long wait_task_inactive(struct task_struct *p, long match_state)
L
Linus Torvalds 已提交
1142 1143
{
	unsigned long flags;
I
Ingo Molnar 已提交
1144
	int running, on_rq;
R
Roland McGrath 已提交
1145
	unsigned long ncsw;
1146
	struct rq *rq;
L
Linus Torvalds 已提交
1147

1148 1149 1150 1151 1152 1153 1154 1155
	for (;;) {
		/*
		 * We do the initial early heuristics without holding
		 * any task-queue locks at all. We'll only try to get
		 * the runqueue lock when things look like they will
		 * work out!
		 */
		rq = task_rq(p);
1156

1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
		/*
		 * If the task is actively running on another CPU
		 * still, just relax and busy-wait without holding
		 * any locks.
		 *
		 * NOTE! Since we don't hold any locks, it's not
		 * even sure that "rq" stays as the right runqueue!
		 * But we don't care, since "task_running()" will
		 * return false if the runqueue has changed and p
		 * is actually now running somewhere else!
		 */
R
Roland McGrath 已提交
1168 1169 1170
		while (task_running(rq, p)) {
			if (match_state && unlikely(p->state != match_state))
				return 0;
1171
			cpu_relax();
R
Roland McGrath 已提交
1172
		}
1173

1174 1175 1176 1177 1178 1179
		/*
		 * Ok, time to look more closely! We need the rq
		 * lock now, to be *sure*. If we're wrong, we'll
		 * just go back and repeat.
		 */
		rq = task_rq_lock(p, &flags);
1180
		trace_sched_wait_task(p);
1181
		running = task_running(rq, p);
P
Peter Zijlstra 已提交
1182
		on_rq = p->on_rq;
R
Roland McGrath 已提交
1183
		ncsw = 0;
1184
		if (!match_state || p->state == match_state)
1185
			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1186
		task_rq_unlock(rq, p, &flags);
1187

R
Roland McGrath 已提交
1188 1189 1190 1191 1192 1193
		/*
		 * If it changed from the expected state, bail out now.
		 */
		if (unlikely(!ncsw))
			break;

1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
		/*
		 * Was it really running after all now that we
		 * checked with the proper locks actually held?
		 *
		 * Oops. Go back and try again..
		 */
		if (unlikely(running)) {
			cpu_relax();
			continue;
		}
1204

1205 1206 1207 1208 1209
		/*
		 * It's not enough that it's not actively running,
		 * it must be off the runqueue _entirely_, and not
		 * preempted!
		 *
1210
		 * So if it was still runnable (but just not actively
1211 1212 1213 1214
		 * running right now), it's preempted, and we should
		 * yield - it could be a while.
		 */
		if (unlikely(on_rq)) {
1215 1216 1217 1218
			ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);

			set_current_state(TASK_UNINTERRUPTIBLE);
			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1219 1220
			continue;
		}
1221

1222 1223 1224 1225 1226 1227 1228
		/*
		 * Ahh, all good. It wasn't running, and it wasn't
		 * runnable, which means that it will never become
		 * running in the future either. We're all done!
		 */
		break;
	}
R
Roland McGrath 已提交
1229 1230

	return ncsw;
L
Linus Torvalds 已提交
1231 1232 1233 1234 1235 1236 1237 1238 1239
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
L
Lucas De Marchi 已提交
1240
 * NOTE: this function doesn't have to take the runqueue lock,
L
Linus Torvalds 已提交
1241 1242 1243 1244 1245
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
1246
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
1247 1248 1249 1250 1251 1252 1253 1254 1255
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}
R
Rusty Russell 已提交
1256
EXPORT_SYMBOL_GPL(kick_process);
N
Nick Piggin 已提交
1257
#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
1258

1259
#ifdef CONFIG_SMP
1260
/*
1261
 * ->cpus_allowed is protected by both rq->lock and p->pi_lock
1262
 */
1263 1264
static int select_fallback_rq(int cpu, struct task_struct *p)
{
1265 1266
	int nid = cpu_to_node(cpu);
	const struct cpumask *nodemask = NULL;
1267 1268
	enum { cpuset, possible, fail } state = cpuset;
	int dest_cpu;
1269

1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
	/*
	 * If the node that the cpu is on has been offlined, cpu_to_node()
	 * will return -1. There is no cpu on the node, and we should
	 * select the cpu on the other node.
	 */
	if (nid != -1) {
		nodemask = cpumask_of_node(nid);

		/* Look for allowed, online CPU in same node. */
		for_each_cpu(dest_cpu, nodemask) {
			if (!cpu_online(dest_cpu))
				continue;
			if (!cpu_active(dest_cpu))
				continue;
			if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
				return dest_cpu;
		}
1287
	}
1288

1289 1290
	for (;;) {
		/* Any allowed, online CPU? */
1291
		for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
1292 1293 1294 1295 1296 1297
			if (!cpu_online(dest_cpu))
				continue;
			if (!cpu_active(dest_cpu))
				continue;
			goto out;
		}
1298

1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
		switch (state) {
		case cpuset:
			/* No more Mr. Nice Guy. */
			cpuset_cpus_allowed_fallback(p);
			state = possible;
			break;

		case possible:
			do_set_cpus_allowed(p, cpu_possible_mask);
			state = fail;
			break;

		case fail:
			BUG();
			break;
		}
	}

out:
	if (state != cpuset) {
		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
		if (p->mm && printk_ratelimit()) {
			printk_sched("process %d (%s) no longer affine to cpu%d\n",
					task_pid_nr(p), p->comm, cpu);
		}
1328 1329 1330 1331 1332
	}

	return dest_cpu;
}

1333
/*
1334
 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
1335
 */
1336
static inline
1337
int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
1338
{
1339
	cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350

	/*
	 * In order not to call set_task_cpu() on a blocking task we need
	 * to rely on ttwu() to place the task on a valid ->cpus_allowed
	 * cpu.
	 *
	 * Since this is common to all placement strategies, this lives here.
	 *
	 * [ this allows ->select_task() to simply return task_cpu(p) and
	 *   not worry about this generic constraint ]
	 */
1351
	if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
P
Peter Zijlstra 已提交
1352
		     !cpu_online(cpu)))
1353
		cpu = select_fallback_rq(task_cpu(p), p);
1354 1355

	return cpu;
1356
}
1357 1358 1359 1360 1361 1362

static void update_avg(u64 *avg, u64 sample)
{
	s64 diff = sample - *avg;
	*avg += diff >> 3;
}
1363 1364
#endif

P
Peter Zijlstra 已提交
1365
static void
1366
ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
T
Tejun Heo 已提交
1367
{
P
Peter Zijlstra 已提交
1368
#ifdef CONFIG_SCHEDSTATS
1369 1370
	struct rq *rq = this_rq();

P
Peter Zijlstra 已提交
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
#ifdef CONFIG_SMP
	int this_cpu = smp_processor_id();

	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
		schedstat_inc(p, se.statistics.nr_wakeups_local);
	} else {
		struct sched_domain *sd;

		schedstat_inc(p, se.statistics.nr_wakeups_remote);
1381
		rcu_read_lock();
P
Peter Zijlstra 已提交
1382 1383 1384 1385 1386 1387
		for_each_domain(this_cpu, sd) {
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
				schedstat_inc(sd, ttwu_wake_remote);
				break;
			}
		}
1388
		rcu_read_unlock();
P
Peter Zijlstra 已提交
1389
	}
1390 1391 1392 1393

	if (wake_flags & WF_MIGRATED)
		schedstat_inc(p, se.statistics.nr_wakeups_migrate);

P
Peter Zijlstra 已提交
1394 1395 1396
#endif /* CONFIG_SMP */

	schedstat_inc(rq, ttwu_count);
T
Tejun Heo 已提交
1397
	schedstat_inc(p, se.statistics.nr_wakeups);
P
Peter Zijlstra 已提交
1398 1399

	if (wake_flags & WF_SYNC)
T
Tejun Heo 已提交
1400
		schedstat_inc(p, se.statistics.nr_wakeups_sync);
P
Peter Zijlstra 已提交
1401 1402 1403 1404 1405 1406

#endif /* CONFIG_SCHEDSTATS */
}

static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
{
T
Tejun Heo 已提交
1407
	activate_task(rq, p, en_flags);
P
Peter Zijlstra 已提交
1408
	p->on_rq = 1;
1409 1410 1411 1412

	/* if a worker is waking up, notify workqueue */
	if (p->flags & PF_WQ_WORKER)
		wq_worker_waking_up(p, cpu_of(rq));
T
Tejun Heo 已提交
1413 1414
}

1415 1416 1417
/*
 * Mark the task runnable and perform wakeup-preemption.
 */
1418
static void
1419
ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
T
Tejun Heo 已提交
1420 1421
{
	check_preempt_curr(rq, p, wake_flags);
1422
	trace_sched_wakeup(p, true);
T
Tejun Heo 已提交
1423 1424 1425 1426 1427 1428

	p->state = TASK_RUNNING;
#ifdef CONFIG_SMP
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);

1429
	if (rq->idle_stamp) {
1430
		u64 delta = rq_clock(rq) - rq->idle_stamp;
1431
		u64 max = 2*rq->max_idle_balance_cost;
T
Tejun Heo 已提交
1432

1433 1434 1435
		update_avg(&rq->avg_idle, delta);

		if (rq->avg_idle > max)
T
Tejun Heo 已提交
1436
			rq->avg_idle = max;
1437

T
Tejun Heo 已提交
1438 1439 1440 1441 1442
		rq->idle_stamp = 0;
	}
#endif
}

1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
static void
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
{
#ifdef CONFIG_SMP
	if (p->sched_contributes_to_load)
		rq->nr_uninterruptible--;
#endif

	ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
	ttwu_do_wakeup(rq, p, wake_flags);
}

/*
 * Called in case the task @p isn't fully descheduled from its runqueue,
 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
 * since all we need to do is flip p->state to TASK_RUNNING, since
 * the task is still ->on_rq.
 */
static int ttwu_remote(struct task_struct *p, int wake_flags)
{
	struct rq *rq;
	int ret = 0;

	rq = __task_rq_lock(p);
	if (p->on_rq) {
1468 1469
		/* check_preempt_curr() may use rq clock */
		update_rq_clock(rq);
1470 1471 1472 1473 1474 1475 1476 1477
		ttwu_do_wakeup(rq, p, wake_flags);
		ret = 1;
	}
	__task_rq_unlock(rq);

	return ret;
}

1478
#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
1479
static void sched_ttwu_pending(void)
1480 1481
{
	struct rq *rq = this_rq();
P
Peter Zijlstra 已提交
1482 1483
	struct llist_node *llist = llist_del_all(&rq->wake_list);
	struct task_struct *p;
1484 1485 1486

	raw_spin_lock(&rq->lock);

P
Peter Zijlstra 已提交
1487 1488 1489
	while (llist) {
		p = llist_entry(llist, struct task_struct, wake_entry);
		llist = llist_next(llist);
1490 1491 1492 1493 1494 1495 1496 1497
		ttwu_do_activate(rq, p, 0);
	}

	raw_spin_unlock(&rq->lock);
}

void scheduler_ipi(void)
{
1498 1499 1500 1501 1502
	/*
	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
	 * TIF_NEED_RESCHED remotely (for the first time) will also send
	 * this IPI.
	 */
1503
	preempt_fold_need_resched();
1504

1505 1506 1507
	if (llist_empty(&this_rq()->wake_list)
			&& !tick_nohz_full_cpu(smp_processor_id())
			&& !got_nohz_idle_kick())
1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
		return;

	/*
	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
	 * traditionally all their work was done from the interrupt return
	 * path. Now that we actually do some work, we need to make sure
	 * we do call them.
	 *
	 * Some archs already do call them, luckily irq_enter/exit nest
	 * properly.
	 *
	 * Arguably we should visit all archs and update all handlers,
	 * however a fair share of IPIs are still resched only so this would
	 * somewhat pessimize the simple resched case.
	 */
	irq_enter();
1524
	tick_nohz_full_check();
P
Peter Zijlstra 已提交
1525
	sched_ttwu_pending();
1526 1527 1528 1529

	/*
	 * Check if someone kicked us for doing the nohz idle load balance.
	 */
1530
	if (unlikely(got_nohz_idle_kick())) {
1531
		this_rq()->idle_balance = 1;
1532
		raise_softirq_irqoff(SCHED_SOFTIRQ);
1533
	}
1534
	irq_exit();
1535 1536 1537 1538
}

static void ttwu_queue_remote(struct task_struct *p, int cpu)
{
P
Peter Zijlstra 已提交
1539
	if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list))
1540 1541
		smp_send_reschedule(cpu);
}
1542

1543
bool cpus_share_cache(int this_cpu, int that_cpu)
1544 1545 1546
{
	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
}
1547
#endif /* CONFIG_SMP */
1548

1549 1550 1551 1552
static void ttwu_queue(struct task_struct *p, int cpu)
{
	struct rq *rq = cpu_rq(cpu);

1553
#if defined(CONFIG_SMP)
1554
	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
1555
		sched_clock_cpu(cpu); /* sync clocks x-cpu */
1556 1557 1558 1559 1560
		ttwu_queue_remote(p, cpu);
		return;
	}
#endif

1561 1562 1563
	raw_spin_lock(&rq->lock);
	ttwu_do_activate(rq, p, 0);
	raw_spin_unlock(&rq->lock);
T
Tejun Heo 已提交
1564 1565 1566
}

/**
L
Linus Torvalds 已提交
1567
 * try_to_wake_up - wake up a thread
T
Tejun Heo 已提交
1568
 * @p: the thread to be awakened
L
Linus Torvalds 已提交
1569
 * @state: the mask of task states that can be woken
T
Tejun Heo 已提交
1570
 * @wake_flags: wake modifier flags (WF_*)
L
Linus Torvalds 已提交
1571 1572 1573 1574 1575 1576 1577
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
1578
 * Return: %true if @p was woken up, %false if it was already running.
T
Tejun Heo 已提交
1579
 * or @state didn't match @p's state.
L
Linus Torvalds 已提交
1580
 */
1581 1582
static int
try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
L
Linus Torvalds 已提交
1583 1584
{
	unsigned long flags;
1585
	int cpu, success = 0;
P
Peter Zijlstra 已提交
1586

1587 1588 1589 1590 1591 1592 1593
	/*
	 * If we are going to wake up a thread waiting for CONDITION we
	 * need to ensure that CONDITION=1 done by the caller can not be
	 * reordered with p->state check below. This pairs with mb() in
	 * set_current_state() the waiting thread does.
	 */
	smp_mb__before_spinlock();
1594
	raw_spin_lock_irqsave(&p->pi_lock, flags);
P
Peter Zijlstra 已提交
1595
	if (!(p->state & state))
L
Linus Torvalds 已提交
1596 1597
		goto out;

1598
	success = 1; /* we're going to change ->state */
L
Linus Torvalds 已提交
1599 1600
	cpu = task_cpu(p);

1601 1602
	if (p->on_rq && ttwu_remote(p, wake_flags))
		goto stat;
L
Linus Torvalds 已提交
1603 1604

#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
1605
	/*
1606 1607
	 * If the owning (remote) cpu is still in the middle of schedule() with
	 * this task as prev, wait until its done referencing the task.
P
Peter Zijlstra 已提交
1608
	 */
1609
	while (p->on_cpu)
1610
		cpu_relax();
1611
	/*
1612
	 * Pairs with the smp_wmb() in finish_lock_switch().
1613
	 */
1614
	smp_rmb();
L
Linus Torvalds 已提交
1615

1616
	p->sched_contributes_to_load = !!task_contributes_to_load(p);
P
Peter Zijlstra 已提交
1617
	p->state = TASK_WAKING;
1618

1619
	if (p->sched_class->task_waking)
1620
		p->sched_class->task_waking(p);
1621

1622
	cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
1623 1624
	if (task_cpu(p) != cpu) {
		wake_flags |= WF_MIGRATED;
1625
		set_task_cpu(p, cpu);
1626
	}
L
Linus Torvalds 已提交
1627 1628
#endif /* CONFIG_SMP */

1629 1630
	ttwu_queue(p, cpu);
stat:
1631
	ttwu_stat(p, cpu, wake_flags);
L
Linus Torvalds 已提交
1632
out:
1633
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
1634 1635 1636 1637

	return success;
}

T
Tejun Heo 已提交
1638 1639 1640 1641
/**
 * try_to_wake_up_local - try to wake up a local task with rq lock held
 * @p: the thread to be awakened
 *
1642
 * Put @p on the run-queue if it's not already there. The caller must
T
Tejun Heo 已提交
1643
 * ensure that this_rq() is locked, @p is bound to this_rq() and not
1644
 * the current task.
T
Tejun Heo 已提交
1645 1646 1647 1648 1649
 */
static void try_to_wake_up_local(struct task_struct *p)
{
	struct rq *rq = task_rq(p);

1650 1651 1652 1653
	if (WARN_ON_ONCE(rq != this_rq()) ||
	    WARN_ON_ONCE(p == current))
		return;

T
Tejun Heo 已提交
1654 1655
	lockdep_assert_held(&rq->lock);

1656 1657 1658 1659 1660 1661
	if (!raw_spin_trylock(&p->pi_lock)) {
		raw_spin_unlock(&rq->lock);
		raw_spin_lock(&p->pi_lock);
		raw_spin_lock(&rq->lock);
	}

T
Tejun Heo 已提交
1662
	if (!(p->state & TASK_NORMAL))
1663
		goto out;
T
Tejun Heo 已提交
1664

P
Peter Zijlstra 已提交
1665
	if (!p->on_rq)
P
Peter Zijlstra 已提交
1666 1667
		ttwu_activate(rq, p, ENQUEUE_WAKEUP);

1668
	ttwu_do_wakeup(rq, p, 0);
1669
	ttwu_stat(p, smp_processor_id(), 0);
1670 1671
out:
	raw_spin_unlock(&p->pi_lock);
T
Tejun Heo 已提交
1672 1673
}

1674 1675 1676 1677 1678
/**
 * wake_up_process - Wake up a specific process
 * @p: The process to be woken up.
 *
 * Attempt to wake up the nominated process and move it to the set of runnable
1679 1680 1681
 * processes.
 *
 * Return: 1 if the process was woken up, 0 if it was already running.
1682 1683 1684 1685
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
 */
1686
int wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
1687
{
1688 1689
	WARN_ON(task_is_stopped_or_traced(p));
	return try_to_wake_up(p, TASK_NORMAL, 0);
L
Linus Torvalds 已提交
1690 1691 1692
}
EXPORT_SYMBOL(wake_up_process);

1693
int wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
1694 1695 1696 1697 1698 1699 1700
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
1701 1702 1703
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
1704
static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
I
Ingo Molnar 已提交
1705
{
P
Peter Zijlstra 已提交
1706 1707 1708
	p->on_rq			= 0;

	p->se.on_rq			= 0;
I
Ingo Molnar 已提交
1709 1710
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
1711
	p->se.prev_sum_exec_runtime	= 0;
1712
	p->se.nr_migrations		= 0;
P
Peter Zijlstra 已提交
1713
	p->se.vruntime			= 0;
P
Peter Zijlstra 已提交
1714
	INIT_LIST_HEAD(&p->se.group_node);
I
Ingo Molnar 已提交
1715 1716

#ifdef CONFIG_SCHEDSTATS
1717
	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
I
Ingo Molnar 已提交
1718
#endif
N
Nick Piggin 已提交
1719

1720 1721 1722 1723
	RB_CLEAR_NODE(&p->dl.rb_node);
	hrtimer_init(&p->dl.dl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	p->dl.dl_runtime = p->dl.runtime = 0;
	p->dl.dl_deadline = p->dl.deadline = 0;
1724
	p->dl.dl_period = 0;
1725 1726
	p->dl.flags = 0;

P
Peter Zijlstra 已提交
1727
	INIT_LIST_HEAD(&p->rt.run_list);
N
Nick Piggin 已提交
1728

1729 1730 1731
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif
1732 1733 1734

#ifdef CONFIG_NUMA_BALANCING
	if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
1735
		p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
1736 1737 1738
		p->mm->numa_scan_seq = 0;
	}

1739 1740 1741 1742 1743
	if (clone_flags & CLONE_VM)
		p->numa_preferred_nid = current->numa_preferred_nid;
	else
		p->numa_preferred_nid = -1;

1744 1745
	p->node_stamp = 0ULL;
	p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
1746
	p->numa_scan_period = sysctl_numa_balancing_scan_delay;
1747
	p->numa_work.next = &p->numa_work;
1748 1749
	p->numa_faults_memory = NULL;
	p->numa_faults_buffer_memory = NULL;
1750 1751
	p->last_task_numa_placement = 0;
	p->last_sum_exec_runtime = 0;
1752 1753 1754

	INIT_LIST_HEAD(&p->numa_entry);
	p->numa_group = NULL;
1755
#endif /* CONFIG_NUMA_BALANCING */
I
Ingo Molnar 已提交
1756 1757
}

1758
#ifdef CONFIG_NUMA_BALANCING
1759
#ifdef CONFIG_SCHED_DEBUG
1760 1761 1762 1763 1764 1765 1766
void set_numabalancing_state(bool enabled)
{
	if (enabled)
		sched_feat_set("NUMA");
	else
		sched_feat_set("NO_NUMA");
}
1767 1768 1769 1770 1771 1772
#else
__read_mostly bool numabalancing_enabled;

void set_numabalancing_state(bool enabled)
{
	numabalancing_enabled = enabled;
I
Ingo Molnar 已提交
1773
}
1774
#endif /* CONFIG_SCHED_DEBUG */
1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797

#ifdef CONFIG_PROC_SYSCTL
int sysctl_numa_balancing(struct ctl_table *table, int write,
			 void __user *buffer, size_t *lenp, loff_t *ppos)
{
	struct ctl_table t;
	int err;
	int state = numabalancing_enabled;

	if (write && !capable(CAP_SYS_ADMIN))
		return -EPERM;

	t = *table;
	t.data = &state;
	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
	if (err < 0)
		return err;
	if (write)
		set_numabalancing_state(state);
	return err;
}
#endif
#endif
I
Ingo Molnar 已提交
1798 1799 1800 1801

/*
 * fork()/clone()-time setup:
 */
1802
int sched_fork(unsigned long clone_flags, struct task_struct *p)
I
Ingo Molnar 已提交
1803
{
1804
	unsigned long flags;
I
Ingo Molnar 已提交
1805 1806
	int cpu = get_cpu();

1807
	__sched_fork(clone_flags, p);
1808
	/*
1809
	 * We mark the process as running here. This guarantees that
1810 1811 1812
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
1813
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
1814

1815 1816 1817 1818 1819
	/*
	 * Make sure we do not leak PI boosting priority to the child.
	 */
	p->prio = current->normal_prio;

1820 1821 1822 1823
	/*
	 * Revert to default priority/policy on fork if requested.
	 */
	if (unlikely(p->sched_reset_on_fork)) {
1824
		if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
1825
			p->policy = SCHED_NORMAL;
1826
			p->static_prio = NICE_TO_PRIO(0);
1827 1828 1829 1830 1831 1832
			p->rt_priority = 0;
		} else if (PRIO_TO_NICE(p->static_prio) < 0)
			p->static_prio = NICE_TO_PRIO(0);

		p->prio = p->normal_prio = __normal_prio(p);
		set_load_weight(p);
1833

1834 1835 1836 1837 1838 1839
		/*
		 * We don't need the reset flag anymore after the fork. It has
		 * fulfilled its duty:
		 */
		p->sched_reset_on_fork = 0;
	}
1840

1841 1842 1843 1844 1845 1846
	if (dl_prio(p->prio)) {
		put_cpu();
		return -EAGAIN;
	} else if (rt_prio(p->prio)) {
		p->sched_class = &rt_sched_class;
	} else {
H
Hiroshi Shimamoto 已提交
1847
		p->sched_class = &fair_sched_class;
1848
	}
1849

P
Peter Zijlstra 已提交
1850 1851 1852
	if (p->sched_class->task_fork)
		p->sched_class->task_fork(p);

1853 1854 1855 1856 1857 1858 1859
	/*
	 * The child is not yet in the pid-hash so no cgroup attach races,
	 * and the cgroup is pinned to this child due to cgroup_fork()
	 * is ran before sched_fork().
	 *
	 * Silence PROVE_RCU.
	 */
1860
	raw_spin_lock_irqsave(&p->pi_lock, flags);
1861
	set_task_cpu(p, cpu);
1862
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1863

1864
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
1865
	if (likely(sched_info_on()))
1866
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
1867
#endif
P
Peter Zijlstra 已提交
1868 1869
#if defined(CONFIG_SMP)
	p->on_cpu = 0;
1870
#endif
1871
	init_task_preempt_count(p);
1872
#ifdef CONFIG_SMP
1873
	plist_node_init(&p->pushable_tasks, MAX_PRIO);
1874
	RB_CLEAR_NODE(&p->pushable_dl_tasks);
1875
#endif
1876

N
Nick Piggin 已提交
1877
	put_cpu();
1878
	return 0;
L
Linus Torvalds 已提交
1879 1880
}

1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902
unsigned long to_ratio(u64 period, u64 runtime)
{
	if (runtime == RUNTIME_INF)
		return 1ULL << 20;

	/*
	 * Doing this here saves a lot of checks in all
	 * the calling paths, and returning zero seems
	 * safe for them anyway.
	 */
	if (period == 0)
		return 0;

	return div64_u64(runtime << 20, period);
}

#ifdef CONFIG_SMP
inline struct dl_bw *dl_bw_of(int i)
{
	return &cpu_rq(i)->rd->dl_bw;
}

1903
static inline int dl_bw_cpus(int i)
1904
{
1905 1906 1907 1908 1909 1910 1911
	struct root_domain *rd = cpu_rq(i)->rd;
	int cpus = 0;

	for_each_cpu_and(i, rd->span, cpu_active_mask)
		cpus++;

	return cpus;
1912 1913 1914 1915 1916 1917 1918
}
#else
inline struct dl_bw *dl_bw_of(int i)
{
	return &cpu_rq(i)->dl.dl_bw;
}

1919
static inline int dl_bw_cpus(int i)
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959
{
	return 1;
}
#endif

static inline
void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw)
{
	dl_b->total_bw -= tsk_bw;
}

static inline
void __dl_add(struct dl_bw *dl_b, u64 tsk_bw)
{
	dl_b->total_bw += tsk_bw;
}

static inline
bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw)
{
	return dl_b->bw != -1 &&
	       dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw;
}

/*
 * We must be sure that accepting a new task (or allowing changing the
 * parameters of an existing one) is consistent with the bandwidth
 * constraints. If yes, this function also accordingly updates the currently
 * allocated bandwidth to reflect the new situation.
 *
 * This function is called while holding p's rq->lock.
 */
static int dl_overflow(struct task_struct *p, int policy,
		       const struct sched_attr *attr)
{

	struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
	u64 period = attr->sched_period;
	u64 runtime = attr->sched_runtime;
	u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
1960
	int cpus, err = -1;
1961 1962 1963 1964 1965 1966 1967 1968 1969 1970

	if (new_bw == p->dl.dl_bw)
		return 0;

	/*
	 * Either if a task, enters, leave, or stays -deadline but changes
	 * its parameters, we may need to update accordingly the total
	 * allocated bandwidth of the container.
	 */
	raw_spin_lock(&dl_b->lock);
1971
	cpus = dl_bw_cpus(task_cpu(p));
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991
	if (dl_policy(policy) && !task_has_dl_policy(p) &&
	    !__dl_overflow(dl_b, cpus, 0, new_bw)) {
		__dl_add(dl_b, new_bw);
		err = 0;
	} else if (dl_policy(policy) && task_has_dl_policy(p) &&
		   !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
		__dl_clear(dl_b, p->dl.dl_bw);
		__dl_add(dl_b, new_bw);
		err = 0;
	} else if (!dl_policy(policy) && task_has_dl_policy(p)) {
		__dl_clear(dl_b, p->dl.dl_bw);
		err = 0;
	}
	raw_spin_unlock(&dl_b->lock);

	return err;
}

extern void init_dl_bw(struct dl_bw *dl_b);

L
Linus Torvalds 已提交
1992 1993 1994 1995 1996 1997 1998
/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
1999
void wake_up_new_task(struct task_struct *p)
L
Linus Torvalds 已提交
2000 2001
{
	unsigned long flags;
I
Ingo Molnar 已提交
2002
	struct rq *rq;
2003

2004
	raw_spin_lock_irqsave(&p->pi_lock, flags);
2005 2006 2007 2008 2009 2010
#ifdef CONFIG_SMP
	/*
	 * Fork balancing, do it here and not earlier because:
	 *  - cpus_allowed can change in the fork path
	 *  - any previously selected cpu might disappear through hotplug
	 */
2011
	set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
2012 2013
#endif

2014 2015
	/* Initialize new task's runnable average */
	init_task_runnable_average(p);
2016
	rq = __task_rq_lock(p);
P
Peter Zijlstra 已提交
2017
	activate_task(rq, p, 0);
P
Peter Zijlstra 已提交
2018
	p->on_rq = 1;
2019
	trace_sched_wakeup_new(p, true);
P
Peter Zijlstra 已提交
2020
	check_preempt_curr(rq, p, WF_FORK);
2021
#ifdef CONFIG_SMP
2022 2023
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);
2024
#endif
2025
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
2026 2027
}

2028 2029 2030
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
2031
 * preempt_notifier_register - tell me when current is being preempted & rescheduled
R
Randy Dunlap 已提交
2032
 * @notifier: notifier struct to register
2033 2034 2035 2036 2037 2038 2039 2040 2041
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
2042
 * @notifier: notifier struct to unregister
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;

2056
	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2057 2058 2059 2060 2061 2062 2063 2064 2065
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;

2066
	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2067 2068 2069
		notifier->ops->sched_out(notifier, next);
}

2070
#else /* !CONFIG_PREEMPT_NOTIFIERS */
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

2082
#endif /* CONFIG_PREEMPT_NOTIFIERS */
2083

2084 2085 2086
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
2087
 * @prev: the current task that is being switched out
2088 2089 2090 2091 2092 2093 2094 2095 2096
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
2097 2098 2099
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
2100
{
2101
	trace_sched_switch(prev, next);
2102
	sched_info_switch(rq, prev, next);
2103
	perf_event_task_sched_out(prev, next);
2104
	fire_sched_out_preempt_notifiers(prev, next);
2105 2106 2107 2108
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
2109 2110
/**
 * finish_task_switch - clean up after a task-switch
2111
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
2112 2113
 * @prev: the thread we just switched away from.
 *
2114 2115 2116 2117
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
2118 2119
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
I
Ingo Molnar 已提交
2120
 * so, we finish that here outside of the runqueue lock. (Doing it
L
Linus Torvalds 已提交
2121 2122 2123
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
A
Alexey Dobriyan 已提交
2124
static void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
2125 2126 2127
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
2128
	long prev_state;
L
Linus Torvalds 已提交
2129 2130 2131 2132 2133

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
2134
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
2135 2136
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
2137
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
2138 2139 2140 2141 2142
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
2143
	prev_state = prev->state;
2144
	vtime_task_switch(prev);
2145
	finish_arch_switch(prev);
2146
	perf_event_task_sched_in(prev, current);
2147
	finish_lock_switch(rq, prev);
2148
	finish_arch_post_lock_switch();
S
Steven Rostedt 已提交
2149

2150
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
2151 2152
	if (mm)
		mmdrop(mm);
2153
	if (unlikely(prev_state == TASK_DEAD)) {
2154 2155
		task_numa_free(prev);

2156 2157 2158
		if (prev->sched_class->task_dead)
			prev->sched_class->task_dead(prev);

2159 2160 2161
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
2162
		 */
2163
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
2164
		put_task_struct(prev);
2165
	}
2166 2167

	tick_nohz_task_switch(current);
L
Linus Torvalds 已提交
2168 2169
}

2170 2171 2172 2173 2174 2175 2176 2177
#ifdef CONFIG_SMP

/* rq->lock is NOT held, but preemption is disabled */
static inline void post_schedule(struct rq *rq)
{
	if (rq->post_schedule) {
		unsigned long flags;

2178
		raw_spin_lock_irqsave(&rq->lock, flags);
2179 2180
		if (rq->curr->sched_class->post_schedule)
			rq->curr->sched_class->post_schedule(rq);
2181
		raw_spin_unlock_irqrestore(&rq->lock, flags);
2182 2183 2184 2185 2186 2187

		rq->post_schedule = 0;
	}
}

#else
2188

2189 2190
static inline void post_schedule(struct rq *rq)
{
L
Linus Torvalds 已提交
2191 2192
}

2193 2194
#endif

L
Linus Torvalds 已提交
2195 2196 2197 2198
/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
2199
asmlinkage void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
2200 2201
	__releases(rq->lock)
{
2202 2203
	struct rq *rq = this_rq();

2204
	finish_task_switch(rq, prev);
2205

2206 2207 2208 2209 2210
	/*
	 * FIXME: do we need to worry about rq being invalidated by the
	 * task_switch?
	 */
	post_schedule(rq);
2211

2212 2213 2214 2215
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
2216
	if (current->set_child_tid)
2217
		put_user(task_pid_vnr(current), current->set_child_tid);
L
Linus Torvalds 已提交
2218 2219 2220 2221 2222 2223
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
2224
static inline void
2225
context_switch(struct rq *rq, struct task_struct *prev,
2226
	       struct task_struct *next)
L
Linus Torvalds 已提交
2227
{
I
Ingo Molnar 已提交
2228
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
2229

2230
	prepare_task_switch(rq, prev, next);
2231

I
Ingo Molnar 已提交
2232 2233
	mm = next->mm;
	oldmm = prev->active_mm;
2234 2235 2236 2237 2238
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
2239
	arch_start_context_switch(prev);
2240

2241
	if (!mm) {
L
Linus Torvalds 已提交
2242 2243 2244 2245 2246 2247
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

2248
	if (!prev->mm) {
L
Linus Torvalds 已提交
2249 2250 2251
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
2252 2253 2254 2255 2256 2257 2258
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2259
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2260
#endif
L
Linus Torvalds 已提交
2261

2262
	context_tracking_task_switch(prev, next);
L
Linus Torvalds 已提交
2263 2264 2265
	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
2266 2267 2268 2269 2270 2271 2272
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
2273 2274 2275
}

/*
2276
 * nr_running and nr_context_switches:
L
Linus Torvalds 已提交
2277 2278
 *
 * externally visible scheduler statistics: current number of runnable
2279
 * threads, total number of context switches performed since bootup.
L
Linus Torvalds 已提交
2280 2281 2282 2283 2284 2285 2286 2287 2288
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
2289
}
L
Linus Torvalds 已提交
2290 2291

unsigned long long nr_context_switches(void)
2292
{
2293 2294
	int i;
	unsigned long long sum = 0;
2295

2296
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2297
		sum += cpu_rq(i)->nr_switches;
2298

L
Linus Torvalds 已提交
2299 2300
	return sum;
}
2301

L
Linus Torvalds 已提交
2302 2303 2304
unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;
2305

2306
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2307
		sum += atomic_read(&cpu_rq(i)->nr_iowait);
2308

L
Linus Torvalds 已提交
2309 2310
	return sum;
}
2311

2312
unsigned long nr_iowait_cpu(int cpu)
2313
{
2314
	struct rq *this = cpu_rq(cpu);
2315 2316
	return atomic_read(&this->nr_iowait);
}
2317

I
Ingo Molnar 已提交
2318
#ifdef CONFIG_SMP
2319

2320
/*
P
Peter Zijlstra 已提交
2321 2322
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
2323
 */
P
Peter Zijlstra 已提交
2324
void sched_exec(void)
2325
{
P
Peter Zijlstra 已提交
2326
	struct task_struct *p = current;
L
Linus Torvalds 已提交
2327
	unsigned long flags;
2328
	int dest_cpu;
2329

2330
	raw_spin_lock_irqsave(&p->pi_lock, flags);
2331
	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
2332 2333
	if (dest_cpu == smp_processor_id())
		goto unlock;
P
Peter Zijlstra 已提交
2334

2335
	if (likely(cpu_active(dest_cpu))) {
2336
		struct migration_arg arg = { p, dest_cpu };
2337

2338 2339
		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
2340 2341
		return;
	}
2342
unlock:
2343
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
2344
}
I
Ingo Molnar 已提交
2345

L
Linus Torvalds 已提交
2346 2347 2348
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);
2349
DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
L
Linus Torvalds 已提交
2350 2351

EXPORT_PER_CPU_SYMBOL(kstat);
2352
EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
L
Linus Torvalds 已提交
2353 2354

/*
2355
 * Return any ns on the sched_clock that have not yet been accounted in
2356
 * @p in case that task is currently running.
2357 2358
 *
 * Called with task_rq_lock() held on @rq.
L
Linus Torvalds 已提交
2359
 */
2360 2361 2362 2363 2364 2365
static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
{
	u64 ns = 0;

	if (task_current(rq, p)) {
		update_rq_clock(rq);
2366
		ns = rq_clock_task(rq) - p->se.exec_start;
2367 2368 2369 2370 2371 2372 2373
		if ((s64)ns < 0)
			ns = 0;
	}

	return ns;
}

2374
unsigned long long task_delta_exec(struct task_struct *p)
L
Linus Torvalds 已提交
2375 2376
{
	unsigned long flags;
2377
	struct rq *rq;
2378
	u64 ns = 0;
2379

2380
	rq = task_rq_lock(p, &flags);
2381
	ns = do_task_delta_exec(p, rq);
2382
	task_rq_unlock(rq, p, &flags);
2383

2384 2385
	return ns;
}
2386

2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397
/*
 * Return accounted runtime for the task.
 * In case the task is currently running, return the runtime plus current's
 * pending runtime that have not been accounted yet.
 */
unsigned long long task_sched_runtime(struct task_struct *p)
{
	unsigned long flags;
	struct rq *rq;
	u64 ns = 0;

2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411
#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
	/*
	 * 64-bit doesn't need locks to atomically read a 64bit value.
	 * So we have a optimization chance when the task's delta_exec is 0.
	 * Reading ->on_cpu is racy, but this is ok.
	 *
	 * If we race with it leaving cpu, we'll take a lock. So we're correct.
	 * If we race with it entering cpu, unaccounted time is 0. This is
	 * indistinguishable from the read occurring a few cycles earlier.
	 */
	if (!p->on_cpu)
		return p->se.sum_exec_runtime;
#endif

2412 2413
	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
2414
	task_rq_unlock(rq, p, &flags);
2415 2416 2417

	return ns;
}
2418

2419 2420 2421 2422 2423 2424 2425 2426
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
2427
	struct task_struct *curr = rq->curr;
2428 2429

	sched_clock_tick();
I
Ingo Molnar 已提交
2430

2431
	raw_spin_lock(&rq->lock);
2432
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
2433
	curr->sched_class->task_tick(rq, curr, 0);
2434
	update_cpu_load_active(rq);
2435
	raw_spin_unlock(&rq->lock);
2436

2437
	perf_event_task_tick();
2438

2439
#ifdef CONFIG_SMP
2440
	rq->idle_balance = idle_cpu(cpu);
2441
	trigger_load_balance(rq);
2442
#endif
2443
	rq_last_tick_reset(rq);
L
Linus Torvalds 已提交
2444 2445
}

2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456
#ifdef CONFIG_NO_HZ_FULL
/**
 * scheduler_tick_max_deferment
 *
 * Keep at least one tick per second when a single
 * active task is running because the scheduler doesn't
 * yet completely support full dynticks environment.
 *
 * This makes sure that uptime, CFS vruntime, load
 * balancing, etc... continue to move forward, even
 * with a very low granularity.
2457 2458
 *
 * Return: Maximum deferment in nanoseconds.
2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
 */
u64 scheduler_tick_max_deferment(void)
{
	struct rq *rq = this_rq();
	unsigned long next, now = ACCESS_ONCE(jiffies);

	next = rq->last_sched_tick + HZ;

	if (time_before_eq(next, now))
		return 0;

2470
	return jiffies_to_nsecs(next - now);
L
Linus Torvalds 已提交
2471
}
2472
#endif
L
Linus Torvalds 已提交
2473

2474
notrace unsigned long get_parent_ip(unsigned long addr)
2475 2476 2477 2478 2479 2480 2481 2482
{
	if (in_lock_functions(addr)) {
		addr = CALLER_ADDR2;
		if (in_lock_functions(addr))
			addr = CALLER_ADDR3;
	}
	return addr;
}
L
Linus Torvalds 已提交
2483

2484 2485 2486
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
				defined(CONFIG_PREEMPT_TRACER))

2487
void __kprobes preempt_count_add(int val)
L
Linus Torvalds 已提交
2488
{
2489
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2490 2491 2492
	/*
	 * Underflow?
	 */
2493 2494
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
2495
#endif
2496
	__preempt_count_add(val);
2497
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2498 2499 2500
	/*
	 * Spinlock count overflowing soon?
	 */
2501 2502
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
2503 2504 2505
#endif
	if (preempt_count() == val)
		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
L
Linus Torvalds 已提交
2506
}
2507
EXPORT_SYMBOL(preempt_count_add);
L
Linus Torvalds 已提交
2508

2509
void __kprobes preempt_count_sub(int val)
L
Linus Torvalds 已提交
2510
{
2511
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2512 2513 2514
	/*
	 * Underflow?
	 */
2515
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
2516
		return;
L
Linus Torvalds 已提交
2517 2518 2519
	/*
	 * Is the spinlock portion underflowing?
	 */
2520 2521 2522
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;
2523
#endif
2524

2525 2526
	if (preempt_count() == val)
		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
2527
	__preempt_count_sub(val);
L
Linus Torvalds 已提交
2528
}
2529
EXPORT_SYMBOL(preempt_count_sub);
L
Linus Torvalds 已提交
2530 2531 2532 2533

#endif

/*
I
Ingo Molnar 已提交
2534
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
2535
 */
I
Ingo Molnar 已提交
2536
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
2537
{
2538 2539 2540
	if (oops_in_progress)
		return;

P
Peter Zijlstra 已提交
2541 2542
	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
		prev->comm, prev->pid, preempt_count());
2543

I
Ingo Molnar 已提交
2544
	debug_show_held_locks(prev);
2545
	print_modules();
I
Ingo Molnar 已提交
2546 2547
	if (irqs_disabled())
		print_irqtrace_events(prev);
2548
	dump_stack();
2549
	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
I
Ingo Molnar 已提交
2550
}
L
Linus Torvalds 已提交
2551

I
Ingo Molnar 已提交
2552 2553 2554 2555 2556
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
2557
	/*
I
Ingo Molnar 已提交
2558
	 * Test if we are atomic. Since do_exit() needs to call into
2559 2560
	 * schedule() atomically, we ignore that path. Otherwise whine
	 * if we are scheduling when we should not.
L
Linus Torvalds 已提交
2561
	 */
2562
	if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD))
I
Ingo Molnar 已提交
2563
		__schedule_bug(prev);
2564
	rcu_sleep_check();
I
Ingo Molnar 已提交
2565

L
Linus Torvalds 已提交
2566 2567
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

2568
	schedstat_inc(this_rq(), sched_count);
I
Ingo Molnar 已提交
2569 2570 2571 2572 2573 2574
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
2575
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
2576
{
2577
	const struct sched_class *class;
I
Ingo Molnar 已提交
2578
	struct task_struct *p;
L
Linus Torvalds 已提交
2579 2580

	/*
I
Ingo Molnar 已提交
2581 2582
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
2583
	 */
2584 2585
	if (likely(prev->sched_class == &fair_sched_class &&
		   rq->nr_running == rq->cfs.h_nr_running)) {
2586
		p = fair_sched_class.pick_next_task(rq, prev);
I
Ingo Molnar 已提交
2587 2588
		if (likely(p))
			return p;
L
Linus Torvalds 已提交
2589 2590
	}

2591
	for_each_class(class) {
2592
		p = class->pick_next_task(rq, prev);
I
Ingo Molnar 已提交
2593 2594 2595
		if (p)
			return p;
	}
2596 2597

	BUG(); /* the idle class will always have a runnable task */
I
Ingo Molnar 已提交
2598
}
L
Linus Torvalds 已提交
2599

I
Ingo Molnar 已提交
2600
/*
2601
 * __schedule() is the main scheduler function.
2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635
 *
 * The main means of driving the scheduler and thus entering this function are:
 *
 *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
 *
 *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
 *      paths. For example, see arch/x86/entry_64.S.
 *
 *      To drive preemption between tasks, the scheduler sets the flag in timer
 *      interrupt handler scheduler_tick().
 *
 *   3. Wakeups don't really cause entry into schedule(). They add a
 *      task to the run-queue and that's it.
 *
 *      Now, if the new task added to the run-queue preempts the current
 *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
 *      called on the nearest possible occasion:
 *
 *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
 *
 *         - in syscall or exception context, at the next outmost
 *           preempt_enable(). (this might be as soon as the wake_up()'s
 *           spin_unlock()!)
 *
 *         - in IRQ context, return from interrupt-handler to
 *           preemptible context
 *
 *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
 *         then at the next:
 *
 *          - cond_resched() call
 *          - explicit schedule() call
 *          - return from syscall or exception to user-space
 *          - return from interrupt-handler to user-space
I
Ingo Molnar 已提交
2636
 */
2637
static void __sched __schedule(void)
I
Ingo Molnar 已提交
2638 2639
{
	struct task_struct *prev, *next;
2640
	unsigned long *switch_count;
I
Ingo Molnar 已提交
2641
	struct rq *rq;
2642
	int cpu;
I
Ingo Molnar 已提交
2643

2644 2645
need_resched:
	preempt_disable();
I
Ingo Molnar 已提交
2646 2647
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
2648
	rcu_note_context_switch(cpu);
I
Ingo Molnar 已提交
2649 2650 2651
	prev = rq->curr;

	schedule_debug(prev);
L
Linus Torvalds 已提交
2652

2653
	if (sched_feat(HRTICK))
M
Mike Galbraith 已提交
2654
		hrtick_clear(rq);
P
Peter Zijlstra 已提交
2655

2656 2657 2658 2659 2660 2661
	/*
	 * Make sure that signal_pending_state()->signal_pending() below
	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
	 * done by the caller to avoid the race with signal_wake_up().
	 */
	smp_mb__before_spinlock();
2662
	raw_spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
2663

2664
	switch_count = &prev->nivcsw;
L
Linus Torvalds 已提交
2665
	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
T
Tejun Heo 已提交
2666
		if (unlikely(signal_pending_state(prev->state, prev))) {
L
Linus Torvalds 已提交
2667
			prev->state = TASK_RUNNING;
T
Tejun Heo 已提交
2668
		} else {
2669 2670 2671
			deactivate_task(rq, prev, DEQUEUE_SLEEP);
			prev->on_rq = 0;

T
Tejun Heo 已提交
2672
			/*
2673 2674 2675
			 * If a worker went to sleep, notify and ask workqueue
			 * whether it wants to wake up a task to maintain
			 * concurrency.
T
Tejun Heo 已提交
2676 2677 2678 2679 2680 2681 2682 2683 2684
			 */
			if (prev->flags & PF_WQ_WORKER) {
				struct task_struct *to_wakeup;

				to_wakeup = wq_worker_sleeping(prev, cpu);
				if (to_wakeup)
					try_to_wake_up_local(to_wakeup);
			}
		}
I
Ingo Molnar 已提交
2685
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
2686 2687
	}

2688 2689 2690 2691
	if (prev->on_rq || rq->skip_clock_update < 0)
		update_rq_clock(rq);

	next = pick_next_task(rq, prev);
2692
	clear_tsk_need_resched(prev);
2693
	clear_preempt_need_resched();
2694
	rq->skip_clock_update = 0;
L
Linus Torvalds 已提交
2695 2696 2697 2698 2699 2700

	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
2701
		context_switch(rq, prev, next); /* unlocks the rq */
P
Peter Zijlstra 已提交
2702
		/*
2703 2704 2705 2706
		 * The context switch have flipped the stack from under us
		 * and restored the local variables which were saved when
		 * this task called schedule() in the past. prev == current
		 * is still correct, but it can be moved to another cpu/rq.
P
Peter Zijlstra 已提交
2707 2708 2709
		 */
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
2710
	} else
2711
		raw_spin_unlock_irq(&rq->lock);
L
Linus Torvalds 已提交
2712

2713
	post_schedule(rq);
L
Linus Torvalds 已提交
2714

2715
	sched_preempt_enable_no_resched();
2716
	if (need_resched())
L
Linus Torvalds 已提交
2717 2718
		goto need_resched;
}
2719

2720 2721
static inline void sched_submit_work(struct task_struct *tsk)
{
2722
	if (!tsk->state || tsk_is_pi_blocked(tsk))
2723 2724 2725 2726 2727 2728 2729 2730 2731
		return;
	/*
	 * If we are going to sleep and we have plugged IO queued,
	 * make sure to submit it to avoid deadlocks.
	 */
	if (blk_needs_flush_plug(tsk))
		blk_schedule_flush_plug(tsk);
}

S
Simon Kirby 已提交
2732
asmlinkage void __sched schedule(void)
2733
{
2734 2735 2736
	struct task_struct *tsk = current;

	sched_submit_work(tsk);
2737 2738
	__schedule();
}
L
Linus Torvalds 已提交
2739 2740
EXPORT_SYMBOL(schedule);

2741
#ifdef CONFIG_CONTEXT_TRACKING
2742 2743 2744 2745 2746 2747 2748 2749
asmlinkage void __sched schedule_user(void)
{
	/*
	 * If we come here after a random call to set_need_resched(),
	 * or we have been woken up remotely but the IPI has not yet arrived,
	 * we haven't yet exited the RCU idle mode. Do it here manually until
	 * we find a better solution.
	 */
2750
	user_exit();
2751
	schedule();
2752
	user_enter();
2753 2754 2755
}
#endif

2756 2757 2758 2759 2760 2761 2762
/**
 * schedule_preempt_disabled - called with preemption disabled
 *
 * Returns with preemption disabled. Note: preempt_count must be 1
 */
void __sched schedule_preempt_disabled(void)
{
2763
	sched_preempt_enable_no_resched();
2764 2765 2766 2767
	schedule();
	preempt_disable();
}

L
Linus Torvalds 已提交
2768 2769
#ifdef CONFIG_PREEMPT
/*
2770
 * this is the entry point to schedule() from in-kernel preemption
I
Ingo Molnar 已提交
2771
 * off of preempt_enable. Kernel preemptions off return from interrupt
L
Linus Torvalds 已提交
2772 2773
 * occur there and call schedule directly.
 */
2774
asmlinkage void __sched notrace preempt_schedule(void)
L
Linus Torvalds 已提交
2775 2776 2777
{
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
I
Ingo Molnar 已提交
2778
	 * we do not want to preempt the current task. Just return..
L
Linus Torvalds 已提交
2779
	 */
2780
	if (likely(!preemptible()))
L
Linus Torvalds 已提交
2781 2782
		return;

2783
	do {
2784
		__preempt_count_add(PREEMPT_ACTIVE);
2785
		__schedule();
2786
		__preempt_count_sub(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
2787

2788 2789 2790 2791 2792
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
2793
	} while (need_resched());
L
Linus Torvalds 已提交
2794 2795
}
EXPORT_SYMBOL(preempt_schedule);
2796
#endif /* CONFIG_PREEMPT */
L
Linus Torvalds 已提交
2797 2798

/*
2799
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
2800 2801 2802 2803 2804 2805
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
asmlinkage void __sched preempt_schedule_irq(void)
{
2806
	enum ctx_state prev_state;
2807

2808
	/* Catch callers which need to be fixed */
2809
	BUG_ON(preempt_count() || !irqs_disabled());
L
Linus Torvalds 已提交
2810

2811 2812
	prev_state = exception_enter();

2813
	do {
2814
		__preempt_count_add(PREEMPT_ACTIVE);
2815
		local_irq_enable();
2816
		__schedule();
2817
		local_irq_disable();
2818
		__preempt_count_sub(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
2819

2820 2821 2822 2823 2824
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
2825
	} while (need_resched());
2826 2827

	exception_exit(prev_state);
L
Linus Torvalds 已提交
2828 2829
}

P
Peter Zijlstra 已提交
2830
int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
I
Ingo Molnar 已提交
2831
			  void *key)
L
Linus Torvalds 已提交
2832
{
P
Peter Zijlstra 已提交
2833
	return try_to_wake_up(curr->private, mode, wake_flags);
L
Linus Torvalds 已提交
2834 2835 2836
}
EXPORT_SYMBOL(default_wake_function);

2837 2838
static long __sched
sleep_on_common(wait_queue_head_t *q, int state, long timeout)
L
Linus Torvalds 已提交
2839
{
I
Ingo Molnar 已提交
2840 2841 2842 2843
	unsigned long flags;
	wait_queue_t wait;

	init_waitqueue_entry(&wait, current);
L
Linus Torvalds 已提交
2844

2845
	__set_current_state(state);
L
Linus Torvalds 已提交
2846

2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860
	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, &wait);
	spin_unlock(&q->lock);
	timeout = schedule_timeout(timeout);
	spin_lock_irq(&q->lock);
	__remove_wait_queue(q, &wait);
	spin_unlock_irqrestore(&q->lock, flags);

	return timeout;
}

void __sched interruptible_sleep_on(wait_queue_head_t *q)
{
	sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
2861 2862 2863
}
EXPORT_SYMBOL(interruptible_sleep_on);

I
Ingo Molnar 已提交
2864
long __sched
I
Ingo Molnar 已提交
2865
interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
2866
{
2867
	return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
2868 2869 2870
}
EXPORT_SYMBOL(interruptible_sleep_on_timeout);

I
Ingo Molnar 已提交
2871
void __sched sleep_on(wait_queue_head_t *q)
L
Linus Torvalds 已提交
2872
{
2873
	sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
L
Linus Torvalds 已提交
2874 2875 2876
}
EXPORT_SYMBOL(sleep_on);

I
Ingo Molnar 已提交
2877
long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
L
Linus Torvalds 已提交
2878
{
2879
	return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
L
Linus Torvalds 已提交
2880 2881 2882
}
EXPORT_SYMBOL(sleep_on_timeout);

2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
 * Used by the rt_mutex code to implement priority inheritance logic.
 */
2895
void rt_mutex_setprio(struct task_struct *p, int prio)
2896
{
2897
	int oldprio, on_rq, running, enqueue_flag = 0;
2898
	struct rq *rq;
2899
	const struct sched_class *prev_class;
2900

2901
	BUG_ON(prio > MAX_PRIO);
2902

2903
	rq = __task_rq_lock(p);
2904

2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922
	/*
	 * Idle task boosting is a nono in general. There is one
	 * exception, when PREEMPT_RT and NOHZ is active:
	 *
	 * The idle task calls get_next_timer_interrupt() and holds
	 * the timer wheel base->lock on the CPU and another CPU wants
	 * to access the timer (probably to cancel it). We can safely
	 * ignore the boosting request, as the idle CPU runs this code
	 * with interrupts disabled and will complete the lock
	 * protected section without being interrupted. So there is no
	 * real need to boost.
	 */
	if (unlikely(p == rq->idle)) {
		WARN_ON(p != rq->curr);
		WARN_ON(p->pi_blocked_on);
		goto out_unlock;
	}

2923
	trace_sched_pi_setprio(p, prio);
2924
	p->pi_top_task = rt_mutex_get_top_task(p);
2925
	oldprio = p->prio;
2926
	prev_class = p->sched_class;
P
Peter Zijlstra 已提交
2927
	on_rq = p->on_rq;
2928
	running = task_current(rq, p);
2929
	if (on_rq)
2930
		dequeue_task(rq, p, 0);
2931 2932
	if (running)
		p->sched_class->put_prev_task(rq, p);
I
Ingo Molnar 已提交
2933

2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950
	/*
	 * Boosting condition are:
	 * 1. -rt task is running and holds mutex A
	 *      --> -dl task blocks on mutex A
	 *
	 * 2. -dl task is running and holds mutex A
	 *      --> -dl task blocks on mutex A and could preempt the
	 *          running task
	 */
	if (dl_prio(prio)) {
		if (!dl_prio(p->normal_prio) || (p->pi_top_task &&
			dl_entity_preempt(&p->pi_top_task->dl, &p->dl))) {
			p->dl.dl_boosted = 1;
			p->dl.dl_throttled = 0;
			enqueue_flag = ENQUEUE_REPLENISH;
		} else
			p->dl.dl_boosted = 0;
2951
		p->sched_class = &dl_sched_class;
2952 2953 2954 2955 2956
	} else if (rt_prio(prio)) {
		if (dl_prio(oldprio))
			p->dl.dl_boosted = 0;
		if (oldprio < prio)
			enqueue_flag = ENQUEUE_HEAD;
I
Ingo Molnar 已提交
2957
		p->sched_class = &rt_sched_class;
2958 2959 2960
	} else {
		if (dl_prio(oldprio))
			p->dl.dl_boosted = 0;
I
Ingo Molnar 已提交
2961
		p->sched_class = &fair_sched_class;
2962
	}
I
Ingo Molnar 已提交
2963

2964 2965
	p->prio = prio;

2966 2967
	if (running)
		p->sched_class->set_curr_task(rq);
P
Peter Zijlstra 已提交
2968
	if (on_rq)
2969
		enqueue_task(rq, p, enqueue_flag);
2970

P
Peter Zijlstra 已提交
2971
	check_class_changed(rq, p, prev_class, oldprio);
2972
out_unlock:
2973
	__task_rq_unlock(rq);
2974 2975
}
#endif
2976

2977
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
2978
{
I
Ingo Molnar 已提交
2979
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
2980
	unsigned long flags;
2981
	struct rq *rq;
L
Linus Torvalds 已提交
2982

2983
	if (task_nice(p) == nice || nice < -20 || nice > 19)
L
Linus Torvalds 已提交
2984 2985 2986 2987 2988 2989 2990 2991 2992 2993
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
2994
	 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
L
Linus Torvalds 已提交
2995
	 */
2996
	if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
2997 2998 2999
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
P
Peter Zijlstra 已提交
3000
	on_rq = p->on_rq;
3001
	if (on_rq)
3002
		dequeue_task(rq, p, 0);
L
Linus Torvalds 已提交
3003 3004

	p->static_prio = NICE_TO_PRIO(nice);
3005
	set_load_weight(p);
3006 3007 3008
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
3009

I
Ingo Molnar 已提交
3010
	if (on_rq) {
3011
		enqueue_task(rq, p, 0);
L
Linus Torvalds 已提交
3012
		/*
3013 3014
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
3015
		 */
3016
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
3017 3018 3019
			resched_task(rq->curr);
	}
out_unlock:
3020
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
3021 3022 3023
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
3024 3025 3026 3027 3028
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
3029
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
3030
{
3031 3032
	/* convert nice value [19,-20] to rlimit style value [1,40] */
	int nice_rlim = 20 - nice;
3033

3034
	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
M
Matt Mackall 已提交
3035 3036 3037
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
3038 3039 3040 3041 3042 3043 3044 3045 3046
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
3047
SYSCALL_DEFINE1(nice, int, increment)
L
Linus Torvalds 已提交
3048
{
3049
	long nice, retval;
L
Linus Torvalds 已提交
3050 3051 3052 3053 3054 3055

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
M
Matt Mackall 已提交
3056 3057
	if (increment < -40)
		increment = -40;
L
Linus Torvalds 已提交
3058 3059 3060
	if (increment > 40)
		increment = 40;

3061
	nice = task_nice(current) + increment;
L
Linus Torvalds 已提交
3062 3063 3064 3065 3066
	if (nice < -20)
		nice = -20;
	if (nice > 19)
		nice = 19;

M
Matt Mackall 已提交
3067 3068 3069
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
3084
 * Return: The priority value as seen by users in /proc.
L
Linus Torvalds 已提交
3085 3086 3087
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
3088
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
3089 3090 3091 3092 3093 3094 3095
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
3096 3097
 *
 * Return: 1 if the CPU is currently idle. 0 otherwise.
L
Linus Torvalds 已提交
3098 3099 3100
 */
int idle_cpu(int cpu)
{
T
Thomas Gleixner 已提交
3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114
	struct rq *rq = cpu_rq(cpu);

	if (rq->curr != rq->idle)
		return 0;

	if (rq->nr_running)
		return 0;

#ifdef CONFIG_SMP
	if (!llist_empty(&rq->wake_list))
		return 0;
#endif

	return 1;
L
Linus Torvalds 已提交
3115 3116 3117 3118 3119
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
3120 3121
 *
 * Return: The idle task for the cpu @cpu.
L
Linus Torvalds 已提交
3122
 */
3123
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
3124 3125 3126 3127 3128 3129 3130
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
3131 3132
 *
 * The task of @pid, if found. %NULL otherwise.
L
Linus Torvalds 已提交
3133
 */
A
Alexey Dobriyan 已提交
3134
static struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
3135
{
3136
	return pid ? find_task_by_vpid(pid) : current;
L
Linus Torvalds 已提交
3137 3138
}

3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154
/*
 * This function initializes the sched_dl_entity of a newly becoming
 * SCHED_DEADLINE task.
 *
 * Only the static values are considered here, the actual runtime and the
 * absolute deadline will be properly calculated when the task is enqueued
 * for the first time with its new policy.
 */
static void
__setparam_dl(struct task_struct *p, const struct sched_attr *attr)
{
	struct sched_dl_entity *dl_se = &p->dl;

	init_dl_task_timer(dl_se);
	dl_se->dl_runtime = attr->sched_runtime;
	dl_se->dl_deadline = attr->sched_deadline;
3155
	dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
3156
	dl_se->flags = attr->sched_flags;
3157
	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
3158 3159 3160 3161
	dl_se->dl_throttled = 0;
	dl_se->dl_new = 1;
}

3162 3163 3164
/* Actually do priority change: must hold pi & rq lock. */
static void __setscheduler(struct rq *rq, struct task_struct *p,
			   const struct sched_attr *attr)
L
Linus Torvalds 已提交
3165
{
3166 3167
	int policy = attr->sched_policy;

3168 3169 3170
	if (policy == -1) /* setparam */
		policy = p->policy;

L
Linus Torvalds 已提交
3171
	p->policy = policy;
3172

3173 3174
	if (dl_policy(policy))
		__setparam_dl(p, attr);
3175
	else if (fair_policy(policy))
3176 3177
		p->static_prio = NICE_TO_PRIO(attr->sched_nice);

3178 3179 3180 3181 3182 3183 3184
	/*
	 * __sched_setscheduler() ensures attr->sched_priority == 0 when
	 * !rt_policy. Always setting this ensures that things like
	 * getparam()/getattr() don't report silly values for !rt tasks.
	 */
	p->rt_priority = attr->sched_priority;

3185 3186
	p->normal_prio = normal_prio(p);
	p->prio = rt_mutex_getprio(p);
3187

3188 3189 3190
	if (dl_prio(p->prio))
		p->sched_class = &dl_sched_class;
	else if (rt_prio(p->prio))
3191 3192 3193
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;
3194

3195
	set_load_weight(p);
L
Linus Torvalds 已提交
3196
}
3197 3198 3199 3200 3201 3202 3203 3204 3205

static void
__getparam_dl(struct task_struct *p, struct sched_attr *attr)
{
	struct sched_dl_entity *dl_se = &p->dl;

	attr->sched_priority = p->rt_priority;
	attr->sched_runtime = dl_se->dl_runtime;
	attr->sched_deadline = dl_se->dl_deadline;
3206
	attr->sched_period = dl_se->dl_period;
3207 3208 3209 3210 3211 3212
	attr->sched_flags = dl_se->flags;
}

/*
 * This function validates the new parameters of a -deadline task.
 * We ask for the deadline not being zero, and greater or equal
3213
 * than the runtime, as well as the period of being zero or
3214 3215 3216
 * greater than deadline. Furthermore, we have to be sure that
 * user parameters are above the internal resolution (1us); we
 * check sched_runtime only since it is always the smaller one.
3217 3218 3219 3220 3221
 */
static bool
__checkparam_dl(const struct sched_attr *attr)
{
	return attr && attr->sched_deadline != 0 &&
3222 3223
		(attr->sched_period == 0 ||
		(s64)(attr->sched_period   - attr->sched_deadline) >= 0) &&
3224 3225
		(s64)(attr->sched_deadline - attr->sched_runtime ) >= 0  &&
		attr->sched_runtime >= (2 << (DL_SCALE - 1));
3226 3227
}

3228 3229 3230 3231 3232 3233 3234 3235 3236 3237
/*
 * check the target process has a UID that matches the current process's
 */
static bool check_same_owner(struct task_struct *p)
{
	const struct cred *cred = current_cred(), *pcred;
	bool match;

	rcu_read_lock();
	pcred = __task_cred(p);
3238 3239
	match = (uid_eq(cred->euid, pcred->euid) ||
		 uid_eq(cred->euid, pcred->uid));
3240 3241 3242 3243
	rcu_read_unlock();
	return match;
}

3244 3245 3246
static int __sched_setscheduler(struct task_struct *p,
				const struct sched_attr *attr,
				bool user)
L
Linus Torvalds 已提交
3247
{
3248
	int retval, oldprio, oldpolicy = -1, on_rq, running;
3249
	int policy = attr->sched_policy;
L
Linus Torvalds 已提交
3250
	unsigned long flags;
3251
	const struct sched_class *prev_class;
3252
	struct rq *rq;
3253
	int reset_on_fork;
L
Linus Torvalds 已提交
3254

3255 3256
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
3257 3258
recheck:
	/* double check policy once rq lock held */
3259 3260
	if (policy < 0) {
		reset_on_fork = p->sched_reset_on_fork;
L
Linus Torvalds 已提交
3261
		policy = oldpolicy = p->policy;
3262
	} else {
3263
		reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
3264

3265 3266
		if (policy != SCHED_DEADLINE &&
				policy != SCHED_FIFO && policy != SCHED_RR &&
3267 3268 3269 3270 3271
				policy != SCHED_NORMAL && policy != SCHED_BATCH &&
				policy != SCHED_IDLE)
			return -EINVAL;
	}

3272 3273 3274
	if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
		return -EINVAL;

L
Linus Torvalds 已提交
3275 3276
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
3277 3278
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
3279
	 */
3280
	if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
3281
	    (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
3282
		return -EINVAL;
3283 3284
	if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
	    (rt_policy(policy) != (attr->sched_priority != 0)))
L
Linus Torvalds 已提交
3285 3286
		return -EINVAL;

3287 3288 3289
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
3290
	if (user && !capable(CAP_SYS_NICE)) {
3291
		if (fair_policy(policy)) {
3292
			if (attr->sched_nice < task_nice(p) &&
3293
			    !can_nice(p, attr->sched_nice))
3294 3295 3296
				return -EPERM;
		}

3297
		if (rt_policy(policy)) {
3298 3299
			unsigned long rlim_rtprio =
					task_rlimit(p, RLIMIT_RTPRIO);
3300 3301 3302 3303 3304 3305

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
3306 3307
			if (attr->sched_priority > p->rt_priority &&
			    attr->sched_priority > rlim_rtprio)
3308 3309
				return -EPERM;
		}
3310

I
Ingo Molnar 已提交
3311
		/*
3312 3313
		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
I
Ingo Molnar 已提交
3314
		 */
3315
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
3316
			if (!can_nice(p, task_nice(p)))
3317 3318
				return -EPERM;
		}
3319

3320
		/* can't change other user's priorities */
3321
		if (!check_same_owner(p))
3322
			return -EPERM;
3323 3324 3325 3326

		/* Normal users shall not reset the sched_reset_on_fork flag */
		if (p->sched_reset_on_fork && !reset_on_fork)
			return -EPERM;
3327
	}
L
Linus Torvalds 已提交
3328

3329
	if (user) {
3330
		retval = security_task_setscheduler(p);
3331 3332 3333 3334
		if (retval)
			return retval;
	}

3335 3336 3337
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
3338
	 *
L
Lucas De Marchi 已提交
3339
	 * To be able to change p->policy safely, the appropriate
L
Linus Torvalds 已提交
3340 3341
	 * runqueue lock must be held.
	 */
3342
	rq = task_rq_lock(p, &flags);
3343

3344 3345 3346 3347
	/*
	 * Changing the policy of the stop threads its a very bad idea
	 */
	if (p == rq->stop) {
3348
		task_rq_unlock(rq, p, &flags);
3349 3350 3351
		return -EINVAL;
	}

3352 3353 3354
	/*
	 * If not changing anything there's no need to proceed further:
	 */
3355
	if (unlikely(policy == p->policy)) {
3356
		if (fair_policy(policy) && attr->sched_nice != task_nice(p))
3357 3358 3359
			goto change;
		if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
			goto change;
3360 3361
		if (dl_policy(policy))
			goto change;
3362

3363
		task_rq_unlock(rq, p, &flags);
3364 3365
		return 0;
	}
3366
change:
3367

3368
	if (user) {
3369
#ifdef CONFIG_RT_GROUP_SCHED
3370 3371 3372 3373 3374
		/*
		 * Do not allow realtime tasks into groups that have no runtime
		 * assigned.
		 */
		if (rt_bandwidth_enabled() && rt_policy(policy) &&
3375 3376
				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
				!task_group_is_autogroup(task_group(p))) {
3377
			task_rq_unlock(rq, p, &flags);
3378 3379 3380
			return -EPERM;
		}
#endif
3381 3382 3383 3384 3385 3386 3387 3388 3389
#ifdef CONFIG_SMP
		if (dl_bandwidth_enabled() && dl_policy(policy)) {
			cpumask_t *span = rq->rd->span;

			/*
			 * Don't allow tasks with an affinity mask smaller than
			 * the entire root_domain to become SCHED_DEADLINE. We
			 * will also fail if there's no bandwidth available.
			 */
3390 3391
			if (!cpumask_subset(span, &p->cpus_allowed) ||
			    rq->rd->dl_bw.bw == 0) {
3392 3393 3394 3395 3396 3397
				task_rq_unlock(rq, p, &flags);
				return -EPERM;
			}
		}
#endif
	}
3398

L
Linus Torvalds 已提交
3399 3400 3401
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
3402
		task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
3403 3404
		goto recheck;
	}
3405 3406 3407 3408 3409 3410

	/*
	 * If setscheduling to SCHED_DEADLINE (or changing the parameters
	 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
	 * is available.
	 */
3411
	if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
3412 3413 3414 3415
		task_rq_unlock(rq, p, &flags);
		return -EBUSY;
	}

P
Peter Zijlstra 已提交
3416
	on_rq = p->on_rq;
3417
	running = task_current(rq, p);
3418
	if (on_rq)
3419
		dequeue_task(rq, p, 0);
3420 3421
	if (running)
		p->sched_class->put_prev_task(rq, p);
3422

3423 3424
	p->sched_reset_on_fork = reset_on_fork;

L
Linus Torvalds 已提交
3425
	oldprio = p->prio;
3426
	prev_class = p->sched_class;
3427
	__setscheduler(rq, p, attr);
3428

3429 3430
	if (running)
		p->sched_class->set_curr_task(rq);
P
Peter Zijlstra 已提交
3431
	if (on_rq)
3432
		enqueue_task(rq, p, 0);
3433

P
Peter Zijlstra 已提交
3434
	check_class_changed(rq, p, prev_class, oldprio);
3435
	task_rq_unlock(rq, p, &flags);
3436

3437 3438
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
3439 3440
	return 0;
}
3441

3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461
static int _sched_setscheduler(struct task_struct *p, int policy,
			       const struct sched_param *param, bool check)
{
	struct sched_attr attr = {
		.sched_policy   = policy,
		.sched_priority = param->sched_priority,
		.sched_nice	= PRIO_TO_NICE(p->static_prio),
	};

	/*
	 * Fixup the legacy SCHED_RESET_ON_FORK hack
	 */
	if (policy & SCHED_RESET_ON_FORK) {
		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
		policy &= ~SCHED_RESET_ON_FORK;
		attr.sched_policy = policy;
	}

	return __sched_setscheduler(p, &attr, check);
}
3462 3463 3464 3465 3466 3467
/**
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
3468 3469
 * Return: 0 on success. An error code otherwise.
 *
3470 3471 3472
 * NOTE that the task may be already dead.
 */
int sched_setscheduler(struct task_struct *p, int policy,
3473
		       const struct sched_param *param)
3474
{
3475
	return _sched_setscheduler(p, policy, param, true);
3476
}
L
Linus Torvalds 已提交
3477 3478
EXPORT_SYMBOL_GPL(sched_setscheduler);

3479 3480 3481 3482 3483 3484
int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
{
	return __sched_setscheduler(p, attr, true);
}
EXPORT_SYMBOL_GPL(sched_setattr);

3485 3486 3487 3488 3489 3490 3491 3492 3493 3494
/**
 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * Just like sched_setscheduler, only don't bother checking if the
 * current context has permission.  For example, this is needed in
 * stop_machine(): we create temporary high priority worker threads,
 * but our caller might not have that capability.
3495 3496
 *
 * Return: 0 on success. An error code otherwise.
3497 3498
 */
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
3499
			       const struct sched_param *param)
3500
{
3501
	return _sched_setscheduler(p, policy, param, false);
3502 3503
}

I
Ingo Molnar 已提交
3504 3505
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
3506 3507 3508
{
	struct sched_param lparam;
	struct task_struct *p;
3509
	int retval;
L
Linus Torvalds 已提交
3510 3511 3512 3513 3514

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
3515 3516 3517

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
3518
	p = find_process_by_pid(pid);
3519 3520 3521
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
3522

L
Linus Torvalds 已提交
3523 3524 3525
	return retval;
}

3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598
/*
 * Mimics kernel/events/core.c perf_copy_attr().
 */
static int sched_copy_attr(struct sched_attr __user *uattr,
			   struct sched_attr *attr)
{
	u32 size;
	int ret;

	if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = SCHED_ATTR_SIZE_VER0;

	if (size < SCHED_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
	 */
	if (size > sizeof(*attr)) {
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;

		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;

		for (; addr < end; addr++) {
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
		size = sizeof(*attr);
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * XXX: do we want to be lenient like existing syscalls; or do we want
	 * to be strict and return an error on out-of-bounds values?
	 */
	attr->sched_nice = clamp(attr->sched_nice, -20, 19);

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

L
Linus Torvalds 已提交
3599 3600 3601 3602 3603
/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
3604 3605
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
3606
 */
3607 3608
SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
		struct sched_param __user *, param)
L
Linus Torvalds 已提交
3609
{
3610 3611 3612 3613
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
3614 3615 3616 3617 3618 3619 3620
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
3621 3622
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
3623
 */
3624
SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
3625 3626 3627 3628
{
	return do_sched_setscheduler(pid, -1, param);
}

3629 3630 3631
/**
 * sys_sched_setattr - same as above, but with extended sched_attr
 * @pid: the pid in question.
J
Juri Lelli 已提交
3632
 * @uattr: structure containing the extended parameters.
3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655
 */
SYSCALL_DEFINE2(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr)
{
	struct sched_attr attr;
	struct task_struct *p;
	int retval;

	if (!uattr || pid < 0)
		return -EINVAL;

	if (sched_copy_attr(uattr, &attr))
		return -EFAULT;

	rcu_read_lock();
	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (p != NULL)
		retval = sched_setattr(p, &attr);
	rcu_read_unlock();

	return retval;
}

L
Linus Torvalds 已提交
3656 3657 3658
/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
3659 3660 3661
 *
 * Return: On success, the policy of the thread. Otherwise, a negative error
 * code.
L
Linus Torvalds 已提交
3662
 */
3663
SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
L
Linus Torvalds 已提交
3664
{
3665
	struct task_struct *p;
3666
	int retval;
L
Linus Torvalds 已提交
3667 3668

	if (pid < 0)
3669
		return -EINVAL;
L
Linus Torvalds 已提交
3670 3671

	retval = -ESRCH;
3672
	rcu_read_lock();
L
Linus Torvalds 已提交
3673 3674 3675 3676
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
3677 3678
			retval = p->policy
				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
L
Linus Torvalds 已提交
3679
	}
3680
	rcu_read_unlock();
L
Linus Torvalds 已提交
3681 3682 3683 3684
	return retval;
}

/**
3685
 * sys_sched_getparam - get the RT priority of a thread
L
Linus Torvalds 已提交
3686 3687
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
3688 3689 3690
 *
 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
 * code.
L
Linus Torvalds 已提交
3691
 */
3692
SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
3693 3694
{
	struct sched_param lp;
3695
	struct task_struct *p;
3696
	int retval;
L
Linus Torvalds 已提交
3697 3698

	if (!param || pid < 0)
3699
		return -EINVAL;
L
Linus Torvalds 已提交
3700

3701
	rcu_read_lock();
L
Linus Torvalds 已提交
3702 3703 3704 3705 3706 3707 3708 3709 3710
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

3711 3712 3713 3714
	if (task_has_dl_policy(p)) {
		retval = -EINVAL;
		goto out_unlock;
	}
L
Linus Torvalds 已提交
3715
	lp.sched_priority = p->rt_priority;
3716
	rcu_read_unlock();
L
Linus Torvalds 已提交
3717 3718 3719 3720 3721 3722 3723 3724 3725

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

	return retval;

out_unlock:
3726
	rcu_read_unlock();
L
Linus Torvalds 已提交
3727 3728 3729
	return retval;
}

3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771
static int sched_read_attr(struct sched_attr __user *uattr,
			   struct sched_attr *attr,
			   unsigned int usize)
{
	int ret;

	if (!access_ok(VERIFY_WRITE, uattr, usize))
		return -EFAULT;

	/*
	 * If we're handed a smaller struct than we know of,
	 * ensure all the unknown bits are 0 - i.e. old
	 * user-space does not get uncomplete information.
	 */
	if (usize < sizeof(*attr)) {
		unsigned char *addr;
		unsigned char *end;

		addr = (void *)attr + usize;
		end  = (void *)attr + sizeof(*attr);

		for (; addr < end; addr++) {
			if (*addr)
				goto err_size;
		}

		attr->size = usize;
	}

	ret = copy_to_user(uattr, attr, usize);
	if (ret)
		return -EFAULT;

out:
	return ret;

err_size:
	ret = -E2BIG;
	goto out;
}

/**
3772
 * sys_sched_getattr - similar to sched_getparam, but with sched_attr
3773
 * @pid: the pid in question.
J
Juri Lelli 已提交
3774
 * @uattr: structure containing the extended parameters.
3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800
 * @size: sizeof(attr) for fwd/bwd comp.
 */
SYSCALL_DEFINE3(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
		unsigned int, size)
{
	struct sched_attr attr = {
		.size = sizeof(struct sched_attr),
	};
	struct task_struct *p;
	int retval;

	if (!uattr || pid < 0 || size > PAGE_SIZE ||
	    size < SCHED_ATTR_SIZE_VER0)
		return -EINVAL;

	rcu_read_lock();
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	attr.sched_policy = p->policy;
3801 3802
	if (p->sched_reset_on_fork)
		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
3803 3804 3805
	if (task_has_dl_policy(p))
		__getparam_dl(p, &attr);
	else if (task_has_rt_policy(p))
3806 3807
		attr.sched_priority = p->rt_priority;
	else
3808
		attr.sched_nice = task_nice(p);
3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819

	rcu_read_unlock();

	retval = sched_read_attr(uattr, &attr, size);
	return retval;

out_unlock:
	rcu_read_unlock();
	return retval;
}

3820
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
L
Linus Torvalds 已提交
3821
{
3822
	cpumask_var_t cpus_allowed, new_mask;
3823 3824
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
3825

3826
	rcu_read_lock();
L
Linus Torvalds 已提交
3827 3828 3829

	p = find_process_by_pid(pid);
	if (!p) {
3830
		rcu_read_unlock();
L
Linus Torvalds 已提交
3831 3832 3833
		return -ESRCH;
	}

3834
	/* Prevent p going away */
L
Linus Torvalds 已提交
3835
	get_task_struct(p);
3836
	rcu_read_unlock();
L
Linus Torvalds 已提交
3837

3838 3839 3840 3841
	if (p->flags & PF_NO_SETAFFINITY) {
		retval = -EINVAL;
		goto out_put_task;
	}
3842 3843 3844 3845 3846 3847 3848 3849
	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_put_task;
	}
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_free_cpus_allowed;
	}
L
Linus Torvalds 已提交
3850
	retval = -EPERM;
E
Eric W. Biederman 已提交
3851 3852 3853 3854 3855 3856 3857 3858
	if (!check_same_owner(p)) {
		rcu_read_lock();
		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
			rcu_read_unlock();
			goto out_unlock;
		}
		rcu_read_unlock();
	}
L
Linus Torvalds 已提交
3859

3860
	retval = security_task_setscheduler(p);
3861 3862 3863
	if (retval)
		goto out_unlock;

3864 3865 3866 3867

	cpuset_cpus_allowed(p, cpus_allowed);
	cpumask_and(new_mask, in_mask, cpus_allowed);

3868 3869 3870 3871 3872 3873 3874 3875 3876 3877
	/*
	 * Since bandwidth control happens on root_domain basis,
	 * if admission test is enabled, we only admit -deadline
	 * tasks allowed to run on all the CPUs in the task's
	 * root_domain.
	 */
#ifdef CONFIG_SMP
	if (task_has_dl_policy(p)) {
		const struct cpumask *span = task_rq(p)->rd->span;

3878
		if (dl_bandwidth_enabled() && !cpumask_subset(span, new_mask)) {
3879 3880 3881 3882 3883
			retval = -EBUSY;
			goto out_unlock;
		}
	}
#endif
P
Peter Zijlstra 已提交
3884
again:
3885
	retval = set_cpus_allowed_ptr(p, new_mask);
L
Linus Torvalds 已提交
3886

P
Paul Menage 已提交
3887
	if (!retval) {
3888 3889
		cpuset_cpus_allowed(p, cpus_allowed);
		if (!cpumask_subset(new_mask, cpus_allowed)) {
P
Paul Menage 已提交
3890 3891 3892 3893 3894
			/*
			 * We must have raced with a concurrent cpuset
			 * update. Just reset the cpus_allowed to the
			 * cpuset's cpus_allowed
			 */
3895
			cpumask_copy(new_mask, cpus_allowed);
P
Paul Menage 已提交
3896 3897 3898
			goto again;
		}
	}
L
Linus Torvalds 已提交
3899
out_unlock:
3900 3901 3902 3903
	free_cpumask_var(new_mask);
out_free_cpus_allowed:
	free_cpumask_var(cpus_allowed);
out_put_task:
L
Linus Torvalds 已提交
3904 3905 3906 3907 3908
	put_task_struct(p);
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
3909
			     struct cpumask *new_mask)
L
Linus Torvalds 已提交
3910
{
3911 3912 3913 3914 3915
	if (len < cpumask_size())
		cpumask_clear(new_mask);
	else if (len > cpumask_size())
		len = cpumask_size();

L
Linus Torvalds 已提交
3916 3917 3918 3919 3920 3921 3922 3923
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
3924 3925
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
3926
 */
3927 3928
SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
3929
{
3930
	cpumask_var_t new_mask;
L
Linus Torvalds 已提交
3931 3932
	int retval;

3933 3934
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
3935

3936 3937 3938 3939 3940
	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
	if (retval == 0)
		retval = sched_setaffinity(pid, new_mask);
	free_cpumask_var(new_mask);
	return retval;
L
Linus Torvalds 已提交
3941 3942
}

3943
long sched_getaffinity(pid_t pid, struct cpumask *mask)
L
Linus Torvalds 已提交
3944
{
3945
	struct task_struct *p;
3946
	unsigned long flags;
L
Linus Torvalds 已提交
3947 3948
	int retval;

3949
	rcu_read_lock();
L
Linus Torvalds 已提交
3950 3951 3952 3953 3954 3955

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

3956 3957 3958 3959
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

3960
	raw_spin_lock_irqsave(&p->pi_lock, flags);
3961
	cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
3962
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
3963 3964

out_unlock:
3965
	rcu_read_unlock();
L
Linus Torvalds 已提交
3966

3967
	return retval;
L
Linus Torvalds 已提交
3968 3969 3970 3971 3972 3973 3974
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
3975 3976
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
3977
 */
3978 3979
SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
3980 3981
{
	int ret;
3982
	cpumask_var_t mask;
L
Linus Torvalds 已提交
3983

A
Anton Blanchard 已提交
3984
	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
3985 3986
		return -EINVAL;
	if (len & (sizeof(unsigned long)-1))
L
Linus Torvalds 已提交
3987 3988
		return -EINVAL;

3989 3990
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
3991

3992 3993
	ret = sched_getaffinity(pid, mask);
	if (ret == 0) {
3994
		size_t retlen = min_t(size_t, len, cpumask_size());
3995 3996

		if (copy_to_user(user_mask_ptr, mask, retlen))
3997 3998
			ret = -EFAULT;
		else
3999
			ret = retlen;
4000 4001
	}
	free_cpumask_var(mask);
L
Linus Torvalds 已提交
4002

4003
	return ret;
L
Linus Torvalds 已提交
4004 4005 4006 4007 4008
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
4009 4010
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
4011 4012
 *
 * Return: 0.
L
Linus Torvalds 已提交
4013
 */
4014
SYSCALL_DEFINE0(sched_yield)
L
Linus Torvalds 已提交
4015
{
4016
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
4017

4018
	schedstat_inc(rq, yld_count);
4019
	current->sched_class->yield_task(rq);
L
Linus Torvalds 已提交
4020 4021 4022 4023 4024 4025

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
4026
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
4027
	do_raw_spin_unlock(&rq->lock);
4028
	sched_preempt_enable_no_resched();
L
Linus Torvalds 已提交
4029 4030 4031 4032 4033 4034

	schedule();

	return 0;
}

A
Andrew Morton 已提交
4035
static void __cond_resched(void)
L
Linus Torvalds 已提交
4036
{
4037
	__preempt_count_add(PREEMPT_ACTIVE);
4038
	__schedule();
4039
	__preempt_count_sub(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4040 4041
}

4042
int __sched _cond_resched(void)
L
Linus Torvalds 已提交
4043
{
P
Peter Zijlstra 已提交
4044
	if (should_resched()) {
L
Linus Torvalds 已提交
4045 4046 4047 4048 4049
		__cond_resched();
		return 1;
	}
	return 0;
}
4050
EXPORT_SYMBOL(_cond_resched);
L
Linus Torvalds 已提交
4051 4052

/*
4053
 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
L
Linus Torvalds 已提交
4054 4055
 * call schedule, and on return reacquire the lock.
 *
I
Ingo Molnar 已提交
4056
 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
L
Linus Torvalds 已提交
4057 4058 4059
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
4060
int __cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
4061
{
P
Peter Zijlstra 已提交
4062
	int resched = should_resched();
J
Jan Kara 已提交
4063 4064
	int ret = 0;

4065 4066
	lockdep_assert_held(lock);

N
Nick Piggin 已提交
4067
	if (spin_needbreak(lock) || resched) {
L
Linus Torvalds 已提交
4068
		spin_unlock(lock);
P
Peter Zijlstra 已提交
4069
		if (resched)
N
Nick Piggin 已提交
4070 4071 4072
			__cond_resched();
		else
			cpu_relax();
J
Jan Kara 已提交
4073
		ret = 1;
L
Linus Torvalds 已提交
4074 4075
		spin_lock(lock);
	}
J
Jan Kara 已提交
4076
	return ret;
L
Linus Torvalds 已提交
4077
}
4078
EXPORT_SYMBOL(__cond_resched_lock);
L
Linus Torvalds 已提交
4079

4080
int __sched __cond_resched_softirq(void)
L
Linus Torvalds 已提交
4081 4082 4083
{
	BUG_ON(!in_softirq());

P
Peter Zijlstra 已提交
4084
	if (should_resched()) {
4085
		local_bh_enable();
L
Linus Torvalds 已提交
4086 4087 4088 4089 4090 4091
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
4092
EXPORT_SYMBOL(__cond_resched_softirq);
L
Linus Torvalds 已提交
4093 4094 4095 4096

/**
 * yield - yield the current processor to other threads.
 *
P
Peter Zijlstra 已提交
4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114
 * Do not ever use this function, there's a 99% chance you're doing it wrong.
 *
 * The scheduler is at all times free to pick the calling task as the most
 * eligible task to run, if removing the yield() call from your code breaks
 * it, its already broken.
 *
 * Typical broken usage is:
 *
 * while (!event)
 * 	yield();
 *
 * where one assumes that yield() will let 'the other' process run that will
 * make event true. If the current task is a SCHED_FIFO task that will never
 * happen. Never use yield() as a progress guarantee!!
 *
 * If you want to use yield() to wait for something, use wait_event().
 * If you want to use yield() to be 'nice' for others, use cond_resched().
 * If you still want to use yield(), do not!
L
Linus Torvalds 已提交
4115 4116 4117 4118 4119 4120 4121 4122
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

4123 4124 4125 4126
/**
 * yield_to - yield the current processor to another thread in
 * your thread group, or accelerate that thread toward the
 * processor it's on.
R
Randy Dunlap 已提交
4127 4128
 * @p: target task
 * @preempt: whether task preemption is allowed or not
4129 4130 4131 4132
 *
 * It's the caller's job to ensure that the target task struct
 * can't go away on us before we can do any checks.
 *
4133
 * Return:
4134 4135 4136
 *	true (>0) if we indeed boosted the target task.
 *	false (0) if we failed to boost the target.
 *	-ESRCH if there's no task to yield to.
4137 4138 4139 4140 4141 4142
 */
bool __sched yield_to(struct task_struct *p, bool preempt)
{
	struct task_struct *curr = current;
	struct rq *rq, *p_rq;
	unsigned long flags;
4143
	int yielded = 0;
4144 4145 4146 4147 4148 4149

	local_irq_save(flags);
	rq = this_rq();

again:
	p_rq = task_rq(p);
4150 4151 4152 4153 4154 4155 4156 4157 4158
	/*
	 * If we're the only runnable task on the rq and target rq also
	 * has only one task, there's absolutely no point in yielding.
	 */
	if (rq->nr_running == 1 && p_rq->nr_running == 1) {
		yielded = -ESRCH;
		goto out_irq;
	}

4159
	double_rq_lock(rq, p_rq);
4160
	if (task_rq(p) != p_rq) {
4161 4162 4163 4164 4165
		double_rq_unlock(rq, p_rq);
		goto again;
	}

	if (!curr->sched_class->yield_to_task)
4166
		goto out_unlock;
4167 4168

	if (curr->sched_class != p->sched_class)
4169
		goto out_unlock;
4170 4171

	if (task_running(p_rq, p) || p->state)
4172
		goto out_unlock;
4173 4174

	yielded = curr->sched_class->yield_to_task(rq, p, preempt);
4175
	if (yielded) {
4176
		schedstat_inc(rq, yld_count);
4177 4178 4179 4180 4181 4182 4183
		/*
		 * Make p's CPU reschedule; pick_next_entity takes care of
		 * fairness.
		 */
		if (preempt && rq != p_rq)
			resched_task(p_rq->curr);
	}
4184

4185
out_unlock:
4186
	double_rq_unlock(rq, p_rq);
4187
out_irq:
4188 4189
	local_irq_restore(flags);

4190
	if (yielded > 0)
4191 4192 4193 4194 4195 4196
		schedule();

	return yielded;
}
EXPORT_SYMBOL_GPL(yield_to);

L
Linus Torvalds 已提交
4197
/*
I
Ingo Molnar 已提交
4198
 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
L
Linus Torvalds 已提交
4199 4200 4201 4202
 * that process accounting knows that this is a task in IO wait state.
 */
void __sched io_schedule(void)
{
4203
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
4204

4205
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4206
	atomic_inc(&rq->nr_iowait);
4207
	blk_flush_plug(current);
4208
	current->in_iowait = 1;
L
Linus Torvalds 已提交
4209
	schedule();
4210
	current->in_iowait = 0;
L
Linus Torvalds 已提交
4211
	atomic_dec(&rq->nr_iowait);
4212
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4213 4214 4215 4216 4217
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
4218
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
4219 4220
	long ret;

4221
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4222
	atomic_inc(&rq->nr_iowait);
4223
	blk_flush_plug(current);
4224
	current->in_iowait = 1;
L
Linus Torvalds 已提交
4225
	ret = schedule_timeout(timeout);
4226
	current->in_iowait = 0;
L
Linus Torvalds 已提交
4227
	atomic_dec(&rq->nr_iowait);
4228
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4229 4230 4231 4232 4233 4234 4235
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
4236 4237 4238
 * Return: On success, this syscall returns the maximum
 * rt_priority that can be used by a given scheduling class.
 * On failure, a negative error code is returned.
L
Linus Torvalds 已提交
4239
 */
4240
SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
L
Linus Torvalds 已提交
4241 4242 4243 4244 4245 4246 4247 4248
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
4249
	case SCHED_DEADLINE:
L
Linus Torvalds 已提交
4250
	case SCHED_NORMAL:
4251
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4252
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4253 4254 4255 4256 4257 4258 4259 4260 4261 4262
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
4263 4264 4265
 * Return: On success, this syscall returns the minimum
 * rt_priority that can be used by a given scheduling class.
 * On failure, a negative error code is returned.
L
Linus Torvalds 已提交
4266
 */
4267
SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
L
Linus Torvalds 已提交
4268 4269 4270 4271 4272 4273 4274 4275
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
4276
	case SCHED_DEADLINE:
L
Linus Torvalds 已提交
4277
	case SCHED_NORMAL:
4278
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4279
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
4292 4293 4294
 *
 * Return: On success, 0 and the timeslice is in @interval. Otherwise,
 * an error code.
L
Linus Torvalds 已提交
4295
 */
4296
SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
4297
		struct timespec __user *, interval)
L
Linus Torvalds 已提交
4298
{
4299
	struct task_struct *p;
D
Dmitry Adamushko 已提交
4300
	unsigned int time_slice;
4301 4302
	unsigned long flags;
	struct rq *rq;
4303
	int retval;
L
Linus Torvalds 已提交
4304 4305 4306
	struct timespec t;

	if (pid < 0)
4307
		return -EINVAL;
L
Linus Torvalds 已提交
4308 4309

	retval = -ESRCH;
4310
	rcu_read_lock();
L
Linus Torvalds 已提交
4311 4312 4313 4314 4315 4316 4317 4318
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4319
	rq = task_rq_lock(p, &flags);
4320 4321 4322
	time_slice = 0;
	if (p->sched_class->get_rr_interval)
		time_slice = p->sched_class->get_rr_interval(rq, p);
4323
	task_rq_unlock(rq, p, &flags);
D
Dmitry Adamushko 已提交
4324

4325
	rcu_read_unlock();
D
Dmitry Adamushko 已提交
4326
	jiffies_to_timespec(time_slice, &t);
L
Linus Torvalds 已提交
4327 4328
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
	return retval;
4329

L
Linus Torvalds 已提交
4330
out_unlock:
4331
	rcu_read_unlock();
L
Linus Torvalds 已提交
4332 4333 4334
	return retval;
}

4335
static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
4336

4337
void sched_show_task(struct task_struct *p)
L
Linus Torvalds 已提交
4338 4339
{
	unsigned long free = 0;
4340
	int ppid;
4341
	unsigned state;
L
Linus Torvalds 已提交
4342 4343

	state = p->state ? __ffs(p->state) + 1 : 0;
4344
	printk(KERN_INFO "%-15.15s %c", p->comm,
4345
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4346
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
4347
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
4348
		printk(KERN_CONT " running  ");
L
Linus Torvalds 已提交
4349
	else
P
Peter Zijlstra 已提交
4350
		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4351 4352
#else
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
4353
		printk(KERN_CONT "  running task    ");
L
Linus Torvalds 已提交
4354
	else
P
Peter Zijlstra 已提交
4355
		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4356 4357
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
4358
	free = stack_not_used(p);
L
Linus Torvalds 已提交
4359
#endif
4360 4361 4362
	rcu_read_lock();
	ppid = task_pid_nr(rcu_dereference(p->real_parent));
	rcu_read_unlock();
P
Peter Zijlstra 已提交
4363
	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
4364
		task_pid_nr(p), ppid,
4365
		(unsigned long)task_thread_info(p)->flags);
L
Linus Torvalds 已提交
4366

4367
	print_worker_info(KERN_INFO, p);
4368
	show_stack(p, NULL);
L
Linus Torvalds 已提交
4369 4370
}

I
Ingo Molnar 已提交
4371
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
4372
{
4373
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
4374

4375
#if BITS_PER_LONG == 32
P
Peter Zijlstra 已提交
4376 4377
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
4378
#else
P
Peter Zijlstra 已提交
4379 4380
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
4381
#endif
4382
	rcu_read_lock();
L
Linus Torvalds 已提交
4383 4384 4385
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
L
Lucas De Marchi 已提交
4386
		 * console might take a lot of time:
L
Linus Torvalds 已提交
4387 4388
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
4389
		if (!state_filter || (p->state & state_filter))
4390
			sched_show_task(p);
L
Linus Torvalds 已提交
4391 4392
	} while_each_thread(g, p);

4393 4394
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
4395 4396 4397
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
4398
	rcu_read_unlock();
I
Ingo Molnar 已提交
4399 4400 4401
	/*
	 * Only show locks if all tasks are dumped:
	 */
4402
	if (!state_filter)
I
Ingo Molnar 已提交
4403
		debug_show_all_locks();
L
Linus Torvalds 已提交
4404 4405
}

4406
void init_idle_bootup_task(struct task_struct *idle)
I
Ingo Molnar 已提交
4407
{
I
Ingo Molnar 已提交
4408
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
4409 4410
}

4411 4412 4413 4414 4415 4416 4417 4418
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
4419
void init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
4420
{
4421
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4422 4423
	unsigned long flags;

4424
	raw_spin_lock_irqsave(&rq->lock, flags);
4425

4426
	__sched_fork(0, idle);
4427
	idle->state = TASK_RUNNING;
I
Ingo Molnar 已提交
4428 4429
	idle->se.exec_start = sched_clock();

4430
	do_set_cpus_allowed(idle, cpumask_of(cpu));
4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441
	/*
	 * We're having a chicken and egg problem, even though we are
	 * holding rq->lock, the cpu isn't yet set to this cpu so the
	 * lockdep check in task_group() will fail.
	 *
	 * Similar case to sched_fork(). / Alternatively we could
	 * use task_rq_lock() here and obtain the other rq->lock.
	 *
	 * Silence PROVE_RCU
	 */
	rcu_read_lock();
I
Ingo Molnar 已提交
4442
	__set_task_cpu(idle, cpu);
4443
	rcu_read_unlock();
L
Linus Torvalds 已提交
4444 4445

	rq->curr = rq->idle = idle;
P
Peter Zijlstra 已提交
4446 4447
#if defined(CONFIG_SMP)
	idle->on_cpu = 1;
4448
#endif
4449
	raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
4450 4451

	/* Set the preempt count _outside_ the spinlocks! */
4452
	init_idle_preempt_count(idle, cpu);
4453

I
Ingo Molnar 已提交
4454 4455 4456 4457
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
4458
	ftrace_graph_init_idle_task(idle, cpu);
4459
	vtime_init_idle(idle, cpu);
4460 4461 4462
#if defined(CONFIG_SMP)
	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
#endif
I
Ingo Molnar 已提交
4463 4464
}

L
Linus Torvalds 已提交
4465
#ifdef CONFIG_SMP
4466 4467 4468 4469
void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
	if (p->sched_class && p->sched_class->set_cpus_allowed)
		p->sched_class->set_cpus_allowed(p, new_mask);
4470 4471

	cpumask_copy(&p->cpus_allowed, new_mask);
4472
	p->nr_cpus_allowed = cpumask_weight(new_mask);
4473 4474
}

L
Linus Torvalds 已提交
4475 4476 4477
/*
 * This is how migration works:
 *
4478 4479 4480 4481 4482 4483
 * 1) we invoke migration_cpu_stop() on the target CPU using
 *    stop_one_cpu().
 * 2) stopper starts to run (implicitly forcing the migrated thread
 *    off the CPU)
 * 3) it checks whether the migrated task is still in the wrong runqueue.
 * 4) if it's in the wrong runqueue then the migration thread removes
L
Linus Torvalds 已提交
4484
 *    it and puts it into the right queue.
4485 4486
 * 5) stopper completes and stop_one_cpu() returns and the migration
 *    is done.
L
Linus Torvalds 已提交
4487 4488 4489 4490 4491 4492 4493 4494
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
I
Ingo Molnar 已提交
4495
 * task must not exit() & deallocate itself prematurely. The
L
Linus Torvalds 已提交
4496 4497
 * call is not atomic; no spinlocks may be held.
 */
4498
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
L
Linus Torvalds 已提交
4499 4500
{
	unsigned long flags;
4501
	struct rq *rq;
4502
	unsigned int dest_cpu;
4503
	int ret = 0;
L
Linus Torvalds 已提交
4504 4505

	rq = task_rq_lock(p, &flags);
4506

4507 4508 4509
	if (cpumask_equal(&p->cpus_allowed, new_mask))
		goto out;

4510
	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
L
Linus Torvalds 已提交
4511 4512 4513 4514
		ret = -EINVAL;
		goto out;
	}

4515
	do_set_cpus_allowed(p, new_mask);
4516

L
Linus Torvalds 已提交
4517
	/* Can the task run on the task's current CPU? If so, we're done */
4518
	if (cpumask_test_cpu(task_cpu(p), new_mask))
L
Linus Torvalds 已提交
4519 4520
		goto out;

4521
	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
4522
	if (p->on_rq) {
4523
		struct migration_arg arg = { p, dest_cpu };
L
Linus Torvalds 已提交
4524
		/* Need help from migration thread: drop lock and wait. */
4525
		task_rq_unlock(rq, p, &flags);
4526
		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
4527 4528 4529 4530
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
4531
	task_rq_unlock(rq, p, &flags);
4532

L
Linus Torvalds 已提交
4533 4534
	return ret;
}
4535
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
L
Linus Torvalds 已提交
4536 4537

/*
I
Ingo Molnar 已提交
4538
 * Move (not current) task off this cpu, onto dest cpu. We're doing
L
Linus Torvalds 已提交
4539 4540 4541 4542 4543 4544
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
4545 4546
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
4547
 */
4548
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
4549
{
4550
	struct rq *rq_dest, *rq_src;
4551
	int ret = 0;
L
Linus Torvalds 已提交
4552

4553
	if (unlikely(!cpu_active(dest_cpu)))
4554
		return ret;
L
Linus Torvalds 已提交
4555 4556 4557 4558

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

4559
	raw_spin_lock(&p->pi_lock);
L
Linus Torvalds 已提交
4560 4561 4562
	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
L
Linus Torvalds 已提交
4563
		goto done;
L
Linus Torvalds 已提交
4564
	/* Affinity changed (again). */
4565
	if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
L
Linus Torvalds 已提交
4566
		goto fail;
L
Linus Torvalds 已提交
4567

4568 4569 4570 4571
	/*
	 * If we're not on a rq, the next wake-up will ensure we're
	 * placed properly.
	 */
P
Peter Zijlstra 已提交
4572
	if (p->on_rq) {
4573
		dequeue_task(rq_src, p, 0);
4574
		set_task_cpu(p, dest_cpu);
4575
		enqueue_task(rq_dest, p, 0);
4576
		check_preempt_curr(rq_dest, p, 0);
L
Linus Torvalds 已提交
4577
	}
L
Linus Torvalds 已提交
4578
done:
4579
	ret = 1;
L
Linus Torvalds 已提交
4580
fail:
L
Linus Torvalds 已提交
4581
	double_rq_unlock(rq_src, rq_dest);
4582
	raw_spin_unlock(&p->pi_lock);
4583
	return ret;
L
Linus Torvalds 已提交
4584 4585
}

4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600
#ifdef CONFIG_NUMA_BALANCING
/* Migrate current task p to target_cpu */
int migrate_task_to(struct task_struct *p, int target_cpu)
{
	struct migration_arg arg = { p, target_cpu };
	int curr_cpu = task_cpu(p);

	if (curr_cpu == target_cpu)
		return 0;

	if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
		return -EINVAL;

	/* TODO: This is not properly updating schedstats */

4601
	trace_sched_move_numa(p, curr_cpu, target_cpu);
4602 4603
	return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
}
4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631

/*
 * Requeue a task on a given node and accurately track the number of NUMA
 * tasks on the runqueues
 */
void sched_setnuma(struct task_struct *p, int nid)
{
	struct rq *rq;
	unsigned long flags;
	bool on_rq, running;

	rq = task_rq_lock(p, &flags);
	on_rq = p->on_rq;
	running = task_current(rq, p);

	if (on_rq)
		dequeue_task(rq, p, 0);
	if (running)
		p->sched_class->put_prev_task(rq, p);

	p->numa_preferred_nid = nid;

	if (running)
		p->sched_class->set_curr_task(rq);
	if (on_rq)
		enqueue_task(rq, p, 0);
	task_rq_unlock(rq, p, &flags);
}
4632 4633
#endif

L
Linus Torvalds 已提交
4634
/*
4635 4636 4637
 * migration_cpu_stop - this will be executed by a highprio stopper thread
 * and performs thread migration by bumping thread off CPU then
 * 'pushing' onto another runqueue.
L
Linus Torvalds 已提交
4638
 */
4639
static int migration_cpu_stop(void *data)
L
Linus Torvalds 已提交
4640
{
4641
	struct migration_arg *arg = data;
4642

4643 4644 4645 4646
	/*
	 * The original target cpu might have gone down and we might
	 * be on another cpu but it doesn't matter.
	 */
4647
	local_irq_disable();
4648
	__migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
4649
	local_irq_enable();
L
Linus Torvalds 已提交
4650
	return 0;
4651 4652
}

L
Linus Torvalds 已提交
4653
#ifdef CONFIG_HOTPLUG_CPU
4654

4655
/*
4656 4657
 * Ensures that the idle task is using init_mm right before its cpu goes
 * offline.
4658
 */
4659
void idle_task_exit(void)
L
Linus Torvalds 已提交
4660
{
4661
	struct mm_struct *mm = current->active_mm;
4662

4663
	BUG_ON(cpu_online(smp_processor_id()));
4664

4665 4666 4667
	if (mm != &init_mm)
		switch_mm(mm, &init_mm, current);
	mmdrop(mm);
L
Linus Torvalds 已提交
4668 4669 4670
}

/*
4671 4672 4673 4674 4675
 * Since this CPU is going 'away' for a while, fold any nr_active delta
 * we might have. Assumes we're called after migrate_tasks() so that the
 * nr_active count is stable.
 *
 * Also see the comment "Global load-average calculations".
L
Linus Torvalds 已提交
4676
 */
4677
static void calc_load_migrate(struct rq *rq)
L
Linus Torvalds 已提交
4678
{
4679 4680 4681
	long delta = calc_load_fold_active(rq);
	if (delta)
		atomic_long_add(delta, &calc_load_tasks);
L
Linus Torvalds 已提交
4682 4683
}

4684
/*
4685 4686 4687 4688 4689 4690
 * Migrate all tasks from the rq, sleeping tasks will be migrated by
 * try_to_wake_up()->select_task_rq().
 *
 * Called with rq->lock held even though we'er in stop_machine() and
 * there's no concurrency possible, we hold the required locks anyway
 * because of lock validation efforts.
L
Linus Torvalds 已提交
4691
 */
4692
static void migrate_tasks(unsigned int dead_cpu)
L
Linus Torvalds 已提交
4693
{
4694
	struct rq *rq = cpu_rq(dead_cpu);
4695 4696
	struct task_struct *next, *stop = rq->stop;
	int dest_cpu;
L
Linus Torvalds 已提交
4697 4698

	/*
4699 4700 4701 4702 4703 4704 4705
	 * Fudge the rq selection such that the below task selection loop
	 * doesn't get stuck on the currently eligible stop task.
	 *
	 * We're currently inside stop_machine() and the rq is either stuck
	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
	 * either way we should never end up calling schedule() until we're
	 * done here.
L
Linus Torvalds 已提交
4706
	 */
4707
	rq->stop = NULL;
4708

4709 4710 4711 4712 4713 4714 4715
	/*
	 * put_prev_task() and pick_next_task() sched
	 * class method both need to have an up-to-date
	 * value of rq->clock[_task]
	 */
	update_rq_clock(rq);

I
Ingo Molnar 已提交
4716
	for ( ; ; ) {
4717 4718 4719 4720 4721
		/*
		 * There's this thread running, bail when that's the only
		 * remaining thread.
		 */
		if (rq->nr_running == 1)
I
Ingo Molnar 已提交
4722
			break;
4723

4724
		next = pick_next_task(rq, NULL);
4725
		BUG_ON(!next);
D
Dmitry Adamushko 已提交
4726
		next->sched_class->put_prev_task(rq, next);
4727

4728 4729 4730 4731 4732 4733 4734
		/* Find suitable destination for @next, with force if needed. */
		dest_cpu = select_fallback_rq(dead_cpu, next);
		raw_spin_unlock(&rq->lock);

		__migrate_task(next, dead_cpu, dest_cpu);

		raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
4735
	}
4736

4737
	rq->stop = stop;
4738
}
4739

L
Linus Torvalds 已提交
4740 4741
#endif /* CONFIG_HOTPLUG_CPU */

4742 4743 4744
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
4745 4746
	{
		.procname	= "sched_domain",
4747
		.mode		= 0555,
4748
	},
4749
	{}
4750 4751 4752
};

static struct ctl_table sd_ctl_root[] = {
4753 4754
	{
		.procname	= "kernel",
4755
		.mode		= 0555,
4756 4757
		.child		= sd_ctl_dir,
	},
4758
	{}
4759 4760 4761 4762 4763
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
4764
		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
4765 4766 4767 4768

	return entry;
}

4769 4770
static void sd_free_ctl_entry(struct ctl_table **tablep)
{
4771
	struct ctl_table *entry;
4772

4773 4774 4775
	/*
	 * In the intermediate directories, both the child directory and
	 * procname are dynamically allocated and could fail but the mode
I
Ingo Molnar 已提交
4776
	 * will always be set. In the lowest directory the names are
4777 4778 4779
	 * static strings and all have proc handlers.
	 */
	for (entry = *tablep; entry->mode; entry++) {
4780 4781
		if (entry->child)
			sd_free_ctl_entry(&entry->child);
4782 4783 4784
		if (entry->proc_handler == NULL)
			kfree(entry->procname);
	}
4785 4786 4787 4788 4789

	kfree(*tablep);
	*tablep = NULL;
}

4790
static int min_load_idx = 0;
4791
static int max_load_idx = CPU_LOAD_IDX_MAX-1;
4792

4793
static void
4794
set_table_entry(struct ctl_table *entry,
4795
		const char *procname, void *data, int maxlen,
4796 4797
		umode_t mode, proc_handler *proc_handler,
		bool load_idx)
4798 4799 4800 4801 4802 4803
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
4804 4805 4806 4807 4808

	if (load_idx) {
		entry->extra1 = &min_load_idx;
		entry->extra2 = &max_load_idx;
	}
4809 4810 4811 4812 4813
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
4814
	struct ctl_table *table = sd_alloc_ctl_entry(13);
4815

4816 4817 4818
	if (table == NULL)
		return NULL;

4819
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
4820
		sizeof(long), 0644, proc_doulongvec_minmax, false);
4821
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
4822
		sizeof(long), 0644, proc_doulongvec_minmax, false);
4823
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
4824
		sizeof(int), 0644, proc_dointvec_minmax, true);
4825
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
4826
		sizeof(int), 0644, proc_dointvec_minmax, true);
4827
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
4828
		sizeof(int), 0644, proc_dointvec_minmax, true);
4829
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
4830
		sizeof(int), 0644, proc_dointvec_minmax, true);
4831
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
4832
		sizeof(int), 0644, proc_dointvec_minmax, true);
4833
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
4834
		sizeof(int), 0644, proc_dointvec_minmax, false);
4835
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
4836
		sizeof(int), 0644, proc_dointvec_minmax, false);
4837
	set_table_entry(&table[9], "cache_nice_tries",
4838
		&sd->cache_nice_tries,
4839
		sizeof(int), 0644, proc_dointvec_minmax, false);
4840
	set_table_entry(&table[10], "flags", &sd->flags,
4841
		sizeof(int), 0644, proc_dointvec_minmax, false);
4842
	set_table_entry(&table[11], "name", sd->name,
4843
		CORENAME_MAX_SIZE, 0444, proc_dostring, false);
4844
	/* &table[12] is terminator */
4845 4846 4847 4848

	return table;
}

4849
static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
4850 4851 4852 4853 4854 4855 4856 4857 4858
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);
4859 4860
	if (table == NULL)
		return NULL;
4861 4862 4863 4864 4865

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
4866
		entry->mode = 0555;
4867 4868 4869 4870 4871 4872 4873 4874
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
4875
static void register_sched_domain_sysctl(void)
4876
{
4877
	int i, cpu_num = num_possible_cpus();
4878 4879 4880
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

4881 4882 4883
	WARN_ON(sd_ctl_dir[0].child);
	sd_ctl_dir[0].child = entry;

4884 4885 4886
	if (entry == NULL)
		return;

4887
	for_each_possible_cpu(i) {
4888 4889
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
4890
		entry->mode = 0555;
4891
		entry->child = sd_alloc_ctl_cpu_table(i);
4892
		entry++;
4893
	}
4894 4895

	WARN_ON(sd_sysctl_header);
4896 4897
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
4898

4899
/* may be called multiple times per register */
4900 4901
static void unregister_sched_domain_sysctl(void)
{
4902 4903
	if (sd_sysctl_header)
		unregister_sysctl_table(sd_sysctl_header);
4904
	sd_sysctl_header = NULL;
4905 4906
	if (sd_ctl_dir[0].child)
		sd_free_ctl_entry(&sd_ctl_dir[0].child);
4907
}
4908
#else
4909 4910 4911 4912
static void register_sched_domain_sysctl(void)
{
}
static void unregister_sched_domain_sysctl(void)
4913 4914 4915 4916
{
}
#endif

4917 4918 4919 4920 4921
static void set_rq_online(struct rq *rq)
{
	if (!rq->online) {
		const struct sched_class *class;

4922
		cpumask_set_cpu(rq->cpu, rq->rd->online);
4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941
		rq->online = 1;

		for_each_class(class) {
			if (class->rq_online)
				class->rq_online(rq);
		}
	}
}

static void set_rq_offline(struct rq *rq)
{
	if (rq->online) {
		const struct sched_class *class;

		for_each_class(class) {
			if (class->rq_offline)
				class->rq_offline(rq);
		}

4942
		cpumask_clear_cpu(rq->cpu, rq->rd->online);
4943 4944 4945 4946
		rq->online = 0;
	}
}

L
Linus Torvalds 已提交
4947 4948 4949 4950
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
4951
static int
4952
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
4953
{
4954
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
4955
	unsigned long flags;
4956
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4957

4958
	switch (action & ~CPU_TASKS_FROZEN) {
4959

L
Linus Torvalds 已提交
4960
	case CPU_UP_PREPARE:
4961
		rq->calc_load_update = calc_load_update;
L
Linus Torvalds 已提交
4962
		break;
4963

L
Linus Torvalds 已提交
4964
	case CPU_ONLINE:
4965
		/* Update our root-domain */
4966
		raw_spin_lock_irqsave(&rq->lock, flags);
4967
		if (rq->rd) {
4968
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
4969 4970

			set_rq_online(rq);
4971
		}
4972
		raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
4973
		break;
4974

L
Linus Torvalds 已提交
4975
#ifdef CONFIG_HOTPLUG_CPU
4976
	case CPU_DYING:
4977
		sched_ttwu_pending();
G
Gregory Haskins 已提交
4978
		/* Update our root-domain */
4979
		raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
4980
		if (rq->rd) {
4981
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
4982
			set_rq_offline(rq);
G
Gregory Haskins 已提交
4983
		}
4984 4985
		migrate_tasks(cpu);
		BUG_ON(rq->nr_running != 1); /* the migration thread */
4986
		raw_spin_unlock_irqrestore(&rq->lock, flags);
4987
		break;
4988

4989
	case CPU_DEAD:
4990
		calc_load_migrate(rq);
G
Gregory Haskins 已提交
4991
		break;
L
Linus Torvalds 已提交
4992 4993
#endif
	}
4994 4995 4996

	update_max_interval();

L
Linus Torvalds 已提交
4997 4998 4999
	return NOTIFY_OK;
}

5000 5001 5002
/*
 * Register at high priority so that task migration (migrate_all_tasks)
 * happens before everything else.  This has to be lower priority than
5003
 * the notifier in the perf_event subsystem, though.
L
Linus Torvalds 已提交
5004
 */
5005
static struct notifier_block migration_notifier = {
L
Linus Torvalds 已提交
5006
	.notifier_call = migration_call,
5007
	.priority = CPU_PRI_MIGRATION,
L
Linus Torvalds 已提交
5008 5009
};

5010
static int sched_cpu_active(struct notifier_block *nfb,
5011 5012 5013
				      unsigned long action, void *hcpu)
{
	switch (action & ~CPU_TASKS_FROZEN) {
5014
	case CPU_STARTING:
5015 5016 5017 5018 5019 5020 5021 5022
	case CPU_DOWN_FAILED:
		set_cpu_active((long)hcpu, true);
		return NOTIFY_OK;
	default:
		return NOTIFY_DONE;
	}
}

5023
static int sched_cpu_inactive(struct notifier_block *nfb,
5024 5025
					unsigned long action, void *hcpu)
{
5026 5027 5028
	unsigned long flags;
	long cpu = (long)hcpu;

5029 5030
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_DOWN_PREPARE:
5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046
		set_cpu_active(cpu, false);

		/* explicitly allow suspend */
		if (!(action & CPU_TASKS_FROZEN)) {
			struct dl_bw *dl_b = dl_bw_of(cpu);
			bool overflow;
			int cpus;

			raw_spin_lock_irqsave(&dl_b->lock, flags);
			cpus = dl_bw_cpus(cpu);
			overflow = __dl_overflow(dl_b, cpus, 0, 0);
			raw_spin_unlock_irqrestore(&dl_b->lock, flags);

			if (overflow)
				return notifier_from_errno(-EBUSY);
		}
5047 5048
		return NOTIFY_OK;
	}
5049 5050

	return NOTIFY_DONE;
5051 5052
}

5053
static int __init migration_init(void)
L
Linus Torvalds 已提交
5054 5055
{
	void *cpu = (void *)(long)smp_processor_id();
5056
	int err;
5057

5058
	/* Initialize migration for the boot CPU */
5059 5060
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
5061 5062
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
5063

5064 5065 5066 5067
	/* Register cpu active notifiers */
	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);

5068
	return 0;
L
Linus Torvalds 已提交
5069
}
5070
early_initcall(migration_init);
L
Linus Torvalds 已提交
5071 5072 5073
#endif

#ifdef CONFIG_SMP
5074

5075 5076
static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */

5077
#ifdef CONFIG_SCHED_DEBUG
I
Ingo Molnar 已提交
5078

5079
static __read_mostly int sched_debug_enabled;
5080

5081
static int __init sched_debug_setup(char *str)
5082
{
5083
	sched_debug_enabled = 1;
5084 5085 5086

	return 0;
}
5087 5088 5089 5090 5091 5092
early_param("sched_debug", sched_debug_setup);

static inline bool sched_debug(void)
{
	return sched_debug_enabled;
}
5093

5094
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5095
				  struct cpumask *groupmask)
L
Linus Torvalds 已提交
5096
{
I
Ingo Molnar 已提交
5097
	struct sched_group *group = sd->groups;
5098
	char str[256];
L
Linus Torvalds 已提交
5099

R
Rusty Russell 已提交
5100
	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
5101
	cpumask_clear(groupmask);
I
Ingo Molnar 已提交
5102 5103 5104 5105

	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);

	if (!(sd->flags & SD_LOAD_BALANCE)) {
P
Peter Zijlstra 已提交
5106
		printk("does not load-balance\n");
I
Ingo Molnar 已提交
5107
		if (sd->parent)
P
Peter Zijlstra 已提交
5108 5109
			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
					" has parent");
I
Ingo Molnar 已提交
5110
		return -1;
N
Nick Piggin 已提交
5111 5112
	}

P
Peter Zijlstra 已提交
5113
	printk(KERN_CONT "span %s level %s\n", str, sd->name);
I
Ingo Molnar 已提交
5114

5115
	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
P
Peter Zijlstra 已提交
5116 5117
		printk(KERN_ERR "ERROR: domain->span does not contain "
				"CPU%d\n", cpu);
I
Ingo Molnar 已提交
5118
	}
5119
	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5120 5121
		printk(KERN_ERR "ERROR: domain->groups does not contain"
				" CPU%d\n", cpu);
I
Ingo Molnar 已提交
5122
	}
L
Linus Torvalds 已提交
5123

I
Ingo Molnar 已提交
5124
	printk(KERN_DEBUG "%*s groups:", level + 1, "");
L
Linus Torvalds 已提交
5125
	do {
I
Ingo Molnar 已提交
5126
		if (!group) {
P
Peter Zijlstra 已提交
5127 5128
			printk("\n");
			printk(KERN_ERR "ERROR: group is NULL\n");
L
Linus Torvalds 已提交
5129 5130 5131
			break;
		}

5132 5133 5134 5135 5136 5137
		/*
		 * Even though we initialize ->power to something semi-sane,
		 * we leave power_orig unset. This allows us to detect if
		 * domain iteration is still funny without causing /0 traps.
		 */
		if (!group->sgp->power_orig) {
P
Peter Zijlstra 已提交
5138 5139 5140
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: domain->cpu_power not "
					"set\n");
I
Ingo Molnar 已提交
5141 5142
			break;
		}
L
Linus Torvalds 已提交
5143

5144
		if (!cpumask_weight(sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5145 5146
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: empty group\n");
I
Ingo Molnar 已提交
5147 5148
			break;
		}
L
Linus Torvalds 已提交
5149

5150 5151
		if (!(sd->flags & SD_OVERLAP) &&
		    cpumask_intersects(groupmask, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5152 5153
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: repeated CPUs\n");
I
Ingo Molnar 已提交
5154 5155
			break;
		}
L
Linus Torvalds 已提交
5156

5157
		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
L
Linus Torvalds 已提交
5158

R
Rusty Russell 已提交
5159
		cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
5160

P
Peter Zijlstra 已提交
5161
		printk(KERN_CONT " %s", str);
5162
		if (group->sgp->power != SCHED_POWER_SCALE) {
P
Peter Zijlstra 已提交
5163
			printk(KERN_CONT " (cpu_power = %d)",
5164
				group->sgp->power);
5165
		}
L
Linus Torvalds 已提交
5166

I
Ingo Molnar 已提交
5167 5168
		group = group->next;
	} while (group != sd->groups);
P
Peter Zijlstra 已提交
5169
	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
5170

5171
	if (!cpumask_equal(sched_domain_span(sd), groupmask))
P
Peter Zijlstra 已提交
5172
		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
L
Linus Torvalds 已提交
5173

5174 5175
	if (sd->parent &&
	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
P
Peter Zijlstra 已提交
5176 5177
		printk(KERN_ERR "ERROR: parent span is not a superset "
			"of domain->span\n");
I
Ingo Molnar 已提交
5178 5179
	return 0;
}
L
Linus Torvalds 已提交
5180

I
Ingo Molnar 已提交
5181 5182 5183
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;
L
Linus Torvalds 已提交
5184

5185
	if (!sched_debug_enabled)
5186 5187
		return;

I
Ingo Molnar 已提交
5188 5189 5190 5191
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}
L
Linus Torvalds 已提交
5192

I
Ingo Molnar 已提交
5193 5194 5195
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	for (;;) {
5196
		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
I
Ingo Molnar 已提交
5197
			break;
L
Linus Torvalds 已提交
5198 5199
		level++;
		sd = sd->parent;
5200
		if (!sd)
I
Ingo Molnar 已提交
5201 5202
			break;
	}
L
Linus Torvalds 已提交
5203
}
5204
#else /* !CONFIG_SCHED_DEBUG */
5205
# define sched_domain_debug(sd, cpu) do { } while (0)
5206 5207 5208 5209
static inline bool sched_debug(void)
{
	return false;
}
5210
#endif /* CONFIG_SCHED_DEBUG */
L
Linus Torvalds 已提交
5211

5212
static int sd_degenerate(struct sched_domain *sd)
5213
{
5214
	if (cpumask_weight(sched_domain_span(sd)) == 1)
5215 5216 5217 5218 5219 5220
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
5221 5222 5223
			 SD_BALANCE_EXEC |
			 SD_SHARE_CPUPOWER |
			 SD_SHARE_PKG_RESOURCES)) {
5224 5225 5226 5227 5228
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
5229
	if (sd->flags & (SD_WAKE_AFFINE))
5230 5231 5232 5233 5234
		return 0;

	return 1;
}

5235 5236
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5237 5238 5239 5240 5241 5242
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

5243
	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
5244 5245 5246 5247 5248 5249 5250
		return 0;

	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
5251 5252
				SD_BALANCE_EXEC |
				SD_SHARE_CPUPOWER |
5253 5254
				SD_SHARE_PKG_RESOURCES |
				SD_PREFER_SIBLING);
5255 5256
		if (nr_node_ids == 1)
			pflags &= ~SD_SERIALIZE;
5257 5258 5259 5260 5261 5262 5263
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

5264
static void free_rootdomain(struct rcu_head *rcu)
5265
{
5266
	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
5267

5268
	cpupri_cleanup(&rd->cpupri);
5269
	cpudl_cleanup(&rd->cpudl);
5270
	free_cpumask_var(rd->dlo_mask);
5271 5272 5273 5274 5275 5276
	free_cpumask_var(rd->rto_mask);
	free_cpumask_var(rd->online);
	free_cpumask_var(rd->span);
	kfree(rd);
}

G
Gregory Haskins 已提交
5277 5278
static void rq_attach_root(struct rq *rq, struct root_domain *rd)
{
I
Ingo Molnar 已提交
5279
	struct root_domain *old_rd = NULL;
G
Gregory Haskins 已提交
5280 5281
	unsigned long flags;

5282
	raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
5283 5284

	if (rq->rd) {
I
Ingo Molnar 已提交
5285
		old_rd = rq->rd;
G
Gregory Haskins 已提交
5286

5287
		if (cpumask_test_cpu(rq->cpu, old_rd->online))
5288
			set_rq_offline(rq);
G
Gregory Haskins 已提交
5289

5290
		cpumask_clear_cpu(rq->cpu, old_rd->span);
5291

I
Ingo Molnar 已提交
5292
		/*
5293
		 * If we dont want to free the old_rd yet then
I
Ingo Molnar 已提交
5294 5295 5296 5297 5298
		 * set old_rd to NULL to skip the freeing later
		 * in this function:
		 */
		if (!atomic_dec_and_test(&old_rd->refcount))
			old_rd = NULL;
G
Gregory Haskins 已提交
5299 5300 5301 5302 5303
	}

	atomic_inc(&rd->refcount);
	rq->rd = rd;

5304
	cpumask_set_cpu(rq->cpu, rd->span);
5305
	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
5306
		set_rq_online(rq);
G
Gregory Haskins 已提交
5307

5308
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
5309 5310

	if (old_rd)
5311
		call_rcu_sched(&old_rd->rcu, free_rootdomain);
G
Gregory Haskins 已提交
5312 5313
}

5314
static int init_rootdomain(struct root_domain *rd)
G
Gregory Haskins 已提交
5315 5316 5317
{
	memset(rd, 0, sizeof(*rd));

5318
	if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
5319
		goto out;
5320
	if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
5321
		goto free_span;
5322
	if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
5323
		goto free_online;
5324 5325
	if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
		goto free_dlo_mask;
5326

5327
	init_dl_bw(&rd->dl_bw);
5328 5329
	if (cpudl_init(&rd->cpudl) != 0)
		goto free_dlo_mask;
5330

5331
	if (cpupri_init(&rd->cpupri) != 0)
5332
		goto free_rto_mask;
5333
	return 0;
5334

5335 5336
free_rto_mask:
	free_cpumask_var(rd->rto_mask);
5337 5338
free_dlo_mask:
	free_cpumask_var(rd->dlo_mask);
5339 5340 5341 5342
free_online:
	free_cpumask_var(rd->online);
free_span:
	free_cpumask_var(rd->span);
5343
out:
5344
	return -ENOMEM;
G
Gregory Haskins 已提交
5345 5346
}

5347 5348 5349 5350 5351 5352
/*
 * By default the system creates a single root-domain with all cpus as
 * members (mimicking the global state we have today).
 */
struct root_domain def_root_domain;

G
Gregory Haskins 已提交
5353 5354
static void init_defrootdomain(void)
{
5355
	init_rootdomain(&def_root_domain);
5356

G
Gregory Haskins 已提交
5357 5358 5359
	atomic_set(&def_root_domain.refcount, 1);
}

5360
static struct root_domain *alloc_rootdomain(void)
G
Gregory Haskins 已提交
5361 5362 5363 5364 5365 5366 5367
{
	struct root_domain *rd;

	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
	if (!rd)
		return NULL;

5368
	if (init_rootdomain(rd) != 0) {
5369 5370 5371
		kfree(rd);
		return NULL;
	}
G
Gregory Haskins 已提交
5372 5373 5374 5375

	return rd;
}

5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394
static void free_sched_groups(struct sched_group *sg, int free_sgp)
{
	struct sched_group *tmp, *first;

	if (!sg)
		return;

	first = sg;
	do {
		tmp = sg->next;

		if (free_sgp && atomic_dec_and_test(&sg->sgp->ref))
			kfree(sg->sgp);

		kfree(sg);
		sg = tmp;
	} while (sg != first);
}

5395 5396 5397
static void free_sched_domain(struct rcu_head *rcu)
{
	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
5398 5399 5400 5401 5402 5403 5404 5405

	/*
	 * If its an overlapping domain it has private groups, iterate and
	 * nuke them all.
	 */
	if (sd->flags & SD_OVERLAP) {
		free_sched_groups(sd->groups, 1);
	} else if (atomic_dec_and_test(&sd->groups->ref)) {
5406
		kfree(sd->groups->sgp);
5407
		kfree(sd->groups);
5408
	}
5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422
	kfree(sd);
}

static void destroy_sched_domain(struct sched_domain *sd, int cpu)
{
	call_rcu(&sd->rcu, free_sched_domain);
}

static void destroy_sched_domains(struct sched_domain *sd, int cpu)
{
	for (; sd; sd = sd->parent)
		destroy_sched_domain(sd, cpu);
}

5423 5424 5425 5426 5427 5428 5429
/*
 * Keep a special pointer to the highest sched_domain that has
 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
 * allows us to avoid some pointer chasing select_idle_sibling().
 *
 * Also keep a unique ID per domain (we use the first cpu number in
 * the cpumask of the domain), this allows us to quickly tell if
5430
 * two cpus are in the same cache domain, see cpus_share_cache().
5431 5432
 */
DEFINE_PER_CPU(struct sched_domain *, sd_llc);
5433
DEFINE_PER_CPU(int, sd_llc_size);
5434
DEFINE_PER_CPU(int, sd_llc_id);
5435
DEFINE_PER_CPU(struct sched_domain *, sd_numa);
5436 5437
DEFINE_PER_CPU(struct sched_domain *, sd_busy);
DEFINE_PER_CPU(struct sched_domain *, sd_asym);
5438 5439 5440 5441

static void update_top_cache_domain(int cpu)
{
	struct sched_domain *sd;
5442
	struct sched_domain *busy_sd = NULL;
5443
	int id = cpu;
5444
	int size = 1;
5445 5446

	sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
5447
	if (sd) {
5448
		id = cpumask_first(sched_domain_span(sd));
5449
		size = cpumask_weight(sched_domain_span(sd));
5450
		busy_sd = sd->parent; /* sd_busy */
5451
	}
5452
	rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd);
5453 5454

	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
5455
	per_cpu(sd_llc_size, cpu) = size;
5456
	per_cpu(sd_llc_id, cpu) = id;
5457 5458 5459

	sd = lowest_flag_domain(cpu, SD_NUMA);
	rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
5460 5461 5462

	sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
	rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
5463 5464
}

L
Linus Torvalds 已提交
5465
/*
I
Ingo Molnar 已提交
5466
 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
L
Linus Torvalds 已提交
5467 5468
 * hold the hotplug lock.
 */
I
Ingo Molnar 已提交
5469 5470
static void
cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
L
Linus Torvalds 已提交
5471
{
5472
	struct rq *rq = cpu_rq(cpu);
5473 5474 5475
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
5476
	for (tmp = sd; tmp; ) {
5477 5478 5479
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
5480

5481
		if (sd_parent_degenerate(tmp, parent)) {
5482
			tmp->parent = parent->parent;
5483 5484
			if (parent->parent)
				parent->parent->child = tmp;
5485 5486 5487 5488 5489 5490 5491
			/*
			 * Transfer SD_PREFER_SIBLING down in case of a
			 * degenerate parent; the spans match for this
			 * so the property transfers.
			 */
			if (parent->flags & SD_PREFER_SIBLING)
				tmp->flags |= SD_PREFER_SIBLING;
5492
			destroy_sched_domain(parent, cpu);
5493 5494
		} else
			tmp = tmp->parent;
5495 5496
	}

5497
	if (sd && sd_degenerate(sd)) {
5498
		tmp = sd;
5499
		sd = sd->parent;
5500
		destroy_sched_domain(tmp, cpu);
5501 5502 5503
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
5504

5505
	sched_domain_debug(sd, cpu);
L
Linus Torvalds 已提交
5506

G
Gregory Haskins 已提交
5507
	rq_attach_root(rq, rd);
5508
	tmp = rq->sd;
N
Nick Piggin 已提交
5509
	rcu_assign_pointer(rq->sd, sd);
5510
	destroy_sched_domains(tmp, cpu);
5511 5512

	update_top_cache_domain(cpu);
L
Linus Torvalds 已提交
5513 5514 5515
}

/* cpus with isolated domains */
5516
static cpumask_var_t cpu_isolated_map;
L
Linus Torvalds 已提交
5517 5518 5519 5520

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
R
Rusty Russell 已提交
5521
	alloc_bootmem_cpumask_var(&cpu_isolated_map);
R
Rusty Russell 已提交
5522
	cpulist_parse(str, cpu_isolated_map);
L
Linus Torvalds 已提交
5523 5524 5525
	return 1;
}

I
Ingo Molnar 已提交
5526
__setup("isolcpus=", isolated_cpu_setup);
L
Linus Torvalds 已提交
5527

5528 5529 5530 5531 5532
static const struct cpumask *cpu_cpu_mask(int cpu)
{
	return cpumask_of_node(cpu_to_node(cpu));
}

5533 5534 5535
struct sd_data {
	struct sched_domain **__percpu sd;
	struct sched_group **__percpu sg;
5536
	struct sched_group_power **__percpu sgp;
5537 5538
};

5539
struct s_data {
5540
	struct sched_domain ** __percpu sd;
5541 5542 5543
	struct root_domain	*rd;
};

5544 5545
enum s_alloc {
	sa_rootdomain,
5546
	sa_sd,
5547
	sa_sd_storage,
5548 5549 5550
	sa_none,
};

5551 5552 5553
struct sched_domain_topology_level;

typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu);
5554 5555
typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);

5556 5557
#define SDTL_OVERLAP	0x01

5558
struct sched_domain_topology_level {
5559 5560
	sched_domain_init_f init;
	sched_domain_mask_f mask;
5561
	int		    flags;
5562
	int		    numa_level;
5563
	struct sd_data      data;
5564 5565
};

P
Peter Zijlstra 已提交
5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603
/*
 * Build an iteration mask that can exclude certain CPUs from the upwards
 * domain traversal.
 *
 * Asymmetric node setups can result in situations where the domain tree is of
 * unequal depth, make sure to skip domains that already cover the entire
 * range.
 *
 * In that case build_sched_domains() will have terminated the iteration early
 * and our sibling sd spans will be empty. Domains should always include the
 * cpu they're built on, so check that.
 *
 */
static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
{
	const struct cpumask *span = sched_domain_span(sd);
	struct sd_data *sdd = sd->private;
	struct sched_domain *sibling;
	int i;

	for_each_cpu(i, span) {
		sibling = *per_cpu_ptr(sdd->sd, i);
		if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
			continue;

		cpumask_set_cpu(i, sched_group_mask(sg));
	}
}

/*
 * Return the canonical balance cpu for this group, this is the first cpu
 * of this group that's also in the iteration mask.
 */
int group_balance_cpu(struct sched_group *sg)
{
	return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
}

5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621
static int
build_overlap_sched_groups(struct sched_domain *sd, int cpu)
{
	struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
	const struct cpumask *span = sched_domain_span(sd);
	struct cpumask *covered = sched_domains_tmpmask;
	struct sd_data *sdd = sd->private;
	struct sched_domain *child;
	int i;

	cpumask_clear(covered);

	for_each_cpu(i, span) {
		struct cpumask *sg_span;

		if (cpumask_test_cpu(i, covered))
			continue;

P
Peter Zijlstra 已提交
5622 5623 5624 5625 5626 5627
		child = *per_cpu_ptr(sdd->sd, i);

		/* See the comment near build_group_mask(). */
		if (!cpumask_test_cpu(i, sched_domain_span(child)))
			continue;

5628
		sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
5629
				GFP_KERNEL, cpu_to_node(cpu));
5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642

		if (!sg)
			goto fail;

		sg_span = sched_group_cpus(sg);
		if (child->child) {
			child = child->child;
			cpumask_copy(sg_span, sched_domain_span(child));
		} else
			cpumask_set_cpu(i, sg_span);

		cpumask_or(covered, covered, sg_span);

P
Peter Zijlstra 已提交
5643
		sg->sgp = *per_cpu_ptr(sdd->sgp, i);
P
Peter Zijlstra 已提交
5644 5645 5646
		if (atomic_inc_return(&sg->sgp->ref) == 1)
			build_group_mask(sd, sg);

5647 5648 5649 5650 5651 5652
		/*
		 * Initialize sgp->power such that even if we mess up the
		 * domains and no possible iteration will get us here, we won't
		 * die on a /0 trap.
		 */
		sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span);
5653
		sg->sgp->power_orig = sg->sgp->power;
5654

P
Peter Zijlstra 已提交
5655 5656 5657 5658 5659
		/*
		 * Make sure the first group of this domain contains the
		 * canonical balance cpu. Otherwise the sched_domain iteration
		 * breaks. See update_sg_lb_stats().
		 */
P
Peter Zijlstra 已提交
5660
		if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
P
Peter Zijlstra 已提交
5661
		    group_balance_cpu(sg) == cpu)
5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680
			groups = sg;

		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
		last->next = first;
	}
	sd->groups = groups;

	return 0;

fail:
	free_sched_groups(first, 0);

	return -ENOMEM;
}

5681
static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
L
Linus Torvalds 已提交
5682
{
5683 5684
	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
	struct sched_domain *child = sd->child;
L
Linus Torvalds 已提交
5685

5686 5687
	if (child)
		cpu = cpumask_first(sched_domain_span(child));
5688

5689
	if (sg) {
5690
		*sg = *per_cpu_ptr(sdd->sg, cpu);
5691
		(*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu);
5692
		atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */
5693
	}
5694 5695

	return cpu;
5696 5697
}

5698
/*
5699 5700 5701
 * build_sched_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
 * and ->cpu_power to 0.
5702 5703
 *
 * Assumes the sched_domain tree is fully constructed
5704
 */
5705 5706
static int
build_sched_groups(struct sched_domain *sd, int cpu)
L
Linus Torvalds 已提交
5707
{
5708 5709 5710
	struct sched_group *first = NULL, *last = NULL;
	struct sd_data *sdd = sd->private;
	const struct cpumask *span = sched_domain_span(sd);
5711
	struct cpumask *covered;
5712
	int i;
5713

5714 5715 5716
	get_group(cpu, sdd, &sd->groups);
	atomic_inc(&sd->groups->ref);

5717
	if (cpu != cpumask_first(span))
5718 5719
		return 0;

5720 5721 5722
	lockdep_assert_held(&sched_domains_mutex);
	covered = sched_domains_tmpmask;

5723
	cpumask_clear(covered);
5724

5725 5726
	for_each_cpu(i, span) {
		struct sched_group *sg;
5727
		int group, j;
5728

5729 5730
		if (cpumask_test_cpu(i, covered))
			continue;
5731

5732
		group = get_group(i, sdd, &sg);
5733
		cpumask_clear(sched_group_cpus(sg));
5734
		sg->sgp->power = 0;
P
Peter Zijlstra 已提交
5735
		cpumask_setall(sched_group_mask(sg));
5736

5737 5738 5739
		for_each_cpu(j, span) {
			if (get_group(j, sdd, NULL) != group)
				continue;
5740

5741 5742 5743
			cpumask_set_cpu(j, covered);
			cpumask_set_cpu(j, sched_group_cpus(sg));
		}
5744

5745 5746 5747 5748 5749 5750 5751
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
5752 5753

	return 0;
5754
}
5755

5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767
/*
 * Initialize sched groups cpu_power.
 *
 * cpu_power indicates the capacity of sched group, which is used while
 * distributing the load between different sched groups in a sched domain.
 * Typically cpu_power for all the groups in a sched domain will be same unless
 * there are asymmetries in the topology. If there are asymmetries, group
 * having more cpu_power will pickup more load compared to the group having
 * less cpu_power.
 */
static void init_sched_groups_power(int cpu, struct sched_domain *sd)
{
5768
	struct sched_group *sg = sd->groups;
5769

5770
	WARN_ON(!sg);
5771 5772 5773 5774 5775

	do {
		sg->group_weight = cpumask_weight(sched_group_cpus(sg));
		sg = sg->next;
	} while (sg != sd->groups);
5776

P
Peter Zijlstra 已提交
5777
	if (cpu != group_balance_cpu(sg))
5778
		return;
5779

5780
	update_group_power(sd, cpu);
5781
	atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight);
5782 5783
}

5784 5785 5786
int __weak arch_sd_sibling_asym_packing(void)
{
       return 0*SD_ASYM_PACKING;
5787 5788
}

5789 5790 5791 5792 5793
/*
 * Initializers for schedule domains
 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
 */

5794 5795 5796 5797 5798 5799
#ifdef CONFIG_SCHED_DEBUG
# define SD_INIT_NAME(sd, type)		sd->name = #type
#else
# define SD_INIT_NAME(sd, type)		do { } while (0)
#endif

5800 5801 5802 5803 5804 5805 5806 5807 5808
#define SD_INIT_FUNC(type)						\
static noinline struct sched_domain *					\
sd_init_##type(struct sched_domain_topology_level *tl, int cpu) 	\
{									\
	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);	\
	*sd = SD_##type##_INIT;						\
	SD_INIT_NAME(sd, type);						\
	sd->private = &tl->data;					\
	return sd;							\
5809 5810 5811 5812 5813 5814 5815 5816 5817
}

SD_INIT_FUNC(CPU)
#ifdef CONFIG_SCHED_SMT
 SD_INIT_FUNC(SIBLING)
#endif
#ifdef CONFIG_SCHED_MC
 SD_INIT_FUNC(MC)
#endif
5818 5819 5820
#ifdef CONFIG_SCHED_BOOK
 SD_INIT_FUNC(BOOK)
#endif
5821

5822
static int default_relax_domain_level = -1;
5823
int sched_domain_level_max;
5824 5825 5826

static int __init setup_relax_domain_level(char *str)
{
5827 5828
	if (kstrtoint(str, 0, &default_relax_domain_level))
		pr_warn("Unable to set relax_domain_level\n");
5829

5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847
	return 1;
}
__setup("relax_domain_level=", setup_relax_domain_level);

static void set_domain_attribute(struct sched_domain *sd,
				 struct sched_domain_attr *attr)
{
	int request;

	if (!attr || attr->relax_domain_level < 0) {
		if (default_relax_domain_level < 0)
			return;
		else
			request = default_relax_domain_level;
	} else
		request = attr->relax_domain_level;
	if (request < sd->level) {
		/* turn off idle balance on this domain */
5848
		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
5849 5850
	} else {
		/* turn on idle balance on this domain */
5851
		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
5852 5853 5854
	}
}

5855 5856 5857
static void __sdt_free(const struct cpumask *cpu_map);
static int __sdt_alloc(const struct cpumask *cpu_map);

5858 5859 5860 5861 5862
static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
				 const struct cpumask *cpu_map)
{
	switch (what) {
	case sa_rootdomain:
5863 5864
		if (!atomic_read(&d->rd->refcount))
			free_rootdomain(&d->rd->rcu); /* fall through */
5865 5866
	case sa_sd:
		free_percpu(d->sd); /* fall through */
5867
	case sa_sd_storage:
5868
		__sdt_free(cpu_map); /* fall through */
5869 5870 5871 5872
	case sa_none:
		break;
	}
}
5873

5874 5875 5876
static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
						   const struct cpumask *cpu_map)
{
5877 5878
	memset(d, 0, sizeof(*d));

5879 5880
	if (__sdt_alloc(cpu_map))
		return sa_sd_storage;
5881 5882 5883
	d->sd = alloc_percpu(struct sched_domain *);
	if (!d->sd)
		return sa_sd_storage;
5884
	d->rd = alloc_rootdomain();
5885
	if (!d->rd)
5886
		return sa_sd;
5887 5888
	return sa_rootdomain;
}
G
Gregory Haskins 已提交
5889

5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901
/*
 * NULL the sd_data elements we've used to build the sched_domain and
 * sched_group structure so that the subsequent __free_domain_allocs()
 * will not free the data we're using.
 */
static void claim_allocations(int cpu, struct sched_domain *sd)
{
	struct sd_data *sdd = sd->private;

	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
	*per_cpu_ptr(sdd->sd, cpu) = NULL;

5902
	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
5903
		*per_cpu_ptr(sdd->sg, cpu) = NULL;
5904 5905

	if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref))
5906
		*per_cpu_ptr(sdd->sgp, cpu) = NULL;
5907 5908
}

5909 5910
#ifdef CONFIG_SCHED_SMT
static const struct cpumask *cpu_smt_mask(int cpu)
5911
{
5912
	return topology_thread_cpumask(cpu);
5913
}
5914
#endif
5915

5916 5917 5918
/*
 * Topology list, bottom-up.
 */
5919
static struct sched_domain_topology_level default_topology[] = {
5920 5921
#ifdef CONFIG_SCHED_SMT
	{ sd_init_SIBLING, cpu_smt_mask, },
5922
#endif
5923
#ifdef CONFIG_SCHED_MC
5924
	{ sd_init_MC, cpu_coregroup_mask, },
5925
#endif
5926 5927 5928 5929
#ifdef CONFIG_SCHED_BOOK
	{ sd_init_BOOK, cpu_book_mask, },
#endif
	{ sd_init_CPU, cpu_cpu_mask, },
5930 5931 5932 5933 5934
	{ NULL, },
};

static struct sched_domain_topology_level *sched_domain_topology = default_topology;

5935 5936 5937
#define for_each_sd_topology(tl)			\
	for (tl = sched_domain_topology; tl->init; tl++)

5938 5939 5940 5941 5942 5943 5944 5945 5946
#ifdef CONFIG_NUMA

static int sched_domains_numa_levels;
static int *sched_domains_numa_distance;
static struct cpumask ***sched_domains_numa_masks;
static int sched_domains_curr_level;

static inline int sd_local_flags(int level)
{
5947
	if (sched_domains_numa_distance[level] > RECLAIM_DISTANCE)
5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964
		return 0;

	return SD_BALANCE_EXEC | SD_BALANCE_FORK | SD_WAKE_AFFINE;
}

static struct sched_domain *
sd_numa_init(struct sched_domain_topology_level *tl, int cpu)
{
	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
	int level = tl->numa_level;
	int sd_weight = cpumask_weight(
			sched_domains_numa_masks[level][cpu_to_node(cpu)]);

	*sd = (struct sched_domain){
		.min_interval		= sd_weight,
		.max_interval		= 2*sd_weight,
		.busy_factor		= 32,
5965
		.imbalance_pct		= 125,
5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982
		.cache_nice_tries	= 2,
		.busy_idx		= 3,
		.idle_idx		= 2,
		.newidle_idx		= 0,
		.wake_idx		= 0,
		.forkexec_idx		= 0,

		.flags			= 1*SD_LOAD_BALANCE
					| 1*SD_BALANCE_NEWIDLE
					| 0*SD_BALANCE_EXEC
					| 0*SD_BALANCE_FORK
					| 0*SD_BALANCE_WAKE
					| 0*SD_WAKE_AFFINE
					| 0*SD_SHARE_CPUPOWER
					| 0*SD_SHARE_PKG_RESOURCES
					| 1*SD_SERIALIZE
					| 0*SD_PREFER_SIBLING
5983
					| 1*SD_NUMA
5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004
					| sd_local_flags(level)
					,
		.last_balance		= jiffies,
		.balance_interval	= sd_weight,
	};
	SD_INIT_NAME(sd, NUMA);
	sd->private = &tl->data;

	/*
	 * Ugly hack to pass state to sd_numa_mask()...
	 */
	sched_domains_curr_level = tl->numa_level;

	return sd;
}

static const struct cpumask *sd_numa_mask(int cpu)
{
	return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
}

6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040
static void sched_numa_warn(const char *str)
{
	static int done = false;
	int i,j;

	if (done)
		return;

	done = true;

	printk(KERN_WARNING "ERROR: %s\n\n", str);

	for (i = 0; i < nr_node_ids; i++) {
		printk(KERN_WARNING "  ");
		for (j = 0; j < nr_node_ids; j++)
			printk(KERN_CONT "%02d ", node_distance(i,j));
		printk(KERN_CONT "\n");
	}
	printk(KERN_WARNING "\n");
}

static bool find_numa_distance(int distance)
{
	int i;

	if (distance == node_distance(0, 0))
		return true;

	for (i = 0; i < sched_domains_numa_levels; i++) {
		if (sched_domains_numa_distance[i] == distance)
			return true;
	}

	return false;
}

6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061
static void sched_init_numa(void)
{
	int next_distance, curr_distance = node_distance(0, 0);
	struct sched_domain_topology_level *tl;
	int level = 0;
	int i, j, k;

	sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
	if (!sched_domains_numa_distance)
		return;

	/*
	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
	 * unique distances in the node_distance() table.
	 *
	 * Assumes node_distance(0,j) includes all distances in
	 * node_distance(i,j) in order to avoid cubic time.
	 */
	next_distance = curr_distance;
	for (i = 0; i < nr_node_ids; i++) {
		for (j = 0; j < nr_node_ids; j++) {
6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085
			for (k = 0; k < nr_node_ids; k++) {
				int distance = node_distance(i, k);

				if (distance > curr_distance &&
				    (distance < next_distance ||
				     next_distance == curr_distance))
					next_distance = distance;

				/*
				 * While not a strong assumption it would be nice to know
				 * about cases where if node A is connected to B, B is not
				 * equally connected to A.
				 */
				if (sched_debug() && node_distance(k, i) != distance)
					sched_numa_warn("Node-distance not symmetric");

				if (sched_debug() && i && !find_numa_distance(distance))
					sched_numa_warn("Node-0 not representative");
			}
			if (next_distance != curr_distance) {
				sched_domains_numa_distance[level++] = next_distance;
				sched_domains_numa_levels = level;
				curr_distance = next_distance;
			} else break;
6086
		}
6087 6088 6089 6090 6091 6092

		/*
		 * In case of sched_debug() we verify the above assumption.
		 */
		if (!sched_debug())
			break;
6093 6094 6095 6096 6097
	}
	/*
	 * 'level' contains the number of unique distances, excluding the
	 * identity distance node_distance(i,i).
	 *
V
Viresh Kumar 已提交
6098
	 * The sched_domains_numa_distance[] array includes the actual distance
6099 6100 6101
	 * numbers.
	 */

6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112
	/*
	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
	 * the array will contain less then 'level' members. This could be
	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
	 * in other functions.
	 *
	 * We reset it to 'level' at the end of this function.
	 */
	sched_domains_numa_levels = 0;

6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127
	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
	if (!sched_domains_numa_masks)
		return;

	/*
	 * Now for each level, construct a mask per node which contains all
	 * cpus of nodes that are that many hops away from us.
	 */
	for (i = 0; i < level; i++) {
		sched_domains_numa_masks[i] =
			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
		if (!sched_domains_numa_masks[i])
			return;

		for (j = 0; j < nr_node_ids; j++) {
6128
			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
6129 6130 6131 6132 6133 6134
			if (!mask)
				return;

			sched_domains_numa_masks[i][j] = mask;

			for (k = 0; k < nr_node_ids; k++) {
6135
				if (node_distance(j, k) > sched_domains_numa_distance[i])
6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166
					continue;

				cpumask_or(mask, mask, cpumask_of_node(k));
			}
		}
	}

	tl = kzalloc((ARRAY_SIZE(default_topology) + level) *
			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
	if (!tl)
		return;

	/*
	 * Copy the default topology bits..
	 */
	for (i = 0; default_topology[i].init; i++)
		tl[i] = default_topology[i];

	/*
	 * .. and append 'j' levels of NUMA goodness.
	 */
	for (j = 0; j < level; i++, j++) {
		tl[i] = (struct sched_domain_topology_level){
			.init = sd_numa_init,
			.mask = sd_numa_mask,
			.flags = SDTL_OVERLAP,
			.numa_level = j,
		};
	}

	sched_domain_topology = tl;
6167 6168

	sched_domains_numa_levels = level;
6169
}
6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216

static void sched_domains_numa_masks_set(int cpu)
{
	int i, j;
	int node = cpu_to_node(cpu);

	for (i = 0; i < sched_domains_numa_levels; i++) {
		for (j = 0; j < nr_node_ids; j++) {
			if (node_distance(j, node) <= sched_domains_numa_distance[i])
				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
		}
	}
}

static void sched_domains_numa_masks_clear(int cpu)
{
	int i, j;
	for (i = 0; i < sched_domains_numa_levels; i++) {
		for (j = 0; j < nr_node_ids; j++)
			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
	}
}

/*
 * Update sched_domains_numa_masks[level][node] array when new cpus
 * are onlined.
 */
static int sched_domains_numa_masks_update(struct notifier_block *nfb,
					   unsigned long action,
					   void *hcpu)
{
	int cpu = (long)hcpu;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_ONLINE:
		sched_domains_numa_masks_set(cpu);
		break;

	case CPU_DEAD:
		sched_domains_numa_masks_clear(cpu);
		break;

	default:
		return NOTIFY_DONE;
	}

	return NOTIFY_OK;
6217 6218 6219 6220 6221
}
#else
static inline void sched_init_numa(void)
{
}
6222 6223 6224 6225 6226 6227 6228

static int sched_domains_numa_masks_update(struct notifier_block *nfb,
					   unsigned long action,
					   void *hcpu)
{
	return 0;
}
6229 6230
#endif /* CONFIG_NUMA */

6231 6232 6233 6234 6235
static int __sdt_alloc(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

6236
	for_each_sd_topology(tl) {
6237 6238 6239 6240 6241 6242 6243 6244 6245 6246
		struct sd_data *sdd = &tl->data;

		sdd->sd = alloc_percpu(struct sched_domain *);
		if (!sdd->sd)
			return -ENOMEM;

		sdd->sg = alloc_percpu(struct sched_group *);
		if (!sdd->sg)
			return -ENOMEM;

6247 6248 6249 6250
		sdd->sgp = alloc_percpu(struct sched_group_power *);
		if (!sdd->sgp)
			return -ENOMEM;

6251 6252 6253
		for_each_cpu(j, cpu_map) {
			struct sched_domain *sd;
			struct sched_group *sg;
6254
			struct sched_group_power *sgp;
6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267

		       	sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sd)
				return -ENOMEM;

			*per_cpu_ptr(sdd->sd, j) = sd;

			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sg)
				return -ENOMEM;

6268 6269
			sg->next = sg;

6270
			*per_cpu_ptr(sdd->sg, j) = sg;
6271

P
Peter Zijlstra 已提交
6272
			sgp = kzalloc_node(sizeof(struct sched_group_power) + cpumask_size(),
6273 6274 6275 6276 6277
					GFP_KERNEL, cpu_to_node(j));
			if (!sgp)
				return -ENOMEM;

			*per_cpu_ptr(sdd->sgp, j) = sgp;
6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288
		}
	}

	return 0;
}

static void __sdt_free(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

6289
	for_each_sd_topology(tl) {
6290 6291 6292
		struct sd_data *sdd = &tl->data;

		for_each_cpu(j, cpu_map) {
6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305
			struct sched_domain *sd;

			if (sdd->sd) {
				sd = *per_cpu_ptr(sdd->sd, j);
				if (sd && (sd->flags & SD_OVERLAP))
					free_sched_groups(sd->groups, 0);
				kfree(*per_cpu_ptr(sdd->sd, j));
			}

			if (sdd->sg)
				kfree(*per_cpu_ptr(sdd->sg, j));
			if (sdd->sgp)
				kfree(*per_cpu_ptr(sdd->sgp, j));
6306 6307
		}
		free_percpu(sdd->sd);
6308
		sdd->sd = NULL;
6309
		free_percpu(sdd->sg);
6310
		sdd->sg = NULL;
6311
		free_percpu(sdd->sgp);
6312
		sdd->sgp = NULL;
6313 6314 6315
	}
}

6316
struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
6317 6318
		const struct cpumask *cpu_map, struct sched_domain_attr *attr,
		struct sched_domain *child, int cpu)
6319
{
6320
	struct sched_domain *sd = tl->init(tl, cpu);
6321
	if (!sd)
6322
		return child;
6323 6324

	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
6325 6326 6327
	if (child) {
		sd->level = child->level + 1;
		sched_domain_level_max = max(sched_domain_level_max, sd->level);
6328
		child->parent = sd;
6329
		sd->child = child;
6330
	}
6331
	set_domain_attribute(sd, attr);
6332 6333 6334 6335

	return sd;
}

6336 6337 6338 6339
/*
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
 */
6340 6341
static int build_sched_domains(const struct cpumask *cpu_map,
			       struct sched_domain_attr *attr)
6342
{
6343
	enum s_alloc alloc_state;
6344
	struct sched_domain *sd;
6345
	struct s_data d;
6346
	int i, ret = -ENOMEM;
6347

6348 6349 6350
	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
	if (alloc_state != sa_rootdomain)
		goto error;
6351

6352
	/* Set up domains for cpus specified by the cpu_map. */
6353
	for_each_cpu(i, cpu_map) {
6354 6355
		struct sched_domain_topology_level *tl;

6356
		sd = NULL;
6357
		for_each_sd_topology(tl) {
6358
			sd = build_sched_domain(tl, cpu_map, attr, sd, i);
6359 6360
			if (tl == sched_domain_topology)
				*per_cpu_ptr(d.sd, i) = sd;
6361 6362
			if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
				sd->flags |= SD_OVERLAP;
6363 6364
			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
				break;
6365
		}
6366 6367 6368 6369 6370 6371
	}

	/* Build the groups for the domains */
	for_each_cpu(i, cpu_map) {
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			sd->span_weight = cpumask_weight(sched_domain_span(sd));
6372 6373 6374 6375 6376 6377 6378
			if (sd->flags & SD_OVERLAP) {
				if (build_overlap_sched_groups(sd, i))
					goto error;
			} else {
				if (build_sched_groups(sd, i))
					goto error;
			}
6379
		}
6380
	}
6381

L
Linus Torvalds 已提交
6382
	/* Calculate CPU power for physical packages and nodes */
6383 6384 6385
	for (i = nr_cpumask_bits-1; i >= 0; i--) {
		if (!cpumask_test_cpu(i, cpu_map))
			continue;
6386

6387 6388
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			claim_allocations(i, sd);
6389
			init_sched_groups_power(i, sd);
6390
		}
6391
	}
6392

L
Linus Torvalds 已提交
6393
	/* Attach the domains */
6394
	rcu_read_lock();
6395
	for_each_cpu(i, cpu_map) {
6396
		sd = *per_cpu_ptr(d.sd, i);
6397
		cpu_attach_domain(sd, d.rd, i);
L
Linus Torvalds 已提交
6398
	}
6399
	rcu_read_unlock();
6400

6401
	ret = 0;
6402
error:
6403
	__free_domain_allocs(&d, alloc_state, cpu_map);
6404
	return ret;
L
Linus Torvalds 已提交
6405
}
P
Paul Jackson 已提交
6406

6407
static cpumask_var_t *doms_cur;	/* current sched domains */
P
Paul Jackson 已提交
6408
static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
I
Ingo Molnar 已提交
6409 6410
static struct sched_domain_attr *dattr_cur;
				/* attribues of custom domains in 'doms_cur' */
P
Paul Jackson 已提交
6411 6412 6413

/*
 * Special case: If a kmalloc of a doms_cur partition (array of
6414 6415
 * cpumask) fails, then fallback to a single sched domain,
 * as determined by the single cpumask fallback_doms.
P
Paul Jackson 已提交
6416
 */
6417
static cpumask_var_t fallback_doms;
P
Paul Jackson 已提交
6418

6419 6420 6421 6422 6423 6424
/*
 * arch_update_cpu_topology lets virtualized architectures update the
 * cpu core maps. It is supposed to return 1 if the topology changed
 * or 0 if it stayed the same.
 */
int __attribute__((weak)) arch_update_cpu_topology(void)
6425
{
6426
	return 0;
6427 6428
}

6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453
cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
{
	int i;
	cpumask_var_t *doms;

	doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
	if (!doms)
		return NULL;
	for (i = 0; i < ndoms; i++) {
		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
			free_sched_domains(doms, i);
			return NULL;
		}
	}
	return doms;
}

void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
{
	unsigned int i;
	for (i = 0; i < ndoms; i++)
		free_cpumask_var(doms[i]);
	kfree(doms);
}

6454
/*
I
Ingo Molnar 已提交
6455
 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
P
Paul Jackson 已提交
6456 6457
 * For now this just excludes isolated cpus, but could be used to
 * exclude other special cases in the future.
6458
 */
6459
static int init_sched_domains(const struct cpumask *cpu_map)
6460
{
6461 6462
	int err;

6463
	arch_update_cpu_topology();
P
Paul Jackson 已提交
6464
	ndoms_cur = 1;
6465
	doms_cur = alloc_sched_domains(ndoms_cur);
P
Paul Jackson 已提交
6466
	if (!doms_cur)
6467 6468
		doms_cur = &fallback_doms;
	cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
6469
	err = build_sched_domains(doms_cur[0], NULL);
6470
	register_sched_domain_sysctl();
6471 6472

	return err;
6473 6474 6475 6476 6477 6478
}

/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
6479
static void detach_destroy_domains(const struct cpumask *cpu_map)
6480 6481 6482
{
	int i;

6483
	rcu_read_lock();
6484
	for_each_cpu(i, cpu_map)
G
Gregory Haskins 已提交
6485
		cpu_attach_domain(NULL, &def_root_domain, i);
6486
	rcu_read_unlock();
6487 6488
}

6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504
/* handle null as "default" */
static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
			struct sched_domain_attr *new, int idx_new)
{
	struct sched_domain_attr tmp;

	/* fast path */
	if (!new && !cur)
		return 1;

	tmp = SD_ATTR_INIT;
	return !memcmp(cur ? (cur + idx_cur) : &tmp,
			new ? (new + idx_new) : &tmp,
			sizeof(struct sched_domain_attr));
}

P
Paul Jackson 已提交
6505 6506
/*
 * Partition sched domains as specified by the 'ndoms_new'
I
Ingo Molnar 已提交
6507
 * cpumasks in the array doms_new[] of cpumasks. This compares
P
Paul Jackson 已提交
6508 6509 6510
 * doms_new[] to the current sched domain partitioning, doms_cur[].
 * It destroys each deleted domain and builds each new domain.
 *
6511
 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
I
Ingo Molnar 已提交
6512 6513 6514
 * The masks don't intersect (don't overlap.) We should setup one
 * sched domain for each mask. CPUs not in any of the cpumasks will
 * not be load balanced. If the same cpumask appears both in the
P
Paul Jackson 已提交
6515 6516 6517
 * current 'doms_cur' domains and in the new 'doms_new', we can leave
 * it as it is.
 *
6518 6519 6520 6521 6522 6523
 * The passed in 'doms_new' should be allocated using
 * alloc_sched_domains.  This routine takes ownership of it and will
 * free_sched_domains it when done with it. If the caller failed the
 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
 * and partition_sched_domains() will fallback to the single partition
 * 'fallback_doms', it also forces the domains to be rebuilt.
P
Paul Jackson 已提交
6524
 *
6525
 * If doms_new == NULL it will be replaced with cpu_online_mask.
6526 6527
 * ndoms_new == 0 is a special case for destroying existing domains,
 * and it will not create the default domain.
6528
 *
P
Paul Jackson 已提交
6529 6530
 * Call with hotplug lock held
 */
6531
void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
6532
			     struct sched_domain_attr *dattr_new)
P
Paul Jackson 已提交
6533
{
6534
	int i, j, n;
6535
	int new_topology;
P
Paul Jackson 已提交
6536

6537
	mutex_lock(&sched_domains_mutex);
6538

6539 6540 6541
	/* always unregister in case we don't destroy any domains */
	unregister_sched_domain_sysctl();

6542 6543 6544
	/* Let architecture update cpu core mappings. */
	new_topology = arch_update_cpu_topology();

6545
	n = doms_new ? ndoms_new : 0;
P
Paul Jackson 已提交
6546 6547 6548

	/* Destroy deleted domains */
	for (i = 0; i < ndoms_cur; i++) {
6549
		for (j = 0; j < n && !new_topology; j++) {
6550
			if (cpumask_equal(doms_cur[i], doms_new[j])
6551
			    && dattrs_equal(dattr_cur, i, dattr_new, j))
P
Paul Jackson 已提交
6552 6553 6554
				goto match1;
		}
		/* no match - a current sched domain not in new doms_new[] */
6555
		detach_destroy_domains(doms_cur[i]);
P
Paul Jackson 已提交
6556 6557 6558 6559
match1:
		;
	}

6560
	n = ndoms_cur;
6561
	if (doms_new == NULL) {
6562
		n = 0;
6563
		doms_new = &fallback_doms;
6564
		cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
6565
		WARN_ON_ONCE(dattr_new);
6566 6567
	}

P
Paul Jackson 已提交
6568 6569
	/* Build new domains */
	for (i = 0; i < ndoms_new; i++) {
6570
		for (j = 0; j < n && !new_topology; j++) {
6571
			if (cpumask_equal(doms_new[i], doms_cur[j])
6572
			    && dattrs_equal(dattr_new, i, dattr_cur, j))
P
Paul Jackson 已提交
6573 6574 6575
				goto match2;
		}
		/* no match - add a new doms_new */
6576
		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
P
Paul Jackson 已提交
6577 6578 6579 6580 6581
match2:
		;
	}

	/* Remember the new sched domains */
6582 6583
	if (doms_cur != &fallback_doms)
		free_sched_domains(doms_cur, ndoms_cur);
6584
	kfree(dattr_cur);	/* kfree(NULL) is safe */
P
Paul Jackson 已提交
6585
	doms_cur = doms_new;
6586
	dattr_cur = dattr_new;
P
Paul Jackson 已提交
6587
	ndoms_cur = ndoms_new;
6588 6589

	register_sched_domain_sysctl();
6590

6591
	mutex_unlock(&sched_domains_mutex);
P
Paul Jackson 已提交
6592 6593
}

6594 6595
static int num_cpus_frozen;	/* used to mark begin/end of suspend/resume */

L
Linus Torvalds 已提交
6596
/*
6597 6598 6599
 * Update cpusets according to cpu_active mask.  If cpusets are
 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
 * around partition_sched_domains().
6600 6601 6602
 *
 * If we come here as part of a suspend/resume, don't touch cpusets because we
 * want to restore it back to its original state upon resume anyway.
L
Linus Torvalds 已提交
6603
 */
6604 6605
static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
			     void *hcpu)
6606
{
6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628
	switch (action) {
	case CPU_ONLINE_FROZEN:
	case CPU_DOWN_FAILED_FROZEN:

		/*
		 * num_cpus_frozen tracks how many CPUs are involved in suspend
		 * resume sequence. As long as this is not the last online
		 * operation in the resume sequence, just build a single sched
		 * domain, ignoring cpusets.
		 */
		num_cpus_frozen--;
		if (likely(num_cpus_frozen)) {
			partition_sched_domains(1, NULL, NULL);
			break;
		}

		/*
		 * This is the last CPU online operation. So fall through and
		 * restore the original sched domains by considering the
		 * cpuset configurations.
		 */

6629
	case CPU_ONLINE:
6630
	case CPU_DOWN_FAILED:
6631
		cpuset_update_active_cpus(true);
6632
		break;
6633 6634 6635
	default:
		return NOTIFY_DONE;
	}
6636
	return NOTIFY_OK;
6637
}
6638

6639 6640
static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
			       void *hcpu)
6641
{
6642
	switch (action) {
6643
	case CPU_DOWN_PREPARE:
6644
		cpuset_update_active_cpus(false);
6645 6646 6647 6648 6649
		break;
	case CPU_DOWN_PREPARE_FROZEN:
		num_cpus_frozen++;
		partition_sched_domains(1, NULL, NULL);
		break;
6650 6651 6652
	default:
		return NOTIFY_DONE;
	}
6653
	return NOTIFY_OK;
6654 6655
}

L
Linus Torvalds 已提交
6656 6657
void __init sched_init_smp(void)
{
6658 6659 6660
	cpumask_var_t non_isolated_cpus;

	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
6661
	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
6662

6663 6664
	sched_init_numa();

6665 6666 6667 6668 6669
	/*
	 * There's no userspace yet to cause hotplug operations; hence all the
	 * cpu masks are stable and all blatant races in the below code cannot
	 * happen.
	 */
6670
	mutex_lock(&sched_domains_mutex);
6671
	init_sched_domains(cpu_active_mask);
6672 6673 6674
	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
	if (cpumask_empty(non_isolated_cpus))
		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
6675
	mutex_unlock(&sched_domains_mutex);
6676

6677
	hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
6678 6679
	hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
	hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
6680

6681
	init_hrtick();
6682 6683

	/* Move init over to a non-isolated CPU */
6684
	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
6685
		BUG();
I
Ingo Molnar 已提交
6686
	sched_init_granularity();
6687
	free_cpumask_var(non_isolated_cpus);
6688

6689
	init_sched_rt_class();
6690
	init_sched_dl_class();
L
Linus Torvalds 已提交
6691 6692 6693 6694
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
6695
	sched_init_granularity();
L
Linus Torvalds 已提交
6696 6697 6698
}
#endif /* CONFIG_SMP */

6699 6700
const_debug unsigned int sysctl_timer_migration = 1;

L
Linus Torvalds 已提交
6701 6702 6703 6704 6705 6706 6707
int in_sched_functions(unsigned long addr)
{
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

6708
#ifdef CONFIG_CGROUP_SCHED
6709 6710 6711 6712
/*
 * Default task group.
 * Every task in system belongs to this group at bootup.
 */
6713
struct task_group root_task_group;
6714
LIST_HEAD(task_groups);
6715
#endif
P
Peter Zijlstra 已提交
6716

6717
DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
P
Peter Zijlstra 已提交
6718

L
Linus Torvalds 已提交
6719 6720
void __init sched_init(void)
{
I
Ingo Molnar 已提交
6721
	int i, j;
6722 6723 6724 6725 6726 6727 6728
	unsigned long alloc_size = 0, ptr;

#ifdef CONFIG_FAIR_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6729
#endif
6730
#ifdef CONFIG_CPUMASK_OFFSTACK
6731
	alloc_size += num_possible_cpus() * cpumask_size();
6732 6733
#endif
	if (alloc_size) {
6734
		ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
6735 6736

#ifdef CONFIG_FAIR_GROUP_SCHED
6737
		root_task_group.se = (struct sched_entity **)ptr;
6738 6739
		ptr += nr_cpu_ids * sizeof(void **);

6740
		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
6741
		ptr += nr_cpu_ids * sizeof(void **);
6742

6743
#endif /* CONFIG_FAIR_GROUP_SCHED */
6744
#ifdef CONFIG_RT_GROUP_SCHED
6745
		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
6746 6747
		ptr += nr_cpu_ids * sizeof(void **);

6748
		root_task_group.rt_rq = (struct rt_rq **)ptr;
6749 6750
		ptr += nr_cpu_ids * sizeof(void **);

6751
#endif /* CONFIG_RT_GROUP_SCHED */
6752 6753
#ifdef CONFIG_CPUMASK_OFFSTACK
		for_each_possible_cpu(i) {
6754
			per_cpu(load_balance_mask, i) = (void *)ptr;
6755 6756 6757
			ptr += cpumask_size();
		}
#endif /* CONFIG_CPUMASK_OFFSTACK */
6758
	}
I
Ingo Molnar 已提交
6759

6760 6761 6762
	init_rt_bandwidth(&def_rt_bandwidth,
			global_rt_period(), global_rt_runtime());
	init_dl_bandwidth(&def_dl_bandwidth,
6763
			global_rt_period(), global_rt_runtime());
6764

G
Gregory Haskins 已提交
6765 6766 6767 6768
#ifdef CONFIG_SMP
	init_defrootdomain();
#endif

6769
#ifdef CONFIG_RT_GROUP_SCHED
6770
	init_rt_bandwidth(&root_task_group.rt_bandwidth,
6771
			global_rt_period(), global_rt_runtime());
6772
#endif /* CONFIG_RT_GROUP_SCHED */
6773

D
Dhaval Giani 已提交
6774
#ifdef CONFIG_CGROUP_SCHED
6775 6776
	list_add(&root_task_group.list, &task_groups);
	INIT_LIST_HEAD(&root_task_group.children);
6777
	INIT_LIST_HEAD(&root_task_group.siblings);
6778
	autogroup_init(&init_task);
6779

D
Dhaval Giani 已提交
6780
#endif /* CONFIG_CGROUP_SCHED */
P
Peter Zijlstra 已提交
6781

6782
	for_each_possible_cpu(i) {
6783
		struct rq *rq;
L
Linus Torvalds 已提交
6784 6785

		rq = cpu_rq(i);
6786
		raw_spin_lock_init(&rq->lock);
N
Nick Piggin 已提交
6787
		rq->nr_running = 0;
6788 6789
		rq->calc_load_active = 0;
		rq->calc_load_update = jiffies + LOAD_FREQ;
6790
		init_cfs_rq(&rq->cfs);
P
Peter Zijlstra 已提交
6791
		init_rt_rq(&rq->rt, rq);
6792
		init_dl_rq(&rq->dl, rq);
I
Ingo Molnar 已提交
6793
#ifdef CONFIG_FAIR_GROUP_SCHED
6794
		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
P
Peter Zijlstra 已提交
6795
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
D
Dhaval Giani 已提交
6796
		/*
6797
		 * How much cpu bandwidth does root_task_group get?
D
Dhaval Giani 已提交
6798 6799 6800 6801
		 *
		 * In case of task-groups formed thr' the cgroup filesystem, it
		 * gets 100% of the cpu resources in the system. This overall
		 * system cpu resource is divided among the tasks of
6802
		 * root_task_group and its child task-groups in a fair manner,
D
Dhaval Giani 已提交
6803 6804 6805
		 * based on each entity's (task or task-group's) weight
		 * (se->load.weight).
		 *
6806
		 * In other words, if root_task_group has 10 tasks of weight
D
Dhaval Giani 已提交
6807 6808 6809
		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
		 * then A0's share of the cpu resource is:
		 *
6810
		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
D
Dhaval Giani 已提交
6811
		 *
6812 6813
		 * We achieve this by letting root_task_group's tasks sit
		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
D
Dhaval Giani 已提交
6814
		 */
6815
		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
6816
		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
D
Dhaval Giani 已提交
6817 6818 6819
#endif /* CONFIG_FAIR_GROUP_SCHED */

		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
6820
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
6821
		INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
6822
		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
I
Ingo Molnar 已提交
6823
#endif
L
Linus Torvalds 已提交
6824

I
Ingo Molnar 已提交
6825 6826
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
6827 6828 6829

		rq->last_load_update_tick = jiffies;

L
Linus Torvalds 已提交
6830
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
6831
		rq->sd = NULL;
G
Gregory Haskins 已提交
6832
		rq->rd = NULL;
6833
		rq->cpu_power = SCHED_POWER_SCALE;
6834
		rq->post_schedule = 0;
L
Linus Torvalds 已提交
6835
		rq->active_balance = 0;
I
Ingo Molnar 已提交
6836
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
6837
		rq->push_cpu = 0;
6838
		rq->cpu = i;
6839
		rq->online = 0;
6840 6841
		rq->idle_stamp = 0;
		rq->avg_idle = 2*sysctl_sched_migration_cost;
6842
		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
6843 6844 6845

		INIT_LIST_HEAD(&rq->cfs_tasks);

6846
		rq_attach_root(rq, &def_root_domain);
6847
#ifdef CONFIG_NO_HZ_COMMON
6848
		rq->nohz_flags = 0;
6849
#endif
6850 6851 6852
#ifdef CONFIG_NO_HZ_FULL
		rq->last_sched_tick = 0;
#endif
L
Linus Torvalds 已提交
6853
#endif
P
Peter Zijlstra 已提交
6854
		init_rq_hrtick(rq);
L
Linus Torvalds 已提交
6855 6856 6857
		atomic_set(&rq->nr_iowait, 0);
	}

6858
	set_load_weight(&init_task);
6859

6860 6861 6862 6863
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

L
Linus Torvalds 已提交
6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
6877 6878 6879

	calc_load_update = jiffies + LOAD_FREQ;

I
Ingo Molnar 已提交
6880 6881 6882 6883
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
6884

6885
#ifdef CONFIG_SMP
6886
	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
R
Rusty Russell 已提交
6887 6888 6889
	/* May be allocated at isolcpus cmdline parse time */
	if (cpu_isolated_map == NULL)
		zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
6890
	idle_thread_set_boot_cpu();
6891 6892
#endif
	init_sched_fair_class();
6893

6894
	scheduler_running = 1;
L
Linus Torvalds 已提交
6895 6896
}

6897
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
6898 6899
static inline int preempt_count_equals(int preempt_offset)
{
6900
	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
6901

A
Arnd Bergmann 已提交
6902
	return (nested == preempt_offset);
6903 6904
}

6905
void __might_sleep(const char *file, int line, int preempt_offset)
L
Linus Torvalds 已提交
6906 6907 6908
{
	static unsigned long prev_jiffy;	/* ratelimiting */

6909
	rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
6910 6911
	if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
	    system_state != SYSTEM_RUNNING || oops_in_progress)
I
Ingo Molnar 已提交
6912 6913 6914 6915 6916
		return;
	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
		return;
	prev_jiffy = jiffies;

P
Peter Zijlstra 已提交
6917 6918 6919 6920 6921 6922 6923
	printk(KERN_ERR
		"BUG: sleeping function called from invalid context at %s:%d\n",
			file, line);
	printk(KERN_ERR
		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
			in_atomic(), irqs_disabled(),
			current->pid, current->comm);
I
Ingo Molnar 已提交
6924 6925 6926 6927 6928

	debug_show_held_locks(current);
	if (irqs_disabled())
		print_irqtrace_events(current);
	dump_stack();
L
Linus Torvalds 已提交
6929 6930 6931 6932 6933
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
6934 6935
static void normalize_task(struct rq *rq, struct task_struct *p)
{
P
Peter Zijlstra 已提交
6936
	const struct sched_class *prev_class = p->sched_class;
6937 6938 6939
	struct sched_attr attr = {
		.sched_policy = SCHED_NORMAL,
	};
P
Peter Zijlstra 已提交
6940
	int old_prio = p->prio;
6941
	int on_rq;
6942

P
Peter Zijlstra 已提交
6943
	on_rq = p->on_rq;
6944
	if (on_rq)
6945
		dequeue_task(rq, p, 0);
6946
	__setscheduler(rq, p, &attr);
6947
	if (on_rq) {
6948
		enqueue_task(rq, p, 0);
6949 6950
		resched_task(rq->curr);
	}
P
Peter Zijlstra 已提交
6951 6952

	check_class_changed(rq, p, prev_class, old_prio);
6953 6954
}

L
Linus Torvalds 已提交
6955 6956
void normalize_rt_tasks(void)
{
6957
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
6958
	unsigned long flags;
6959
	struct rq *rq;
L
Linus Torvalds 已提交
6960

6961
	read_lock_irqsave(&tasklist_lock, flags);
6962
	do_each_thread(g, p) {
6963 6964 6965 6966 6967 6968
		/*
		 * Only normalize user tasks:
		 */
		if (!p->mm)
			continue;

I
Ingo Molnar 已提交
6969 6970
		p->se.exec_start		= 0;
#ifdef CONFIG_SCHEDSTATS
6971 6972 6973
		p->se.statistics.wait_start	= 0;
		p->se.statistics.sleep_start	= 0;
		p->se.statistics.block_start	= 0;
I
Ingo Molnar 已提交
6974
#endif
I
Ingo Molnar 已提交
6975

6976
		if (!dl_task(p) && !rt_task(p)) {
I
Ingo Molnar 已提交
6977 6978 6979 6980
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
6981
			if (task_nice(p) < 0 && p->mm)
I
Ingo Molnar 已提交
6982
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
6983
			continue;
I
Ingo Molnar 已提交
6984
		}
L
Linus Torvalds 已提交
6985

6986
		raw_spin_lock(&p->pi_lock);
6987
		rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
6988

6989
		normalize_task(rq, p);
6990

6991
		__task_rq_unlock(rq);
6992
		raw_spin_unlock(&p->pi_lock);
6993 6994
	} while_each_thread(g, p);

6995
	read_unlock_irqrestore(&tasklist_lock, flags);
L
Linus Torvalds 已提交
6996 6997 6998
}

#endif /* CONFIG_MAGIC_SYSRQ */
6999

7000
#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
7001
/*
7002
 * These functions are only useful for the IA64 MCA handling, or kdb.
7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7016 7017
 *
 * Return: The current task for @cpu.
7018
 */
7019
struct task_struct *curr_task(int cpu)
7020 7021 7022 7023
{
	return cpu_curr(cpu);
}

7024 7025 7026
#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */

#ifdef CONFIG_IA64
7027 7028 7029 7030 7031 7032
/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
I
Ingo Molnar 已提交
7033 7034
 * are serviced on a separate stack. It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner. This function
7035 7036 7037 7038 7039 7040 7041
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
7042
void set_curr_task(int cpu, struct task_struct *p)
7043 7044 7045 7046 7047
{
	cpu_curr(cpu) = p;
}

#endif
S
Srivatsa Vaddagiri 已提交
7048

D
Dhaval Giani 已提交
7049
#ifdef CONFIG_CGROUP_SCHED
7050 7051 7052
/* task_group_lock serializes the addition/removal of task groups */
static DEFINE_SPINLOCK(task_group_lock);

7053 7054 7055 7056
static void free_sched_group(struct task_group *tg)
{
	free_fair_sched_group(tg);
	free_rt_sched_group(tg);
7057
	autogroup_free(tg);
7058 7059 7060 7061
	kfree(tg);
}

/* allocate runqueue etc for a new task group */
7062
struct task_group *sched_create_group(struct task_group *parent)
7063 7064 7065 7066 7067 7068 7069
{
	struct task_group *tg;

	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
	if (!tg)
		return ERR_PTR(-ENOMEM);

7070
	if (!alloc_fair_sched_group(tg, parent))
7071 7072
		goto err;

7073
	if (!alloc_rt_sched_group(tg, parent))
7074 7075
		goto err;

7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086
	return tg;

err:
	free_sched_group(tg);
	return ERR_PTR(-ENOMEM);
}

void sched_online_group(struct task_group *tg, struct task_group *parent)
{
	unsigned long flags;

7087
	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
7088
	list_add_rcu(&tg->list, &task_groups);
P
Peter Zijlstra 已提交
7089 7090 7091 7092 7093

	WARN_ON(!parent); /* root should already exist */

	tg->parent = parent;
	INIT_LIST_HEAD(&tg->children);
7094
	list_add_rcu(&tg->siblings, &parent->children);
7095
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
7096 7097
}

7098
/* rcu callback to free various structures associated with a task group */
P
Peter Zijlstra 已提交
7099
static void free_sched_group_rcu(struct rcu_head *rhp)
S
Srivatsa Vaddagiri 已提交
7100 7101
{
	/* now it should be safe to free those cfs_rqs */
P
Peter Zijlstra 已提交
7102
	free_sched_group(container_of(rhp, struct task_group, rcu));
S
Srivatsa Vaddagiri 已提交
7103 7104
}

7105
/* Destroy runqueue etc associated with a task group */
7106
void sched_destroy_group(struct task_group *tg)
7107 7108 7109 7110 7111 7112
{
	/* wait for possible concurrent references to cfs_rqs complete */
	call_rcu(&tg->rcu, free_sched_group_rcu);
}

void sched_offline_group(struct task_group *tg)
S
Srivatsa Vaddagiri 已提交
7113
{
7114
	unsigned long flags;
7115
	int i;
S
Srivatsa Vaddagiri 已提交
7116

7117 7118
	/* end participation in shares distribution */
	for_each_possible_cpu(i)
7119
		unregister_fair_sched_group(tg, i);
7120 7121

	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
7122
	list_del_rcu(&tg->list);
P
Peter Zijlstra 已提交
7123
	list_del_rcu(&tg->siblings);
7124
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
7125 7126
}

7127
/* change task's runqueue when it moves between groups.
I
Ingo Molnar 已提交
7128 7129 7130
 *	The caller of this function should have put the task in its new group
 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
 *	reflect its new group.
7131 7132
 */
void sched_move_task(struct task_struct *tsk)
S
Srivatsa Vaddagiri 已提交
7133
{
P
Peter Zijlstra 已提交
7134
	struct task_group *tg;
S
Srivatsa Vaddagiri 已提交
7135 7136 7137 7138 7139 7140
	int on_rq, running;
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

7141
	running = task_current(rq, tsk);
P
Peter Zijlstra 已提交
7142
	on_rq = tsk->on_rq;
S
Srivatsa Vaddagiri 已提交
7143

7144
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
7145
		dequeue_task(rq, tsk, 0);
7146 7147
	if (unlikely(running))
		tsk->sched_class->put_prev_task(rq, tsk);
S
Srivatsa Vaddagiri 已提交
7148

7149
	tg = container_of(task_css_check(tsk, cpu_cgroup_subsys_id,
P
Peter Zijlstra 已提交
7150 7151 7152 7153 7154
				lockdep_is_held(&tsk->sighand->siglock)),
			  struct task_group, css);
	tg = autogroup_task_group(tsk, tg);
	tsk->sched_task_group = tg;

P
Peter Zijlstra 已提交
7155
#ifdef CONFIG_FAIR_GROUP_SCHED
7156 7157 7158
	if (tsk->sched_class->task_move_group)
		tsk->sched_class->task_move_group(tsk, on_rq);
	else
P
Peter Zijlstra 已提交
7159
#endif
7160
		set_task_rq(tsk, task_cpu(tsk));
P
Peter Zijlstra 已提交
7161

7162 7163 7164
	if (unlikely(running))
		tsk->sched_class->set_curr_task(rq);
	if (on_rq)
7165
		enqueue_task(rq, tsk, 0);
S
Srivatsa Vaddagiri 已提交
7166

7167
	task_rq_unlock(rq, tsk, &flags);
S
Srivatsa Vaddagiri 已提交
7168
}
D
Dhaval Giani 已提交
7169
#endif /* CONFIG_CGROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
7170

7171 7172 7173 7174 7175
#ifdef CONFIG_RT_GROUP_SCHED
/*
 * Ensure that the real time constraints are schedulable.
 */
static DEFINE_MUTEX(rt_constraints_mutex);
P
Peter Zijlstra 已提交
7176

P
Peter Zijlstra 已提交
7177 7178
/* Must be called with tasklist_lock held */
static inline int tg_has_rt_tasks(struct task_group *tg)
7179
{
P
Peter Zijlstra 已提交
7180
	struct task_struct *g, *p;
7181

P
Peter Zijlstra 已提交
7182
	do_each_thread(g, p) {
7183
		if (rt_task(p) && task_rq(p)->rt.tg == tg)
P
Peter Zijlstra 已提交
7184 7185
			return 1;
	} while_each_thread(g, p);
7186

P
Peter Zijlstra 已提交
7187 7188
	return 0;
}
7189

P
Peter Zijlstra 已提交
7190 7191 7192 7193 7194
struct rt_schedulable_data {
	struct task_group *tg;
	u64 rt_period;
	u64 rt_runtime;
};
7195

7196
static int tg_rt_schedulable(struct task_group *tg, void *data)
P
Peter Zijlstra 已提交
7197 7198 7199 7200 7201
{
	struct rt_schedulable_data *d = data;
	struct task_group *child;
	unsigned long total, sum = 0;
	u64 period, runtime;
7202

P
Peter Zijlstra 已提交
7203 7204
	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	runtime = tg->rt_bandwidth.rt_runtime;
7205

P
Peter Zijlstra 已提交
7206 7207 7208
	if (tg == d->tg) {
		period = d->rt_period;
		runtime = d->rt_runtime;
7209 7210
	}

7211 7212 7213 7214 7215
	/*
	 * Cannot have more runtime than the period.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
P
Peter Zijlstra 已提交
7216

7217 7218 7219
	/*
	 * Ensure we don't starve existing RT tasks.
	 */
P
Peter Zijlstra 已提交
7220 7221
	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
		return -EBUSY;
P
Peter Zijlstra 已提交
7222

P
Peter Zijlstra 已提交
7223
	total = to_ratio(period, runtime);
P
Peter Zijlstra 已提交
7224

7225 7226 7227 7228 7229
	/*
	 * Nobody can have more than the global setting allows.
	 */
	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
		return -EINVAL;
P
Peter Zijlstra 已提交
7230

7231 7232 7233
	/*
	 * The sum of our children's runtime should not exceed our own.
	 */
P
Peter Zijlstra 已提交
7234 7235 7236
	list_for_each_entry_rcu(child, &tg->children, siblings) {
		period = ktime_to_ns(child->rt_bandwidth.rt_period);
		runtime = child->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
7237

P
Peter Zijlstra 已提交
7238 7239 7240 7241
		if (child == d->tg) {
			period = d->rt_period;
			runtime = d->rt_runtime;
		}
P
Peter Zijlstra 已提交
7242

P
Peter Zijlstra 已提交
7243
		sum += to_ratio(period, runtime);
P
Peter Zijlstra 已提交
7244
	}
P
Peter Zijlstra 已提交
7245

P
Peter Zijlstra 已提交
7246 7247 7248 7249
	if (sum > total)
		return -EINVAL;

	return 0;
P
Peter Zijlstra 已提交
7250 7251
}

P
Peter Zijlstra 已提交
7252
static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
7253
{
7254 7255
	int ret;

P
Peter Zijlstra 已提交
7256 7257 7258 7259 7260 7261
	struct rt_schedulable_data data = {
		.tg = tg,
		.rt_period = period,
		.rt_runtime = runtime,
	};

7262 7263 7264 7265 7266
	rcu_read_lock();
	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
	rcu_read_unlock();

	return ret;
7267 7268
}

7269
static int tg_set_rt_bandwidth(struct task_group *tg,
7270
		u64 rt_period, u64 rt_runtime)
P
Peter Zijlstra 已提交
7271
{
P
Peter Zijlstra 已提交
7272
	int i, err = 0;
P
Peter Zijlstra 已提交
7273 7274

	mutex_lock(&rt_constraints_mutex);
7275
	read_lock(&tasklist_lock);
P
Peter Zijlstra 已提交
7276 7277
	err = __rt_schedulable(tg, rt_period, rt_runtime);
	if (err)
P
Peter Zijlstra 已提交
7278
		goto unlock;
P
Peter Zijlstra 已提交
7279

7280
	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
7281 7282
	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
	tg->rt_bandwidth.rt_runtime = rt_runtime;
P
Peter Zijlstra 已提交
7283 7284 7285 7286

	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = tg->rt_rq[i];

7287
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7288
		rt_rq->rt_runtime = rt_runtime;
7289
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7290
	}
7291
	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
P
Peter Zijlstra 已提交
7292
unlock:
7293
	read_unlock(&tasklist_lock);
P
Peter Zijlstra 已提交
7294 7295 7296
	mutex_unlock(&rt_constraints_mutex);

	return err;
P
Peter Zijlstra 已提交
7297 7298
}

7299
static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
7300 7301 7302 7303 7304 7305 7306 7307
{
	u64 rt_runtime, rt_period;

	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
	if (rt_runtime_us < 0)
		rt_runtime = RUNTIME_INF;

7308
	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7309 7310
}

7311
static long sched_group_rt_runtime(struct task_group *tg)
P
Peter Zijlstra 已提交
7312 7313 7314
{
	u64 rt_runtime_us;

7315
	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
7316 7317
		return -1;

7318
	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
7319 7320 7321
	do_div(rt_runtime_us, NSEC_PER_USEC);
	return rt_runtime_us;
}
7322

7323
static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
7324 7325 7326 7327 7328 7329
{
	u64 rt_runtime, rt_period;

	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
	rt_runtime = tg->rt_bandwidth.rt_runtime;

7330 7331 7332
	if (rt_period == 0)
		return -EINVAL;

7333
	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7334 7335
}

7336
static long sched_group_rt_period(struct task_group *tg)
7337 7338 7339 7340 7341 7342 7343
{
	u64 rt_period_us;

	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
	do_div(rt_period_us, NSEC_PER_USEC);
	return rt_period_us;
}
7344
#endif /* CONFIG_RT_GROUP_SCHED */
7345

7346
#ifdef CONFIG_RT_GROUP_SCHED
7347 7348 7349 7350 7351
static int sched_rt_global_constraints(void)
{
	int ret = 0;

	mutex_lock(&rt_constraints_mutex);
P
Peter Zijlstra 已提交
7352
	read_lock(&tasklist_lock);
7353
	ret = __rt_schedulable(NULL, 0, 0);
P
Peter Zijlstra 已提交
7354
	read_unlock(&tasklist_lock);
7355 7356 7357 7358
	mutex_unlock(&rt_constraints_mutex);

	return ret;
}
7359

7360
static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
7361 7362 7363 7364 7365 7366 7367 7368
{
	/* Don't accept realtime tasks when there is no way for them to run */
	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
		return 0;

	return 1;
}

7369
#else /* !CONFIG_RT_GROUP_SCHED */
7370 7371
static int sched_rt_global_constraints(void)
{
P
Peter Zijlstra 已提交
7372
	unsigned long flags;
7373
	int i, ret = 0;
7374

7375
	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
7376 7377 7378
	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = &cpu_rq(i)->rt;

7379
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7380
		rt_rq->rt_runtime = global_rt_runtime();
7381
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7382
	}
7383
	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
7384

7385
	return ret;
7386
}
7387
#endif /* CONFIG_RT_GROUP_SCHED */
7388

7389 7390
static int sched_dl_global_constraints(void)
{
7391 7392
	u64 runtime = global_rt_runtime();
	u64 period = global_rt_period();
7393
	u64 new_bw = to_ratio(period, runtime);
7394
	int cpu, ret = 0;
7395 7396 7397 7398 7399 7400 7401 7402 7403 7404

	/*
	 * Here we want to check the bandwidth not being set to some
	 * value smaller than the currently allocated bandwidth in
	 * any of the root_domains.
	 *
	 * FIXME: Cycling on all the CPUs is overdoing, but simpler than
	 * cycling on root_domains... Discussion on different/better
	 * solutions is welcome!
	 */
7405 7406
	for_each_possible_cpu(cpu) {
		struct dl_bw *dl_b = dl_bw_of(cpu);
7407 7408

		raw_spin_lock(&dl_b->lock);
7409 7410
		if (new_bw < dl_b->total_bw)
			ret = -EBUSY;
7411
		raw_spin_unlock(&dl_b->lock);
7412 7413 7414

		if (ret)
			break;
7415 7416
	}

7417
	return ret;
7418 7419
}

7420
static void sched_dl_do_global(void)
7421
{
7422 7423
	u64 new_bw = -1;
	int cpu;
7424

7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439
	def_dl_bandwidth.dl_period = global_rt_period();
	def_dl_bandwidth.dl_runtime = global_rt_runtime();

	if (global_rt_runtime() != RUNTIME_INF)
		new_bw = to_ratio(global_rt_period(), global_rt_runtime());

	/*
	 * FIXME: As above...
	 */
	for_each_possible_cpu(cpu) {
		struct dl_bw *dl_b = dl_bw_of(cpu);

		raw_spin_lock(&dl_b->lock);
		dl_b->bw = new_bw;
		raw_spin_unlock(&dl_b->lock);
7440
	}
7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457
}

static int sched_rt_global_validate(void)
{
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

	if (sysctl_sched_rt_runtime > sysctl_sched_rt_period)
		return -EINVAL;

	return 0;
}

static void sched_rt_do_global(void)
{
	def_rt_bandwidth.rt_runtime = global_rt_runtime();
	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
7458 7459
}

7460
int sched_rt_handler(struct ctl_table *table, int write,
7461
		void __user *buffer, size_t *lenp,
7462 7463 7464 7465
		loff_t *ppos)
{
	int old_period, old_runtime;
	static DEFINE_MUTEX(mutex);
7466
	int ret;
7467 7468 7469 7470 7471

	mutex_lock(&mutex);
	old_period = sysctl_sched_rt_period;
	old_runtime = sysctl_sched_rt_runtime;

7472
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
7473 7474

	if (!ret && write) {
7475 7476 7477 7478
		ret = sched_rt_global_validate();
		if (ret)
			goto undo;

7479
		ret = sched_rt_global_constraints();
7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493
		if (ret)
			goto undo;

		ret = sched_dl_global_constraints();
		if (ret)
			goto undo;

		sched_rt_do_global();
		sched_dl_do_global();
	}
	if (0) {
undo:
		sysctl_sched_rt_period = old_period;
		sysctl_sched_rt_runtime = old_runtime;
7494 7495 7496 7497 7498
	}
	mutex_unlock(&mutex);

	return ret;
}
7499

7500
int sched_rr_handler(struct ctl_table *table, int write,
7501 7502 7503 7504 7505 7506 7507 7508
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
7509 7510
	/* make sure that internally we keep jiffies */
	/* also, writing zero resets timeslice to default */
7511
	if (!ret && write) {
7512 7513
		sched_rr_timeslice = sched_rr_timeslice <= 0 ?
			RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
7514 7515 7516 7517 7518
	}
	mutex_unlock(&mutex);
	return ret;
}

7519
#ifdef CONFIG_CGROUP_SCHED
7520

7521
static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
7522
{
7523
	return css ? container_of(css, struct task_group, css) : NULL;
7524 7525
}

7526 7527
static struct cgroup_subsys_state *
cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
7528
{
7529 7530
	struct task_group *parent = css_tg(parent_css);
	struct task_group *tg;
7531

7532
	if (!parent) {
7533
		/* This is early initialization for the top cgroup */
7534
		return &root_task_group.css;
7535 7536
	}

7537
	tg = sched_create_group(parent);
7538 7539 7540 7541 7542 7543
	if (IS_ERR(tg))
		return ERR_PTR(-ENOMEM);

	return &tg->css;
}

7544
static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
7545
{
7546 7547
	struct task_group *tg = css_tg(css);
	struct task_group *parent = css_tg(css_parent(css));
7548

T
Tejun Heo 已提交
7549 7550
	if (parent)
		sched_online_group(tg, parent);
7551 7552 7553
	return 0;
}

7554
static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
7555
{
7556
	struct task_group *tg = css_tg(css);
7557 7558 7559 7560

	sched_destroy_group(tg);
}

7561
static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
7562
{
7563
	struct task_group *tg = css_tg(css);
7564 7565 7566 7567

	sched_offline_group(tg);
}

7568
static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
7569
				 struct cgroup_taskset *tset)
7570
{
7571 7572
	struct task_struct *task;

7573
	cgroup_taskset_for_each(task, css, tset) {
7574
#ifdef CONFIG_RT_GROUP_SCHED
7575
		if (!sched_rt_can_attach(css_tg(css), task))
7576
			return -EINVAL;
7577
#else
7578 7579 7580
		/* We don't support RT-tasks being in separate groups */
		if (task->sched_class != &fair_sched_class)
			return -EINVAL;
7581
#endif
7582
	}
7583 7584
	return 0;
}
7585

7586
static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
7587
			      struct cgroup_taskset *tset)
7588
{
7589 7590
	struct task_struct *task;

7591
	cgroup_taskset_for_each(task, css, tset)
7592
		sched_move_task(task);
7593 7594
}

7595 7596 7597
static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
			    struct cgroup_subsys_state *old_css,
			    struct task_struct *task)
7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609
{
	/*
	 * cgroup_exit() is called in the copy_process() failure path.
	 * Ignore this case since the task hasn't ran yet, this avoids
	 * trying to poke a half freed task state from generic code.
	 */
	if (!(task->flags & PF_EXITING))
		return;

	sched_move_task(task);
}

7610
#ifdef CONFIG_FAIR_GROUP_SCHED
7611 7612
static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
				struct cftype *cftype, u64 shareval)
7613
{
7614
	return sched_group_set_shares(css_tg(css), scale_load(shareval));
7615 7616
}

7617 7618
static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
			       struct cftype *cft)
7619
{
7620
	struct task_group *tg = css_tg(css);
7621

7622
	return (u64) scale_load_down(tg->shares);
7623
}
7624 7625

#ifdef CONFIG_CFS_BANDWIDTH
7626 7627
static DEFINE_MUTEX(cfs_constraints_mutex);

7628 7629 7630
const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */

7631 7632
static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);

7633 7634
static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
{
7635
	int i, ret = 0, runtime_enabled, runtime_was_enabled;
7636
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656

	if (tg == &root_task_group)
		return -EINVAL;

	/*
	 * Ensure we have at some amount of bandwidth every period.  This is
	 * to prevent reaching a state of large arrears when throttled via
	 * entity_tick() resulting in prolonged exit starvation.
	 */
	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
		return -EINVAL;

	/*
	 * Likewise, bound things on the otherside by preventing insane quota
	 * periods.  This also allows us to normalize in computing quota
	 * feasibility.
	 */
	if (period > max_cfs_quota_period)
		return -EINVAL;

7657 7658 7659 7660 7661
	mutex_lock(&cfs_constraints_mutex);
	ret = __cfs_schedulable(tg, period, quota);
	if (ret)
		goto out_unlock;

7662
	runtime_enabled = quota != RUNTIME_INF;
7663
	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
7664 7665 7666 7667 7668 7669
	/*
	 * If we need to toggle cfs_bandwidth_used, off->on must occur
	 * before making related changes, and on->off must occur afterwards
	 */
	if (runtime_enabled && !runtime_was_enabled)
		cfs_bandwidth_usage_inc();
7670 7671 7672
	raw_spin_lock_irq(&cfs_b->lock);
	cfs_b->period = ns_to_ktime(period);
	cfs_b->quota = quota;
7673

P
Paul Turner 已提交
7674
	__refill_cfs_bandwidth_runtime(cfs_b);
7675 7676 7677 7678 7679 7680
	/* restart the period timer (if active) to handle new period expiry */
	if (runtime_enabled && cfs_b->timer_active) {
		/* force a reprogram */
		cfs_b->timer_active = 0;
		__start_cfs_bandwidth(cfs_b);
	}
7681 7682 7683 7684
	raw_spin_unlock_irq(&cfs_b->lock);

	for_each_possible_cpu(i) {
		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
7685
		struct rq *rq = cfs_rq->rq;
7686 7687

		raw_spin_lock_irq(&rq->lock);
7688
		cfs_rq->runtime_enabled = runtime_enabled;
7689
		cfs_rq->runtime_remaining = 0;
7690

7691
		if (cfs_rq->throttled)
7692
			unthrottle_cfs_rq(cfs_rq);
7693 7694
		raw_spin_unlock_irq(&rq->lock);
	}
7695 7696
	if (runtime_was_enabled && !runtime_enabled)
		cfs_bandwidth_usage_dec();
7697 7698
out_unlock:
	mutex_unlock(&cfs_constraints_mutex);
7699

7700
	return ret;
7701 7702 7703 7704 7705 7706
}

int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
{
	u64 quota, period;

7707
	period = ktime_to_ns(tg->cfs_bandwidth.period);
7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719
	if (cfs_quota_us < 0)
		quota = RUNTIME_INF;
	else
		quota = (u64)cfs_quota_us * NSEC_PER_USEC;

	return tg_set_cfs_bandwidth(tg, period, quota);
}

long tg_get_cfs_quota(struct task_group *tg)
{
	u64 quota_us;

7720
	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
7721 7722
		return -1;

7723
	quota_us = tg->cfs_bandwidth.quota;
7724 7725 7726 7727 7728 7729 7730 7731 7732 7733
	do_div(quota_us, NSEC_PER_USEC);

	return quota_us;
}

int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
{
	u64 quota, period;

	period = (u64)cfs_period_us * NSEC_PER_USEC;
7734
	quota = tg->cfs_bandwidth.quota;
7735 7736 7737 7738 7739 7740 7741 7742

	return tg_set_cfs_bandwidth(tg, period, quota);
}

long tg_get_cfs_period(struct task_group *tg)
{
	u64 cfs_period_us;

7743
	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
7744 7745 7746 7747 7748
	do_div(cfs_period_us, NSEC_PER_USEC);

	return cfs_period_us;
}

7749 7750
static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
				  struct cftype *cft)
7751
{
7752
	return tg_get_cfs_quota(css_tg(css));
7753 7754
}

7755 7756
static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
				   struct cftype *cftype, s64 cfs_quota_us)
7757
{
7758
	return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
7759 7760
}

7761 7762
static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
				   struct cftype *cft)
7763
{
7764
	return tg_get_cfs_period(css_tg(css));
7765 7766
}

7767 7768
static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
				    struct cftype *cftype, u64 cfs_period_us)
7769
{
7770
	return tg_set_cfs_period(css_tg(css), cfs_period_us);
7771 7772
}

7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804
struct cfs_schedulable_data {
	struct task_group *tg;
	u64 period, quota;
};

/*
 * normalize group quota/period to be quota/max_period
 * note: units are usecs
 */
static u64 normalize_cfs_quota(struct task_group *tg,
			       struct cfs_schedulable_data *d)
{
	u64 quota, period;

	if (tg == d->tg) {
		period = d->period;
		quota = d->quota;
	} else {
		period = tg_get_cfs_period(tg);
		quota = tg_get_cfs_quota(tg);
	}

	/* note: these should typically be equivalent */
	if (quota == RUNTIME_INF || quota == -1)
		return RUNTIME_INF;

	return to_ratio(period, quota);
}

static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
{
	struct cfs_schedulable_data *d = data;
7805
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7806 7807 7808 7809 7810
	s64 quota = 0, parent_quota = -1;

	if (!tg->parent) {
		quota = RUNTIME_INF;
	} else {
7811
		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831

		quota = normalize_cfs_quota(tg, d);
		parent_quota = parent_b->hierarchal_quota;

		/*
		 * ensure max(child_quota) <= parent_quota, inherit when no
		 * limit is set
		 */
		if (quota == RUNTIME_INF)
			quota = parent_quota;
		else if (parent_quota != RUNTIME_INF && quota > parent_quota)
			return -EINVAL;
	}
	cfs_b->hierarchal_quota = quota;

	return 0;
}

static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
{
7832
	int ret;
7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843
	struct cfs_schedulable_data data = {
		.tg = tg,
		.period = period,
		.quota = quota,
	};

	if (quota != RUNTIME_INF) {
		do_div(data.period, NSEC_PER_USEC);
		do_div(data.quota, NSEC_PER_USEC);
	}

7844 7845 7846 7847 7848
	rcu_read_lock();
	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
	rcu_read_unlock();

	return ret;
7849
}
7850

7851
static int cpu_stats_show(struct seq_file *sf, void *v)
7852
{
7853
	struct task_group *tg = css_tg(seq_css(sf));
7854
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7855

7856 7857 7858
	seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
	seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
	seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
7859 7860 7861

	return 0;
}
7862
#endif /* CONFIG_CFS_BANDWIDTH */
7863
#endif /* CONFIG_FAIR_GROUP_SCHED */
7864

7865
#ifdef CONFIG_RT_GROUP_SCHED
7866 7867
static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
				struct cftype *cft, s64 val)
P
Peter Zijlstra 已提交
7868
{
7869
	return sched_group_set_rt_runtime(css_tg(css), val);
P
Peter Zijlstra 已提交
7870 7871
}

7872 7873
static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
			       struct cftype *cft)
P
Peter Zijlstra 已提交
7874
{
7875
	return sched_group_rt_runtime(css_tg(css));
P
Peter Zijlstra 已提交
7876
}
7877

7878 7879
static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
				    struct cftype *cftype, u64 rt_period_us)
7880
{
7881
	return sched_group_set_rt_period(css_tg(css), rt_period_us);
7882 7883
}

7884 7885
static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
				   struct cftype *cft)
7886
{
7887
	return sched_group_rt_period(css_tg(css));
7888
}
7889
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
7890

7891
static struct cftype cpu_files[] = {
7892
#ifdef CONFIG_FAIR_GROUP_SCHED
7893 7894
	{
		.name = "shares",
7895 7896
		.read_u64 = cpu_shares_read_u64,
		.write_u64 = cpu_shares_write_u64,
7897
	},
7898
#endif
7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909
#ifdef CONFIG_CFS_BANDWIDTH
	{
		.name = "cfs_quota_us",
		.read_s64 = cpu_cfs_quota_read_s64,
		.write_s64 = cpu_cfs_quota_write_s64,
	},
	{
		.name = "cfs_period_us",
		.read_u64 = cpu_cfs_period_read_u64,
		.write_u64 = cpu_cfs_period_write_u64,
	},
7910 7911
	{
		.name = "stat",
7912
		.seq_show = cpu_stats_show,
7913
	},
7914
#endif
7915
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
7916
	{
P
Peter Zijlstra 已提交
7917
		.name = "rt_runtime_us",
7918 7919
		.read_s64 = cpu_rt_runtime_read,
		.write_s64 = cpu_rt_runtime_write,
P
Peter Zijlstra 已提交
7920
	},
7921 7922
	{
		.name = "rt_period_us",
7923 7924
		.read_u64 = cpu_rt_period_read_uint,
		.write_u64 = cpu_rt_period_write_uint,
7925
	},
7926
#endif
7927
	{ }	/* terminate */
7928 7929 7930
};

struct cgroup_subsys cpu_cgroup_subsys = {
I
Ingo Molnar 已提交
7931
	.name		= "cpu",
7932 7933
	.css_alloc	= cpu_cgroup_css_alloc,
	.css_free	= cpu_cgroup_css_free,
7934 7935
	.css_online	= cpu_cgroup_css_online,
	.css_offline	= cpu_cgroup_css_offline,
7936 7937
	.can_attach	= cpu_cgroup_can_attach,
	.attach		= cpu_cgroup_attach,
7938
	.exit		= cpu_cgroup_exit,
I
Ingo Molnar 已提交
7939
	.subsys_id	= cpu_cgroup_subsys_id,
7940
	.base_cftypes	= cpu_files,
7941 7942 7943
	.early_init	= 1,
};

7944
#endif	/* CONFIG_CGROUP_SCHED */
7945

7946 7947 7948 7949 7950
void dump_cpu_task(int cpu)
{
	pr_info("Task dump for CPU %d:\n", cpu);
	sched_show_task(cpu_curr(cpu));
}