core.c 191.2 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  kernel/sched/core.c
L
Linus Torvalds 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
I
Ingo Molnar 已提交
19 20 21 22 23 24
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
25 26
 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 *              Thomas Gleixner, Mike Kravetz
L
Linus Torvalds 已提交
27 28 29 30 31 32
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
33
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
34 35 36
#include <linux/highmem.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
37
#include <linux/capability.h>
L
Linus Torvalds 已提交
38 39
#include <linux/completion.h>
#include <linux/kernel_stat.h>
40
#include <linux/debug_locks.h>
41
#include <linux/perf_event.h>
L
Linus Torvalds 已提交
42 43 44
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
45
#include <linux/freezer.h>
46
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
47 48
#include <linux/blkdev.h>
#include <linux/delay.h>
49
#include <linux/pid_namespace.h>
L
Linus Torvalds 已提交
50 51 52 53 54 55 56
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
57
#include <linux/proc_fs.h>
L
Linus Torvalds 已提交
58
#include <linux/seq_file.h>
59
#include <linux/sysctl.h>
L
Linus Torvalds 已提交
60 61
#include <linux/syscalls.h>
#include <linux/times.h>
62
#include <linux/tsacct_kern.h>
63
#include <linux/kprobes.h>
64
#include <linux/delayacct.h>
65
#include <linux/unistd.h>
J
Jens Axboe 已提交
66
#include <linux/pagemap.h>
P
Peter Zijlstra 已提交
67
#include <linux/hrtimer.h>
R
Reynes Philippe 已提交
68
#include <linux/tick.h>
P
Peter Zijlstra 已提交
69 70
#include <linux/debugfs.h>
#include <linux/ctype.h>
71
#include <linux/ftrace.h>
72
#include <linux/slab.h>
73
#include <linux/init_task.h>
A
Al Viro 已提交
74
#include <linux/binfmts.h>
75
#include <linux/context_tracking.h>
76
#include <linux/compiler.h>
L
Linus Torvalds 已提交
77

78
#include <asm/switch_to.h>
79
#include <asm/tlb.h>
80
#include <asm/irq_regs.h>
81
#include <asm/mutex.h>
G
Glauber Costa 已提交
82 83 84
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#endif
L
Linus Torvalds 已提交
85

86
#include "sched.h"
87
#include "../workqueue_internal.h"
88
#include "../smpboot.h"
89

90
#define CREATE_TRACE_POINTS
91
#include <trace/events/sched.h>
92

93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
#ifdef smp_mb__before_atomic
void __smp_mb__before_atomic(void)
{
	smp_mb__before_atomic();
}
EXPORT_SYMBOL(__smp_mb__before_atomic);
#endif

#ifdef smp_mb__after_atomic
void __smp_mb__after_atomic(void)
{
	smp_mb__after_atomic();
}
EXPORT_SYMBOL(__smp_mb__after_atomic);
#endif

109
void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
110
{
111 112
	unsigned long delta;
	ktime_t soft, hard, now;
113

114 115 116 117 118 119
	for (;;) {
		if (hrtimer_active(period_timer))
			break;

		now = hrtimer_cb_get_time(period_timer);
		hrtimer_forward(period_timer, now, period);
120

121 122 123 124 125 126 127 128
		soft = hrtimer_get_softexpires(period_timer);
		hard = hrtimer_get_expires(period_timer);
		delta = ktime_to_ns(ktime_sub(hard, soft));
		__hrtimer_start_range_ns(period_timer, soft, delta,
					 HRTIMER_MODE_ABS_PINNED, 0);
	}
}

129 130
DEFINE_MUTEX(sched_domains_mutex);
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
131

132
static void update_rq_clock_task(struct rq *rq, s64 delta);
133

134
void update_rq_clock(struct rq *rq)
135
{
136
	s64 delta;
137

138
	if (rq->skip_clock_update > 0)
139
		return;
140

141 142 143
	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
	rq->clock += delta;
	update_rq_clock_task(rq, delta);
144 145
}

I
Ingo Molnar 已提交
146 147 148
/*
 * Debugging: various feature bits
 */
P
Peter Zijlstra 已提交
149 150 151 152

#define SCHED_FEAT(name, enabled)	\
	(1UL << __SCHED_FEAT_##name) * enabled |

I
Ingo Molnar 已提交
153
const_debug unsigned int sysctl_sched_features =
154
#include "features.h"
P
Peter Zijlstra 已提交
155 156 157 158 159 160 161 162
	0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG
#define SCHED_FEAT(name, enabled)	\
	#name ,

163
static const char * const sched_feat_names[] = {
164
#include "features.h"
P
Peter Zijlstra 已提交
165 166 167 168
};

#undef SCHED_FEAT

L
Li Zefan 已提交
169
static int sched_feat_show(struct seq_file *m, void *v)
P
Peter Zijlstra 已提交
170 171 172
{
	int i;

173
	for (i = 0; i < __SCHED_FEAT_NR; i++) {
L
Li Zefan 已提交
174 175 176
		if (!(sysctl_sched_features & (1UL << i)))
			seq_puts(m, "NO_");
		seq_printf(m, "%s ", sched_feat_names[i]);
P
Peter Zijlstra 已提交
177
	}
L
Li Zefan 已提交
178
	seq_puts(m, "\n");
P
Peter Zijlstra 已提交
179

L
Li Zefan 已提交
180
	return 0;
P
Peter Zijlstra 已提交
181 182
}

183 184
#ifdef HAVE_JUMP_LABEL

185 186
#define jump_label_key__true  STATIC_KEY_INIT_TRUE
#define jump_label_key__false STATIC_KEY_INIT_FALSE
187 188 189 190

#define SCHED_FEAT(name, enabled)	\
	jump_label_key__##enabled ,

191
struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
192 193 194 195 196 197 198
#include "features.h"
};

#undef SCHED_FEAT

static void sched_feat_disable(int i)
{
199 200
	if (static_key_enabled(&sched_feat_keys[i]))
		static_key_slow_dec(&sched_feat_keys[i]);
201 202 203 204
}

static void sched_feat_enable(int i)
{
205 206
	if (!static_key_enabled(&sched_feat_keys[i]))
		static_key_slow_inc(&sched_feat_keys[i]);
207 208 209 210 211 212
}
#else
static void sched_feat_disable(int i) { };
static void sched_feat_enable(int i) { };
#endif /* HAVE_JUMP_LABEL */

213
static int sched_feat_set(char *cmp)
P
Peter Zijlstra 已提交
214 215
{
	int i;
216
	int neg = 0;
P
Peter Zijlstra 已提交
217

H
Hillf Danton 已提交
218
	if (strncmp(cmp, "NO_", 3) == 0) {
P
Peter Zijlstra 已提交
219 220 221 222
		neg = 1;
		cmp += 3;
	}

223
	for (i = 0; i < __SCHED_FEAT_NR; i++) {
224
		if (strcmp(cmp, sched_feat_names[i]) == 0) {
225
			if (neg) {
P
Peter Zijlstra 已提交
226
				sysctl_sched_features &= ~(1UL << i);
227 228
				sched_feat_disable(i);
			} else {
P
Peter Zijlstra 已提交
229
				sysctl_sched_features |= (1UL << i);
230 231
				sched_feat_enable(i);
			}
P
Peter Zijlstra 已提交
232 233 234 235
			break;
		}
	}

236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
	return i;
}

static ssize_t
sched_feat_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
	char buf[64];
	char *cmp;
	int i;

	if (cnt > 63)
		cnt = 63;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;
	cmp = strstrip(buf);

	i = sched_feat_set(cmp);
257
	if (i == __SCHED_FEAT_NR)
P
Peter Zijlstra 已提交
258 259
		return -EINVAL;

260
	*ppos += cnt;
P
Peter Zijlstra 已提交
261 262 263 264

	return cnt;
}

L
Li Zefan 已提交
265 266 267 268 269
static int sched_feat_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_feat_show, NULL);
}

270
static const struct file_operations sched_feat_fops = {
L
Li Zefan 已提交
271 272 273 274 275
	.open		= sched_feat_open,
	.write		= sched_feat_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
P
Peter Zijlstra 已提交
276 277 278 279 280 281 282 283 284 285
};

static __init int sched_init_debug(void)
{
	debugfs_create_file("sched_features", 0644, NULL, NULL,
			&sched_feat_fops);

	return 0;
}
late_initcall(sched_init_debug);
286
#endif /* CONFIG_SCHED_DEBUG */
I
Ingo Molnar 已提交
287

288 289 290 291 292 293
/*
 * Number of tasks to iterate in a single balance run.
 * Limited because this is done with IRQs disabled.
 */
const_debug unsigned int sysctl_sched_nr_migrate = 32;

294 295 296 297 298 299 300 301
/*
 * period over which we average the RT time consumption, measured
 * in ms.
 *
 * default: 1s
 */
const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;

P
Peter Zijlstra 已提交
302
/*
P
Peter Zijlstra 已提交
303
 * period over which we measure -rt task cpu usage in us.
P
Peter Zijlstra 已提交
304 305
 * default: 1s
 */
P
Peter Zijlstra 已提交
306
unsigned int sysctl_sched_rt_period = 1000000;
P
Peter Zijlstra 已提交
307

308
__read_mostly int scheduler_running;
309

P
Peter Zijlstra 已提交
310 311 312 313 314
/*
 * part of the period that we allow rt tasks to run in us.
 * default: 0.95s
 */
int sysctl_sched_rt_runtime = 950000;
P
Peter Zijlstra 已提交
315

316
/*
317
 * __task_rq_lock - lock the rq @p resides on.
318
 */
319
static inline struct rq *__task_rq_lock(struct task_struct *p)
320 321
	__acquires(rq->lock)
{
322 323
	struct rq *rq;

324 325
	lockdep_assert_held(&p->pi_lock);

326
	for (;;) {
327
		rq = task_rq(p);
328
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
329
		if (likely(rq == task_rq(p)))
330
			return rq;
331
		raw_spin_unlock(&rq->lock);
332 333 334
	}
}

L
Linus Torvalds 已提交
335
/*
336
 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
L
Linus Torvalds 已提交
337
 */
338
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
339
	__acquires(p->pi_lock)
L
Linus Torvalds 已提交
340 341
	__acquires(rq->lock)
{
342
	struct rq *rq;
L
Linus Torvalds 已提交
343

344
	for (;;) {
345
		raw_spin_lock_irqsave(&p->pi_lock, *flags);
346
		rq = task_rq(p);
347
		raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
348
		if (likely(rq == task_rq(p)))
349
			return rq;
350 351
		raw_spin_unlock(&rq->lock);
		raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
352 353 354
	}
}

A
Alexey Dobriyan 已提交
355
static void __task_rq_unlock(struct rq *rq)
356 357
	__releases(rq->lock)
{
358
	raw_spin_unlock(&rq->lock);
359 360
}

361 362
static inline void
task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
L
Linus Torvalds 已提交
363
	__releases(rq->lock)
364
	__releases(p->pi_lock)
L
Linus Torvalds 已提交
365
{
366 367
	raw_spin_unlock(&rq->lock);
	raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
L
Linus Torvalds 已提交
368 369 370
}

/*
371
 * this_rq_lock - lock this runqueue and disable interrupts.
L
Linus Torvalds 已提交
372
 */
A
Alexey Dobriyan 已提交
373
static struct rq *this_rq_lock(void)
L
Linus Torvalds 已提交
374 375
	__acquires(rq->lock)
{
376
	struct rq *rq;
L
Linus Torvalds 已提交
377 378 379

	local_irq_disable();
	rq = this_rq();
380
	raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
381 382 383 384

	return rq;
}

P
Peter Zijlstra 已提交
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
#ifdef CONFIG_SCHED_HRTICK
/*
 * Use HR-timers to deliver accurate preemption points.
 */

static void hrtick_clear(struct rq *rq)
{
	if (hrtimer_active(&rq->hrtick_timer))
		hrtimer_cancel(&rq->hrtick_timer);
}

/*
 * High-resolution timer tick.
 * Runs from hardirq context with interrupts disabled.
 */
static enum hrtimer_restart hrtick(struct hrtimer *timer)
{
	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

406
	raw_spin_lock(&rq->lock);
407
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
408
	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
409
	raw_spin_unlock(&rq->lock);
P
Peter Zijlstra 已提交
410 411 412 413

	return HRTIMER_NORESTART;
}

414
#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
415 416 417 418 419 420 421 422 423

static int __hrtick_restart(struct rq *rq)
{
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = hrtimer_get_softexpires(timer);

	return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0);
}

424 425 426 427
/*
 * called from hardirq (IPI) context
 */
static void __hrtick_start(void *arg)
428
{
429
	struct rq *rq = arg;
430

431
	raw_spin_lock(&rq->lock);
P
Peter Zijlstra 已提交
432
	__hrtick_restart(rq);
433
	rq->hrtick_csd_pending = 0;
434
	raw_spin_unlock(&rq->lock);
435 436
}

437 438 439 440 441
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
442
void hrtick_start(struct rq *rq, u64 delay)
443
{
444 445
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
446

447
	hrtimer_set_expires(timer, time);
448 449

	if (rq == this_rq()) {
P
Peter Zijlstra 已提交
450
		__hrtick_restart(rq);
451
	} else if (!rq->hrtick_csd_pending) {
452
		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
453 454
		rq->hrtick_csd_pending = 1;
	}
455 456 457 458 459 460 461 462 463 464 465 466 467 468
}

static int
hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
	int cpu = (int)(long)hcpu;

	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
469
		hrtick_clear(cpu_rq(cpu));
470 471 472 473 474 475
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

476
static __init void init_hrtick(void)
477 478 479
{
	hotcpu_notifier(hotplug_hrtick, 0);
}
480 481 482 483 484 485
#else
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
486
void hrtick_start(struct rq *rq, u64 delay)
487
{
488
	__hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
489
			HRTIMER_MODE_REL_PINNED, 0);
490
}
491

A
Andrew Morton 已提交
492
static inline void init_hrtick(void)
P
Peter Zijlstra 已提交
493 494
{
}
495
#endif /* CONFIG_SMP */
P
Peter Zijlstra 已提交
496

497
static void init_rq_hrtick(struct rq *rq)
P
Peter Zijlstra 已提交
498
{
499 500
#ifdef CONFIG_SMP
	rq->hrtick_csd_pending = 0;
P
Peter Zijlstra 已提交
501

502 503 504 505
	rq->hrtick_csd.flags = 0;
	rq->hrtick_csd.func = __hrtick_start;
	rq->hrtick_csd.info = rq;
#endif
P
Peter Zijlstra 已提交
506

507 508
	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rq->hrtick_timer.function = hrtick;
P
Peter Zijlstra 已提交
509
}
A
Andrew Morton 已提交
510
#else	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
511 512 513 514 515 516 517 518
static inline void hrtick_clear(struct rq *rq)
{
}

static inline void init_rq_hrtick(struct rq *rq)
{
}

519 520 521
static inline void init_hrtick(void)
{
}
A
Andrew Morton 已提交
522
#endif	/* CONFIG_SCHED_HRTICK */
P
Peter Zijlstra 已提交
523

524 525 526 527 528 529 530 531 532 533 534 535 536 537
/*
 * cmpxchg based fetch_or, macro so it works for different integer types
 */
#define fetch_or(ptr, val)						\
({	typeof(*(ptr)) __old, __val = *(ptr);				\
 	for (;;) {							\
 		__old = cmpxchg((ptr), __val, __val | (val));		\
 		if (__old == __val)					\
 			break;						\
 		__val = __old;						\
 	}								\
 	__old;								\
})

538
#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
539 540 541 542 543 544 545 546 547 548
/*
 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
 * this avoids any races wrt polling state changes and thereby avoids
 * spurious IPIs.
 */
static bool set_nr_and_not_polling(struct task_struct *p)
{
	struct thread_info *ti = task_thread_info(p);
	return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
}
549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573

/*
 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
 *
 * If this returns true, then the idle task promises to call
 * sched_ttwu_pending() and reschedule soon.
 */
static bool set_nr_if_polling(struct task_struct *p)
{
	struct thread_info *ti = task_thread_info(p);
	typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags);

	for (;;) {
		if (!(val & _TIF_POLLING_NRFLAG))
			return false;
		if (val & _TIF_NEED_RESCHED)
			return true;
		old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
		if (old == val)
			break;
		val = old;
	}
	return true;
}

574 575 576 577 578 579
#else
static bool set_nr_and_not_polling(struct task_struct *p)
{
	set_tsk_need_resched(p);
	return true;
}
580 581 582 583 584 585 586

#ifdef CONFIG_SMP
static bool set_nr_if_polling(struct task_struct *p)
{
	return false;
}
#endif
587 588
#endif

I
Ingo Molnar 已提交
589 590 591 592 593 594 595
/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
596
void resched_task(struct task_struct *p)
I
Ingo Molnar 已提交
597 598 599
{
	int cpu;

600
	lockdep_assert_held(&task_rq(p)->lock);
I
Ingo Molnar 已提交
601

602
	if (test_tsk_need_resched(p))
I
Ingo Molnar 已提交
603 604 605
		return;

	cpu = task_cpu(p);
606

607
	if (cpu == smp_processor_id()) {
608
		set_tsk_need_resched(p);
609
		set_preempt_need_resched();
I
Ingo Molnar 已提交
610
		return;
611
	}
I
Ingo Molnar 已提交
612

613
	if (set_nr_and_not_polling(p))
I
Ingo Molnar 已提交
614
		smp_send_reschedule(cpu);
615 616
	else
		trace_sched_wake_idle_without_ipi(cpu);
I
Ingo Molnar 已提交
617 618
}

619
void resched_cpu(int cpu)
I
Ingo Molnar 已提交
620 621 622 623
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

624
	if (!raw_spin_trylock_irqsave(&rq->lock, flags))
I
Ingo Molnar 已提交
625 626
		return;
	resched_task(cpu_curr(cpu));
627
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
628
}
629

630
#ifdef CONFIG_SMP
631
#ifdef CONFIG_NO_HZ_COMMON
632 633 634 635 636 637 638 639
/*
 * In the semi idle case, use the nearest busy cpu for migrating timers
 * from an idle cpu.  This is good for power-savings.
 *
 * We don't do similar optimization for completely idle system, as
 * selecting an idle cpu will add more delays to the timers than intended
 * (as that cpu's timer base may not be uptodate wrt jiffies etc).
 */
640
int get_nohz_timer_target(int pinned)
641 642 643 644 645
{
	int cpu = smp_processor_id();
	int i;
	struct sched_domain *sd;

646 647 648
	if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu))
		return cpu;

649
	rcu_read_lock();
650
	for_each_domain(cpu, sd) {
651 652 653 654 655 656
		for_each_cpu(i, sched_domain_span(sd)) {
			if (!idle_cpu(i)) {
				cpu = i;
				goto unlock;
			}
		}
657
	}
658 659
unlock:
	rcu_read_unlock();
660 661
	return cpu;
}
662 663 664 665 666 667 668 669 670 671
/*
 * When add_timer_on() enqueues a timer into the timer wheel of an
 * idle CPU then this timer might expire before the next timer event
 * which is scheduled to wake up that CPU. In case of a completely
 * idle system the next event might even be infinite time into the
 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 * leaves the inner idle loop so the newly added timer is taken into
 * account when the CPU goes back to idle and evaluates the timer
 * wheel for the next timer event.
 */
672
static void wake_up_idle_cpu(int cpu)
673 674 675 676 677 678
{
	struct rq *rq = cpu_rq(cpu);

	if (cpu == smp_processor_id())
		return;

679
	if (set_nr_and_not_polling(rq->idle))
680
		smp_send_reschedule(cpu);
681 682
	else
		trace_sched_wake_idle_without_ipi(cpu);
683 684
}

685
static bool wake_up_full_nohz_cpu(int cpu)
686
{
687
	if (tick_nohz_full_cpu(cpu)) {
688 689 690 691 692 693 694 695 696 697 698
		if (cpu != smp_processor_id() ||
		    tick_nohz_tick_stopped())
			smp_send_reschedule(cpu);
		return true;
	}

	return false;
}

void wake_up_nohz_cpu(int cpu)
{
699
	if (!wake_up_full_nohz_cpu(cpu))
700 701 702
		wake_up_idle_cpu(cpu);
}

703
static inline bool got_nohz_idle_kick(void)
704
{
705
	int cpu = smp_processor_id();
706 707 708 709 710 711 712 713 714 715 716 717 718

	if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
		return false;

	if (idle_cpu(cpu) && !need_resched())
		return true;

	/*
	 * We can't run Idle Load Balance on this CPU for this time so we
	 * cancel it and clear NOHZ_BALANCE_KICK
	 */
	clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
	return false;
719 720
}

721
#else /* CONFIG_NO_HZ_COMMON */
722

723
static inline bool got_nohz_idle_kick(void)
P
Peter Zijlstra 已提交
724
{
725
	return false;
P
Peter Zijlstra 已提交
726 727
}

728
#endif /* CONFIG_NO_HZ_COMMON */
729

730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746
#ifdef CONFIG_NO_HZ_FULL
bool sched_can_stop_tick(void)
{
       struct rq *rq;

       rq = this_rq();

       /* Make sure rq->nr_running update is visible after the IPI */
       smp_rmb();

       /* More than one running task need preemption */
       if (rq->nr_running > 1)
               return false;

       return true;
}
#endif /* CONFIG_NO_HZ_FULL */
747

748
void sched_avg_update(struct rq *rq)
749
{
750 751
	s64 period = sched_avg_period();

752
	while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
753 754 755 756 757 758
		/*
		 * Inline assembly required to prevent the compiler
		 * optimising this loop into a divmod call.
		 * See __iter_div_u64_rem() for another example of this.
		 */
		asm("" : "+rm" (rq->age_stamp));
759 760 761
		rq->age_stamp += period;
		rq->rt_avg /= 2;
	}
762 763
}

764
#endif /* CONFIG_SMP */
765

766 767
#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
768
/*
769 770 771 772
 * Iterate task_group tree rooted at *from, calling @down when first entering a
 * node and @up when leaving it for the final time.
 *
 * Caller must hold rcu_lock or sufficient equivalent.
773
 */
774
int walk_tg_tree_from(struct task_group *from,
775
			     tg_visitor down, tg_visitor up, void *data)
776 777
{
	struct task_group *parent, *child;
P
Peter Zijlstra 已提交
778
	int ret;
779

780 781
	parent = from;

782
down:
P
Peter Zijlstra 已提交
783 784
	ret = (*down)(parent, data);
	if (ret)
785
		goto out;
786 787 788 789 790 791 792
	list_for_each_entry_rcu(child, &parent->children, siblings) {
		parent = child;
		goto down;

up:
		continue;
	}
P
Peter Zijlstra 已提交
793
	ret = (*up)(parent, data);
794 795
	if (ret || parent == from)
		goto out;
796 797 798 799 800

	child = parent;
	parent = parent->parent;
	if (parent)
		goto up;
801
out:
P
Peter Zijlstra 已提交
802
	return ret;
803 804
}

805
int tg_nop(struct task_group *tg, void *data)
P
Peter Zijlstra 已提交
806
{
807
	return 0;
P
Peter Zijlstra 已提交
808
}
809 810
#endif

811 812
static void set_load_weight(struct task_struct *p)
{
N
Nikhil Rao 已提交
813 814 815
	int prio = p->static_prio - MAX_RT_PRIO;
	struct load_weight *load = &p->se.load;

I
Ingo Molnar 已提交
816 817 818 819
	/*
	 * SCHED_IDLE tasks get minimal weight:
	 */
	if (p->policy == SCHED_IDLE) {
820
		load->weight = scale_load(WEIGHT_IDLEPRIO);
N
Nikhil Rao 已提交
821
		load->inv_weight = WMULT_IDLEPRIO;
I
Ingo Molnar 已提交
822 823
		return;
	}
824

825
	load->weight = scale_load(prio_to_weight[prio]);
N
Nikhil Rao 已提交
826
	load->inv_weight = prio_to_wmult[prio];
827 828
}

829
static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
830
{
831
	update_rq_clock(rq);
832
	sched_info_queued(rq, p);
833
	p->sched_class->enqueue_task(rq, p, flags);
834 835
}

836
static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
837
{
838
	update_rq_clock(rq);
839
	sched_info_dequeued(rq, p);
840
	p->sched_class->dequeue_task(rq, p, flags);
841 842
}

843
void activate_task(struct rq *rq, struct task_struct *p, int flags)
844 845 846 847
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible--;

848
	enqueue_task(rq, p, flags);
849 850
}

851
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
852 853 854 855
{
	if (task_contributes_to_load(p))
		rq->nr_uninterruptible++;

856
	dequeue_task(rq, p, flags);
857 858
}

859
static void update_rq_clock_task(struct rq *rq, s64 delta)
860
{
861 862 863 864 865 866 867 868
/*
 * In theory, the compile should just see 0 here, and optimize out the call
 * to sched_rt_avg_update. But I don't trust it...
 */
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
	s64 steal = 0, irq_delta = 0;
#endif
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
869
	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890

	/*
	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
	 * this case when a previous update_rq_clock() happened inside a
	 * {soft,}irq region.
	 *
	 * When this happens, we stop ->clock_task and only update the
	 * prev_irq_time stamp to account for the part that fit, so that a next
	 * update will consume the rest. This ensures ->clock_task is
	 * monotonic.
	 *
	 * It does however cause some slight miss-attribution of {soft,}irq
	 * time, a more accurate solution would be to update the irq_time using
	 * the current rq->clock timestamp, except that would require using
	 * atomic ops.
	 */
	if (irq_delta > delta)
		irq_delta = delta;

	rq->prev_irq_time += irq_delta;
	delta -= irq_delta;
891 892
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
893
	if (static_key_false((&paravirt_steal_rq_enabled))) {
894 895 896 897 898 899 900 901 902 903 904
		steal = paravirt_steal_clock(cpu_of(rq));
		steal -= rq->prev_steal_time_rq;

		if (unlikely(steal > delta))
			steal = delta;

		rq->prev_steal_time_rq += steal;
		delta -= steal;
	}
#endif

905 906
	rq->clock_task += delta;

907
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
908
	if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
909 910
		sched_rt_avg_update(rq, irq_delta + steal);
#endif
911 912
}

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
void sched_set_stop_task(int cpu, struct task_struct *stop)
{
	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	struct task_struct *old_stop = cpu_rq(cpu)->stop;

	if (stop) {
		/*
		 * Make it appear like a SCHED_FIFO task, its something
		 * userspace knows about and won't get confused about.
		 *
		 * Also, it will make PI more or less work without too
		 * much confusion -- but then, stop work should not
		 * rely on PI working anyway.
		 */
		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);

		stop->sched_class = &stop_sched_class;
	}

	cpu_rq(cpu)->stop = stop;

	if (old_stop) {
		/*
		 * Reset it back to a normal scheduling class so that
		 * it can die in pieces.
		 */
		old_stop->sched_class = &rt_sched_class;
	}
}

943
/*
I
Ingo Molnar 已提交
944
 * __normal_prio - return the priority that is based on the static prio
945 946 947
 */
static inline int __normal_prio(struct task_struct *p)
{
I
Ingo Molnar 已提交
948
	return p->static_prio;
949 950
}

951 952 953 954 955 956 957
/*
 * Calculate the expected normal priority: i.e. priority
 * without taking RT-inheritance into account. Might be
 * boosted by interactivity modifiers. Changes upon fork,
 * setprio syscalls, and whenever the interactivity
 * estimator recalculates.
 */
958
static inline int normal_prio(struct task_struct *p)
959 960 961
{
	int prio;

962 963 964
	if (task_has_dl_policy(p))
		prio = MAX_DL_PRIO-1;
	else if (task_has_rt_policy(p))
965 966 967 968 969 970 971 972 973 974 975 976 977
		prio = MAX_RT_PRIO-1 - p->rt_priority;
	else
		prio = __normal_prio(p);
	return prio;
}

/*
 * Calculate the current priority, i.e. the priority
 * taken into account by the scheduler. This value might
 * be boosted by RT tasks, or might be boosted by
 * interactivity modifiers. Will be RT if the task got
 * RT-boosted. If not then it returns p->normal_prio.
 */
978
static int effective_prio(struct task_struct *p)
979 980 981 982 983 984 985 986 987 988 989 990
{
	p->normal_prio = normal_prio(p);
	/*
	 * If we are RT tasks or we were boosted to RT priority,
	 * keep the priority unchanged. Otherwise, update priority
	 * to the normal priority:
	 */
	if (!rt_prio(p->prio))
		return p->normal_prio;
	return p->prio;
}

L
Linus Torvalds 已提交
991 992 993
/**
 * task_curr - is this task currently executing on a CPU?
 * @p: the task in question.
994 995
 *
 * Return: 1 if the task is currently executing. 0 otherwise.
L
Linus Torvalds 已提交
996
 */
997
inline int task_curr(const struct task_struct *p)
L
Linus Torvalds 已提交
998 999 1000 1001
{
	return cpu_curr(task_cpu(p)) == p;
}

1002 1003
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
				       const struct sched_class *prev_class,
P
Peter Zijlstra 已提交
1004
				       int oldprio)
1005 1006 1007
{
	if (prev_class != p->sched_class) {
		if (prev_class->switched_from)
P
Peter Zijlstra 已提交
1008 1009
			prev_class->switched_from(rq, p);
		p->sched_class->switched_to(rq, p);
1010
	} else if (oldprio != p->prio || dl_task(p))
P
Peter Zijlstra 已提交
1011
		p->sched_class->prio_changed(rq, p, oldprio);
1012 1013
}

1014
void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
{
	const struct sched_class *class;

	if (p->sched_class == rq->curr->sched_class) {
		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
	} else {
		for_each_class(class) {
			if (class == rq->curr->sched_class)
				break;
			if (class == p->sched_class) {
				resched_task(rq->curr);
				break;
			}
		}
	}

	/*
	 * A queue event has occurred, and we're going to schedule.  In
	 * this case, we can save a useless back to back clock update.
	 */
P
Peter Zijlstra 已提交
1035
	if (rq->curr->on_rq && test_tsk_need_resched(rq->curr))
1036 1037 1038
		rq->skip_clock_update = 1;
}

L
Linus Torvalds 已提交
1039
#ifdef CONFIG_SMP
I
Ingo Molnar 已提交
1040
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
I
Ingo Molnar 已提交
1041
{
1042 1043 1044 1045 1046
#ifdef CONFIG_SCHED_DEBUG
	/*
	 * We should never call set_task_cpu() on a blocked task,
	 * ttwu() will sort out the placement.
	 */
P
Peter Zijlstra 已提交
1047
	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
1048
			!(task_preempt_count(p) & PREEMPT_ACTIVE));
1049 1050

#ifdef CONFIG_LOCKDEP
1051 1052 1053 1054 1055
	/*
	 * The caller should hold either p->pi_lock or rq->lock, when changing
	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
	 *
	 * sched_move_task() holds both and thus holding either pins the cgroup,
P
Peter Zijlstra 已提交
1056
	 * see task_group().
1057 1058 1059 1060
	 *
	 * Furthermore, all task_rq users should acquire both locks, see
	 * task_rq_lock().
	 */
1061 1062 1063
	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
				      lockdep_is_held(&task_rq(p)->lock)));
#endif
1064 1065
#endif

1066
	trace_sched_migrate_task(p, new_cpu);
1067

1068
	if (task_cpu(p) != new_cpu) {
1069 1070
		if (p->sched_class->migrate_task_rq)
			p->sched_class->migrate_task_rq(p, new_cpu);
1071
		p->se.nr_migrations++;
1072
		perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
1073
	}
I
Ingo Molnar 已提交
1074 1075

	__set_task_cpu(p, new_cpu);
I
Ingo Molnar 已提交
1076 1077
}

1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
static void __migrate_swap_task(struct task_struct *p, int cpu)
{
	if (p->on_rq) {
		struct rq *src_rq, *dst_rq;

		src_rq = task_rq(p);
		dst_rq = cpu_rq(cpu);

		deactivate_task(src_rq, p, 0);
		set_task_cpu(p, cpu);
		activate_task(dst_rq, p, 0);
		check_preempt_curr(dst_rq, p, 0);
	} else {
		/*
		 * Task isn't running anymore; make it appear like we migrated
		 * it before it went to sleep. This means on wakeup we make the
		 * previous cpu our targer instead of where it really is.
		 */
		p->wake_cpu = cpu;
	}
}

struct migration_swap_arg {
	struct task_struct *src_task, *dst_task;
	int src_cpu, dst_cpu;
};

static int migrate_swap_stop(void *data)
{
	struct migration_swap_arg *arg = data;
	struct rq *src_rq, *dst_rq;
	int ret = -EAGAIN;

	src_rq = cpu_rq(arg->src_cpu);
	dst_rq = cpu_rq(arg->dst_cpu);

1114 1115
	double_raw_lock(&arg->src_task->pi_lock,
			&arg->dst_task->pi_lock);
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
	double_rq_lock(src_rq, dst_rq);
	if (task_cpu(arg->dst_task) != arg->dst_cpu)
		goto unlock;

	if (task_cpu(arg->src_task) != arg->src_cpu)
		goto unlock;

	if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
		goto unlock;

	if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
		goto unlock;

	__migrate_swap_task(arg->src_task, arg->dst_cpu);
	__migrate_swap_task(arg->dst_task, arg->src_cpu);

	ret = 0;

unlock:
	double_rq_unlock(src_rq, dst_rq);
1136 1137
	raw_spin_unlock(&arg->dst_task->pi_lock);
	raw_spin_unlock(&arg->src_task->pi_lock);
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159

	return ret;
}

/*
 * Cross migrate two tasks
 */
int migrate_swap(struct task_struct *cur, struct task_struct *p)
{
	struct migration_swap_arg arg;
	int ret = -EINVAL;

	arg = (struct migration_swap_arg){
		.src_task = cur,
		.src_cpu = task_cpu(cur),
		.dst_task = p,
		.dst_cpu = task_cpu(p),
	};

	if (arg.src_cpu == arg.dst_cpu)
		goto out;

1160 1161 1162 1163
	/*
	 * These three tests are all lockless; this is OK since all of them
	 * will be re-checked with proper locks held further down the line.
	 */
1164 1165 1166 1167 1168 1169 1170 1171 1172
	if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
		goto out;

	if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
		goto out;

	if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
		goto out;

1173
	trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
1174 1175 1176 1177 1178 1179
	ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);

out:
	return ret;
}

1180
struct migration_arg {
1181
	struct task_struct *task;
L
Linus Torvalds 已提交
1182
	int dest_cpu;
1183
};
L
Linus Torvalds 已提交
1184

1185 1186
static int migration_cpu_stop(void *data);

L
Linus Torvalds 已提交
1187 1188 1189
/*
 * wait_task_inactive - wait for a thread to unschedule.
 *
R
Roland McGrath 已提交
1190 1191 1192 1193 1194 1195 1196
 * If @match_state is nonzero, it's the @p->state value just checked and
 * not expected to change.  If it changes, i.e. @p might have woken up,
 * then return zero.  When we succeed in waiting for @p to be off its CPU,
 * we return a positive number (its total switch count).  If a second call
 * a short while later returns the same number, the caller can be sure that
 * @p has remained unscheduled the whole time.
 *
L
Linus Torvalds 已提交
1197 1198 1199 1200 1201 1202
 * The caller must ensure that the task *will* unschedule sometime soon,
 * else this function might spin for a *long* time. This function can't
 * be called with interrupts off, or it may introduce deadlock with
 * smp_call_function() if an IPI is sent by the same process we are
 * waiting to become inactive.
 */
R
Roland McGrath 已提交
1203
unsigned long wait_task_inactive(struct task_struct *p, long match_state)
L
Linus Torvalds 已提交
1204 1205
{
	unsigned long flags;
I
Ingo Molnar 已提交
1206
	int running, on_rq;
R
Roland McGrath 已提交
1207
	unsigned long ncsw;
1208
	struct rq *rq;
L
Linus Torvalds 已提交
1209

1210 1211 1212 1213 1214 1215 1216 1217
	for (;;) {
		/*
		 * We do the initial early heuristics without holding
		 * any task-queue locks at all. We'll only try to get
		 * the runqueue lock when things look like they will
		 * work out!
		 */
		rq = task_rq(p);
1218

1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229
		/*
		 * If the task is actively running on another CPU
		 * still, just relax and busy-wait without holding
		 * any locks.
		 *
		 * NOTE! Since we don't hold any locks, it's not
		 * even sure that "rq" stays as the right runqueue!
		 * But we don't care, since "task_running()" will
		 * return false if the runqueue has changed and p
		 * is actually now running somewhere else!
		 */
R
Roland McGrath 已提交
1230 1231 1232
		while (task_running(rq, p)) {
			if (match_state && unlikely(p->state != match_state))
				return 0;
1233
			cpu_relax();
R
Roland McGrath 已提交
1234
		}
1235

1236 1237 1238 1239 1240 1241
		/*
		 * Ok, time to look more closely! We need the rq
		 * lock now, to be *sure*. If we're wrong, we'll
		 * just go back and repeat.
		 */
		rq = task_rq_lock(p, &flags);
1242
		trace_sched_wait_task(p);
1243
		running = task_running(rq, p);
P
Peter Zijlstra 已提交
1244
		on_rq = p->on_rq;
R
Roland McGrath 已提交
1245
		ncsw = 0;
1246
		if (!match_state || p->state == match_state)
1247
			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1248
		task_rq_unlock(rq, p, &flags);
1249

R
Roland McGrath 已提交
1250 1251 1252 1253 1254 1255
		/*
		 * If it changed from the expected state, bail out now.
		 */
		if (unlikely(!ncsw))
			break;

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
		/*
		 * Was it really running after all now that we
		 * checked with the proper locks actually held?
		 *
		 * Oops. Go back and try again..
		 */
		if (unlikely(running)) {
			cpu_relax();
			continue;
		}
1266

1267 1268 1269 1270 1271
		/*
		 * It's not enough that it's not actively running,
		 * it must be off the runqueue _entirely_, and not
		 * preempted!
		 *
1272
		 * So if it was still runnable (but just not actively
1273 1274 1275 1276
		 * running right now), it's preempted, and we should
		 * yield - it could be a while.
		 */
		if (unlikely(on_rq)) {
1277 1278 1279 1280
			ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);

			set_current_state(TASK_UNINTERRUPTIBLE);
			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1281 1282
			continue;
		}
1283

1284 1285 1286 1287 1288 1289 1290
		/*
		 * Ahh, all good. It wasn't running, and it wasn't
		 * runnable, which means that it will never become
		 * running in the future either. We're all done!
		 */
		break;
	}
R
Roland McGrath 已提交
1291 1292

	return ncsw;
L
Linus Torvalds 已提交
1293 1294 1295 1296 1297 1298 1299 1300 1301
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
L
Lucas De Marchi 已提交
1302
 * NOTE: this function doesn't have to take the runqueue lock,
L
Linus Torvalds 已提交
1303 1304 1305 1306 1307
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
1308
void kick_process(struct task_struct *p)
L
Linus Torvalds 已提交
1309 1310 1311 1312 1313 1314 1315 1316 1317
{
	int cpu;

	preempt_disable();
	cpu = task_cpu(p);
	if ((cpu != smp_processor_id()) && task_curr(p))
		smp_send_reschedule(cpu);
	preempt_enable();
}
R
Rusty Russell 已提交
1318
EXPORT_SYMBOL_GPL(kick_process);
N
Nick Piggin 已提交
1319
#endif /* CONFIG_SMP */
L
Linus Torvalds 已提交
1320

1321
#ifdef CONFIG_SMP
1322
/*
1323
 * ->cpus_allowed is protected by both rq->lock and p->pi_lock
1324
 */
1325 1326
static int select_fallback_rq(int cpu, struct task_struct *p)
{
1327 1328
	int nid = cpu_to_node(cpu);
	const struct cpumask *nodemask = NULL;
1329 1330
	enum { cpuset, possible, fail } state = cpuset;
	int dest_cpu;
1331

1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
	/*
	 * If the node that the cpu is on has been offlined, cpu_to_node()
	 * will return -1. There is no cpu on the node, and we should
	 * select the cpu on the other node.
	 */
	if (nid != -1) {
		nodemask = cpumask_of_node(nid);

		/* Look for allowed, online CPU in same node. */
		for_each_cpu(dest_cpu, nodemask) {
			if (!cpu_online(dest_cpu))
				continue;
			if (!cpu_active(dest_cpu))
				continue;
			if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
				return dest_cpu;
		}
1349
	}
1350

1351 1352
	for (;;) {
		/* Any allowed, online CPU? */
1353
		for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
1354 1355 1356 1357 1358 1359
			if (!cpu_online(dest_cpu))
				continue;
			if (!cpu_active(dest_cpu))
				continue;
			goto out;
		}
1360

1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
		switch (state) {
		case cpuset:
			/* No more Mr. Nice Guy. */
			cpuset_cpus_allowed_fallback(p);
			state = possible;
			break;

		case possible:
			do_set_cpus_allowed(p, cpu_possible_mask);
			state = fail;
			break;

		case fail:
			BUG();
			break;
		}
	}

out:
	if (state != cpuset) {
		/*
		 * Don't tell them about moving exiting tasks or
		 * kernel threads (both mm NULL), since they never
		 * leave kernel.
		 */
		if (p->mm && printk_ratelimit()) {
1387
			printk_deferred("process %d (%s) no longer affine to cpu%d\n",
1388 1389
					task_pid_nr(p), p->comm, cpu);
		}
1390 1391 1392 1393 1394
	}

	return dest_cpu;
}

1395
/*
1396
 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
1397
 */
1398
static inline
1399
int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
1400
{
1401
	cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412

	/*
	 * In order not to call set_task_cpu() on a blocking task we need
	 * to rely on ttwu() to place the task on a valid ->cpus_allowed
	 * cpu.
	 *
	 * Since this is common to all placement strategies, this lives here.
	 *
	 * [ this allows ->select_task() to simply return task_cpu(p) and
	 *   not worry about this generic constraint ]
	 */
1413
	if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
P
Peter Zijlstra 已提交
1414
		     !cpu_online(cpu)))
1415
		cpu = select_fallback_rq(task_cpu(p), p);
1416 1417

	return cpu;
1418
}
1419 1420 1421 1422 1423 1424

static void update_avg(u64 *avg, u64 sample)
{
	s64 diff = sample - *avg;
	*avg += diff >> 3;
}
1425 1426
#endif

P
Peter Zijlstra 已提交
1427
static void
1428
ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
T
Tejun Heo 已提交
1429
{
P
Peter Zijlstra 已提交
1430
#ifdef CONFIG_SCHEDSTATS
1431 1432
	struct rq *rq = this_rq();

P
Peter Zijlstra 已提交
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
#ifdef CONFIG_SMP
	int this_cpu = smp_processor_id();

	if (cpu == this_cpu) {
		schedstat_inc(rq, ttwu_local);
		schedstat_inc(p, se.statistics.nr_wakeups_local);
	} else {
		struct sched_domain *sd;

		schedstat_inc(p, se.statistics.nr_wakeups_remote);
1443
		rcu_read_lock();
P
Peter Zijlstra 已提交
1444 1445 1446 1447 1448 1449
		for_each_domain(this_cpu, sd) {
			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
				schedstat_inc(sd, ttwu_wake_remote);
				break;
			}
		}
1450
		rcu_read_unlock();
P
Peter Zijlstra 已提交
1451
	}
1452 1453 1454 1455

	if (wake_flags & WF_MIGRATED)
		schedstat_inc(p, se.statistics.nr_wakeups_migrate);

P
Peter Zijlstra 已提交
1456 1457 1458
#endif /* CONFIG_SMP */

	schedstat_inc(rq, ttwu_count);
T
Tejun Heo 已提交
1459
	schedstat_inc(p, se.statistics.nr_wakeups);
P
Peter Zijlstra 已提交
1460 1461

	if (wake_flags & WF_SYNC)
T
Tejun Heo 已提交
1462
		schedstat_inc(p, se.statistics.nr_wakeups_sync);
P
Peter Zijlstra 已提交
1463 1464 1465 1466 1467 1468

#endif /* CONFIG_SCHEDSTATS */
}

static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
{
T
Tejun Heo 已提交
1469
	activate_task(rq, p, en_flags);
P
Peter Zijlstra 已提交
1470
	p->on_rq = 1;
1471 1472 1473 1474

	/* if a worker is waking up, notify workqueue */
	if (p->flags & PF_WQ_WORKER)
		wq_worker_waking_up(p, cpu_of(rq));
T
Tejun Heo 已提交
1475 1476
}

1477 1478 1479
/*
 * Mark the task runnable and perform wakeup-preemption.
 */
1480
static void
1481
ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
T
Tejun Heo 已提交
1482 1483
{
	check_preempt_curr(rq, p, wake_flags);
1484
	trace_sched_wakeup(p, true);
T
Tejun Heo 已提交
1485 1486 1487 1488 1489 1490

	p->state = TASK_RUNNING;
#ifdef CONFIG_SMP
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);

1491
	if (rq->idle_stamp) {
1492
		u64 delta = rq_clock(rq) - rq->idle_stamp;
1493
		u64 max = 2*rq->max_idle_balance_cost;
T
Tejun Heo 已提交
1494

1495 1496 1497
		update_avg(&rq->avg_idle, delta);

		if (rq->avg_idle > max)
T
Tejun Heo 已提交
1498
			rq->avg_idle = max;
1499

T
Tejun Heo 已提交
1500 1501 1502 1503 1504
		rq->idle_stamp = 0;
	}
#endif
}

1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
static void
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
{
#ifdef CONFIG_SMP
	if (p->sched_contributes_to_load)
		rq->nr_uninterruptible--;
#endif

	ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
	ttwu_do_wakeup(rq, p, wake_flags);
}

/*
 * Called in case the task @p isn't fully descheduled from its runqueue,
 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
 * since all we need to do is flip p->state to TASK_RUNNING, since
 * the task is still ->on_rq.
 */
static int ttwu_remote(struct task_struct *p, int wake_flags)
{
	struct rq *rq;
	int ret = 0;

	rq = __task_rq_lock(p);
	if (p->on_rq) {
1530 1531
		/* check_preempt_curr() may use rq clock */
		update_rq_clock(rq);
1532 1533 1534 1535 1536 1537 1538 1539
		ttwu_do_wakeup(rq, p, wake_flags);
		ret = 1;
	}
	__task_rq_unlock(rq);

	return ret;
}

1540
#ifdef CONFIG_SMP
1541
void sched_ttwu_pending(void)
1542 1543
{
	struct rq *rq = this_rq();
P
Peter Zijlstra 已提交
1544 1545
	struct llist_node *llist = llist_del_all(&rq->wake_list);
	struct task_struct *p;
1546
	unsigned long flags;
1547

1548 1549 1550 1551
	if (!llist)
		return;

	raw_spin_lock_irqsave(&rq->lock, flags);
1552

P
Peter Zijlstra 已提交
1553 1554 1555
	while (llist) {
		p = llist_entry(llist, struct task_struct, wake_entry);
		llist = llist_next(llist);
1556 1557 1558
		ttwu_do_activate(rq, p, 0);
	}

1559
	raw_spin_unlock_irqrestore(&rq->lock, flags);
1560 1561 1562 1563
}

void scheduler_ipi(void)
{
1564 1565 1566 1567 1568
	/*
	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
	 * TIF_NEED_RESCHED remotely (for the first time) will also send
	 * this IPI.
	 */
1569
	preempt_fold_need_resched();
1570

1571 1572 1573
	if (llist_empty(&this_rq()->wake_list)
			&& !tick_nohz_full_cpu(smp_processor_id())
			&& !got_nohz_idle_kick())
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
		return;

	/*
	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
	 * traditionally all their work was done from the interrupt return
	 * path. Now that we actually do some work, we need to make sure
	 * we do call them.
	 *
	 * Some archs already do call them, luckily irq_enter/exit nest
	 * properly.
	 *
	 * Arguably we should visit all archs and update all handlers,
	 * however a fair share of IPIs are still resched only so this would
	 * somewhat pessimize the simple resched case.
	 */
	irq_enter();
1590
	tick_nohz_full_check();
P
Peter Zijlstra 已提交
1591
	sched_ttwu_pending();
1592 1593 1594 1595

	/*
	 * Check if someone kicked us for doing the nohz idle load balance.
	 */
1596
	if (unlikely(got_nohz_idle_kick())) {
1597
		this_rq()->idle_balance = 1;
1598
		raise_softirq_irqoff(SCHED_SOFTIRQ);
1599
	}
1600
	irq_exit();
1601 1602 1603 1604
}

static void ttwu_queue_remote(struct task_struct *p, int cpu)
{
1605 1606 1607 1608 1609 1610 1611 1612
	struct rq *rq = cpu_rq(cpu);

	if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
		if (!set_nr_if_polling(rq->idle))
			smp_send_reschedule(cpu);
		else
			trace_sched_wake_idle_without_ipi(cpu);
	}
1613
}
1614

1615
bool cpus_share_cache(int this_cpu, int that_cpu)
1616 1617 1618
{
	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
}
1619
#endif /* CONFIG_SMP */
1620

1621 1622 1623 1624
static void ttwu_queue(struct task_struct *p, int cpu)
{
	struct rq *rq = cpu_rq(cpu);

1625
#if defined(CONFIG_SMP)
1626
	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
1627
		sched_clock_cpu(cpu); /* sync clocks x-cpu */
1628 1629 1630 1631 1632
		ttwu_queue_remote(p, cpu);
		return;
	}
#endif

1633 1634 1635
	raw_spin_lock(&rq->lock);
	ttwu_do_activate(rq, p, 0);
	raw_spin_unlock(&rq->lock);
T
Tejun Heo 已提交
1636 1637 1638
}

/**
L
Linus Torvalds 已提交
1639
 * try_to_wake_up - wake up a thread
T
Tejun Heo 已提交
1640
 * @p: the thread to be awakened
L
Linus Torvalds 已提交
1641
 * @state: the mask of task states that can be woken
T
Tejun Heo 已提交
1642
 * @wake_flags: wake modifier flags (WF_*)
L
Linus Torvalds 已提交
1643 1644 1645 1646 1647 1648 1649
 *
 * Put it on the run-queue if it's not already there. The "current"
 * thread is always on the run-queue (except when the actual
 * re-schedule is in progress), and as such you're allowed to do
 * the simpler "current->state = TASK_RUNNING" to mark yourself
 * runnable without the overhead of this.
 *
1650
 * Return: %true if @p was woken up, %false if it was already running.
T
Tejun Heo 已提交
1651
 * or @state didn't match @p's state.
L
Linus Torvalds 已提交
1652
 */
1653 1654
static int
try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
L
Linus Torvalds 已提交
1655 1656
{
	unsigned long flags;
1657
	int cpu, success = 0;
P
Peter Zijlstra 已提交
1658

1659 1660 1661 1662 1663 1664 1665
	/*
	 * If we are going to wake up a thread waiting for CONDITION we
	 * need to ensure that CONDITION=1 done by the caller can not be
	 * reordered with p->state check below. This pairs with mb() in
	 * set_current_state() the waiting thread does.
	 */
	smp_mb__before_spinlock();
1666
	raw_spin_lock_irqsave(&p->pi_lock, flags);
P
Peter Zijlstra 已提交
1667
	if (!(p->state & state))
L
Linus Torvalds 已提交
1668 1669
		goto out;

1670
	success = 1; /* we're going to change ->state */
L
Linus Torvalds 已提交
1671 1672
	cpu = task_cpu(p);

1673 1674
	if (p->on_rq && ttwu_remote(p, wake_flags))
		goto stat;
L
Linus Torvalds 已提交
1675 1676

#ifdef CONFIG_SMP
P
Peter Zijlstra 已提交
1677
	/*
1678 1679
	 * If the owning (remote) cpu is still in the middle of schedule() with
	 * this task as prev, wait until its done referencing the task.
P
Peter Zijlstra 已提交
1680
	 */
1681
	while (p->on_cpu)
1682
		cpu_relax();
1683
	/*
1684
	 * Pairs with the smp_wmb() in finish_lock_switch().
1685
	 */
1686
	smp_rmb();
L
Linus Torvalds 已提交
1687

1688
	p->sched_contributes_to_load = !!task_contributes_to_load(p);
P
Peter Zijlstra 已提交
1689
	p->state = TASK_WAKING;
1690

1691
	if (p->sched_class->task_waking)
1692
		p->sched_class->task_waking(p);
1693

1694
	cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
1695 1696
	if (task_cpu(p) != cpu) {
		wake_flags |= WF_MIGRATED;
1697
		set_task_cpu(p, cpu);
1698
	}
L
Linus Torvalds 已提交
1699 1700
#endif /* CONFIG_SMP */

1701 1702
	ttwu_queue(p, cpu);
stat:
1703
	ttwu_stat(p, cpu, wake_flags);
L
Linus Torvalds 已提交
1704
out:
1705
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
1706 1707 1708 1709

	return success;
}

T
Tejun Heo 已提交
1710 1711 1712 1713
/**
 * try_to_wake_up_local - try to wake up a local task with rq lock held
 * @p: the thread to be awakened
 *
1714
 * Put @p on the run-queue if it's not already there. The caller must
T
Tejun Heo 已提交
1715
 * ensure that this_rq() is locked, @p is bound to this_rq() and not
1716
 * the current task.
T
Tejun Heo 已提交
1717 1718 1719 1720 1721
 */
static void try_to_wake_up_local(struct task_struct *p)
{
	struct rq *rq = task_rq(p);

1722 1723 1724 1725
	if (WARN_ON_ONCE(rq != this_rq()) ||
	    WARN_ON_ONCE(p == current))
		return;

T
Tejun Heo 已提交
1726 1727
	lockdep_assert_held(&rq->lock);

1728 1729 1730 1731 1732 1733
	if (!raw_spin_trylock(&p->pi_lock)) {
		raw_spin_unlock(&rq->lock);
		raw_spin_lock(&p->pi_lock);
		raw_spin_lock(&rq->lock);
	}

T
Tejun Heo 已提交
1734
	if (!(p->state & TASK_NORMAL))
1735
		goto out;
T
Tejun Heo 已提交
1736

P
Peter Zijlstra 已提交
1737
	if (!p->on_rq)
P
Peter Zijlstra 已提交
1738 1739
		ttwu_activate(rq, p, ENQUEUE_WAKEUP);

1740
	ttwu_do_wakeup(rq, p, 0);
1741
	ttwu_stat(p, smp_processor_id(), 0);
1742 1743
out:
	raw_spin_unlock(&p->pi_lock);
T
Tejun Heo 已提交
1744 1745
}

1746 1747 1748 1749 1750
/**
 * wake_up_process - Wake up a specific process
 * @p: The process to be woken up.
 *
 * Attempt to wake up the nominated process and move it to the set of runnable
1751 1752 1753
 * processes.
 *
 * Return: 1 if the process was woken up, 0 if it was already running.
1754 1755 1756 1757
 *
 * It may be assumed that this function implies a write memory barrier before
 * changing the task state if and only if any tasks are woken up.
 */
1758
int wake_up_process(struct task_struct *p)
L
Linus Torvalds 已提交
1759
{
1760 1761
	WARN_ON(task_is_stopped_or_traced(p));
	return try_to_wake_up(p, TASK_NORMAL, 0);
L
Linus Torvalds 已提交
1762 1763 1764
}
EXPORT_SYMBOL(wake_up_process);

1765
int wake_up_state(struct task_struct *p, unsigned int state)
L
Linus Torvalds 已提交
1766 1767 1768 1769 1770 1771 1772
{
	return try_to_wake_up(p, state, 0);
}

/*
 * Perform scheduler related setup for a newly forked process p.
 * p is forked by current.
I
Ingo Molnar 已提交
1773 1774 1775
 *
 * __sched_fork() is basic setup used by init_idle() too:
 */
1776
static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
I
Ingo Molnar 已提交
1777
{
P
Peter Zijlstra 已提交
1778 1779 1780
	p->on_rq			= 0;

	p->se.on_rq			= 0;
I
Ingo Molnar 已提交
1781 1782
	p->se.exec_start		= 0;
	p->se.sum_exec_runtime		= 0;
1783
	p->se.prev_sum_exec_runtime	= 0;
1784
	p->se.nr_migrations		= 0;
P
Peter Zijlstra 已提交
1785
	p->se.vruntime			= 0;
P
Peter Zijlstra 已提交
1786
	INIT_LIST_HEAD(&p->se.group_node);
I
Ingo Molnar 已提交
1787 1788

#ifdef CONFIG_SCHEDSTATS
1789
	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
I
Ingo Molnar 已提交
1790
#endif
N
Nick Piggin 已提交
1791

1792 1793 1794 1795
	RB_CLEAR_NODE(&p->dl.rb_node);
	hrtimer_init(&p->dl.dl_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	p->dl.dl_runtime = p->dl.runtime = 0;
	p->dl.dl_deadline = p->dl.deadline = 0;
1796
	p->dl.dl_period = 0;
1797 1798
	p->dl.flags = 0;

P
Peter Zijlstra 已提交
1799
	INIT_LIST_HEAD(&p->rt.run_list);
N
Nick Piggin 已提交
1800

1801 1802 1803
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&p->preempt_notifiers);
#endif
1804 1805 1806

#ifdef CONFIG_NUMA_BALANCING
	if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
1807
		p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
1808 1809 1810
		p->mm->numa_scan_seq = 0;
	}

1811 1812 1813 1814 1815
	if (clone_flags & CLONE_VM)
		p->numa_preferred_nid = current->numa_preferred_nid;
	else
		p->numa_preferred_nid = -1;

1816 1817
	p->node_stamp = 0ULL;
	p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
1818
	p->numa_scan_period = sysctl_numa_balancing_scan_delay;
1819
	p->numa_work.next = &p->numa_work;
1820 1821
	p->numa_faults_memory = NULL;
	p->numa_faults_buffer_memory = NULL;
1822 1823
	p->last_task_numa_placement = 0;
	p->last_sum_exec_runtime = 0;
1824 1825 1826

	INIT_LIST_HEAD(&p->numa_entry);
	p->numa_group = NULL;
1827
#endif /* CONFIG_NUMA_BALANCING */
I
Ingo Molnar 已提交
1828 1829
}

1830
#ifdef CONFIG_NUMA_BALANCING
1831
#ifdef CONFIG_SCHED_DEBUG
1832 1833 1834 1835 1836 1837 1838
void set_numabalancing_state(bool enabled)
{
	if (enabled)
		sched_feat_set("NUMA");
	else
		sched_feat_set("NO_NUMA");
}
1839 1840 1841 1842 1843 1844
#else
__read_mostly bool numabalancing_enabled;

void set_numabalancing_state(bool enabled)
{
	numabalancing_enabled = enabled;
I
Ingo Molnar 已提交
1845
}
1846
#endif /* CONFIG_SCHED_DEBUG */
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869

#ifdef CONFIG_PROC_SYSCTL
int sysctl_numa_balancing(struct ctl_table *table, int write,
			 void __user *buffer, size_t *lenp, loff_t *ppos)
{
	struct ctl_table t;
	int err;
	int state = numabalancing_enabled;

	if (write && !capable(CAP_SYS_ADMIN))
		return -EPERM;

	t = *table;
	t.data = &state;
	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
	if (err < 0)
		return err;
	if (write)
		set_numabalancing_state(state);
	return err;
}
#endif
#endif
I
Ingo Molnar 已提交
1870 1871 1872 1873

/*
 * fork()/clone()-time setup:
 */
1874
int sched_fork(unsigned long clone_flags, struct task_struct *p)
I
Ingo Molnar 已提交
1875
{
1876
	unsigned long flags;
I
Ingo Molnar 已提交
1877 1878
	int cpu = get_cpu();

1879
	__sched_fork(clone_flags, p);
1880
	/*
1881
	 * We mark the process as running here. This guarantees that
1882 1883 1884
	 * nobody will actually run it, and a signal or other external
	 * event cannot wake it up and insert it on the runqueue either.
	 */
1885
	p->state = TASK_RUNNING;
I
Ingo Molnar 已提交
1886

1887 1888 1889 1890 1891
	/*
	 * Make sure we do not leak PI boosting priority to the child.
	 */
	p->prio = current->normal_prio;

1892 1893 1894 1895
	/*
	 * Revert to default priority/policy on fork if requested.
	 */
	if (unlikely(p->sched_reset_on_fork)) {
1896
		if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
1897
			p->policy = SCHED_NORMAL;
1898
			p->static_prio = NICE_TO_PRIO(0);
1899 1900 1901 1902 1903 1904
			p->rt_priority = 0;
		} else if (PRIO_TO_NICE(p->static_prio) < 0)
			p->static_prio = NICE_TO_PRIO(0);

		p->prio = p->normal_prio = __normal_prio(p);
		set_load_weight(p);
1905

1906 1907 1908 1909 1910 1911
		/*
		 * We don't need the reset flag anymore after the fork. It has
		 * fulfilled its duty:
		 */
		p->sched_reset_on_fork = 0;
	}
1912

1913 1914 1915 1916 1917 1918
	if (dl_prio(p->prio)) {
		put_cpu();
		return -EAGAIN;
	} else if (rt_prio(p->prio)) {
		p->sched_class = &rt_sched_class;
	} else {
H
Hiroshi Shimamoto 已提交
1919
		p->sched_class = &fair_sched_class;
1920
	}
1921

P
Peter Zijlstra 已提交
1922 1923 1924
	if (p->sched_class->task_fork)
		p->sched_class->task_fork(p);

1925 1926 1927 1928 1929 1930 1931
	/*
	 * The child is not yet in the pid-hash so no cgroup attach races,
	 * and the cgroup is pinned to this child due to cgroup_fork()
	 * is ran before sched_fork().
	 *
	 * Silence PROVE_RCU.
	 */
1932
	raw_spin_lock_irqsave(&p->pi_lock, flags);
1933
	set_task_cpu(p, cpu);
1934
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1935

1936
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
I
Ingo Molnar 已提交
1937
	if (likely(sched_info_on()))
1938
		memset(&p->sched_info, 0, sizeof(p->sched_info));
L
Linus Torvalds 已提交
1939
#endif
P
Peter Zijlstra 已提交
1940 1941
#if defined(CONFIG_SMP)
	p->on_cpu = 0;
1942
#endif
1943
	init_task_preempt_count(p);
1944
#ifdef CONFIG_SMP
1945
	plist_node_init(&p->pushable_tasks, MAX_PRIO);
1946
	RB_CLEAR_NODE(&p->pushable_dl_tasks);
1947
#endif
1948

N
Nick Piggin 已提交
1949
	put_cpu();
1950
	return 0;
L
Linus Torvalds 已提交
1951 1952
}

1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
unsigned long to_ratio(u64 period, u64 runtime)
{
	if (runtime == RUNTIME_INF)
		return 1ULL << 20;

	/*
	 * Doing this here saves a lot of checks in all
	 * the calling paths, and returning zero seems
	 * safe for them anyway.
	 */
	if (period == 0)
		return 0;

	return div64_u64(runtime << 20, period);
}

#ifdef CONFIG_SMP
inline struct dl_bw *dl_bw_of(int i)
{
	return &cpu_rq(i)->rd->dl_bw;
}

1975
static inline int dl_bw_cpus(int i)
1976
{
1977 1978 1979 1980 1981 1982 1983
	struct root_domain *rd = cpu_rq(i)->rd;
	int cpus = 0;

	for_each_cpu_and(i, rd->span, cpu_active_mask)
		cpus++;

	return cpus;
1984 1985 1986 1987 1988 1989 1990
}
#else
inline struct dl_bw *dl_bw_of(int i)
{
	return &cpu_rq(i)->dl.dl_bw;
}

1991
static inline int dl_bw_cpus(int i)
1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028
{
	return 1;
}
#endif

static inline
void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw)
{
	dl_b->total_bw -= tsk_bw;
}

static inline
void __dl_add(struct dl_bw *dl_b, u64 tsk_bw)
{
	dl_b->total_bw += tsk_bw;
}

static inline
bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw)
{
	return dl_b->bw != -1 &&
	       dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw;
}

/*
 * We must be sure that accepting a new task (or allowing changing the
 * parameters of an existing one) is consistent with the bandwidth
 * constraints. If yes, this function also accordingly updates the currently
 * allocated bandwidth to reflect the new situation.
 *
 * This function is called while holding p's rq->lock.
 */
static int dl_overflow(struct task_struct *p, int policy,
		       const struct sched_attr *attr)
{

	struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
2029
	u64 period = attr->sched_period ?: attr->sched_deadline;
2030 2031
	u64 runtime = attr->sched_runtime;
	u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
2032
	int cpus, err = -1;
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042

	if (new_bw == p->dl.dl_bw)
		return 0;

	/*
	 * Either if a task, enters, leave, or stays -deadline but changes
	 * its parameters, we may need to update accordingly the total
	 * allocated bandwidth of the container.
	 */
	raw_spin_lock(&dl_b->lock);
2043
	cpus = dl_bw_cpus(task_cpu(p));
2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
	if (dl_policy(policy) && !task_has_dl_policy(p) &&
	    !__dl_overflow(dl_b, cpus, 0, new_bw)) {
		__dl_add(dl_b, new_bw);
		err = 0;
	} else if (dl_policy(policy) && task_has_dl_policy(p) &&
		   !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
		__dl_clear(dl_b, p->dl.dl_bw);
		__dl_add(dl_b, new_bw);
		err = 0;
	} else if (!dl_policy(policy) && task_has_dl_policy(p)) {
		__dl_clear(dl_b, p->dl.dl_bw);
		err = 0;
	}
	raw_spin_unlock(&dl_b->lock);

	return err;
}

extern void init_dl_bw(struct dl_bw *dl_b);

L
Linus Torvalds 已提交
2064 2065 2066 2067 2068 2069 2070
/*
 * wake_up_new_task - wake up a newly created task for the first time.
 *
 * This function will do some initial scheduler statistics housekeeping
 * that must be done for every newly created context, then puts the task
 * on the runqueue and wakes it.
 */
2071
void wake_up_new_task(struct task_struct *p)
L
Linus Torvalds 已提交
2072 2073
{
	unsigned long flags;
I
Ingo Molnar 已提交
2074
	struct rq *rq;
2075

2076
	raw_spin_lock_irqsave(&p->pi_lock, flags);
2077 2078 2079 2080 2081 2082
#ifdef CONFIG_SMP
	/*
	 * Fork balancing, do it here and not earlier because:
	 *  - cpus_allowed can change in the fork path
	 *  - any previously selected cpu might disappear through hotplug
	 */
2083
	set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
2084 2085
#endif

2086 2087
	/* Initialize new task's runnable average */
	init_task_runnable_average(p);
2088
	rq = __task_rq_lock(p);
P
Peter Zijlstra 已提交
2089
	activate_task(rq, p, 0);
P
Peter Zijlstra 已提交
2090
	p->on_rq = 1;
2091
	trace_sched_wakeup_new(p, true);
P
Peter Zijlstra 已提交
2092
	check_preempt_curr(rq, p, WF_FORK);
2093
#ifdef CONFIG_SMP
2094 2095
	if (p->sched_class->task_woken)
		p->sched_class->task_woken(rq, p);
2096
#endif
2097
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
2098 2099
}

2100 2101 2102
#ifdef CONFIG_PREEMPT_NOTIFIERS

/**
2103
 * preempt_notifier_register - tell me when current is being preempted & rescheduled
R
Randy Dunlap 已提交
2104
 * @notifier: notifier struct to register
2105 2106 2107 2108 2109 2110 2111 2112 2113
 */
void preempt_notifier_register(struct preempt_notifier *notifier)
{
	hlist_add_head(&notifier->link, &current->preempt_notifiers);
}
EXPORT_SYMBOL_GPL(preempt_notifier_register);

/**
 * preempt_notifier_unregister - no longer interested in preemption notifications
R
Randy Dunlap 已提交
2114
 * @notifier: notifier struct to unregister
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127
 *
 * This is safe to call from within a preemption notifier.
 */
void preempt_notifier_unregister(struct preempt_notifier *notifier)
{
	hlist_del(&notifier->link);
}
EXPORT_SYMBOL_GPL(preempt_notifier_unregister);

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
	struct preempt_notifier *notifier;

2128
	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2129 2130 2131 2132 2133 2134 2135 2136 2137
		notifier->ops->sched_in(notifier, raw_smp_processor_id());
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
	struct preempt_notifier *notifier;

2138
	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2139 2140 2141
		notifier->ops->sched_out(notifier, next);
}

2142
#else /* !CONFIG_PREEMPT_NOTIFIERS */
2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153

static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
{
}

static void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
				 struct task_struct *next)
{
}

2154
#endif /* CONFIG_PREEMPT_NOTIFIERS */
2155

2156 2157 2158
/**
 * prepare_task_switch - prepare to switch tasks
 * @rq: the runqueue preparing to switch
R
Randy Dunlap 已提交
2159
 * @prev: the current task that is being switched out
2160 2161 2162 2163 2164 2165 2166 2167 2168
 * @next: the task we are going to switch to.
 *
 * This is called with the rq lock held and interrupts off. It must
 * be paired with a subsequent finish_task_switch after the context
 * switch.
 *
 * prepare_task_switch sets up locking and calls architecture specific
 * hooks.
 */
2169 2170 2171
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
		    struct task_struct *next)
2172
{
2173
	trace_sched_switch(prev, next);
2174
	sched_info_switch(rq, prev, next);
2175
	perf_event_task_sched_out(prev, next);
2176
	fire_sched_out_preempt_notifiers(prev, next);
2177 2178 2179 2180
	prepare_lock_switch(rq, next);
	prepare_arch_switch(next);
}

L
Linus Torvalds 已提交
2181 2182
/**
 * finish_task_switch - clean up after a task-switch
2183
 * @rq: runqueue associated with task-switch
L
Linus Torvalds 已提交
2184 2185
 * @prev: the thread we just switched away from.
 *
2186 2187 2188 2189
 * finish_task_switch must be called after the context switch, paired
 * with a prepare_task_switch call before the context switch.
 * finish_task_switch will reconcile locking set up by prepare_task_switch,
 * and do any other architecture-specific cleanup actions.
L
Linus Torvalds 已提交
2190 2191
 *
 * Note that we may have delayed dropping an mm in context_switch(). If
I
Ingo Molnar 已提交
2192
 * so, we finish that here outside of the runqueue lock. (Doing it
L
Linus Torvalds 已提交
2193 2194 2195
 * with the lock held can cause deadlocks; see schedule() for
 * details.)
 */
A
Alexey Dobriyan 已提交
2196
static void finish_task_switch(struct rq *rq, struct task_struct *prev)
L
Linus Torvalds 已提交
2197 2198 2199
	__releases(rq->lock)
{
	struct mm_struct *mm = rq->prev_mm;
O
Oleg Nesterov 已提交
2200
	long prev_state;
L
Linus Torvalds 已提交
2201 2202 2203 2204 2205

	rq->prev_mm = NULL;

	/*
	 * A task struct has one reference for the use as "current".
2206
	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
O
Oleg Nesterov 已提交
2207 2208
	 * schedule one last time. The schedule call will never return, and
	 * the scheduled task must drop that reference.
2209
	 * The test for TASK_DEAD must occur while the runqueue locks are
L
Linus Torvalds 已提交
2210 2211 2212 2213 2214
	 * still held, otherwise prev could be scheduled on another cpu, die
	 * there before we look at prev->state, and then the reference would
	 * be dropped twice.
	 *		Manfred Spraul <manfred@colorfullife.com>
	 */
O
Oleg Nesterov 已提交
2215
	prev_state = prev->state;
2216
	vtime_task_switch(prev);
2217
	finish_arch_switch(prev);
2218
	perf_event_task_sched_in(prev, current);
2219
	finish_lock_switch(rq, prev);
2220
	finish_arch_post_lock_switch();
S
Steven Rostedt 已提交
2221

2222
	fire_sched_in_preempt_notifiers(current);
L
Linus Torvalds 已提交
2223 2224
	if (mm)
		mmdrop(mm);
2225
	if (unlikely(prev_state == TASK_DEAD)) {
2226 2227 2228
		if (prev->sched_class->task_dead)
			prev->sched_class->task_dead(prev);

2229 2230 2231
		/*
		 * Remove function-return probe instances associated with this
		 * task and put them back on the free list.
I
Ingo Molnar 已提交
2232
		 */
2233
		kprobe_flush_task(prev);
L
Linus Torvalds 已提交
2234
		put_task_struct(prev);
2235
	}
2236 2237

	tick_nohz_task_switch(current);
L
Linus Torvalds 已提交
2238 2239
}

2240 2241 2242 2243 2244 2245 2246 2247
#ifdef CONFIG_SMP

/* rq->lock is NOT held, but preemption is disabled */
static inline void post_schedule(struct rq *rq)
{
	if (rq->post_schedule) {
		unsigned long flags;

2248
		raw_spin_lock_irqsave(&rq->lock, flags);
2249 2250
		if (rq->curr->sched_class->post_schedule)
			rq->curr->sched_class->post_schedule(rq);
2251
		raw_spin_unlock_irqrestore(&rq->lock, flags);
2252 2253 2254 2255 2256 2257

		rq->post_schedule = 0;
	}
}

#else
2258

2259 2260
static inline void post_schedule(struct rq *rq)
{
L
Linus Torvalds 已提交
2261 2262
}

2263 2264
#endif

L
Linus Torvalds 已提交
2265 2266 2267 2268
/**
 * schedule_tail - first thing a freshly forked thread must call.
 * @prev: the thread we just switched away from.
 */
2269
asmlinkage __visible void schedule_tail(struct task_struct *prev)
L
Linus Torvalds 已提交
2270 2271
	__releases(rq->lock)
{
2272 2273
	struct rq *rq = this_rq();

2274
	finish_task_switch(rq, prev);
2275

2276 2277 2278 2279 2280
	/*
	 * FIXME: do we need to worry about rq being invalidated by the
	 * task_switch?
	 */
	post_schedule(rq);
2281

2282 2283 2284 2285
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
	/* In this case, finish_task_switch does not reenable preemption */
	preempt_enable();
#endif
L
Linus Torvalds 已提交
2286
	if (current->set_child_tid)
2287
		put_user(task_pid_vnr(current), current->set_child_tid);
L
Linus Torvalds 已提交
2288 2289 2290 2291 2292 2293
}

/*
 * context_switch - switch to the new MM and the new
 * thread's register state.
 */
I
Ingo Molnar 已提交
2294
static inline void
2295
context_switch(struct rq *rq, struct task_struct *prev,
2296
	       struct task_struct *next)
L
Linus Torvalds 已提交
2297
{
I
Ingo Molnar 已提交
2298
	struct mm_struct *mm, *oldmm;
L
Linus Torvalds 已提交
2299

2300
	prepare_task_switch(rq, prev, next);
2301

I
Ingo Molnar 已提交
2302 2303
	mm = next->mm;
	oldmm = prev->active_mm;
2304 2305 2306 2307 2308
	/*
	 * For paravirt, this is coupled with an exit in switch_to to
	 * combine the page table reload and the switch backend into
	 * one hypercall.
	 */
2309
	arch_start_context_switch(prev);
2310

2311
	if (!mm) {
L
Linus Torvalds 已提交
2312 2313 2314 2315 2316 2317
		next->active_mm = oldmm;
		atomic_inc(&oldmm->mm_count);
		enter_lazy_tlb(oldmm, next);
	} else
		switch_mm(oldmm, mm, next);

2318
	if (!prev->mm) {
L
Linus Torvalds 已提交
2319 2320 2321
		prev->active_mm = NULL;
		rq->prev_mm = oldmm;
	}
2322 2323 2324 2325 2326 2327 2328
	/*
	 * Since the runqueue lock will be released by the next
	 * task (which is an invalid locking op but in the case
	 * of the scheduler it's an obvious special-case), so we
	 * do an early lockdep release here:
	 */
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2329
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2330
#endif
L
Linus Torvalds 已提交
2331

2332
	context_tracking_task_switch(prev, next);
L
Linus Torvalds 已提交
2333 2334 2335
	/* Here we just switch the register state and the stack. */
	switch_to(prev, next, prev);

I
Ingo Molnar 已提交
2336 2337 2338 2339 2340 2341 2342
	barrier();
	/*
	 * this_rq must be evaluated again because prev may have moved
	 * CPUs since it called schedule(), thus the 'rq' on its stack
	 * frame will be invalid.
	 */
	finish_task_switch(this_rq(), prev);
L
Linus Torvalds 已提交
2343 2344 2345
}

/*
2346
 * nr_running and nr_context_switches:
L
Linus Torvalds 已提交
2347 2348
 *
 * externally visible scheduler statistics: current number of runnable
2349
 * threads, total number of context switches performed since bootup.
L
Linus Torvalds 已提交
2350 2351 2352 2353 2354 2355 2356 2357 2358
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
2359
}
L
Linus Torvalds 已提交
2360 2361

unsigned long long nr_context_switches(void)
2362
{
2363 2364
	int i;
	unsigned long long sum = 0;
2365

2366
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2367
		sum += cpu_rq(i)->nr_switches;
2368

L
Linus Torvalds 已提交
2369 2370
	return sum;
}
2371

L
Linus Torvalds 已提交
2372 2373 2374
unsigned long nr_iowait(void)
{
	unsigned long i, sum = 0;
2375

2376
	for_each_possible_cpu(i)
L
Linus Torvalds 已提交
2377
		sum += atomic_read(&cpu_rq(i)->nr_iowait);
2378

L
Linus Torvalds 已提交
2379 2380
	return sum;
}
2381

2382
unsigned long nr_iowait_cpu(int cpu)
2383
{
2384
	struct rq *this = cpu_rq(cpu);
2385 2386
	return atomic_read(&this->nr_iowait);
}
2387

I
Ingo Molnar 已提交
2388
#ifdef CONFIG_SMP
2389

2390
/*
P
Peter Zijlstra 已提交
2391 2392
 * sched_exec - execve() is a valuable balancing opportunity, because at
 * this point the task has the smallest effective memory and cache footprint.
2393
 */
P
Peter Zijlstra 已提交
2394
void sched_exec(void)
2395
{
P
Peter Zijlstra 已提交
2396
	struct task_struct *p = current;
L
Linus Torvalds 已提交
2397
	unsigned long flags;
2398
	int dest_cpu;
2399

2400
	raw_spin_lock_irqsave(&p->pi_lock, flags);
2401
	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
2402 2403
	if (dest_cpu == smp_processor_id())
		goto unlock;
P
Peter Zijlstra 已提交
2404

2405
	if (likely(cpu_active(dest_cpu))) {
2406
		struct migration_arg arg = { p, dest_cpu };
2407

2408 2409
		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
2410 2411
		return;
	}
2412
unlock:
2413
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
2414
}
I
Ingo Molnar 已提交
2415

L
Linus Torvalds 已提交
2416 2417 2418
#endif

DEFINE_PER_CPU(struct kernel_stat, kstat);
2419
DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
L
Linus Torvalds 已提交
2420 2421

EXPORT_PER_CPU_SYMBOL(kstat);
2422
EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
L
Linus Torvalds 已提交
2423 2424

/*
2425
 * Return any ns on the sched_clock that have not yet been accounted in
2426
 * @p in case that task is currently running.
2427 2428
 *
 * Called with task_rq_lock() held on @rq.
L
Linus Torvalds 已提交
2429
 */
2430 2431 2432 2433 2434 2435
static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
{
	u64 ns = 0;

	if (task_current(rq, p)) {
		update_rq_clock(rq);
2436
		ns = rq_clock_task(rq) - p->se.exec_start;
2437 2438 2439 2440 2441 2442 2443
		if ((s64)ns < 0)
			ns = 0;
	}

	return ns;
}

2444
unsigned long long task_delta_exec(struct task_struct *p)
L
Linus Torvalds 已提交
2445 2446
{
	unsigned long flags;
2447
	struct rq *rq;
2448
	u64 ns = 0;
2449

2450
	rq = task_rq_lock(p, &flags);
2451
	ns = do_task_delta_exec(p, rq);
2452
	task_rq_unlock(rq, p, &flags);
2453

2454 2455
	return ns;
}
2456

2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
/*
 * Return accounted runtime for the task.
 * In case the task is currently running, return the runtime plus current's
 * pending runtime that have not been accounted yet.
 */
unsigned long long task_sched_runtime(struct task_struct *p)
{
	unsigned long flags;
	struct rq *rq;
	u64 ns = 0;

2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
	/*
	 * 64-bit doesn't need locks to atomically read a 64bit value.
	 * So we have a optimization chance when the task's delta_exec is 0.
	 * Reading ->on_cpu is racy, but this is ok.
	 *
	 * If we race with it leaving cpu, we'll take a lock. So we're correct.
	 * If we race with it entering cpu, unaccounted time is 0. This is
	 * indistinguishable from the read occurring a few cycles earlier.
	 */
	if (!p->on_cpu)
		return p->se.sum_exec_runtime;
#endif

2482 2483
	rq = task_rq_lock(p, &flags);
	ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
2484
	task_rq_unlock(rq, p, &flags);
2485 2486 2487

	return ns;
}
2488

2489 2490 2491 2492 2493 2494 2495 2496
/*
 * This function gets called by the timer code, with HZ frequency.
 * We call it with interrupts disabled.
 */
void scheduler_tick(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
I
Ingo Molnar 已提交
2497
	struct task_struct *curr = rq->curr;
2498 2499

	sched_clock_tick();
I
Ingo Molnar 已提交
2500

2501
	raw_spin_lock(&rq->lock);
2502
	update_rq_clock(rq);
P
Peter Zijlstra 已提交
2503
	curr->sched_class->task_tick(rq, curr, 0);
2504
	update_cpu_load_active(rq);
2505
	raw_spin_unlock(&rq->lock);
2506

2507
	perf_event_task_tick();
2508

2509
#ifdef CONFIG_SMP
2510
	rq->idle_balance = idle_cpu(cpu);
2511
	trigger_load_balance(rq);
2512
#endif
2513
	rq_last_tick_reset(rq);
L
Linus Torvalds 已提交
2514 2515
}

2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
#ifdef CONFIG_NO_HZ_FULL
/**
 * scheduler_tick_max_deferment
 *
 * Keep at least one tick per second when a single
 * active task is running because the scheduler doesn't
 * yet completely support full dynticks environment.
 *
 * This makes sure that uptime, CFS vruntime, load
 * balancing, etc... continue to move forward, even
 * with a very low granularity.
2527 2528
 *
 * Return: Maximum deferment in nanoseconds.
2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539
 */
u64 scheduler_tick_max_deferment(void)
{
	struct rq *rq = this_rq();
	unsigned long next, now = ACCESS_ONCE(jiffies);

	next = rq->last_sched_tick + HZ;

	if (time_before_eq(next, now))
		return 0;

2540
	return jiffies_to_nsecs(next - now);
L
Linus Torvalds 已提交
2541
}
2542
#endif
L
Linus Torvalds 已提交
2543

2544
notrace unsigned long get_parent_ip(unsigned long addr)
2545 2546 2547 2548 2549 2550 2551 2552
{
	if (in_lock_functions(addr)) {
		addr = CALLER_ADDR2;
		if (in_lock_functions(addr))
			addr = CALLER_ADDR3;
	}
	return addr;
}
L
Linus Torvalds 已提交
2553

2554 2555 2556
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
				defined(CONFIG_PREEMPT_TRACER))

2557
void preempt_count_add(int val)
L
Linus Torvalds 已提交
2558
{
2559
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2560 2561 2562
	/*
	 * Underflow?
	 */
2563 2564
	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
		return;
2565
#endif
2566
	__preempt_count_add(val);
2567
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2568 2569 2570
	/*
	 * Spinlock count overflowing soon?
	 */
2571 2572
	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
				PREEMPT_MASK - 10);
2573
#endif
2574 2575 2576 2577 2578 2579 2580
	if (preempt_count() == val) {
		unsigned long ip = get_parent_ip(CALLER_ADDR1);
#ifdef CONFIG_DEBUG_PREEMPT
		current->preempt_disable_ip = ip;
#endif
		trace_preempt_off(CALLER_ADDR0, ip);
	}
L
Linus Torvalds 已提交
2581
}
2582
EXPORT_SYMBOL(preempt_count_add);
2583
NOKPROBE_SYMBOL(preempt_count_add);
L
Linus Torvalds 已提交
2584

2585
void preempt_count_sub(int val)
L
Linus Torvalds 已提交
2586
{
2587
#ifdef CONFIG_DEBUG_PREEMPT
L
Linus Torvalds 已提交
2588 2589 2590
	/*
	 * Underflow?
	 */
2591
	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
2592
		return;
L
Linus Torvalds 已提交
2593 2594 2595
	/*
	 * Is the spinlock portion underflowing?
	 */
2596 2597 2598
	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
			!(preempt_count() & PREEMPT_MASK)))
		return;
2599
#endif
2600

2601 2602
	if (preempt_count() == val)
		trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
2603
	__preempt_count_sub(val);
L
Linus Torvalds 已提交
2604
}
2605
EXPORT_SYMBOL(preempt_count_sub);
2606
NOKPROBE_SYMBOL(preempt_count_sub);
L
Linus Torvalds 已提交
2607 2608 2609 2610

#endif

/*
I
Ingo Molnar 已提交
2611
 * Print scheduling while atomic bug:
L
Linus Torvalds 已提交
2612
 */
I
Ingo Molnar 已提交
2613
static noinline void __schedule_bug(struct task_struct *prev)
L
Linus Torvalds 已提交
2614
{
2615 2616 2617
	if (oops_in_progress)
		return;

P
Peter Zijlstra 已提交
2618 2619
	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
		prev->comm, prev->pid, preempt_count());
2620

I
Ingo Molnar 已提交
2621
	debug_show_held_locks(prev);
2622
	print_modules();
I
Ingo Molnar 已提交
2623 2624
	if (irqs_disabled())
		print_irqtrace_events(prev);
2625 2626 2627 2628 2629 2630 2631
#ifdef CONFIG_DEBUG_PREEMPT
	if (in_atomic_preempt_off()) {
		pr_err("Preemption disabled at:");
		print_ip_sym(current->preempt_disable_ip);
		pr_cont("\n");
	}
#endif
2632
	dump_stack();
2633
	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
I
Ingo Molnar 已提交
2634
}
L
Linus Torvalds 已提交
2635

I
Ingo Molnar 已提交
2636 2637 2638 2639 2640
/*
 * Various schedule()-time debugging checks and statistics:
 */
static inline void schedule_debug(struct task_struct *prev)
{
L
Linus Torvalds 已提交
2641
	/*
I
Ingo Molnar 已提交
2642
	 * Test if we are atomic. Since do_exit() needs to call into
2643 2644
	 * schedule() atomically, we ignore that path. Otherwise whine
	 * if we are scheduling when we should not.
L
Linus Torvalds 已提交
2645
	 */
2646
	if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD))
I
Ingo Molnar 已提交
2647
		__schedule_bug(prev);
2648
	rcu_sleep_check();
I
Ingo Molnar 已提交
2649

L
Linus Torvalds 已提交
2650 2651
	profile_hit(SCHED_PROFILING, __builtin_return_address(0));

2652
	schedstat_inc(this_rq(), sched_count);
I
Ingo Molnar 已提交
2653 2654 2655 2656 2657 2658
}

/*
 * Pick up the highest-prio task:
 */
static inline struct task_struct *
2659
pick_next_task(struct rq *rq, struct task_struct *prev)
I
Ingo Molnar 已提交
2660
{
2661
	const struct sched_class *class = &fair_sched_class;
I
Ingo Molnar 已提交
2662
	struct task_struct *p;
L
Linus Torvalds 已提交
2663 2664

	/*
I
Ingo Molnar 已提交
2665 2666
	 * Optimization: we know that if all tasks are in
	 * the fair class we can call that function directly:
L
Linus Torvalds 已提交
2667
	 */
2668
	if (likely(prev->sched_class == class &&
2669
		   rq->nr_running == rq->cfs.h_nr_running)) {
2670
		p = fair_sched_class.pick_next_task(rq, prev);
2671 2672 2673 2674 2675 2676 2677 2678
		if (unlikely(p == RETRY_TASK))
			goto again;

		/* assumes fair_sched_class->next == idle_sched_class */
		if (unlikely(!p))
			p = idle_sched_class.pick_next_task(rq, prev);

		return p;
L
Linus Torvalds 已提交
2679 2680
	}

2681
again:
2682
	for_each_class(class) {
2683
		p = class->pick_next_task(rq, prev);
2684 2685 2686
		if (p) {
			if (unlikely(p == RETRY_TASK))
				goto again;
I
Ingo Molnar 已提交
2687
			return p;
2688
		}
I
Ingo Molnar 已提交
2689
	}
2690 2691

	BUG(); /* the idle class will always have a runnable task */
I
Ingo Molnar 已提交
2692
}
L
Linus Torvalds 已提交
2693

I
Ingo Molnar 已提交
2694
/*
2695
 * __schedule() is the main scheduler function.
2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729
 *
 * The main means of driving the scheduler and thus entering this function are:
 *
 *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
 *
 *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
 *      paths. For example, see arch/x86/entry_64.S.
 *
 *      To drive preemption between tasks, the scheduler sets the flag in timer
 *      interrupt handler scheduler_tick().
 *
 *   3. Wakeups don't really cause entry into schedule(). They add a
 *      task to the run-queue and that's it.
 *
 *      Now, if the new task added to the run-queue preempts the current
 *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
 *      called on the nearest possible occasion:
 *
 *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
 *
 *         - in syscall or exception context, at the next outmost
 *           preempt_enable(). (this might be as soon as the wake_up()'s
 *           spin_unlock()!)
 *
 *         - in IRQ context, return from interrupt-handler to
 *           preemptible context
 *
 *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
 *         then at the next:
 *
 *          - cond_resched() call
 *          - explicit schedule() call
 *          - return from syscall or exception to user-space
 *          - return from interrupt-handler to user-space
I
Ingo Molnar 已提交
2730
 */
2731
static void __sched __schedule(void)
I
Ingo Molnar 已提交
2732 2733
{
	struct task_struct *prev, *next;
2734
	unsigned long *switch_count;
I
Ingo Molnar 已提交
2735
	struct rq *rq;
2736
	int cpu;
I
Ingo Molnar 已提交
2737

2738 2739
need_resched:
	preempt_disable();
I
Ingo Molnar 已提交
2740 2741
	cpu = smp_processor_id();
	rq = cpu_rq(cpu);
2742
	rcu_note_context_switch(cpu);
I
Ingo Molnar 已提交
2743 2744 2745
	prev = rq->curr;

	schedule_debug(prev);
L
Linus Torvalds 已提交
2746

2747
	if (sched_feat(HRTICK))
M
Mike Galbraith 已提交
2748
		hrtick_clear(rq);
P
Peter Zijlstra 已提交
2749

2750 2751 2752 2753 2754 2755
	/*
	 * Make sure that signal_pending_state()->signal_pending() below
	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
	 * done by the caller to avoid the race with signal_wake_up().
	 */
	smp_mb__before_spinlock();
2756
	raw_spin_lock_irq(&rq->lock);
L
Linus Torvalds 已提交
2757

2758
	switch_count = &prev->nivcsw;
L
Linus Torvalds 已提交
2759
	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
T
Tejun Heo 已提交
2760
		if (unlikely(signal_pending_state(prev->state, prev))) {
L
Linus Torvalds 已提交
2761
			prev->state = TASK_RUNNING;
T
Tejun Heo 已提交
2762
		} else {
2763 2764 2765
			deactivate_task(rq, prev, DEQUEUE_SLEEP);
			prev->on_rq = 0;

T
Tejun Heo 已提交
2766
			/*
2767 2768 2769
			 * If a worker went to sleep, notify and ask workqueue
			 * whether it wants to wake up a task to maintain
			 * concurrency.
T
Tejun Heo 已提交
2770 2771 2772 2773 2774 2775 2776 2777 2778
			 */
			if (prev->flags & PF_WQ_WORKER) {
				struct task_struct *to_wakeup;

				to_wakeup = wq_worker_sleeping(prev, cpu);
				if (to_wakeup)
					try_to_wake_up_local(to_wakeup);
			}
		}
I
Ingo Molnar 已提交
2779
		switch_count = &prev->nvcsw;
L
Linus Torvalds 已提交
2780 2781
	}

2782 2783 2784 2785
	if (prev->on_rq || rq->skip_clock_update < 0)
		update_rq_clock(rq);

	next = pick_next_task(rq, prev);
2786
	clear_tsk_need_resched(prev);
2787
	clear_preempt_need_resched();
2788
	rq->skip_clock_update = 0;
L
Linus Torvalds 已提交
2789 2790 2791 2792 2793 2794

	if (likely(prev != next)) {
		rq->nr_switches++;
		rq->curr = next;
		++*switch_count;

I
Ingo Molnar 已提交
2795
		context_switch(rq, prev, next); /* unlocks the rq */
P
Peter Zijlstra 已提交
2796
		/*
2797 2798 2799 2800
		 * The context switch have flipped the stack from under us
		 * and restored the local variables which were saved when
		 * this task called schedule() in the past. prev == current
		 * is still correct, but it can be moved to another cpu/rq.
P
Peter Zijlstra 已提交
2801 2802 2803
		 */
		cpu = smp_processor_id();
		rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
2804
	} else
2805
		raw_spin_unlock_irq(&rq->lock);
L
Linus Torvalds 已提交
2806

2807
	post_schedule(rq);
L
Linus Torvalds 已提交
2808

2809
	sched_preempt_enable_no_resched();
2810
	if (need_resched())
L
Linus Torvalds 已提交
2811 2812
		goto need_resched;
}
2813

2814 2815
static inline void sched_submit_work(struct task_struct *tsk)
{
2816
	if (!tsk->state || tsk_is_pi_blocked(tsk))
2817 2818 2819 2820 2821 2822 2823 2824 2825
		return;
	/*
	 * If we are going to sleep and we have plugged IO queued,
	 * make sure to submit it to avoid deadlocks.
	 */
	if (blk_needs_flush_plug(tsk))
		blk_schedule_flush_plug(tsk);
}

2826
asmlinkage __visible void __sched schedule(void)
2827
{
2828 2829 2830
	struct task_struct *tsk = current;

	sched_submit_work(tsk);
2831 2832
	__schedule();
}
L
Linus Torvalds 已提交
2833 2834
EXPORT_SYMBOL(schedule);

2835
#ifdef CONFIG_CONTEXT_TRACKING
2836
asmlinkage __visible void __sched schedule_user(void)
2837 2838 2839 2840 2841 2842 2843
{
	/*
	 * If we come here after a random call to set_need_resched(),
	 * or we have been woken up remotely but the IPI has not yet arrived,
	 * we haven't yet exited the RCU idle mode. Do it here manually until
	 * we find a better solution.
	 */
2844
	user_exit();
2845
	schedule();
2846
	user_enter();
2847 2848 2849
}
#endif

2850 2851 2852 2853 2854 2855 2856
/**
 * schedule_preempt_disabled - called with preemption disabled
 *
 * Returns with preemption disabled. Note: preempt_count must be 1
 */
void __sched schedule_preempt_disabled(void)
{
2857
	sched_preempt_enable_no_resched();
2858 2859 2860 2861
	schedule();
	preempt_disable();
}

L
Linus Torvalds 已提交
2862 2863
#ifdef CONFIG_PREEMPT
/*
2864
 * this is the entry point to schedule() from in-kernel preemption
I
Ingo Molnar 已提交
2865
 * off of preempt_enable. Kernel preemptions off return from interrupt
L
Linus Torvalds 已提交
2866 2867
 * occur there and call schedule directly.
 */
2868
asmlinkage __visible void __sched notrace preempt_schedule(void)
L
Linus Torvalds 已提交
2869 2870 2871
{
	/*
	 * If there is a non-zero preempt_count or interrupts are disabled,
I
Ingo Molnar 已提交
2872
	 * we do not want to preempt the current task. Just return..
L
Linus Torvalds 已提交
2873
	 */
2874
	if (likely(!preemptible()))
L
Linus Torvalds 已提交
2875 2876
		return;

2877
	do {
2878
		__preempt_count_add(PREEMPT_ACTIVE);
2879
		__schedule();
2880
		__preempt_count_sub(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
2881

2882 2883 2884 2885 2886
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
2887
	} while (need_resched());
L
Linus Torvalds 已提交
2888
}
2889
NOKPROBE_SYMBOL(preempt_schedule);
L
Linus Torvalds 已提交
2890
EXPORT_SYMBOL(preempt_schedule);
2891
#endif /* CONFIG_PREEMPT */
L
Linus Torvalds 已提交
2892 2893

/*
2894
 * this is the entry point to schedule() from kernel preemption
L
Linus Torvalds 已提交
2895 2896 2897 2898
 * off of irq context.
 * Note, that this is called and return with irqs disabled. This will
 * protect us against recursive calling from irq.
 */
2899
asmlinkage __visible void __sched preempt_schedule_irq(void)
L
Linus Torvalds 已提交
2900
{
2901
	enum ctx_state prev_state;
2902

2903
	/* Catch callers which need to be fixed */
2904
	BUG_ON(preempt_count() || !irqs_disabled());
L
Linus Torvalds 已提交
2905

2906 2907
	prev_state = exception_enter();

2908
	do {
2909
		__preempt_count_add(PREEMPT_ACTIVE);
2910
		local_irq_enable();
2911
		__schedule();
2912
		local_irq_disable();
2913
		__preempt_count_sub(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
2914

2915 2916 2917 2918 2919
		/*
		 * Check again in case we missed a preemption opportunity
		 * between schedule and now.
		 */
		barrier();
2920
	} while (need_resched());
2921 2922

	exception_exit(prev_state);
L
Linus Torvalds 已提交
2923 2924
}

P
Peter Zijlstra 已提交
2925
int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
I
Ingo Molnar 已提交
2926
			  void *key)
L
Linus Torvalds 已提交
2927
{
P
Peter Zijlstra 已提交
2928
	return try_to_wake_up(curr->private, mode, wake_flags);
L
Linus Torvalds 已提交
2929 2930 2931
}
EXPORT_SYMBOL(default_wake_function);

2932 2933 2934 2935 2936 2937 2938 2939 2940 2941
#ifdef CONFIG_RT_MUTEXES

/*
 * rt_mutex_setprio - set the current priority of a task
 * @p: task
 * @prio: prio value (kernel-internal form)
 *
 * This function changes the 'effective' priority of a task. It does
 * not touch ->normal_prio like __setscheduler().
 *
2942 2943
 * Used by the rt_mutex code to implement priority inheritance
 * logic. Call site only calls if the priority of the task changed.
2944
 */
2945
void rt_mutex_setprio(struct task_struct *p, int prio)
2946
{
2947
	int oldprio, on_rq, running, enqueue_flag = 0;
2948
	struct rq *rq;
2949
	const struct sched_class *prev_class;
2950

2951
	BUG_ON(prio > MAX_PRIO);
2952

2953
	rq = __task_rq_lock(p);
2954

2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972
	/*
	 * Idle task boosting is a nono in general. There is one
	 * exception, when PREEMPT_RT and NOHZ is active:
	 *
	 * The idle task calls get_next_timer_interrupt() and holds
	 * the timer wheel base->lock on the CPU and another CPU wants
	 * to access the timer (probably to cancel it). We can safely
	 * ignore the boosting request, as the idle CPU runs this code
	 * with interrupts disabled and will complete the lock
	 * protected section without being interrupted. So there is no
	 * real need to boost.
	 */
	if (unlikely(p == rq->idle)) {
		WARN_ON(p != rq->curr);
		WARN_ON(p->pi_blocked_on);
		goto out_unlock;
	}

2973
	trace_sched_pi_setprio(p, prio);
2974
	p->pi_top_task = rt_mutex_get_top_task(p);
2975
	oldprio = p->prio;
2976
	prev_class = p->sched_class;
P
Peter Zijlstra 已提交
2977
	on_rq = p->on_rq;
2978
	running = task_current(rq, p);
2979
	if (on_rq)
2980
		dequeue_task(rq, p, 0);
2981 2982
	if (running)
		p->sched_class->put_prev_task(rq, p);
I
Ingo Molnar 已提交
2983

2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000
	/*
	 * Boosting condition are:
	 * 1. -rt task is running and holds mutex A
	 *      --> -dl task blocks on mutex A
	 *
	 * 2. -dl task is running and holds mutex A
	 *      --> -dl task blocks on mutex A and could preempt the
	 *          running task
	 */
	if (dl_prio(prio)) {
		if (!dl_prio(p->normal_prio) || (p->pi_top_task &&
			dl_entity_preempt(&p->pi_top_task->dl, &p->dl))) {
			p->dl.dl_boosted = 1;
			p->dl.dl_throttled = 0;
			enqueue_flag = ENQUEUE_REPLENISH;
		} else
			p->dl.dl_boosted = 0;
3001
		p->sched_class = &dl_sched_class;
3002 3003 3004 3005 3006
	} else if (rt_prio(prio)) {
		if (dl_prio(oldprio))
			p->dl.dl_boosted = 0;
		if (oldprio < prio)
			enqueue_flag = ENQUEUE_HEAD;
I
Ingo Molnar 已提交
3007
		p->sched_class = &rt_sched_class;
3008 3009 3010
	} else {
		if (dl_prio(oldprio))
			p->dl.dl_boosted = 0;
I
Ingo Molnar 已提交
3011
		p->sched_class = &fair_sched_class;
3012
	}
I
Ingo Molnar 已提交
3013

3014 3015
	p->prio = prio;

3016 3017
	if (running)
		p->sched_class->set_curr_task(rq);
P
Peter Zijlstra 已提交
3018
	if (on_rq)
3019
		enqueue_task(rq, p, enqueue_flag);
3020

P
Peter Zijlstra 已提交
3021
	check_class_changed(rq, p, prev_class, oldprio);
3022
out_unlock:
3023
	__task_rq_unlock(rq);
3024 3025
}
#endif
3026

3027
void set_user_nice(struct task_struct *p, long nice)
L
Linus Torvalds 已提交
3028
{
I
Ingo Molnar 已提交
3029
	int old_prio, delta, on_rq;
L
Linus Torvalds 已提交
3030
	unsigned long flags;
3031
	struct rq *rq;
L
Linus Torvalds 已提交
3032

3033
	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
L
Linus Torvalds 已提交
3034 3035 3036 3037 3038 3039 3040 3041 3042 3043
		return;
	/*
	 * We have to be careful, if called from sys_setpriority(),
	 * the task might be in the middle of scheduling on another CPU.
	 */
	rq = task_rq_lock(p, &flags);
	/*
	 * The RT priorities are set via sched_setscheduler(), but we still
	 * allow the 'normal' nice value to be set - but as expected
	 * it wont have any effect on scheduling until the task is
3044
	 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
L
Linus Torvalds 已提交
3045
	 */
3046
	if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
L
Linus Torvalds 已提交
3047 3048 3049
		p->static_prio = NICE_TO_PRIO(nice);
		goto out_unlock;
	}
P
Peter Zijlstra 已提交
3050
	on_rq = p->on_rq;
3051
	if (on_rq)
3052
		dequeue_task(rq, p, 0);
L
Linus Torvalds 已提交
3053 3054

	p->static_prio = NICE_TO_PRIO(nice);
3055
	set_load_weight(p);
3056 3057 3058
	old_prio = p->prio;
	p->prio = effective_prio(p);
	delta = p->prio - old_prio;
L
Linus Torvalds 已提交
3059

I
Ingo Molnar 已提交
3060
	if (on_rq) {
3061
		enqueue_task(rq, p, 0);
L
Linus Torvalds 已提交
3062
		/*
3063 3064
		 * If the task increased its priority or is running and
		 * lowered its priority, then reschedule its CPU:
L
Linus Torvalds 已提交
3065
		 */
3066
		if (delta < 0 || (delta > 0 && task_running(rq, p)))
L
Linus Torvalds 已提交
3067 3068 3069
			resched_task(rq->curr);
	}
out_unlock:
3070
	task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
3071 3072 3073
}
EXPORT_SYMBOL(set_user_nice);

M
Matt Mackall 已提交
3074 3075 3076 3077 3078
/*
 * can_nice - check if a task can reduce its nice value
 * @p: task
 * @nice: nice value
 */
3079
int can_nice(const struct task_struct *p, const int nice)
M
Matt Mackall 已提交
3080
{
3081
	/* convert nice value [19,-20] to rlimit style value [1,40] */
3082
	int nice_rlim = nice_to_rlimit(nice);
3083

3084
	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
M
Matt Mackall 已提交
3085 3086 3087
		capable(CAP_SYS_NICE));
}

L
Linus Torvalds 已提交
3088 3089 3090 3091 3092 3093 3094 3095 3096
#ifdef __ARCH_WANT_SYS_NICE

/*
 * sys_nice - change the priority of the current process.
 * @increment: priority increment
 *
 * sys_setpriority is a more generic, but much slower function that
 * does similar things.
 */
3097
SYSCALL_DEFINE1(nice, int, increment)
L
Linus Torvalds 已提交
3098
{
3099
	long nice, retval;
L
Linus Torvalds 已提交
3100 3101 3102 3103 3104 3105

	/*
	 * Setpriority might change our priority at the same moment.
	 * We don't have to worry. Conceptually one call occurs first
	 * and we have a single winner.
	 */
3106
	increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
3107
	nice = task_nice(current) + increment;
L
Linus Torvalds 已提交
3108

3109
	nice = clamp_val(nice, MIN_NICE, MAX_NICE);
M
Matt Mackall 已提交
3110 3111 3112
	if (increment < 0 && !can_nice(current, nice))
		return -EPERM;

L
Linus Torvalds 已提交
3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126
	retval = security_task_setnice(current, nice);
	if (retval)
		return retval;

	set_user_nice(current, nice);
	return 0;
}

#endif

/**
 * task_prio - return the priority value of a given task.
 * @p: the task in question.
 *
3127
 * Return: The priority value as seen by users in /proc.
L
Linus Torvalds 已提交
3128 3129 3130
 * RT tasks are offset by -200. Normal tasks are centered
 * around 0, value goes from -16 to +15.
 */
3131
int task_prio(const struct task_struct *p)
L
Linus Torvalds 已提交
3132 3133 3134 3135 3136 3137 3138
{
	return p->prio - MAX_RT_PRIO;
}

/**
 * idle_cpu - is a given cpu idle currently?
 * @cpu: the processor in question.
3139 3140
 *
 * Return: 1 if the CPU is currently idle. 0 otherwise.
L
Linus Torvalds 已提交
3141 3142 3143
 */
int idle_cpu(int cpu)
{
T
Thomas Gleixner 已提交
3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157
	struct rq *rq = cpu_rq(cpu);

	if (rq->curr != rq->idle)
		return 0;

	if (rq->nr_running)
		return 0;

#ifdef CONFIG_SMP
	if (!llist_empty(&rq->wake_list))
		return 0;
#endif

	return 1;
L
Linus Torvalds 已提交
3158 3159 3160 3161 3162
}

/**
 * idle_task - return the idle task for a given cpu.
 * @cpu: the processor in question.
3163 3164
 *
 * Return: The idle task for the cpu @cpu.
L
Linus Torvalds 已提交
3165
 */
3166
struct task_struct *idle_task(int cpu)
L
Linus Torvalds 已提交
3167 3168 3169 3170 3171 3172 3173
{
	return cpu_rq(cpu)->idle;
}

/**
 * find_process_by_pid - find a process with a matching PID value.
 * @pid: the pid in question.
3174 3175
 *
 * The task of @pid, if found. %NULL otherwise.
L
Linus Torvalds 已提交
3176
 */
A
Alexey Dobriyan 已提交
3177
static struct task_struct *find_process_by_pid(pid_t pid)
L
Linus Torvalds 已提交
3178
{
3179
	return pid ? find_task_by_vpid(pid) : current;
L
Linus Torvalds 已提交
3180 3181
}

3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197
/*
 * This function initializes the sched_dl_entity of a newly becoming
 * SCHED_DEADLINE task.
 *
 * Only the static values are considered here, the actual runtime and the
 * absolute deadline will be properly calculated when the task is enqueued
 * for the first time with its new policy.
 */
static void
__setparam_dl(struct task_struct *p, const struct sched_attr *attr)
{
	struct sched_dl_entity *dl_se = &p->dl;

	init_dl_task_timer(dl_se);
	dl_se->dl_runtime = attr->sched_runtime;
	dl_se->dl_deadline = attr->sched_deadline;
3198
	dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
3199
	dl_se->flags = attr->sched_flags;
3200
	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
3201 3202
	dl_se->dl_throttled = 0;
	dl_se->dl_new = 1;
3203
	dl_se->dl_yielded = 0;
3204 3205
}

3206 3207
static void __setscheduler_params(struct task_struct *p,
		const struct sched_attr *attr)
L
Linus Torvalds 已提交
3208
{
3209 3210
	int policy = attr->sched_policy;

3211 3212 3213
	if (policy == -1) /* setparam */
		policy = p->policy;

L
Linus Torvalds 已提交
3214
	p->policy = policy;
3215

3216 3217
	if (dl_policy(policy))
		__setparam_dl(p, attr);
3218
	else if (fair_policy(policy))
3219 3220
		p->static_prio = NICE_TO_PRIO(attr->sched_nice);

3221 3222 3223 3224 3225 3226
	/*
	 * __sched_setscheduler() ensures attr->sched_priority == 0 when
	 * !rt_policy. Always setting this ensures that things like
	 * getparam()/getattr() don't report silly values for !rt tasks.
	 */
	p->rt_priority = attr->sched_priority;
3227
	p->normal_prio = normal_prio(p);
3228 3229
	set_load_weight(p);
}
3230

3231 3232 3233 3234 3235
/* Actually do priority change: must hold pi & rq lock. */
static void __setscheduler(struct rq *rq, struct task_struct *p,
			   const struct sched_attr *attr)
{
	__setscheduler_params(p, attr);
3236

3237 3238 3239 3240 3241 3242
	/*
	 * If we get here, there was no pi waiters boosting the
	 * task. It is safe to use the normal prio.
	 */
	p->prio = normal_prio(p);

3243 3244 3245
	if (dl_prio(p->prio))
		p->sched_class = &dl_sched_class;
	else if (rt_prio(p->prio))
3246 3247 3248
		p->sched_class = &rt_sched_class;
	else
		p->sched_class = &fair_sched_class;
L
Linus Torvalds 已提交
3249
}
3250 3251 3252 3253 3254 3255 3256 3257 3258

static void
__getparam_dl(struct task_struct *p, struct sched_attr *attr)
{
	struct sched_dl_entity *dl_se = &p->dl;

	attr->sched_priority = p->rt_priority;
	attr->sched_runtime = dl_se->dl_runtime;
	attr->sched_deadline = dl_se->dl_deadline;
3259
	attr->sched_period = dl_se->dl_period;
3260 3261 3262 3263 3264 3265
	attr->sched_flags = dl_se->flags;
}

/*
 * This function validates the new parameters of a -deadline task.
 * We ask for the deadline not being zero, and greater or equal
3266
 * than the runtime, as well as the period of being zero or
3267
 * greater than deadline. Furthermore, we have to be sure that
3268 3269 3270 3271
 * user parameters are above the internal resolution of 1us (we
 * check sched_runtime only since it is always the smaller one) and
 * below 2^63 ns (we have to check both sched_deadline and
 * sched_period, as the latter can be zero).
3272 3273 3274 3275
 */
static bool
__checkparam_dl(const struct sched_attr *attr)
{
3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301
	/* deadline != 0 */
	if (attr->sched_deadline == 0)
		return false;

	/*
	 * Since we truncate DL_SCALE bits, make sure we're at least
	 * that big.
	 */
	if (attr->sched_runtime < (1ULL << DL_SCALE))
		return false;

	/*
	 * Since we use the MSB for wrap-around and sign issues, make
	 * sure it's not set (mind that period can be equal to zero).
	 */
	if (attr->sched_deadline & (1ULL << 63) ||
	    attr->sched_period & (1ULL << 63))
		return false;

	/* runtime <= deadline <= period (if period != 0) */
	if ((attr->sched_period != 0 &&
	     attr->sched_period < attr->sched_deadline) ||
	    attr->sched_deadline < attr->sched_runtime)
		return false;

	return true;
3302 3303
}

3304 3305 3306 3307 3308 3309 3310 3311 3312 3313
/*
 * check the target process has a UID that matches the current process's
 */
static bool check_same_owner(struct task_struct *p)
{
	const struct cred *cred = current_cred(), *pcred;
	bool match;

	rcu_read_lock();
	pcred = __task_cred(p);
3314 3315
	match = (uid_eq(cred->euid, pcred->euid) ||
		 uid_eq(cred->euid, pcred->uid));
3316 3317 3318 3319
	rcu_read_unlock();
	return match;
}

3320 3321 3322
static int __sched_setscheduler(struct task_struct *p,
				const struct sched_attr *attr,
				bool user)
L
Linus Torvalds 已提交
3323
{
3324 3325
	int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
		      MAX_RT_PRIO - 1 - attr->sched_priority;
3326
	int retval, oldprio, oldpolicy = -1, on_rq, running;
3327
	int policy = attr->sched_policy;
L
Linus Torvalds 已提交
3328
	unsigned long flags;
3329
	const struct sched_class *prev_class;
3330
	struct rq *rq;
3331
	int reset_on_fork;
L
Linus Torvalds 已提交
3332

3333 3334
	/* may grab non-irq protected spin_locks */
	BUG_ON(in_interrupt());
L
Linus Torvalds 已提交
3335 3336
recheck:
	/* double check policy once rq lock held */
3337 3338
	if (policy < 0) {
		reset_on_fork = p->sched_reset_on_fork;
L
Linus Torvalds 已提交
3339
		policy = oldpolicy = p->policy;
3340
	} else {
3341
		reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
3342

3343 3344
		if (policy != SCHED_DEADLINE &&
				policy != SCHED_FIFO && policy != SCHED_RR &&
3345 3346 3347 3348 3349
				policy != SCHED_NORMAL && policy != SCHED_BATCH &&
				policy != SCHED_IDLE)
			return -EINVAL;
	}

3350 3351 3352
	if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
		return -EINVAL;

L
Linus Torvalds 已提交
3353 3354
	/*
	 * Valid priorities for SCHED_FIFO and SCHED_RR are
I
Ingo Molnar 已提交
3355 3356
	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
	 * SCHED_BATCH and SCHED_IDLE is 0.
L
Linus Torvalds 已提交
3357
	 */
3358
	if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
3359
	    (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
L
Linus Torvalds 已提交
3360
		return -EINVAL;
3361 3362
	if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
	    (rt_policy(policy) != (attr->sched_priority != 0)))
L
Linus Torvalds 已提交
3363 3364
		return -EINVAL;

3365 3366 3367
	/*
	 * Allow unprivileged RT tasks to decrease priority:
	 */
3368
	if (user && !capable(CAP_SYS_NICE)) {
3369
		if (fair_policy(policy)) {
3370
			if (attr->sched_nice < task_nice(p) &&
3371
			    !can_nice(p, attr->sched_nice))
3372 3373 3374
				return -EPERM;
		}

3375
		if (rt_policy(policy)) {
3376 3377
			unsigned long rlim_rtprio =
					task_rlimit(p, RLIMIT_RTPRIO);
3378 3379 3380 3381 3382 3383

			/* can't set/change the rt policy */
			if (policy != p->policy && !rlim_rtprio)
				return -EPERM;

			/* can't increase priority */
3384 3385
			if (attr->sched_priority > p->rt_priority &&
			    attr->sched_priority > rlim_rtprio)
3386 3387
				return -EPERM;
		}
3388

3389 3390 3391 3392 3393 3394 3395 3396 3397
		 /*
		  * Can't set/change SCHED_DEADLINE policy at all for now
		  * (safest behavior); in the future we would like to allow
		  * unprivileged DL tasks to increase their relative deadline
		  * or reduce their runtime (both ways reducing utilization)
		  */
		if (dl_policy(policy))
			return -EPERM;

I
Ingo Molnar 已提交
3398
		/*
3399 3400
		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
I
Ingo Molnar 已提交
3401
		 */
3402
		if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
3403
			if (!can_nice(p, task_nice(p)))
3404 3405
				return -EPERM;
		}
3406

3407
		/* can't change other user's priorities */
3408
		if (!check_same_owner(p))
3409
			return -EPERM;
3410 3411 3412 3413

		/* Normal users shall not reset the sched_reset_on_fork flag */
		if (p->sched_reset_on_fork && !reset_on_fork)
			return -EPERM;
3414
	}
L
Linus Torvalds 已提交
3415

3416
	if (user) {
3417
		retval = security_task_setscheduler(p);
3418 3419 3420 3421
		if (retval)
			return retval;
	}

3422 3423 3424
	/*
	 * make sure no PI-waiters arrive (or leave) while we are
	 * changing the priority of the task:
3425
	 *
L
Lucas De Marchi 已提交
3426
	 * To be able to change p->policy safely, the appropriate
L
Linus Torvalds 已提交
3427 3428
	 * runqueue lock must be held.
	 */
3429
	rq = task_rq_lock(p, &flags);
3430

3431 3432 3433 3434
	/*
	 * Changing the policy of the stop threads its a very bad idea
	 */
	if (p == rq->stop) {
3435
		task_rq_unlock(rq, p, &flags);
3436 3437 3438
		return -EINVAL;
	}

3439
	/*
3440 3441
	 * If not changing anything there's no need to proceed further,
	 * but store a possible modification of reset_on_fork.
3442
	 */
3443
	if (unlikely(policy == p->policy)) {
3444
		if (fair_policy(policy) && attr->sched_nice != task_nice(p))
3445 3446 3447
			goto change;
		if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
			goto change;
3448 3449
		if (dl_policy(policy))
			goto change;
3450

3451
		p->sched_reset_on_fork = reset_on_fork;
3452
		task_rq_unlock(rq, p, &flags);
3453 3454
		return 0;
	}
3455
change:
3456

3457
	if (user) {
3458
#ifdef CONFIG_RT_GROUP_SCHED
3459 3460 3461 3462 3463
		/*
		 * Do not allow realtime tasks into groups that have no runtime
		 * assigned.
		 */
		if (rt_bandwidth_enabled() && rt_policy(policy) &&
3464 3465
				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
				!task_group_is_autogroup(task_group(p))) {
3466
			task_rq_unlock(rq, p, &flags);
3467 3468 3469
			return -EPERM;
		}
#endif
3470 3471 3472 3473 3474 3475 3476 3477 3478
#ifdef CONFIG_SMP
		if (dl_bandwidth_enabled() && dl_policy(policy)) {
			cpumask_t *span = rq->rd->span;

			/*
			 * Don't allow tasks with an affinity mask smaller than
			 * the entire root_domain to become SCHED_DEADLINE. We
			 * will also fail if there's no bandwidth available.
			 */
3479 3480
			if (!cpumask_subset(span, &p->cpus_allowed) ||
			    rq->rd->dl_bw.bw == 0) {
3481 3482 3483 3484 3485 3486
				task_rq_unlock(rq, p, &flags);
				return -EPERM;
			}
		}
#endif
	}
3487

L
Linus Torvalds 已提交
3488 3489 3490
	/* recheck policy now with rq lock held */
	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
		policy = oldpolicy = -1;
3491
		task_rq_unlock(rq, p, &flags);
L
Linus Torvalds 已提交
3492 3493
		goto recheck;
	}
3494 3495 3496 3497 3498 3499

	/*
	 * If setscheduling to SCHED_DEADLINE (or changing the parameters
	 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
	 * is available.
	 */
3500
	if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
3501 3502 3503 3504
		task_rq_unlock(rq, p, &flags);
		return -EBUSY;
	}

3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522
	p->sched_reset_on_fork = reset_on_fork;
	oldprio = p->prio;

	/*
	 * Special case for priority boosted tasks.
	 *
	 * If the new priority is lower or equal (user space view)
	 * than the current (boosted) priority, we just store the new
	 * normal parameters and do not touch the scheduler class and
	 * the runqueue. This will be done when the task deboost
	 * itself.
	 */
	if (rt_mutex_check_prio(p, newprio)) {
		__setscheduler_params(p, attr);
		task_rq_unlock(rq, p, &flags);
		return 0;
	}

P
Peter Zijlstra 已提交
3523
	on_rq = p->on_rq;
3524
	running = task_current(rq, p);
3525
	if (on_rq)
3526
		dequeue_task(rq, p, 0);
3527 3528
	if (running)
		p->sched_class->put_prev_task(rq, p);
3529

3530
	prev_class = p->sched_class;
3531
	__setscheduler(rq, p, attr);
3532

3533 3534
	if (running)
		p->sched_class->set_curr_task(rq);
3535 3536 3537 3538 3539 3540 3541
	if (on_rq) {
		/*
		 * We enqueue to tail when the priority of a task is
		 * increased (user space view).
		 */
		enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0);
	}
3542

P
Peter Zijlstra 已提交
3543
	check_class_changed(rq, p, prev_class, oldprio);
3544
	task_rq_unlock(rq, p, &flags);
3545

3546 3547
	rt_mutex_adjust_pi(p);

L
Linus Torvalds 已提交
3548 3549
	return 0;
}
3550

3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570
static int _sched_setscheduler(struct task_struct *p, int policy,
			       const struct sched_param *param, bool check)
{
	struct sched_attr attr = {
		.sched_policy   = policy,
		.sched_priority = param->sched_priority,
		.sched_nice	= PRIO_TO_NICE(p->static_prio),
	};

	/*
	 * Fixup the legacy SCHED_RESET_ON_FORK hack
	 */
	if (policy & SCHED_RESET_ON_FORK) {
		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
		policy &= ~SCHED_RESET_ON_FORK;
		attr.sched_policy = policy;
	}

	return __sched_setscheduler(p, &attr, check);
}
3571 3572 3573 3574 3575 3576
/**
 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
3577 3578
 * Return: 0 on success. An error code otherwise.
 *
3579 3580 3581
 * NOTE that the task may be already dead.
 */
int sched_setscheduler(struct task_struct *p, int policy,
3582
		       const struct sched_param *param)
3583
{
3584
	return _sched_setscheduler(p, policy, param, true);
3585
}
L
Linus Torvalds 已提交
3586 3587
EXPORT_SYMBOL_GPL(sched_setscheduler);

3588 3589 3590 3591 3592 3593
int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
{
	return __sched_setscheduler(p, attr, true);
}
EXPORT_SYMBOL_GPL(sched_setattr);

3594 3595 3596 3597 3598 3599 3600 3601 3602 3603
/**
 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
 * @p: the task in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
 *
 * Just like sched_setscheduler, only don't bother checking if the
 * current context has permission.  For example, this is needed in
 * stop_machine(): we create temporary high priority worker threads,
 * but our caller might not have that capability.
3604 3605
 *
 * Return: 0 on success. An error code otherwise.
3606 3607
 */
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
3608
			       const struct sched_param *param)
3609
{
3610
	return _sched_setscheduler(p, policy, param, false);
3611 3612
}

I
Ingo Molnar 已提交
3613 3614
static int
do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
L
Linus Torvalds 已提交
3615 3616 3617
{
	struct sched_param lparam;
	struct task_struct *p;
3618
	int retval;
L
Linus Torvalds 已提交
3619 3620 3621 3622 3623

	if (!param || pid < 0)
		return -EINVAL;
	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
		return -EFAULT;
3624 3625 3626

	rcu_read_lock();
	retval = -ESRCH;
L
Linus Torvalds 已提交
3627
	p = find_process_by_pid(pid);
3628 3629 3630
	if (p != NULL)
		retval = sched_setscheduler(p, policy, &lparam);
	rcu_read_unlock();
3631

L
Linus Torvalds 已提交
3632 3633 3634
	return retval;
}

3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696
/*
 * Mimics kernel/events/core.c perf_copy_attr().
 */
static int sched_copy_attr(struct sched_attr __user *uattr,
			   struct sched_attr *attr)
{
	u32 size;
	int ret;

	if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = SCHED_ATTR_SIZE_VER0;

	if (size < SCHED_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
	 */
	if (size > sizeof(*attr)) {
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;

		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;

		for (; addr < end; addr++) {
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
		size = sizeof(*attr);
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

	/*
	 * XXX: do we want to be lenient like existing syscalls; or do we want
	 * to be strict and return an error on out-of-bounds values?
	 */
3697
	attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
3698

3699
	return 0;
3700 3701 3702

err_size:
	put_user(sizeof(*attr), &uattr->size);
3703
	return -E2BIG;
3704 3705
}

L
Linus Torvalds 已提交
3706 3707 3708 3709 3710
/**
 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
 * @pid: the pid in question.
 * @policy: new policy.
 * @param: structure containing the new RT priority.
3711 3712
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
3713
 */
3714 3715
SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
		struct sched_param __user *, param)
L
Linus Torvalds 已提交
3716
{
3717 3718 3719 3720
	/* negative values for policy are not valid */
	if (policy < 0)
		return -EINVAL;

L
Linus Torvalds 已提交
3721 3722 3723 3724 3725 3726 3727
	return do_sched_setscheduler(pid, policy, param);
}

/**
 * sys_sched_setparam - set/change the RT priority of a thread
 * @pid: the pid in question.
 * @param: structure containing the new RT priority.
3728 3729
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
3730
 */
3731
SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
3732 3733 3734 3735
{
	return do_sched_setscheduler(pid, -1, param);
}

3736 3737 3738
/**
 * sys_sched_setattr - same as above, but with extended sched_attr
 * @pid: the pid in question.
J
Juri Lelli 已提交
3739
 * @uattr: structure containing the extended parameters.
3740
 * @flags: for future extension.
3741
 */
3742 3743
SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
			       unsigned int, flags)
3744 3745 3746 3747 3748
{
	struct sched_attr attr;
	struct task_struct *p;
	int retval;

3749
	if (!uattr || pid < 0 || flags)
3750 3751
		return -EINVAL;

3752 3753 3754
	retval = sched_copy_attr(uattr, &attr);
	if (retval)
		return retval;
3755

3756
	if ((int)attr.sched_policy < 0)
3757
		return -EINVAL;
3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768

	rcu_read_lock();
	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (p != NULL)
		retval = sched_setattr(p, &attr);
	rcu_read_unlock();

	return retval;
}

L
Linus Torvalds 已提交
3769 3770 3771
/**
 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
 * @pid: the pid in question.
3772 3773 3774
 *
 * Return: On success, the policy of the thread. Otherwise, a negative error
 * code.
L
Linus Torvalds 已提交
3775
 */
3776
SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
L
Linus Torvalds 已提交
3777
{
3778
	struct task_struct *p;
3779
	int retval;
L
Linus Torvalds 已提交
3780 3781

	if (pid < 0)
3782
		return -EINVAL;
L
Linus Torvalds 已提交
3783 3784

	retval = -ESRCH;
3785
	rcu_read_lock();
L
Linus Torvalds 已提交
3786 3787 3788 3789
	p = find_process_by_pid(pid);
	if (p) {
		retval = security_task_getscheduler(p);
		if (!retval)
3790 3791
			retval = p->policy
				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
L
Linus Torvalds 已提交
3792
	}
3793
	rcu_read_unlock();
L
Linus Torvalds 已提交
3794 3795 3796 3797
	return retval;
}

/**
3798
 * sys_sched_getparam - get the RT priority of a thread
L
Linus Torvalds 已提交
3799 3800
 * @pid: the pid in question.
 * @param: structure containing the RT priority.
3801 3802 3803
 *
 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
 * code.
L
Linus Torvalds 已提交
3804
 */
3805
SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
L
Linus Torvalds 已提交
3806
{
3807
	struct sched_param lp = { .sched_priority = 0 };
3808
	struct task_struct *p;
3809
	int retval;
L
Linus Torvalds 已提交
3810 3811

	if (!param || pid < 0)
3812
		return -EINVAL;
L
Linus Torvalds 已提交
3813

3814
	rcu_read_lock();
L
Linus Torvalds 已提交
3815 3816 3817 3818 3819 3820 3821 3822 3823
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

3824 3825
	if (task_has_rt_policy(p))
		lp.sched_priority = p->rt_priority;
3826
	rcu_read_unlock();
L
Linus Torvalds 已提交
3827 3828 3829 3830 3831 3832 3833 3834 3835

	/*
	 * This one might sleep, we cannot do it with a spinlock held ...
	 */
	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;

	return retval;

out_unlock:
3836
	rcu_read_unlock();
L
Linus Torvalds 已提交
3837 3838 3839
	return retval;
}

3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862
static int sched_read_attr(struct sched_attr __user *uattr,
			   struct sched_attr *attr,
			   unsigned int usize)
{
	int ret;

	if (!access_ok(VERIFY_WRITE, uattr, usize))
		return -EFAULT;

	/*
	 * If we're handed a smaller struct than we know of,
	 * ensure all the unknown bits are 0 - i.e. old
	 * user-space does not get uncomplete information.
	 */
	if (usize < sizeof(*attr)) {
		unsigned char *addr;
		unsigned char *end;

		addr = (void *)attr + usize;
		end  = (void *)attr + sizeof(*attr);

		for (; addr < end; addr++) {
			if (*addr)
3863
				return -EFBIG;
3864 3865 3866 3867 3868
		}

		attr->size = usize;
	}

3869
	ret = copy_to_user(uattr, attr, attr->size);
3870 3871 3872
	if (ret)
		return -EFAULT;

3873
	return 0;
3874 3875 3876
}

/**
3877
 * sys_sched_getattr - similar to sched_getparam, but with sched_attr
3878
 * @pid: the pid in question.
J
Juri Lelli 已提交
3879
 * @uattr: structure containing the extended parameters.
3880
 * @size: sizeof(attr) for fwd/bwd comp.
3881
 * @flags: for future extension.
3882
 */
3883 3884
SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
		unsigned int, size, unsigned int, flags)
3885 3886 3887 3888 3889 3890 3891 3892
{
	struct sched_attr attr = {
		.size = sizeof(struct sched_attr),
	};
	struct task_struct *p;
	int retval;

	if (!uattr || pid < 0 || size > PAGE_SIZE ||
3893
	    size < SCHED_ATTR_SIZE_VER0 || flags)
3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906
		return -EINVAL;

	rcu_read_lock();
	p = find_process_by_pid(pid);
	retval = -ESRCH;
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

	attr.sched_policy = p->policy;
3907 3908
	if (p->sched_reset_on_fork)
		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
3909 3910 3911
	if (task_has_dl_policy(p))
		__getparam_dl(p, &attr);
	else if (task_has_rt_policy(p))
3912 3913
		attr.sched_priority = p->rt_priority;
	else
3914
		attr.sched_nice = task_nice(p);
3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925

	rcu_read_unlock();

	retval = sched_read_attr(uattr, &attr, size);
	return retval;

out_unlock:
	rcu_read_unlock();
	return retval;
}

3926
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
L
Linus Torvalds 已提交
3927
{
3928
	cpumask_var_t cpus_allowed, new_mask;
3929 3930
	struct task_struct *p;
	int retval;
L
Linus Torvalds 已提交
3931

3932
	rcu_read_lock();
L
Linus Torvalds 已提交
3933 3934 3935

	p = find_process_by_pid(pid);
	if (!p) {
3936
		rcu_read_unlock();
L
Linus Torvalds 已提交
3937 3938 3939
		return -ESRCH;
	}

3940
	/* Prevent p going away */
L
Linus Torvalds 已提交
3941
	get_task_struct(p);
3942
	rcu_read_unlock();
L
Linus Torvalds 已提交
3943

3944 3945 3946 3947
	if (p->flags & PF_NO_SETAFFINITY) {
		retval = -EINVAL;
		goto out_put_task;
	}
3948 3949 3950 3951 3952 3953 3954 3955
	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_put_task;
	}
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
		retval = -ENOMEM;
		goto out_free_cpus_allowed;
	}
L
Linus Torvalds 已提交
3956
	retval = -EPERM;
E
Eric W. Biederman 已提交
3957 3958 3959 3960 3961 3962 3963 3964
	if (!check_same_owner(p)) {
		rcu_read_lock();
		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
			rcu_read_unlock();
			goto out_unlock;
		}
		rcu_read_unlock();
	}
L
Linus Torvalds 已提交
3965

3966
	retval = security_task_setscheduler(p);
3967 3968 3969
	if (retval)
		goto out_unlock;

3970 3971 3972 3973

	cpuset_cpus_allowed(p, cpus_allowed);
	cpumask_and(new_mask, in_mask, cpus_allowed);

3974 3975 3976 3977 3978 3979 3980 3981 3982 3983
	/*
	 * Since bandwidth control happens on root_domain basis,
	 * if admission test is enabled, we only admit -deadline
	 * tasks allowed to run on all the CPUs in the task's
	 * root_domain.
	 */
#ifdef CONFIG_SMP
	if (task_has_dl_policy(p)) {
		const struct cpumask *span = task_rq(p)->rd->span;

3984
		if (dl_bandwidth_enabled() && !cpumask_subset(span, new_mask)) {
3985 3986 3987 3988 3989
			retval = -EBUSY;
			goto out_unlock;
		}
	}
#endif
P
Peter Zijlstra 已提交
3990
again:
3991
	retval = set_cpus_allowed_ptr(p, new_mask);
L
Linus Torvalds 已提交
3992

P
Paul Menage 已提交
3993
	if (!retval) {
3994 3995
		cpuset_cpus_allowed(p, cpus_allowed);
		if (!cpumask_subset(new_mask, cpus_allowed)) {
P
Paul Menage 已提交
3996 3997 3998 3999 4000
			/*
			 * We must have raced with a concurrent cpuset
			 * update. Just reset the cpus_allowed to the
			 * cpuset's cpus_allowed
			 */
4001
			cpumask_copy(new_mask, cpus_allowed);
P
Paul Menage 已提交
4002 4003 4004
			goto again;
		}
	}
L
Linus Torvalds 已提交
4005
out_unlock:
4006 4007 4008 4009
	free_cpumask_var(new_mask);
out_free_cpus_allowed:
	free_cpumask_var(cpus_allowed);
out_put_task:
L
Linus Torvalds 已提交
4010 4011 4012 4013 4014
	put_task_struct(p);
	return retval;
}

static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
4015
			     struct cpumask *new_mask)
L
Linus Torvalds 已提交
4016
{
4017 4018 4019 4020 4021
	if (len < cpumask_size())
		cpumask_clear(new_mask);
	else if (len > cpumask_size())
		len = cpumask_size();

L
Linus Torvalds 已提交
4022 4023 4024 4025 4026 4027 4028 4029
	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
}

/**
 * sys_sched_setaffinity - set the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to the new cpu mask
4030 4031
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
4032
 */
4033 4034
SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
4035
{
4036
	cpumask_var_t new_mask;
L
Linus Torvalds 已提交
4037 4038
	int retval;

4039 4040
	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
4041

4042 4043 4044 4045 4046
	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
	if (retval == 0)
		retval = sched_setaffinity(pid, new_mask);
	free_cpumask_var(new_mask);
	return retval;
L
Linus Torvalds 已提交
4047 4048
}

4049
long sched_getaffinity(pid_t pid, struct cpumask *mask)
L
Linus Torvalds 已提交
4050
{
4051
	struct task_struct *p;
4052
	unsigned long flags;
L
Linus Torvalds 已提交
4053 4054
	int retval;

4055
	rcu_read_lock();
L
Linus Torvalds 已提交
4056 4057 4058 4059 4060 4061

	retval = -ESRCH;
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

4062 4063 4064 4065
	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4066
	raw_spin_lock_irqsave(&p->pi_lock, flags);
4067
	cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
4068
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
L
Linus Torvalds 已提交
4069 4070

out_unlock:
4071
	rcu_read_unlock();
L
Linus Torvalds 已提交
4072

4073
	return retval;
L
Linus Torvalds 已提交
4074 4075 4076 4077 4078 4079 4080
}

/**
 * sys_sched_getaffinity - get the cpu affinity of a process
 * @pid: pid of the process
 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
 * @user_mask_ptr: user-space pointer to hold the current cpu mask
4081 4082
 *
 * Return: 0 on success. An error code otherwise.
L
Linus Torvalds 已提交
4083
 */
4084 4085
SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
		unsigned long __user *, user_mask_ptr)
L
Linus Torvalds 已提交
4086 4087
{
	int ret;
4088
	cpumask_var_t mask;
L
Linus Torvalds 已提交
4089

A
Anton Blanchard 已提交
4090
	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
4091 4092
		return -EINVAL;
	if (len & (sizeof(unsigned long)-1))
L
Linus Torvalds 已提交
4093 4094
		return -EINVAL;

4095 4096
	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
		return -ENOMEM;
L
Linus Torvalds 已提交
4097

4098 4099
	ret = sched_getaffinity(pid, mask);
	if (ret == 0) {
4100
		size_t retlen = min_t(size_t, len, cpumask_size());
4101 4102

		if (copy_to_user(user_mask_ptr, mask, retlen))
4103 4104
			ret = -EFAULT;
		else
4105
			ret = retlen;
4106 4107
	}
	free_cpumask_var(mask);
L
Linus Torvalds 已提交
4108

4109
	return ret;
L
Linus Torvalds 已提交
4110 4111 4112 4113 4114
}

/**
 * sys_sched_yield - yield the current processor to other threads.
 *
I
Ingo Molnar 已提交
4115 4116
 * This function yields the current CPU to other tasks. If there are no
 * other threads running on this CPU then this function will return.
4117 4118
 *
 * Return: 0.
L
Linus Torvalds 已提交
4119
 */
4120
SYSCALL_DEFINE0(sched_yield)
L
Linus Torvalds 已提交
4121
{
4122
	struct rq *rq = this_rq_lock();
L
Linus Torvalds 已提交
4123

4124
	schedstat_inc(rq, yld_count);
4125
	current->sched_class->yield_task(rq);
L
Linus Torvalds 已提交
4126 4127 4128 4129 4130 4131

	/*
	 * Since we are going to call schedule() anyway, there's
	 * no need to preempt or enable interrupts:
	 */
	__release(rq->lock);
4132
	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
4133
	do_raw_spin_unlock(&rq->lock);
4134
	sched_preempt_enable_no_resched();
L
Linus Torvalds 已提交
4135 4136 4137 4138 4139 4140

	schedule();

	return 0;
}

A
Andrew Morton 已提交
4141
static void __cond_resched(void)
L
Linus Torvalds 已提交
4142
{
4143
	__preempt_count_add(PREEMPT_ACTIVE);
4144
	__schedule();
4145
	__preempt_count_sub(PREEMPT_ACTIVE);
L
Linus Torvalds 已提交
4146 4147
}

4148
int __sched _cond_resched(void)
L
Linus Torvalds 已提交
4149
{
4150
	rcu_cond_resched();
P
Peter Zijlstra 已提交
4151
	if (should_resched()) {
L
Linus Torvalds 已提交
4152 4153 4154 4155 4156
		__cond_resched();
		return 1;
	}
	return 0;
}
4157
EXPORT_SYMBOL(_cond_resched);
L
Linus Torvalds 已提交
4158 4159

/*
4160
 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
L
Linus Torvalds 已提交
4161 4162
 * call schedule, and on return reacquire the lock.
 *
I
Ingo Molnar 已提交
4163
 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
L
Linus Torvalds 已提交
4164 4165 4166
 * operations here to prevent schedule() from being called twice (once via
 * spin_unlock(), once by hand).
 */
4167
int __cond_resched_lock(spinlock_t *lock)
L
Linus Torvalds 已提交
4168
{
4169
	bool need_rcu_resched = rcu_should_resched();
P
Peter Zijlstra 已提交
4170
	int resched = should_resched();
J
Jan Kara 已提交
4171 4172
	int ret = 0;

4173 4174
	lockdep_assert_held(lock);

4175
	if (spin_needbreak(lock) || resched || need_rcu_resched) {
L
Linus Torvalds 已提交
4176
		spin_unlock(lock);
P
Peter Zijlstra 已提交
4177
		if (resched)
N
Nick Piggin 已提交
4178
			__cond_resched();
4179 4180
		else if (unlikely(need_rcu_resched))
			rcu_resched();
N
Nick Piggin 已提交
4181 4182
		else
			cpu_relax();
J
Jan Kara 已提交
4183
		ret = 1;
L
Linus Torvalds 已提交
4184 4185
		spin_lock(lock);
	}
J
Jan Kara 已提交
4186
	return ret;
L
Linus Torvalds 已提交
4187
}
4188
EXPORT_SYMBOL(__cond_resched_lock);
L
Linus Torvalds 已提交
4189

4190
int __sched __cond_resched_softirq(void)
L
Linus Torvalds 已提交
4191 4192 4193
{
	BUG_ON(!in_softirq());

4194
	rcu_cond_resched();  /* BH disabled OK, just recording QSes. */
P
Peter Zijlstra 已提交
4195
	if (should_resched()) {
4196
		local_bh_enable();
L
Linus Torvalds 已提交
4197 4198 4199 4200 4201 4202
		__cond_resched();
		local_bh_disable();
		return 1;
	}
	return 0;
}
4203
EXPORT_SYMBOL(__cond_resched_softirq);
L
Linus Torvalds 已提交
4204 4205 4206 4207

/**
 * yield - yield the current processor to other threads.
 *
P
Peter Zijlstra 已提交
4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225
 * Do not ever use this function, there's a 99% chance you're doing it wrong.
 *
 * The scheduler is at all times free to pick the calling task as the most
 * eligible task to run, if removing the yield() call from your code breaks
 * it, its already broken.
 *
 * Typical broken usage is:
 *
 * while (!event)
 * 	yield();
 *
 * where one assumes that yield() will let 'the other' process run that will
 * make event true. If the current task is a SCHED_FIFO task that will never
 * happen. Never use yield() as a progress guarantee!!
 *
 * If you want to use yield() to wait for something, use wait_event().
 * If you want to use yield() to be 'nice' for others, use cond_resched().
 * If you still want to use yield(), do not!
L
Linus Torvalds 已提交
4226 4227 4228 4229 4230 4231 4232 4233
 */
void __sched yield(void)
{
	set_current_state(TASK_RUNNING);
	sys_sched_yield();
}
EXPORT_SYMBOL(yield);

4234 4235 4236 4237
/**
 * yield_to - yield the current processor to another thread in
 * your thread group, or accelerate that thread toward the
 * processor it's on.
R
Randy Dunlap 已提交
4238 4239
 * @p: target task
 * @preempt: whether task preemption is allowed or not
4240 4241 4242 4243
 *
 * It's the caller's job to ensure that the target task struct
 * can't go away on us before we can do any checks.
 *
4244
 * Return:
4245 4246 4247
 *	true (>0) if we indeed boosted the target task.
 *	false (0) if we failed to boost the target.
 *	-ESRCH if there's no task to yield to.
4248
 */
4249
int __sched yield_to(struct task_struct *p, bool preempt)
4250 4251 4252 4253
{
	struct task_struct *curr = current;
	struct rq *rq, *p_rq;
	unsigned long flags;
4254
	int yielded = 0;
4255 4256 4257 4258 4259 4260

	local_irq_save(flags);
	rq = this_rq();

again:
	p_rq = task_rq(p);
4261 4262 4263 4264 4265 4266 4267 4268 4269
	/*
	 * If we're the only runnable task on the rq and target rq also
	 * has only one task, there's absolutely no point in yielding.
	 */
	if (rq->nr_running == 1 && p_rq->nr_running == 1) {
		yielded = -ESRCH;
		goto out_irq;
	}

4270
	double_rq_lock(rq, p_rq);
4271
	if (task_rq(p) != p_rq) {
4272 4273 4274 4275 4276
		double_rq_unlock(rq, p_rq);
		goto again;
	}

	if (!curr->sched_class->yield_to_task)
4277
		goto out_unlock;
4278 4279

	if (curr->sched_class != p->sched_class)
4280
		goto out_unlock;
4281 4282

	if (task_running(p_rq, p) || p->state)
4283
		goto out_unlock;
4284 4285

	yielded = curr->sched_class->yield_to_task(rq, p, preempt);
4286
	if (yielded) {
4287
		schedstat_inc(rq, yld_count);
4288 4289 4290 4291 4292 4293 4294
		/*
		 * Make p's CPU reschedule; pick_next_entity takes care of
		 * fairness.
		 */
		if (preempt && rq != p_rq)
			resched_task(p_rq->curr);
	}
4295

4296
out_unlock:
4297
	double_rq_unlock(rq, p_rq);
4298
out_irq:
4299 4300
	local_irq_restore(flags);

4301
	if (yielded > 0)
4302 4303 4304 4305 4306 4307
		schedule();

	return yielded;
}
EXPORT_SYMBOL_GPL(yield_to);

L
Linus Torvalds 已提交
4308
/*
I
Ingo Molnar 已提交
4309
 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
L
Linus Torvalds 已提交
4310 4311 4312 4313
 * that process accounting knows that this is a task in IO wait state.
 */
void __sched io_schedule(void)
{
4314
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
4315

4316
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4317
	atomic_inc(&rq->nr_iowait);
4318
	blk_flush_plug(current);
4319
	current->in_iowait = 1;
L
Linus Torvalds 已提交
4320
	schedule();
4321
	current->in_iowait = 0;
L
Linus Torvalds 已提交
4322
	atomic_dec(&rq->nr_iowait);
4323
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4324 4325 4326 4327 4328
}
EXPORT_SYMBOL(io_schedule);

long __sched io_schedule_timeout(long timeout)
{
4329
	struct rq *rq = raw_rq();
L
Linus Torvalds 已提交
4330 4331
	long ret;

4332
	delayacct_blkio_start();
L
Linus Torvalds 已提交
4333
	atomic_inc(&rq->nr_iowait);
4334
	blk_flush_plug(current);
4335
	current->in_iowait = 1;
L
Linus Torvalds 已提交
4336
	ret = schedule_timeout(timeout);
4337
	current->in_iowait = 0;
L
Linus Torvalds 已提交
4338
	atomic_dec(&rq->nr_iowait);
4339
	delayacct_blkio_end();
L
Linus Torvalds 已提交
4340 4341 4342 4343 4344 4345 4346
	return ret;
}

/**
 * sys_sched_get_priority_max - return maximum RT priority.
 * @policy: scheduling class.
 *
4347 4348 4349
 * Return: On success, this syscall returns the maximum
 * rt_priority that can be used by a given scheduling class.
 * On failure, a negative error code is returned.
L
Linus Torvalds 已提交
4350
 */
4351
SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
L
Linus Torvalds 已提交
4352 4353 4354 4355 4356 4357 4358 4359
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = MAX_USER_RT_PRIO-1;
		break;
4360
	case SCHED_DEADLINE:
L
Linus Torvalds 已提交
4361
	case SCHED_NORMAL:
4362
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4363
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4364 4365 4366 4367 4368 4369 4370 4371 4372 4373
		ret = 0;
		break;
	}
	return ret;
}

/**
 * sys_sched_get_priority_min - return minimum RT priority.
 * @policy: scheduling class.
 *
4374 4375 4376
 * Return: On success, this syscall returns the minimum
 * rt_priority that can be used by a given scheduling class.
 * On failure, a negative error code is returned.
L
Linus Torvalds 已提交
4377
 */
4378
SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
L
Linus Torvalds 已提交
4379 4380 4381 4382 4383 4384 4385 4386
{
	int ret = -EINVAL;

	switch (policy) {
	case SCHED_FIFO:
	case SCHED_RR:
		ret = 1;
		break;
4387
	case SCHED_DEADLINE:
L
Linus Torvalds 已提交
4388
	case SCHED_NORMAL:
4389
	case SCHED_BATCH:
I
Ingo Molnar 已提交
4390
	case SCHED_IDLE:
L
Linus Torvalds 已提交
4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402
		ret = 0;
	}
	return ret;
}

/**
 * sys_sched_rr_get_interval - return the default timeslice of a process.
 * @pid: pid of the process.
 * @interval: userspace pointer to the timeslice value.
 *
 * this syscall writes the default timeslice value of a given process
 * into the user-space timespec buffer. A value of '0' means infinity.
4403 4404 4405
 *
 * Return: On success, 0 and the timeslice is in @interval. Otherwise,
 * an error code.
L
Linus Torvalds 已提交
4406
 */
4407
SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
4408
		struct timespec __user *, interval)
L
Linus Torvalds 已提交
4409
{
4410
	struct task_struct *p;
D
Dmitry Adamushko 已提交
4411
	unsigned int time_slice;
4412 4413
	unsigned long flags;
	struct rq *rq;
4414
	int retval;
L
Linus Torvalds 已提交
4415 4416 4417
	struct timespec t;

	if (pid < 0)
4418
		return -EINVAL;
L
Linus Torvalds 已提交
4419 4420

	retval = -ESRCH;
4421
	rcu_read_lock();
L
Linus Torvalds 已提交
4422 4423 4424 4425 4426 4427 4428 4429
	p = find_process_by_pid(pid);
	if (!p)
		goto out_unlock;

	retval = security_task_getscheduler(p);
	if (retval)
		goto out_unlock;

4430
	rq = task_rq_lock(p, &flags);
4431 4432 4433
	time_slice = 0;
	if (p->sched_class->get_rr_interval)
		time_slice = p->sched_class->get_rr_interval(rq, p);
4434
	task_rq_unlock(rq, p, &flags);
D
Dmitry Adamushko 已提交
4435

4436
	rcu_read_unlock();
D
Dmitry Adamushko 已提交
4437
	jiffies_to_timespec(time_slice, &t);
L
Linus Torvalds 已提交
4438 4439
	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
	return retval;
4440

L
Linus Torvalds 已提交
4441
out_unlock:
4442
	rcu_read_unlock();
L
Linus Torvalds 已提交
4443 4444 4445
	return retval;
}

4446
static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
4447

4448
void sched_show_task(struct task_struct *p)
L
Linus Torvalds 已提交
4449 4450
{
	unsigned long free = 0;
4451
	int ppid;
4452
	unsigned state;
L
Linus Torvalds 已提交
4453 4454

	state = p->state ? __ffs(p->state) + 1 : 0;
4455
	printk(KERN_INFO "%-15.15s %c", p->comm,
4456
		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4457
#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
4458
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
4459
		printk(KERN_CONT " running  ");
L
Linus Torvalds 已提交
4460
	else
P
Peter Zijlstra 已提交
4461
		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4462 4463
#else
	if (state == TASK_RUNNING)
P
Peter Zijlstra 已提交
4464
		printk(KERN_CONT "  running task    ");
L
Linus Torvalds 已提交
4465
	else
P
Peter Zijlstra 已提交
4466
		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
L
Linus Torvalds 已提交
4467 4468
#endif
#ifdef CONFIG_DEBUG_STACK_USAGE
4469
	free = stack_not_used(p);
L
Linus Torvalds 已提交
4470
#endif
4471 4472 4473
	rcu_read_lock();
	ppid = task_pid_nr(rcu_dereference(p->real_parent));
	rcu_read_unlock();
P
Peter Zijlstra 已提交
4474
	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
4475
		task_pid_nr(p), ppid,
4476
		(unsigned long)task_thread_info(p)->flags);
L
Linus Torvalds 已提交
4477

4478
	print_worker_info(KERN_INFO, p);
4479
	show_stack(p, NULL);
L
Linus Torvalds 已提交
4480 4481
}

I
Ingo Molnar 已提交
4482
void show_state_filter(unsigned long state_filter)
L
Linus Torvalds 已提交
4483
{
4484
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
4485

4486
#if BITS_PER_LONG == 32
P
Peter Zijlstra 已提交
4487 4488
	printk(KERN_INFO
		"  task                PC stack   pid father\n");
L
Linus Torvalds 已提交
4489
#else
P
Peter Zijlstra 已提交
4490 4491
	printk(KERN_INFO
		"  task                        PC stack   pid father\n");
L
Linus Torvalds 已提交
4492
#endif
4493
	rcu_read_lock();
L
Linus Torvalds 已提交
4494 4495 4496
	do_each_thread(g, p) {
		/*
		 * reset the NMI-timeout, listing all files on a slow
L
Lucas De Marchi 已提交
4497
		 * console might take a lot of time:
L
Linus Torvalds 已提交
4498 4499
		 */
		touch_nmi_watchdog();
I
Ingo Molnar 已提交
4500
		if (!state_filter || (p->state & state_filter))
4501
			sched_show_task(p);
L
Linus Torvalds 已提交
4502 4503
	} while_each_thread(g, p);

4504 4505
	touch_all_softlockup_watchdogs();

I
Ingo Molnar 已提交
4506 4507 4508
#ifdef CONFIG_SCHED_DEBUG
	sysrq_sched_debug_show();
#endif
4509
	rcu_read_unlock();
I
Ingo Molnar 已提交
4510 4511 4512
	/*
	 * Only show locks if all tasks are dumped:
	 */
4513
	if (!state_filter)
I
Ingo Molnar 已提交
4514
		debug_show_all_locks();
L
Linus Torvalds 已提交
4515 4516
}

4517
void init_idle_bootup_task(struct task_struct *idle)
I
Ingo Molnar 已提交
4518
{
I
Ingo Molnar 已提交
4519
	idle->sched_class = &idle_sched_class;
I
Ingo Molnar 已提交
4520 4521
}

4522 4523 4524 4525 4526 4527 4528 4529
/**
 * init_idle - set up an idle thread for a given CPU
 * @idle: task in question
 * @cpu: cpu the idle task belongs to
 *
 * NOTE: this function does not set the idle thread's NEED_RESCHED
 * flag, to make booting more robust.
 */
4530
void init_idle(struct task_struct *idle, int cpu)
L
Linus Torvalds 已提交
4531
{
4532
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
4533 4534
	unsigned long flags;

4535
	raw_spin_lock_irqsave(&rq->lock, flags);
4536

4537
	__sched_fork(0, idle);
4538
	idle->state = TASK_RUNNING;
I
Ingo Molnar 已提交
4539 4540
	idle->se.exec_start = sched_clock();

4541
	do_set_cpus_allowed(idle, cpumask_of(cpu));
4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552
	/*
	 * We're having a chicken and egg problem, even though we are
	 * holding rq->lock, the cpu isn't yet set to this cpu so the
	 * lockdep check in task_group() will fail.
	 *
	 * Similar case to sched_fork(). / Alternatively we could
	 * use task_rq_lock() here and obtain the other rq->lock.
	 *
	 * Silence PROVE_RCU
	 */
	rcu_read_lock();
I
Ingo Molnar 已提交
4553
	__set_task_cpu(idle, cpu);
4554
	rcu_read_unlock();
L
Linus Torvalds 已提交
4555 4556

	rq->curr = rq->idle = idle;
4557
	idle->on_rq = 1;
P
Peter Zijlstra 已提交
4558 4559
#if defined(CONFIG_SMP)
	idle->on_cpu = 1;
4560
#endif
4561
	raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
4562 4563

	/* Set the preempt count _outside_ the spinlocks! */
4564
	init_idle_preempt_count(idle, cpu);
4565

I
Ingo Molnar 已提交
4566 4567 4568 4569
	/*
	 * The idle tasks have their own, simple scheduling class:
	 */
	idle->sched_class = &idle_sched_class;
4570
	ftrace_graph_init_idle_task(idle, cpu);
4571
	vtime_init_idle(idle, cpu);
4572 4573 4574
#if defined(CONFIG_SMP)
	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
#endif
I
Ingo Molnar 已提交
4575 4576
}

L
Linus Torvalds 已提交
4577
#ifdef CONFIG_SMP
4578 4579 4580 4581
void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
	if (p->sched_class && p->sched_class->set_cpus_allowed)
		p->sched_class->set_cpus_allowed(p, new_mask);
4582 4583

	cpumask_copy(&p->cpus_allowed, new_mask);
4584
	p->nr_cpus_allowed = cpumask_weight(new_mask);
4585 4586
}

L
Linus Torvalds 已提交
4587 4588 4589
/*
 * This is how migration works:
 *
4590 4591 4592 4593 4594 4595
 * 1) we invoke migration_cpu_stop() on the target CPU using
 *    stop_one_cpu().
 * 2) stopper starts to run (implicitly forcing the migrated thread
 *    off the CPU)
 * 3) it checks whether the migrated task is still in the wrong runqueue.
 * 4) if it's in the wrong runqueue then the migration thread removes
L
Linus Torvalds 已提交
4596
 *    it and puts it into the right queue.
4597 4598
 * 5) stopper completes and stop_one_cpu() returns and the migration
 *    is done.
L
Linus Torvalds 已提交
4599 4600 4601 4602 4603 4604 4605 4606
 */

/*
 * Change a given task's CPU affinity. Migrate the thread to a
 * proper CPU and schedule it away if the CPU it's executing on
 * is removed from the allowed bitmask.
 *
 * NOTE: the caller must have a valid reference to the task, the
I
Ingo Molnar 已提交
4607
 * task must not exit() & deallocate itself prematurely. The
L
Linus Torvalds 已提交
4608 4609
 * call is not atomic; no spinlocks may be held.
 */
4610
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
L
Linus Torvalds 已提交
4611 4612
{
	unsigned long flags;
4613
	struct rq *rq;
4614
	unsigned int dest_cpu;
4615
	int ret = 0;
L
Linus Torvalds 已提交
4616 4617

	rq = task_rq_lock(p, &flags);
4618

4619 4620 4621
	if (cpumask_equal(&p->cpus_allowed, new_mask))
		goto out;

4622
	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
L
Linus Torvalds 已提交
4623 4624 4625 4626
		ret = -EINVAL;
		goto out;
	}

4627
	do_set_cpus_allowed(p, new_mask);
4628

L
Linus Torvalds 已提交
4629
	/* Can the task run on the task's current CPU? If so, we're done */
4630
	if (cpumask_test_cpu(task_cpu(p), new_mask))
L
Linus Torvalds 已提交
4631 4632
		goto out;

4633
	dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
4634
	if (p->on_rq) {
4635
		struct migration_arg arg = { p, dest_cpu };
L
Linus Torvalds 已提交
4636
		/* Need help from migration thread: drop lock and wait. */
4637
		task_rq_unlock(rq, p, &flags);
4638
		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
L
Linus Torvalds 已提交
4639 4640 4641 4642
		tlb_migrate_finish(p->mm);
		return 0;
	}
out:
4643
	task_rq_unlock(rq, p, &flags);
4644

L
Linus Torvalds 已提交
4645 4646
	return ret;
}
4647
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
L
Linus Torvalds 已提交
4648 4649

/*
I
Ingo Molnar 已提交
4650
 * Move (not current) task off this cpu, onto dest cpu. We're doing
L
Linus Torvalds 已提交
4651 4652 4653 4654 4655 4656
 * this because either it can't run here any more (set_cpus_allowed()
 * away from this CPU, or CPU going down), or because we're
 * attempting to rebalance this task on exec (sched_exec).
 *
 * So we race with normal scheduler movements, but that's OK, as long
 * as the task is no longer on this CPU.
4657 4658
 *
 * Returns non-zero if task was successfully migrated.
L
Linus Torvalds 已提交
4659
 */
4660
static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
L
Linus Torvalds 已提交
4661
{
4662
	struct rq *rq_dest, *rq_src;
4663
	int ret = 0;
L
Linus Torvalds 已提交
4664

4665
	if (unlikely(!cpu_active(dest_cpu)))
4666
		return ret;
L
Linus Torvalds 已提交
4667 4668 4669 4670

	rq_src = cpu_rq(src_cpu);
	rq_dest = cpu_rq(dest_cpu);

4671
	raw_spin_lock(&p->pi_lock);
L
Linus Torvalds 已提交
4672 4673 4674
	double_rq_lock(rq_src, rq_dest);
	/* Already moved. */
	if (task_cpu(p) != src_cpu)
L
Linus Torvalds 已提交
4675
		goto done;
L
Linus Torvalds 已提交
4676
	/* Affinity changed (again). */
4677
	if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
L
Linus Torvalds 已提交
4678
		goto fail;
L
Linus Torvalds 已提交
4679

4680 4681 4682 4683
	/*
	 * If we're not on a rq, the next wake-up will ensure we're
	 * placed properly.
	 */
P
Peter Zijlstra 已提交
4684
	if (p->on_rq) {
4685
		dequeue_task(rq_src, p, 0);
4686
		set_task_cpu(p, dest_cpu);
4687
		enqueue_task(rq_dest, p, 0);
4688
		check_preempt_curr(rq_dest, p, 0);
L
Linus Torvalds 已提交
4689
	}
L
Linus Torvalds 已提交
4690
done:
4691
	ret = 1;
L
Linus Torvalds 已提交
4692
fail:
L
Linus Torvalds 已提交
4693
	double_rq_unlock(rq_src, rq_dest);
4694
	raw_spin_unlock(&p->pi_lock);
4695
	return ret;
L
Linus Torvalds 已提交
4696 4697
}

4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712
#ifdef CONFIG_NUMA_BALANCING
/* Migrate current task p to target_cpu */
int migrate_task_to(struct task_struct *p, int target_cpu)
{
	struct migration_arg arg = { p, target_cpu };
	int curr_cpu = task_cpu(p);

	if (curr_cpu == target_cpu)
		return 0;

	if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
		return -EINVAL;

	/* TODO: This is not properly updating schedstats */

4713
	trace_sched_move_numa(p, curr_cpu, target_cpu);
4714 4715
	return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
}
4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743

/*
 * Requeue a task on a given node and accurately track the number of NUMA
 * tasks on the runqueues
 */
void sched_setnuma(struct task_struct *p, int nid)
{
	struct rq *rq;
	unsigned long flags;
	bool on_rq, running;

	rq = task_rq_lock(p, &flags);
	on_rq = p->on_rq;
	running = task_current(rq, p);

	if (on_rq)
		dequeue_task(rq, p, 0);
	if (running)
		p->sched_class->put_prev_task(rq, p);

	p->numa_preferred_nid = nid;

	if (running)
		p->sched_class->set_curr_task(rq);
	if (on_rq)
		enqueue_task(rq, p, 0);
	task_rq_unlock(rq, p, &flags);
}
4744 4745
#endif

L
Linus Torvalds 已提交
4746
/*
4747 4748 4749
 * migration_cpu_stop - this will be executed by a highprio stopper thread
 * and performs thread migration by bumping thread off CPU then
 * 'pushing' onto another runqueue.
L
Linus Torvalds 已提交
4750
 */
4751
static int migration_cpu_stop(void *data)
L
Linus Torvalds 已提交
4752
{
4753
	struct migration_arg *arg = data;
4754

4755 4756 4757 4758
	/*
	 * The original target cpu might have gone down and we might
	 * be on another cpu but it doesn't matter.
	 */
4759
	local_irq_disable();
4760
	__migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
4761
	local_irq_enable();
L
Linus Torvalds 已提交
4762
	return 0;
4763 4764
}

L
Linus Torvalds 已提交
4765
#ifdef CONFIG_HOTPLUG_CPU
4766

4767
/*
4768 4769
 * Ensures that the idle task is using init_mm right before its cpu goes
 * offline.
4770
 */
4771
void idle_task_exit(void)
L
Linus Torvalds 已提交
4772
{
4773
	struct mm_struct *mm = current->active_mm;
4774

4775
	BUG_ON(cpu_online(smp_processor_id()));
4776

4777
	if (mm != &init_mm) {
4778
		switch_mm(mm, &init_mm, current);
4779 4780
		finish_arch_post_lock_switch();
	}
4781
	mmdrop(mm);
L
Linus Torvalds 已提交
4782 4783 4784
}

/*
4785 4786 4787 4788 4789
 * Since this CPU is going 'away' for a while, fold any nr_active delta
 * we might have. Assumes we're called after migrate_tasks() so that the
 * nr_active count is stable.
 *
 * Also see the comment "Global load-average calculations".
L
Linus Torvalds 已提交
4790
 */
4791
static void calc_load_migrate(struct rq *rq)
L
Linus Torvalds 已提交
4792
{
4793 4794 4795
	long delta = calc_load_fold_active(rq);
	if (delta)
		atomic_long_add(delta, &calc_load_tasks);
L
Linus Torvalds 已提交
4796 4797
}

4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813
static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
{
}

static const struct sched_class fake_sched_class = {
	.put_prev_task = put_prev_task_fake,
};

static struct task_struct fake_task = {
	/*
	 * Avoid pull_{rt,dl}_task()
	 */
	.prio = MAX_PRIO + 1,
	.sched_class = &fake_sched_class,
};

4814
/*
4815 4816 4817 4818 4819 4820
 * Migrate all tasks from the rq, sleeping tasks will be migrated by
 * try_to_wake_up()->select_task_rq().
 *
 * Called with rq->lock held even though we'er in stop_machine() and
 * there's no concurrency possible, we hold the required locks anyway
 * because of lock validation efforts.
L
Linus Torvalds 已提交
4821
 */
4822
static void migrate_tasks(unsigned int dead_cpu)
L
Linus Torvalds 已提交
4823
{
4824
	struct rq *rq = cpu_rq(dead_cpu);
4825 4826
	struct task_struct *next, *stop = rq->stop;
	int dest_cpu;
L
Linus Torvalds 已提交
4827 4828

	/*
4829 4830 4831 4832 4833 4834 4835
	 * Fudge the rq selection such that the below task selection loop
	 * doesn't get stuck on the currently eligible stop task.
	 *
	 * We're currently inside stop_machine() and the rq is either stuck
	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
	 * either way we should never end up calling schedule() until we're
	 * done here.
L
Linus Torvalds 已提交
4836
	 */
4837
	rq->stop = NULL;
4838

4839 4840 4841 4842 4843 4844 4845
	/*
	 * put_prev_task() and pick_next_task() sched
	 * class method both need to have an up-to-date
	 * value of rq->clock[_task]
	 */
	update_rq_clock(rq);

I
Ingo Molnar 已提交
4846
	for ( ; ; ) {
4847 4848 4849 4850 4851
		/*
		 * There's this thread running, bail when that's the only
		 * remaining thread.
		 */
		if (rq->nr_running == 1)
I
Ingo Molnar 已提交
4852
			break;
4853

4854
		next = pick_next_task(rq, &fake_task);
4855
		BUG_ON(!next);
D
Dmitry Adamushko 已提交
4856
		next->sched_class->put_prev_task(rq, next);
4857

4858 4859 4860 4861 4862 4863 4864
		/* Find suitable destination for @next, with force if needed. */
		dest_cpu = select_fallback_rq(dead_cpu, next);
		raw_spin_unlock(&rq->lock);

		__migrate_task(next, dead_cpu, dest_cpu);

		raw_spin_lock(&rq->lock);
L
Linus Torvalds 已提交
4865
	}
4866

4867
	rq->stop = stop;
4868
}
4869

L
Linus Torvalds 已提交
4870 4871
#endif /* CONFIG_HOTPLUG_CPU */

4872 4873 4874
#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)

static struct ctl_table sd_ctl_dir[] = {
4875 4876
	{
		.procname	= "sched_domain",
4877
		.mode		= 0555,
4878
	},
4879
	{}
4880 4881 4882
};

static struct ctl_table sd_ctl_root[] = {
4883 4884
	{
		.procname	= "kernel",
4885
		.mode		= 0555,
4886 4887
		.child		= sd_ctl_dir,
	},
4888
	{}
4889 4890 4891 4892 4893
};

static struct ctl_table *sd_alloc_ctl_entry(int n)
{
	struct ctl_table *entry =
4894
		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
4895 4896 4897 4898

	return entry;
}

4899 4900
static void sd_free_ctl_entry(struct ctl_table **tablep)
{
4901
	struct ctl_table *entry;
4902

4903 4904 4905
	/*
	 * In the intermediate directories, both the child directory and
	 * procname are dynamically allocated and could fail but the mode
I
Ingo Molnar 已提交
4906
	 * will always be set. In the lowest directory the names are
4907 4908 4909
	 * static strings and all have proc handlers.
	 */
	for (entry = *tablep; entry->mode; entry++) {
4910 4911
		if (entry->child)
			sd_free_ctl_entry(&entry->child);
4912 4913 4914
		if (entry->proc_handler == NULL)
			kfree(entry->procname);
	}
4915 4916 4917 4918 4919

	kfree(*tablep);
	*tablep = NULL;
}

4920
static int min_load_idx = 0;
4921
static int max_load_idx = CPU_LOAD_IDX_MAX-1;
4922

4923
static void
4924
set_table_entry(struct ctl_table *entry,
4925
		const char *procname, void *data, int maxlen,
4926 4927
		umode_t mode, proc_handler *proc_handler,
		bool load_idx)
4928 4929 4930 4931 4932 4933
{
	entry->procname = procname;
	entry->data = data;
	entry->maxlen = maxlen;
	entry->mode = mode;
	entry->proc_handler = proc_handler;
4934 4935 4936 4937 4938

	if (load_idx) {
		entry->extra1 = &min_load_idx;
		entry->extra2 = &max_load_idx;
	}
4939 4940 4941 4942 4943
}

static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
4944
	struct ctl_table *table = sd_alloc_ctl_entry(14);
4945

4946 4947 4948
	if (table == NULL)
		return NULL;

4949
	set_table_entry(&table[0], "min_interval", &sd->min_interval,
4950
		sizeof(long), 0644, proc_doulongvec_minmax, false);
4951
	set_table_entry(&table[1], "max_interval", &sd->max_interval,
4952
		sizeof(long), 0644, proc_doulongvec_minmax, false);
4953
	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
4954
		sizeof(int), 0644, proc_dointvec_minmax, true);
4955
	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
4956
		sizeof(int), 0644, proc_dointvec_minmax, true);
4957
	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
4958
		sizeof(int), 0644, proc_dointvec_minmax, true);
4959
	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
4960
		sizeof(int), 0644, proc_dointvec_minmax, true);
4961
	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
4962
		sizeof(int), 0644, proc_dointvec_minmax, true);
4963
	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
4964
		sizeof(int), 0644, proc_dointvec_minmax, false);
4965
	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
4966
		sizeof(int), 0644, proc_dointvec_minmax, false);
4967
	set_table_entry(&table[9], "cache_nice_tries",
4968
		&sd->cache_nice_tries,
4969
		sizeof(int), 0644, proc_dointvec_minmax, false);
4970
	set_table_entry(&table[10], "flags", &sd->flags,
4971
		sizeof(int), 0644, proc_dointvec_minmax, false);
4972 4973 4974 4975
	set_table_entry(&table[11], "max_newidle_lb_cost",
		&sd->max_newidle_lb_cost,
		sizeof(long), 0644, proc_doulongvec_minmax, false);
	set_table_entry(&table[12], "name", sd->name,
4976
		CORENAME_MAX_SIZE, 0444, proc_dostring, false);
4977
	/* &table[13] is terminator */
4978 4979 4980 4981

	return table;
}

4982
static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
4983 4984 4985 4986 4987 4988 4989 4990 4991
{
	struct ctl_table *entry, *table;
	struct sched_domain *sd;
	int domain_num = 0, i;
	char buf[32];

	for_each_domain(cpu, sd)
		domain_num++;
	entry = table = sd_alloc_ctl_entry(domain_num + 1);
4992 4993
	if (table == NULL)
		return NULL;
4994 4995 4996 4997 4998

	i = 0;
	for_each_domain(cpu, sd) {
		snprintf(buf, 32, "domain%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
4999
		entry->mode = 0555;
5000 5001 5002 5003 5004 5005 5006 5007
		entry->child = sd_alloc_ctl_domain_table(sd);
		entry++;
		i++;
	}
	return table;
}

static struct ctl_table_header *sd_sysctl_header;
5008
static void register_sched_domain_sysctl(void)
5009
{
5010
	int i, cpu_num = num_possible_cpus();
5011 5012 5013
	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
	char buf[32];

5014 5015 5016
	WARN_ON(sd_ctl_dir[0].child);
	sd_ctl_dir[0].child = entry;

5017 5018 5019
	if (entry == NULL)
		return;

5020
	for_each_possible_cpu(i) {
5021 5022
		snprintf(buf, 32, "cpu%d", i);
		entry->procname = kstrdup(buf, GFP_KERNEL);
5023
		entry->mode = 0555;
5024
		entry->child = sd_alloc_ctl_cpu_table(i);
5025
		entry++;
5026
	}
5027 5028

	WARN_ON(sd_sysctl_header);
5029 5030
	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
}
5031

5032
/* may be called multiple times per register */
5033 5034
static void unregister_sched_domain_sysctl(void)
{
5035 5036
	if (sd_sysctl_header)
		unregister_sysctl_table(sd_sysctl_header);
5037
	sd_sysctl_header = NULL;
5038 5039
	if (sd_ctl_dir[0].child)
		sd_free_ctl_entry(&sd_ctl_dir[0].child);
5040
}
5041
#else
5042 5043 5044 5045
static void register_sched_domain_sysctl(void)
{
}
static void unregister_sched_domain_sysctl(void)
5046 5047 5048 5049
{
}
#endif

5050 5051 5052 5053 5054
static void set_rq_online(struct rq *rq)
{
	if (!rq->online) {
		const struct sched_class *class;

5055
		cpumask_set_cpu(rq->cpu, rq->rd->online);
5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074
		rq->online = 1;

		for_each_class(class) {
			if (class->rq_online)
				class->rq_online(rq);
		}
	}
}

static void set_rq_offline(struct rq *rq)
{
	if (rq->online) {
		const struct sched_class *class;

		for_each_class(class) {
			if (class->rq_offline)
				class->rq_offline(rq);
		}

5075
		cpumask_clear_cpu(rq->cpu, rq->rd->online);
5076 5077 5078 5079
		rq->online = 0;
	}
}

L
Linus Torvalds 已提交
5080 5081 5082 5083
/*
 * migration_call - callback that gets triggered when a CPU is added.
 * Here we can start up the necessary migration thread for the new CPU.
 */
5084
static int
5085
migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
L
Linus Torvalds 已提交
5086
{
5087
	int cpu = (long)hcpu;
L
Linus Torvalds 已提交
5088
	unsigned long flags;
5089
	struct rq *rq = cpu_rq(cpu);
L
Linus Torvalds 已提交
5090

5091
	switch (action & ~CPU_TASKS_FROZEN) {
5092

L
Linus Torvalds 已提交
5093
	case CPU_UP_PREPARE:
5094
		rq->calc_load_update = calc_load_update;
L
Linus Torvalds 已提交
5095
		break;
5096

L
Linus Torvalds 已提交
5097
	case CPU_ONLINE:
5098
		/* Update our root-domain */
5099
		raw_spin_lock_irqsave(&rq->lock, flags);
5100
		if (rq->rd) {
5101
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
5102 5103

			set_rq_online(rq);
5104
		}
5105
		raw_spin_unlock_irqrestore(&rq->lock, flags);
L
Linus Torvalds 已提交
5106
		break;
5107

L
Linus Torvalds 已提交
5108
#ifdef CONFIG_HOTPLUG_CPU
5109
	case CPU_DYING:
5110
		sched_ttwu_pending();
G
Gregory Haskins 已提交
5111
		/* Update our root-domain */
5112
		raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
5113
		if (rq->rd) {
5114
			BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
5115
			set_rq_offline(rq);
G
Gregory Haskins 已提交
5116
		}
5117 5118
		migrate_tasks(cpu);
		BUG_ON(rq->nr_running != 1); /* the migration thread */
5119
		raw_spin_unlock_irqrestore(&rq->lock, flags);
5120
		break;
5121

5122
	case CPU_DEAD:
5123
		calc_load_migrate(rq);
G
Gregory Haskins 已提交
5124
		break;
L
Linus Torvalds 已提交
5125 5126
#endif
	}
5127 5128 5129

	update_max_interval();

L
Linus Torvalds 已提交
5130 5131 5132
	return NOTIFY_OK;
}

5133 5134 5135
/*
 * Register at high priority so that task migration (migrate_all_tasks)
 * happens before everything else.  This has to be lower priority than
5136
 * the notifier in the perf_event subsystem, though.
L
Linus Torvalds 已提交
5137
 */
5138
static struct notifier_block migration_notifier = {
L
Linus Torvalds 已提交
5139
	.notifier_call = migration_call,
5140
	.priority = CPU_PRI_MIGRATION,
L
Linus Torvalds 已提交
5141 5142
};

5143 5144 5145 5146 5147 5148 5149
static void __cpuinit set_cpu_rq_start_time(void)
{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);
	rq->age_stamp = sched_clock_cpu(cpu);
}

5150
static int sched_cpu_active(struct notifier_block *nfb,
5151 5152 5153
				      unsigned long action, void *hcpu)
{
	switch (action & ~CPU_TASKS_FROZEN) {
5154 5155 5156
	case CPU_STARTING:
		set_cpu_rq_start_time();
		return NOTIFY_OK;
5157 5158 5159 5160 5161 5162 5163 5164
	case CPU_DOWN_FAILED:
		set_cpu_active((long)hcpu, true);
		return NOTIFY_OK;
	default:
		return NOTIFY_DONE;
	}
}

5165
static int sched_cpu_inactive(struct notifier_block *nfb,
5166 5167
					unsigned long action, void *hcpu)
{
5168 5169 5170
	unsigned long flags;
	long cpu = (long)hcpu;

5171 5172
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_DOWN_PREPARE:
5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188
		set_cpu_active(cpu, false);

		/* explicitly allow suspend */
		if (!(action & CPU_TASKS_FROZEN)) {
			struct dl_bw *dl_b = dl_bw_of(cpu);
			bool overflow;
			int cpus;

			raw_spin_lock_irqsave(&dl_b->lock, flags);
			cpus = dl_bw_cpus(cpu);
			overflow = __dl_overflow(dl_b, cpus, 0, 0);
			raw_spin_unlock_irqrestore(&dl_b->lock, flags);

			if (overflow)
				return notifier_from_errno(-EBUSY);
		}
5189 5190
		return NOTIFY_OK;
	}
5191 5192

	return NOTIFY_DONE;
5193 5194
}

5195
static int __init migration_init(void)
L
Linus Torvalds 已提交
5196 5197
{
	void *cpu = (void *)(long)smp_processor_id();
5198
	int err;
5199

5200
	/* Initialize migration for the boot CPU */
5201 5202
	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
	BUG_ON(err == NOTIFY_BAD);
L
Linus Torvalds 已提交
5203 5204
	migration_call(&migration_notifier, CPU_ONLINE, cpu);
	register_cpu_notifier(&migration_notifier);
5205

5206 5207 5208 5209
	/* Register cpu active notifiers */
	cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
	cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);

5210
	return 0;
L
Linus Torvalds 已提交
5211
}
5212
early_initcall(migration_init);
L
Linus Torvalds 已提交
5213 5214 5215
#endif

#ifdef CONFIG_SMP
5216

5217 5218
static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */

5219
#ifdef CONFIG_SCHED_DEBUG
I
Ingo Molnar 已提交
5220

5221
static __read_mostly int sched_debug_enabled;
5222

5223
static int __init sched_debug_setup(char *str)
5224
{
5225
	sched_debug_enabled = 1;
5226 5227 5228

	return 0;
}
5229 5230 5231 5232 5233 5234
early_param("sched_debug", sched_debug_setup);

static inline bool sched_debug(void)
{
	return sched_debug_enabled;
}
5235

5236
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5237
				  struct cpumask *groupmask)
L
Linus Torvalds 已提交
5238
{
I
Ingo Molnar 已提交
5239
	struct sched_group *group = sd->groups;
5240
	char str[256];
L
Linus Torvalds 已提交
5241

R
Rusty Russell 已提交
5242
	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
5243
	cpumask_clear(groupmask);
I
Ingo Molnar 已提交
5244 5245 5246 5247

	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);

	if (!(sd->flags & SD_LOAD_BALANCE)) {
P
Peter Zijlstra 已提交
5248
		printk("does not load-balance\n");
I
Ingo Molnar 已提交
5249
		if (sd->parent)
P
Peter Zijlstra 已提交
5250 5251
			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
					" has parent");
I
Ingo Molnar 已提交
5252
		return -1;
N
Nick Piggin 已提交
5253 5254
	}

P
Peter Zijlstra 已提交
5255
	printk(KERN_CONT "span %s level %s\n", str, sd->name);
I
Ingo Molnar 已提交
5256

5257
	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
P
Peter Zijlstra 已提交
5258 5259
		printk(KERN_ERR "ERROR: domain->span does not contain "
				"CPU%d\n", cpu);
I
Ingo Molnar 已提交
5260
	}
5261
	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5262 5263
		printk(KERN_ERR "ERROR: domain->groups does not contain"
				" CPU%d\n", cpu);
I
Ingo Molnar 已提交
5264
	}
L
Linus Torvalds 已提交
5265

I
Ingo Molnar 已提交
5266
	printk(KERN_DEBUG "%*s groups:", level + 1, "");
L
Linus Torvalds 已提交
5267
	do {
I
Ingo Molnar 已提交
5268
		if (!group) {
P
Peter Zijlstra 已提交
5269 5270
			printk("\n");
			printk(KERN_ERR "ERROR: group is NULL\n");
L
Linus Torvalds 已提交
5271 5272 5273
			break;
		}

5274
		/*
5275 5276
		 * Even though we initialize ->capacity to something semi-sane,
		 * we leave capacity_orig unset. This allows us to detect if
5277 5278
		 * domain iteration is still funny without causing /0 traps.
		 */
5279
		if (!group->sgc->capacity_orig) {
P
Peter Zijlstra 已提交
5280
			printk(KERN_CONT "\n");
5281
			printk(KERN_ERR "ERROR: domain->cpu_capacity not set\n");
I
Ingo Molnar 已提交
5282 5283
			break;
		}
L
Linus Torvalds 已提交
5284

5285
		if (!cpumask_weight(sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5286 5287
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: empty group\n");
I
Ingo Molnar 已提交
5288 5289
			break;
		}
L
Linus Torvalds 已提交
5290

5291 5292
		if (!(sd->flags & SD_OVERLAP) &&
		    cpumask_intersects(groupmask, sched_group_cpus(group))) {
P
Peter Zijlstra 已提交
5293 5294
			printk(KERN_CONT "\n");
			printk(KERN_ERR "ERROR: repeated CPUs\n");
I
Ingo Molnar 已提交
5295 5296
			break;
		}
L
Linus Torvalds 已提交
5297

5298
		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
L
Linus Torvalds 已提交
5299

R
Rusty Russell 已提交
5300
		cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
5301

P
Peter Zijlstra 已提交
5302
		printk(KERN_CONT " %s", str);
5303
		if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
5304 5305
			printk(KERN_CONT " (cpu_capacity = %d)",
				group->sgc->capacity);
5306
		}
L
Linus Torvalds 已提交
5307

I
Ingo Molnar 已提交
5308 5309
		group = group->next;
	} while (group != sd->groups);
P
Peter Zijlstra 已提交
5310
	printk(KERN_CONT "\n");
L
Linus Torvalds 已提交
5311

5312
	if (!cpumask_equal(sched_domain_span(sd), groupmask))
P
Peter Zijlstra 已提交
5313
		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
L
Linus Torvalds 已提交
5314

5315 5316
	if (sd->parent &&
	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
P
Peter Zijlstra 已提交
5317 5318
		printk(KERN_ERR "ERROR: parent span is not a superset "
			"of domain->span\n");
I
Ingo Molnar 已提交
5319 5320
	return 0;
}
L
Linus Torvalds 已提交
5321

I
Ingo Molnar 已提交
5322 5323 5324
static void sched_domain_debug(struct sched_domain *sd, int cpu)
{
	int level = 0;
L
Linus Torvalds 已提交
5325

5326
	if (!sched_debug_enabled)
5327 5328
		return;

I
Ingo Molnar 已提交
5329 5330 5331 5332
	if (!sd) {
		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
		return;
	}
L
Linus Torvalds 已提交
5333

I
Ingo Molnar 已提交
5334 5335 5336
	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);

	for (;;) {
5337
		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
I
Ingo Molnar 已提交
5338
			break;
L
Linus Torvalds 已提交
5339 5340
		level++;
		sd = sd->parent;
5341
		if (!sd)
I
Ingo Molnar 已提交
5342 5343
			break;
	}
L
Linus Torvalds 已提交
5344
}
5345
#else /* !CONFIG_SCHED_DEBUG */
5346
# define sched_domain_debug(sd, cpu) do { } while (0)
5347 5348 5349 5350
static inline bool sched_debug(void)
{
	return false;
}
5351
#endif /* CONFIG_SCHED_DEBUG */
L
Linus Torvalds 已提交
5352

5353
static int sd_degenerate(struct sched_domain *sd)
5354
{
5355
	if (cpumask_weight(sched_domain_span(sd)) == 1)
5356 5357 5358 5359 5360 5361
		return 1;

	/* Following flags need at least 2 groups */
	if (sd->flags & (SD_LOAD_BALANCE |
			 SD_BALANCE_NEWIDLE |
			 SD_BALANCE_FORK |
5362
			 SD_BALANCE_EXEC |
5363
			 SD_SHARE_CPUCAPACITY |
5364 5365
			 SD_SHARE_PKG_RESOURCES |
			 SD_SHARE_POWERDOMAIN)) {
5366 5367 5368 5369 5370
		if (sd->groups != sd->groups->next)
			return 0;
	}

	/* Following flags don't use groups */
5371
	if (sd->flags & (SD_WAKE_AFFINE))
5372 5373 5374 5375 5376
		return 0;

	return 1;
}

5377 5378
static int
sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5379 5380 5381 5382 5383 5384
{
	unsigned long cflags = sd->flags, pflags = parent->flags;

	if (sd_degenerate(parent))
		return 1;

5385
	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
5386 5387 5388 5389 5390 5391 5392
		return 0;

	/* Flags needing groups don't count if only 1 group in parent */
	if (parent->groups == parent->groups->next) {
		pflags &= ~(SD_LOAD_BALANCE |
				SD_BALANCE_NEWIDLE |
				SD_BALANCE_FORK |
5393
				SD_BALANCE_EXEC |
5394
				SD_SHARE_CPUCAPACITY |
5395
				SD_SHARE_PKG_RESOURCES |
5396 5397
				SD_PREFER_SIBLING |
				SD_SHARE_POWERDOMAIN);
5398 5399
		if (nr_node_ids == 1)
			pflags &= ~SD_SERIALIZE;
5400 5401 5402 5403 5404 5405 5406
	}
	if (~cflags & pflags)
		return 0;

	return 1;
}

5407
static void free_rootdomain(struct rcu_head *rcu)
5408
{
5409
	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
5410

5411
	cpupri_cleanup(&rd->cpupri);
5412
	cpudl_cleanup(&rd->cpudl);
5413
	free_cpumask_var(rd->dlo_mask);
5414 5415 5416 5417 5418 5419
	free_cpumask_var(rd->rto_mask);
	free_cpumask_var(rd->online);
	free_cpumask_var(rd->span);
	kfree(rd);
}

G
Gregory Haskins 已提交
5420 5421
static void rq_attach_root(struct rq *rq, struct root_domain *rd)
{
I
Ingo Molnar 已提交
5422
	struct root_domain *old_rd = NULL;
G
Gregory Haskins 已提交
5423 5424
	unsigned long flags;

5425
	raw_spin_lock_irqsave(&rq->lock, flags);
G
Gregory Haskins 已提交
5426 5427

	if (rq->rd) {
I
Ingo Molnar 已提交
5428
		old_rd = rq->rd;
G
Gregory Haskins 已提交
5429

5430
		if (cpumask_test_cpu(rq->cpu, old_rd->online))
5431
			set_rq_offline(rq);
G
Gregory Haskins 已提交
5432

5433
		cpumask_clear_cpu(rq->cpu, old_rd->span);
5434

I
Ingo Molnar 已提交
5435
		/*
5436
		 * If we dont want to free the old_rd yet then
I
Ingo Molnar 已提交
5437 5438 5439 5440 5441
		 * set old_rd to NULL to skip the freeing later
		 * in this function:
		 */
		if (!atomic_dec_and_test(&old_rd->refcount))
			old_rd = NULL;
G
Gregory Haskins 已提交
5442 5443 5444 5445 5446
	}

	atomic_inc(&rd->refcount);
	rq->rd = rd;

5447
	cpumask_set_cpu(rq->cpu, rd->span);
5448
	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
5449
		set_rq_online(rq);
G
Gregory Haskins 已提交
5450

5451
	raw_spin_unlock_irqrestore(&rq->lock, flags);
I
Ingo Molnar 已提交
5452 5453

	if (old_rd)
5454
		call_rcu_sched(&old_rd->rcu, free_rootdomain);
G
Gregory Haskins 已提交
5455 5456
}

5457
static int init_rootdomain(struct root_domain *rd)
G
Gregory Haskins 已提交
5458 5459 5460
{
	memset(rd, 0, sizeof(*rd));

5461
	if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
5462
		goto out;
5463
	if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
5464
		goto free_span;
5465
	if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
5466
		goto free_online;
5467 5468
	if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
		goto free_dlo_mask;
5469

5470
	init_dl_bw(&rd->dl_bw);
5471 5472
	if (cpudl_init(&rd->cpudl) != 0)
		goto free_dlo_mask;
5473

5474
	if (cpupri_init(&rd->cpupri) != 0)
5475
		goto free_rto_mask;
5476
	return 0;
5477

5478 5479
free_rto_mask:
	free_cpumask_var(rd->rto_mask);
5480 5481
free_dlo_mask:
	free_cpumask_var(rd->dlo_mask);
5482 5483 5484 5485
free_online:
	free_cpumask_var(rd->online);
free_span:
	free_cpumask_var(rd->span);
5486
out:
5487
	return -ENOMEM;
G
Gregory Haskins 已提交
5488 5489
}

5490 5491 5492 5493 5494 5495
/*
 * By default the system creates a single root-domain with all cpus as
 * members (mimicking the global state we have today).
 */
struct root_domain def_root_domain;

G
Gregory Haskins 已提交
5496 5497
static void init_defrootdomain(void)
{
5498
	init_rootdomain(&def_root_domain);
5499

G
Gregory Haskins 已提交
5500 5501 5502
	atomic_set(&def_root_domain.refcount, 1);
}

5503
static struct root_domain *alloc_rootdomain(void)
G
Gregory Haskins 已提交
5504 5505 5506 5507 5508 5509 5510
{
	struct root_domain *rd;

	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
	if (!rd)
		return NULL;

5511
	if (init_rootdomain(rd) != 0) {
5512 5513 5514
		kfree(rd);
		return NULL;
	}
G
Gregory Haskins 已提交
5515 5516 5517 5518

	return rd;
}

5519
static void free_sched_groups(struct sched_group *sg, int free_sgc)
5520 5521 5522 5523 5524 5525 5526 5527 5528 5529
{
	struct sched_group *tmp, *first;

	if (!sg)
		return;

	first = sg;
	do {
		tmp = sg->next;

5530 5531
		if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
			kfree(sg->sgc);
5532 5533 5534 5535 5536 5537

		kfree(sg);
		sg = tmp;
	} while (sg != first);
}

5538 5539 5540
static void free_sched_domain(struct rcu_head *rcu)
{
	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
5541 5542 5543 5544 5545 5546 5547 5548

	/*
	 * If its an overlapping domain it has private groups, iterate and
	 * nuke them all.
	 */
	if (sd->flags & SD_OVERLAP) {
		free_sched_groups(sd->groups, 1);
	} else if (atomic_dec_and_test(&sd->groups->ref)) {
5549
		kfree(sd->groups->sgc);
5550
		kfree(sd->groups);
5551
	}
5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565
	kfree(sd);
}

static void destroy_sched_domain(struct sched_domain *sd, int cpu)
{
	call_rcu(&sd->rcu, free_sched_domain);
}

static void destroy_sched_domains(struct sched_domain *sd, int cpu)
{
	for (; sd; sd = sd->parent)
		destroy_sched_domain(sd, cpu);
}

5566 5567 5568 5569 5570 5571 5572
/*
 * Keep a special pointer to the highest sched_domain that has
 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
 * allows us to avoid some pointer chasing select_idle_sibling().
 *
 * Also keep a unique ID per domain (we use the first cpu number in
 * the cpumask of the domain), this allows us to quickly tell if
5573
 * two cpus are in the same cache domain, see cpus_share_cache().
5574 5575
 */
DEFINE_PER_CPU(struct sched_domain *, sd_llc);
5576
DEFINE_PER_CPU(int, sd_llc_size);
5577
DEFINE_PER_CPU(int, sd_llc_id);
5578
DEFINE_PER_CPU(struct sched_domain *, sd_numa);
5579 5580
DEFINE_PER_CPU(struct sched_domain *, sd_busy);
DEFINE_PER_CPU(struct sched_domain *, sd_asym);
5581 5582 5583 5584

static void update_top_cache_domain(int cpu)
{
	struct sched_domain *sd;
5585
	struct sched_domain *busy_sd = NULL;
5586
	int id = cpu;
5587
	int size = 1;
5588 5589

	sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
5590
	if (sd) {
5591
		id = cpumask_first(sched_domain_span(sd));
5592
		size = cpumask_weight(sched_domain_span(sd));
5593
		busy_sd = sd->parent; /* sd_busy */
5594
	}
5595
	rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd);
5596 5597

	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
5598
	per_cpu(sd_llc_size, cpu) = size;
5599
	per_cpu(sd_llc_id, cpu) = id;
5600 5601 5602

	sd = lowest_flag_domain(cpu, SD_NUMA);
	rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
5603 5604 5605

	sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
	rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
5606 5607
}

L
Linus Torvalds 已提交
5608
/*
I
Ingo Molnar 已提交
5609
 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
L
Linus Torvalds 已提交
5610 5611
 * hold the hotplug lock.
 */
I
Ingo Molnar 已提交
5612 5613
static void
cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
L
Linus Torvalds 已提交
5614
{
5615
	struct rq *rq = cpu_rq(cpu);
5616 5617 5618
	struct sched_domain *tmp;

	/* Remove the sched domains which do not contribute to scheduling. */
5619
	for (tmp = sd; tmp; ) {
5620 5621 5622
		struct sched_domain *parent = tmp->parent;
		if (!parent)
			break;
5623

5624
		if (sd_parent_degenerate(tmp, parent)) {
5625
			tmp->parent = parent->parent;
5626 5627
			if (parent->parent)
				parent->parent->child = tmp;
5628 5629 5630 5631 5632 5633 5634
			/*
			 * Transfer SD_PREFER_SIBLING down in case of a
			 * degenerate parent; the spans match for this
			 * so the property transfers.
			 */
			if (parent->flags & SD_PREFER_SIBLING)
				tmp->flags |= SD_PREFER_SIBLING;
5635
			destroy_sched_domain(parent, cpu);
5636 5637
		} else
			tmp = tmp->parent;
5638 5639
	}

5640
	if (sd && sd_degenerate(sd)) {
5641
		tmp = sd;
5642
		sd = sd->parent;
5643
		destroy_sched_domain(tmp, cpu);
5644 5645 5646
		if (sd)
			sd->child = NULL;
	}
L
Linus Torvalds 已提交
5647

5648
	sched_domain_debug(sd, cpu);
L
Linus Torvalds 已提交
5649

G
Gregory Haskins 已提交
5650
	rq_attach_root(rq, rd);
5651
	tmp = rq->sd;
N
Nick Piggin 已提交
5652
	rcu_assign_pointer(rq->sd, sd);
5653
	destroy_sched_domains(tmp, cpu);
5654 5655

	update_top_cache_domain(cpu);
L
Linus Torvalds 已提交
5656 5657 5658
}

/* cpus with isolated domains */
5659
static cpumask_var_t cpu_isolated_map;
L
Linus Torvalds 已提交
5660 5661 5662 5663

/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
R
Rusty Russell 已提交
5664
	alloc_bootmem_cpumask_var(&cpu_isolated_map);
R
Rusty Russell 已提交
5665
	cpulist_parse(str, cpu_isolated_map);
L
Linus Torvalds 已提交
5666 5667 5668
	return 1;
}

I
Ingo Molnar 已提交
5669
__setup("isolcpus=", isolated_cpu_setup);
L
Linus Torvalds 已提交
5670

5671
struct s_data {
5672
	struct sched_domain ** __percpu sd;
5673 5674 5675
	struct root_domain	*rd;
};

5676 5677
enum s_alloc {
	sa_rootdomain,
5678
	sa_sd,
5679
	sa_sd_storage,
5680 5681 5682
	sa_none,
};

P
Peter Zijlstra 已提交
5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720
/*
 * Build an iteration mask that can exclude certain CPUs from the upwards
 * domain traversal.
 *
 * Asymmetric node setups can result in situations where the domain tree is of
 * unequal depth, make sure to skip domains that already cover the entire
 * range.
 *
 * In that case build_sched_domains() will have terminated the iteration early
 * and our sibling sd spans will be empty. Domains should always include the
 * cpu they're built on, so check that.
 *
 */
static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
{
	const struct cpumask *span = sched_domain_span(sd);
	struct sd_data *sdd = sd->private;
	struct sched_domain *sibling;
	int i;

	for_each_cpu(i, span) {
		sibling = *per_cpu_ptr(sdd->sd, i);
		if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
			continue;

		cpumask_set_cpu(i, sched_group_mask(sg));
	}
}

/*
 * Return the canonical balance cpu for this group, this is the first cpu
 * of this group that's also in the iteration mask.
 */
int group_balance_cpu(struct sched_group *sg)
{
	return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
}

5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738
static int
build_overlap_sched_groups(struct sched_domain *sd, int cpu)
{
	struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
	const struct cpumask *span = sched_domain_span(sd);
	struct cpumask *covered = sched_domains_tmpmask;
	struct sd_data *sdd = sd->private;
	struct sched_domain *child;
	int i;

	cpumask_clear(covered);

	for_each_cpu(i, span) {
		struct cpumask *sg_span;

		if (cpumask_test_cpu(i, covered))
			continue;

P
Peter Zijlstra 已提交
5739 5740 5741 5742 5743 5744
		child = *per_cpu_ptr(sdd->sd, i);

		/* See the comment near build_group_mask(). */
		if (!cpumask_test_cpu(i, sched_domain_span(child)))
			continue;

5745
		sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
5746
				GFP_KERNEL, cpu_to_node(cpu));
5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759

		if (!sg)
			goto fail;

		sg_span = sched_group_cpus(sg);
		if (child->child) {
			child = child->child;
			cpumask_copy(sg_span, sched_domain_span(child));
		} else
			cpumask_set_cpu(i, sg_span);

		cpumask_or(covered, covered, sg_span);

5760 5761
		sg->sgc = *per_cpu_ptr(sdd->sgc, i);
		if (atomic_inc_return(&sg->sgc->ref) == 1)
P
Peter Zijlstra 已提交
5762 5763
			build_group_mask(sd, sg);

5764
		/*
5765
		 * Initialize sgc->capacity such that even if we mess up the
5766 5767 5768
		 * domains and no possible iteration will get us here, we won't
		 * die on a /0 trap.
		 */
5769
		sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
5770
		sg->sgc->capacity_orig = sg->sgc->capacity;
5771

P
Peter Zijlstra 已提交
5772 5773 5774 5775 5776
		/*
		 * Make sure the first group of this domain contains the
		 * canonical balance cpu. Otherwise the sched_domain iteration
		 * breaks. See update_sg_lb_stats().
		 */
P
Peter Zijlstra 已提交
5777
		if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
P
Peter Zijlstra 已提交
5778
		    group_balance_cpu(sg) == cpu)
5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797
			groups = sg;

		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
		last->next = first;
	}
	sd->groups = groups;

	return 0;

fail:
	free_sched_groups(first, 0);

	return -ENOMEM;
}

5798
static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
L
Linus Torvalds 已提交
5799
{
5800 5801
	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
	struct sched_domain *child = sd->child;
L
Linus Torvalds 已提交
5802

5803 5804
	if (child)
		cpu = cpumask_first(sched_domain_span(child));
5805

5806
	if (sg) {
5807
		*sg = *per_cpu_ptr(sdd->sg, cpu);
5808 5809
		(*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu);
		atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */
5810
	}
5811 5812

	return cpu;
5813 5814
}

5815
/*
5816 5817
 * build_sched_groups will build a circular linked list of the groups
 * covered by the given span, and will set each group's ->cpumask correctly,
5818
 * and ->cpu_capacity to 0.
5819 5820
 *
 * Assumes the sched_domain tree is fully constructed
5821
 */
5822 5823
static int
build_sched_groups(struct sched_domain *sd, int cpu)
L
Linus Torvalds 已提交
5824
{
5825 5826 5827
	struct sched_group *first = NULL, *last = NULL;
	struct sd_data *sdd = sd->private;
	const struct cpumask *span = sched_domain_span(sd);
5828
	struct cpumask *covered;
5829
	int i;
5830

5831 5832 5833
	get_group(cpu, sdd, &sd->groups);
	atomic_inc(&sd->groups->ref);

5834
	if (cpu != cpumask_first(span))
5835 5836
		return 0;

5837 5838 5839
	lockdep_assert_held(&sched_domains_mutex);
	covered = sched_domains_tmpmask;

5840
	cpumask_clear(covered);
5841

5842 5843
	for_each_cpu(i, span) {
		struct sched_group *sg;
5844
		int group, j;
5845

5846 5847
		if (cpumask_test_cpu(i, covered))
			continue;
5848

5849
		group = get_group(i, sdd, &sg);
P
Peter Zijlstra 已提交
5850
		cpumask_setall(sched_group_mask(sg));
5851

5852 5853 5854
		for_each_cpu(j, span) {
			if (get_group(j, sdd, NULL) != group)
				continue;
5855

5856 5857 5858
			cpumask_set_cpu(j, covered);
			cpumask_set_cpu(j, sched_group_cpus(sg));
		}
5859

5860 5861 5862 5863 5864 5865 5866
		if (!first)
			first = sg;
		if (last)
			last->next = sg;
		last = sg;
	}
	last->next = first;
5867 5868

	return 0;
5869
}
5870

5871
/*
5872
 * Initialize sched groups cpu_capacity.
5873
 *
5874
 * cpu_capacity indicates the capacity of sched group, which is used while
5875
 * distributing the load between different sched groups in a sched domain.
5876 5877 5878 5879
 * Typically cpu_capacity for all the groups in a sched domain will be same
 * unless there are asymmetries in the topology. If there are asymmetries,
 * group having more cpu_capacity will pickup more load compared to the
 * group having less cpu_capacity.
5880
 */
5881
static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
5882
{
5883
	struct sched_group *sg = sd->groups;
5884

5885
	WARN_ON(!sg);
5886 5887 5888 5889 5890

	do {
		sg->group_weight = cpumask_weight(sched_group_cpus(sg));
		sg = sg->next;
	} while (sg != sd->groups);
5891

P
Peter Zijlstra 已提交
5892
	if (cpu != group_balance_cpu(sg))
5893
		return;
5894

5895 5896
	update_group_capacity(sd, cpu);
	atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight);
5897 5898
}

5899 5900 5901 5902 5903
/*
 * Initializers for schedule domains
 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
 */

5904
static int default_relax_domain_level = -1;
5905
int sched_domain_level_max;
5906 5907 5908

static int __init setup_relax_domain_level(char *str)
{
5909 5910
	if (kstrtoint(str, 0, &default_relax_domain_level))
		pr_warn("Unable to set relax_domain_level\n");
5911

5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929
	return 1;
}
__setup("relax_domain_level=", setup_relax_domain_level);

static void set_domain_attribute(struct sched_domain *sd,
				 struct sched_domain_attr *attr)
{
	int request;

	if (!attr || attr->relax_domain_level < 0) {
		if (default_relax_domain_level < 0)
			return;
		else
			request = default_relax_domain_level;
	} else
		request = attr->relax_domain_level;
	if (request < sd->level) {
		/* turn off idle balance on this domain */
5930
		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
5931 5932
	} else {
		/* turn on idle balance on this domain */
5933
		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
5934 5935 5936
	}
}

5937 5938 5939
static void __sdt_free(const struct cpumask *cpu_map);
static int __sdt_alloc(const struct cpumask *cpu_map);

5940 5941 5942 5943 5944
static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
				 const struct cpumask *cpu_map)
{
	switch (what) {
	case sa_rootdomain:
5945 5946
		if (!atomic_read(&d->rd->refcount))
			free_rootdomain(&d->rd->rcu); /* fall through */
5947 5948
	case sa_sd:
		free_percpu(d->sd); /* fall through */
5949
	case sa_sd_storage:
5950
		__sdt_free(cpu_map); /* fall through */
5951 5952 5953 5954
	case sa_none:
		break;
	}
}
5955

5956 5957 5958
static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
						   const struct cpumask *cpu_map)
{
5959 5960
	memset(d, 0, sizeof(*d));

5961 5962
	if (__sdt_alloc(cpu_map))
		return sa_sd_storage;
5963 5964 5965
	d->sd = alloc_percpu(struct sched_domain *);
	if (!d->sd)
		return sa_sd_storage;
5966
	d->rd = alloc_rootdomain();
5967
	if (!d->rd)
5968
		return sa_sd;
5969 5970
	return sa_rootdomain;
}
G
Gregory Haskins 已提交
5971

5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983
/*
 * NULL the sd_data elements we've used to build the sched_domain and
 * sched_group structure so that the subsequent __free_domain_allocs()
 * will not free the data we're using.
 */
static void claim_allocations(int cpu, struct sched_domain *sd)
{
	struct sd_data *sdd = sd->private;

	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
	*per_cpu_ptr(sdd->sd, cpu) = NULL;

5984
	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
5985
		*per_cpu_ptr(sdd->sg, cpu) = NULL;
5986

5987 5988
	if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
		*per_cpu_ptr(sdd->sgc, cpu) = NULL;
5989 5990
}

5991 5992 5993 5994 5995
#ifdef CONFIG_NUMA
static int sched_domains_numa_levels;
static int *sched_domains_numa_distance;
static struct cpumask ***sched_domains_numa_masks;
static int sched_domains_curr_level;
5996
#endif
5997

5998 5999 6000
/*
 * SD_flags allowed in topology descriptions.
 *
6001
 * SD_SHARE_CPUCAPACITY      - describes SMT topologies
6002 6003
 * SD_SHARE_PKG_RESOURCES - describes shared caches
 * SD_NUMA                - describes NUMA topologies
6004
 * SD_SHARE_POWERDOMAIN   - describes shared power domain
6005 6006 6007 6008 6009
 *
 * Odd one out:
 * SD_ASYM_PACKING        - describes SMT quirks
 */
#define TOPOLOGY_SD_FLAGS		\
6010
	(SD_SHARE_CPUCAPACITY |		\
6011 6012
	 SD_SHARE_PKG_RESOURCES |	\
	 SD_NUMA |			\
6013 6014
	 SD_ASYM_PACKING |		\
	 SD_SHARE_POWERDOMAIN)
6015 6016

static struct sched_domain *
6017
sd_init(struct sched_domain_topology_level *tl, int cpu)
6018 6019
{
	struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035
	int sd_weight, sd_flags = 0;

#ifdef CONFIG_NUMA
	/*
	 * Ugly hack to pass state to sd_numa_mask()...
	 */
	sched_domains_curr_level = tl->numa_level;
#endif

	sd_weight = cpumask_weight(tl->mask(cpu));

	if (tl->sd_flags)
		sd_flags = (*tl->sd_flags)();
	if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
			"wrong sd_flags in topology description\n"))
		sd_flags &= ~TOPOLOGY_SD_FLAGS;
6036 6037 6038 6039 6040

	*sd = (struct sched_domain){
		.min_interval		= sd_weight,
		.max_interval		= 2*sd_weight,
		.busy_factor		= 32,
6041
		.imbalance_pct		= 125,
6042 6043 6044 6045

		.cache_nice_tries	= 0,
		.busy_idx		= 0,
		.idle_idx		= 0,
6046 6047 6048 6049 6050 6051
		.newidle_idx		= 0,
		.wake_idx		= 0,
		.forkexec_idx		= 0,

		.flags			= 1*SD_LOAD_BALANCE
					| 1*SD_BALANCE_NEWIDLE
6052 6053
					| 1*SD_BALANCE_EXEC
					| 1*SD_BALANCE_FORK
6054
					| 0*SD_BALANCE_WAKE
6055
					| 1*SD_WAKE_AFFINE
6056
					| 0*SD_SHARE_CPUCAPACITY
6057
					| 0*SD_SHARE_PKG_RESOURCES
6058
					| 0*SD_SERIALIZE
6059
					| 0*SD_PREFER_SIBLING
6060 6061
					| 0*SD_NUMA
					| sd_flags
6062
					,
6063

6064 6065
		.last_balance		= jiffies,
		.balance_interval	= sd_weight,
6066
		.smt_gain		= 0,
6067 6068
		.max_newidle_lb_cost	= 0,
		.next_decay_max_lb_cost	= jiffies,
6069 6070 6071
#ifdef CONFIG_SCHED_DEBUG
		.name			= tl->name,
#endif
6072 6073 6074
	};

	/*
6075
	 * Convert topological properties into behaviour.
6076
	 */
6077

6078
	if (sd->flags & SD_SHARE_CPUCAPACITY) {
6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108
		sd->imbalance_pct = 110;
		sd->smt_gain = 1178; /* ~15% */

	} else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
		sd->imbalance_pct = 117;
		sd->cache_nice_tries = 1;
		sd->busy_idx = 2;

#ifdef CONFIG_NUMA
	} else if (sd->flags & SD_NUMA) {
		sd->cache_nice_tries = 2;
		sd->busy_idx = 3;
		sd->idle_idx = 2;

		sd->flags |= SD_SERIALIZE;
		if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
			sd->flags &= ~(SD_BALANCE_EXEC |
				       SD_BALANCE_FORK |
				       SD_WAKE_AFFINE);
		}

#endif
	} else {
		sd->flags |= SD_PREFER_SIBLING;
		sd->cache_nice_tries = 1;
		sd->busy_idx = 2;
		sd->idle_idx = 1;
	}

	sd->private = &tl->data;
6109 6110 6111 6112

	return sd;
}

6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138
/*
 * Topology list, bottom-up.
 */
static struct sched_domain_topology_level default_topology[] = {
#ifdef CONFIG_SCHED_SMT
	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
#endif
#ifdef CONFIG_SCHED_MC
	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
#endif
	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
	{ NULL, },
};

struct sched_domain_topology_level *sched_domain_topology = default_topology;

#define for_each_sd_topology(tl)			\
	for (tl = sched_domain_topology; tl->mask; tl++)

void set_sched_topology(struct sched_domain_topology_level *tl)
{
	sched_domain_topology = tl;
}

#ifdef CONFIG_NUMA

6139 6140 6141 6142 6143
static const struct cpumask *sd_numa_mask(int cpu)
{
	return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
}

6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179
static void sched_numa_warn(const char *str)
{
	static int done = false;
	int i,j;

	if (done)
		return;

	done = true;

	printk(KERN_WARNING "ERROR: %s\n\n", str);

	for (i = 0; i < nr_node_ids; i++) {
		printk(KERN_WARNING "  ");
		for (j = 0; j < nr_node_ids; j++)
			printk(KERN_CONT "%02d ", node_distance(i,j));
		printk(KERN_CONT "\n");
	}
	printk(KERN_WARNING "\n");
}

static bool find_numa_distance(int distance)
{
	int i;

	if (distance == node_distance(0, 0))
		return true;

	for (i = 0; i < sched_domains_numa_levels; i++) {
		if (sched_domains_numa_distance[i] == distance)
			return true;
	}

	return false;
}

6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200
static void sched_init_numa(void)
{
	int next_distance, curr_distance = node_distance(0, 0);
	struct sched_domain_topology_level *tl;
	int level = 0;
	int i, j, k;

	sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
	if (!sched_domains_numa_distance)
		return;

	/*
	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
	 * unique distances in the node_distance() table.
	 *
	 * Assumes node_distance(0,j) includes all distances in
	 * node_distance(i,j) in order to avoid cubic time.
	 */
	next_distance = curr_distance;
	for (i = 0; i < nr_node_ids; i++) {
		for (j = 0; j < nr_node_ids; j++) {
6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224
			for (k = 0; k < nr_node_ids; k++) {
				int distance = node_distance(i, k);

				if (distance > curr_distance &&
				    (distance < next_distance ||
				     next_distance == curr_distance))
					next_distance = distance;

				/*
				 * While not a strong assumption it would be nice to know
				 * about cases where if node A is connected to B, B is not
				 * equally connected to A.
				 */
				if (sched_debug() && node_distance(k, i) != distance)
					sched_numa_warn("Node-distance not symmetric");

				if (sched_debug() && i && !find_numa_distance(distance))
					sched_numa_warn("Node-0 not representative");
			}
			if (next_distance != curr_distance) {
				sched_domains_numa_distance[level++] = next_distance;
				sched_domains_numa_levels = level;
				curr_distance = next_distance;
			} else break;
6225
		}
6226 6227 6228 6229 6230 6231

		/*
		 * In case of sched_debug() we verify the above assumption.
		 */
		if (!sched_debug())
			break;
6232 6233 6234 6235 6236
	}
	/*
	 * 'level' contains the number of unique distances, excluding the
	 * identity distance node_distance(i,i).
	 *
V
Viresh Kumar 已提交
6237
	 * The sched_domains_numa_distance[] array includes the actual distance
6238 6239 6240
	 * numbers.
	 */

6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251
	/*
	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
	 * the array will contain less then 'level' members. This could be
	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
	 * in other functions.
	 *
	 * We reset it to 'level' at the end of this function.
	 */
	sched_domains_numa_levels = 0;

6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266
	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
	if (!sched_domains_numa_masks)
		return;

	/*
	 * Now for each level, construct a mask per node which contains all
	 * cpus of nodes that are that many hops away from us.
	 */
	for (i = 0; i < level; i++) {
		sched_domains_numa_masks[i] =
			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
		if (!sched_domains_numa_masks[i])
			return;

		for (j = 0; j < nr_node_ids; j++) {
6267
			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
6268 6269 6270 6271 6272 6273
			if (!mask)
				return;

			sched_domains_numa_masks[i][j] = mask;

			for (k = 0; k < nr_node_ids; k++) {
6274
				if (node_distance(j, k) > sched_domains_numa_distance[i])
6275 6276 6277 6278 6279 6280 6281
					continue;

				cpumask_or(mask, mask, cpumask_of_node(k));
			}
		}
	}

6282 6283 6284
	/* Compute default topology size */
	for (i = 0; sched_domain_topology[i].mask; i++);

6285
	tl = kzalloc((i + level + 1) *
6286 6287 6288 6289 6290 6291 6292
			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
	if (!tl)
		return;

	/*
	 * Copy the default topology bits..
	 */
6293 6294
	for (i = 0; sched_domain_topology[i].mask; i++)
		tl[i] = sched_domain_topology[i];
6295 6296 6297 6298 6299 6300 6301

	/*
	 * .. and append 'j' levels of NUMA goodness.
	 */
	for (j = 0; j < level; i++, j++) {
		tl[i] = (struct sched_domain_topology_level){
			.mask = sd_numa_mask,
6302
			.sd_flags = cpu_numa_flags,
6303 6304
			.flags = SDTL_OVERLAP,
			.numa_level = j,
6305
			SD_INIT_NAME(NUMA)
6306 6307 6308 6309
		};
	}

	sched_domain_topology = tl;
6310 6311

	sched_domains_numa_levels = level;
6312
}
6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359

static void sched_domains_numa_masks_set(int cpu)
{
	int i, j;
	int node = cpu_to_node(cpu);

	for (i = 0; i < sched_domains_numa_levels; i++) {
		for (j = 0; j < nr_node_ids; j++) {
			if (node_distance(j, node) <= sched_domains_numa_distance[i])
				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
		}
	}
}

static void sched_domains_numa_masks_clear(int cpu)
{
	int i, j;
	for (i = 0; i < sched_domains_numa_levels; i++) {
		for (j = 0; j < nr_node_ids; j++)
			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
	}
}

/*
 * Update sched_domains_numa_masks[level][node] array when new cpus
 * are onlined.
 */
static int sched_domains_numa_masks_update(struct notifier_block *nfb,
					   unsigned long action,
					   void *hcpu)
{
	int cpu = (long)hcpu;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_ONLINE:
		sched_domains_numa_masks_set(cpu);
		break;

	case CPU_DEAD:
		sched_domains_numa_masks_clear(cpu);
		break;

	default:
		return NOTIFY_DONE;
	}

	return NOTIFY_OK;
6360 6361 6362 6363 6364
}
#else
static inline void sched_init_numa(void)
{
}
6365 6366 6367 6368 6369 6370 6371

static int sched_domains_numa_masks_update(struct notifier_block *nfb,
					   unsigned long action,
					   void *hcpu)
{
	return 0;
}
6372 6373
#endif /* CONFIG_NUMA */

6374 6375 6376 6377 6378
static int __sdt_alloc(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

6379
	for_each_sd_topology(tl) {
6380 6381 6382 6383 6384 6385 6386 6387 6388 6389
		struct sd_data *sdd = &tl->data;

		sdd->sd = alloc_percpu(struct sched_domain *);
		if (!sdd->sd)
			return -ENOMEM;

		sdd->sg = alloc_percpu(struct sched_group *);
		if (!sdd->sg)
			return -ENOMEM;

6390 6391
		sdd->sgc = alloc_percpu(struct sched_group_capacity *);
		if (!sdd->sgc)
6392 6393
			return -ENOMEM;

6394 6395 6396
		for_each_cpu(j, cpu_map) {
			struct sched_domain *sd;
			struct sched_group *sg;
6397
			struct sched_group_capacity *sgc;
6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410

		       	sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sd)
				return -ENOMEM;

			*per_cpu_ptr(sdd->sd, j) = sd;

			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
					GFP_KERNEL, cpu_to_node(j));
			if (!sg)
				return -ENOMEM;

6411 6412
			sg->next = sg;

6413
			*per_cpu_ptr(sdd->sg, j) = sg;
6414

6415
			sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
6416
					GFP_KERNEL, cpu_to_node(j));
6417
			if (!sgc)
6418 6419
				return -ENOMEM;

6420
			*per_cpu_ptr(sdd->sgc, j) = sgc;
6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431
		}
	}

	return 0;
}

static void __sdt_free(const struct cpumask *cpu_map)
{
	struct sched_domain_topology_level *tl;
	int j;

6432
	for_each_sd_topology(tl) {
6433 6434 6435
		struct sd_data *sdd = &tl->data;

		for_each_cpu(j, cpu_map) {
6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446
			struct sched_domain *sd;

			if (sdd->sd) {
				sd = *per_cpu_ptr(sdd->sd, j);
				if (sd && (sd->flags & SD_OVERLAP))
					free_sched_groups(sd->groups, 0);
				kfree(*per_cpu_ptr(sdd->sd, j));
			}

			if (sdd->sg)
				kfree(*per_cpu_ptr(sdd->sg, j));
6447 6448
			if (sdd->sgc)
				kfree(*per_cpu_ptr(sdd->sgc, j));
6449 6450
		}
		free_percpu(sdd->sd);
6451
		sdd->sd = NULL;
6452
		free_percpu(sdd->sg);
6453
		sdd->sg = NULL;
6454 6455
		free_percpu(sdd->sgc);
		sdd->sgc = NULL;
6456 6457 6458
	}
}

6459
struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
6460 6461
		const struct cpumask *cpu_map, struct sched_domain_attr *attr,
		struct sched_domain *child, int cpu)
6462
{
6463
	struct sched_domain *sd = sd_init(tl, cpu);
6464
	if (!sd)
6465
		return child;
6466 6467

	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
6468 6469 6470
	if (child) {
		sd->level = child->level + 1;
		sched_domain_level_max = max(sched_domain_level_max, sd->level);
6471
		child->parent = sd;
6472
		sd->child = child;
6473
	}
6474
	set_domain_attribute(sd, attr);
6475 6476 6477 6478

	return sd;
}

6479 6480 6481 6482
/*
 * Build sched domains for a given set of cpus and attach the sched domains
 * to the individual cpus
 */
6483 6484
static int build_sched_domains(const struct cpumask *cpu_map,
			       struct sched_domain_attr *attr)
6485
{
6486
	enum s_alloc alloc_state;
6487
	struct sched_domain *sd;
6488
	struct s_data d;
6489
	int i, ret = -ENOMEM;
6490

6491 6492 6493
	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
	if (alloc_state != sa_rootdomain)
		goto error;
6494

6495
	/* Set up domains for cpus specified by the cpu_map. */
6496
	for_each_cpu(i, cpu_map) {
6497 6498
		struct sched_domain_topology_level *tl;

6499
		sd = NULL;
6500
		for_each_sd_topology(tl) {
6501
			sd = build_sched_domain(tl, cpu_map, attr, sd, i);
6502 6503
			if (tl == sched_domain_topology)
				*per_cpu_ptr(d.sd, i) = sd;
6504 6505
			if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
				sd->flags |= SD_OVERLAP;
6506 6507
			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
				break;
6508
		}
6509 6510 6511 6512 6513 6514
	}

	/* Build the groups for the domains */
	for_each_cpu(i, cpu_map) {
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			sd->span_weight = cpumask_weight(sched_domain_span(sd));
6515 6516 6517 6518 6519 6520 6521
			if (sd->flags & SD_OVERLAP) {
				if (build_overlap_sched_groups(sd, i))
					goto error;
			} else {
				if (build_sched_groups(sd, i))
					goto error;
			}
6522
		}
6523
	}
6524

6525
	/* Calculate CPU capacity for physical packages and nodes */
6526 6527 6528
	for (i = nr_cpumask_bits-1; i >= 0; i--) {
		if (!cpumask_test_cpu(i, cpu_map))
			continue;
6529

6530 6531
		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
			claim_allocations(i, sd);
6532
			init_sched_groups_capacity(i, sd);
6533
		}
6534
	}
6535

L
Linus Torvalds 已提交
6536
	/* Attach the domains */
6537
	rcu_read_lock();
6538
	for_each_cpu(i, cpu_map) {
6539
		sd = *per_cpu_ptr(d.sd, i);
6540
		cpu_attach_domain(sd, d.rd, i);
L
Linus Torvalds 已提交
6541
	}
6542
	rcu_read_unlock();
6543

6544
	ret = 0;
6545
error:
6546
	__free_domain_allocs(&d, alloc_state, cpu_map);
6547
	return ret;
L
Linus Torvalds 已提交
6548
}
P
Paul Jackson 已提交
6549

6550
static cpumask_var_t *doms_cur;	/* current sched domains */
P
Paul Jackson 已提交
6551
static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
I
Ingo Molnar 已提交
6552 6553
static struct sched_domain_attr *dattr_cur;
				/* attribues of custom domains in 'doms_cur' */
P
Paul Jackson 已提交
6554 6555 6556

/*
 * Special case: If a kmalloc of a doms_cur partition (array of
6557 6558
 * cpumask) fails, then fallback to a single sched domain,
 * as determined by the single cpumask fallback_doms.
P
Paul Jackson 已提交
6559
 */
6560
static cpumask_var_t fallback_doms;
P
Paul Jackson 已提交
6561

6562 6563 6564 6565 6566
/*
 * arch_update_cpu_topology lets virtualized architectures update the
 * cpu core maps. It is supposed to return 1 if the topology changed
 * or 0 if it stayed the same.
 */
6567
int __weak arch_update_cpu_topology(void)
6568
{
6569
	return 0;
6570 6571
}

6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596
cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
{
	int i;
	cpumask_var_t *doms;

	doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
	if (!doms)
		return NULL;
	for (i = 0; i < ndoms; i++) {
		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
			free_sched_domains(doms, i);
			return NULL;
		}
	}
	return doms;
}

void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
{
	unsigned int i;
	for (i = 0; i < ndoms; i++)
		free_cpumask_var(doms[i]);
	kfree(doms);
}

6597
/*
I
Ingo Molnar 已提交
6598
 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
P
Paul Jackson 已提交
6599 6600
 * For now this just excludes isolated cpus, but could be used to
 * exclude other special cases in the future.
6601
 */
6602
static int init_sched_domains(const struct cpumask *cpu_map)
6603
{
6604 6605
	int err;

6606
	arch_update_cpu_topology();
P
Paul Jackson 已提交
6607
	ndoms_cur = 1;
6608
	doms_cur = alloc_sched_domains(ndoms_cur);
P
Paul Jackson 已提交
6609
	if (!doms_cur)
6610 6611
		doms_cur = &fallback_doms;
	cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
6612
	err = build_sched_domains(doms_cur[0], NULL);
6613
	register_sched_domain_sysctl();
6614 6615

	return err;
6616 6617 6618 6619 6620 6621
}

/*
 * Detach sched domains from a group of cpus specified in cpu_map
 * These cpus will now be attached to the NULL domain
 */
6622
static void detach_destroy_domains(const struct cpumask *cpu_map)
6623 6624 6625
{
	int i;

6626
	rcu_read_lock();
6627
	for_each_cpu(i, cpu_map)
G
Gregory Haskins 已提交
6628
		cpu_attach_domain(NULL, &def_root_domain, i);
6629
	rcu_read_unlock();
6630 6631
}

6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647
/* handle null as "default" */
static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
			struct sched_domain_attr *new, int idx_new)
{
	struct sched_domain_attr tmp;

	/* fast path */
	if (!new && !cur)
		return 1;

	tmp = SD_ATTR_INIT;
	return !memcmp(cur ? (cur + idx_cur) : &tmp,
			new ? (new + idx_new) : &tmp,
			sizeof(struct sched_domain_attr));
}

P
Paul Jackson 已提交
6648 6649
/*
 * Partition sched domains as specified by the 'ndoms_new'
I
Ingo Molnar 已提交
6650
 * cpumasks in the array doms_new[] of cpumasks. This compares
P
Paul Jackson 已提交
6651 6652 6653
 * doms_new[] to the current sched domain partitioning, doms_cur[].
 * It destroys each deleted domain and builds each new domain.
 *
6654
 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
I
Ingo Molnar 已提交
6655 6656 6657
 * The masks don't intersect (don't overlap.) We should setup one
 * sched domain for each mask. CPUs not in any of the cpumasks will
 * not be load balanced. If the same cpumask appears both in the
P
Paul Jackson 已提交
6658 6659 6660
 * current 'doms_cur' domains and in the new 'doms_new', we can leave
 * it as it is.
 *
6661 6662 6663 6664 6665 6666
 * The passed in 'doms_new' should be allocated using
 * alloc_sched_domains.  This routine takes ownership of it and will
 * free_sched_domains it when done with it. If the caller failed the
 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
 * and partition_sched_domains() will fallback to the single partition
 * 'fallback_doms', it also forces the domains to be rebuilt.
P
Paul Jackson 已提交
6667
 *
6668
 * If doms_new == NULL it will be replaced with cpu_online_mask.
6669 6670
 * ndoms_new == 0 is a special case for destroying existing domains,
 * and it will not create the default domain.
6671
 *
P
Paul Jackson 已提交
6672 6673
 * Call with hotplug lock held
 */
6674
void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
6675
			     struct sched_domain_attr *dattr_new)
P
Paul Jackson 已提交
6676
{
6677
	int i, j, n;
6678
	int new_topology;
P
Paul Jackson 已提交
6679

6680
	mutex_lock(&sched_domains_mutex);
6681

6682 6683 6684
	/* always unregister in case we don't destroy any domains */
	unregister_sched_domain_sysctl();

6685 6686 6687
	/* Let architecture update cpu core mappings. */
	new_topology = arch_update_cpu_topology();

6688
	n = doms_new ? ndoms_new : 0;
P
Paul Jackson 已提交
6689 6690 6691

	/* Destroy deleted domains */
	for (i = 0; i < ndoms_cur; i++) {
6692
		for (j = 0; j < n && !new_topology; j++) {
6693
			if (cpumask_equal(doms_cur[i], doms_new[j])
6694
			    && dattrs_equal(dattr_cur, i, dattr_new, j))
P
Paul Jackson 已提交
6695 6696 6697
				goto match1;
		}
		/* no match - a current sched domain not in new doms_new[] */
6698
		detach_destroy_domains(doms_cur[i]);
P
Paul Jackson 已提交
6699 6700 6701 6702
match1:
		;
	}

6703
	n = ndoms_cur;
6704
	if (doms_new == NULL) {
6705
		n = 0;
6706
		doms_new = &fallback_doms;
6707
		cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
6708
		WARN_ON_ONCE(dattr_new);
6709 6710
	}

P
Paul Jackson 已提交
6711 6712
	/* Build new domains */
	for (i = 0; i < ndoms_new; i++) {
6713
		for (j = 0; j < n && !new_topology; j++) {
6714
			if (cpumask_equal(doms_new[i], doms_cur[j])
6715
			    && dattrs_equal(dattr_new, i, dattr_cur, j))
P
Paul Jackson 已提交
6716 6717 6718
				goto match2;
		}
		/* no match - add a new doms_new */
6719
		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
P
Paul Jackson 已提交
6720 6721 6722 6723 6724
match2:
		;
	}

	/* Remember the new sched domains */
6725 6726
	if (doms_cur != &fallback_doms)
		free_sched_domains(doms_cur, ndoms_cur);
6727
	kfree(dattr_cur);	/* kfree(NULL) is safe */
P
Paul Jackson 已提交
6728
	doms_cur = doms_new;
6729
	dattr_cur = dattr_new;
P
Paul Jackson 已提交
6730
	ndoms_cur = ndoms_new;
6731 6732

	register_sched_domain_sysctl();
6733

6734
	mutex_unlock(&sched_domains_mutex);
P
Paul Jackson 已提交
6735 6736
}

6737 6738
static int num_cpus_frozen;	/* used to mark begin/end of suspend/resume */

L
Linus Torvalds 已提交
6739
/*
6740 6741 6742
 * Update cpusets according to cpu_active mask.  If cpusets are
 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
 * around partition_sched_domains().
6743 6744 6745
 *
 * If we come here as part of a suspend/resume, don't touch cpusets because we
 * want to restore it back to its original state upon resume anyway.
L
Linus Torvalds 已提交
6746
 */
6747 6748
static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
			     void *hcpu)
6749
{
6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771
	switch (action) {
	case CPU_ONLINE_FROZEN:
	case CPU_DOWN_FAILED_FROZEN:

		/*
		 * num_cpus_frozen tracks how many CPUs are involved in suspend
		 * resume sequence. As long as this is not the last online
		 * operation in the resume sequence, just build a single sched
		 * domain, ignoring cpusets.
		 */
		num_cpus_frozen--;
		if (likely(num_cpus_frozen)) {
			partition_sched_domains(1, NULL, NULL);
			break;
		}

		/*
		 * This is the last CPU online operation. So fall through and
		 * restore the original sched domains by considering the
		 * cpuset configurations.
		 */

6772
	case CPU_ONLINE:
6773
	case CPU_DOWN_FAILED:
6774
		cpuset_update_active_cpus(true);
6775
		break;
6776 6777 6778
	default:
		return NOTIFY_DONE;
	}
6779
	return NOTIFY_OK;
6780
}
6781

6782 6783
static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
			       void *hcpu)
6784
{
6785
	switch (action) {
6786
	case CPU_DOWN_PREPARE:
6787
		cpuset_update_active_cpus(false);
6788 6789 6790 6791 6792
		break;
	case CPU_DOWN_PREPARE_FROZEN:
		num_cpus_frozen++;
		partition_sched_domains(1, NULL, NULL);
		break;
6793 6794 6795
	default:
		return NOTIFY_DONE;
	}
6796
	return NOTIFY_OK;
6797 6798
}

L
Linus Torvalds 已提交
6799 6800
void __init sched_init_smp(void)
{
6801 6802 6803
	cpumask_var_t non_isolated_cpus;

	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
6804
	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
6805

6806 6807
	sched_init_numa();

6808 6809 6810 6811 6812
	/*
	 * There's no userspace yet to cause hotplug operations; hence all the
	 * cpu masks are stable and all blatant races in the below code cannot
	 * happen.
	 */
6813
	mutex_lock(&sched_domains_mutex);
6814
	init_sched_domains(cpu_active_mask);
6815 6816 6817
	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
	if (cpumask_empty(non_isolated_cpus))
		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
6818
	mutex_unlock(&sched_domains_mutex);
6819

6820
	hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
6821 6822
	hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
	hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
6823

6824
	init_hrtick();
6825 6826

	/* Move init over to a non-isolated CPU */
6827
	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
6828
		BUG();
I
Ingo Molnar 已提交
6829
	sched_init_granularity();
6830
	free_cpumask_var(non_isolated_cpus);
6831

6832
	init_sched_rt_class();
6833
	init_sched_dl_class();
L
Linus Torvalds 已提交
6834 6835 6836 6837
}
#else
void __init sched_init_smp(void)
{
I
Ingo Molnar 已提交
6838
	sched_init_granularity();
L
Linus Torvalds 已提交
6839 6840 6841
}
#endif /* CONFIG_SMP */

6842 6843
const_debug unsigned int sysctl_timer_migration = 1;

L
Linus Torvalds 已提交
6844 6845 6846 6847 6848 6849 6850
int in_sched_functions(unsigned long addr)
{
	return in_lock_functions(addr) ||
		(addr >= (unsigned long)__sched_text_start
		&& addr < (unsigned long)__sched_text_end);
}

6851
#ifdef CONFIG_CGROUP_SCHED
6852 6853 6854 6855
/*
 * Default task group.
 * Every task in system belongs to this group at bootup.
 */
6856
struct task_group root_task_group;
6857
LIST_HEAD(task_groups);
6858
#endif
P
Peter Zijlstra 已提交
6859

6860
DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
P
Peter Zijlstra 已提交
6861

L
Linus Torvalds 已提交
6862 6863
void __init sched_init(void)
{
I
Ingo Molnar 已提交
6864
	int i, j;
6865 6866 6867 6868 6869 6870 6871
	unsigned long alloc_size = 0, ptr;

#ifdef CONFIG_FAIR_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
#ifdef CONFIG_RT_GROUP_SCHED
	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
6872
#endif
6873
#ifdef CONFIG_CPUMASK_OFFSTACK
6874
	alloc_size += num_possible_cpus() * cpumask_size();
6875 6876
#endif
	if (alloc_size) {
6877
		ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
6878 6879

#ifdef CONFIG_FAIR_GROUP_SCHED
6880
		root_task_group.se = (struct sched_entity **)ptr;
6881 6882
		ptr += nr_cpu_ids * sizeof(void **);

6883
		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
6884
		ptr += nr_cpu_ids * sizeof(void **);
6885

6886
#endif /* CONFIG_FAIR_GROUP_SCHED */
6887
#ifdef CONFIG_RT_GROUP_SCHED
6888
		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
6889 6890
		ptr += nr_cpu_ids * sizeof(void **);

6891
		root_task_group.rt_rq = (struct rt_rq **)ptr;
6892 6893
		ptr += nr_cpu_ids * sizeof(void **);

6894
#endif /* CONFIG_RT_GROUP_SCHED */
6895 6896
#ifdef CONFIG_CPUMASK_OFFSTACK
		for_each_possible_cpu(i) {
6897
			per_cpu(load_balance_mask, i) = (void *)ptr;
6898 6899 6900
			ptr += cpumask_size();
		}
#endif /* CONFIG_CPUMASK_OFFSTACK */
6901
	}
I
Ingo Molnar 已提交
6902

6903 6904 6905
	init_rt_bandwidth(&def_rt_bandwidth,
			global_rt_period(), global_rt_runtime());
	init_dl_bandwidth(&def_dl_bandwidth,
6906
			global_rt_period(), global_rt_runtime());
6907

G
Gregory Haskins 已提交
6908 6909 6910 6911
#ifdef CONFIG_SMP
	init_defrootdomain();
#endif

6912
#ifdef CONFIG_RT_GROUP_SCHED
6913
	init_rt_bandwidth(&root_task_group.rt_bandwidth,
6914
			global_rt_period(), global_rt_runtime());
6915
#endif /* CONFIG_RT_GROUP_SCHED */
6916

D
Dhaval Giani 已提交
6917
#ifdef CONFIG_CGROUP_SCHED
6918 6919
	list_add(&root_task_group.list, &task_groups);
	INIT_LIST_HEAD(&root_task_group.children);
6920
	INIT_LIST_HEAD(&root_task_group.siblings);
6921
	autogroup_init(&init_task);
6922

D
Dhaval Giani 已提交
6923
#endif /* CONFIG_CGROUP_SCHED */
P
Peter Zijlstra 已提交
6924

6925
	for_each_possible_cpu(i) {
6926
		struct rq *rq;
L
Linus Torvalds 已提交
6927 6928

		rq = cpu_rq(i);
6929
		raw_spin_lock_init(&rq->lock);
N
Nick Piggin 已提交
6930
		rq->nr_running = 0;
6931 6932
		rq->calc_load_active = 0;
		rq->calc_load_update = jiffies + LOAD_FREQ;
6933
		init_cfs_rq(&rq->cfs);
P
Peter Zijlstra 已提交
6934
		init_rt_rq(&rq->rt, rq);
6935
		init_dl_rq(&rq->dl, rq);
I
Ingo Molnar 已提交
6936
#ifdef CONFIG_FAIR_GROUP_SCHED
6937
		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
P
Peter Zijlstra 已提交
6938
		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
D
Dhaval Giani 已提交
6939
		/*
6940
		 * How much cpu bandwidth does root_task_group get?
D
Dhaval Giani 已提交
6941 6942 6943 6944
		 *
		 * In case of task-groups formed thr' the cgroup filesystem, it
		 * gets 100% of the cpu resources in the system. This overall
		 * system cpu resource is divided among the tasks of
6945
		 * root_task_group and its child task-groups in a fair manner,
D
Dhaval Giani 已提交
6946 6947 6948
		 * based on each entity's (task or task-group's) weight
		 * (se->load.weight).
		 *
6949
		 * In other words, if root_task_group has 10 tasks of weight
D
Dhaval Giani 已提交
6950 6951 6952
		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
		 * then A0's share of the cpu resource is:
		 *
6953
		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
D
Dhaval Giani 已提交
6954
		 *
6955 6956
		 * We achieve this by letting root_task_group's tasks sit
		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
D
Dhaval Giani 已提交
6957
		 */
6958
		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
6959
		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
D
Dhaval Giani 已提交
6960 6961 6962
#endif /* CONFIG_FAIR_GROUP_SCHED */

		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
6963
#ifdef CONFIG_RT_GROUP_SCHED
6964
		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
I
Ingo Molnar 已提交
6965
#endif
L
Linus Torvalds 已提交
6966

I
Ingo Molnar 已提交
6967 6968
		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
			rq->cpu_load[j] = 0;
6969 6970 6971

		rq->last_load_update_tick = jiffies;

L
Linus Torvalds 已提交
6972
#ifdef CONFIG_SMP
N
Nick Piggin 已提交
6973
		rq->sd = NULL;
G
Gregory Haskins 已提交
6974
		rq->rd = NULL;
6975
		rq->cpu_capacity = SCHED_CAPACITY_SCALE;
6976
		rq->post_schedule = 0;
L
Linus Torvalds 已提交
6977
		rq->active_balance = 0;
I
Ingo Molnar 已提交
6978
		rq->next_balance = jiffies;
L
Linus Torvalds 已提交
6979
		rq->push_cpu = 0;
6980
		rq->cpu = i;
6981
		rq->online = 0;
6982 6983
		rq->idle_stamp = 0;
		rq->avg_idle = 2*sysctl_sched_migration_cost;
6984
		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
6985 6986 6987

		INIT_LIST_HEAD(&rq->cfs_tasks);

6988
		rq_attach_root(rq, &def_root_domain);
6989
#ifdef CONFIG_NO_HZ_COMMON
6990
		rq->nohz_flags = 0;
6991
#endif
6992 6993 6994
#ifdef CONFIG_NO_HZ_FULL
		rq->last_sched_tick = 0;
#endif
L
Linus Torvalds 已提交
6995
#endif
P
Peter Zijlstra 已提交
6996
		init_rq_hrtick(rq);
L
Linus Torvalds 已提交
6997 6998 6999
		atomic_set(&rq->nr_iowait, 0);
	}

7000
	set_load_weight(&init_task);
7001

7002 7003 7004 7005
#ifdef CONFIG_PREEMPT_NOTIFIERS
	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
#endif

L
Linus Torvalds 已提交
7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018
	/*
	 * The boot idle thread does lazy MMU switching as well:
	 */
	atomic_inc(&init_mm.mm_count);
	enter_lazy_tlb(&init_mm, current);

	/*
	 * Make us the idle thread. Technically, schedule() should not be
	 * called from this thread, however somewhere below it might be,
	 * but because we are the idle thread, we just pick up running again
	 * when this runqueue becomes "idle".
	 */
	init_idle(current, smp_processor_id());
7019 7020 7021

	calc_load_update = jiffies + LOAD_FREQ;

I
Ingo Molnar 已提交
7022 7023 7024 7025
	/*
	 * During early bootup we pretend to be a normal task:
	 */
	current->sched_class = &fair_sched_class;
7026

7027
#ifdef CONFIG_SMP
7028
	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
R
Rusty Russell 已提交
7029 7030 7031
	/* May be allocated at isolcpus cmdline parse time */
	if (cpu_isolated_map == NULL)
		zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
7032
	idle_thread_set_boot_cpu();
7033
	set_cpu_rq_start_time();
7034 7035
#endif
	init_sched_fair_class();
7036

7037
	scheduler_running = 1;
L
Linus Torvalds 已提交
7038 7039
}

7040
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
7041 7042
static inline int preempt_count_equals(int preempt_offset)
{
7043
	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
7044

A
Arnd Bergmann 已提交
7045
	return (nested == preempt_offset);
7046 7047
}

7048
void __might_sleep(const char *file, int line, int preempt_offset)
L
Linus Torvalds 已提交
7049 7050 7051
{
	static unsigned long prev_jiffy;	/* ratelimiting */

7052
	rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
7053 7054
	if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
	     !is_idle_task(current)) ||
7055
	    system_state != SYSTEM_RUNNING || oops_in_progress)
I
Ingo Molnar 已提交
7056 7057 7058 7059 7060
		return;
	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
		return;
	prev_jiffy = jiffies;

P
Peter Zijlstra 已提交
7061 7062 7063 7064 7065 7066 7067
	printk(KERN_ERR
		"BUG: sleeping function called from invalid context at %s:%d\n",
			file, line);
	printk(KERN_ERR
		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
			in_atomic(), irqs_disabled(),
			current->pid, current->comm);
I
Ingo Molnar 已提交
7068 7069 7070 7071

	debug_show_held_locks(current);
	if (irqs_disabled())
		print_irqtrace_events(current);
7072 7073 7074 7075 7076 7077 7078
#ifdef CONFIG_DEBUG_PREEMPT
	if (!preempt_count_equals(preempt_offset)) {
		pr_err("Preemption disabled at:");
		print_ip_sym(current->preempt_disable_ip);
		pr_cont("\n");
	}
#endif
I
Ingo Molnar 已提交
7079
	dump_stack();
L
Linus Torvalds 已提交
7080 7081 7082 7083 7084
}
EXPORT_SYMBOL(__might_sleep);
#endif

#ifdef CONFIG_MAGIC_SYSRQ
7085 7086
static void normalize_task(struct rq *rq, struct task_struct *p)
{
P
Peter Zijlstra 已提交
7087
	const struct sched_class *prev_class = p->sched_class;
7088 7089 7090
	struct sched_attr attr = {
		.sched_policy = SCHED_NORMAL,
	};
P
Peter Zijlstra 已提交
7091
	int old_prio = p->prio;
7092
	int on_rq;
7093

P
Peter Zijlstra 已提交
7094
	on_rq = p->on_rq;
7095
	if (on_rq)
7096
		dequeue_task(rq, p, 0);
7097
	__setscheduler(rq, p, &attr);
7098
	if (on_rq) {
7099
		enqueue_task(rq, p, 0);
7100 7101
		resched_task(rq->curr);
	}
P
Peter Zijlstra 已提交
7102 7103

	check_class_changed(rq, p, prev_class, old_prio);
7104 7105
}

L
Linus Torvalds 已提交
7106 7107
void normalize_rt_tasks(void)
{
7108
	struct task_struct *g, *p;
L
Linus Torvalds 已提交
7109
	unsigned long flags;
7110
	struct rq *rq;
L
Linus Torvalds 已提交
7111

7112
	read_lock_irqsave(&tasklist_lock, flags);
7113
	do_each_thread(g, p) {
7114 7115 7116 7117 7118 7119
		/*
		 * Only normalize user tasks:
		 */
		if (!p->mm)
			continue;

I
Ingo Molnar 已提交
7120 7121
		p->se.exec_start		= 0;
#ifdef CONFIG_SCHEDSTATS
7122 7123 7124
		p->se.statistics.wait_start	= 0;
		p->se.statistics.sleep_start	= 0;
		p->se.statistics.block_start	= 0;
I
Ingo Molnar 已提交
7125
#endif
I
Ingo Molnar 已提交
7126

7127
		if (!dl_task(p) && !rt_task(p)) {
I
Ingo Molnar 已提交
7128 7129 7130 7131
			/*
			 * Renice negative nice level userspace
			 * tasks back to 0:
			 */
7132
			if (task_nice(p) < 0 && p->mm)
I
Ingo Molnar 已提交
7133
				set_user_nice(p, 0);
L
Linus Torvalds 已提交
7134
			continue;
I
Ingo Molnar 已提交
7135
		}
L
Linus Torvalds 已提交
7136

7137
		raw_spin_lock(&p->pi_lock);
7138
		rq = __task_rq_lock(p);
L
Linus Torvalds 已提交
7139

7140
		normalize_task(rq, p);
7141

7142
		__task_rq_unlock(rq);
7143
		raw_spin_unlock(&p->pi_lock);
7144 7145
	} while_each_thread(g, p);

7146
	read_unlock_irqrestore(&tasklist_lock, flags);
L
Linus Torvalds 已提交
7147 7148 7149
}

#endif /* CONFIG_MAGIC_SYSRQ */
7150

7151
#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
7152
/*
7153
 * These functions are only useful for the IA64 MCA handling, or kdb.
7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166
 *
 * They can only be called when the whole system has been
 * stopped - every CPU needs to be quiescent, and no scheduling
 * activity can take place. Using them for anything else would
 * be a serious bug, and as a result, they aren't even visible
 * under any other configuration.
 */

/**
 * curr_task - return the current task for a given cpu.
 * @cpu: the processor in question.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7167 7168
 *
 * Return: The current task for @cpu.
7169
 */
7170
struct task_struct *curr_task(int cpu)
7171 7172 7173 7174
{
	return cpu_curr(cpu);
}

7175 7176 7177
#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */

#ifdef CONFIG_IA64
7178 7179 7180 7181 7182 7183
/**
 * set_curr_task - set the current task for a given cpu.
 * @cpu: the processor in question.
 * @p: the task pointer to set.
 *
 * Description: This function must only be used when non-maskable interrupts
I
Ingo Molnar 已提交
7184 7185
 * are serviced on a separate stack. It allows the architecture to switch the
 * notion of the current task on a cpu in a non-blocking manner. This function
7186 7187 7188 7189 7190 7191 7192
 * must be called with all CPU's synchronized, and interrupts disabled, the
 * and caller must save the original value of the current task (see
 * curr_task() above) and restore that value before reenabling interrupts and
 * re-starting the system.
 *
 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
 */
7193
void set_curr_task(int cpu, struct task_struct *p)
7194 7195 7196 7197 7198
{
	cpu_curr(cpu) = p;
}

#endif
S
Srivatsa Vaddagiri 已提交
7199

D
Dhaval Giani 已提交
7200
#ifdef CONFIG_CGROUP_SCHED
7201 7202 7203
/* task_group_lock serializes the addition/removal of task groups */
static DEFINE_SPINLOCK(task_group_lock);

7204 7205 7206 7207
static void free_sched_group(struct task_group *tg)
{
	free_fair_sched_group(tg);
	free_rt_sched_group(tg);
7208
	autogroup_free(tg);
7209 7210 7211 7212
	kfree(tg);
}

/* allocate runqueue etc for a new task group */
7213
struct task_group *sched_create_group(struct task_group *parent)
7214 7215 7216 7217 7218 7219 7220
{
	struct task_group *tg;

	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
	if (!tg)
		return ERR_PTR(-ENOMEM);

7221
	if (!alloc_fair_sched_group(tg, parent))
7222 7223
		goto err;

7224
	if (!alloc_rt_sched_group(tg, parent))
7225 7226
		goto err;

7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237
	return tg;

err:
	free_sched_group(tg);
	return ERR_PTR(-ENOMEM);
}

void sched_online_group(struct task_group *tg, struct task_group *parent)
{
	unsigned long flags;

7238
	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
7239
	list_add_rcu(&tg->list, &task_groups);
P
Peter Zijlstra 已提交
7240 7241 7242 7243 7244

	WARN_ON(!parent); /* root should already exist */

	tg->parent = parent;
	INIT_LIST_HEAD(&tg->children);
7245
	list_add_rcu(&tg->siblings, &parent->children);
7246
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
7247 7248
}

7249
/* rcu callback to free various structures associated with a task group */
P
Peter Zijlstra 已提交
7250
static void free_sched_group_rcu(struct rcu_head *rhp)
S
Srivatsa Vaddagiri 已提交
7251 7252
{
	/* now it should be safe to free those cfs_rqs */
P
Peter Zijlstra 已提交
7253
	free_sched_group(container_of(rhp, struct task_group, rcu));
S
Srivatsa Vaddagiri 已提交
7254 7255
}

7256
/* Destroy runqueue etc associated with a task group */
7257
void sched_destroy_group(struct task_group *tg)
7258 7259 7260 7261 7262 7263
{
	/* wait for possible concurrent references to cfs_rqs complete */
	call_rcu(&tg->rcu, free_sched_group_rcu);
}

void sched_offline_group(struct task_group *tg)
S
Srivatsa Vaddagiri 已提交
7264
{
7265
	unsigned long flags;
7266
	int i;
S
Srivatsa Vaddagiri 已提交
7267

7268 7269
	/* end participation in shares distribution */
	for_each_possible_cpu(i)
7270
		unregister_fair_sched_group(tg, i);
7271 7272

	spin_lock_irqsave(&task_group_lock, flags);
P
Peter Zijlstra 已提交
7273
	list_del_rcu(&tg->list);
P
Peter Zijlstra 已提交
7274
	list_del_rcu(&tg->siblings);
7275
	spin_unlock_irqrestore(&task_group_lock, flags);
S
Srivatsa Vaddagiri 已提交
7276 7277
}

7278
/* change task's runqueue when it moves between groups.
I
Ingo Molnar 已提交
7279 7280 7281
 *	The caller of this function should have put the task in its new group
 *	by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
 *	reflect its new group.
7282 7283
 */
void sched_move_task(struct task_struct *tsk)
S
Srivatsa Vaddagiri 已提交
7284
{
P
Peter Zijlstra 已提交
7285
	struct task_group *tg;
S
Srivatsa Vaddagiri 已提交
7286 7287 7288 7289 7290 7291
	int on_rq, running;
	unsigned long flags;
	struct rq *rq;

	rq = task_rq_lock(tsk, &flags);

7292
	running = task_current(rq, tsk);
P
Peter Zijlstra 已提交
7293
	on_rq = tsk->on_rq;
S
Srivatsa Vaddagiri 已提交
7294

7295
	if (on_rq)
S
Srivatsa Vaddagiri 已提交
7296
		dequeue_task(rq, tsk, 0);
7297 7298
	if (unlikely(running))
		tsk->sched_class->put_prev_task(rq, tsk);
S
Srivatsa Vaddagiri 已提交
7299

7300
	tg = container_of(task_css_check(tsk, cpu_cgrp_id,
P
Peter Zijlstra 已提交
7301 7302 7303 7304 7305
				lockdep_is_held(&tsk->sighand->siglock)),
			  struct task_group, css);
	tg = autogroup_task_group(tsk, tg);
	tsk->sched_task_group = tg;

P
Peter Zijlstra 已提交
7306
#ifdef CONFIG_FAIR_GROUP_SCHED
7307 7308 7309
	if (tsk->sched_class->task_move_group)
		tsk->sched_class->task_move_group(tsk, on_rq);
	else
P
Peter Zijlstra 已提交
7310
#endif
7311
		set_task_rq(tsk, task_cpu(tsk));
P
Peter Zijlstra 已提交
7312

7313 7314 7315
	if (unlikely(running))
		tsk->sched_class->set_curr_task(rq);
	if (on_rq)
7316
		enqueue_task(rq, tsk, 0);
S
Srivatsa Vaddagiri 已提交
7317

7318
	task_rq_unlock(rq, tsk, &flags);
S
Srivatsa Vaddagiri 已提交
7319
}
D
Dhaval Giani 已提交
7320
#endif /* CONFIG_CGROUP_SCHED */
S
Srivatsa Vaddagiri 已提交
7321

7322 7323 7324 7325 7326
#ifdef CONFIG_RT_GROUP_SCHED
/*
 * Ensure that the real time constraints are schedulable.
 */
static DEFINE_MUTEX(rt_constraints_mutex);
P
Peter Zijlstra 已提交
7327

P
Peter Zijlstra 已提交
7328 7329
/* Must be called with tasklist_lock held */
static inline int tg_has_rt_tasks(struct task_group *tg)
7330
{
P
Peter Zijlstra 已提交
7331
	struct task_struct *g, *p;
7332

P
Peter Zijlstra 已提交
7333
	do_each_thread(g, p) {
7334
		if (rt_task(p) && task_rq(p)->rt.tg == tg)
P
Peter Zijlstra 已提交
7335 7336
			return 1;
	} while_each_thread(g, p);
7337

P
Peter Zijlstra 已提交
7338 7339
	return 0;
}
7340

P
Peter Zijlstra 已提交
7341 7342 7343 7344 7345
struct rt_schedulable_data {
	struct task_group *tg;
	u64 rt_period;
	u64 rt_runtime;
};
7346

7347
static int tg_rt_schedulable(struct task_group *tg, void *data)
P
Peter Zijlstra 已提交
7348 7349 7350 7351 7352
{
	struct rt_schedulable_data *d = data;
	struct task_group *child;
	unsigned long total, sum = 0;
	u64 period, runtime;
7353

P
Peter Zijlstra 已提交
7354 7355
	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	runtime = tg->rt_bandwidth.rt_runtime;
7356

P
Peter Zijlstra 已提交
7357 7358 7359
	if (tg == d->tg) {
		period = d->rt_period;
		runtime = d->rt_runtime;
7360 7361
	}

7362 7363 7364 7365 7366
	/*
	 * Cannot have more runtime than the period.
	 */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;
P
Peter Zijlstra 已提交
7367

7368 7369 7370
	/*
	 * Ensure we don't starve existing RT tasks.
	 */
P
Peter Zijlstra 已提交
7371 7372
	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
		return -EBUSY;
P
Peter Zijlstra 已提交
7373

P
Peter Zijlstra 已提交
7374
	total = to_ratio(period, runtime);
P
Peter Zijlstra 已提交
7375

7376 7377 7378 7379 7380
	/*
	 * Nobody can have more than the global setting allows.
	 */
	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
		return -EINVAL;
P
Peter Zijlstra 已提交
7381

7382 7383 7384
	/*
	 * The sum of our children's runtime should not exceed our own.
	 */
P
Peter Zijlstra 已提交
7385 7386 7387
	list_for_each_entry_rcu(child, &tg->children, siblings) {
		period = ktime_to_ns(child->rt_bandwidth.rt_period);
		runtime = child->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
7388

P
Peter Zijlstra 已提交
7389 7390 7391 7392
		if (child == d->tg) {
			period = d->rt_period;
			runtime = d->rt_runtime;
		}
P
Peter Zijlstra 已提交
7393

P
Peter Zijlstra 已提交
7394
		sum += to_ratio(period, runtime);
P
Peter Zijlstra 已提交
7395
	}
P
Peter Zijlstra 已提交
7396

P
Peter Zijlstra 已提交
7397 7398 7399 7400
	if (sum > total)
		return -EINVAL;

	return 0;
P
Peter Zijlstra 已提交
7401 7402
}

P
Peter Zijlstra 已提交
7403
static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
7404
{
7405 7406
	int ret;

P
Peter Zijlstra 已提交
7407 7408 7409 7410 7411 7412
	struct rt_schedulable_data data = {
		.tg = tg,
		.rt_period = period,
		.rt_runtime = runtime,
	};

7413 7414 7415 7416 7417
	rcu_read_lock();
	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
	rcu_read_unlock();

	return ret;
7418 7419
}

7420
static int tg_set_rt_bandwidth(struct task_group *tg,
7421
		u64 rt_period, u64 rt_runtime)
P
Peter Zijlstra 已提交
7422
{
P
Peter Zijlstra 已提交
7423
	int i, err = 0;
P
Peter Zijlstra 已提交
7424 7425

	mutex_lock(&rt_constraints_mutex);
7426
	read_lock(&tasklist_lock);
P
Peter Zijlstra 已提交
7427 7428
	err = __rt_schedulable(tg, rt_period, rt_runtime);
	if (err)
P
Peter Zijlstra 已提交
7429
		goto unlock;
P
Peter Zijlstra 已提交
7430

7431
	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
7432 7433
	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
	tg->rt_bandwidth.rt_runtime = rt_runtime;
P
Peter Zijlstra 已提交
7434 7435 7436 7437

	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = tg->rt_rq[i];

7438
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7439
		rt_rq->rt_runtime = rt_runtime;
7440
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7441
	}
7442
	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
P
Peter Zijlstra 已提交
7443
unlock:
7444
	read_unlock(&tasklist_lock);
P
Peter Zijlstra 已提交
7445 7446 7447
	mutex_unlock(&rt_constraints_mutex);

	return err;
P
Peter Zijlstra 已提交
7448 7449
}

7450
static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
7451 7452 7453 7454 7455 7456 7457 7458
{
	u64 rt_runtime, rt_period;

	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
	if (rt_runtime_us < 0)
		rt_runtime = RUNTIME_INF;

7459
	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7460 7461
}

7462
static long sched_group_rt_runtime(struct task_group *tg)
P
Peter Zijlstra 已提交
7463 7464 7465
{
	u64 rt_runtime_us;

7466
	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
P
Peter Zijlstra 已提交
7467 7468
		return -1;

7469
	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
P
Peter Zijlstra 已提交
7470 7471 7472
	do_div(rt_runtime_us, NSEC_PER_USEC);
	return rt_runtime_us;
}
7473

7474
static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
7475 7476 7477 7478 7479 7480
{
	u64 rt_runtime, rt_period;

	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
	rt_runtime = tg->rt_bandwidth.rt_runtime;

7481 7482 7483
	if (rt_period == 0)
		return -EINVAL;

7484
	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
7485 7486
}

7487
static long sched_group_rt_period(struct task_group *tg)
7488 7489 7490 7491 7492 7493 7494
{
	u64 rt_period_us;

	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
	do_div(rt_period_us, NSEC_PER_USEC);
	return rt_period_us;
}
7495
#endif /* CONFIG_RT_GROUP_SCHED */
7496

7497
#ifdef CONFIG_RT_GROUP_SCHED
7498 7499 7500 7501 7502
static int sched_rt_global_constraints(void)
{
	int ret = 0;

	mutex_lock(&rt_constraints_mutex);
P
Peter Zijlstra 已提交
7503
	read_lock(&tasklist_lock);
7504
	ret = __rt_schedulable(NULL, 0, 0);
P
Peter Zijlstra 已提交
7505
	read_unlock(&tasklist_lock);
7506 7507 7508 7509
	mutex_unlock(&rt_constraints_mutex);

	return ret;
}
7510

7511
static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
7512 7513 7514 7515 7516 7517 7518 7519
{
	/* Don't accept realtime tasks when there is no way for them to run */
	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
		return 0;

	return 1;
}

7520
#else /* !CONFIG_RT_GROUP_SCHED */
7521 7522
static int sched_rt_global_constraints(void)
{
P
Peter Zijlstra 已提交
7523
	unsigned long flags;
7524
	int i, ret = 0;
7525

7526
	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
7527 7528 7529
	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = &cpu_rq(i)->rt;

7530
		raw_spin_lock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7531
		rt_rq->rt_runtime = global_rt_runtime();
7532
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
P
Peter Zijlstra 已提交
7533
	}
7534
	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
P
Peter Zijlstra 已提交
7535

7536
	return ret;
7537
}
7538
#endif /* CONFIG_RT_GROUP_SCHED */
7539

7540 7541
static int sched_dl_global_constraints(void)
{
7542 7543
	u64 runtime = global_rt_runtime();
	u64 period = global_rt_period();
7544
	u64 new_bw = to_ratio(period, runtime);
7545
	int cpu, ret = 0;
7546
	unsigned long flags;
7547 7548 7549 7550 7551 7552 7553 7554 7555 7556

	/*
	 * Here we want to check the bandwidth not being set to some
	 * value smaller than the currently allocated bandwidth in
	 * any of the root_domains.
	 *
	 * FIXME: Cycling on all the CPUs is overdoing, but simpler than
	 * cycling on root_domains... Discussion on different/better
	 * solutions is welcome!
	 */
7557 7558
	for_each_possible_cpu(cpu) {
		struct dl_bw *dl_b = dl_bw_of(cpu);
7559

7560
		raw_spin_lock_irqsave(&dl_b->lock, flags);
7561 7562
		if (new_bw < dl_b->total_bw)
			ret = -EBUSY;
7563
		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
7564 7565 7566

		if (ret)
			break;
7567 7568
	}

7569
	return ret;
7570 7571
}

7572
static void sched_dl_do_global(void)
7573
{
7574 7575
	u64 new_bw = -1;
	int cpu;
7576
	unsigned long flags;
7577

7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589
	def_dl_bandwidth.dl_period = global_rt_period();
	def_dl_bandwidth.dl_runtime = global_rt_runtime();

	if (global_rt_runtime() != RUNTIME_INF)
		new_bw = to_ratio(global_rt_period(), global_rt_runtime());

	/*
	 * FIXME: As above...
	 */
	for_each_possible_cpu(cpu) {
		struct dl_bw *dl_b = dl_bw_of(cpu);

7590
		raw_spin_lock_irqsave(&dl_b->lock, flags);
7591
		dl_b->bw = new_bw;
7592
		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
7593
	}
7594 7595 7596 7597 7598 7599 7600
}

static int sched_rt_global_validate(void)
{
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

7601 7602
	if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
		(sysctl_sched_rt_runtime > sysctl_sched_rt_period))
7603 7604 7605 7606 7607 7608 7609 7610 7611
		return -EINVAL;

	return 0;
}

static void sched_rt_do_global(void)
{
	def_rt_bandwidth.rt_runtime = global_rt_runtime();
	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
7612 7613
}

7614
int sched_rt_handler(struct ctl_table *table, int write,
7615
		void __user *buffer, size_t *lenp,
7616 7617 7618 7619
		loff_t *ppos)
{
	int old_period, old_runtime;
	static DEFINE_MUTEX(mutex);
7620
	int ret;
7621 7622 7623 7624 7625

	mutex_lock(&mutex);
	old_period = sysctl_sched_rt_period;
	old_runtime = sysctl_sched_rt_runtime;

7626
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
7627 7628

	if (!ret && write) {
7629 7630 7631 7632
		ret = sched_rt_global_validate();
		if (ret)
			goto undo;

7633
		ret = sched_rt_global_constraints();
7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647
		if (ret)
			goto undo;

		ret = sched_dl_global_constraints();
		if (ret)
			goto undo;

		sched_rt_do_global();
		sched_dl_do_global();
	}
	if (0) {
undo:
		sysctl_sched_rt_period = old_period;
		sysctl_sched_rt_runtime = old_runtime;
7648 7649 7650 7651 7652
	}
	mutex_unlock(&mutex);

	return ret;
}
7653

7654
int sched_rr_handler(struct ctl_table *table, int write,
7655 7656 7657 7658 7659 7660 7661 7662
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
7663 7664
	/* make sure that internally we keep jiffies */
	/* also, writing zero resets timeslice to default */
7665
	if (!ret && write) {
7666 7667
		sched_rr_timeslice = sched_rr_timeslice <= 0 ?
			RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
7668 7669 7670 7671 7672
	}
	mutex_unlock(&mutex);
	return ret;
}

7673
#ifdef CONFIG_CGROUP_SCHED
7674

7675
static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
7676
{
7677
	return css ? container_of(css, struct task_group, css) : NULL;
7678 7679
}

7680 7681
static struct cgroup_subsys_state *
cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
7682
{
7683 7684
	struct task_group *parent = css_tg(parent_css);
	struct task_group *tg;
7685

7686
	if (!parent) {
7687
		/* This is early initialization for the top cgroup */
7688
		return &root_task_group.css;
7689 7690
	}

7691
	tg = sched_create_group(parent);
7692 7693 7694 7695 7696 7697
	if (IS_ERR(tg))
		return ERR_PTR(-ENOMEM);

	return &tg->css;
}

7698
static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
7699
{
7700
	struct task_group *tg = css_tg(css);
T
Tejun Heo 已提交
7701
	struct task_group *parent = css_tg(css->parent);
7702

T
Tejun Heo 已提交
7703 7704
	if (parent)
		sched_online_group(tg, parent);
7705 7706 7707
	return 0;
}

7708
static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
7709
{
7710
	struct task_group *tg = css_tg(css);
7711 7712 7713 7714

	sched_destroy_group(tg);
}

7715
static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
7716
{
7717
	struct task_group *tg = css_tg(css);
7718 7719 7720 7721

	sched_offline_group(tg);
}

7722
static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
7723
				 struct cgroup_taskset *tset)
7724
{
7725 7726
	struct task_struct *task;

7727
	cgroup_taskset_for_each(task, tset) {
7728
#ifdef CONFIG_RT_GROUP_SCHED
7729
		if (!sched_rt_can_attach(css_tg(css), task))
7730
			return -EINVAL;
7731
#else
7732 7733 7734
		/* We don't support RT-tasks being in separate groups */
		if (task->sched_class != &fair_sched_class)
			return -EINVAL;
7735
#endif
7736
	}
7737 7738
	return 0;
}
7739

7740
static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
7741
			      struct cgroup_taskset *tset)
7742
{
7743 7744
	struct task_struct *task;

7745
	cgroup_taskset_for_each(task, tset)
7746
		sched_move_task(task);
7747 7748
}

7749 7750 7751
static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
			    struct cgroup_subsys_state *old_css,
			    struct task_struct *task)
7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763
{
	/*
	 * cgroup_exit() is called in the copy_process() failure path.
	 * Ignore this case since the task hasn't ran yet, this avoids
	 * trying to poke a half freed task state from generic code.
	 */
	if (!(task->flags & PF_EXITING))
		return;

	sched_move_task(task);
}

7764
#ifdef CONFIG_FAIR_GROUP_SCHED
7765 7766
static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
				struct cftype *cftype, u64 shareval)
7767
{
7768
	return sched_group_set_shares(css_tg(css), scale_load(shareval));
7769 7770
}

7771 7772
static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
			       struct cftype *cft)
7773
{
7774
	struct task_group *tg = css_tg(css);
7775

7776
	return (u64) scale_load_down(tg->shares);
7777
}
7778 7779

#ifdef CONFIG_CFS_BANDWIDTH
7780 7781
static DEFINE_MUTEX(cfs_constraints_mutex);

7782 7783 7784
const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */

7785 7786
static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);

7787 7788
static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
{
7789
	int i, ret = 0, runtime_enabled, runtime_was_enabled;
7790
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810

	if (tg == &root_task_group)
		return -EINVAL;

	/*
	 * Ensure we have at some amount of bandwidth every period.  This is
	 * to prevent reaching a state of large arrears when throttled via
	 * entity_tick() resulting in prolonged exit starvation.
	 */
	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
		return -EINVAL;

	/*
	 * Likewise, bound things on the otherside by preventing insane quota
	 * periods.  This also allows us to normalize in computing quota
	 * feasibility.
	 */
	if (period > max_cfs_quota_period)
		return -EINVAL;

7811 7812 7813 7814 7815
	mutex_lock(&cfs_constraints_mutex);
	ret = __cfs_schedulable(tg, period, quota);
	if (ret)
		goto out_unlock;

7816
	runtime_enabled = quota != RUNTIME_INF;
7817
	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
7818 7819 7820 7821 7822 7823
	/*
	 * If we need to toggle cfs_bandwidth_used, off->on must occur
	 * before making related changes, and on->off must occur afterwards
	 */
	if (runtime_enabled && !runtime_was_enabled)
		cfs_bandwidth_usage_inc();
7824 7825 7826
	raw_spin_lock_irq(&cfs_b->lock);
	cfs_b->period = ns_to_ktime(period);
	cfs_b->quota = quota;
7827

P
Paul Turner 已提交
7828
	__refill_cfs_bandwidth_runtime(cfs_b);
7829 7830 7831
	/* restart the period timer (if active) to handle new period expiry */
	if (runtime_enabled && cfs_b->timer_active) {
		/* force a reprogram */
7832
		__start_cfs_bandwidth(cfs_b, true);
7833
	}
7834 7835 7836 7837
	raw_spin_unlock_irq(&cfs_b->lock);

	for_each_possible_cpu(i) {
		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
7838
		struct rq *rq = cfs_rq->rq;
7839 7840

		raw_spin_lock_irq(&rq->lock);
7841
		cfs_rq->runtime_enabled = runtime_enabled;
7842
		cfs_rq->runtime_remaining = 0;
7843

7844
		if (cfs_rq->throttled)
7845
			unthrottle_cfs_rq(cfs_rq);
7846 7847
		raw_spin_unlock_irq(&rq->lock);
	}
7848 7849
	if (runtime_was_enabled && !runtime_enabled)
		cfs_bandwidth_usage_dec();
7850 7851
out_unlock:
	mutex_unlock(&cfs_constraints_mutex);
7852

7853
	return ret;
7854 7855 7856 7857 7858 7859
}

int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
{
	u64 quota, period;

7860
	period = ktime_to_ns(tg->cfs_bandwidth.period);
7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872
	if (cfs_quota_us < 0)
		quota = RUNTIME_INF;
	else
		quota = (u64)cfs_quota_us * NSEC_PER_USEC;

	return tg_set_cfs_bandwidth(tg, period, quota);
}

long tg_get_cfs_quota(struct task_group *tg)
{
	u64 quota_us;

7873
	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
7874 7875
		return -1;

7876
	quota_us = tg->cfs_bandwidth.quota;
7877 7878 7879 7880 7881 7882 7883 7884 7885 7886
	do_div(quota_us, NSEC_PER_USEC);

	return quota_us;
}

int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
{
	u64 quota, period;

	period = (u64)cfs_period_us * NSEC_PER_USEC;
7887
	quota = tg->cfs_bandwidth.quota;
7888 7889 7890 7891 7892 7893 7894 7895

	return tg_set_cfs_bandwidth(tg, period, quota);
}

long tg_get_cfs_period(struct task_group *tg)
{
	u64 cfs_period_us;

7896
	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
7897 7898 7899 7900 7901
	do_div(cfs_period_us, NSEC_PER_USEC);

	return cfs_period_us;
}

7902 7903
static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
				  struct cftype *cft)
7904
{
7905
	return tg_get_cfs_quota(css_tg(css));
7906 7907
}

7908 7909
static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
				   struct cftype *cftype, s64 cfs_quota_us)
7910
{
7911
	return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
7912 7913
}

7914 7915
static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
				   struct cftype *cft)
7916
{
7917
	return tg_get_cfs_period(css_tg(css));
7918 7919
}

7920 7921
static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
				    struct cftype *cftype, u64 cfs_period_us)
7922
{
7923
	return tg_set_cfs_period(css_tg(css), cfs_period_us);
7924 7925
}

7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957
struct cfs_schedulable_data {
	struct task_group *tg;
	u64 period, quota;
};

/*
 * normalize group quota/period to be quota/max_period
 * note: units are usecs
 */
static u64 normalize_cfs_quota(struct task_group *tg,
			       struct cfs_schedulable_data *d)
{
	u64 quota, period;

	if (tg == d->tg) {
		period = d->period;
		quota = d->quota;
	} else {
		period = tg_get_cfs_period(tg);
		quota = tg_get_cfs_quota(tg);
	}

	/* note: these should typically be equivalent */
	if (quota == RUNTIME_INF || quota == -1)
		return RUNTIME_INF;

	return to_ratio(period, quota);
}

static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
{
	struct cfs_schedulable_data *d = data;
7958
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7959 7960 7961 7962 7963
	s64 quota = 0, parent_quota = -1;

	if (!tg->parent) {
		quota = RUNTIME_INF;
	} else {
7964
		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984

		quota = normalize_cfs_quota(tg, d);
		parent_quota = parent_b->hierarchal_quota;

		/*
		 * ensure max(child_quota) <= parent_quota, inherit when no
		 * limit is set
		 */
		if (quota == RUNTIME_INF)
			quota = parent_quota;
		else if (parent_quota != RUNTIME_INF && quota > parent_quota)
			return -EINVAL;
	}
	cfs_b->hierarchal_quota = quota;

	return 0;
}

static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
{
7985
	int ret;
7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996
	struct cfs_schedulable_data data = {
		.tg = tg,
		.period = period,
		.quota = quota,
	};

	if (quota != RUNTIME_INF) {
		do_div(data.period, NSEC_PER_USEC);
		do_div(data.quota, NSEC_PER_USEC);
	}

7997 7998 7999 8000 8001
	rcu_read_lock();
	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
	rcu_read_unlock();

	return ret;
8002
}
8003

8004
static int cpu_stats_show(struct seq_file *sf, void *v)
8005
{
8006
	struct task_group *tg = css_tg(seq_css(sf));
8007
	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8008

8009 8010 8011
	seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
	seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
	seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
8012 8013 8014

	return 0;
}
8015
#endif /* CONFIG_CFS_BANDWIDTH */
8016
#endif /* CONFIG_FAIR_GROUP_SCHED */
8017

8018
#ifdef CONFIG_RT_GROUP_SCHED
8019 8020
static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
				struct cftype *cft, s64 val)
P
Peter Zijlstra 已提交
8021
{
8022
	return sched_group_set_rt_runtime(css_tg(css), val);
P
Peter Zijlstra 已提交
8023 8024
}

8025 8026
static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
			       struct cftype *cft)
P
Peter Zijlstra 已提交
8027
{
8028
	return sched_group_rt_runtime(css_tg(css));
P
Peter Zijlstra 已提交
8029
}
8030

8031 8032
static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
				    struct cftype *cftype, u64 rt_period_us)
8033
{
8034
	return sched_group_set_rt_period(css_tg(css), rt_period_us);
8035 8036
}

8037 8038
static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
				   struct cftype *cft)
8039
{
8040
	return sched_group_rt_period(css_tg(css));
8041
}
8042
#endif /* CONFIG_RT_GROUP_SCHED */
P
Peter Zijlstra 已提交
8043

8044
static struct cftype cpu_files[] = {
8045
#ifdef CONFIG_FAIR_GROUP_SCHED
8046 8047
	{
		.name = "shares",
8048 8049
		.read_u64 = cpu_shares_read_u64,
		.write_u64 = cpu_shares_write_u64,
8050
	},
8051
#endif
8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062
#ifdef CONFIG_CFS_BANDWIDTH
	{
		.name = "cfs_quota_us",
		.read_s64 = cpu_cfs_quota_read_s64,
		.write_s64 = cpu_cfs_quota_write_s64,
	},
	{
		.name = "cfs_period_us",
		.read_u64 = cpu_cfs_period_read_u64,
		.write_u64 = cpu_cfs_period_write_u64,
	},
8063 8064
	{
		.name = "stat",
8065
		.seq_show = cpu_stats_show,
8066
	},
8067
#endif
8068
#ifdef CONFIG_RT_GROUP_SCHED
P
Peter Zijlstra 已提交
8069
	{
P
Peter Zijlstra 已提交
8070
		.name = "rt_runtime_us",
8071 8072
		.read_s64 = cpu_rt_runtime_read,
		.write_s64 = cpu_rt_runtime_write,
P
Peter Zijlstra 已提交
8073
	},
8074 8075
	{
		.name = "rt_period_us",
8076 8077
		.read_u64 = cpu_rt_period_read_uint,
		.write_u64 = cpu_rt_period_write_uint,
8078
	},
8079
#endif
8080
	{ }	/* terminate */
8081 8082
};

8083
struct cgroup_subsys cpu_cgrp_subsys = {
8084 8085
	.css_alloc	= cpu_cgroup_css_alloc,
	.css_free	= cpu_cgroup_css_free,
8086 8087
	.css_online	= cpu_cgroup_css_online,
	.css_offline	= cpu_cgroup_css_offline,
8088 8089
	.can_attach	= cpu_cgroup_can_attach,
	.attach		= cpu_cgroup_attach,
8090
	.exit		= cpu_cgroup_exit,
8091
	.base_cftypes	= cpu_files,
8092 8093 8094
	.early_init	= 1,
};

8095
#endif	/* CONFIG_CGROUP_SCHED */
8096

8097 8098 8099 8100 8101
void dump_cpu_task(int cpu)
{
	pr_info("Task dump for CPU %d:\n", cpu);
	sched_show_task(cpu_curr(cpu));
}