cpufreq_schedutil.c 17.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * CPUFreq governor based on scheduler-provided CPU utilization data.
 *
 * Copyright (C) 2016, Intel Corporation
 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

12 13
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14
#include <linux/cpufreq.h>
15
#include <linux/kthread.h>
16
#include <uapi/linux/sched/types.h>
17 18 19 20 21
#include <linux/slab.h>
#include <trace/events/power.h>

#include "sched.h"

22 23
#define SUGOV_KTHREAD_PRIORITY	50

24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
struct sugov_tunables {
	struct gov_attr_set attr_set;
	unsigned int rate_limit_us;
};

struct sugov_policy {
	struct cpufreq_policy *policy;

	struct sugov_tunables *tunables;
	struct list_head tunables_hook;

	raw_spinlock_t update_lock;  /* For shared policies */
	u64 last_freq_update_time;
	s64 freq_update_delay_ns;
	unsigned int next_freq;
39
	unsigned int cached_raw_freq;
40 41 42

	/* The next fields are only needed if fast switch cannot be used. */
	struct irq_work irq_work;
43
	struct kthread_work work;
44
	struct mutex work_lock;
45 46
	struct kthread_worker worker;
	struct task_struct *thread;
47 48 49 50 51 52 53 54 55
	bool work_in_progress;

	bool need_freq_update;
};

struct sugov_cpu {
	struct update_util_data update_util;
	struct sugov_policy *sg_policy;

56 57 58
	unsigned long iowait_boost;
	unsigned long iowait_boost_max;
	u64 last_update;
59

60 61 62
	/* The fields below are only needed when sharing a policy. */
	unsigned long util;
	unsigned long max;
63
	unsigned int flags;
64 65 66 67 68

	/* The field below is for single-CPU policies only. */
#ifdef CONFIG_NO_HZ_COMMON
	unsigned long saved_idle_calls;
#endif
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
};

static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);

/************************ Governor internals ***********************/

static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
{
	s64 delta_ns;

	if (sg_policy->work_in_progress)
		return false;

	if (unlikely(sg_policy->need_freq_update)) {
		sg_policy->need_freq_update = false;
		/*
		 * This happens when limits change, so forget the previous
		 * next_freq value and force an update.
		 */
		sg_policy->next_freq = UINT_MAX;
		return true;
	}

	delta_ns = time - sg_policy->last_freq_update_time;
	return delta_ns >= sg_policy->freq_update_delay_ns;
}

static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
				unsigned int next_freq)
{
	struct cpufreq_policy *policy = sg_policy->policy;

101 102 103 104
	if (sg_policy->next_freq == next_freq)
		return;

	sg_policy->next_freq = next_freq;
105 106 107 108 109 110 111 112 113
	sg_policy->last_freq_update_time = time;

	if (policy->fast_switch_enabled) {
		next_freq = cpufreq_driver_fast_switch(policy, next_freq);
		if (next_freq == CPUFREQ_ENTRY_INVALID)
			return;

		policy->cur = next_freq;
		trace_cpu_frequency(next_freq, smp_processor_id());
114
	} else {
115 116 117 118 119 120 121
		sg_policy->work_in_progress = true;
		irq_work_queue(&sg_policy->irq_work);
	}
}

/**
 * get_next_freq - Compute a new frequency for a given cpufreq policy.
122
 * @sg_policy: schedutil policy object to compute the new frequency for.
123 124 125 126 127 128 129 130 131 132 133 134 135 136
 * @util: Current CPU utilization.
 * @max: CPU capacity.
 *
 * If the utilization is frequency-invariant, choose the new frequency to be
 * proportional to it, that is
 *
 * next_freq = C * max_freq * util / max
 *
 * Otherwise, approximate the would-be frequency-invariant utilization by
 * util_raw * (curr_freq / max_freq) which leads to
 *
 * next_freq = C * curr_freq * util_raw / max
 *
 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
137 138 139 140
 *
 * The lowest driver-supported frequency which is equal or greater than the raw
 * next_freq (as calculated above) is returned, subject to policy min/max and
 * cpufreq driver limitations.
141
 */
142 143
static unsigned int get_next_freq(struct sugov_policy *sg_policy,
				  unsigned long util, unsigned long max)
144
{
145
	struct cpufreq_policy *policy = sg_policy->policy;
146 147 148
	unsigned int freq = arch_scale_freq_invariant() ?
				policy->cpuinfo.max_freq : policy->cur;

149 150
	freq = (freq + (freq >> 2)) * util / max;

151
	if (freq == sg_policy->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
152
		return sg_policy->next_freq;
153
	sg_policy->cached_raw_freq = freq;
154
	return cpufreq_driver_resolve_freq(policy, freq);
155 156
}

157 158 159
static void sugov_get_util(unsigned long *util, unsigned long *max)
{
	struct rq *rq = this_rq();
160 161 162
	unsigned long cfs_max;

	cfs_max = arch_scale_cpu_capacity(NULL, smp_processor_id());
163 164 165 166 167

	*util = min(rq->cfs.avg.util_avg, cfs_max);
	*max = cfs_max;
}

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
				   unsigned int flags)
{
	if (flags & SCHED_CPUFREQ_IOWAIT) {
		sg_cpu->iowait_boost = sg_cpu->iowait_boost_max;
	} else if (sg_cpu->iowait_boost) {
		s64 delta_ns = time - sg_cpu->last_update;

		/* Clear iowait_boost if the CPU apprears to have been idle. */
		if (delta_ns > TICK_NSEC)
			sg_cpu->iowait_boost = 0;
	}
}

static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util,
			       unsigned long *max)
{
	unsigned long boost_util = sg_cpu->iowait_boost;
	unsigned long boost_max = sg_cpu->iowait_boost_max;

	if (!boost_util)
		return;

	if (*util * boost_max < *max * boost_util) {
		*util = boost_util;
		*max = boost_max;
	}
	sg_cpu->iowait_boost >>= 1;
}

198 199 200 201 202 203 204 205 206 207 208 209 210
#ifdef CONFIG_NO_HZ_COMMON
static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
{
	unsigned long idle_calls = tick_nohz_get_idle_calls();
	bool ret = idle_calls == sg_cpu->saved_idle_calls;

	sg_cpu->saved_idle_calls = idle_calls;
	return ret;
}
#else
static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
#endif /* CONFIG_NO_HZ_COMMON */

211
static void sugov_update_single(struct update_util_data *hook, u64 time,
212
				unsigned int flags)
213 214 215 216
{
	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
	struct cpufreq_policy *policy = sg_policy->policy;
217
	unsigned long util, max;
218
	unsigned int next_f;
219
	bool busy;
220

221 222 223
	sugov_set_iowait_boost(sg_cpu, time, flags);
	sg_cpu->last_update = time;

224 225 226
	if (!sugov_should_update_freq(sg_policy, time))
		return;

227 228
	busy = sugov_cpu_is_busy(sg_cpu);

229 230 231 232
	if (flags & SCHED_CPUFREQ_RT_DL) {
		next_f = policy->cpuinfo.max_freq;
	} else {
		sugov_get_util(&util, &max);
233
		sugov_iowait_boost(sg_cpu, &util, &max);
234
		next_f = get_next_freq(sg_policy, util, max);
235 236 237 238 239 240
		/*
		 * Do not reduce the frequency if the CPU has not been idle
		 * recently, as the reduction is likely to be premature then.
		 */
		if (busy && next_f < sg_policy->next_freq)
			next_f = sg_policy->next_freq;
241
	}
242 243 244
	sugov_update_commit(sg_policy, time, next_f);
}

245
static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
246
{
247
	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
248
	struct cpufreq_policy *policy = sg_policy->policy;
249
	unsigned long util = 0, max = 1;
250 251 252
	unsigned int j;

	for_each_cpu(j, policy->cpus) {
253
		struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
254 255 256 257 258 259 260 261
		unsigned long j_util, j_max;
		s64 delta_ns;

		/*
		 * If the CPU utilization was last updated before the previous
		 * frequency update and the time elapsed between the last update
		 * of the CPU utilization and the last frequency update is long
		 * enough, don't take the CPU into account as it probably is
262
		 * idle now (and clear iowait_boost for it).
263
		 */
264
		delta_ns = time - j_sg_cpu->last_update;
265 266
		if (delta_ns > TICK_NSEC) {
			j_sg_cpu->iowait_boost = 0;
267
			continue;
268
		}
269
		if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL)
270
			return policy->cpuinfo.max_freq;
271

272
		j_util = j_sg_cpu->util;
273 274 275 276 277
		j_max = j_sg_cpu->max;
		if (j_util * max > j_max * util) {
			util = j_util;
			max = j_max;
		}
278 279

		sugov_iowait_boost(j_sg_cpu, &util, &max);
280 281
	}

282
	return get_next_freq(sg_policy, util, max);
283 284 285
}

static void sugov_update_shared(struct update_util_data *hook, u64 time,
286
				unsigned int flags)
287 288 289
{
	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
290
	unsigned long util, max;
291 292
	unsigned int next_f;

293 294
	sugov_get_util(&util, &max);

295 296 297 298
	raw_spin_lock(&sg_policy->update_lock);

	sg_cpu->util = util;
	sg_cpu->max = max;
299
	sg_cpu->flags = flags;
300 301

	sugov_set_iowait_boost(sg_cpu, time, flags);
302 303 304
	sg_cpu->last_update = time;

	if (sugov_should_update_freq(sg_policy, time)) {
305 306 307
		if (flags & SCHED_CPUFREQ_RT_DL)
			next_f = sg_policy->policy->cpuinfo.max_freq;
		else
308
			next_f = sugov_next_freq_shared(sg_cpu, time);
309

310 311 312 313 314 315
		sugov_update_commit(sg_policy, time, next_f);
	}

	raw_spin_unlock(&sg_policy->update_lock);
}

316
static void sugov_work(struct kthread_work *work)
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
{
	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);

	mutex_lock(&sg_policy->work_lock);
	__cpufreq_driver_target(sg_policy->policy, sg_policy->next_freq,
				CPUFREQ_RELATION_L);
	mutex_unlock(&sg_policy->work_lock);

	sg_policy->work_in_progress = false;
}

static void sugov_irq_work(struct irq_work *irq_work)
{
	struct sugov_policy *sg_policy;

	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);
333 334

	/*
335 336 337
	 * For RT and deadline tasks, the schedutil governor shoots the
	 * frequency to maximum. Special care must be taken to ensure that this
	 * kthread doesn't result in the same behavior.
338 339
	 *
	 * This is (mostly) guaranteed by the work_in_progress flag. The flag is
340 341
	 * updated only at the end of the sugov_work() function and before that
	 * the schedutil governor rejects all other frequency scaling requests.
342
	 *
343
	 * There is a very rare case though, where the RT thread yields right
344 345 346 347
	 * after the work_in_progress flag is cleared. The effects of that are
	 * neglected for now.
	 */
	kthread_queue_work(&sg_policy->worker, &sg_policy->work);
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
}

/************************** sysfs interface ************************/

static struct sugov_tunables *global_tunables;
static DEFINE_MUTEX(global_tunables_lock);

static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set)
{
	return container_of(attr_set, struct sugov_tunables, attr_set);
}

static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
{
	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

	return sprintf(buf, "%u\n", tunables->rate_limit_us);
}

static ssize_t rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf,
				   size_t count)
{
	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
	struct sugov_policy *sg_policy;
	unsigned int rate_limit_us;

	if (kstrtouint(buf, 10, &rate_limit_us))
		return -EINVAL;

	tunables->rate_limit_us = rate_limit_us;

	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;

	return count;
}

static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us);

static struct attribute *sugov_attributes[] = {
	&rate_limit_us.attr,
	NULL
};

static struct kobj_type sugov_tunables_ktype = {
	.default_attrs = sugov_attributes,
	.sysfs_ops = &governor_sysfs_ops,
};

/********************** cpufreq governor interface *********************/

static struct cpufreq_governor schedutil_gov;

static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy)
{
	struct sugov_policy *sg_policy;

	sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);
	if (!sg_policy)
		return NULL;

	sg_policy->policy = policy;
	raw_spin_lock_init(&sg_policy->update_lock);
	return sg_policy;
}

static void sugov_policy_free(struct sugov_policy *sg_policy)
{
	kfree(sg_policy);
}

419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
static int sugov_kthread_create(struct sugov_policy *sg_policy)
{
	struct task_struct *thread;
	struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 };
	struct cpufreq_policy *policy = sg_policy->policy;
	int ret;

	/* kthread only required for slow path */
	if (policy->fast_switch_enabled)
		return 0;

	kthread_init_work(&sg_policy->work, sugov_work);
	kthread_init_worker(&sg_policy->worker);
	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
				"sugov:%d",
				cpumask_first(policy->related_cpus));
	if (IS_ERR(thread)) {
		pr_err("failed to create sugov thread: %ld\n", PTR_ERR(thread));
		return PTR_ERR(thread);
	}

	ret = sched_setscheduler_nocheck(thread, SCHED_FIFO, &param);
	if (ret) {
		kthread_stop(thread);
		pr_warn("%s: failed to set SCHED_FIFO\n", __func__);
		return ret;
	}

	sg_policy->thread = thread;
	kthread_bind_mask(thread, policy->related_cpus);
449 450 451
	init_irq_work(&sg_policy->irq_work, sugov_irq_work);
	mutex_init(&sg_policy->work_lock);

452 453 454 455 456 457 458 459 460 461 462 463 464
	wake_up_process(thread);

	return 0;
}

static void sugov_kthread_stop(struct sugov_policy *sg_policy)
{
	/* kthread only required for slow path */
	if (sg_policy->policy->fast_switch_enabled)
		return;

	kthread_flush_worker(&sg_policy->worker);
	kthread_stop(sg_policy->thread);
465
	mutex_destroy(&sg_policy->work_lock);
466 467
}

468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)
{
	struct sugov_tunables *tunables;

	tunables = kzalloc(sizeof(*tunables), GFP_KERNEL);
	if (tunables) {
		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
		if (!have_governor_per_policy())
			global_tunables = tunables;
	}
	return tunables;
}

static void sugov_tunables_free(struct sugov_tunables *tunables)
{
	if (!have_governor_per_policy())
		global_tunables = NULL;

	kfree(tunables);
}

static int sugov_init(struct cpufreq_policy *policy)
{
	struct sugov_policy *sg_policy;
	struct sugov_tunables *tunables;
	int ret = 0;

	/* State should be equivalent to EXIT */
	if (policy->governor_data)
		return -EBUSY;

499 500
	cpufreq_enable_fast_switch(policy);

501
	sg_policy = sugov_policy_alloc(policy);
502 503 504 505
	if (!sg_policy) {
		ret = -ENOMEM;
		goto disable_fast_switch;
	}
506

507 508 509 510
	ret = sugov_kthread_create(sg_policy);
	if (ret)
		goto free_sg_policy;

511 512 513 514 515
	mutex_lock(&global_tunables_lock);

	if (global_tunables) {
		if (WARN_ON(have_governor_per_policy())) {
			ret = -EINVAL;
516
			goto stop_kthread;
517 518 519 520 521 522 523 524 525 526 527
		}
		policy->governor_data = sg_policy;
		sg_policy->tunables = global_tunables;

		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
		goto out;
	}

	tunables = sugov_tunables_alloc(sg_policy);
	if (!tunables) {
		ret = -ENOMEM;
528
		goto stop_kthread;
529 530
	}

531 532 533 534 535 536 537 538 539 540
	if (policy->transition_delay_us) {
		tunables->rate_limit_us = policy->transition_delay_us;
	} else {
		unsigned int lat;

		tunables->rate_limit_us = LATENCY_MULTIPLIER;
		lat = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
		if (lat)
			tunables->rate_limit_us *= lat;
	}
541 542 543 544 545 546 547 548 549 550

	policy->governor_data = sg_policy;
	sg_policy->tunables = tunables;

	ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
				   get_governor_parent_kobj(policy), "%s",
				   schedutil_gov.name);
	if (ret)
		goto fail;

551
out:
552 553 554
	mutex_unlock(&global_tunables_lock);
	return 0;

555
fail:
556 557 558
	policy->governor_data = NULL;
	sugov_tunables_free(tunables);

559 560 561
stop_kthread:
	sugov_kthread_stop(sg_policy);

562
free_sg_policy:
563 564 565
	mutex_unlock(&global_tunables_lock);

	sugov_policy_free(sg_policy);
566 567 568 569

disable_fast_switch:
	cpufreq_disable_fast_switch(policy);

570
	pr_err("initialization failed (error %d)\n", ret);
571 572 573
	return ret;
}

574
static void sugov_exit(struct cpufreq_policy *policy)
575 576 577 578 579 580 581 582 583 584 585 586 587 588
{
	struct sugov_policy *sg_policy = policy->governor_data;
	struct sugov_tunables *tunables = sg_policy->tunables;
	unsigned int count;

	mutex_lock(&global_tunables_lock);

	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
	policy->governor_data = NULL;
	if (!count)
		sugov_tunables_free(tunables);

	mutex_unlock(&global_tunables_lock);

589
	sugov_kthread_stop(sg_policy);
590
	sugov_policy_free(sg_policy);
591
	cpufreq_disable_fast_switch(policy);
592 593 594 595 596 597 598 599 600 601 602 603
}

static int sugov_start(struct cpufreq_policy *policy)
{
	struct sugov_policy *sg_policy = policy->governor_data;
	unsigned int cpu;

	sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
	sg_policy->last_freq_update_time = 0;
	sg_policy->next_freq = UINT_MAX;
	sg_policy->work_in_progress = false;
	sg_policy->need_freq_update = false;
604
	sg_policy->cached_raw_freq = 0;
605 606 607 608

	for_each_cpu(cpu, policy->cpus) {
		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);

609
		memset(sg_cpu, 0, sizeof(*sg_cpu));
610
		sg_cpu->sg_policy = sg_policy;
611 612 613 614 615 616
		sg_cpu->flags = SCHED_CPUFREQ_RT;
		sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
					     policy_is_shared(policy) ?
							sugov_update_shared :
							sugov_update_single);
617 618 619 620
	}
	return 0;
}

621
static void sugov_stop(struct cpufreq_policy *policy)
622 623 624 625 626 627 628 629 630
{
	struct sugov_policy *sg_policy = policy->governor_data;
	unsigned int cpu;

	for_each_cpu(cpu, policy->cpus)
		cpufreq_remove_update_util_hook(cpu);

	synchronize_sched();

631 632 633 634
	if (!policy->fast_switch_enabled) {
		irq_work_sync(&sg_policy->irq_work);
		kthread_cancel_work_sync(&sg_policy->work);
	}
635 636
}

637
static void sugov_limits(struct cpufreq_policy *policy)
638 639 640 641 642
{
	struct sugov_policy *sg_policy = policy->governor_data;

	if (!policy->fast_switch_enabled) {
		mutex_lock(&sg_policy->work_lock);
643
		cpufreq_policy_apply_limits(policy);
644 645 646 647 648 649 650 651 652
		mutex_unlock(&sg_policy->work_lock);
	}

	sg_policy->need_freq_update = true;
}

static struct cpufreq_governor schedutil_gov = {
	.name = "schedutil",
	.owner = THIS_MODULE,
653 654 655 656 657
	.init = sugov_init,
	.exit = sugov_exit,
	.start = sugov_start,
	.stop = sugov_stop,
	.limits = sugov_limits,
658 659 660 661 662 663 664 665
};

#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
struct cpufreq_governor *cpufreq_default_governor(void)
{
	return &schedutil_gov;
}
#endif
666 667 668 669 670 671

static int __init sugov_register(void)
{
	return cpufreq_register_governor(&schedutil_gov);
}
fs_initcall(sugov_register);