cpufreq_schedutil.c 19.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * CPUFreq governor based on scheduler-provided CPU utilization data.
 *
 * Copyright (C) 2016, Intel Corporation
 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

12 13
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14
#include <linux/cpufreq.h>
15
#include <linux/kthread.h>
16
#include <uapi/linux/sched/types.h>
17 18 19 20 21
#include <linux/slab.h>
#include <trace/events/power.h>

#include "sched.h"

22 23
#define SUGOV_KTHREAD_PRIORITY	50

24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
struct sugov_tunables {
	struct gov_attr_set attr_set;
	unsigned int rate_limit_us;
};

struct sugov_policy {
	struct cpufreq_policy *policy;

	struct sugov_tunables *tunables;
	struct list_head tunables_hook;

	raw_spinlock_t update_lock;  /* For shared policies */
	u64 last_freq_update_time;
	s64 freq_update_delay_ns;
	unsigned int next_freq;
39
	unsigned int cached_raw_freq;
40 41 42

	/* The next fields are only needed if fast switch cannot be used. */
	struct irq_work irq_work;
43
	struct kthread_work work;
44
	struct mutex work_lock;
45 46
	struct kthread_worker worker;
	struct task_struct *thread;
47 48 49 50 51 52 53 54
	bool work_in_progress;

	bool need_freq_update;
};

struct sugov_cpu {
	struct update_util_data update_util;
	struct sugov_policy *sg_policy;
55
	unsigned int cpu;
56

57
	bool iowait_boost_pending;
58 59
	unsigned int iowait_boost;
	unsigned int iowait_boost_max;
60
	u64 last_update;
61

62 63 64
	/* The fields below are only needed when sharing a policy. */
	unsigned long util;
	unsigned long max;
65
	unsigned int flags;
66 67 68 69 70

	/* The field below is for single-CPU policies only. */
#ifdef CONFIG_NO_HZ_COMMON
	unsigned long saved_idle_calls;
#endif
71 72 73 74 75 76 77 78 79 80
};

static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);

/************************ Governor internals ***********************/

static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
{
	s64 delta_ns;

81 82 83 84 85 86
	/*
	 * Since cpufreq_update_util() is called with rq->lock held for
	 * the @target_cpu, our per-cpu data is fully serialized.
	 *
	 * However, drivers cannot in general deal with cross-cpu
	 * requests, so while get_next_freq() will work, our
87
	 * sugov_update_commit() call may not for the fast switching platforms.
88 89 90 91
	 *
	 * Hence stop here for remote requests if they aren't supported
	 * by the hardware, as calculating the frequency is pointless if
	 * we cannot in fact act on it.
92 93 94 95
	 *
	 * For the slow switching platforms, the kthread is always scheduled on
	 * the right set of CPUs and any CPU can find the next frequency and
	 * schedule the kthread.
96
	 */
97 98
	if (sg_policy->policy->fast_switch_enabled &&
	    !cpufreq_can_do_remote_dvfs(sg_policy->policy))
99 100
		return false;

101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
	if (sg_policy->work_in_progress)
		return false;

	if (unlikely(sg_policy->need_freq_update)) {
		sg_policy->need_freq_update = false;
		/*
		 * This happens when limits change, so forget the previous
		 * next_freq value and force an update.
		 */
		sg_policy->next_freq = UINT_MAX;
		return true;
	}

	delta_ns = time - sg_policy->last_freq_update_time;
	return delta_ns >= sg_policy->freq_update_delay_ns;
}

static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
				unsigned int next_freq)
{
	struct cpufreq_policy *policy = sg_policy->policy;

123 124 125 126
	if (sg_policy->next_freq == next_freq)
		return;

	sg_policy->next_freq = next_freq;
127 128 129 130
	sg_policy->last_freq_update_time = time;

	if (policy->fast_switch_enabled) {
		next_freq = cpufreq_driver_fast_switch(policy, next_freq);
131
		if (!next_freq)
132 133 134 135
			return;

		policy->cur = next_freq;
		trace_cpu_frequency(next_freq, smp_processor_id());
136
	} else {
137 138 139 140 141 142 143
		sg_policy->work_in_progress = true;
		irq_work_queue(&sg_policy->irq_work);
	}
}

/**
 * get_next_freq - Compute a new frequency for a given cpufreq policy.
144
 * @sg_policy: schedutil policy object to compute the new frequency for.
145 146 147 148 149 150 151 152 153 154 155 156 157 158
 * @util: Current CPU utilization.
 * @max: CPU capacity.
 *
 * If the utilization is frequency-invariant, choose the new frequency to be
 * proportional to it, that is
 *
 * next_freq = C * max_freq * util / max
 *
 * Otherwise, approximate the would-be frequency-invariant utilization by
 * util_raw * (curr_freq / max_freq) which leads to
 *
 * next_freq = C * curr_freq * util_raw / max
 *
 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
159 160 161 162
 *
 * The lowest driver-supported frequency which is equal or greater than the raw
 * next_freq (as calculated above) is returned, subject to policy min/max and
 * cpufreq driver limitations.
163
 */
164 165
static unsigned int get_next_freq(struct sugov_policy *sg_policy,
				  unsigned long util, unsigned long max)
166
{
167
	struct cpufreq_policy *policy = sg_policy->policy;
168 169 170
	unsigned int freq = arch_scale_freq_invariant() ?
				policy->cpuinfo.max_freq : policy->cur;

171 172
	freq = (freq + (freq >> 2)) * util / max;

173
	if (freq == sg_policy->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
174
		return sg_policy->next_freq;
175
	sg_policy->cached_raw_freq = freq;
176
	return cpufreq_driver_resolve_freq(policy, freq);
177 178
}

179
static void sugov_get_util(unsigned long *util, unsigned long *max, int cpu)
180
{
181
	struct rq *rq = cpu_rq(cpu);
182 183
	unsigned long util_cfs = cpu_util_cfs(rq);
	unsigned long util_dl  = cpu_util_dl(rq);
184

185
	*max = arch_scale_cpu_capacity(NULL, cpu);
186

187 188 189 190 191 192
	/*
	 * Ideally we would like to set util_dl as min/guaranteed freq and
	 * util_cfs + util_dl as requested freq. However, cpufreq is not yet
	 * ready for such an interface. So, we only do the latter for now.
	 */
	*util = min(util_cfs + util_dl, *max);
193 194
}

195
static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time)
196
{
197
	if (sg_cpu->flags & SCHED_CPUFREQ_IOWAIT) {
198 199 200 201 202 203 204 205 206 207 208 209
		if (sg_cpu->iowait_boost_pending)
			return;

		sg_cpu->iowait_boost_pending = true;

		if (sg_cpu->iowait_boost) {
			sg_cpu->iowait_boost <<= 1;
			if (sg_cpu->iowait_boost > sg_cpu->iowait_boost_max)
				sg_cpu->iowait_boost = sg_cpu->iowait_boost_max;
		} else {
			sg_cpu->iowait_boost = sg_cpu->sg_policy->policy->min;
		}
210 211 212 213
	} else if (sg_cpu->iowait_boost) {
		s64 delta_ns = time - sg_cpu->last_update;

		/* Clear iowait_boost if the CPU apprears to have been idle. */
214
		if (delta_ns > TICK_NSEC) {
215
			sg_cpu->iowait_boost = 0;
216 217
			sg_cpu->iowait_boost_pending = false;
		}
218 219 220 221 222 223
	}
}

static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util,
			       unsigned long *max)
{
224
	unsigned int boost_util, boost_max;
225

226
	if (!sg_cpu->iowait_boost)
227 228
		return;

229 230 231 232 233 234 235 236 237 238 239 240 241
	if (sg_cpu->iowait_boost_pending) {
		sg_cpu->iowait_boost_pending = false;
	} else {
		sg_cpu->iowait_boost >>= 1;
		if (sg_cpu->iowait_boost < sg_cpu->sg_policy->policy->min) {
			sg_cpu->iowait_boost = 0;
			return;
		}
	}

	boost_util = sg_cpu->iowait_boost;
	boost_max = sg_cpu->iowait_boost_max;

242 243 244 245 246 247
	if (*util * boost_max < *max * boost_util) {
		*util = boost_util;
		*max = boost_max;
	}
}

248 249 250
#ifdef CONFIG_NO_HZ_COMMON
static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
{
251
	unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);
252 253 254 255 256 257 258 259 260
	bool ret = idle_calls == sg_cpu->saved_idle_calls;

	sg_cpu->saved_idle_calls = idle_calls;
	return ret;
}
#else
static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
#endif /* CONFIG_NO_HZ_COMMON */

261
static void sugov_update_single(struct update_util_data *hook, u64 time,
262
				unsigned int flags)
263 264 265 266
{
	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
	struct cpufreq_policy *policy = sg_policy->policy;
267
	unsigned long util, max;
268
	unsigned int next_f;
269
	bool busy;
270

271
	sugov_set_iowait_boost(sg_cpu, time);
272 273
	sg_cpu->last_update = time;

274 275 276
	if (!sugov_should_update_freq(sg_policy, time))
		return;

277 278
	busy = sugov_cpu_is_busy(sg_cpu);

279
	if (flags & SCHED_CPUFREQ_RT) {
280 281
		next_f = policy->cpuinfo.max_freq;
	} else {
282
		sugov_get_util(&util, &max, sg_cpu->cpu);
283
		sugov_iowait_boost(sg_cpu, &util, &max);
284
		next_f = get_next_freq(sg_policy, util, max);
285 286 287 288
		/*
		 * Do not reduce the frequency if the CPU has not been idle
		 * recently, as the reduction is likely to be premature then.
		 */
289
		if (busy && next_f < sg_policy->next_freq) {
290
			next_f = sg_policy->next_freq;
291 292 293 294

			/* Reset cached freq as next_freq has changed */
			sg_policy->cached_raw_freq = 0;
		}
295
	}
296 297 298
	sugov_update_commit(sg_policy, time, next_f);
}

299
static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
300
{
301
	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
302
	struct cpufreq_policy *policy = sg_policy->policy;
303
	unsigned long util = 0, max = 1;
304 305 306
	unsigned int j;

	for_each_cpu(j, policy->cpus) {
307
		struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
308 309 310 311 312 313 314 315
		unsigned long j_util, j_max;
		s64 delta_ns;

		/*
		 * If the CPU utilization was last updated before the previous
		 * frequency update and the time elapsed between the last update
		 * of the CPU utilization and the last frequency update is long
		 * enough, don't take the CPU into account as it probably is
316
		 * idle now (and clear iowait_boost for it).
317
		 */
318
		delta_ns = time - j_sg_cpu->last_update;
319 320
		if (delta_ns > TICK_NSEC) {
			j_sg_cpu->iowait_boost = 0;
321
			j_sg_cpu->iowait_boost_pending = false;
322
			continue;
323
		}
324
		if (j_sg_cpu->flags & SCHED_CPUFREQ_RT)
325
			return policy->cpuinfo.max_freq;
326

327
		j_util = j_sg_cpu->util;
328 329 330 331 332
		j_max = j_sg_cpu->max;
		if (j_util * max > j_max * util) {
			util = j_util;
			max = j_max;
		}
333 334

		sugov_iowait_boost(j_sg_cpu, &util, &max);
335 336
	}

337
	return get_next_freq(sg_policy, util, max);
338 339 340
}

static void sugov_update_shared(struct update_util_data *hook, u64 time,
341
				unsigned int flags)
342 343 344
{
	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
345
	unsigned long util, max;
346 347
	unsigned int next_f;

348
	sugov_get_util(&util, &max, sg_cpu->cpu);
349

350 351 352 353
	raw_spin_lock(&sg_policy->update_lock);

	sg_cpu->util = util;
	sg_cpu->max = max;
354
	sg_cpu->flags = flags;
355

356
	sugov_set_iowait_boost(sg_cpu, time);
357 358 359
	sg_cpu->last_update = time;

	if (sugov_should_update_freq(sg_policy, time)) {
360
		if (flags & SCHED_CPUFREQ_RT)
361 362
			next_f = sg_policy->policy->cpuinfo.max_freq;
		else
363
			next_f = sugov_next_freq_shared(sg_cpu, time);
364

365 366 367 368 369 370
		sugov_update_commit(sg_policy, time, next_f);
	}

	raw_spin_unlock(&sg_policy->update_lock);
}

371
static void sugov_work(struct kthread_work *work)
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
{
	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);

	mutex_lock(&sg_policy->work_lock);
	__cpufreq_driver_target(sg_policy->policy, sg_policy->next_freq,
				CPUFREQ_RELATION_L);
	mutex_unlock(&sg_policy->work_lock);

	sg_policy->work_in_progress = false;
}

static void sugov_irq_work(struct irq_work *irq_work)
{
	struct sugov_policy *sg_policy;

	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);
388 389

	/*
390 391 392
	 * For RT tasks, the schedutil governor shoots the frequency to maximum.
	 * Special care must be taken to ensure that this kthread doesn't result
	 * in the same behavior.
393 394
	 *
	 * This is (mostly) guaranteed by the work_in_progress flag. The flag is
395 396
	 * updated only at the end of the sugov_work() function and before that
	 * the schedutil governor rejects all other frequency scaling requests.
397
	 *
398
	 * There is a very rare case though, where the RT thread yields right
399 400 401 402
	 * after the work_in_progress flag is cleared. The effects of that are
	 * neglected for now.
	 */
	kthread_queue_work(&sg_policy->worker, &sg_policy->work);
403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
}

/************************** sysfs interface ************************/

static struct sugov_tunables *global_tunables;
static DEFINE_MUTEX(global_tunables_lock);

static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr_set)
{
	return container_of(attr_set, struct sugov_tunables, attr_set);
}

static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
{
	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

	return sprintf(buf, "%u\n", tunables->rate_limit_us);
}

static ssize_t rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf,
				   size_t count)
{
	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
	struct sugov_policy *sg_policy;
	unsigned int rate_limit_us;

	if (kstrtouint(buf, 10, &rate_limit_us))
		return -EINVAL;

	tunables->rate_limit_us = rate_limit_us;

	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;

	return count;
}

static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us);

static struct attribute *sugov_attributes[] = {
	&rate_limit_us.attr,
	NULL
};

static struct kobj_type sugov_tunables_ktype = {
	.default_attrs = sugov_attributes,
	.sysfs_ops = &governor_sysfs_ops,
};

/********************** cpufreq governor interface *********************/

static struct cpufreq_governor schedutil_gov;

static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy)
{
	struct sugov_policy *sg_policy;

	sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);
	if (!sg_policy)
		return NULL;

	sg_policy->policy = policy;
	raw_spin_lock_init(&sg_policy->update_lock);
	return sg_policy;
}

static void sugov_policy_free(struct sugov_policy *sg_policy)
{
	kfree(sg_policy);
}

474 475 476
static int sugov_kthread_create(struct sugov_policy *sg_policy)
{
	struct task_struct *thread;
477 478 479 480 481 482 483 484 485 486 487 488 489 490
	struct sched_attr attr = {
		.size = sizeof(struct sched_attr),
		.sched_policy = SCHED_DEADLINE,
		.sched_flags = SCHED_FLAG_SUGOV,
		.sched_nice = 0,
		.sched_priority = 0,
		/*
		 * Fake (unused) bandwidth; workaround to "fix"
		 * priority inheritance.
		 */
		.sched_runtime	=  1000000,
		.sched_deadline = 10000000,
		.sched_period	= 10000000,
	};
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
	struct cpufreq_policy *policy = sg_policy->policy;
	int ret;

	/* kthread only required for slow path */
	if (policy->fast_switch_enabled)
		return 0;

	kthread_init_work(&sg_policy->work, sugov_work);
	kthread_init_worker(&sg_policy->worker);
	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
				"sugov:%d",
				cpumask_first(policy->related_cpus));
	if (IS_ERR(thread)) {
		pr_err("failed to create sugov thread: %ld\n", PTR_ERR(thread));
		return PTR_ERR(thread);
	}

508
	ret = sched_setattr_nocheck(thread, &attr);
509 510
	if (ret) {
		kthread_stop(thread);
511
		pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__);
512 513 514 515
		return ret;
	}

	sg_policy->thread = thread;
516 517 518 519 520

	/* Kthread is bound to all CPUs by default */
	if (!policy->dvfs_possible_from_any_cpu)
		kthread_bind_mask(thread, policy->related_cpus);

521 522 523
	init_irq_work(&sg_policy->irq_work, sugov_irq_work);
	mutex_init(&sg_policy->work_lock);

524 525 526 527 528 529 530 531 532 533 534 535 536
	wake_up_process(thread);

	return 0;
}

static void sugov_kthread_stop(struct sugov_policy *sg_policy)
{
	/* kthread only required for slow path */
	if (sg_policy->policy->fast_switch_enabled)
		return;

	kthread_flush_worker(&sg_policy->worker);
	kthread_stop(sg_policy->thread);
537
	mutex_destroy(&sg_policy->work_lock);
538 539
}

540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)
{
	struct sugov_tunables *tunables;

	tunables = kzalloc(sizeof(*tunables), GFP_KERNEL);
	if (tunables) {
		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
		if (!have_governor_per_policy())
			global_tunables = tunables;
	}
	return tunables;
}

static void sugov_tunables_free(struct sugov_tunables *tunables)
{
	if (!have_governor_per_policy())
		global_tunables = NULL;

	kfree(tunables);
}

static int sugov_init(struct cpufreq_policy *policy)
{
	struct sugov_policy *sg_policy;
	struct sugov_tunables *tunables;
	int ret = 0;

	/* State should be equivalent to EXIT */
	if (policy->governor_data)
		return -EBUSY;

571 572
	cpufreq_enable_fast_switch(policy);

573
	sg_policy = sugov_policy_alloc(policy);
574 575 576 577
	if (!sg_policy) {
		ret = -ENOMEM;
		goto disable_fast_switch;
	}
578

579 580 581 582
	ret = sugov_kthread_create(sg_policy);
	if (ret)
		goto free_sg_policy;

583 584 585 586 587
	mutex_lock(&global_tunables_lock);

	if (global_tunables) {
		if (WARN_ON(have_governor_per_policy())) {
			ret = -EINVAL;
588
			goto stop_kthread;
589 590 591 592 593 594 595 596 597 598 599
		}
		policy->governor_data = sg_policy;
		sg_policy->tunables = global_tunables;

		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
		goto out;
	}

	tunables = sugov_tunables_alloc(sg_policy);
	if (!tunables) {
		ret = -ENOMEM;
600
		goto stop_kthread;
601 602
	}

603
	tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);
604 605 606 607 608 609 610 611 612 613

	policy->governor_data = sg_policy;
	sg_policy->tunables = tunables;

	ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
				   get_governor_parent_kobj(policy), "%s",
				   schedutil_gov.name);
	if (ret)
		goto fail;

614
out:
615 616 617
	mutex_unlock(&global_tunables_lock);
	return 0;

618
fail:
619 620 621
	policy->governor_data = NULL;
	sugov_tunables_free(tunables);

622 623 624
stop_kthread:
	sugov_kthread_stop(sg_policy);

625
free_sg_policy:
626 627 628
	mutex_unlock(&global_tunables_lock);

	sugov_policy_free(sg_policy);
629 630 631 632

disable_fast_switch:
	cpufreq_disable_fast_switch(policy);

633
	pr_err("initialization failed (error %d)\n", ret);
634 635 636
	return ret;
}

637
static void sugov_exit(struct cpufreq_policy *policy)
638 639 640 641 642 643 644 645 646 647 648 649 650 651
{
	struct sugov_policy *sg_policy = policy->governor_data;
	struct sugov_tunables *tunables = sg_policy->tunables;
	unsigned int count;

	mutex_lock(&global_tunables_lock);

	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
	policy->governor_data = NULL;
	if (!count)
		sugov_tunables_free(tunables);

	mutex_unlock(&global_tunables_lock);

652
	sugov_kthread_stop(sg_policy);
653
	sugov_policy_free(sg_policy);
654
	cpufreq_disable_fast_switch(policy);
655 656 657 658 659 660 661 662 663 664 665 666
}

static int sugov_start(struct cpufreq_policy *policy)
{
	struct sugov_policy *sg_policy = policy->governor_data;
	unsigned int cpu;

	sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
	sg_policy->last_freq_update_time = 0;
	sg_policy->next_freq = UINT_MAX;
	sg_policy->work_in_progress = false;
	sg_policy->need_freq_update = false;
667
	sg_policy->cached_raw_freq = 0;
668 669 670 671

	for_each_cpu(cpu, policy->cpus) {
		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);

672
		memset(sg_cpu, 0, sizeof(*sg_cpu));
673
		sg_cpu->cpu = cpu;
674
		sg_cpu->sg_policy = sg_policy;
675
		sg_cpu->flags = 0;
676
		sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
677 678 679 680 681
	}

	for_each_cpu(cpu, policy->cpus) {
		struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);

682 683 684 685
		cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
					     policy_is_shared(policy) ?
							sugov_update_shared :
							sugov_update_single);
686 687 688 689
	}
	return 0;
}

690
static void sugov_stop(struct cpufreq_policy *policy)
691 692 693 694 695 696 697 698 699
{
	struct sugov_policy *sg_policy = policy->governor_data;
	unsigned int cpu;

	for_each_cpu(cpu, policy->cpus)
		cpufreq_remove_update_util_hook(cpu);

	synchronize_sched();

700 701 702 703
	if (!policy->fast_switch_enabled) {
		irq_work_sync(&sg_policy->irq_work);
		kthread_cancel_work_sync(&sg_policy->work);
	}
704 705
}

706
static void sugov_limits(struct cpufreq_policy *policy)
707 708 709 710 711
{
	struct sugov_policy *sg_policy = policy->governor_data;

	if (!policy->fast_switch_enabled) {
		mutex_lock(&sg_policy->work_lock);
712
		cpufreq_policy_apply_limits(policy);
713 714 715 716 717 718 719 720 721
		mutex_unlock(&sg_policy->work_lock);
	}

	sg_policy->need_freq_update = true;
}

static struct cpufreq_governor schedutil_gov = {
	.name = "schedutil",
	.owner = THIS_MODULE,
722
	.dynamic_switching = true,
723 724 725 726 727
	.init = sugov_init,
	.exit = sugov_exit,
	.start = sugov_start,
	.stop = sugov_stop,
	.limits = sugov_limits,
728 729 730 731 732 733 734 735
};

#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
struct cpufreq_governor *cpufreq_default_governor(void)
{
	return &schedutil_gov;
}
#endif
736 737 738 739 740 741

static int __init sugov_register(void)
{
	return cpufreq_register_governor(&schedutil_gov);
}
fs_initcall(sugov_register);