cpufreq_ondemand.c 13.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 *  drivers/cpufreq/cpufreq_ondemand.c
 *
 *  Copyright (C)  2001 Russell King
 *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *                      Jun Nakajima <jun.nakajima@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/ctype.h>
#include <linux/cpufreq.h>
#include <linux/sysctl.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/sysfs.h>
#include <linux/sched.h>
#include <linux/kmod.h>
#include <linux/workqueue.h>
#include <linux/jiffies.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>

/*
 * dbs is used in this file as a shortform for demandbased switching
 * It helps to keep variable names smaller, simpler
 */

#define DEF_FREQUENCY_UP_THRESHOLD		(80)
37
#define MIN_FREQUENCY_UP_THRESHOLD		(11)
L
Linus Torvalds 已提交
38 39 40 41 42 43 44 45 46 47 48 49 50
#define MAX_FREQUENCY_UP_THRESHOLD		(100)

/* 
 * The polling frequency of this governor depends on the capability of 
 * the processor. Default polling frequency is 1000 times the transition
 * latency of the processor. The governor will work on any processor with 
 * transition latency <= 10mS, using appropriate sampling 
 * rate.
 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
 * this governor will not work.
 * All times here are in uS.
 */
static unsigned int 				def_sampling_rate;
51 52 53 54
#define MIN_SAMPLING_RATE_RATIO			(2)
/* for correct statistics, we need at least 10 ticks between each measure */
#define MIN_STAT_SAMPLING_RATE			(MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10))
#define MIN_SAMPLING_RATE			(def_sampling_rate / MIN_SAMPLING_RATE_RATIO)
L
Linus Torvalds 已提交
55 56
#define MAX_SAMPLING_RATE			(500 * def_sampling_rate)
#define DEF_SAMPLING_RATE_LATENCY_MULTIPLIER	(1000)
57 58
#define DEF_SAMPLING_DOWN_FACTOR		(1)
#define MAX_SAMPLING_DOWN_FACTOR		(10)
L
Linus Torvalds 已提交
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
#define TRANSITION_LATENCY_LIMIT		(10 * 1000)

static void do_dbs_timer(void *data);

struct cpu_dbs_info_s {
	struct cpufreq_policy 	*cur_policy;
	unsigned int 		prev_cpu_idle_up;
	unsigned int 		prev_cpu_idle_down;
	unsigned int 		enable;
};
static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);

static unsigned int dbs_enable;	/* number of CPUs using this policy */

static DECLARE_MUTEX 	(dbs_sem);
static DECLARE_WORK	(dbs_work, do_dbs_timer, NULL);

struct dbs_tuners {
	unsigned int 		sampling_rate;
	unsigned int		sampling_down_factor;
	unsigned int		up_threshold;
80
	unsigned int		ignore_nice;
L
Linus Torvalds 已提交
81 82 83 84 85 86 87
};

static struct dbs_tuners dbs_tuners_ins = {
	.up_threshold 		= DEF_FREQUENCY_UP_THRESHOLD,
	.sampling_down_factor 	= DEF_SAMPLING_DOWN_FACTOR,
};

88 89 90 91 92 93 94 95 96
static inline unsigned int get_cpu_idle_time(unsigned int cpu)
{
	return	kstat_cpu(cpu).cpustat.idle +
		kstat_cpu(cpu).cpustat.iowait +
		( !dbs_tuners_ins.ignore_nice ? 
		  kstat_cpu(cpu).cpustat.nice :
		  0);
}

L
Linus Torvalds 已提交
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
/************************** sysfs interface ************************/
static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
{
	return sprintf (buf, "%u\n", MAX_SAMPLING_RATE);
}

static ssize_t show_sampling_rate_min(struct cpufreq_policy *policy, char *buf)
{
	return sprintf (buf, "%u\n", MIN_SAMPLING_RATE);
}

#define define_one_ro(_name) 					\
static struct freq_attr _name =  				\
__ATTR(_name, 0444, show_##_name, NULL)

define_one_ro(sampling_rate_max);
define_one_ro(sampling_rate_min);

/* cpufreq_ondemand Governor Tunables */
#define show_one(file_name, object)					\
static ssize_t show_##file_name						\
(struct cpufreq_policy *unused, char *buf)				\
{									\
	return sprintf(buf, "%u\n", dbs_tuners_ins.object);		\
}
show_one(sampling_rate, sampling_rate);
show_one(sampling_down_factor, sampling_down_factor);
show_one(up_threshold, up_threshold);
125
show_one(ignore_nice, ignore_nice);
L
Linus Torvalds 已提交
126 127 128 129 130 131 132 133 134 135

static ssize_t store_sampling_down_factor(struct cpufreq_policy *unused, 
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf (buf, "%u", &input);
	if (ret != 1 )
		return -EINVAL;

136 137 138
	if (input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
		return -EINVAL;

L
Linus Torvalds 已提交
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
	down(&dbs_sem);
	dbs_tuners_ins.sampling_down_factor = input;
	up(&dbs_sem);

	return count;
}

static ssize_t store_sampling_rate(struct cpufreq_policy *unused, 
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf (buf, "%u", &input);

	down(&dbs_sem);
	if (ret != 1 || input > MAX_SAMPLING_RATE || input < MIN_SAMPLING_RATE) {
		up(&dbs_sem);
		return -EINVAL;
	}

	dbs_tuners_ins.sampling_rate = input;
	up(&dbs_sem);

	return count;
}

static ssize_t store_up_threshold(struct cpufreq_policy *unused, 
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf (buf, "%u", &input);

	down(&dbs_sem);
	if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD || 
174
			input < MIN_FREQUENCY_UP_THRESHOLD) {
L
Linus Torvalds 已提交
175 176 177 178 179 180 181 182 183 184
		up(&dbs_sem);
		return -EINVAL;
	}

	dbs_tuners_ins.up_threshold = input;
	up(&dbs_sem);

	return count;
}

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
static ssize_t store_ignore_nice(struct cpufreq_policy *policy,
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;

	unsigned int j;
	
	ret = sscanf (buf, "%u", &input);
	if ( ret != 1 )
		return -EINVAL;

	if ( input > 1 )
		input = 1;
	
	down(&dbs_sem);
	if ( input == dbs_tuners_ins.ignore_nice ) { /* nothing to do */
		up(&dbs_sem);
		return count;
	}
	dbs_tuners_ins.ignore_nice = input;

	/* we need to re-evaluate prev_cpu_idle_up and prev_cpu_idle_down */
208
	for_each_online_cpu(j) {
209 210
		struct cpu_dbs_info_s *j_dbs_info;
		j_dbs_info = &per_cpu(cpu_dbs_info, j);
211
		j_dbs_info->prev_cpu_idle_up = get_cpu_idle_time(j);
212 213 214 215 216 217 218
		j_dbs_info->prev_cpu_idle_down = j_dbs_info->prev_cpu_idle_up;
	}
	up(&dbs_sem);

	return count;
}

L
Linus Torvalds 已提交
219 220 221 222 223 224 225
#define define_one_rw(_name) \
static struct freq_attr _name = \
__ATTR(_name, 0644, show_##_name, store_##_name)

define_one_rw(sampling_rate);
define_one_rw(sampling_down_factor);
define_one_rw(up_threshold);
226
define_one_rw(ignore_nice);
L
Linus Torvalds 已提交
227 228 229 230 231 232 233

static struct attribute * dbs_attributes[] = {
	&sampling_rate_max.attr,
	&sampling_rate_min.attr,
	&sampling_rate.attr,
	&sampling_down_factor.attr,
	&up_threshold.attr,
234
	&ignore_nice.attr,
L
Linus Torvalds 已提交
235 236 237 238 239 240 241 242 243 244 245 246
	NULL
};

static struct attribute_group dbs_attr_group = {
	.attrs = dbs_attributes,
	.name = "ondemand",
};

/************************** sysfs end ************************/

static void dbs_check_cpu(int cpu)
{
247 248
	unsigned int idle_ticks, up_idle_ticks, total_ticks;
	unsigned int freq_next;
L
Linus Torvalds 已提交
249 250 251 252 253 254 255 256 257 258 259 260 261
	unsigned int freq_down_sampling_rate;
	static int down_skip[NR_CPUS];
	struct cpu_dbs_info_s *this_dbs_info;

	struct cpufreq_policy *policy;
	unsigned int j;

	this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
	if (!this_dbs_info->enable)
		return;

	policy = this_dbs_info->cur_policy;
	/* 
262 263 264 265 266
	 * Every sampling_rate, we check, if current idle time is less
	 * than 20% (default), then we try to increase frequency
	 * Every sampling_rate*sampling_down_factor, we look for a the lowest
	 * frequency which can sustain the load while keeping idle time over
	 * 30%. If such a frequency exist, we try to decrease to this frequency.
L
Linus Torvalds 已提交
267 268 269
	 *
	 * Any frequency increase takes it to the maximum frequency. 
	 * Frequency reduction happens at minimum steps of 
270
	 * 5% (default) of current frequency 
L
Linus Torvalds 已提交
271 272 273
	 */

	/* Check for frequency increase */
274
	idle_ticks = UINT_MAX;
L
Linus Torvalds 已提交
275
	for_each_cpu_mask(j, policy->cpus) {
276
		unsigned int tmp_idle_ticks, total_idle_ticks;
L
Linus Torvalds 已提交
277 278 279
		struct cpu_dbs_info_s *j_dbs_info;

		j_dbs_info = &per_cpu(cpu_dbs_info, j);
280
		total_idle_ticks = get_cpu_idle_time(j);
L
Linus Torvalds 已提交
281 282 283 284 285 286 287 288 289 290 291
		tmp_idle_ticks = total_idle_ticks -
			j_dbs_info->prev_cpu_idle_up;
		j_dbs_info->prev_cpu_idle_up = total_idle_ticks;

		if (tmp_idle_ticks < idle_ticks)
			idle_ticks = tmp_idle_ticks;
	}

	/* Scale idle ticks by 100 and compare with up and down ticks */
	idle_ticks *= 100;
	up_idle_ticks = (100 - dbs_tuners_ins.up_threshold) *
292
			usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
L
Linus Torvalds 已提交
293 294

	if (idle_ticks < up_idle_ticks) {
295
		down_skip[cpu] = 0;
296 297 298 299 300 301 302
		for_each_cpu_mask(j, policy->cpus) {
			struct cpu_dbs_info_s *j_dbs_info;

			j_dbs_info = &per_cpu(cpu_dbs_info, j);
			j_dbs_info->prev_cpu_idle_down = 
					j_dbs_info->prev_cpu_idle_up;
		}
303 304 305 306
		/* if we are already at full speed then break out early */
		if (policy->cur == policy->max)
			return;
		
L
Linus Torvalds 已提交
307 308 309 310 311 312 313 314 315 316
		__cpufreq_driver_target(policy, policy->max, 
			CPUFREQ_RELATION_H);
		return;
	}

	/* Check for frequency decrease */
	down_skip[cpu]++;
	if (down_skip[cpu] < dbs_tuners_ins.sampling_down_factor)
		return;

317
	idle_ticks = UINT_MAX;
L
Linus Torvalds 已提交
318
	for_each_cpu_mask(j, policy->cpus) {
319
		unsigned int tmp_idle_ticks, total_idle_ticks;
L
Linus Torvalds 已提交
320 321 322
		struct cpu_dbs_info_s *j_dbs_info;

		j_dbs_info = &per_cpu(cpu_dbs_info, j);
323 324
		/* Check for frequency decrease */
		total_idle_ticks = j_dbs_info->prev_cpu_idle_up;
L
Linus Torvalds 已提交
325 326 327 328 329 330 331 332 333
		tmp_idle_ticks = total_idle_ticks -
			j_dbs_info->prev_cpu_idle_down;
		j_dbs_info->prev_cpu_idle_down = total_idle_ticks;

		if (tmp_idle_ticks < idle_ticks)
			idle_ticks = tmp_idle_ticks;
	}

	down_skip[cpu] = 0;
334 335 336
	/* if we cannot reduce the frequency anymore, break out early */
	if (policy->cur == policy->min)
		return;
L
Linus Torvalds 已提交
337

338
	/* Compute how many ticks there are between two measurements */
L
Linus Torvalds 已提交
339 340
	freq_down_sampling_rate = dbs_tuners_ins.sampling_rate *
		dbs_tuners_ins.sampling_down_factor;
341
	total_ticks = usecs_to_jiffies(freq_down_sampling_rate);
342

343 344 345 346 347 348 349 350
	/*
	 * The optimal frequency is the frequency that is the lowest that
	 * can support the current CPU usage without triggering the up
	 * policy. To be safe, we focus 10 points under the threshold.
	 */
	freq_next = ((total_ticks - idle_ticks) * 100) / total_ticks;
	freq_next = (freq_next * policy->cur) / 
			(dbs_tuners_ins.up_threshold - 10);
L
Linus Torvalds 已提交
351

352 353
	if (freq_next <= ((policy->cur * 95) / 100))
		__cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L);
L
Linus Torvalds 已提交
354 355 356 357 358 359
}

static void do_dbs_timer(void *data)
{ 
	int i;
	down(&dbs_sem);
360 361
	for_each_online_cpu(i)
		dbs_check_cpu(i);
L
Linus Torvalds 已提交
362
	schedule_delayed_work(&dbs_work, 
363
			usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
L
Linus Torvalds 已提交
364 365 366 367 368 369 370
	up(&dbs_sem);
} 

static inline void dbs_timer_init(void)
{
	INIT_WORK(&dbs_work, do_dbs_timer, NULL);
	schedule_delayed_work(&dbs_work,
371
			usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
L
Linus Torvalds 已提交
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
	return;
}

static inline void dbs_timer_exit(void)
{
	cancel_delayed_work(&dbs_work);
	return;
}

static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
				   unsigned int event)
{
	unsigned int cpu = policy->cpu;
	struct cpu_dbs_info_s *this_dbs_info;
	unsigned int j;

	this_dbs_info = &per_cpu(cpu_dbs_info, cpu);

	switch (event) {
	case CPUFREQ_GOV_START:
		if ((!cpu_online(cpu)) || 
		    (!policy->cur))
			return -EINVAL;

		if (policy->cpuinfo.transition_latency >
				(TRANSITION_LATENCY_LIMIT * 1000))
			return -EINVAL;
		if (this_dbs_info->enable) /* Already enabled */
			break;
		 
		down(&dbs_sem);
		for_each_cpu_mask(j, policy->cpus) {
			struct cpu_dbs_info_s *j_dbs_info;
			j_dbs_info = &per_cpu(cpu_dbs_info, j);
			j_dbs_info->cur_policy = policy;
		
408
			j_dbs_info->prev_cpu_idle_up = get_cpu_idle_time(j);
409 410
			j_dbs_info->prev_cpu_idle_down
				= j_dbs_info->prev_cpu_idle_up;
L
Linus Torvalds 已提交
411 412 413 414 415 416 417 418 419 420 421
		}
		this_dbs_info->enable = 1;
		sysfs_create_group(&policy->kobj, &dbs_attr_group);
		dbs_enable++;
		/*
		 * Start the timerschedule work, when this governor
		 * is used for first time
		 */
		if (dbs_enable == 1) {
			unsigned int latency;
			/* policy latency is in nS. Convert it to uS first */
422 423 424
			latency = policy->cpuinfo.transition_latency / 1000;
			if (latency == 0)
				latency = 1;
L
Linus Torvalds 已提交
425

426
			def_sampling_rate = latency *
L
Linus Torvalds 已提交
427
					DEF_SAMPLING_RATE_LATENCY_MULTIPLIER;
428 429 430 431

			if (def_sampling_rate < MIN_STAT_SAMPLING_RATE)
				def_sampling_rate = MIN_STAT_SAMPLING_RATE;

L
Linus Torvalds 已提交
432
			dbs_tuners_ins.sampling_rate = def_sampling_rate;
433
			dbs_tuners_ins.ignore_nice = 0;
L
Linus Torvalds 已提交
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472

			dbs_timer_init();
		}
		
		up(&dbs_sem);
		break;

	case CPUFREQ_GOV_STOP:
		down(&dbs_sem);
		this_dbs_info->enable = 0;
		sysfs_remove_group(&policy->kobj, &dbs_attr_group);
		dbs_enable--;
		/*
		 * Stop the timerschedule work, when this governor
		 * is used for first time
		 */
		if (dbs_enable == 0) 
			dbs_timer_exit();
		
		up(&dbs_sem);

		break;

	case CPUFREQ_GOV_LIMITS:
		down(&dbs_sem);
		if (policy->max < this_dbs_info->cur_policy->cur)
			__cpufreq_driver_target(
					this_dbs_info->cur_policy,
				       	policy->max, CPUFREQ_RELATION_H);
		else if (policy->min > this_dbs_info->cur_policy->cur)
			__cpufreq_driver_target(
					this_dbs_info->cur_policy,
				       	policy->min, CPUFREQ_RELATION_L);
		up(&dbs_sem);
		break;
	}
	return 0;
}

473
static struct cpufreq_governor cpufreq_gov_dbs = {
L
Linus Torvalds 已提交
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
	.name		= "ondemand",
	.governor	= cpufreq_governor_dbs,
	.owner		= THIS_MODULE,
};

static int __init cpufreq_gov_dbs_init(void)
{
	return cpufreq_register_governor(&cpufreq_gov_dbs);
}

static void __exit cpufreq_gov_dbs_exit(void)
{
	/* Make sure that the scheduled work is indeed not running */
	flush_scheduled_work();

	cpufreq_unregister_governor(&cpufreq_gov_dbs);
}


MODULE_AUTHOR ("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
MODULE_DESCRIPTION ("'cpufreq_ondemand' - A dynamic cpufreq governor for "
		"Low Latency Frequency Transition capable processors");
MODULE_LICENSE ("GPL");

module_init(cpufreq_gov_dbs_init);
module_exit(cpufreq_gov_dbs_exit);