cpufreq_ondemand.c 14.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
/*
 *  drivers/cpufreq/cpufreq_ondemand.c
 *
 *  Copyright (C)  2001 Russell King
 *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *                      Jun Nakajima <jun.nakajima@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/ctype.h>
#include <linux/cpufreq.h>
#include <linux/sysctl.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/sysfs.h>
#include <linux/sched.h>
#include <linux/kmod.h>
#include <linux/workqueue.h>
#include <linux/jiffies.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>

/*
 * dbs is used in this file as a shortform for demandbased switching
 * It helps to keep variable names smaller, simpler
 */

#define DEF_FREQUENCY_UP_THRESHOLD		(80)
#define MIN_FREQUENCY_UP_THRESHOLD		(0)
#define MAX_FREQUENCY_UP_THRESHOLD		(100)

#define DEF_FREQUENCY_DOWN_THRESHOLD		(20)
#define MIN_FREQUENCY_DOWN_THRESHOLD		(0)
#define MAX_FREQUENCY_DOWN_THRESHOLD		(100)

/* 
 * The polling frequency of this governor depends on the capability of 
 * the processor. Default polling frequency is 1000 times the transition
 * latency of the processor. The governor will work on any processor with 
 * transition latency <= 10mS, using appropriate sampling 
 * rate.
 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
 * this governor will not work.
 * All times here are in uS.
 */
static unsigned int 				def_sampling_rate;
#define MIN_SAMPLING_RATE			(def_sampling_rate / 2)
#define MAX_SAMPLING_RATE			(500 * def_sampling_rate)
#define DEF_SAMPLING_RATE_LATENCY_MULTIPLIER	(1000)
#define DEF_SAMPLING_DOWN_FACTOR		(10)
#define TRANSITION_LATENCY_LIMIT		(10 * 1000)

static void do_dbs_timer(void *data);

struct cpu_dbs_info_s {
	struct cpufreq_policy 	*cur_policy;
	unsigned int 		prev_cpu_idle_up;
	unsigned int 		prev_cpu_idle_down;
	unsigned int 		enable;
};
static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);

static unsigned int dbs_enable;	/* number of CPUs using this policy */

static DECLARE_MUTEX 	(dbs_sem);
static DECLARE_WORK	(dbs_work, do_dbs_timer, NULL);

struct dbs_tuners {
	unsigned int 		sampling_rate;
	unsigned int		sampling_down_factor;
	unsigned int		up_threshold;
	unsigned int		down_threshold;
81
	unsigned int		ignore_nice;
L
Linus Torvalds 已提交
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
};

static struct dbs_tuners dbs_tuners_ins = {
	.up_threshold 		= DEF_FREQUENCY_UP_THRESHOLD,
	.down_threshold 	= DEF_FREQUENCY_DOWN_THRESHOLD,
	.sampling_down_factor 	= DEF_SAMPLING_DOWN_FACTOR,
};

/************************** sysfs interface ************************/
static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
{
	return sprintf (buf, "%u\n", MAX_SAMPLING_RATE);
}

static ssize_t show_sampling_rate_min(struct cpufreq_policy *policy, char *buf)
{
	return sprintf (buf, "%u\n", MIN_SAMPLING_RATE);
}

#define define_one_ro(_name) 					\
static struct freq_attr _name =  				\
__ATTR(_name, 0444, show_##_name, NULL)

define_one_ro(sampling_rate_max);
define_one_ro(sampling_rate_min);

/* cpufreq_ondemand Governor Tunables */
#define show_one(file_name, object)					\
static ssize_t show_##file_name						\
(struct cpufreq_policy *unused, char *buf)				\
{									\
	return sprintf(buf, "%u\n", dbs_tuners_ins.object);		\
}
show_one(sampling_rate, sampling_rate);
show_one(sampling_down_factor, sampling_down_factor);
show_one(up_threshold, up_threshold);
show_one(down_threshold, down_threshold);
119
show_one(ignore_nice, ignore_nice);
L
Linus Torvalds 已提交
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197

static ssize_t store_sampling_down_factor(struct cpufreq_policy *unused, 
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf (buf, "%u", &input);
	if (ret != 1 )
		return -EINVAL;

	down(&dbs_sem);
	dbs_tuners_ins.sampling_down_factor = input;
	up(&dbs_sem);

	return count;
}

static ssize_t store_sampling_rate(struct cpufreq_policy *unused, 
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf (buf, "%u", &input);

	down(&dbs_sem);
	if (ret != 1 || input > MAX_SAMPLING_RATE || input < MIN_SAMPLING_RATE) {
		up(&dbs_sem);
		return -EINVAL;
	}

	dbs_tuners_ins.sampling_rate = input;
	up(&dbs_sem);

	return count;
}

static ssize_t store_up_threshold(struct cpufreq_policy *unused, 
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf (buf, "%u", &input);

	down(&dbs_sem);
	if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD || 
			input < MIN_FREQUENCY_UP_THRESHOLD ||
			input <= dbs_tuners_ins.down_threshold) {
		up(&dbs_sem);
		return -EINVAL;
	}

	dbs_tuners_ins.up_threshold = input;
	up(&dbs_sem);

	return count;
}

static ssize_t store_down_threshold(struct cpufreq_policy *unused, 
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;
	ret = sscanf (buf, "%u", &input);

	down(&dbs_sem);
	if (ret != 1 || input > MAX_FREQUENCY_DOWN_THRESHOLD || 
			input < MIN_FREQUENCY_DOWN_THRESHOLD ||
			input >= dbs_tuners_ins.up_threshold) {
		up(&dbs_sem);
		return -EINVAL;
	}

	dbs_tuners_ins.down_threshold = input;
	up(&dbs_sem);

	return count;
}

198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
static ssize_t store_ignore_nice(struct cpufreq_policy *policy,
		const char *buf, size_t count)
{
	unsigned int input;
	int ret;

	unsigned int j;
	
	ret = sscanf (buf, "%u", &input);
	if ( ret != 1 )
		return -EINVAL;

	if ( input > 1 )
		input = 1;
	
	down(&dbs_sem);
	if ( input == dbs_tuners_ins.ignore_nice ) { /* nothing to do */
		up(&dbs_sem);
		return count;
	}
	dbs_tuners_ins.ignore_nice = input;

	/* we need to re-evaluate prev_cpu_idle_up and prev_cpu_idle_down */
	for_each_cpu_mask(j, policy->cpus) {
		struct cpu_dbs_info_s *j_dbs_info;
		j_dbs_info = &per_cpu(cpu_dbs_info, j);
		j_dbs_info->cur_policy = policy;

		j_dbs_info->prev_cpu_idle_up =
			kstat_cpu(j).cpustat.idle +
			kstat_cpu(j).cpustat.iowait +
			( !dbs_tuners_ins.ignore_nice
			  ? kstat_cpu(j).cpustat.nice : 0 );
		j_dbs_info->prev_cpu_idle_down = j_dbs_info->prev_cpu_idle_up;
	}
	up(&dbs_sem);

	return count;
}

L
Linus Torvalds 已提交
238 239 240 241 242 243 244 245
#define define_one_rw(_name) \
static struct freq_attr _name = \
__ATTR(_name, 0644, show_##_name, store_##_name)

define_one_rw(sampling_rate);
define_one_rw(sampling_down_factor);
define_one_rw(up_threshold);
define_one_rw(down_threshold);
246
define_one_rw(ignore_nice);
L
Linus Torvalds 已提交
247 248 249 250 251 252 253 254

static struct attribute * dbs_attributes[] = {
	&sampling_rate_max.attr,
	&sampling_rate_min.attr,
	&sampling_rate.attr,
	&sampling_down_factor.attr,
	&up_threshold.attr,
	&down_threshold.attr,
255
	&ignore_nice.attr,
L
Linus Torvalds 已提交
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
	NULL
};

static struct attribute_group dbs_attr_group = {
	.attrs = dbs_attributes,
	.name = "ondemand",
};

/************************** sysfs end ************************/

static void dbs_check_cpu(int cpu)
{
	unsigned int idle_ticks, up_idle_ticks, down_idle_ticks;
	unsigned int total_idle_ticks;
	unsigned int freq_down_step;
	unsigned int freq_down_sampling_rate;
	static int down_skip[NR_CPUS];
	struct cpu_dbs_info_s *this_dbs_info;

	struct cpufreq_policy *policy;
	unsigned int j;

	this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
	if (!this_dbs_info->enable)
		return;

	policy = this_dbs_info->cur_policy;
	/* 
	 * The default safe range is 20% to 80% 
	 * Every sampling_rate, we check
	 * 	- If current idle time is less than 20%, then we try to 
	 * 	  increase frequency
	 * Every sampling_rate*sampling_down_factor, we check
	 * 	- If current idle time is more than 80%, then we try to
	 * 	  decrease frequency
	 *
	 * Any frequency increase takes it to the maximum frequency. 
	 * Frequency reduction happens at minimum steps of 
	 * 5% of max_frequency 
	 */

	/* Check for frequency increase */
	total_idle_ticks = kstat_cpu(cpu).cpustat.idle +
		kstat_cpu(cpu).cpustat.iowait;
300 301 302
	  /* consider 'nice' tasks as 'idle' time too if required */
	  if (dbs_tuners_ins.ignore_nice == 0)
		total_idle_ticks += kstat_cpu(cpu).cpustat.nice;
L
Linus Torvalds 已提交
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
	idle_ticks = total_idle_ticks -
		this_dbs_info->prev_cpu_idle_up;
	this_dbs_info->prev_cpu_idle_up = total_idle_ticks;
	

	for_each_cpu_mask(j, policy->cpus) {
		unsigned int tmp_idle_ticks;
		struct cpu_dbs_info_s *j_dbs_info;

		if (j == cpu)
			continue;

		j_dbs_info = &per_cpu(cpu_dbs_info, j);
		/* Check for frequency increase */
		total_idle_ticks = kstat_cpu(j).cpustat.idle +
			kstat_cpu(j).cpustat.iowait;
319 320 321
		  /* consider 'nice' too? */
		  if (dbs_tuners_ins.ignore_nice == 0)
			   total_idle_ticks += kstat_cpu(j).cpustat.nice;
L
Linus Torvalds 已提交
322 323 324 325 326 327 328 329 330 331 332
		tmp_idle_ticks = total_idle_ticks -
			j_dbs_info->prev_cpu_idle_up;
		j_dbs_info->prev_cpu_idle_up = total_idle_ticks;

		if (tmp_idle_ticks < idle_ticks)
			idle_ticks = tmp_idle_ticks;
	}

	/* Scale idle ticks by 100 and compare with up and down ticks */
	idle_ticks *= 100;
	up_idle_ticks = (100 - dbs_tuners_ins.up_threshold) *
333
			usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
L
Linus Torvalds 已提交
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349

	if (idle_ticks < up_idle_ticks) {
		__cpufreq_driver_target(policy, policy->max, 
			CPUFREQ_RELATION_H);
		down_skip[cpu] = 0;
		this_dbs_info->prev_cpu_idle_down = total_idle_ticks;
		return;
	}

	/* Check for frequency decrease */
	down_skip[cpu]++;
	if (down_skip[cpu] < dbs_tuners_ins.sampling_down_factor)
		return;

	total_idle_ticks = kstat_cpu(cpu).cpustat.idle +
		kstat_cpu(cpu).cpustat.iowait;
350 351 352
	  /* consider 'nice' too? */
	  if (dbs_tuners_ins.ignore_nice == 0)
		  total_idle_ticks += kstat_cpu(cpu).cpustat.nice;
L
Linus Torvalds 已提交
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
	idle_ticks = total_idle_ticks -
		this_dbs_info->prev_cpu_idle_down;
	this_dbs_info->prev_cpu_idle_down = total_idle_ticks;

	for_each_cpu_mask(j, policy->cpus) {
		unsigned int tmp_idle_ticks;
		struct cpu_dbs_info_s *j_dbs_info;

		if (j == cpu)
			continue;

		j_dbs_info = &per_cpu(cpu_dbs_info, j);
		/* Check for frequency increase */
		total_idle_ticks = kstat_cpu(j).cpustat.idle +
			kstat_cpu(j).cpustat.iowait;
368 369 370
		  /* consider 'nice' too? */
		  if (dbs_tuners_ins.ignore_nice == 0)
			total_idle_ticks += kstat_cpu(j).cpustat.nice;
L
Linus Torvalds 已提交
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
		tmp_idle_ticks = total_idle_ticks -
			j_dbs_info->prev_cpu_idle_down;
		j_dbs_info->prev_cpu_idle_down = total_idle_ticks;

		if (tmp_idle_ticks < idle_ticks)
			idle_ticks = tmp_idle_ticks;
	}

	/* Scale idle ticks by 100 and compare with up and down ticks */
	idle_ticks *= 100;
	down_skip[cpu] = 0;

	freq_down_sampling_rate = dbs_tuners_ins.sampling_rate *
		dbs_tuners_ins.sampling_down_factor;
	down_idle_ticks = (100 - dbs_tuners_ins.down_threshold) *
386
			usecs_to_jiffies(freq_down_sampling_rate);
L
Linus Torvalds 已提交
387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405

	if (idle_ticks > down_idle_ticks ) {
		freq_down_step = (5 * policy->max) / 100;

		/* max freq cannot be less than 100. But who knows.... */
		if (unlikely(freq_down_step == 0))
			freq_down_step = 5;

		__cpufreq_driver_target(policy,
			policy->cur - freq_down_step, 
			CPUFREQ_RELATION_H);
		return;
	}
}

static void do_dbs_timer(void *data)
{ 
	int i;
	down(&dbs_sem);
406 407
	for_each_online_cpu(i)
		dbs_check_cpu(i);
L
Linus Torvalds 已提交
408
	schedule_delayed_work(&dbs_work, 
409
			usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
L
Linus Torvalds 已提交
410 411 412 413 414 415 416
	up(&dbs_sem);
} 

static inline void dbs_timer_init(void)
{
	INIT_WORK(&dbs_work, do_dbs_timer, NULL);
	schedule_delayed_work(&dbs_work,
417
			usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
L
Linus Torvalds 已提交
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
	return;
}

static inline void dbs_timer_exit(void)
{
	cancel_delayed_work(&dbs_work);
	return;
}

static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
				   unsigned int event)
{
	unsigned int cpu = policy->cpu;
	struct cpu_dbs_info_s *this_dbs_info;
	unsigned int j;

	this_dbs_info = &per_cpu(cpu_dbs_info, cpu);

	switch (event) {
	case CPUFREQ_GOV_START:
		if ((!cpu_online(cpu)) || 
		    (!policy->cur))
			return -EINVAL;

		if (policy->cpuinfo.transition_latency >
				(TRANSITION_LATENCY_LIMIT * 1000))
			return -EINVAL;
		if (this_dbs_info->enable) /* Already enabled */
			break;
		 
		down(&dbs_sem);
		for_each_cpu_mask(j, policy->cpus) {
			struct cpu_dbs_info_s *j_dbs_info;
			j_dbs_info = &per_cpu(cpu_dbs_info, j);
			j_dbs_info->cur_policy = policy;
		
			j_dbs_info->prev_cpu_idle_up = 
				kstat_cpu(j).cpustat.idle +
456 457 458 459 460
				kstat_cpu(j).cpustat.iowait +
				( !dbs_tuners_ins.ignore_nice
				  ? kstat_cpu(j).cpustat.nice : 0 );
			j_dbs_info->prev_cpu_idle_down
				= j_dbs_info->prev_cpu_idle_up;
L
Linus Torvalds 已提交
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
		}
		this_dbs_info->enable = 1;
		sysfs_create_group(&policy->kobj, &dbs_attr_group);
		dbs_enable++;
		/*
		 * Start the timerschedule work, when this governor
		 * is used for first time
		 */
		if (dbs_enable == 1) {
			unsigned int latency;
			/* policy latency is in nS. Convert it to uS first */

			latency = policy->cpuinfo.transition_latency;
			if (latency < 1000)
				latency = 1000;

			def_sampling_rate = (latency / 1000) *
					DEF_SAMPLING_RATE_LATENCY_MULTIPLIER;
			dbs_tuners_ins.sampling_rate = def_sampling_rate;
480
			dbs_tuners_ins.ignore_nice = 0;
L
Linus Torvalds 已提交
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519

			dbs_timer_init();
		}
		
		up(&dbs_sem);
		break;

	case CPUFREQ_GOV_STOP:
		down(&dbs_sem);
		this_dbs_info->enable = 0;
		sysfs_remove_group(&policy->kobj, &dbs_attr_group);
		dbs_enable--;
		/*
		 * Stop the timerschedule work, when this governor
		 * is used for first time
		 */
		if (dbs_enable == 0) 
			dbs_timer_exit();
		
		up(&dbs_sem);

		break;

	case CPUFREQ_GOV_LIMITS:
		down(&dbs_sem);
		if (policy->max < this_dbs_info->cur_policy->cur)
			__cpufreq_driver_target(
					this_dbs_info->cur_policy,
				       	policy->max, CPUFREQ_RELATION_H);
		else if (policy->min > this_dbs_info->cur_policy->cur)
			__cpufreq_driver_target(
					this_dbs_info->cur_policy,
				       	policy->min, CPUFREQ_RELATION_L);
		up(&dbs_sem);
		break;
	}
	return 0;
}

520
static struct cpufreq_governor cpufreq_gov_dbs = {
L
Linus Torvalds 已提交
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
	.name		= "ondemand",
	.governor	= cpufreq_governor_dbs,
	.owner		= THIS_MODULE,
};

static int __init cpufreq_gov_dbs_init(void)
{
	return cpufreq_register_governor(&cpufreq_gov_dbs);
}

static void __exit cpufreq_gov_dbs_exit(void)
{
	/* Make sure that the scheduled work is indeed not running */
	flush_scheduled_work();

	cpufreq_unregister_governor(&cpufreq_gov_dbs);
}


MODULE_AUTHOR ("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
MODULE_DESCRIPTION ("'cpufreq_ondemand' - A dynamic cpufreq governor for "
		"Low Latency Frequency Transition capable processors");
MODULE_LICENSE ("GPL");

module_init(cpufreq_gov_dbs_init);
module_exit(cpufreq_gov_dbs_exit);