cpufreq_conservative.c 16.4 KB
Newer Older
1 2 3 4 5 6
/*
 *  drivers/cpufreq/cpufreq_conservative.c
 *
 *  Copyright (C)  2001 Russell King
 *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *                      Jun Nakajima <jun.nakajima@intel.com>
7
 *            (C)  2009 Alexander Clouter <alex@digriz.org.uk>
8 9 10 11 12 13 14 15 16 17
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/cpufreq.h>
A
Andrew Morton 已提交
18
#include <linux/cpu.h>
19 20
#include <linux/jiffies.h>
#include <linux/kernel_stat.h>
21
#include <linux/mutex.h>
22 23 24 25 26
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/ktime.h>
#include <linux/sched.h>

27 28 29 30 31 32 33 34
/*
 * dbs is used in this file as a shortform for demandbased switching
 * It helps to keep variable names smaller, simpler
 */

#define DEF_FREQUENCY_UP_THRESHOLD		(80)
#define DEF_FREQUENCY_DOWN_THRESHOLD		(20)

35 36
/*
 * The polling frequency of this governor depends on the capability of
37
 * the processor. Default polling frequency is 1000 times the transition
38 39
 * latency of the processor. The governor will work on any processor with
 * transition latency <= 10mS, using appropriate sampling
40
 * rate.
41 42
 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
 * this governor will not work.
43 44
 * All times here are in uS.
 */
45
#define MIN_SAMPLING_RATE_RATIO			(2)
46

47 48
static unsigned int min_sampling_rate;

49
#define LATENCY_MULTIPLIER			(1000)
50
#define MIN_LATENCY_MULTIPLIER			(100)
51 52
#define DEF_SAMPLING_DOWN_FACTOR		(1)
#define MAX_SAMPLING_DOWN_FACTOR		(10)
53
#define TRANSITION_LATENCY_LIMIT		(10 * 1000 * 1000)
54

D
David Howells 已提交
55
static void do_dbs_timer(struct work_struct *work);
56 57

struct cpu_dbs_info_s {
58 59 60
	cputime64_t prev_cpu_idle;
	cputime64_t prev_cpu_wall;
	cputime64_t prev_cpu_nice;
61
	struct cpufreq_policy *cur_policy;
62
	struct delayed_work work;
63 64
	unsigned int down_skip;
	unsigned int requested_freq;
65 66
	int cpu;
	unsigned int enable:1;
67 68 69 70 71 72
	/*
	 * percpu mutex that serializes governor limit change with
	 * do_dbs_timer invocation. We do not want do_dbs_timer to run
	 * when user is changing the governor or limits.
	 */
	struct mutex timer_mutex;
73
};
74
static DEFINE_PER_CPU(struct cpu_dbs_info_s, cs_cpu_dbs_info);
75 76 77

static unsigned int dbs_enable;	/* number of CPUs using this policy */

78
/*
79
 * dbs_mutex protects dbs_enable in governor start/stop.
80
 */
81
static DEFINE_MUTEX(dbs_mutex);
82

83
static struct dbs_tuners {
84 85 86 87 88 89
	unsigned int sampling_rate;
	unsigned int sampling_down_factor;
	unsigned int up_threshold;
	unsigned int down_threshold;
	unsigned int ignore_nice;
	unsigned int freq_step;
90
} dbs_tuners_ins = {
91 92 93 94 95
	.up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
	.down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD,
	.sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR,
	.ignore_nice = 0,
	.freq_step = 5,
96 97
};

98 99
static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
							cputime64_t *wall)
100
{
101 102 103 104 105 106 107
	cputime64_t idle_time;
	cputime64_t cur_wall_time;
	cputime64_t busy_time;

	cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
	busy_time = cputime64_add(kstat_cpu(cpu).cpustat.user,
			kstat_cpu(cpu).cpustat.system);
108

109 110 111 112
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.irq);
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.softirq);
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.steal);
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.nice);
113

114 115
	idle_time = cputime64_sub(cur_wall_time, busy_time);
	if (wall)
116
		*wall = (cputime64_t)jiffies_to_usecs(cur_wall_time);
117

118
	return (cputime64_t)jiffies_to_usecs(idle_time);
119 120 121 122 123 124 125 126 127 128
}

static inline cputime64_t get_cpu_idle_time(unsigned int cpu, cputime64_t *wall)
{
	u64 idle_time = get_cpu_idle_time_us(cpu, wall);

	if (idle_time == -1ULL)
		return get_cpu_idle_time_jiffy(cpu, wall);

	return idle_time;
129 130
}

131 132 133 134 135 136
/* keep track of frequency transitions */
static int
dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
		     void *data)
{
	struct cpufreq_freqs *freq = data;
137
	struct cpu_dbs_info_s *this_dbs_info = &per_cpu(cs_cpu_dbs_info,
138 139
							freq->cpu);

140 141
	struct cpufreq_policy *policy;

142 143 144
	if (!this_dbs_info->enable)
		return 0;

145 146 147 148 149 150 151 152 153 154
	policy = this_dbs_info->cur_policy;

	/*
	 * we only care if our internally tracked freq moves outside
	 * the 'valid' ranges of freqency available to us otherwise
	 * we do not change it
	*/
	if (this_dbs_info->requested_freq > policy->max
			|| this_dbs_info->requested_freq < policy->min)
		this_dbs_info->requested_freq = freq->new;
155 156 157 158 159 160 161 162

	return 0;
}

static struct notifier_block dbs_cpufreq_notifier_block = {
	.notifier_call = dbs_cpufreq_notifier
};

163
/************************** sysfs interface ************************/
164 165
static ssize_t show_sampling_rate_min(struct kobject *kobj,
				      struct attribute *attr, char *buf)
166
{
167
	return sprintf(buf, "%u\n", min_sampling_rate);
168 169
}

170
define_one_global_ro(sampling_rate_min);
171 172 173 174

/* cpufreq_conservative Governor Tunables */
#define show_one(file_name, object)					\
static ssize_t show_##file_name						\
175
(struct kobject *kobj, struct attribute *attr, char *buf)		\
176 177 178 179 180 181 182
{									\
	return sprintf(buf, "%u\n", dbs_tuners_ins.object);		\
}
show_one(sampling_rate, sampling_rate);
show_one(sampling_down_factor, sampling_down_factor);
show_one(up_threshold, up_threshold);
show_one(down_threshold, down_threshold);
183
show_one(ignore_nice_load, ignore_nice);
184 185
show_one(freq_step, freq_step);

186 187 188
static ssize_t store_sampling_down_factor(struct kobject *a,
					  struct attribute *b,
					  const char *buf, size_t count)
189 190 191
{
	unsigned int input;
	int ret;
192
	ret = sscanf(buf, "%u", &input);
193

194
	if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
195 196 197 198 199 200
		return -EINVAL;

	dbs_tuners_ins.sampling_down_factor = input;
	return count;
}

201 202
static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b,
				   const char *buf, size_t count)
203 204 205
{
	unsigned int input;
	int ret;
206
	ret = sscanf(buf, "%u", &input);
207

208
	if (ret != 1)
209
		return -EINVAL;
210

211
	dbs_tuners_ins.sampling_rate = max(input, min_sampling_rate);
212 213 214
	return count;
}

215 216
static ssize_t store_up_threshold(struct kobject *a, struct attribute *b,
				  const char *buf, size_t count)
217 218 219
{
	unsigned int input;
	int ret;
220
	ret = sscanf(buf, "%u", &input);
221

222
	if (ret != 1 || input > 100 ||
223
			input <= dbs_tuners_ins.down_threshold)
224 225 226 227 228 229
		return -EINVAL;

	dbs_tuners_ins.up_threshold = input;
	return count;
}

230 231
static ssize_t store_down_threshold(struct kobject *a, struct attribute *b,
				    const char *buf, size_t count)
232 233 234
{
	unsigned int input;
	int ret;
235
	ret = sscanf(buf, "%u", &input);
236

237 238
	/* cannot be lower than 11 otherwise freq will not fall */
	if (ret != 1 || input < 11 || input > 100 ||
239
			input >= dbs_tuners_ins.up_threshold)
240 241 242 243 244 245
		return -EINVAL;

	dbs_tuners_ins.down_threshold = input;
	return count;
}

246 247
static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
				      const char *buf, size_t count)
248 249 250 251 252
{
	unsigned int input;
	int ret;

	unsigned int j;
253 254 255

	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
256 257
		return -EINVAL;

258
	if (input > 1)
259
		input = 1;
260

261
	if (input == dbs_tuners_ins.ignore_nice) /* nothing to do */
262
		return count;
263

264 265
	dbs_tuners_ins.ignore_nice = input;

266
	/* we need to re-evaluate prev_cpu_idle */
267
	for_each_online_cpu(j) {
268
		struct cpu_dbs_info_s *dbs_info;
269
		dbs_info = &per_cpu(cs_cpu_dbs_info, j);
270 271 272 273
		dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
						&dbs_info->prev_cpu_wall);
		if (dbs_tuners_ins.ignore_nice)
			dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
274 275 276 277
	}
	return count;
}

278 279
static ssize_t store_freq_step(struct kobject *a, struct attribute *b,
			       const char *buf, size_t count)
280 281 282
{
	unsigned int input;
	int ret;
283
	ret = sscanf(buf, "%u", &input);
284

285
	if (ret != 1)
286 287
		return -EINVAL;

288
	if (input > 100)
289
		input = 100;
290

291 292 293 294 295 296
	/* no need to test here if freq_step is zero as the user might actually
	 * want this, they would be crazy though :) */
	dbs_tuners_ins.freq_step = input;
	return count;
}

297 298 299 300 301 302
define_one_global_rw(sampling_rate);
define_one_global_rw(sampling_down_factor);
define_one_global_rw(up_threshold);
define_one_global_rw(down_threshold);
define_one_global_rw(ignore_nice_load);
define_one_global_rw(freq_step);
303

304
static struct attribute *dbs_attributes[] = {
305 306 307 308 309
	&sampling_rate_min.attr,
	&sampling_rate.attr,
	&sampling_down_factor.attr,
	&up_threshold.attr,
	&down_threshold.attr,
310
	&ignore_nice_load.attr,
311 312 313 314 315 316 317 318 319 320 321
	&freq_step.attr,
	NULL
};

static struct attribute_group dbs_attr_group = {
	.attrs = dbs_attributes,
	.name = "conservative",
};

/************************** sysfs end ************************/

322
static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
323
{
324
	unsigned int load = 0;
325
	unsigned int max_load = 0;
326
	unsigned int freq_target;
327

328 329
	struct cpufreq_policy *policy;
	unsigned int j;
330

331 332
	policy = this_dbs_info->cur_policy;

333
	/*
334 335 336 337
	 * Every sampling_rate, we check, if current idle time is less
	 * than 20% (default), then we try to increase frequency
	 * Every sampling_rate*sampling_down_factor, we check, if current
	 * idle time is more than 80%, then we try to decrease frequency
338
	 *
339 340
	 * Any frequency increase takes it to the maximum frequency.
	 * Frequency reduction happens at minimum steps of
341
	 * 5% (default) of maximum frequency
342 343
	 */

344 345 346 347 348
	/* Get Absolute Load */
	for_each_cpu(j, policy->cpus) {
		struct cpu_dbs_info_s *j_dbs_info;
		cputime64_t cur_wall_time, cur_idle_time;
		unsigned int idle_time, wall_time;
349

350
		j_dbs_info = &per_cpu(cs_cpu_dbs_info, j);
351 352 353 354 355 356

		cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);

		wall_time = (unsigned int) cputime64_sub(cur_wall_time,
				j_dbs_info->prev_cpu_wall);
		j_dbs_info->prev_cpu_wall = cur_wall_time;
357

358 359 360
		idle_time = (unsigned int) cputime64_sub(cur_idle_time,
				j_dbs_info->prev_cpu_idle);
		j_dbs_info->prev_cpu_idle = cur_idle_time;
361

362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
		if (dbs_tuners_ins.ignore_nice) {
			cputime64_t cur_nice;
			unsigned long cur_nice_jiffies;

			cur_nice = cputime64_sub(kstat_cpu(j).cpustat.nice,
					 j_dbs_info->prev_cpu_nice);
			/*
			 * Assumption: nice time between sampling periods will
			 * be less than 2^32 jiffies for 32 bit sys
			 */
			cur_nice_jiffies = (unsigned long)
					cputime64_to_jiffies64(cur_nice);

			j_dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
			idle_time += jiffies_to_usecs(cur_nice_jiffies);
		}

		if (unlikely(!wall_time || wall_time < idle_time))
			continue;

		load = 100 * (wall_time - idle_time) / wall_time;
383 384 385

		if (load > max_load)
			max_load = load;
386 387 388 389 390 391 392 393
	}

	/*
	 * break out if we 'cannot' reduce the speed as the user might
	 * want freq_step to be zero
	 */
	if (dbs_tuners_ins.freq_step == 0)
		return;
394

395
	/* Check for frequency increase */
396
	if (max_load > dbs_tuners_ins.up_threshold) {
397
		this_dbs_info->down_skip = 0;
398

399
		/* if we are already at full speed then break out early */
400
		if (this_dbs_info->requested_freq == policy->max)
401
			return;
402

403
		freq_target = (dbs_tuners_ins.freq_step * policy->max) / 100;
404 405

		/* max freq cannot be less than 100. But who knows.... */
406 407
		if (unlikely(freq_target == 0))
			freq_target = 5;
408

409
		this_dbs_info->requested_freq += freq_target;
410 411
		if (this_dbs_info->requested_freq > policy->max)
			this_dbs_info->requested_freq = policy->max;
412

413
		__cpufreq_driver_target(policy, this_dbs_info->requested_freq,
414 415 416 417
			CPUFREQ_RELATION_H);
		return;
	}

418 419 420 421 422
	/*
	 * The optimal frequency is the frequency that is the lowest that
	 * can support the current CPU usage without triggering the up
	 * policy. To be safe, we focus 10 points under the threshold.
	 */
423
	if (max_load < (dbs_tuners_ins.down_threshold - 10)) {
424
		freq_target = (dbs_tuners_ins.freq_step * policy->max) / 100;
425

426
		this_dbs_info->requested_freq -= freq_target;
427 428
		if (this_dbs_info->requested_freq < policy->min)
			this_dbs_info->requested_freq = policy->min;
429

430 431 432 433 434 435
		/*
		 * if we cannot reduce the frequency anymore, break out early
		 */
		if (policy->cur == policy->min)
			return;

436
		__cpufreq_driver_target(policy, this_dbs_info->requested_freq,
437
				CPUFREQ_RELATION_H);
438 439 440 441
		return;
	}
}

D
David Howells 已提交
442
static void do_dbs_timer(struct work_struct *work)
443
{
444 445 446 447 448 449 450 451 452
	struct cpu_dbs_info_s *dbs_info =
		container_of(work, struct cpu_dbs_info_s, work.work);
	unsigned int cpu = dbs_info->cpu;

	/* We want all CPUs to do sampling nearly on same jiffy */
	int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);

	delay -= jiffies % delay;

453
	mutex_lock(&dbs_info->timer_mutex);
454 455 456

	dbs_check_cpu(dbs_info);

457
	schedule_delayed_work_on(cpu, &dbs_info->work, delay);
458
	mutex_unlock(&dbs_info->timer_mutex);
459
}
460

461
static inline void dbs_timer_init(struct cpu_dbs_info_s *dbs_info)
462
{
463 464 465 466 467 468
	/* We want all CPUs to do sampling nearly on same jiffy */
	int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
	delay -= jiffies % delay;

	dbs_info->enable = 1;
	INIT_DELAYED_WORK_DEFERRABLE(&dbs_info->work, do_dbs_timer);
469
	schedule_delayed_work_on(dbs_info->cpu, &dbs_info->work, delay);
470 471
}

472
static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info)
473
{
474
	dbs_info->enable = 0;
475
	cancel_delayed_work_sync(&dbs_info->work);
476 477 478 479 480 481 482 483
}

static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
				   unsigned int event)
{
	unsigned int cpu = policy->cpu;
	struct cpu_dbs_info_s *this_dbs_info;
	unsigned int j;
J
Jeff Garzik 已提交
484
	int rc;
485

486
	this_dbs_info = &per_cpu(cs_cpu_dbs_info, cpu);
487 488 489

	switch (event) {
	case CPUFREQ_GOV_START:
490
		if ((!cpu_online(cpu)) || (!policy->cur))
491 492
			return -EINVAL;

493
		mutex_lock(&dbs_mutex);
J
Jeff Garzik 已提交
494

495
		for_each_cpu(j, policy->cpus) {
496
			struct cpu_dbs_info_s *j_dbs_info;
497
			j_dbs_info = &per_cpu(cs_cpu_dbs_info, j);
498
			j_dbs_info->cur_policy = policy;
499

500 501 502 503 504 505
			j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
						&j_dbs_info->prev_cpu_wall);
			if (dbs_tuners_ins.ignore_nice) {
				j_dbs_info->prev_cpu_nice =
						kstat_cpu(j).cpustat.nice;
			}
506
		}
507 508
		this_dbs_info->down_skip = 0;
		this_dbs_info->requested_freq = policy->cur;
J
Jeff Garzik 已提交
509

510
		mutex_init(&this_dbs_info->timer_mutex);
511 512 513 514 515 516 517 518
		dbs_enable++;
		/*
		 * Start the timerschedule work, when this governor
		 * is used for first time
		 */
		if (dbs_enable == 1) {
			unsigned int latency;
			/* policy latency is in nS. Convert it to uS first */
519 520 521
			latency = policy->cpuinfo.transition_latency / 1000;
			if (latency == 0)
				latency = 1;
522

523 524 525 526 527 528 529
			rc = sysfs_create_group(cpufreq_global_kobject,
						&dbs_attr_group);
			if (rc) {
				mutex_unlock(&dbs_mutex);
				return rc;
			}

530 531 532 533 534 535 536 537 538 539 540 541
			/*
			 * conservative does not implement micro like ondemand
			 * governor, thus we are bound to jiffes/HZ
			 */
			min_sampling_rate =
				MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10);
			/* Bring kernel and HW constraints together */
			min_sampling_rate = max(min_sampling_rate,
					MIN_LATENCY_MULTIPLIER * latency);
			dbs_tuners_ins.sampling_rate =
				max(min_sampling_rate,
				    latency * LATENCY_MULTIPLIER);
542

543 544 545
			cpufreq_register_notifier(
					&dbs_cpufreq_notifier_block,
					CPUFREQ_TRANSITION_NOTIFIER);
546
		}
547
		mutex_unlock(&dbs_mutex);
548

549 550
		dbs_timer_init(this_dbs_info);

551 552 553
		break;

	case CPUFREQ_GOV_STOP:
554
		dbs_timer_exit(this_dbs_info);
555 556

		mutex_lock(&dbs_mutex);
557
		dbs_enable--;
558
		mutex_destroy(&this_dbs_info->timer_mutex);
559

560 561 562 563
		/*
		 * Stop the timerschedule work, when this governor
		 * is used for first time
		 */
564
		if (dbs_enable == 0)
565 566 567 568
			cpufreq_unregister_notifier(
					&dbs_cpufreq_notifier_block,
					CPUFREQ_TRANSITION_NOTIFIER);

569
		mutex_unlock(&dbs_mutex);
570 571 572
		if (!dbs_enable)
			sysfs_remove_group(cpufreq_global_kobject,
					   &dbs_attr_group);
573 574 575 576

		break;

	case CPUFREQ_GOV_LIMITS:
577
		mutex_lock(&this_dbs_info->timer_mutex);
578 579 580
		if (policy->max < this_dbs_info->cur_policy->cur)
			__cpufreq_driver_target(
					this_dbs_info->cur_policy,
581
					policy->max, CPUFREQ_RELATION_H);
582 583 584
		else if (policy->min > this_dbs_info->cur_policy->cur)
			__cpufreq_driver_target(
					this_dbs_info->cur_policy,
585
					policy->min, CPUFREQ_RELATION_L);
586
		mutex_unlock(&this_dbs_info->timer_mutex);
587

588 589 590 591 592
		break;
	}
	return 0;
}

593 594 595
#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE
static
#endif
596 597 598 599 600
struct cpufreq_governor cpufreq_gov_conservative = {
	.name			= "conservative",
	.governor		= cpufreq_governor_dbs,
	.max_transition_latency	= TRANSITION_LATENCY_LIMIT,
	.owner			= THIS_MODULE,
601 602 603 604
};

static int __init cpufreq_gov_dbs_init(void)
{
605
	return cpufreq_register_governor(&cpufreq_gov_conservative);
606 607 608 609
}

static void __exit cpufreq_gov_dbs_exit(void)
{
610
	cpufreq_unregister_governor(&cpufreq_gov_conservative);
611 612 613
}


614
MODULE_AUTHOR("Alexander Clouter <alex@digriz.org.uk>");
615
MODULE_DESCRIPTION("'cpufreq_conservative' - A dynamic cpufreq governor for "
616 617
		"Low Latency Frequency Transition capable processors "
		"optimised for use in a battery environment");
618
MODULE_LICENSE("GPL");
619

620 621 622
#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE
fs_initcall(cpufreq_gov_dbs_init);
#else
623
module_init(cpufreq_gov_dbs_init);
624
#endif
625
module_exit(cpufreq_gov_dbs_exit);