cpufreq_ondemand.c 21.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 *  drivers/cpufreq/cpufreq_ondemand.c
 *
 *  Copyright (C)  2001 Russell King
 *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *                      Jun Nakajima <jun.nakajima@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/cpufreq.h>
A
Andrew Morton 已提交
17
#include <linux/cpu.h>
L
Linus Torvalds 已提交
18 19
#include <linux/jiffies.h>
#include <linux/kernel_stat.h>
20
#include <linux/mutex.h>
21 22 23
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/ktime.h>
24
#include <linux/sched.h>
L
Linus Torvalds 已提交
25 26 27 28 29 30

/*
 * dbs is used in this file as a shortform for demandbased switching
 * It helps to keep variable names smaller, simpler
 */

31
#define DEF_FREQUENCY_DOWN_DIFFERENTIAL		(10)
L
Linus Torvalds 已提交
32
#define DEF_FREQUENCY_UP_THRESHOLD		(80)
33 34
#define DEF_SAMPLING_DOWN_FACTOR		(1)
#define MAX_SAMPLING_DOWN_FACTOR		(100000)
35 36
#define MICRO_FREQUENCY_DOWN_DIFFERENTIAL	(3)
#define MICRO_FREQUENCY_UP_THRESHOLD		(95)
37
#define MICRO_FREQUENCY_MIN_SAMPLE_RATE		(10000)
38
#define MIN_FREQUENCY_UP_THRESHOLD		(11)
L
Linus Torvalds 已提交
39 40
#define MAX_FREQUENCY_UP_THRESHOLD		(100)

41 42
/*
 * The polling frequency of this governor depends on the capability of
L
Linus Torvalds 已提交
43
 * the processor. Default polling frequency is 1000 times the transition
44 45
 * latency of the processor. The governor will work on any processor with
 * transition latency <= 10mS, using appropriate sampling
L
Linus Torvalds 已提交
46 47 48 49 50
 * rate.
 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
 * this governor will not work.
 * All times here are in uS.
 */
51
#define MIN_SAMPLING_RATE_RATIO			(2)
52

53 54
static unsigned int min_sampling_rate;

55
#define LATENCY_MULTIPLIER			(1000)
56
#define MIN_LATENCY_MULTIPLIER			(100)
57
#define TRANSITION_LATENCY_LIMIT		(10 * 1000 * 1000)
L
Linus Torvalds 已提交
58

D
David Howells 已提交
59
static void do_dbs_timer(struct work_struct *work);
60 61 62 63 64 65 66 67 68 69 70 71
static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
				unsigned int event);

#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
static
#endif
struct cpufreq_governor cpufreq_gov_ondemand = {
       .name                   = "ondemand",
       .governor               = cpufreq_governor_dbs,
       .max_transition_latency = TRANSITION_LATENCY_LIMIT,
       .owner                  = THIS_MODULE,
};
D
David Howells 已提交
72 73

/* Sampling types */
74
enum {DBS_NORMAL_SAMPLE, DBS_SUB_SAMPLE};
L
Linus Torvalds 已提交
75 76

struct cpu_dbs_info_s {
77
	cputime64_t prev_cpu_idle;
78
	cputime64_t prev_cpu_iowait;
79
	cputime64_t prev_cpu_wall;
80
	cputime64_t prev_cpu_nice;
81
	struct cpufreq_policy *cur_policy;
82
	struct delayed_work work;
83 84 85 86
	struct cpufreq_frequency_table *freq_table;
	unsigned int freq_lo;
	unsigned int freq_lo_jiffies;
	unsigned int freq_hi_jiffies;
87
	unsigned int rate_mult;
88
	int cpu;
89 90 91 92 93 94 95
	unsigned int sample_type:1;
	/*
	 * percpu mutex that serializes governor limit change with
	 * do_dbs_timer invocation. We do not want do_dbs_timer to run
	 * when user is changing the governor or limits.
	 */
	struct mutex timer_mutex;
L
Linus Torvalds 已提交
96
};
97
static DEFINE_PER_CPU(struct cpu_dbs_info_s, od_cpu_dbs_info);
L
Linus Torvalds 已提交
98 99 100

static unsigned int dbs_enable;	/* number of CPUs using this policy */

101
/*
102
 * dbs_mutex protects data in dbs_tuners_ins from concurrent changes on
103
 * different CPUs. It protects dbs_enable in governor start/stop.
104
 */
105
static DEFINE_MUTEX(dbs_mutex);
L
Linus Torvalds 已提交
106

107
static struct dbs_tuners {
108 109
	unsigned int sampling_rate;
	unsigned int up_threshold;
110
	unsigned int down_differential;
111
	unsigned int ignore_nice;
112
	unsigned int sampling_down_factor;
113
	unsigned int powersave_bias;
114
	unsigned int io_is_busy;
115
} dbs_tuners_ins = {
116
	.up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
117
	.sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR,
118
	.down_differential = DEF_FREQUENCY_DOWN_DIFFERENTIAL,
119
	.ignore_nice = 0,
120
	.powersave_bias = 0,
L
Linus Torvalds 已提交
121 122
};

123 124
static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu,
							cputime64_t *wall)
125
{
126
	cputime64_t idle_time;
127
	cputime64_t cur_wall_time;
128
	cputime64_t busy_time;
129

130
	cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
131 132
	busy_time = cputime64_add(kstat_cpu(cpu).cpustat.user,
			kstat_cpu(cpu).cpustat.system);
133

134 135 136
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.irq);
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.softirq);
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.steal);
137
	busy_time = cputime64_add(busy_time, kstat_cpu(cpu).cpustat.nice);
138

139 140
	idle_time = cputime64_sub(cur_wall_time, busy_time);
	if (wall)
141
		*wall = (cputime64_t)jiffies_to_usecs(cur_wall_time);
142

143
	return (cputime64_t)jiffies_to_usecs(idle_time);
144 145
}

146 147 148 149 150 151 152 153 154 155
static inline cputime64_t get_cpu_idle_time(unsigned int cpu, cputime64_t *wall)
{
	u64 idle_time = get_cpu_idle_time_us(cpu, wall);

	if (idle_time == -1ULL)
		return get_cpu_idle_time_jiffy(cpu, wall);

	return idle_time;
}

156 157 158 159 160 161 162 163 164 165
static inline cputime64_t get_cpu_iowait_time(unsigned int cpu, cputime64_t *wall)
{
	u64 iowait_time = get_cpu_iowait_time_us(cpu, wall);

	if (iowait_time == -1ULL)
		return 0;

	return iowait_time;
}

166 167 168 169 170
/*
 * Find right freq to be set now with powersave_bias on.
 * Returns the freq_hi to be used right now and will set freq_hi_jiffies,
 * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs.
 */
171 172 173
static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
					  unsigned int freq_next,
					  unsigned int relation)
174 175 176 177 178
{
	unsigned int freq_req, freq_reduc, freq_avg;
	unsigned int freq_hi, freq_lo;
	unsigned int index = 0;
	unsigned int jiffies_total, jiffies_hi, jiffies_lo;
179 180
	struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
						   policy->cpu);
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220

	if (!dbs_info->freq_table) {
		dbs_info->freq_lo = 0;
		dbs_info->freq_lo_jiffies = 0;
		return freq_next;
	}

	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
			relation, &index);
	freq_req = dbs_info->freq_table[index].frequency;
	freq_reduc = freq_req * dbs_tuners_ins.powersave_bias / 1000;
	freq_avg = freq_req - freq_reduc;

	/* Find freq bounds for freq_avg in freq_table */
	index = 0;
	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
			CPUFREQ_RELATION_H, &index);
	freq_lo = dbs_info->freq_table[index].frequency;
	index = 0;
	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
			CPUFREQ_RELATION_L, &index);
	freq_hi = dbs_info->freq_table[index].frequency;

	/* Find out how long we have to be in hi and lo freqs */
	if (freq_hi == freq_lo) {
		dbs_info->freq_lo = 0;
		dbs_info->freq_lo_jiffies = 0;
		return freq_lo;
	}
	jiffies_total = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
	jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
	jiffies_hi += ((freq_hi - freq_lo) / 2);
	jiffies_hi /= (freq_hi - freq_lo);
	jiffies_lo = jiffies_total - jiffies_hi;
	dbs_info->freq_lo = freq_lo;
	dbs_info->freq_lo_jiffies = jiffies_lo;
	dbs_info->freq_hi_jiffies = jiffies_hi;
	return freq_hi;
}

221 222
static void ondemand_powersave_bias_init_cpu(int cpu)
{
223
	struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
224 225 226 227
	dbs_info->freq_table = cpufreq_frequency_get_table(cpu);
	dbs_info->freq_lo = 0;
}

228 229 230 231
static void ondemand_powersave_bias_init(void)
{
	int i;
	for_each_online_cpu(i) {
232
		ondemand_powersave_bias_init_cpu(i);
233 234 235
	}
}

L
Linus Torvalds 已提交
236
/************************** sysfs interface ************************/
237 238 239

static ssize_t show_sampling_rate_min(struct kobject *kobj,
				      struct attribute *attr, char *buf)
L
Linus Torvalds 已提交
240
{
241
	return sprintf(buf, "%u\n", min_sampling_rate);
L
Linus Torvalds 已提交
242 243
}

244
define_one_global_ro(sampling_rate_min);
L
Linus Torvalds 已提交
245 246 247 248

/* cpufreq_ondemand Governor Tunables */
#define show_one(file_name, object)					\
static ssize_t show_##file_name						\
249
(struct kobject *kobj, struct attribute *attr, char *buf)              \
L
Linus Torvalds 已提交
250 251 252 253
{									\
	return sprintf(buf, "%u\n", dbs_tuners_ins.object);		\
}
show_one(sampling_rate, sampling_rate);
254
show_one(io_is_busy, io_is_busy);
L
Linus Torvalds 已提交
255
show_one(up_threshold, up_threshold);
256
show_one(sampling_down_factor, sampling_down_factor);
257
show_one(ignore_nice_load, ignore_nice);
258
show_one(powersave_bias, powersave_bias);
L
Linus Torvalds 已提交
259

260 261
static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b,
				   const char *buf, size_t count)
L
Linus Torvalds 已提交
262 263 264
{
	unsigned int input;
	int ret;
265
	ret = sscanf(buf, "%u", &input);
266 267
	if (ret != 1)
		return -EINVAL;
L
Linus Torvalds 已提交
268

269
	mutex_lock(&dbs_mutex);
270
	dbs_tuners_ins.sampling_rate = max(input, min_sampling_rate);
271
	mutex_unlock(&dbs_mutex);
L
Linus Torvalds 已提交
272 273 274 275

	return count;
}

276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b,
				   const char *buf, size_t count)
{
	unsigned int input;
	int ret;

	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;

	mutex_lock(&dbs_mutex);
	dbs_tuners_ins.io_is_busy = !!input;
	mutex_unlock(&dbs_mutex);

	return count;
}

293 294
static ssize_t store_up_threshold(struct kobject *a, struct attribute *b,
				  const char *buf, size_t count)
L
Linus Torvalds 已提交
295 296 297
{
	unsigned int input;
	int ret;
298
	ret = sscanf(buf, "%u", &input);
L
Linus Torvalds 已提交
299

300
	if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
301
			input < MIN_FREQUENCY_UP_THRESHOLD) {
L
Linus Torvalds 已提交
302 303 304
		return -EINVAL;
	}

305
	mutex_lock(&dbs_mutex);
L
Linus Torvalds 已提交
306
	dbs_tuners_ins.up_threshold = input;
307
	mutex_unlock(&dbs_mutex);
L
Linus Torvalds 已提交
308 309 310 311

	return count;
}

312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
static ssize_t store_sampling_down_factor(struct kobject *a,
			struct attribute *b, const char *buf, size_t count)
{
	unsigned int input, j;
	int ret;
	ret = sscanf(buf, "%u", &input);

	if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
		return -EINVAL;
	mutex_lock(&dbs_mutex);
	dbs_tuners_ins.sampling_down_factor = input;

	/* Reset down sampling multiplier in case it was active */
	for_each_online_cpu(j) {
		struct cpu_dbs_info_s *dbs_info;
		dbs_info = &per_cpu(od_cpu_dbs_info, j);
		dbs_info->rate_mult = 1;
	}
	mutex_unlock(&dbs_mutex);

	return count;
}

335 336
static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
				      const char *buf, size_t count)
337 338 339 340 341
{
	unsigned int input;
	int ret;

	unsigned int j;
342

343
	ret = sscanf(buf, "%u", &input);
344
	if (ret != 1)
345 346
		return -EINVAL;

347
	if (input > 1)
348
		input = 1;
349

350
	mutex_lock(&dbs_mutex);
351
	if (input == dbs_tuners_ins.ignore_nice) { /* nothing to do */
352
		mutex_unlock(&dbs_mutex);
353 354 355 356
		return count;
	}
	dbs_tuners_ins.ignore_nice = input;

357
	/* we need to re-evaluate prev_cpu_idle */
358
	for_each_online_cpu(j) {
359
		struct cpu_dbs_info_s *dbs_info;
360
		dbs_info = &per_cpu(od_cpu_dbs_info, j);
361 362
		dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
						&dbs_info->prev_cpu_wall);
363 364 365
		if (dbs_tuners_ins.ignore_nice)
			dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;

366
	}
367
	mutex_unlock(&dbs_mutex);
368 369 370 371

	return count;
}

372 373
static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b,
				    const char *buf, size_t count)
374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392
{
	unsigned int input;
	int ret;
	ret = sscanf(buf, "%u", &input);

	if (ret != 1)
		return -EINVAL;

	if (input > 1000)
		input = 1000;

	mutex_lock(&dbs_mutex);
	dbs_tuners_ins.powersave_bias = input;
	ondemand_powersave_bias_init();
	mutex_unlock(&dbs_mutex);

	return count;
}

393
define_one_global_rw(sampling_rate);
394
define_one_global_rw(io_is_busy);
395
define_one_global_rw(up_threshold);
396
define_one_global_rw(sampling_down_factor);
397 398
define_one_global_rw(ignore_nice_load);
define_one_global_rw(powersave_bias);
L
Linus Torvalds 已提交
399

400
static struct attribute *dbs_attributes[] = {
L
Linus Torvalds 已提交
401 402 403
	&sampling_rate_min.attr,
	&sampling_rate.attr,
	&up_threshold.attr,
404
	&sampling_down_factor.attr,
405
	&ignore_nice_load.attr,
406
	&powersave_bias.attr,
407
	&io_is_busy.attr,
L
Linus Torvalds 已提交
408 409 410 411 412 413 414 415 416 417
	NULL
};

static struct attribute_group dbs_attr_group = {
	.attrs = dbs_attributes,
	.name = "ondemand",
};

/************************** sysfs end ************************/

418 419 420 421 422 423 424 425 426 427 428
static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq)
{
	if (dbs_tuners_ins.powersave_bias)
		freq = powersave_bias_target(p, freq, CPUFREQ_RELATION_H);
	else if (p->cur == p->max)
		return;

	__cpufreq_driver_target(p, freq, dbs_tuners_ins.powersave_bias ?
			CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
}

429
static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
L
Linus Torvalds 已提交
430
{
431
	unsigned int max_load_freq;
L
Linus Torvalds 已提交
432 433 434 435

	struct cpufreq_policy *policy;
	unsigned int j;

436
	this_dbs_info->freq_lo = 0;
L
Linus Torvalds 已提交
437
	policy = this_dbs_info->cur_policy;
438

439
	/*
440 441
	 * Every sampling_rate, we check, if current idle time is less
	 * than 20% (default), then we try to increase frequency
442
	 * Every sampling_rate, we look for a the lowest
443 444
	 * frequency which can sustain the load while keeping idle time over
	 * 30%. If such a frequency exist, we try to decrease to this frequency.
L
Linus Torvalds 已提交
445
	 *
446 447 448
	 * Any frequency increase takes it to the maximum frequency.
	 * Frequency reduction happens at minimum steps of
	 * 5% (default) of current frequency
L
Linus Torvalds 已提交
449 450
	 */

451 452 453
	/* Get Absolute Load - in terms of freq */
	max_load_freq = 0;

454
	for_each_cpu(j, policy->cpus) {
L
Linus Torvalds 已提交
455
		struct cpu_dbs_info_s *j_dbs_info;
456 457
		cputime64_t cur_wall_time, cur_idle_time, cur_iowait_time;
		unsigned int idle_time, wall_time, iowait_time;
458 459
		unsigned int load, load_freq;
		int freq_avg;
L
Linus Torvalds 已提交
460

461
		j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
462 463

		cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);
464
		cur_iowait_time = get_cpu_iowait_time(j, &cur_wall_time);
465

466 467 468 469 470
		wall_time = (unsigned int) cputime64_sub(cur_wall_time,
				j_dbs_info->prev_cpu_wall);
		j_dbs_info->prev_cpu_wall = cur_wall_time;

		idle_time = (unsigned int) cputime64_sub(cur_idle_time,
471
				j_dbs_info->prev_cpu_idle);
472
		j_dbs_info->prev_cpu_idle = cur_idle_time;
L
Linus Torvalds 已提交
473

474 475 476 477
		iowait_time = (unsigned int) cputime64_sub(cur_iowait_time,
				j_dbs_info->prev_cpu_iowait);
		j_dbs_info->prev_cpu_iowait = cur_iowait_time;

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
		if (dbs_tuners_ins.ignore_nice) {
			cputime64_t cur_nice;
			unsigned long cur_nice_jiffies;

			cur_nice = cputime64_sub(kstat_cpu(j).cpustat.nice,
					 j_dbs_info->prev_cpu_nice);
			/*
			 * Assumption: nice time between sampling periods will
			 * be less than 2^32 jiffies for 32 bit sys
			 */
			cur_nice_jiffies = (unsigned long)
					cputime64_to_jiffies64(cur_nice);

			j_dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice;
			idle_time += jiffies_to_usecs(cur_nice_jiffies);
		}

495 496 497 498 499 500 501
		/*
		 * For the purpose of ondemand, waiting for disk IO is an
		 * indication that you're performance critical, and not that
		 * the system is actually idle. So subtract the iowait time
		 * from the cpu idle time.
		 */

502
		if (dbs_tuners_ins.io_is_busy && idle_time >= iowait_time)
503 504
			idle_time -= iowait_time;

505
		if (unlikely(!wall_time || wall_time < idle_time))
506 507 508 509 510 511 512 513 514 515 516
			continue;

		load = 100 * (wall_time - idle_time) / wall_time;

		freq_avg = __cpufreq_driver_getavg(policy, j);
		if (freq_avg <= 0)
			freq_avg = policy->cur;

		load_freq = load * freq_avg;
		if (load_freq > max_load_freq)
			max_load_freq = load_freq;
L
Linus Torvalds 已提交
517 518
	}

519
	/* Check for frequency increase */
520
	if (max_load_freq > dbs_tuners_ins.up_threshold * policy->cur) {
521 522 523 524
		/* If switching to max speed, apply sampling_down_factor */
		if (policy->cur < policy->max)
			this_dbs_info->rate_mult =
				dbs_tuners_ins.sampling_down_factor;
525
		dbs_freq_increase(policy, policy->max);
L
Linus Torvalds 已提交
526 527 528 529
		return;
	}

	/* Check for frequency decrease */
530 531 532
	/* if we cannot reduce the frequency anymore, break out early */
	if (policy->cur == policy->min)
		return;
L
Linus Torvalds 已提交
533

534 535 536 537 538
	/*
	 * The optimal frequency is the frequency that is the lowest that
	 * can support the current CPU usage without triggering the up
	 * policy. To be safe, we focus 10 points under the threshold.
	 */
539 540 541
	if (max_load_freq <
	    (dbs_tuners_ins.up_threshold - dbs_tuners_ins.down_differential) *
	     policy->cur) {
542
		unsigned int freq_next;
543 544 545
		freq_next = max_load_freq /
				(dbs_tuners_ins.up_threshold -
				 dbs_tuners_ins.down_differential);
546

547 548 549
		/* No longer fully busy, reset rate_mult */
		this_dbs_info->rate_mult = 1;

550 551 552
		if (freq_next < policy->min)
			freq_next = policy->min;

553 554 555 556 557 558 559 560 561
		if (!dbs_tuners_ins.powersave_bias) {
			__cpufreq_driver_target(policy, freq_next,
					CPUFREQ_RELATION_L);
		} else {
			int freq = powersave_bias_target(policy, freq_next,
					CPUFREQ_RELATION_L);
			__cpufreq_driver_target(policy, freq,
				CPUFREQ_RELATION_L);
		}
562
	}
L
Linus Torvalds 已提交
563 564
}

D
David Howells 已提交
565
static void do_dbs_timer(struct work_struct *work)
566
{
567 568 569 570 571
	struct cpu_dbs_info_s *dbs_info =
		container_of(work, struct cpu_dbs_info_s, work.work);
	unsigned int cpu = dbs_info->cpu;
	int sample_type = dbs_info->sample_type;

572
	int delay;
573

574
	mutex_lock(&dbs_info->timer_mutex);
575

576
	/* Common NORMAL_SAMPLE setup */
D
David Howells 已提交
577
	dbs_info->sample_type = DBS_NORMAL_SAMPLE;
578
	if (!dbs_tuners_ins.powersave_bias ||
D
David Howells 已提交
579
	    sample_type == DBS_NORMAL_SAMPLE) {
580 581 582
		dbs_check_cpu(dbs_info);
		if (dbs_info->freq_lo) {
			/* Setup timer for SUB_SAMPLE */
D
David Howells 已提交
583
			dbs_info->sample_type = DBS_SUB_SAMPLE;
584
			delay = dbs_info->freq_hi_jiffies;
585 586 587 588 589 590 591 592 593
		} else {
			/* We want all CPUs to do sampling nearly on
			 * same jiffy
			 */
			delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate
				* dbs_info->rate_mult);

			if (num_online_cpus() > 1)
				delay -= jiffies % delay;
594 595 596
		}
	} else {
		__cpufreq_driver_target(dbs_info->cur_policy,
597
			dbs_info->freq_lo, CPUFREQ_RELATION_H);
598
		delay = dbs_info->freq_lo_jiffies;
599
	}
600
	schedule_delayed_work_on(cpu, &dbs_info->work, delay);
601
	mutex_unlock(&dbs_info->timer_mutex);
602
}
L
Linus Torvalds 已提交
603

604
static inline void dbs_timer_init(struct cpu_dbs_info_s *dbs_info)
L
Linus Torvalds 已提交
605
{
606 607
	/* We want all CPUs to do sampling nearly on same jiffy */
	int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
608 609 610

	if (num_online_cpus() > 1)
		delay -= jiffies % delay;
611

D
David Howells 已提交
612
	dbs_info->sample_type = DBS_NORMAL_SAMPLE;
613
	INIT_DELAYED_WORK_DEFERRABLE(&dbs_info->work, do_dbs_timer);
614
	schedule_delayed_work_on(dbs_info->cpu, &dbs_info->work, delay);
L
Linus Torvalds 已提交
615 616
}

617
static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info)
L
Linus Torvalds 已提交
618
{
619
	cancel_delayed_work_sync(&dbs_info->work);
L
Linus Torvalds 已提交
620 621
}

622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
/*
 * Not all CPUs want IO time to be accounted as busy; this dependson how
 * efficient idling at a higher frequency/voltage is.
 * Pavel Machek says this is not so for various generations of AMD and old
 * Intel systems.
 * Mike Chan (androidlcom) calis this is also not true for ARM.
 * Because of this, whitelist specific known (series) of CPUs by default, and
 * leave all others up to the user.
 */
static int should_io_be_busy(void)
{
#if defined(CONFIG_X86)
	/*
	 * For Intel, Core 2 (model 15) andl later have an efficient idle.
	 */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
	    boot_cpu_data.x86 == 6 &&
	    boot_cpu_data.x86_model >= 15)
		return 1;
#endif
	return 0;
}

L
Linus Torvalds 已提交
645 646 647 648 649 650
static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
				   unsigned int event)
{
	unsigned int cpu = policy->cpu;
	struct cpu_dbs_info_s *this_dbs_info;
	unsigned int j;
J
Jeff Garzik 已提交
651
	int rc;
L
Linus Torvalds 已提交
652

653
	this_dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
L
Linus Torvalds 已提交
654 655 656

	switch (event) {
	case CPUFREQ_GOV_START:
657
		if ((!cpu_online(cpu)) || (!policy->cur))
L
Linus Torvalds 已提交
658 659
			return -EINVAL;

660
		mutex_lock(&dbs_mutex);
J
Jeff Garzik 已提交
661

662
		dbs_enable++;
663
		for_each_cpu(j, policy->cpus) {
L
Linus Torvalds 已提交
664
			struct cpu_dbs_info_s *j_dbs_info;
665
			j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
L
Linus Torvalds 已提交
666
			j_dbs_info->cur_policy = policy;
667

668 669
			j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
						&j_dbs_info->prev_cpu_wall);
670 671 672 673
			if (dbs_tuners_ins.ignore_nice) {
				j_dbs_info->prev_cpu_nice =
						kstat_cpu(j).cpustat.nice;
			}
L
Linus Torvalds 已提交
674
		}
675
		this_dbs_info->cpu = cpu;
676
		this_dbs_info->rate_mult = 1;
677
		ondemand_powersave_bias_init_cpu(cpu);
L
Linus Torvalds 已提交
678 679 680 681 682 683
		/*
		 * Start the timerschedule work, when this governor
		 * is used for first time
		 */
		if (dbs_enable == 1) {
			unsigned int latency;
684 685 686 687 688 689 690 691

			rc = sysfs_create_group(cpufreq_global_kobject,
						&dbs_attr_group);
			if (rc) {
				mutex_unlock(&dbs_mutex);
				return rc;
			}

L
Linus Torvalds 已提交
692
			/* policy latency is in nS. Convert it to uS first */
693 694 695
			latency = policy->cpuinfo.transition_latency / 1000;
			if (latency == 0)
				latency = 1;
696 697 698 699 700 701
			/* Bring kernel and HW constraints together */
			min_sampling_rate = max(min_sampling_rate,
					MIN_LATENCY_MULTIPLIER * latency);
			dbs_tuners_ins.sampling_rate =
				max(min_sampling_rate,
				    latency * LATENCY_MULTIPLIER);
702
			dbs_tuners_ins.io_is_busy = should_io_be_busy();
L
Linus Torvalds 已提交
703
		}
704
		mutex_unlock(&dbs_mutex);
705

706
		mutex_init(&this_dbs_info->timer_mutex);
707
		dbs_timer_init(this_dbs_info);
L
Linus Torvalds 已提交
708 709 710
		break;

	case CPUFREQ_GOV_STOP:
711
		dbs_timer_exit(this_dbs_info);
712 713

		mutex_lock(&dbs_mutex);
714
		mutex_destroy(&this_dbs_info->timer_mutex);
L
Linus Torvalds 已提交
715
		dbs_enable--;
716
		mutex_unlock(&dbs_mutex);
717 718 719
		if (!dbs_enable)
			sysfs_remove_group(cpufreq_global_kobject,
					   &dbs_attr_group);
L
Linus Torvalds 已提交
720 721 722 723

		break;

	case CPUFREQ_GOV_LIMITS:
724
		mutex_lock(&this_dbs_info->timer_mutex);
L
Linus Torvalds 已提交
725
		if (policy->max < this_dbs_info->cur_policy->cur)
726
			__cpufreq_driver_target(this_dbs_info->cur_policy,
727
				policy->max, CPUFREQ_RELATION_H);
L
Linus Torvalds 已提交
728
		else if (policy->min > this_dbs_info->cur_policy->cur)
729
			__cpufreq_driver_target(this_dbs_info->cur_policy,
730
				policy->min, CPUFREQ_RELATION_L);
731
		mutex_unlock(&this_dbs_info->timer_mutex);
L
Linus Torvalds 已提交
732 733 734 735 736 737 738
		break;
	}
	return 0;
}

static int __init cpufreq_gov_dbs_init(void)
{
739
	cputime64_t wall;
740 741
	u64 idle_time;
	int cpu = get_cpu();
742

743 744
	idle_time = get_cpu_idle_time_us(cpu, &wall);
	put_cpu();
745 746 747 748 749
	if (idle_time != -1ULL) {
		/* Idle micro accounting is supported. Use finer thresholds */
		dbs_tuners_ins.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
		dbs_tuners_ins.down_differential =
					MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
750 751 752 753 754 755 756 757 758 759
		/*
		 * In no_hz/micro accounting case we set the minimum frequency
		 * not depending on HZ, but fixed (very low). The deferred
		 * timer might skip some samples if idle/sleeping as needed.
		*/
		min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
	} else {
		/* For correct statistics, we need 10 ticks for each measure */
		min_sampling_rate =
			MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10);
760
	}
761

762
	return cpufreq_register_governor(&cpufreq_gov_ondemand);
L
Linus Torvalds 已提交
763 764 765 766
}

static void __exit cpufreq_gov_dbs_exit(void)
{
767
	cpufreq_unregister_governor(&cpufreq_gov_ondemand);
L
Linus Torvalds 已提交
768 769 770
}


771 772 773
MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
774
	"Low Latency Frequency Transition capable processors");
775
MODULE_LICENSE("GPL");
L
Linus Torvalds 已提交
776

777 778 779
#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
fs_initcall(cpufreq_gov_dbs_init);
#else
L
Linus Torvalds 已提交
780
module_init(cpufreq_gov_dbs_init);
781
#endif
L
Linus Torvalds 已提交
782
module_exit(cpufreq_gov_dbs_exit);