cpufreq_governor.c 16.2 KB
Newer Older
1 2 3 4 5
/*
 * drivers/cpufreq/cpufreq_governor.c
 *
 * CPUFREQ governors common code
 *
6 7 8 9 10 11
 * Copyright	(C) 2001 Russell King
 *		(C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *		(C) 2003 Jun Nakajima <jun.nakajima@intel.com>
 *		(C) 2009 Alexander Clouter <alex@digriz.org.uk>
 *		(c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
 *
12 13 14 15 16
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

17 18
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

19 20
#include <linux/export.h>
#include <linux/kernel_stat.h>
21
#include <linux/slab.h>
22 23 24

#include "cpufreq_governor.h"

25 26 27
DEFINE_MUTEX(dbs_data_mutex);
EXPORT_SYMBOL_GPL(dbs_data_mutex);

28
static inline struct dbs_data *to_dbs_data(struct kobject *kobj)
29
{
30
	return container_of(kobj, struct dbs_data, kobj);
31 32
}

33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
static inline struct governor_attr *to_gov_attr(struct attribute *attr)
{
	return container_of(attr, struct governor_attr, attr);
}

static ssize_t governor_show(struct kobject *kobj, struct attribute *attr,
			     char *buf)
{
	struct dbs_data *dbs_data = to_dbs_data(kobj);
	struct governor_attr *gattr = to_gov_attr(attr);
	int ret = -EIO;

	if (gattr->show)
		ret = gattr->show(dbs_data, buf);

	return ret;
}

static ssize_t governor_store(struct kobject *kobj, struct attribute *attr,
			      const char *buf, size_t count)
{
	struct dbs_data *dbs_data = to_dbs_data(kobj);
	struct governor_attr *gattr = to_gov_attr(attr);
	int ret = -EIO;

	mutex_lock(&dbs_data->mutex);

	if (gattr->store)
		ret = gattr->store(dbs_data, buf, count);

	mutex_unlock(&dbs_data->mutex);

	return ret;
}

/*
 * Sysfs Ops for accessing governor attributes.
 *
 * All show/store invocations for governor specific sysfs attributes, will first
 * call the below show/store callbacks and the attribute specific callback will
 * be called from within it.
 */
static const struct sysfs_ops governor_sysfs_ops = {
	.show	= governor_show,
	.store	= governor_store,
};

80
void dbs_check_cpu(struct cpufreq_policy *policy)
81
{
82
	int cpu = policy->cpu;
83
	struct dbs_governor *gov = dbs_governor_of(policy);
84 85
	struct policy_dbs_info *policy_dbs = policy->governor_data;
	struct dbs_data *dbs_data = policy_dbs->dbs_data;
86
	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
87 88
	unsigned int sampling_rate = dbs_data->sampling_rate;
	unsigned int ignore_nice = dbs_data->ignore_nice_load;
89 90 91
	unsigned int max_load = 0;
	unsigned int j;

92
	if (gov->governor == GOV_ONDEMAND) {
93
		struct od_cpu_dbs_info_s *od_dbs_info =
94
				gov->get_cpu_dbs_info_s(cpu);
95 96 97 98 99 100 101 102 103 104

		/*
		 * Sometimes, the ondemand governor uses an additional
		 * multiplier to give long delays. So apply this multiplier to
		 * the 'sampling_rate', so as to keep the wake-up-from-idle
		 * detection logic a bit conservative.
		 */
		sampling_rate *= od_dbs_info->rate_mult;

	}
105

106
	/* Get Absolute Load */
107
	for_each_cpu(j, policy->cpus) {
108
		struct cpu_dbs_info *j_cdbs;
109 110
		u64 cur_wall_time, cur_idle_time;
		unsigned int idle_time, wall_time;
111
		unsigned int load;
112
		int io_busy = 0;
113

114
		j_cdbs = gov->get_cpu_cdbs(j);
115

116 117 118 119 120 121
		/*
		 * For the purpose of ondemand, waiting for disk IO is
		 * an indication that you're performance critical, and
		 * not that the system is actually idle. So do not add
		 * the iowait time to the cpu idle time.
		 */
122
		if (gov->governor == GOV_ONDEMAND)
123 124
			io_busy = od_tuners->io_is_busy;
		cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy);
125 126 127 128 129

		wall_time = (unsigned int)
			(cur_wall_time - j_cdbs->prev_cpu_wall);
		j_cdbs->prev_cpu_wall = cur_wall_time;

130 131 132
		if (cur_idle_time < j_cdbs->prev_cpu_idle)
			cur_idle_time = j_cdbs->prev_cpu_idle;

133 134 135 136 137
		idle_time = (unsigned int)
			(cur_idle_time - j_cdbs->prev_cpu_idle);
		j_cdbs->prev_cpu_idle = cur_idle_time;

		if (ignore_nice) {
138
			struct cpu_dbs_info *cdbs = gov->get_cpu_cdbs(cpu);
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
			u64 cur_nice;
			unsigned long cur_nice_jiffies;

			cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE] -
					 cdbs->prev_cpu_nice;
			/*
			 * Assumption: nice time between sampling periods will
			 * be less than 2^32 jiffies for 32 bit sys
			 */
			cur_nice_jiffies = (unsigned long)
					cputime64_to_jiffies64(cur_nice);

			cdbs->prev_cpu_nice =
				kcpustat_cpu(j).cpustat[CPUTIME_NICE];
			idle_time += jiffies_to_usecs(cur_nice_jiffies);
		}

		if (unlikely(!wall_time || wall_time < idle_time))
			continue;

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
		/*
		 * If the CPU had gone completely idle, and a task just woke up
		 * on this CPU now, it would be unfair to calculate 'load' the
		 * usual way for this elapsed time-window, because it will show
		 * near-zero load, irrespective of how CPU intensive that task
		 * actually is. This is undesirable for latency-sensitive bursty
		 * workloads.
		 *
		 * To avoid this, we reuse the 'load' from the previous
		 * time-window and give this task a chance to start with a
		 * reasonably high CPU frequency. (However, we shouldn't over-do
		 * this copy, lest we get stuck at a high load (high frequency)
		 * for too long, even when the current system load has actually
		 * dropped down. So we perform the copy only once, upon the
		 * first wake-up from idle.)
		 *
175 176 177 178
		 * Detecting this situation is easy: the governor's utilization
		 * update handler would not have run during CPU-idle periods.
		 * Hence, an unusually large 'wall_time' (as compared to the
		 * sampling rate) indicates this scenario.
179 180 181 182 183
		 *
		 * prev_load can be zero in two cases and we must recalculate it
		 * for both cases:
		 * - during long idle intervals
		 * - explicitly set to zero
184
		 */
185 186
		if (unlikely(wall_time > (2 * sampling_rate) &&
			     j_cdbs->prev_load)) {
187
			load = j_cdbs->prev_load;
188 189 190 191 192 193 194

			/*
			 * Perform a destructive copy, to ensure that we copy
			 * the previous load only once, upon the first wake-up
			 * from idle.
			 */
			j_cdbs->prev_load = 0;
195 196 197 198
		} else {
			load = 100 * (wall_time - idle_time) / wall_time;
			j_cdbs->prev_load = load;
		}
199 200 201 202 203

		if (load > max_load)
			max_load = load;
	}

204
	gov->gov_check_cpu(cpu, max_load);
205 206 207
}
EXPORT_SYMBOL_GPL(dbs_check_cpu);

208
void gov_set_update_util(struct policy_dbs_info *policy_dbs,
209
			 unsigned int delay_us)
210
{
211
	struct cpufreq_policy *policy = policy_dbs->policy;
212
	struct dbs_governor *gov = dbs_governor_of(policy);
213
	int cpu;
214

215 216
	gov_update_sample_delay(policy_dbs, delay_us);
	policy_dbs->last_sample_time = 0;
217

218
	for_each_cpu(cpu, policy->cpus) {
219
		struct cpu_dbs_info *cdbs = gov->get_cpu_cdbs(cpu);
220 221

		cpufreq_set_update_util_data(cpu, &cdbs->update_util);
222 223
	}
}
224
EXPORT_SYMBOL_GPL(gov_set_update_util);
225

226
static inline void gov_clear_update_util(struct cpufreq_policy *policy)
227 228
{
	int i;
229

230 231 232 233
	for_each_cpu(i, policy->cpus)
		cpufreq_set_update_util_data(i, NULL);

	synchronize_rcu();
234 235
}

236
static void gov_cancel_work(struct policy_dbs_info *policy_dbs)
237
{
238
	/* Tell dbs_update_util_handler() to skip queuing up work items. */
239
	atomic_inc(&policy_dbs->work_count);
240
	/*
241
	 * If dbs_update_util_handler() is already running, it may not notice
242
	 * the incremented work_count, so wait for it to complete to prevent its
243
	 * work item from being queued up after the cancel_work_sync() below.
244
	 */
245 246 247
	gov_clear_update_util(policy_dbs->policy);
	irq_work_sync(&policy_dbs->irq_work);
	cancel_work_sync(&policy_dbs->work);
248
	atomic_set(&policy_dbs->work_count, 0);
249
}
250

251
static void dbs_work_handler(struct work_struct *work)
252
{
253
	struct policy_dbs_info *policy_dbs;
254
	struct cpufreq_policy *policy;
255
	struct dbs_governor *gov;
256
	unsigned int delay;
257

258 259
	policy_dbs = container_of(work, struct policy_dbs_info, work);
	policy = policy_dbs->policy;
260
	gov = dbs_governor_of(policy);
261

262
	/*
263 264
	 * Make sure cpufreq_governor_limits() isn't evaluating load or the
	 * ondemand governor isn't updating the sampling rate in parallel.
265
	 */
266
	mutex_lock(&policy_dbs->timer_mutex);
267
	delay = gov->gov_dbs_timer(policy);
268 269
	policy_dbs->sample_delay_ns = jiffies_to_nsecs(delay);
	mutex_unlock(&policy_dbs->timer_mutex);
270

271 272 273 274 275 276
	/*
	 * If the atomic operation below is reordered with respect to the
	 * sample delay modification, the utilization update handler may end
	 * up using a stale sample delay value.
	 */
	smp_mb__before_atomic();
277
	atomic_dec(&policy_dbs->work_count);
278 279 280 281
}

static void dbs_irq_work(struct irq_work *irq_work)
{
282
	struct policy_dbs_info *policy_dbs;
283

284 285
	policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work);
	schedule_work(&policy_dbs->work);
286 287
}

288
static inline void gov_queue_irq_work(struct policy_dbs_info *policy_dbs)
289
{
290
#ifdef CONFIG_SMP
291
	irq_work_queue_on(&policy_dbs->irq_work, smp_processor_id());
292
#else
293
	irq_work_queue(&policy_dbs->irq_work);
294 295 296 297 298 299 300
#endif
}

static void dbs_update_util_handler(struct update_util_data *data, u64 time,
				    unsigned long util, unsigned long max)
{
	struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util);
301
	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
302 303

	/*
304 305 306 307 308
	 * The work may not be allowed to be queued up right now.
	 * Possible reasons:
	 * - Work has already been queued up or is in progress.
	 * - The governor is being stopped.
	 * - It is too early (too little time from the previous sample).
309
	 */
310
	if (atomic_inc_return(&policy_dbs->work_count) == 1) {
311 312
		u64 delta_ns;

313 314 315 316
		delta_ns = time - policy_dbs->last_sample_time;
		if ((s64)delta_ns >= policy_dbs->sample_delay_ns) {
			policy_dbs->last_sample_time = time;
			gov_queue_irq_work(policy_dbs);
317 318 319
			return;
		}
	}
320
	atomic_dec(&policy_dbs->work_count);
321
}
322

323 324
static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy,
						     struct dbs_governor *gov)
325
{
326
	struct policy_dbs_info *policy_dbs;
327 328 329
	int j;

	/* Allocate memory for the common information for policy->cpus */
330 331
	policy_dbs = kzalloc(sizeof(*policy_dbs), GFP_KERNEL);
	if (!policy_dbs)
332
		return NULL;
333

334
	mutex_init(&policy_dbs->timer_mutex);
335
	atomic_set(&policy_dbs->work_count, 0);
336 337
	init_irq_work(&policy_dbs->irq_work, dbs_irq_work);
	INIT_WORK(&policy_dbs->work, dbs_work_handler);
338 339 340 341 342 343 344 345

	/* Set policy_dbs for all CPUs, online+offline */
	for_each_cpu(j, policy->related_cpus) {
		struct cpu_dbs_info *j_cdbs = gov->get_cpu_cdbs(j);

		j_cdbs->policy_dbs = policy_dbs;
		j_cdbs->update_util.func = dbs_update_util_handler;
	}
346
	return policy_dbs;
347 348
}

349
static void free_policy_dbs_info(struct cpufreq_policy *policy,
350
				 struct dbs_governor *gov)
351
{
352
	struct cpu_dbs_info *cdbs = gov->get_cpu_cdbs(policy->cpu);
353
	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
354 355
	int j;

356
	mutex_destroy(&policy_dbs->timer_mutex);
357

358 359
	for_each_cpu(j, policy->related_cpus) {
		struct cpu_dbs_info *j_cdbs = gov->get_cpu_cdbs(j);
360

361 362 363
		j_cdbs->policy_dbs = NULL;
		j_cdbs->update_util.func = NULL;
	}
364
	kfree(policy_dbs);
365 366
}

367
static int cpufreq_governor_init(struct cpufreq_policy *policy)
368
{
369
	struct dbs_governor *gov = dbs_governor_of(policy);
370
	struct dbs_data *dbs_data = gov->gdbs_data;
371
	struct policy_dbs_info *policy_dbs;
372 373
	unsigned int latency;
	int ret;
374

375 376 377 378
	/* State should be equivalent to EXIT */
	if (policy->governor_data)
		return -EBUSY;

379 380 381
	policy_dbs = alloc_policy_dbs_info(policy, gov);
	if (!policy_dbs)
		return -ENOMEM;
382

383 384 385 386 387
	if (dbs_data) {
		if (WARN_ON(have_governor_per_policy())) {
			ret = -EINVAL;
			goto free_policy_dbs_info;
		}
388
		dbs_data->usage_count++;
389 390
		policy_dbs->dbs_data = dbs_data;
		policy->governor_data = policy_dbs;
391 392
		return 0;
	}
393

394
	dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
395 396 397 398
	if (!dbs_data) {
		ret = -ENOMEM;
		goto free_policy_dbs_info;
	}
399

400
	dbs_data->usage_count = 1;
401
	mutex_init(&dbs_data->mutex);
402

403
	ret = gov->init(dbs_data, !policy->governor->initialized);
404
	if (ret)
405
		goto free_policy_dbs_info;
406

407 408 409 410
	/* policy latency is in ns. Convert it to us first */
	latency = policy->cpuinfo.transition_latency / 1000;
	if (latency == 0)
		latency = 1;
411

412 413 414
	/* Bring kernel and HW constraints together */
	dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
					  MIN_LATENCY_MULTIPLIER * latency);
415 416
	dbs_data->sampling_rate = max(dbs_data->min_sampling_rate,
				      LATENCY_MULTIPLIER * latency);
417

418
	if (!have_governor_per_policy())
419
		gov->gdbs_data = dbs_data;
420

421 422
	policy_dbs->dbs_data = dbs_data;
	policy->governor_data = policy_dbs;
423

424 425 426 427
	gov->kobj_type.sysfs_ops = &governor_sysfs_ops;
	ret = kobject_init_and_add(&dbs_data->kobj, &gov->kobj_type,
				   get_governor_parent_kobj(policy),
				   "%s", gov->gov.name);
428 429
	if (!ret)
		return 0;
430

431
	/* Failure, so roll back. */
432
	pr_err("cpufreq: Governor initialization failed (dbs_data kobject init error %d)\n", ret);
433

434 435
	policy->governor_data = NULL;

436
	if (!have_governor_per_policy())
437 438
		gov->gdbs_data = NULL;
	gov->exit(dbs_data, !policy->governor->initialized);
439 440
	kfree(dbs_data);

441 442
free_policy_dbs_info:
	free_policy_dbs_info(policy, gov);
443 444
	return ret;
}
445

446
static int cpufreq_governor_exit(struct cpufreq_policy *policy)
447
{
448
	struct dbs_governor *gov = dbs_governor_of(policy);
449 450
	struct policy_dbs_info *policy_dbs = policy->governor_data;
	struct dbs_data *dbs_data = policy_dbs->dbs_data;
451 452

	/* State should be equivalent to INIT */
453
	if (policy_dbs->policy)
454
		return -EBUSY;
455

456
	if (!--dbs_data->usage_count) {
457
		kobject_put(&dbs_data->kobj);
458

459 460
		policy->governor_data = NULL;

461
		if (!have_governor_per_policy())
462
			gov->gdbs_data = NULL;
463

464
		gov->exit(dbs_data, policy->governor->initialized == 1);
465
		mutex_destroy(&dbs_data->mutex);
466
		kfree(dbs_data);
467 468
	} else {
		policy->governor_data = NULL;
469
	}
470

471
	free_policy_dbs_info(policy, gov);
472
	return 0;
473
}
474

475
static int cpufreq_governor_start(struct cpufreq_policy *policy)
476
{
477
	struct dbs_governor *gov = dbs_governor_of(policy);
478 479
	struct policy_dbs_info *policy_dbs = policy->governor_data;
	struct dbs_data *dbs_data = policy_dbs->dbs_data;
480 481 482 483 484 485
	unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu;
	int io_busy = 0;

	if (!policy->cur)
		return -EINVAL;

486
	/* State should be equivalent to INIT */
487
	if (policy_dbs->policy)
488 489
		return -EBUSY;

490 491
	sampling_rate = dbs_data->sampling_rate;
	ignore_nice = dbs_data->ignore_nice_load;
492

493
	if (gov->governor == GOV_ONDEMAND) {
494 495
		struct od_dbs_tuners *od_tuners = dbs_data->tuners;

496
		io_busy = od_tuners->io_is_busy;
497 498
	}

499
	for_each_cpu(j, policy->cpus) {
500
		struct cpu_dbs_info *j_cdbs = gov->get_cpu_cdbs(j);
501
		unsigned int prev_load;
502

503 504
		j_cdbs->prev_cpu_idle =
			get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);
505

506 507 508 509
		prev_load = (unsigned int)(j_cdbs->prev_cpu_wall -
					    j_cdbs->prev_cpu_idle);
		j_cdbs->prev_load = 100 * prev_load /
				    (unsigned int)j_cdbs->prev_cpu_wall;
510

511 512 513
		if (ignore_nice)
			j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
	}
514
	policy_dbs->policy = policy;
515

516
	if (gov->governor == GOV_CONSERVATIVE) {
517
		struct cs_cpu_dbs_info_s *cs_dbs_info =
518
			gov->get_cpu_dbs_info_s(cpu);
519

520 521 522
		cs_dbs_info->down_skip = 0;
		cs_dbs_info->requested_freq = policy->cur;
	} else {
523 524
		struct od_ops *od_ops = gov->gov_ops;
		struct od_cpu_dbs_info_s *od_dbs_info = gov->get_cpu_dbs_info_s(cpu);
525

526 527 528 529
		od_dbs_info->rate_mult = 1;
		od_dbs_info->sample_type = OD_NORMAL_SAMPLE;
		od_ops->powersave_bias_init_cpu(cpu);
	}
530

531
	gov_set_update_util(policy_dbs, sampling_rate);
532 533 534
	return 0;
}

535
static int cpufreq_governor_stop(struct cpufreq_policy *policy)
536
{
537
	struct policy_dbs_info *policy_dbs = policy->governor_data;
538

539
	/* State should be equivalent to START */
540
	if (!policy_dbs->policy)
541 542
		return -EBUSY;

543 544
	gov_cancel_work(policy_dbs);
	policy_dbs->policy = NULL;
545

546
	return 0;
547
}
548

549
static int cpufreq_governor_limits(struct cpufreq_policy *policy)
550
{
551
	struct policy_dbs_info *policy_dbs = policy->governor_data;
552

553
	/* State should be equivalent to START */
554
	if (!policy_dbs->policy)
555
		return -EBUSY;
556

557 558 559 560 561
	mutex_lock(&policy_dbs->timer_mutex);
	if (policy->max < policy->cur)
		__cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
	else if (policy->min > policy->cur)
		__cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
562
	dbs_check_cpu(policy);
563
	mutex_unlock(&policy_dbs->timer_mutex);
564 565

	return 0;
566
}
567

568
int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event)
569
{
570
	int ret = -EINVAL;
571

572
	/* Lock governor to block concurrent initialization of governor */
573
	mutex_lock(&dbs_data_mutex);
574

575
	if (event == CPUFREQ_GOV_POLICY_INIT) {
576
		ret = cpufreq_governor_init(policy);
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
	} else if (policy->governor_data) {
		switch (event) {
		case CPUFREQ_GOV_POLICY_EXIT:
			ret = cpufreq_governor_exit(policy);
			break;
		case CPUFREQ_GOV_START:
			ret = cpufreq_governor_start(policy);
			break;
		case CPUFREQ_GOV_STOP:
			ret = cpufreq_governor_stop(policy);
			break;
		case CPUFREQ_GOV_LIMITS:
			ret = cpufreq_governor_limits(policy);
			break;
		}
592
	}
593

594
	mutex_unlock(&dbs_data_mutex);
595
	return ret;
596 597
}
EXPORT_SYMBOL_GPL(cpufreq_governor_dbs);