cpufreq.c 54.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *  linux/drivers/cpufreq/cpufreq.c
 *
 *  Copyright (C) 2001 Russell King
 *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6
 *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
L
Linus Torvalds 已提交
7
 *
8
 *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
9
 *	Added handling for CPU hotplug
10 11
 *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
 *	Fix handling for CPU hotplug -- affected CPUs
12
 *
L
Linus Torvalds 已提交
13 14 15 16 17
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

V
Viresh Kumar 已提交
18 19
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

20
#include <linux/cpu.h>
L
Linus Torvalds 已提交
21 22 23
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/device.h>
24 25 26
#include <linux/init.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
27
#include <linux/mutex.h>
28
#include <linux/slab.h>
29
#include <linux/syscore_ops.h>
30
#include <linux/tick.h>
31 32
#include <trace/events/power.h>

L
Linus Torvalds 已提交
33
/**
D
Dave Jones 已提交
34
 * The "cpufreq driver" - the arch- or hardware-dependent low
L
Linus Torvalds 已提交
35 36 37
 * level driver of CPUFreq support, and its spinlock. This lock
 * also protects the cpufreq_cpu_data array.
 */
38
static struct cpufreq_driver *cpufreq_driver;
39
static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
40
static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
41 42
static DEFINE_RWLOCK(cpufreq_driver_lock);
static DEFINE_MUTEX(cpufreq_governor_lock);
43
static LIST_HEAD(cpufreq_policy_list);
44

45 46
#ifdef CONFIG_HOTPLUG_CPU
/* This one keeps track of the previously set governor of a removed CPU */
47
static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
48
#endif
L
Linus Torvalds 已提交
49

50 51 52 53 54 55 56 57 58 59 60 61 62 63
/*
 * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
 * all cpufreq/hotplug/workqueue/etc related lock issues.
 *
 * The rules for this semaphore:
 * - Any routine that wants to read from the policy structure will
 *   do a down_read on this semaphore.
 * - Any routine that will write to the policy structure and/or may take away
 *   the policy altogether (eg. CPU hotplug), will hold this lock in write
 *   mode before doing so.
 *
 * Additional rules:
 * - Governor routines that can be called in cpufreq hotplug path should not
 *   take this sem as top level hotplug notifier handler takes this.
64 65
 * - Lock should not be held across
 *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
66
 */
67
static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
68 69 70
static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);

#define lock_policy_rwsem(mode, cpu)					\
71
static int lock_policy_rwsem_##mode(int cpu)				\
72
{									\
73
	int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);		\
74 75 76 77 78 79 80 81 82
	BUG_ON(policy_cpu == -1);					\
	down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));		\
									\
	return 0;							\
}

lock_policy_rwsem(read, cpu);
lock_policy_rwsem(write, cpu);

83 84 85 86 87 88
#define unlock_policy_rwsem(mode, cpu)					\
static void unlock_policy_rwsem_##mode(int cpu)				\
{									\
	int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);		\
	BUG_ON(policy_cpu == -1);					\
	up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));		\
89 90
}

91 92
unlock_policy_rwsem(read, cpu);
unlock_policy_rwsem(write, cpu);
93

L
Linus Torvalds 已提交
94
/* internal prototypes */
95 96
static int __cpufreq_governor(struct cpufreq_policy *policy,
		unsigned int event);
97
static unsigned int __cpufreq_get(unsigned int cpu);
98
static void handle_update(struct work_struct *work);
L
Linus Torvalds 已提交
99 100

/**
101 102
 * Two notifier lists: the "policy" list is involved in the
 * validation process for a new CPU frequency policy; the
L
Linus Torvalds 已提交
103 104 105 106
 * "transition" list for kernel code that needs to handle
 * changes to devices when the CPU clock speed changes.
 * The mutex locks both lists.
 */
107
static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
108
static struct srcu_notifier_head cpufreq_transition_notifier_list;
L
Linus Torvalds 已提交
109

110
static bool init_cpufreq_transition_notifier_list_called;
111 112 113
static int __init init_cpufreq_transition_notifier_list(void)
{
	srcu_init_notifier_head(&cpufreq_transition_notifier_list);
114
	init_cpufreq_transition_notifier_list_called = true;
115 116
	return 0;
}
117
pure_initcall(init_cpufreq_transition_notifier_list);
L
Linus Torvalds 已提交
118

119
static int off __read_mostly;
120
static int cpufreq_disabled(void)
121 122 123 124 125 126 127
{
	return off;
}
void disable_cpufreq(void)
{
	off = 1;
}
L
Linus Torvalds 已提交
128
static LIST_HEAD(cpufreq_governor_list);
129
static DEFINE_MUTEX(cpufreq_governor_mutex);
L
Linus Torvalds 已提交
130

131 132
bool have_governor_per_policy(void)
{
133
	return cpufreq_driver->have_governor_per_policy;
134
}
135
EXPORT_SYMBOL_GPL(have_governor_per_policy);
136

137 138 139 140 141 142 143 144 145
struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
{
	if (have_governor_per_policy())
		return &policy->kobj;
	else
		return cpufreq_global_kobject;
}
EXPORT_SYMBOL_GPL(get_governor_parent_kobj);

146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
{
	u64 idle_time;
	u64 cur_wall_time;
	u64 busy_time;

	cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());

	busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
	busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];

	idle_time = cur_wall_time - busy_time;
	if (wall)
		*wall = cputime_to_usecs(cur_wall_time);

	return cputime_to_usecs(idle_time);
}

u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
{
	u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);

	if (idle_time == -1ULL)
		return get_cpu_idle_time_jiffy(cpu, wall);
	else if (!io_busy)
		idle_time += get_cpu_iowait_time_us(cpu, wall);

	return idle_time;
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time);

181
static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
L
Linus Torvalds 已提交
182
{
183
	struct cpufreq_policy *policy;
L
Linus Torvalds 已提交
184 185
	unsigned long flags;

186
	if (cpu >= nr_cpu_ids)
L
Linus Torvalds 已提交
187 188 189
		goto err_out;

	/* get the cpufreq driver */
190
	read_lock_irqsave(&cpufreq_driver_lock, flags);
L
Linus Torvalds 已提交
191

192
	if (!cpufreq_driver)
L
Linus Torvalds 已提交
193 194
		goto err_out_unlock;

195
	if (!try_module_get(cpufreq_driver->owner))
L
Linus Torvalds 已提交
196 197 198
		goto err_out_unlock;

	/* get the CPU */
199
	policy = per_cpu(cpufreq_cpu_data, cpu);
L
Linus Torvalds 已提交
200

201
	if (!policy)
L
Linus Torvalds 已提交
202 203
		goto err_out_put_module;

204
	if (!sysfs && !kobject_get(&policy->kobj))
L
Linus Torvalds 已提交
205 206
		goto err_out_put_module;

207
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
208
	return policy;
L
Linus Torvalds 已提交
209

210
err_out_put_module:
211
	module_put(cpufreq_driver->owner);
212
err_out_unlock:
213
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
214
err_out:
L
Linus Torvalds 已提交
215 216
	return NULL;
}
217 218 219

struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
{
220 221 222
	if (cpufreq_disabled())
		return NULL;

223 224
	return __cpufreq_cpu_get(cpu, false);
}
L
Linus Torvalds 已提交
225 226
EXPORT_SYMBOL_GPL(cpufreq_cpu_get);

227 228 229 230 231
static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
{
	return __cpufreq_cpu_get(cpu, true);
}

232
static void __cpufreq_cpu_put(struct cpufreq_policy *policy, bool sysfs)
233 234
{
	if (!sysfs)
235
		kobject_put(&policy->kobj);
236
	module_put(cpufreq_driver->owner);
237
}
238

239
void cpufreq_cpu_put(struct cpufreq_policy *policy)
L
Linus Torvalds 已提交
240
{
241 242 243
	if (cpufreq_disabled())
		return;

244
	__cpufreq_cpu_put(policy, false);
L
Linus Torvalds 已提交
245 246 247
}
EXPORT_SYMBOL_GPL(cpufreq_cpu_put);

248
static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *policy)
249
{
250
	__cpufreq_cpu_put(policy, true);
251
}
L
Linus Torvalds 已提交
252 253 254 255 256 257 258 259 260 261

/*********************************************************************
 *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 *********************************************************************/

/**
 * adjust_jiffies - adjust the system "loops_per_jiffy"
 *
 * This function alters the system "loops_per_jiffy" for the clock
 * speed change. Note that loops_per_jiffy cannot be updated on SMP
262
 * systems as each CPU might be scaled differently. So, use the arch
L
Linus Torvalds 已提交
263 264 265 266
 * per-CPU loops_per_jiffy value wherever possible.
 */
#ifndef CONFIG_SMP
static unsigned long l_p_j_ref;
267
static unsigned int l_p_j_ref_freq;
L
Linus Torvalds 已提交
268

269
static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
L
Linus Torvalds 已提交
270 271 272 273 274 275 276
{
	if (ci->flags & CPUFREQ_CONST_LOOPS)
		return;

	if (!l_p_j_ref_freq) {
		l_p_j_ref = loops_per_jiffy;
		l_p_j_ref_freq = ci->old;
277
		pr_debug("saving %lu as reference value for loops_per_jiffy; "
278
			"freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
L
Linus Torvalds 已提交
279
	}
280
	if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
281
	    (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
282 283
		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
								ci->new);
284
		pr_debug("scaling loops_per_jiffy to %lu "
285
			"for frequency %u kHz\n", loops_per_jiffy, ci->new);
L
Linus Torvalds 已提交
286 287 288
	}
}
#else
289 290 291 292
static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
{
	return;
}
L
Linus Torvalds 已提交
293 294
#endif

295
static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
296
		struct cpufreq_freqs *freqs, unsigned int state)
L
Linus Torvalds 已提交
297 298 299
{
	BUG_ON(irqs_disabled());

300 301 302
	if (cpufreq_disabled())
		return;

303
	freqs->flags = cpufreq_driver->flags;
304
	pr_debug("notification %u of frequency transition to %u kHz\n",
305
		state, freqs->new);
L
Linus Torvalds 已提交
306 307

	switch (state) {
308

L
Linus Torvalds 已提交
309
	case CPUFREQ_PRECHANGE:
310 311
		if (WARN(policy->transition_ongoing ==
					cpumask_weight(policy->cpus),
312 313 314
				"In middle of another frequency transition\n"))
			return;

315
		policy->transition_ongoing++;
316

317
		/* detect if the driver reported a value as "old frequency"
318 319
		 * which is not equal to what the cpufreq core thinks is
		 * "old frequency".
L
Linus Torvalds 已提交
320
		 */
321
		if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
322 323
			if ((policy) && (policy->cpu == freqs->cpu) &&
			    (policy->cur) && (policy->cur != freqs->old)) {
324
				pr_debug("Warning: CPU frequency is"
325 326 327
					" %u, cpufreq assumed %u kHz.\n",
					freqs->old, policy->cur);
				freqs->old = policy->cur;
L
Linus Torvalds 已提交
328 329
			}
		}
330
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
331
				CPUFREQ_PRECHANGE, freqs);
L
Linus Torvalds 已提交
332 333
		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
		break;
334

L
Linus Torvalds 已提交
335
	case CPUFREQ_POSTCHANGE:
336 337 338 339
		if (WARN(!policy->transition_ongoing,
				"No frequency transition in progress\n"))
			return;

340
		policy->transition_ongoing--;
341

L
Linus Torvalds 已提交
342
		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
343
		pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
344
			(unsigned long)freqs->cpu);
345
		trace_cpu_frequency(freqs->new, freqs->cpu);
346
		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
347
				CPUFREQ_POSTCHANGE, freqs);
348 349
		if (likely(policy) && likely(policy->cpu == freqs->cpu))
			policy->cur = freqs->new;
L
Linus Torvalds 已提交
350 351 352
		break;
	}
}
353

354 355 356 357 358 359 360 361 362 363 364 365 366 367
/**
 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 * on frequency transition.
 *
 * This function calls the transition notifiers and the "adjust_jiffies"
 * function. It is called twice on all CPU frequency changes that have
 * external effects.
 */
void cpufreq_notify_transition(struct cpufreq_policy *policy,
		struct cpufreq_freqs *freqs, unsigned int state)
{
	for_each_cpu(freqs->cpu, policy->cpus)
		__cpufreq_notify_transition(policy, freqs, state);
}
L
Linus Torvalds 已提交
368 369 370 371 372 373 374
EXPORT_SYMBOL_GPL(cpufreq_notify_transition);


/*********************************************************************
 *                          SYSFS INTERFACE                          *
 *********************************************************************/

375 376 377 378 379
static struct cpufreq_governor *__find_governor(const char *str_governor)
{
	struct cpufreq_governor *t;

	list_for_each_entry(t, &cpufreq_governor_list, governor_list)
380
		if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
381 382 383 384 385
			return t;

	return NULL;
}

L
Linus Torvalds 已提交
386 387 388
/**
 * cpufreq_parse_governor - parse a governor string
 */
D
Dave Jones 已提交
389
static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
L
Linus Torvalds 已提交
390 391
				struct cpufreq_governor **governor)
{
392
	int err = -EINVAL;
393 394

	if (!cpufreq_driver)
395 396
		goto out;

397
	if (cpufreq_driver->setpolicy) {
L
Linus Torvalds 已提交
398 399
		if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
			*policy = CPUFREQ_POLICY_PERFORMANCE;
400
			err = 0;
401 402
		} else if (!strnicmp(str_governor, "powersave",
						CPUFREQ_NAME_LEN)) {
L
Linus Torvalds 已提交
403
			*policy = CPUFREQ_POLICY_POWERSAVE;
404
			err = 0;
L
Linus Torvalds 已提交
405
		}
406
	} else if (cpufreq_driver->target) {
L
Linus Torvalds 已提交
407
		struct cpufreq_governor *t;
408

409
		mutex_lock(&cpufreq_governor_mutex);
410 411 412

		t = __find_governor(str_governor);

413
		if (t == NULL) {
414
			int ret;
415

416 417 418
			mutex_unlock(&cpufreq_governor_mutex);
			ret = request_module("cpufreq_%s", str_governor);
			mutex_lock(&cpufreq_governor_mutex);
419

420 421
			if (ret == 0)
				t = __find_governor(str_governor);
422 423
		}

424 425 426
		if (t != NULL) {
			*governor = t;
			err = 0;
L
Linus Torvalds 已提交
427
		}
428

429
		mutex_unlock(&cpufreq_governor_mutex);
L
Linus Torvalds 已提交
430
	}
431
out:
432
	return err;
L
Linus Torvalds 已提交
433 434 435
}

/**
436 437
 * cpufreq_per_cpu_attr_read() / show_##file_name() -
 * print out cpufreq information
L
Linus Torvalds 已提交
438 439 440 441 442
 *
 * Write out information from cpufreq_driver->policy[cpu]; object must be
 * "unsigned int".
 */

443 444
#define show_one(file_name, object)			\
static ssize_t show_##file_name				\
D
Dave Jones 已提交
445
(struct cpufreq_policy *policy, char *buf)		\
446
{							\
447
	return sprintf(buf, "%u\n", policy->object);	\
L
Linus Torvalds 已提交
448 449 450 451
}

show_one(cpuinfo_min_freq, cpuinfo.min_freq);
show_one(cpuinfo_max_freq, cpuinfo.max_freq);
452
show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
L
Linus Torvalds 已提交
453 454 455 456
show_one(scaling_min_freq, min);
show_one(scaling_max_freq, max);
show_one(scaling_cur_freq, cur);

457 458
static int __cpufreq_set_policy(struct cpufreq_policy *policy,
				struct cpufreq_policy *new_policy);
459

L
Linus Torvalds 已提交
460 461 462 463 464
/**
 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 */
#define store_one(file_name, object)			\
static ssize_t store_##file_name					\
D
Dave Jones 已提交
465
(struct cpufreq_policy *policy, const char *buf, size_t count)		\
L
Linus Torvalds 已提交
466
{									\
467
	unsigned int ret;						\
L
Linus Torvalds 已提交
468 469 470 471 472 473
	struct cpufreq_policy new_policy;				\
									\
	ret = cpufreq_get_policy(&new_policy, policy->cpu);		\
	if (ret)							\
		return -EINVAL;						\
									\
474
	ret = sscanf(buf, "%u", &new_policy.object);			\
L
Linus Torvalds 已提交
475 476 477
	if (ret != 1)							\
		return -EINVAL;						\
									\
478 479
	ret = __cpufreq_set_policy(policy, &new_policy);		\
	policy->user_policy.object = policy->object;			\
L
Linus Torvalds 已提交
480 481 482 483
									\
	return ret ? ret : count;					\
}

484 485
store_one(scaling_min_freq, min);
store_one(scaling_max_freq, max);
L
Linus Torvalds 已提交
486 487 488 489

/**
 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 */
D
Dave Jones 已提交
490 491
static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
					char *buf)
L
Linus Torvalds 已提交
492
{
493
	unsigned int cur_freq = __cpufreq_get(policy->cpu);
L
Linus Torvalds 已提交
494 495 496 497 498 499 500 501
	if (!cur_freq)
		return sprintf(buf, "<unknown>");
	return sprintf(buf, "%u\n", cur_freq);
}

/**
 * show_scaling_governor - show the current policy for the specified CPU
 */
D
Dave Jones 已提交
502
static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
L
Linus Torvalds 已提交
503
{
504
	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
L
Linus Torvalds 已提交
505 506 507 508
		return sprintf(buf, "powersave\n");
	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
		return sprintf(buf, "performance\n");
	else if (policy->governor)
509
		return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
510
				policy->governor->name);
L
Linus Torvalds 已提交
511 512 513 514 515 516
	return -EINVAL;
}

/**
 * store_scaling_governor - store policy for the specified CPU
 */
D
Dave Jones 已提交
517 518
static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
					const char *buf, size_t count)
L
Linus Torvalds 已提交
519
{
520
	unsigned int ret;
L
Linus Torvalds 已提交
521 522 523 524 525 526 527
	char	str_governor[16];
	struct cpufreq_policy new_policy;

	ret = cpufreq_get_policy(&new_policy, policy->cpu);
	if (ret)
		return ret;

528
	ret = sscanf(buf, "%15s", str_governor);
L
Linus Torvalds 已提交
529 530 531
	if (ret != 1)
		return -EINVAL;

532 533
	if (cpufreq_parse_governor(str_governor, &new_policy.policy,
						&new_policy.governor))
L
Linus Torvalds 已提交
534 535
		return -EINVAL;

536 537 538 539
	/*
	 * Do not use cpufreq_set_policy here or the user_policy.max
	 * will be wrongly overridden
	 */
540 541 542 543 544
	ret = __cpufreq_set_policy(policy, &new_policy);

	policy->user_policy.policy = policy->policy;
	policy->user_policy.governor = policy->governor;

545 546 547 548
	if (ret)
		return ret;
	else
		return count;
L
Linus Torvalds 已提交
549 550 551 552 553
}

/**
 * show_scaling_driver - show the cpufreq driver currently loaded
 */
D
Dave Jones 已提交
554
static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
L
Linus Torvalds 已提交
555
{
556
	return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
L
Linus Torvalds 已提交
557 558 559 560 561
}

/**
 * show_scaling_available_governors - show the available CPUfreq governors
 */
D
Dave Jones 已提交
562 563
static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
						char *buf)
L
Linus Torvalds 已提交
564 565 566 567
{
	ssize_t i = 0;
	struct cpufreq_governor *t;

568
	if (!cpufreq_driver->target) {
L
Linus Torvalds 已提交
569 570 571 572 573
		i += sprintf(buf, "performance powersave");
		goto out;
	}

	list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
574 575
		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
		    - (CPUFREQ_NAME_LEN + 2)))
L
Linus Torvalds 已提交
576
			goto out;
577
		i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
L
Linus Torvalds 已提交
578
	}
579
out:
L
Linus Torvalds 已提交
580 581 582
	i += sprintf(&buf[i], "\n");
	return i;
}
583

584
ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
L
Linus Torvalds 已提交
585 586 587 588
{
	ssize_t i = 0;
	unsigned int cpu;

589
	for_each_cpu(cpu, mask) {
L
Linus Torvalds 已提交
590 591 592 593
		if (i)
			i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
		if (i >= (PAGE_SIZE - 5))
594
			break;
L
Linus Torvalds 已提交
595 596 597 598
	}
	i += sprintf(&buf[i], "\n");
	return i;
}
599
EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
L
Linus Torvalds 已提交
600

601 602 603 604 605 606
/**
 * show_related_cpus - show the CPUs affected by each transition even if
 * hw coordination is in use
 */
static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
{
607
	return cpufreq_show_cpus(policy->related_cpus, buf);
608 609 610 611 612 613 614
}

/**
 * show_affected_cpus - show the CPUs affected by each transition
 */
static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
{
615
	return cpufreq_show_cpus(policy->cpus, buf);
616 617
}

618
static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
D
Dave Jones 已提交
619
					const char *buf, size_t count)
620 621 622 623
{
	unsigned int freq = 0;
	unsigned int ret;

624
	if (!policy->governor || !policy->governor->store_setspeed)
625 626 627 628 629 630 631 632 633 634 635 636 637
		return -EINVAL;

	ret = sscanf(buf, "%u", &freq);
	if (ret != 1)
		return -EINVAL;

	policy->governor->store_setspeed(policy, freq);

	return count;
}

static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
{
638
	if (!policy->governor || !policy->governor->show_setspeed)
639 640 641 642
		return sprintf(buf, "<unsupported>\n");

	return policy->governor->show_setspeed(policy, buf);
}
L
Linus Torvalds 已提交
643

644
/**
645
 * show_bios_limit - show the current cpufreq HW/BIOS limitation
646 647 648 649 650
 */
static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
{
	unsigned int limit;
	int ret;
651 652
	if (cpufreq_driver->bios_limit) {
		ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
653 654 655 656 657 658
		if (!ret)
			return sprintf(buf, "%u\n", limit);
	}
	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
}

659 660 661 662 663 664 665 666 667 668 669 670 671 672
cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
cpufreq_freq_attr_ro(cpuinfo_min_freq);
cpufreq_freq_attr_ro(cpuinfo_max_freq);
cpufreq_freq_attr_ro(cpuinfo_transition_latency);
cpufreq_freq_attr_ro(scaling_available_governors);
cpufreq_freq_attr_ro(scaling_driver);
cpufreq_freq_attr_ro(scaling_cur_freq);
cpufreq_freq_attr_ro(bios_limit);
cpufreq_freq_attr_ro(related_cpus);
cpufreq_freq_attr_ro(affected_cpus);
cpufreq_freq_attr_rw(scaling_min_freq);
cpufreq_freq_attr_rw(scaling_max_freq);
cpufreq_freq_attr_rw(scaling_governor);
cpufreq_freq_attr_rw(scaling_setspeed);
L
Linus Torvalds 已提交
673

D
Dave Jones 已提交
674
static struct attribute *default_attrs[] = {
L
Linus Torvalds 已提交
675 676
	&cpuinfo_min_freq.attr,
	&cpuinfo_max_freq.attr,
677
	&cpuinfo_transition_latency.attr,
L
Linus Torvalds 已提交
678 679 680
	&scaling_min_freq.attr,
	&scaling_max_freq.attr,
	&affected_cpus.attr,
681
	&related_cpus.attr,
L
Linus Torvalds 已提交
682 683 684
	&scaling_governor.attr,
	&scaling_driver.attr,
	&scaling_available_governors.attr,
685
	&scaling_setspeed.attr,
L
Linus Torvalds 已提交
686 687 688
	NULL
};

689 690
#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
#define to_attr(a) container_of(a, struct freq_attr, attr)
L
Linus Torvalds 已提交
691

692
static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
L
Linus Torvalds 已提交
693
{
D
Dave Jones 已提交
694 695
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
696
	ssize_t ret = -EINVAL;
697
	policy = cpufreq_cpu_get_sysfs(policy->cpu);
L
Linus Torvalds 已提交
698
	if (!policy)
699
		goto no_policy;
700 701

	if (lock_policy_rwsem_read(policy->cpu) < 0)
702
		goto fail;
703

704 705 706 707 708
	if (fattr->show)
		ret = fattr->show(policy, buf);
	else
		ret = -EIO;

709
	unlock_policy_rwsem_read(policy->cpu);
710
fail:
711
	cpufreq_cpu_put_sysfs(policy);
712
no_policy:
L
Linus Torvalds 已提交
713 714 715
	return ret;
}

D
Dave Jones 已提交
716 717
static ssize_t store(struct kobject *kobj, struct attribute *attr,
		     const char *buf, size_t count)
L
Linus Torvalds 已提交
718
{
D
Dave Jones 已提交
719 720
	struct cpufreq_policy *policy = to_policy(kobj);
	struct freq_attr *fattr = to_attr(attr);
721
	ssize_t ret = -EINVAL;
722
	policy = cpufreq_cpu_get_sysfs(policy->cpu);
L
Linus Torvalds 已提交
723
	if (!policy)
724
		goto no_policy;
725 726

	if (lock_policy_rwsem_write(policy->cpu) < 0)
727
		goto fail;
728

729 730 731 732 733
	if (fattr->store)
		ret = fattr->store(policy, buf, count);
	else
		ret = -EIO;

734
	unlock_policy_rwsem_write(policy->cpu);
735
fail:
736
	cpufreq_cpu_put_sysfs(policy);
737
no_policy:
L
Linus Torvalds 已提交
738 739 740
	return ret;
}

D
Dave Jones 已提交
741
static void cpufreq_sysfs_release(struct kobject *kobj)
L
Linus Torvalds 已提交
742
{
D
Dave Jones 已提交
743
	struct cpufreq_policy *policy = to_policy(kobj);
744
	pr_debug("last reference is dropped\n");
L
Linus Torvalds 已提交
745 746 747
	complete(&policy->kobj_unregister);
}

748
static const struct sysfs_ops sysfs_ops = {
L
Linus Torvalds 已提交
749 750 751 752 753 754 755 756 757 758
	.show	= show,
	.store	= store,
};

static struct kobj_type ktype_cpufreq = {
	.sysfs_ops	= &sysfs_ops,
	.default_attrs	= default_attrs,
	.release	= cpufreq_sysfs_release,
};

759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
struct kobject *cpufreq_global_kobject;
EXPORT_SYMBOL(cpufreq_global_kobject);

static int cpufreq_global_kobject_usage;

int cpufreq_get_global_kobject(void)
{
	if (!cpufreq_global_kobject_usage++)
		return kobject_add(cpufreq_global_kobject,
				&cpu_subsys.dev_root->kobj, "%s", "cpufreq");

	return 0;
}
EXPORT_SYMBOL(cpufreq_get_global_kobject);

void cpufreq_put_global_kobject(void)
{
	if (!--cpufreq_global_kobject_usage)
		kobject_del(cpufreq_global_kobject);
}
EXPORT_SYMBOL(cpufreq_put_global_kobject);

int cpufreq_sysfs_create_file(const struct attribute *attr)
{
	int ret = cpufreq_get_global_kobject();

	if (!ret) {
		ret = sysfs_create_file(cpufreq_global_kobject, attr);
		if (ret)
			cpufreq_put_global_kobject();
	}

	return ret;
}
EXPORT_SYMBOL(cpufreq_sysfs_create_file);

void cpufreq_sysfs_remove_file(const struct attribute *attr)
{
	sysfs_remove_file(cpufreq_global_kobject, attr);
	cpufreq_put_global_kobject();
}
EXPORT_SYMBOL(cpufreq_sysfs_remove_file);

802
/* symlink affected CPUs */
803
static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
804 805 806 807 808
{
	unsigned int j;
	int ret = 0;

	for_each_cpu(j, policy->cpus) {
809
		struct device *cpu_dev;
810

811
		if (j == policy->cpu)
812 813
			continue;

814
		pr_debug("Adding link for CPU: %u\n", j);
815 816
		cpu_dev = get_cpu_device(j);
		ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
817
					"cpufreq");
818 819
		if (ret)
			break;
820 821 822 823
	}
	return ret;
}

824
static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
825
				     struct device *dev)
826 827 828 829 830 831
{
	struct freq_attr **drv_attr;
	int ret = 0;

	/* prepare interface data */
	ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
832
				   &dev->kobj, "cpufreq");
833 834 835 836
	if (ret)
		return ret;

	/* set up files for this cpu device */
837
	drv_attr = cpufreq_driver->attr;
838 839 840
	while ((drv_attr) && (*drv_attr)) {
		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
		if (ret)
841
			goto err_out_kobj_put;
842 843
		drv_attr++;
	}
844
	if (cpufreq_driver->get) {
845 846
		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
		if (ret)
847
			goto err_out_kobj_put;
848
	}
849
	if (cpufreq_driver->target) {
850 851
		ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
		if (ret)
852
			goto err_out_kobj_put;
853
	}
854
	if (cpufreq_driver->bios_limit) {
855 856
		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
		if (ret)
857
			goto err_out_kobj_put;
858
	}
859

860
	ret = cpufreq_add_dev_symlink(policy);
861 862 863
	if (ret)
		goto err_out_kobj_put;

864 865 866 867 868 869 870 871 872 873 874 875 876
	return ret;

err_out_kobj_put:
	kobject_put(&policy->kobj);
	wait_for_completion(&policy->kobj_unregister);
	return ret;
}

static void cpufreq_init_policy(struct cpufreq_policy *policy)
{
	struct cpufreq_policy new_policy;
	int ret = 0;

877
	memcpy(&new_policy, policy, sizeof(*policy));
878 879 880 881 882 883 884 885 886
	/* assure that the starting sequence is run in __cpufreq_set_policy */
	policy->governor = NULL;

	/* set default policy */
	ret = __cpufreq_set_policy(policy, &new_policy);
	policy->user_policy.policy = policy->policy;
	policy->user_policy.governor = policy->governor;

	if (ret) {
887
		pr_debug("setting policy failed\n");
888 889
		if (cpufreq_driver->exit)
			cpufreq_driver->exit(policy);
890
	}
891 892
}

893
#ifdef CONFIG_HOTPLUG_CPU
894 895 896
static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
				  unsigned int cpu, struct device *dev,
				  bool frozen)
897
{
898
	int ret = 0, has_target = !!cpufreq_driver->target;
899 900
	unsigned long flags;

901 902
	if (has_target)
		__cpufreq_governor(policy, CPUFREQ_GOV_STOP);
903

904
	lock_policy_rwsem_write(policy->cpu);
V
Viresh Kumar 已提交
905

906
	write_lock_irqsave(&cpufreq_driver_lock, flags);
V
Viresh Kumar 已提交
907

908
	cpumask_set_cpu(cpu, policy->cpus);
V
Viresh Kumar 已提交
909
	per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
910
	per_cpu(cpufreq_cpu_data, cpu) = policy;
911
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
912

913
	unlock_policy_rwsem_write(policy->cpu);
V
Viresh Kumar 已提交
914

915 916 917 918
	if (has_target) {
		__cpufreq_governor(policy, CPUFREQ_GOV_START);
		__cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
	}
919

920
	/* Don't touch sysfs links during light-weight init */
921 922
	if (!frozen)
		ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
923 924

	return ret;
925 926
}
#endif
L
Linus Torvalds 已提交
927

928 929 930 931 932 933 934 935 936 937 938 939 940 941
static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
{
	struct cpufreq_policy *policy;
	unsigned long flags;

	write_lock_irqsave(&cpufreq_driver_lock, flags);

	policy = per_cpu(cpufreq_cpu_data_fallback, cpu);

	write_unlock_irqrestore(&cpufreq_driver_lock, flags);

	return policy;
}

942 943 944 945 946 947 948 949 950 951 952 953 954 955
static struct cpufreq_policy *cpufreq_policy_alloc(void)
{
	struct cpufreq_policy *policy;

	policy = kzalloc(sizeof(*policy), GFP_KERNEL);
	if (!policy)
		return NULL;

	if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
		goto err_free_policy;

	if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
		goto err_free_cpumask;

956
	INIT_LIST_HEAD(&policy->policy_list);
957 958 959 960 961 962 963 964 965 966 967 968
	return policy;

err_free_cpumask:
	free_cpumask_var(policy->cpus);
err_free_policy:
	kfree(policy);

	return NULL;
}

static void cpufreq_policy_free(struct cpufreq_policy *policy)
{
969 970 971 972 973 974
	unsigned long flags;

	write_lock_irqsave(&cpufreq_driver_lock, flags);
	list_del(&policy->policy_list);
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);

975 976 977 978 979
	free_cpumask_var(policy->related_cpus);
	free_cpumask_var(policy->cpus);
	kfree(policy);
}

980 981
static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
			     bool frozen)
L
Linus Torvalds 已提交
982
{
983
	unsigned int j, cpu = dev->id;
984
	int ret = -ENOMEM;
L
Linus Torvalds 已提交
985 986
	struct cpufreq_policy *policy;
	unsigned long flags;
987
#ifdef CONFIG_HOTPLUG_CPU
988
	struct cpufreq_policy *tpolicy;
989
	struct cpufreq_governor *gov;
990
#endif
L
Linus Torvalds 已提交
991

992 993 994
	if (cpu_is_offline(cpu))
		return 0;

995
	pr_debug("adding CPU %u\n", cpu);
L
Linus Torvalds 已提交
996 997 998 999 1000 1001

#ifdef CONFIG_SMP
	/* check whether a different CPU already registered this
	 * CPU because it is in the same boat. */
	policy = cpufreq_cpu_get(cpu);
	if (unlikely(policy)) {
1002
		cpufreq_cpu_put(policy);
L
Linus Torvalds 已提交
1003 1004
		return 0;
	}
1005 1006 1007

#ifdef CONFIG_HOTPLUG_CPU
	/* Check if this cpu was hot-unplugged earlier and has siblings */
1008
	read_lock_irqsave(&cpufreq_driver_lock, flags);
1009 1010
	list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
		if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1011
			read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1012 1013
			return cpufreq_add_policy_cpu(tpolicy, cpu, dev,
					frozen);
V
Viresh Kumar 已提交
1014
		}
1015
	}
1016
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1017
#endif
L
Linus Torvalds 已提交
1018 1019
#endif

1020
	if (!try_module_get(cpufreq_driver->owner)) {
L
Linus Torvalds 已提交
1021 1022 1023 1024
		ret = -EINVAL;
		goto module_out;
	}

1025 1026 1027 1028 1029 1030
	if (frozen)
		/* Restore the saved policy when doing light-weight init */
		policy = cpufreq_policy_restore(cpu);
	else
		policy = cpufreq_policy_alloc();

1031
	if (!policy)
L
Linus Torvalds 已提交
1032
		goto nomem_out;
1033

L
Linus Torvalds 已提交
1034
	policy->cpu = cpu;
1035
	policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1036
	cpumask_copy(policy->cpus, cpumask_of(cpu));
L
Linus Torvalds 已提交
1037

1038
	/* Initially set CPU itself as the policy_cpu */
1039
	per_cpu(cpufreq_policy_cpu, cpu) = cpu;
1040

L
Linus Torvalds 已提交
1041
	init_completion(&policy->kobj_unregister);
1042
	INIT_WORK(&policy->update, handle_update);
L
Linus Torvalds 已提交
1043 1044 1045 1046

	/* call driver. From then on the cpufreq must be able
	 * to accept all calls to ->verify and ->setpolicy for this CPU
	 */
1047
	ret = cpufreq_driver->init(policy);
L
Linus Torvalds 已提交
1048
	if (ret) {
1049
		pr_debug("initialization failed\n");
V
Viresh Kumar 已提交
1050
		goto err_set_policy_cpu;
L
Linus Torvalds 已提交
1051
	}
V
Viresh Kumar 已提交
1052

1053 1054 1055
	/* related cpus should atleast have policy->cpus */
	cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);

V
Viresh Kumar 已提交
1056 1057 1058 1059 1060 1061
	/*
	 * affected cpus must always be the one, which are online. We aren't
	 * managing offline cpus here.
	 */
	cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);

1062 1063
	policy->user_policy.min = policy->min;
	policy->user_policy.max = policy->max;
L
Linus Torvalds 已提交
1064

1065 1066 1067
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
				     CPUFREQ_START, policy);

1068 1069 1070 1071 1072 1073
#ifdef CONFIG_HOTPLUG_CPU
	gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
	if (gov) {
		policy->governor = gov;
		pr_debug("Restoring governor %s for cpu %d\n",
		       policy->governor->name, cpu);
1074
	}
1075
#endif
L
Linus Torvalds 已提交
1076

1077 1078 1079 1080 1081 1082 1083
	write_lock_irqsave(&cpufreq_driver_lock, flags);
	for_each_cpu(j, policy->cpus) {
		per_cpu(cpufreq_cpu_data, j) = policy;
		per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
	}
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);

1084
	if (!frozen) {
1085
		ret = cpufreq_add_dev_interface(policy, dev);
1086 1087
		if (ret)
			goto err_out_unregister;
1088 1089 1090 1091

		write_lock_irqsave(&cpufreq_driver_lock, flags);
		list_add(&policy->policy_list, &cpufreq_policy_list);
		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1092
	}
1093

1094 1095
	cpufreq_init_policy(policy);

1096
	kobject_uevent(&policy->kobj, KOBJ_ADD);
1097
	module_put(cpufreq_driver->owner);
1098
	pr_debug("initialization complete\n");
1099

L
Linus Torvalds 已提交
1100 1101 1102
	return 0;

err_out_unregister:
1103
	write_lock_irqsave(&cpufreq_driver_lock, flags);
1104
	for_each_cpu(j, policy->cpus) {
1105
		per_cpu(cpufreq_cpu_data, j) = NULL;
1106 1107 1108
		if (j != cpu)
			per_cpu(cpufreq_policy_cpu, j) = -1;
	}
1109
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
L
Linus Torvalds 已提交
1110

V
Viresh Kumar 已提交
1111 1112
err_set_policy_cpu:
	per_cpu(cpufreq_policy_cpu, cpu) = -1;
1113
	cpufreq_policy_free(policy);
L
Linus Torvalds 已提交
1114
nomem_out:
1115
	module_put(cpufreq_driver->owner);
1116
module_out:
L
Linus Torvalds 已提交
1117 1118 1119
	return ret;
}

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
/**
 * cpufreq_add_dev - add a CPU device
 *
 * Adds the cpufreq interface for a CPU device.
 *
 * The Oracle says: try running cpufreq registration/unregistration concurrently
 * with with cpu hotplugging and all hell will break loose. Tried to clean this
 * mess up, but more thorough testing is needed. - Mathieu
 */
static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
{
	return __cpufreq_add_dev(dev, sif, false);
}

1134 1135 1136 1137 1138 1139 1140
static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
{
	int j;

	policy->last_cpu = policy->cpu;
	policy->cpu = cpu;

1141
	for_each_cpu(j, policy->cpus)
1142 1143 1144 1145 1146 1147 1148 1149
		per_cpu(cpufreq_policy_cpu, j) = cpu;

#ifdef CONFIG_CPU_FREQ_TABLE
	cpufreq_frequency_table_update_policy_cpu(policy);
#endif
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
			CPUFREQ_UPDATE_POLICY_CPU, policy);
}
L
Linus Torvalds 已提交
1150

1151
static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1152
					   unsigned int old_cpu, bool frozen)
1153 1154 1155 1156 1157 1158
{
	struct device *cpu_dev;
	unsigned long flags;
	int ret;

	/* first sibling now owns the new sysfs dir */
1159
	cpu_dev = get_cpu_device(cpumask_first(policy->cpus));
1160 1161 1162 1163 1164

	/* Don't touch sysfs files during light-weight tear-down */
	if (frozen)
		return cpu_dev->id;

1165
	sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1166
	ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1167 1168 1169 1170
	if (ret) {
		pr_err("%s: Failed to move kobj: %d", __func__, ret);

		WARN_ON(lock_policy_rwsem_write(old_cpu));
1171
		cpumask_set_cpu(old_cpu, policy->cpus);
1172 1173

		write_lock_irqsave(&cpufreq_driver_lock, flags);
1174
		per_cpu(cpufreq_cpu_data, old_cpu) = policy;
1175 1176 1177 1178
		write_unlock_irqrestore(&cpufreq_driver_lock, flags);

		unlock_policy_rwsem_write(old_cpu);

1179
		ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1180 1181 1182 1183 1184 1185 1186 1187
					"cpufreq");

		return -EINVAL;
	}

	return cpu_dev->id;
}

L
Linus Torvalds 已提交
1188
/**
1189
 * __cpufreq_remove_dev - remove a CPU device
L
Linus Torvalds 已提交
1190 1191
 *
 * Removes the cpufreq interface for a CPU device.
1192 1193
 * Caller should already have policy_rwsem in write mode for this CPU.
 * This routine frees the rwsem before returning.
L
Linus Torvalds 已提交
1194
 */
1195
static int __cpufreq_remove_dev(struct device *dev,
1196
				struct subsys_interface *sif, bool frozen)
L
Linus Torvalds 已提交
1197
{
1198 1199
	unsigned int cpu = dev->id, cpus;
	int new_cpu;
L
Linus Torvalds 已提交
1200
	unsigned long flags;
1201
	struct cpufreq_policy *policy;
A
Amerigo Wang 已提交
1202 1203
	struct kobject *kobj;
	struct completion *cmp;
L
Linus Torvalds 已提交
1204

1205
	pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
L
Linus Torvalds 已提交
1206

1207
	write_lock_irqsave(&cpufreq_driver_lock, flags);
V
Viresh Kumar 已提交
1208

1209
	policy = per_cpu(cpufreq_cpu_data, cpu);
V
Viresh Kumar 已提交
1210 1211
	per_cpu(cpufreq_cpu_data, cpu) = NULL;

1212 1213
	/* Save the policy somewhere when doing a light-weight tear-down */
	if (frozen)
1214
		per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1215

1216
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
L
Linus Torvalds 已提交
1217

1218
	if (!policy) {
1219
		pr_debug("%s: No cpu_data found\n", __func__);
L
Linus Torvalds 已提交
1220 1221 1222
		return -EINVAL;
	}

1223
	if (cpufreq_driver->target)
1224
		__cpufreq_governor(policy, CPUFREQ_GOV_STOP);
L
Linus Torvalds 已提交
1225

1226
#ifdef CONFIG_HOTPLUG_CPU
1227
	if (!cpufreq_driver->setpolicy)
1228
		strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1229
			policy->governor->name, CPUFREQ_NAME_LEN);
L
Linus Torvalds 已提交
1230 1231
#endif

V
Viresh Kumar 已提交
1232
	WARN_ON(lock_policy_rwsem_write(cpu));
1233
	cpus = cpumask_weight(policy->cpus);
1234 1235

	if (cpus > 1)
1236
		cpumask_clear_cpu(cpu, policy->cpus);
V
Viresh Kumar 已提交
1237
	unlock_policy_rwsem_write(cpu);
1238

1239
	if (cpu != policy->cpu && !frozen) {
1240 1241
		sysfs_remove_link(&dev->kobj, "cpufreq");
	} else if (cpus > 1) {
1242

1243
		new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu, frozen);
1244
		if (new_cpu >= 0) {
V
Viresh Kumar 已提交
1245
			WARN_ON(lock_policy_rwsem_write(cpu));
1246
			update_policy_cpu(policy, new_cpu);
A
Amerigo Wang 已提交
1247
			unlock_policy_rwsem_write(cpu);
1248 1249 1250 1251 1252

			if (!frozen) {
				pr_debug("%s: policy Kobject moved to cpu: %d "
					 "from: %d\n",__func__, new_cpu, cpu);
			}
L
Linus Torvalds 已提交
1253 1254 1255
		}
	}

1256 1257
	/* If cpu is last user of policy, free policy */
	if (cpus == 1) {
1258
		if (cpufreq_driver->target)
1259
			__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1260

1261 1262
		if (!frozen) {
			lock_policy_rwsem_read(cpu);
1263 1264
			kobj = &policy->kobj;
			cmp = &policy->kobj_unregister;
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276
			unlock_policy_rwsem_read(cpu);
			kobject_put(kobj);

			/*
			 * We need to make sure that the underlying kobj is
			 * actually not referenced anymore by anybody before we
			 * proceed with unloading.
			 */
			pr_debug("waiting for dropping of refcount\n");
			wait_for_completion(cmp);
			pr_debug("wait complete\n");
		}
1277

1278 1279 1280 1281
		/*
		 * Perform the ->exit() even during light-weight tear-down,
		 * since this is a core component, and is essential for the
		 * subsequent light-weight ->init() to succeed.
1282
		 */
1283
		if (cpufreq_driver->exit)
1284
			cpufreq_driver->exit(policy);
1285

1286
		if (!frozen)
1287
			cpufreq_policy_free(policy);
1288 1289
	} else {
		if (cpufreq_driver->target) {
1290 1291
			__cpufreq_governor(policy, CPUFREQ_GOV_START);
			__cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1292
		}
1293
	}
L
Linus Torvalds 已提交
1294

V
Viresh Kumar 已提交
1295
	per_cpu(cpufreq_policy_cpu, cpu) = -1;
L
Linus Torvalds 已提交
1296 1297 1298
	return 0;
}

1299
static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1300
{
1301
	unsigned int cpu = dev->id;
1302
	int retval;
1303 1304 1305 1306

	if (cpu_is_offline(cpu))
		return 0;

1307
	retval = __cpufreq_remove_dev(dev, sif, false);
1308 1309 1310
	return retval;
}

1311
static void handle_update(struct work_struct *work)
L
Linus Torvalds 已提交
1312
{
1313 1314 1315
	struct cpufreq_policy *policy =
		container_of(work, struct cpufreq_policy, update);
	unsigned int cpu = policy->cpu;
1316
	pr_debug("handle_update for cpu %u called\n", cpu);
L
Linus Torvalds 已提交
1317 1318 1319 1320
	cpufreq_update_policy(cpu);
}

/**
1321 1322
 *	cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
 *	in deep trouble.
L
Linus Torvalds 已提交
1323 1324 1325 1326
 *	@cpu: cpu number
 *	@old_freq: CPU frequency the kernel thinks the CPU runs at
 *	@new_freq: CPU frequency the CPU actually runs at
 *
1327 1328
 *	We adjust to current frequency first, and need to clean up later.
 *	So either call to cpufreq_update_policy() or schedule handle_update()).
L
Linus Torvalds 已提交
1329
 */
1330 1331
static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
				unsigned int new_freq)
L
Linus Torvalds 已提交
1332
{
1333
	struct cpufreq_policy *policy;
L
Linus Torvalds 已提交
1334
	struct cpufreq_freqs freqs;
1335 1336
	unsigned long flags;

1337
	pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
L
Linus Torvalds 已提交
1338 1339 1340 1341
	       "core thinks of %u, is %u kHz.\n", old_freq, new_freq);

	freqs.old = old_freq;
	freqs.new = new_freq;
1342 1343 1344 1345 1346 1347 1348

	read_lock_irqsave(&cpufreq_driver_lock, flags);
	policy = per_cpu(cpufreq_cpu_data, cpu);
	read_unlock_irqrestore(&cpufreq_driver_lock, flags);

	cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
	cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
L
Linus Torvalds 已提交
1349 1350
}

1351
/**
D
Dhaval Giani 已提交
1352
 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1353 1354 1355 1356 1357 1358 1359
 * @cpu: CPU number
 *
 * This is the last known freq, without actually getting it from the driver.
 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
 */
unsigned int cpufreq_quick_get(unsigned int cpu)
{
1360
	struct cpufreq_policy *policy;
1361
	unsigned int ret_freq = 0;
1362

1363 1364
	if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
		return cpufreq_driver->get(cpu);
1365 1366

	policy = cpufreq_cpu_get(cpu);
1367
	if (policy) {
1368
		ret_freq = policy->cur;
1369 1370 1371
		cpufreq_cpu_put(policy);
	}

D
Dave Jones 已提交
1372
	return ret_freq;
1373 1374 1375
}
EXPORT_SYMBOL(cpufreq_quick_get);

1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
/**
 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
 * @cpu: CPU number
 *
 * Just return the max possible frequency for a given CPU.
 */
unsigned int cpufreq_quick_get_max(unsigned int cpu)
{
	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
	unsigned int ret_freq = 0;

	if (policy) {
		ret_freq = policy->max;
		cpufreq_cpu_put(policy);
	}

	return ret_freq;
}
EXPORT_SYMBOL(cpufreq_quick_get_max);

1396
static unsigned int __cpufreq_get(unsigned int cpu)
L
Linus Torvalds 已提交
1397
{
1398
	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1399
	unsigned int ret_freq = 0;
1400

1401
	if (!cpufreq_driver->get)
D
Dave Jones 已提交
1402
		return ret_freq;
L
Linus Torvalds 已提交
1403

1404
	ret_freq = cpufreq_driver->get(cpu);
L
Linus Torvalds 已提交
1405

1406
	if (ret_freq && policy->cur &&
1407
		!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1408 1409 1410 1411
		/* verify no discrepancy between actual and
					saved value exists */
		if (unlikely(ret_freq != policy->cur)) {
			cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
L
Linus Torvalds 已提交
1412 1413 1414 1415
			schedule_work(&policy->update);
		}
	}

D
Dave Jones 已提交
1416
	return ret_freq;
1417
}
L
Linus Torvalds 已提交
1418

1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
/**
 * cpufreq_get - get the current CPU frequency (in kHz)
 * @cpu: CPU number
 *
 * Get the CPU current (static) CPU frequency
 */
unsigned int cpufreq_get(unsigned int cpu)
{
	unsigned int ret_freq = 0;
	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);

	if (!policy)
		goto out;

	if (unlikely(lock_policy_rwsem_read(cpu)))
		goto out_policy;

	ret_freq = __cpufreq_get(cpu);

	unlock_policy_rwsem_read(cpu);
L
Linus Torvalds 已提交
1439

1440 1441 1442
out_policy:
	cpufreq_cpu_put(policy);
out:
D
Dave Jones 已提交
1443
	return ret_freq;
L
Linus Torvalds 已提交
1444 1445 1446
}
EXPORT_SYMBOL(cpufreq_get);

1447 1448 1449 1450 1451
static struct subsys_interface cpufreq_interface = {
	.name		= "cpufreq",
	.subsys		= &cpu_subsys,
	.add_dev	= cpufreq_add_dev,
	.remove_dev	= cpufreq_remove_dev,
1452 1453
};

1454
/**
1455 1456 1457 1458
 * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
 *
 * This function is only executed for the boot processor.  The other CPUs
 * have been put offline by means of CPU hotplug.
1459
 */
1460
static int cpufreq_bp_suspend(void)
1461
{
1462
	int ret = 0;
1463

1464
	int cpu = smp_processor_id();
1465
	struct cpufreq_policy *policy;
1466

1467
	pr_debug("suspending cpu %u\n", cpu);
1468

1469
	/* If there's no policy for the boot CPU, we have nothing to do. */
1470 1471
	policy = cpufreq_cpu_get(cpu);
	if (!policy)
1472
		return 0;
1473

1474
	if (cpufreq_driver->suspend) {
1475
		ret = cpufreq_driver->suspend(policy);
1476
		if (ret)
1477
			printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1478
					"step on CPU %u\n", policy->cpu);
1479 1480
	}

1481
	cpufreq_cpu_put(policy);
1482
	return ret;
1483 1484
}

L
Linus Torvalds 已提交
1485
/**
1486
 * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
L
Linus Torvalds 已提交
1487 1488
 *
 *	1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1489 1490 1491 1492 1493
 *	2.) schedule call cpufreq_update_policy() ASAP as interrupts are
 *	    restored. It will verify that the current freq is in sync with
 *	    what we believe it to be. This is a bit later than when it
 *	    should be, but nonethteless it's better than calling
 *	    cpufreq_driver->get() here which might re-enable interrupts...
1494 1495 1496
 *
 * This function is only executed for the boot CPU.  The other CPUs have not
 * been turned on yet.
L
Linus Torvalds 已提交
1497
 */
1498
static void cpufreq_bp_resume(void)
L
Linus Torvalds 已提交
1499
{
1500
	int ret = 0;
1501

1502
	int cpu = smp_processor_id();
1503
	struct cpufreq_policy *policy;
L
Linus Torvalds 已提交
1504

1505
	pr_debug("resuming cpu %u\n", cpu);
L
Linus Torvalds 已提交
1506

1507
	/* If there's no policy for the boot CPU, we have nothing to do. */
1508 1509
	policy = cpufreq_cpu_get(cpu);
	if (!policy)
1510
		return;
L
Linus Torvalds 已提交
1511

1512
	if (cpufreq_driver->resume) {
1513
		ret = cpufreq_driver->resume(policy);
L
Linus Torvalds 已提交
1514 1515
		if (ret) {
			printk(KERN_ERR "cpufreq: resume failed in ->resume "
1516
					"step on CPU %u\n", policy->cpu);
1517
			goto fail;
L
Linus Torvalds 已提交
1518 1519 1520
		}
	}

1521
	schedule_work(&policy->update);
1522

1523
fail:
1524
	cpufreq_cpu_put(policy);
L
Linus Torvalds 已提交
1525 1526
}

1527 1528 1529
static struct syscore_ops cpufreq_syscore_ops = {
	.suspend	= cpufreq_bp_suspend,
	.resume		= cpufreq_bp_resume,
L
Linus Torvalds 已提交
1530 1531
};

1532 1533 1534 1535 1536 1537 1538 1539
/**
 *	cpufreq_get_current_driver - return current driver's name
 *
 *	Return the name string of the currently loaded cpufreq driver
 *	or NULL, if none.
 */
const char *cpufreq_get_current_driver(void)
{
1540 1541 1542 1543
	if (cpufreq_driver)
		return cpufreq_driver->name;

	return NULL;
1544 1545
}
EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
L
Linus Torvalds 已提交
1546 1547 1548 1549 1550 1551 1552 1553 1554 1555

/*********************************************************************
 *                     NOTIFIER LISTS INTERFACE                      *
 *********************************************************************/

/**
 *	cpufreq_register_notifier - register a driver with cpufreq
 *	@nb: notifier function to register
 *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
 *
1556
 *	Add a driver to one of two lists: either a list of drivers that
L
Linus Torvalds 已提交
1557 1558 1559 1560 1561
 *      are notified about clock rate changes (once before and once after
 *      the transition), or a list of drivers that are notified about
 *      changes in cpufreq policy.
 *
 *	This function may sleep, and has the same return conditions as
1562
 *	blocking_notifier_chain_register.
L
Linus Torvalds 已提交
1563 1564 1565 1566 1567
 */
int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
{
	int ret;

1568 1569 1570
	if (cpufreq_disabled())
		return -EINVAL;

1571 1572
	WARN_ON(!init_cpufreq_transition_notifier_list_called);

L
Linus Torvalds 已提交
1573 1574
	switch (list) {
	case CPUFREQ_TRANSITION_NOTIFIER:
1575
		ret = srcu_notifier_chain_register(
1576
				&cpufreq_transition_notifier_list, nb);
L
Linus Torvalds 已提交
1577 1578
		break;
	case CPUFREQ_POLICY_NOTIFIER:
1579 1580
		ret = blocking_notifier_chain_register(
				&cpufreq_policy_notifier_list, nb);
L
Linus Torvalds 已提交
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
		break;
	default:
		ret = -EINVAL;
	}

	return ret;
}
EXPORT_SYMBOL(cpufreq_register_notifier);

/**
 *	cpufreq_unregister_notifier - unregister a driver with cpufreq
 *	@nb: notifier block to be unregistered
1593
 *	@list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
L
Linus Torvalds 已提交
1594 1595 1596 1597
 *
 *	Remove a driver from the CPU frequency notifier list.
 *
 *	This function may sleep, and has the same return conditions as
1598
 *	blocking_notifier_chain_unregister.
L
Linus Torvalds 已提交
1599 1600 1601 1602 1603
 */
int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
{
	int ret;

1604 1605 1606
	if (cpufreq_disabled())
		return -EINVAL;

L
Linus Torvalds 已提交
1607 1608
	switch (list) {
	case CPUFREQ_TRANSITION_NOTIFIER:
1609
		ret = srcu_notifier_chain_unregister(
1610
				&cpufreq_transition_notifier_list, nb);
L
Linus Torvalds 已提交
1611 1612
		break;
	case CPUFREQ_POLICY_NOTIFIER:
1613 1614
		ret = blocking_notifier_chain_unregister(
				&cpufreq_policy_notifier_list, nb);
L
Linus Torvalds 已提交
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633
		break;
	default:
		ret = -EINVAL;
	}

	return ret;
}
EXPORT_SYMBOL(cpufreq_unregister_notifier);


/*********************************************************************
 *                              GOVERNORS                            *
 *********************************************************************/

int __cpufreq_driver_target(struct cpufreq_policy *policy,
			    unsigned int target_freq,
			    unsigned int relation)
{
	int retval = -EINVAL;
1634
	unsigned int old_target_freq = target_freq;
1635

1636 1637
	if (cpufreq_disabled())
		return -ENODEV;
1638 1639
	if (policy->transition_ongoing)
		return -EBUSY;
1640

1641 1642 1643 1644 1645 1646 1647 1648
	/* Make sure that target_freq is within supported range */
	if (target_freq > policy->max)
		target_freq = policy->max;
	if (target_freq < policy->min)
		target_freq = policy->min;

	pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
			policy->cpu, target_freq, relation, old_target_freq);
1649 1650 1651 1652

	if (target_freq == policy->cur)
		return 0;

1653 1654
	if (cpufreq_driver->target)
		retval = cpufreq_driver->target(policy, target_freq, relation);
1655

L
Linus Torvalds 已提交
1656 1657 1658 1659 1660 1661 1662 1663
	return retval;
}
EXPORT_SYMBOL_GPL(__cpufreq_driver_target);

int cpufreq_driver_target(struct cpufreq_policy *policy,
			  unsigned int target_freq,
			  unsigned int relation)
{
1664
	int ret = -EINVAL;
L
Linus Torvalds 已提交
1665

1666
	if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1667
		goto fail;
L
Linus Torvalds 已提交
1668 1669 1670

	ret = __cpufreq_driver_target(policy, target_freq, relation);

1671
	unlock_policy_rwsem_write(policy->cpu);
L
Linus Torvalds 已提交
1672

1673
fail:
L
Linus Torvalds 已提交
1674 1675 1676 1677
	return ret;
}
EXPORT_SYMBOL_GPL(cpufreq_driver_target);

1678 1679 1680
/*
 * when "event" is CPUFREQ_GOV_LIMITS
 */
L
Linus Torvalds 已提交
1681

1682 1683
static int __cpufreq_governor(struct cpufreq_policy *policy,
					unsigned int event)
L
Linus Torvalds 已提交
1684
{
1685
	int ret;
1686 1687 1688 1689 1690 1691 1692 1693 1694 1695

	/* Only must be defined when default governor is known to have latency
	   restrictions, like e.g. conservative or ondemand.
	   That this is the case is already ensured in Kconfig
	*/
#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
	struct cpufreq_governor *gov = &cpufreq_gov_performance;
#else
	struct cpufreq_governor *gov = NULL;
#endif
1696 1697 1698 1699

	if (policy->governor->max_transition_latency &&
	    policy->cpuinfo.transition_latency >
	    policy->governor->max_transition_latency) {
1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
		if (!gov)
			return -EINVAL;
		else {
			printk(KERN_WARNING "%s governor failed, too long"
			       " transition latency of HW, fallback"
			       " to %s governor\n",
			       policy->governor->name,
			       gov->name);
			policy->governor = gov;
		}
1710
	}
L
Linus Torvalds 已提交
1711

1712 1713 1714
	if (event == CPUFREQ_GOV_POLICY_INIT)
		if (!try_module_get(policy->governor->owner))
			return -EINVAL;
L
Linus Torvalds 已提交
1715

1716
	pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1717
						policy->cpu, event);
1718 1719 1720 1721 1722

	mutex_lock(&cpufreq_governor_lock);
	if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
	    (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
		mutex_unlock(&cpufreq_governor_lock);
1723 1724
		if (event == CPUFREQ_GOV_POLICY_INIT)
			module_put(policy->governor->owner);
1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
		return -EBUSY;
	}

	if (event == CPUFREQ_GOV_STOP)
		policy->governor_enabled = false;
	else if (event == CPUFREQ_GOV_START)
		policy->governor_enabled = true;

	mutex_unlock(&cpufreq_governor_lock);

L
Linus Torvalds 已提交
1735 1736
	ret = policy->governor->governor(policy, event);

1737 1738 1739 1740 1741
	if (!ret) {
		if (event == CPUFREQ_GOV_POLICY_INIT)
			policy->governor->initialized++;
		else if (event == CPUFREQ_GOV_POLICY_EXIT)
			policy->governor->initialized--;
1742 1743 1744 1745 1746 1747 1748 1749
	} else {
		/* Restore original values */
		mutex_lock(&cpufreq_governor_lock);
		if (event == CPUFREQ_GOV_STOP)
			policy->governor_enabled = true;
		else if (event == CPUFREQ_GOV_START)
			policy->governor_enabled = false;
		mutex_unlock(&cpufreq_governor_lock);
1750
	}
1751

1752 1753
	if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
			((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
L
Linus Torvalds 已提交
1754 1755 1756 1757 1758 1759 1760
		module_put(policy->governor->owner);

	return ret;
}

int cpufreq_register_governor(struct cpufreq_governor *governor)
{
1761
	int err;
L
Linus Torvalds 已提交
1762 1763 1764 1765

	if (!governor)
		return -EINVAL;

1766 1767 1768
	if (cpufreq_disabled())
		return -ENODEV;

1769
	mutex_lock(&cpufreq_governor_mutex);
1770

1771
	governor->initialized = 0;
1772 1773 1774 1775
	err = -EBUSY;
	if (__find_governor(governor->name) == NULL) {
		err = 0;
		list_add(&governor->governor_list, &cpufreq_governor_list);
L
Linus Torvalds 已提交
1776 1777
	}

1778
	mutex_unlock(&cpufreq_governor_mutex);
1779
	return err;
L
Linus Torvalds 已提交
1780 1781 1782 1783 1784
}
EXPORT_SYMBOL_GPL(cpufreq_register_governor);

void cpufreq_unregister_governor(struct cpufreq_governor *governor)
{
1785 1786 1787 1788
#ifdef CONFIG_HOTPLUG_CPU
	int cpu;
#endif

L
Linus Torvalds 已提交
1789 1790 1791
	if (!governor)
		return;

1792 1793 1794
	if (cpufreq_disabled())
		return;

1795 1796 1797 1798 1799 1800 1801 1802 1803
#ifdef CONFIG_HOTPLUG_CPU
	for_each_present_cpu(cpu) {
		if (cpu_online(cpu))
			continue;
		if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
			strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
	}
#endif

1804
	mutex_lock(&cpufreq_governor_mutex);
L
Linus Torvalds 已提交
1805
	list_del(&governor->governor_list);
1806
	mutex_unlock(&cpufreq_governor_mutex);
L
Linus Torvalds 已提交
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
	return;
}
EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);


/*********************************************************************
 *                          POLICY INTERFACE                         *
 *********************************************************************/

/**
 * cpufreq_get_policy - get the current cpufreq_policy
1818 1819
 * @policy: struct cpufreq_policy into which the current cpufreq_policy
 *	is written
L
Linus Torvalds 已提交
1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
 *
 * Reads the current cpufreq policy.
 */
int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
{
	struct cpufreq_policy *cpu_policy;
	if (!policy)
		return -EINVAL;

	cpu_policy = cpufreq_cpu_get(cpu);
	if (!cpu_policy)
		return -EINVAL;

1833
	memcpy(policy, cpu_policy, sizeof(*policy));
L
Linus Torvalds 已提交
1834 1835 1836 1837 1838 1839

	cpufreq_cpu_put(cpu_policy);
	return 0;
}
EXPORT_SYMBOL(cpufreq_get_policy);

1840
/*
1841 1842
 * data   : current policy.
 * policy : policy to be set.
1843
 */
1844 1845
static int __cpufreq_set_policy(struct cpufreq_policy *policy,
				struct cpufreq_policy *new_policy)
L
Linus Torvalds 已提交
1846
{
1847
	int ret = 0, failed = 1;
L
Linus Torvalds 已提交
1848

1849 1850
	pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
		new_policy->min, new_policy->max);
L
Linus Torvalds 已提交
1851

1852
	memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
L
Linus Torvalds 已提交
1853

1854
	if (new_policy->min > policy->max || new_policy->max < policy->min) {
1855 1856 1857 1858
		ret = -EINVAL;
		goto error_out;
	}

L
Linus Torvalds 已提交
1859
	/* verify the cpu speed can be set within this limit */
1860
	ret = cpufreq_driver->verify(new_policy);
L
Linus Torvalds 已提交
1861 1862 1863 1864
	if (ret)
		goto error_out;

	/* adjust if necessary - all reasons */
1865
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1866
			CPUFREQ_ADJUST, new_policy);
L
Linus Torvalds 已提交
1867 1868

	/* adjust if necessary - hardware incompatibility*/
1869
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1870
			CPUFREQ_INCOMPATIBLE, new_policy);
L
Linus Torvalds 已提交
1871

1872 1873 1874 1875
	/*
	 * verify the cpu speed can be set within this limit, which might be
	 * different to the first one
	 */
1876
	ret = cpufreq_driver->verify(new_policy);
1877
	if (ret)
L
Linus Torvalds 已提交
1878 1879 1880
		goto error_out;

	/* notification of the new policy */
1881
	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1882
			CPUFREQ_NOTIFY, new_policy);
L
Linus Torvalds 已提交
1883

1884 1885
	policy->min = new_policy->min;
	policy->max = new_policy->max;
L
Linus Torvalds 已提交
1886

1887
	pr_debug("new min and max freqs are %u - %u kHz\n",
1888
					policy->min, policy->max);
L
Linus Torvalds 已提交
1889

1890
	if (cpufreq_driver->setpolicy) {
1891
		policy->policy = new_policy->policy;
1892
		pr_debug("setting range\n");
1893
		ret = cpufreq_driver->setpolicy(new_policy);
L
Linus Torvalds 已提交
1894
	} else {
1895
		if (new_policy->governor != policy->governor) {
L
Linus Torvalds 已提交
1896
			/* save old, working values */
1897
			struct cpufreq_governor *old_gov = policy->governor;
L
Linus Torvalds 已提交
1898

1899
			pr_debug("governor switch\n");
L
Linus Torvalds 已提交
1900 1901

			/* end old governor */
1902 1903 1904 1905
			if (policy->governor) {
				__cpufreq_governor(policy, CPUFREQ_GOV_STOP);
				unlock_policy_rwsem_write(new_policy->cpu);
				__cpufreq_governor(policy,
1906
						CPUFREQ_GOV_POLICY_EXIT);
1907
				lock_policy_rwsem_write(new_policy->cpu);
1908
			}
L
Linus Torvalds 已提交
1909 1910

			/* start new governor */
1911 1912 1913
			policy->governor = new_policy->governor;
			if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
				if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
1914
					failed = 0;
1915
				} else {
1916 1917
					unlock_policy_rwsem_write(new_policy->cpu);
					__cpufreq_governor(policy,
1918
							CPUFREQ_GOV_POLICY_EXIT);
1919
					lock_policy_rwsem_write(new_policy->cpu);
1920
				}
1921 1922 1923
			}

			if (failed) {
L
Linus Torvalds 已提交
1924
				/* new governor failed, so re-start old one */
1925
				pr_debug("starting governor %s failed\n",
1926
							policy->governor->name);
L
Linus Torvalds 已提交
1927
				if (old_gov) {
1928 1929
					policy->governor = old_gov;
					__cpufreq_governor(policy,
1930
							CPUFREQ_GOV_POLICY_INIT);
1931
					__cpufreq_governor(policy,
1932
							   CPUFREQ_GOV_START);
L
Linus Torvalds 已提交
1933 1934 1935 1936 1937 1938
				}
				ret = -EINVAL;
				goto error_out;
			}
			/* might be a policy change, too, so fall through */
		}
1939
		pr_debug("governor: change or update limits\n");
1940
		__cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
L
Linus Torvalds 已提交
1941 1942
	}

1943
error_out:
L
Linus Torvalds 已提交
1944 1945 1946 1947 1948 1949 1950
	return ret;
}

/**
 *	cpufreq_update_policy - re-evaluate an existing cpufreq policy
 *	@cpu: CPU which shall be re-evaluated
 *
L
Lucas De Marchi 已提交
1951
 *	Useful for policy notifiers which have different necessities
L
Linus Torvalds 已提交
1952 1953 1954 1955
 *	at different times.
 */
int cpufreq_update_policy(unsigned int cpu)
{
1956 1957
	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
	struct cpufreq_policy new_policy;
1958
	int ret;
L
Linus Torvalds 已提交
1959

1960
	if (!policy) {
1961 1962 1963
		ret = -ENODEV;
		goto no_policy;
	}
L
Linus Torvalds 已提交
1964

1965 1966 1967 1968
	if (unlikely(lock_policy_rwsem_write(cpu))) {
		ret = -EINVAL;
		goto fail;
	}
L
Linus Torvalds 已提交
1969

1970
	pr_debug("updating policy for CPU %u\n", cpu);
1971
	memcpy(&new_policy, policy, sizeof(*policy));
1972 1973 1974 1975
	new_policy.min = policy->user_policy.min;
	new_policy.max = policy->user_policy.max;
	new_policy.policy = policy->user_policy.policy;
	new_policy.governor = policy->user_policy.governor;
L
Linus Torvalds 已提交
1976

1977 1978 1979 1980
	/*
	 * BIOS might change freq behind our back
	 * -> ask driver for current freq and notify governors about a change
	 */
1981
	if (cpufreq_driver->get) {
1982 1983
		new_policy.cur = cpufreq_driver->get(cpu);
		if (!policy->cur) {
1984
			pr_debug("Driver did not initialize current freq");
1985
			policy->cur = new_policy.cur;
1986
		} else {
1987 1988 1989
			if (policy->cur != new_policy.cur && cpufreq_driver->target)
				cpufreq_out_of_sync(cpu, policy->cur,
								new_policy.cur);
1990
		}
1991 1992
	}

1993
	ret = __cpufreq_set_policy(policy, &new_policy);
L
Linus Torvalds 已提交
1994

1995 1996
	unlock_policy_rwsem_write(cpu);

1997
fail:
1998
	cpufreq_cpu_put(policy);
1999
no_policy:
L
Linus Torvalds 已提交
2000 2001 2002 2003
	return ret;
}
EXPORT_SYMBOL(cpufreq_update_policy);

2004
static int cpufreq_cpu_callback(struct notifier_block *nfb,
2005 2006 2007
					unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;
2008
	struct device *dev;
2009
	bool frozen = false;
2010

2011 2012
	dev = get_cpu_device(cpu);
	if (dev) {
2013 2014 2015 2016 2017

		if (action & CPU_TASKS_FROZEN)
			frozen = true;

		switch (action & ~CPU_TASKS_FROZEN) {
2018
		case CPU_ONLINE:
2019
			__cpufreq_add_dev(dev, NULL, frozen);
2020
			cpufreq_update_policy(cpu);
2021
			break;
2022

2023
		case CPU_DOWN_PREPARE:
2024
			__cpufreq_remove_dev(dev, NULL, frozen);
2025
			break;
2026

2027
		case CPU_DOWN_FAILED:
2028
			__cpufreq_add_dev(dev, NULL, frozen);
2029 2030 2031 2032 2033 2034
			break;
		}
	}
	return NOTIFY_OK;
}

2035
static struct notifier_block __refdata cpufreq_cpu_notifier = {
2036
	.notifier_call = cpufreq_cpu_callback,
2037
};
L
Linus Torvalds 已提交
2038 2039 2040 2041 2042 2043 2044 2045 2046 2047

/*********************************************************************
 *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
 *********************************************************************/

/**
 * cpufreq_register_driver - register a CPU Frequency driver
 * @driver_data: A struct cpufreq_driver containing the values#
 * submitted by the CPU Frequency driver.
 *
2048
 * Registers a CPU Frequency driver to this core code. This code
L
Linus Torvalds 已提交
2049
 * returns zero on success, -EBUSY when another driver got here first
2050
 * (and isn't unregistered in the meantime).
L
Linus Torvalds 已提交
2051 2052
 *
 */
2053
int cpufreq_register_driver(struct cpufreq_driver *driver_data)
L
Linus Torvalds 已提交
2054 2055 2056 2057
{
	unsigned long flags;
	int ret;

2058 2059 2060
	if (cpufreq_disabled())
		return -ENODEV;

L
Linus Torvalds 已提交
2061 2062 2063 2064
	if (!driver_data || !driver_data->verify || !driver_data->init ||
	    ((!driver_data->setpolicy) && (!driver_data->target)))
		return -EINVAL;

2065
	pr_debug("trying to register driver %s\n", driver_data->name);
L
Linus Torvalds 已提交
2066 2067 2068 2069

	if (driver_data->setpolicy)
		driver_data->flags |= CPUFREQ_CONST_LOOPS;

2070
	write_lock_irqsave(&cpufreq_driver_lock, flags);
2071
	if (cpufreq_driver) {
2072
		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
L
Linus Torvalds 已提交
2073 2074
		return -EBUSY;
	}
2075
	cpufreq_driver = driver_data;
2076
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
L
Linus Torvalds 已提交
2077

2078
	ret = subsys_interface_register(&cpufreq_interface);
2079 2080
	if (ret)
		goto err_null_driver;
L
Linus Torvalds 已提交
2081

2082
	if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
L
Linus Torvalds 已提交
2083 2084 2085 2086
		int i;
		ret = -ENODEV;

		/* check for at least one working CPU */
2087 2088
		for (i = 0; i < nr_cpu_ids; i++)
			if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
L
Linus Torvalds 已提交
2089
				ret = 0;
2090 2091
				break;
			}
L
Linus Torvalds 已提交
2092 2093 2094

		/* if all ->init() calls failed, unregister */
		if (ret) {
2095
			pr_debug("no CPU initialized for driver %s\n",
2096
							driver_data->name);
2097
			goto err_if_unreg;
L
Linus Torvalds 已提交
2098 2099 2100
		}
	}

2101
	register_hotcpu_notifier(&cpufreq_cpu_notifier);
2102
	pr_debug("driver %s up and running\n", driver_data->name);
L
Linus Torvalds 已提交
2103

2104
	return 0;
2105 2106
err_if_unreg:
	subsys_interface_unregister(&cpufreq_interface);
2107
err_null_driver:
2108
	write_lock_irqsave(&cpufreq_driver_lock, flags);
2109
	cpufreq_driver = NULL;
2110
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
D
Dave Jones 已提交
2111
	return ret;
L
Linus Torvalds 已提交
2112 2113 2114 2115 2116 2117
}
EXPORT_SYMBOL_GPL(cpufreq_register_driver);

/**
 * cpufreq_unregister_driver - unregister the current CPUFreq driver
 *
2118
 * Unregister the current CPUFreq driver. Only call this if you have
L
Linus Torvalds 已提交
2119 2120 2121 2122
 * the right to do so, i.e. if you have succeeded in initialising before!
 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
 * currently not initialised.
 */
2123
int cpufreq_unregister_driver(struct cpufreq_driver *driver)
L
Linus Torvalds 已提交
2124 2125 2126
{
	unsigned long flags;

2127
	if (!cpufreq_driver || (driver != cpufreq_driver))
L
Linus Torvalds 已提交
2128 2129
		return -EINVAL;

2130
	pr_debug("unregistering driver %s\n", driver->name);
L
Linus Torvalds 已提交
2131

2132
	subsys_interface_unregister(&cpufreq_interface);
2133
	unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
L
Linus Torvalds 已提交
2134

2135
	write_lock_irqsave(&cpufreq_driver_lock, flags);
2136
	cpufreq_driver = NULL;
2137
	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
L
Linus Torvalds 已提交
2138 2139 2140 2141

	return 0;
}
EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2142 2143 2144 2145 2146

static int __init cpufreq_core_init(void)
{
	int cpu;

2147 2148 2149
	if (cpufreq_disabled())
		return -ENODEV;

2150
	for_each_possible_cpu(cpu) {
2151
		per_cpu(cpufreq_policy_cpu, cpu) = -1;
2152 2153
		init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
	}
2154

2155
	cpufreq_global_kobject = kobject_create();
2156
	BUG_ON(!cpufreq_global_kobject);
2157
	register_syscore_ops(&cpufreq_syscore_ops);
2158

2159 2160 2161
	return 0;
}
core_initcall(cpufreq_core_init);