intel_pstate.c 32.0 KB
Newer Older
1
/*
2
 * intel_pstate.c: Native P state management for Intel processors
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 * (C) Copyright 2012 Intel Corporation
 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/cpu.h>
#include <linux/cpufreq.h>
#include <linux/sysfs.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/debugfs.h>
28
#include <linux/acpi.h>
29
#include <linux/vmalloc.h>
30 31 32 33 34
#include <trace/events/power.h>

#include <asm/div64.h>
#include <asm/msr.h>
#include <asm/cpu_device_id.h>
35
#include <asm/cpufeature.h>
36

37 38 39
#define BYT_RATIOS		0x66a
#define BYT_VIDS		0x66b
#define BYT_TURBO_RATIOS	0x66c
40
#define BYT_TURBO_VIDS		0x66d
41

42
#define FRAC_BITS 8
43 44
#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
#define fp_toint(X) ((X) >> FRAC_BITS)
45

46 47 48 49 50
static inline int32_t mul_fp(int32_t x, int32_t y)
{
	return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
}

51
static inline int32_t div_fp(s64 x, s64 y)
52
{
53
	return div64_s64((int64_t)x << FRAC_BITS, y);
54 55
}

56 57 58 59 60 61 62 63 64 65 66
static inline int ceiling_fp(int32_t x)
{
	int mask, ret;

	ret = fp_toint(x);
	mask = (1 << FRAC_BITS) - 1;
	if (x & mask)
		ret += 1;
	return ret;
}

67
struct sample {
68
	int32_t core_pct_busy;
69 70
	u64 aperf;
	u64 mperf;
71
	u64 tsc;
72
	int freq;
73
	ktime_t time;
74 75 76 77 78 79
};

struct pstate_data {
	int	current_pstate;
	int	min_pstate;
	int	max_pstate;
80
	int	max_pstate_physical;
81
	int	scaling;
82 83 84
	int	turbo_pstate;
};

85
struct vid_data {
86 87 88
	int min;
	int max;
	int turbo;
89 90 91
	int32_t ratio;
};

92 93 94 95 96 97 98
struct _pid {
	int setpoint;
	int32_t integral;
	int32_t p_gain;
	int32_t i_gain;
	int32_t d_gain;
	int deadband;
99
	int32_t last_err;
100 101 102 103 104 105 106 107
};

struct cpudata {
	int cpu;

	struct timer_list timer;

	struct pstate_data pstate;
108
	struct vid_data vid;
109 110
	struct _pid pid;

111
	ktime_t last_sample_time;
112 113
	u64	prev_aperf;
	u64	prev_mperf;
114
	u64	prev_tsc;
115
	struct sample sample;
116 117 118 119 120 121 122 123 124 125 126 127
};

static struct cpudata **all_cpu_data;
struct pstate_adjust_policy {
	int sample_rate_ms;
	int deadband;
	int setpoint;
	int p_gain_pct;
	int d_gain_pct;
	int i_gain_pct;
};

128 129
struct pstate_funcs {
	int (*get_max)(void);
130
	int (*get_max_physical)(void);
131 132
	int (*get_min)(void);
	int (*get_turbo)(void);
133
	int (*get_scaling)(void);
134 135
	void (*set)(struct cpudata*, int pstate);
	void (*get_vid)(struct cpudata *);
136 137
};

138 139 140
struct cpu_defaults {
	struct pstate_adjust_policy pid_policy;
	struct pstate_funcs funcs;
141 142
};

143 144
static struct pstate_adjust_policy pid_params;
static struct pstate_funcs pstate_funcs;
D
Dirk Brandewie 已提交
145
static int hwp_active;
146

147 148
struct perf_limits {
	int no_turbo;
149
	int turbo_disabled;
150 151 152 153
	int max_perf_pct;
	int min_perf_pct;
	int32_t max_perf;
	int32_t min_perf;
154 155
	int max_policy_pct;
	int max_sysfs_pct;
156 157
	int min_policy_pct;
	int min_sysfs_pct;
158 159
};

160 161 162 163 164 165 166 167 168 169 170 171 172 173
static struct perf_limits performance_limits = {
	.no_turbo = 0,
	.turbo_disabled = 0,
	.max_perf_pct = 100,
	.max_perf = int_tofp(1),
	.min_perf_pct = 100,
	.min_perf = int_tofp(1),
	.max_policy_pct = 100,
	.max_sysfs_pct = 100,
	.min_policy_pct = 0,
	.min_sysfs_pct = 0,
};

static struct perf_limits powersave_limits = {
174
	.no_turbo = 0,
175
	.turbo_disabled = 0,
176 177 178 179
	.max_perf_pct = 100,
	.max_perf = int_tofp(1),
	.min_perf_pct = 0,
	.min_perf = 0,
180 181
	.max_policy_pct = 100,
	.max_sysfs_pct = 100,
182 183
	.min_policy_pct = 0,
	.min_sysfs_pct = 0,
184 185
};

186 187 188 189 190 191
#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
static struct perf_limits *limits = &performance_limits;
#else
static struct perf_limits *limits = &powersave_limits;
#endif

192
static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
193
			     int deadband, int integral) {
194 195 196
	pid->setpoint = setpoint;
	pid->deadband  = deadband;
	pid->integral  = int_tofp(integral);
197
	pid->last_err  = int_tofp(setpoint) - int_tofp(busy);
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
}

static inline void pid_p_gain_set(struct _pid *pid, int percent)
{
	pid->p_gain = div_fp(int_tofp(percent), int_tofp(100));
}

static inline void pid_i_gain_set(struct _pid *pid, int percent)
{
	pid->i_gain = div_fp(int_tofp(percent), int_tofp(100));
}

static inline void pid_d_gain_set(struct _pid *pid, int percent)
{
	pid->d_gain = div_fp(int_tofp(percent), int_tofp(100));
}

215
static signed int pid_calc(struct _pid *pid, int32_t busy)
216
{
217
	signed int result;
218 219 220
	int32_t pterm, dterm, fp_error;
	int32_t integral_limit;

221
	fp_error = int_tofp(pid->setpoint) - busy;
222

223
	if (abs(fp_error) <= int_tofp(pid->deadband))
224 225 226 227 228 229
		return 0;

	pterm = mul_fp(pid->p_gain, fp_error);

	pid->integral += fp_error;

230 231 232 233 234 235 236 237
	/*
	 * We limit the integral here so that it will never
	 * get higher than 30.  This prevents it from becoming
	 * too large an input over long periods of time and allows
	 * it to get factored out sooner.
	 *
	 * The value of 30 was chosen through experimentation.
	 */
238 239 240 241 242 243
	integral_limit = int_tofp(30);
	if (pid->integral > integral_limit)
		pid->integral = integral_limit;
	if (pid->integral < -integral_limit)
		pid->integral = -integral_limit;

244 245
	dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
	pid->last_err = fp_error;
246 247

	result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
248
	result = result + (1 << (FRAC_BITS-1));
249 250 251 252 253
	return (signed int)fp_toint(result);
}

static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
{
254 255 256
	pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
	pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
	pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
257

258
	pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
259 260 261 262 263
}

static inline void intel_pstate_reset_all_pid(void)
{
	unsigned int cpu;
264

265 266 267 268 269 270
	for_each_online_cpu(cpu) {
		if (all_cpu_data[cpu])
			intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
	}
}

271 272 273 274 275 276 277
static inline void update_turbo_state(void)
{
	u64 misc_en;
	struct cpudata *cpu;

	cpu = all_cpu_data[0];
	rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
278
	limits->turbo_disabled =
279 280 281 282
		(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
		 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
}

D
Dirk Brandewie 已提交
283 284
static void intel_pstate_hwp_set(void)
{
285 286 287 288 289 290 291
	int min, hw_min, max, hw_max, cpu, range, adj_range;
	u64 value, cap;

	rdmsrl(MSR_HWP_CAPABILITIES, cap);
	hw_min = HWP_LOWEST_PERF(cap);
	hw_max = HWP_HIGHEST_PERF(cap);
	range = hw_max - hw_min;
D
Dirk Brandewie 已提交
292 293 294 295 296

	get_online_cpus();

	for_each_online_cpu(cpu) {
		rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
297
		adj_range = limits->min_perf_pct * range / 100;
298
		min = hw_min + adj_range;
D
Dirk Brandewie 已提交
299 300 301
		value &= ~HWP_MIN_PERF(~0L);
		value |= HWP_MIN_PERF(min);

302
		adj_range = limits->max_perf_pct * range / 100;
303
		max = hw_min + adj_range;
304
		if (limits->no_turbo) {
305 306 307
			hw_max = HWP_GUARANTEED_PERF(cap);
			if (hw_max < max)
				max = hw_max;
D
Dirk Brandewie 已提交
308 309 310 311 312 313 314 315 316 317
		}

		value &= ~HWP_MAX_PERF(~0L);
		value |= HWP_MAX_PERF(max);
		wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
	}

	put_online_cpus();
}

318 319 320 321 322 323 324
/************************** debugfs begin ************************/
static int pid_param_set(void *data, u64 val)
{
	*(u32 *)data = val;
	intel_pstate_reset_all_pid();
	return 0;
}
325

326 327 328 329 330
static int pid_param_get(void *data, u64 *val)
{
	*val = *(u32 *)data;
	return 0;
}
331
DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
332 333 334 335 336 337 338

struct pid_param {
	char *name;
	void *value;
};

static struct pid_param pid_files[] = {
339 340 341 342 343 344
	{"sample_rate_ms", &pid_params.sample_rate_ms},
	{"d_gain_pct", &pid_params.d_gain_pct},
	{"i_gain_pct", &pid_params.i_gain_pct},
	{"deadband", &pid_params.deadband},
	{"setpoint", &pid_params.setpoint},
	{"p_gain_pct", &pid_params.p_gain_pct},
345 346 347
	{NULL, NULL}
};

348
static void __init intel_pstate_debug_expose_params(void)
349
{
350
	struct dentry *debugfs_parent;
351 352
	int i = 0;

D
Dirk Brandewie 已提交
353 354
	if (hwp_active)
		return;
355 356 357 358 359
	debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
	if (IS_ERR_OR_NULL(debugfs_parent))
		return;
	while (pid_files[i].name) {
		debugfs_create_file(pid_files[i].name, 0660,
360 361
				    debugfs_parent, pid_files[i].value,
				    &fops_pid_param);
362 363 364 365 366 367 368 369 370 371 372
		i++;
	}
}

/************************** debugfs end ************************/

/************************** sysfs begin ************************/
#define show_one(file_name, object)					\
	static ssize_t show_##file_name					\
	(struct kobject *kobj, struct attribute *attr, char *buf)	\
	{								\
373
		return sprintf(buf, "%u\n", limits->object);		\
374 375
	}

376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
static ssize_t show_turbo_pct(struct kobject *kobj,
				struct attribute *attr, char *buf)
{
	struct cpudata *cpu;
	int total, no_turbo, turbo_pct;
	uint32_t turbo_fp;

	cpu = all_cpu_data[0];

	total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
	no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
	turbo_fp = div_fp(int_tofp(no_turbo), int_tofp(total));
	turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
	return sprintf(buf, "%u\n", turbo_pct);
}

392 393 394 395 396 397 398 399 400 401 402
static ssize_t show_num_pstates(struct kobject *kobj,
				struct attribute *attr, char *buf)
{
	struct cpudata *cpu;
	int total;

	cpu = all_cpu_data[0];
	total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
	return sprintf(buf, "%u\n", total);
}

403 404 405 406 407 408
static ssize_t show_no_turbo(struct kobject *kobj,
			     struct attribute *attr, char *buf)
{
	ssize_t ret;

	update_turbo_state();
409 410
	if (limits->turbo_disabled)
		ret = sprintf(buf, "%u\n", limits->turbo_disabled);
411
	else
412
		ret = sprintf(buf, "%u\n", limits->no_turbo);
413 414 415 416

	return ret;
}

417
static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
418
			      const char *buf, size_t count)
419 420 421
{
	unsigned int input;
	int ret;
422

423 424 425
	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;
426 427

	update_turbo_state();
428
	if (limits->turbo_disabled) {
429
		pr_warn("intel_pstate: Turbo disabled by BIOS or unavailable on processor\n");
430
		return -EPERM;
431
	}
D
Dirk Brandewie 已提交
432

433
	limits->no_turbo = clamp_t(int, input, 0, 1);
434

D
Dirk Brandewie 已提交
435 436 437
	if (hwp_active)
		intel_pstate_hwp_set();

438 439 440 441
	return count;
}

static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
442
				  const char *buf, size_t count)
443 444 445
{
	unsigned int input;
	int ret;
446

447 448 449 450
	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;

451 452 453 454 455 456 457 458 459
	limits->max_sysfs_pct = clamp_t(int, input, 0 , 100);
	limits->max_perf_pct = min(limits->max_policy_pct,
				   limits->max_sysfs_pct);
	limits->max_perf_pct = max(limits->min_policy_pct,
				   limits->max_perf_pct);
	limits->max_perf_pct = max(limits->min_perf_pct,
				   limits->max_perf_pct);
	limits->max_perf = div_fp(int_tofp(limits->max_perf_pct),
				  int_tofp(100));
460

D
Dirk Brandewie 已提交
461 462
	if (hwp_active)
		intel_pstate_hwp_set();
463 464 465 466
	return count;
}

static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
467
				  const char *buf, size_t count)
468 469 470
{
	unsigned int input;
	int ret;
471

472 473 474
	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;
475

476 477 478 479 480 481 482 483 484
	limits->min_sysfs_pct = clamp_t(int, input, 0 , 100);
	limits->min_perf_pct = max(limits->min_policy_pct,
				   limits->min_sysfs_pct);
	limits->min_perf_pct = min(limits->max_policy_pct,
				   limits->min_perf_pct);
	limits->min_perf_pct = min(limits->max_perf_pct,
				   limits->min_perf_pct);
	limits->min_perf = div_fp(int_tofp(limits->min_perf_pct),
				  int_tofp(100));
485

D
Dirk Brandewie 已提交
486 487
	if (hwp_active)
		intel_pstate_hwp_set();
488 489 490 491 492 493 494 495 496
	return count;
}

show_one(max_perf_pct, max_perf_pct);
show_one(min_perf_pct, min_perf_pct);

define_one_global_rw(no_turbo);
define_one_global_rw(max_perf_pct);
define_one_global_rw(min_perf_pct);
497
define_one_global_ro(turbo_pct);
498
define_one_global_ro(num_pstates);
499 500 501 502 503

static struct attribute *intel_pstate_attributes[] = {
	&no_turbo.attr,
	&max_perf_pct.attr,
	&min_perf_pct.attr,
504
	&turbo_pct.attr,
505
	&num_pstates.attr,
506 507 508 509 510 511 512
	NULL
};

static struct attribute_group intel_pstate_attr_group = {
	.attrs = intel_pstate_attributes,
};

513
static void __init intel_pstate_sysfs_expose_params(void)
514
{
515
	struct kobject *intel_pstate_kobject;
516 517 518 519 520
	int rc;

	intel_pstate_kobject = kobject_create_and_add("intel_pstate",
						&cpu_subsys.dev_root->kobj);
	BUG_ON(!intel_pstate_kobject);
521
	rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
522 523 524
	BUG_ON(rc);
}
/************************** sysfs end ************************/
D
Dirk Brandewie 已提交
525

526
static void intel_pstate_hwp_enable(struct cpudata *cpudata)
D
Dirk Brandewie 已提交
527
{
528
	wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
D
Dirk Brandewie 已提交
529 530
}

531 532 533
static int byt_get_min_pstate(void)
{
	u64 value;
534

535
	rdmsrl(BYT_RATIOS, value);
D
Dirk Brandewie 已提交
536
	return (value >> 8) & 0x7F;
537 538 539 540 541
}

static int byt_get_max_pstate(void)
{
	u64 value;
542

543
	rdmsrl(BYT_RATIOS, value);
D
Dirk Brandewie 已提交
544
	return (value >> 16) & 0x7F;
545
}
546

547 548 549
static int byt_get_turbo_pstate(void)
{
	u64 value;
550

551
	rdmsrl(BYT_TURBO_RATIOS, value);
D
Dirk Brandewie 已提交
552
	return value & 0x7F;
553 554
}

555 556 557 558 559 560
static void byt_set_pstate(struct cpudata *cpudata, int pstate)
{
	u64 val;
	int32_t vid_fp;
	u32 vid;

561
	val = (u64)pstate << 8;
562
	if (limits->no_turbo && !limits->turbo_disabled)
563 564 565 566 567 568 569
		val |= (u64)1 << 32;

	vid_fp = cpudata->vid.min + mul_fp(
		int_tofp(pstate - cpudata->pstate.min_pstate),
		cpudata->vid.ratio);

	vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
570
	vid = ceiling_fp(vid_fp);
571

572 573 574
	if (pstate > cpudata->pstate.max_pstate)
		vid = cpudata->vid.turbo;

575 576
	val |= vid;

577
	wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
578 579
}

580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
#define BYT_BCLK_FREQS 5
static int byt_freq_table[BYT_BCLK_FREQS] = { 833, 1000, 1333, 1167, 800};

static int byt_get_scaling(void)
{
	u64 value;
	int i;

	rdmsrl(MSR_FSB_FREQ, value);
	i = value & 0x3;

	BUG_ON(i > BYT_BCLK_FREQS);

	return byt_freq_table[i] * 100;
}

596 597 598 599 600
static void byt_get_vid(struct cpudata *cpudata)
{
	u64 value;

	rdmsrl(BYT_VIDS, value);
D
Dirk Brandewie 已提交
601 602
	cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
	cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
603 604 605 606
	cpudata->vid.ratio = div_fp(
		cpudata->vid.max - cpudata->vid.min,
		int_tofp(cpudata->pstate.max_pstate -
			cpudata->pstate.min_pstate));
607 608 609

	rdmsrl(BYT_TURBO_VIDS, value);
	cpudata->vid.turbo = value & 0x7f;
610 611
}

612
static int core_get_min_pstate(void)
613 614
{
	u64 value;
615

616
	rdmsrl(MSR_PLATFORM_INFO, value);
617 618 619
	return (value >> 40) & 0xFF;
}

620
static int core_get_max_pstate_physical(void)
621 622
{
	u64 value;
623

624
	rdmsrl(MSR_PLATFORM_INFO, value);
625 626 627
	return (value >> 8) & 0xFF;
}

628
static int core_get_max_pstate(void)
629
{
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
	u64 tar;
	u64 plat_info;
	int max_pstate;
	int err;

	rdmsrl(MSR_PLATFORM_INFO, plat_info);
	max_pstate = (plat_info >> 8) & 0xFF;

	err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
	if (!err) {
		/* Do some sanity checking for safety */
		if (plat_info & 0x600000000) {
			u64 tdp_ctrl;
			u64 tdp_ratio;
			int tdp_msr;

			err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
			if (err)
				goto skip_tar;

			tdp_msr = MSR_CONFIG_TDP_NOMINAL + tdp_ctrl;
			err = rdmsrl_safe(tdp_msr, &tdp_ratio);
			if (err)
				goto skip_tar;

			if (tdp_ratio - 1 == tar) {
				max_pstate = tar;
				pr_debug("max_pstate=TAC %x\n", max_pstate);
			} else {
				goto skip_tar;
			}
		}
	}
663

664 665
skip_tar:
	return max_pstate;
666 667
}

668
static int core_get_turbo_pstate(void)
669 670 671
{
	u64 value;
	int nont, ret;
672

673
	rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
674
	nont = core_get_max_pstate();
675
	ret = (value) & 255;
676 677 678 679 680
	if (ret <= nont)
		ret = nont;
	return ret;
}

681 682 683 684 685
static inline int core_get_scaling(void)
{
	return 100000;
}

686
static void core_set_pstate(struct cpudata *cpudata, int pstate)
687 688 689
{
	u64 val;

690
	val = (u64)pstate << 8;
691
	if (limits->no_turbo && !limits->turbo_disabled)
692 693
		val |= (u64)1 << 32;

694
	wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
695 696
}

697 698 699 700 701 702 703 704 705 706 707 708 709
static int knl_get_turbo_pstate(void)
{
	u64 value;
	int nont, ret;

	rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
	nont = core_get_max_pstate();
	ret = (((value) >> 8) & 0xFF);
	if (ret <= nont)
		ret = nont;
	return ret;
}

710 711 712 713 714 715 716 717 718 719 720
static struct cpu_defaults core_params = {
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
		.setpoint = 97,
		.p_gain_pct = 20,
		.d_gain_pct = 0,
		.i_gain_pct = 0,
	},
	.funcs = {
		.get_max = core_get_max_pstate,
721
		.get_max_physical = core_get_max_pstate_physical,
722 723
		.get_min = core_get_min_pstate,
		.get_turbo = core_get_turbo_pstate,
724
		.get_scaling = core_get_scaling,
725 726 727 728
		.set = core_set_pstate,
	},
};

729 730 731 732
static struct cpu_defaults byt_params = {
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
733
		.setpoint = 60,
734 735 736 737 738 739
		.p_gain_pct = 14,
		.d_gain_pct = 0,
		.i_gain_pct = 4,
	},
	.funcs = {
		.get_max = byt_get_max_pstate,
740
		.get_max_physical = byt_get_max_pstate,
741
		.get_min = byt_get_min_pstate,
742
		.get_turbo = byt_get_turbo_pstate,
743
		.set = byt_set_pstate,
744
		.get_scaling = byt_get_scaling,
745
		.get_vid = byt_get_vid,
746 747 748
	},
};

749 750 751 752 753 754 755 756 757 758 759
static struct cpu_defaults knl_params = {
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
		.setpoint = 97,
		.p_gain_pct = 20,
		.d_gain_pct = 0,
		.i_gain_pct = 0,
	},
	.funcs = {
		.get_max = core_get_max_pstate,
760
		.get_max_physical = core_get_max_pstate_physical,
761 762
		.get_min = core_get_min_pstate,
		.get_turbo = knl_get_turbo_pstate,
763
		.get_scaling = core_get_scaling,
764 765 766 767
		.set = core_set_pstate,
	},
};

768 769 770
static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
{
	int max_perf = cpu->pstate.turbo_pstate;
771
	int max_perf_adj;
772
	int min_perf;
773

774
	if (limits->no_turbo || limits->turbo_disabled)
775 776
		max_perf = cpu->pstate.max_pstate;

777 778 779 780 781
	/*
	 * performance can be limited by user through sysfs, by cpufreq
	 * policy, or by cpu specific default values determined through
	 * experimentation.
	 */
782 783 784
	max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf), limits->max_perf));
	*max = clamp_t(int, max_perf_adj,
			cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
785

786 787
	min_perf = fp_toint(mul_fp(int_tofp(max_perf), limits->min_perf));
	*min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
788 789
}

790
static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate, bool force)
791 792 793
{
	int max_perf, min_perf;

794 795
	if (force) {
		update_turbo_state();
796

797
		intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
798

799
		pstate = clamp_t(int, pstate, min_perf, max_perf);
800

801 802 803
		if (pstate == cpu->pstate.current_pstate)
			return;
	}
804
	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
805

806 807
	cpu->pstate.current_pstate = pstate;

808
	pstate_funcs.set(cpu, pstate);
809 810 811 812
}

static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
{
813 814
	cpu->pstate.min_pstate = pstate_funcs.get_min();
	cpu->pstate.max_pstate = pstate_funcs.get_max();
815
	cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
816
	cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
817
	cpu->pstate.scaling = pstate_funcs.get_scaling();
818

819 820
	if (pstate_funcs.get_vid)
		pstate_funcs.get_vid(cpu);
821
	intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate, false);
822 823
}

824
static inline void intel_pstate_calc_busy(struct cpudata *cpu)
825
{
826
	struct sample *sample = &cpu->sample;
827
	int64_t core_pct;
828

829
	core_pct = int_tofp(sample->aperf) * int_tofp(100);
830
	core_pct = div64_u64(core_pct, int_tofp(sample->mperf));
831

832
	sample->freq = fp_toint(
833
		mul_fp(int_tofp(
834 835
			cpu->pstate.max_pstate_physical *
			cpu->pstate.scaling / 100),
836
			core_pct));
837

838
	sample->core_pct_busy = (int32_t)core_pct;
839 840 841 842 843
}

static inline void intel_pstate_sample(struct cpudata *cpu)
{
	u64 aperf, mperf;
844
	unsigned long flags;
845
	u64 tsc;
846

847
	local_irq_save(flags);
848 849
	rdmsrl(MSR_IA32_APERF, aperf);
	rdmsrl(MSR_IA32_MPERF, mperf);
850 851 852 853 854
	if (cpu->prev_mperf == mperf) {
		local_irq_restore(flags);
		return;
	}

855
	tsc = rdtsc();
856
	local_irq_restore(flags);
857

858 859
	cpu->last_sample_time = cpu->sample.time;
	cpu->sample.time = ktime_get();
860 861
	cpu->sample.aperf = aperf;
	cpu->sample.mperf = mperf;
862
	cpu->sample.tsc =  tsc;
863 864
	cpu->sample.aperf -= cpu->prev_aperf;
	cpu->sample.mperf -= cpu->prev_mperf;
865
	cpu->sample.tsc -= cpu->prev_tsc;
866

867
	intel_pstate_calc_busy(cpu);
868 869 870

	cpu->prev_aperf = aperf;
	cpu->prev_mperf = mperf;
871
	cpu->prev_tsc = tsc;
872 873
}

D
Dirk Brandewie 已提交
874 875 876 877 878 879 880 881
static inline void intel_hwp_set_sample_time(struct cpudata *cpu)
{
	int delay;

	delay = msecs_to_jiffies(50);
	mod_timer_pinned(&cpu->timer, jiffies + delay);
}

882 883
static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
{
884
	int delay;
885

886
	delay = msecs_to_jiffies(pid_params.sample_rate_ms);
887 888 889
	mod_timer_pinned(&cpu->timer, jiffies + delay);
}

890
static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu)
891
{
892
	int32_t core_busy, max_pstate, current_pstate, sample_ratio;
893
	s64 duration_us;
894
	u32 sample_time;
895

896 897 898 899 900 901 902 903 904 905 906
	/*
	 * core_busy is the ratio of actual performance to max
	 * max_pstate is the max non turbo pstate available
	 * current_pstate was the pstate that was requested during
	 * 	the last sample period.
	 *
	 * We normalize core_busy, which was our actual percent
	 * performance to what we requested during the last sample
	 * period. The result will be a percentage of busy at a
	 * specified pstate.
	 */
907
	core_busy = cpu->sample.core_pct_busy;
908
	max_pstate = int_tofp(cpu->pstate.max_pstate_physical);
909
	current_pstate = int_tofp(cpu->pstate.current_pstate);
910
	core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate));
911

912 913 914 915 916 917 918
	/*
	 * Since we have a deferred timer, it will not fire unless
	 * we are in C0.  So, determine if the actual elapsed time
	 * is significantly greater (3x) than our sample interval.  If it
	 * is, then we were idle for a long enough period of time
	 * to adjust our busyness.
	 */
919
	sample_time = pid_params.sample_rate_ms  * USEC_PER_MSEC;
920 921
	duration_us = ktime_us_delta(cpu->sample.time,
				     cpu->last_sample_time);
922 923
	if (duration_us > sample_time * 3) {
		sample_ratio = div_fp(int_tofp(sample_time),
924
				      int_tofp(duration_us));
925 926 927
		core_busy = mul_fp(core_busy, sample_ratio);
	}

928
	return core_busy;
929 930 931 932
}

static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
{
933
	int32_t busy_scaled;
934
	struct _pid *pid;
935
	signed int ctl;
936 937 938 939
	int from;
	struct sample *sample;

	from = cpu->pstate.current_pstate;
940 941 942 943 944 945

	pid = &cpu->pid;
	busy_scaled = intel_pstate_get_scaled_busy(cpu);

	ctl = pid_calc(pid, busy_scaled);

946
	/* Negative values of ctl increase the pstate and vice versa */
947
	intel_pstate_set_pstate(cpu, cpu->pstate.current_pstate - ctl, true);
948 949 950 951 952 953 954 955 956 957

	sample = &cpu->sample;
	trace_pstate_sample(fp_toint(sample->core_pct_busy),
		fp_toint(busy_scaled),
		from,
		cpu->pstate.current_pstate,
		sample->mperf,
		sample->aperf,
		sample->tsc,
		sample->freq);
958 959
}

D
Dirk Brandewie 已提交
960 961 962 963 964 965 966 967
static void intel_hwp_timer_func(unsigned long __data)
{
	struct cpudata *cpu = (struct cpudata *) __data;

	intel_pstate_sample(cpu);
	intel_hwp_set_sample_time(cpu);
}

968 969 970 971 972
static void intel_pstate_timer_func(unsigned long __data)
{
	struct cpudata *cpu = (struct cpudata *) __data;

	intel_pstate_sample(cpu);
973

974
	intel_pstate_adjust_busy_pstate(cpu);
975

976 977 978 979
	intel_pstate_set_sample_time(cpu);
}

#define ICPU(model, policy) \
980 981
	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
			(unsigned long)&policy }
982 983

static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
984 985
	ICPU(0x2a, core_params),
	ICPU(0x2d, core_params),
986
	ICPU(0x37, byt_params),
987 988
	ICPU(0x3a, core_params),
	ICPU(0x3c, core_params),
989
	ICPU(0x3d, core_params),
990 991 992 993
	ICPU(0x3e, core_params),
	ICPU(0x3f, core_params),
	ICPU(0x45, core_params),
	ICPU(0x46, core_params),
994
	ICPU(0x47, core_params),
995
	ICPU(0x4c, byt_params),
996
	ICPU(0x4e, core_params),
997
	ICPU(0x4f, core_params),
998
	ICPU(0x5e, core_params),
999
	ICPU(0x56, core_params),
1000
	ICPU(0x57, knl_params),
1001 1002 1003 1004
	{}
};
MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);

D
Dirk Brandewie 已提交
1005 1006 1007 1008 1009
static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] = {
	ICPU(0x56, core_params),
	{}
};

1010 1011 1012 1013
static int intel_pstate_init_cpu(unsigned int cpunum)
{
	struct cpudata *cpu;

1014 1015 1016
	if (!all_cpu_data[cpunum])
		all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata),
					       GFP_KERNEL);
1017 1018 1019 1020 1021 1022
	if (!all_cpu_data[cpunum])
		return -ENOMEM;

	cpu = all_cpu_data[cpunum];

	cpu->cpu = cpunum;
1023 1024 1025 1026

	if (hwp_active)
		intel_pstate_hwp_enable(cpu);

1027
	intel_pstate_get_cpu_pstates(cpu);
1028

1029
	init_timer_deferrable(&cpu->timer);
1030
	cpu->timer.data = (unsigned long)cpu;
1031
	cpu->timer.expires = jiffies + HZ/100;
D
Dirk Brandewie 已提交
1032 1033 1034 1035 1036 1037

	if (!hwp_active)
		cpu->timer.function = intel_pstate_timer_func;
	else
		cpu->timer.function = intel_hwp_timer_func;

1038 1039 1040 1041 1042
	intel_pstate_busy_pid_reset(cpu);
	intel_pstate_sample(cpu);

	add_timer_on(&cpu->timer, cpunum);

1043
	pr_debug("intel_pstate: controlling: cpu %d\n", cpunum);
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055

	return 0;
}

static unsigned int intel_pstate_get(unsigned int cpu_num)
{
	struct sample *sample;
	struct cpudata *cpu;

	cpu = all_cpu_data[cpu_num];
	if (!cpu)
		return 0;
1056
	sample = &cpu->sample;
1057 1058 1059 1060 1061
	return sample->freq;
}

static int intel_pstate_set_policy(struct cpufreq_policy *policy)
{
1062 1063 1064
	if (!policy->cpuinfo.max_freq)
		return -ENODEV;

1065 1066
	if (policy->policy == CPUFREQ_POLICY_PERFORMANCE &&
	    policy->max >= policy->cpuinfo.max_freq) {
1067 1068
		pr_debug("intel_pstate: set performance\n");
		limits = &performance_limits;
1069
		return 0;
1070
	}
D
Dirk Brandewie 已提交
1071

1072 1073 1074 1075 1076 1077
	pr_debug("intel_pstate: set powersave\n");
	limits = &powersave_limits;
	limits->min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
	limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, 0 , 100);
	limits->max_policy_pct = (policy->max * 100) / policy->cpuinfo.max_freq;
	limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0 , 100);
1078 1079

	/* Normalize user input to [min_policy_pct, max_policy_pct] */
1080 1081 1082 1083 1084 1085 1086 1087
	limits->min_perf_pct = max(limits->min_policy_pct,
				   limits->min_sysfs_pct);
	limits->min_perf_pct = min(limits->max_policy_pct,
				   limits->min_perf_pct);
	limits->max_perf_pct = min(limits->max_policy_pct,
				   limits->max_sysfs_pct);
	limits->max_perf_pct = max(limits->min_policy_pct,
				   limits->max_perf_pct);
1088 1089

	/* Make sure min_perf_pct <= max_perf_pct */
1090
	limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct);
1091

1092 1093 1094 1095
	limits->min_perf = div_fp(int_tofp(limits->min_perf_pct),
				  int_tofp(100));
	limits->max_perf = div_fp(int_tofp(limits->max_perf_pct),
				  int_tofp(100));
1096

D
Dirk Brandewie 已提交
1097 1098 1099
	if (hwp_active)
		intel_pstate_hwp_set();

1100 1101 1102 1103 1104
	return 0;
}

static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
{
1105
	cpufreq_verify_within_cpu_limits(policy);
1106

1107
	if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
1108
	    policy->policy != CPUFREQ_POLICY_PERFORMANCE)
1109 1110 1111 1112 1113
		return -EINVAL;

	return 0;
}

1114
static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
1115
{
1116 1117
	int cpu_num = policy->cpu;
	struct cpudata *cpu = all_cpu_data[cpu_num];
1118

1119
	pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);
1120

1121
	del_timer_sync(&all_cpu_data[cpu_num]->timer);
D
Dirk Brandewie 已提交
1122 1123 1124
	if (hwp_active)
		return;

1125
	intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate, false);
1126 1127
}

1128
static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
1129 1130
{
	struct cpudata *cpu;
1131
	int rc;
1132 1133 1134 1135 1136 1137 1138

	rc = intel_pstate_init_cpu(policy->cpu);
	if (rc)
		return rc;

	cpu = all_cpu_data[policy->cpu];

1139
	if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100)
1140 1141 1142 1143
		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
	else
		policy->policy = CPUFREQ_POLICY_POWERSAVE;

1144 1145
	policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
	policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1146 1147

	/* cpuinfo and default policy values */
1148 1149 1150
	policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
	policy->cpuinfo.max_freq =
		cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
	cpumask_set_cpu(policy->cpu, policy->cpus);

	return 0;
}

static struct cpufreq_driver intel_pstate_driver = {
	.flags		= CPUFREQ_CONST_LOOPS,
	.verify		= intel_pstate_verify_policy,
	.setpolicy	= intel_pstate_set_policy,
	.get		= intel_pstate_get,
	.init		= intel_pstate_cpu_init,
1163
	.stop_cpu	= intel_pstate_stop_cpu,
1164 1165 1166
	.name		= "intel_pstate",
};

1167
static int __initdata no_load;
D
Dirk Brandewie 已提交
1168
static int __initdata no_hwp;
1169
static int __initdata hwp_only;
1170
static unsigned int force_load;
1171

1172 1173
static int intel_pstate_msrs_not_valid(void)
{
1174
	if (!pstate_funcs.get_max() ||
1175 1176
	    !pstate_funcs.get_min() ||
	    !pstate_funcs.get_turbo())
1177 1178 1179 1180
		return -ENODEV;

	return 0;
}
1181

1182
static void copy_pid_params(struct pstate_adjust_policy *policy)
1183 1184 1185 1186 1187 1188 1189 1190 1191
{
	pid_params.sample_rate_ms = policy->sample_rate_ms;
	pid_params.p_gain_pct = policy->p_gain_pct;
	pid_params.i_gain_pct = policy->i_gain_pct;
	pid_params.d_gain_pct = policy->d_gain_pct;
	pid_params.deadband = policy->deadband;
	pid_params.setpoint = policy->setpoint;
}

1192
static void copy_cpu_funcs(struct pstate_funcs *funcs)
1193 1194
{
	pstate_funcs.get_max   = funcs->get_max;
1195
	pstate_funcs.get_max_physical = funcs->get_max_physical;
1196 1197
	pstate_funcs.get_min   = funcs->get_min;
	pstate_funcs.get_turbo = funcs->get_turbo;
1198
	pstate_funcs.get_scaling = funcs->get_scaling;
1199
	pstate_funcs.set       = funcs->set;
1200
	pstate_funcs.get_vid   = funcs->get_vid;
1201 1202
}

1203
#if IS_ENABLED(CONFIG_ACPI)
1204
#include <acpi/processor.h>
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234

static bool intel_pstate_no_acpi_pss(void)
{
	int i;

	for_each_possible_cpu(i) {
		acpi_status status;
		union acpi_object *pss;
		struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
		struct acpi_processor *pr = per_cpu(processors, i);

		if (!pr)
			continue;

		status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
		if (ACPI_FAILURE(status))
			continue;

		pss = buffer.pointer;
		if (pss && pss->type == ACPI_TYPE_PACKAGE) {
			kfree(pss);
			return false;
		}

		kfree(pss);
	}

	return true;
}

1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
static bool intel_pstate_has_acpi_ppc(void)
{
	int i;

	for_each_possible_cpu(i) {
		struct acpi_processor *pr = per_cpu(processors, i);

		if (!pr)
			continue;
		if (acpi_has_method(pr->handle, "_PPC"))
			return true;
	}
	return false;
}

enum {
	PSS,
	PPC,
};

1255 1256 1257 1258
struct hw_vendor_info {
	u16  valid;
	char oem_id[ACPI_OEM_ID_SIZE];
	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
1259
	int  oem_pwr_table;
1260 1261 1262 1263
};

/* Hardware vendor-specific info that has its own power management modes */
static struct hw_vendor_info vendor_info[] = {
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
	{1, "HP    ", "ProLiant", PSS},
	{1, "ORACLE", "X4-2    ", PPC},
	{1, "ORACLE", "X4-2L   ", PPC},
	{1, "ORACLE", "X4-2B   ", PPC},
	{1, "ORACLE", "X3-2    ", PPC},
	{1, "ORACLE", "X3-2L   ", PPC},
	{1, "ORACLE", "X3-2B   ", PPC},
	{1, "ORACLE", "X4470M2 ", PPC},
	{1, "ORACLE", "X4270M3 ", PPC},
	{1, "ORACLE", "X4270M2 ", PPC},
	{1, "ORACLE", "X4170M2 ", PPC},
1275 1276 1277 1278
	{1, "ORACLE", "X4170 M3", PPC},
	{1, "ORACLE", "X4275 M3", PPC},
	{1, "ORACLE", "X6-2    ", PPC},
	{1, "ORACLE", "Sudbury ", PPC},
1279 1280 1281 1282 1283 1284 1285
	{0, "", ""},
};

static bool intel_pstate_platform_pwr_mgmt_exists(void)
{
	struct acpi_table_header hdr;
	struct hw_vendor_info *v_info;
D
Dirk Brandewie 已提交
1286 1287 1288 1289 1290 1291 1292 1293 1294
	const struct x86_cpu_id *id;
	u64 misc_pwr;

	id = x86_match_cpu(intel_pstate_cpu_oob_ids);
	if (id) {
		rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
		if ( misc_pwr & (1 << 8))
			return true;
	}
1295

1296 1297
	if (acpi_disabled ||
	    ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
1298 1299 1300
		return false;

	for (v_info = vendor_info; v_info->valid; v_info++) {
1301
		if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
1302 1303 1304 1305 1306 1307
			!strncmp(hdr.oem_table_id, v_info->oem_table_id,
						ACPI_OEM_TABLE_ID_SIZE))
			switch (v_info->oem_pwr_table) {
			case PSS:
				return intel_pstate_no_acpi_pss();
			case PPC:
1308 1309
				return intel_pstate_has_acpi_ppc() &&
					(!force_load);
1310
			}
1311 1312 1313 1314 1315 1316
	}

	return false;
}
#else /* CONFIG_ACPI not enabled */
static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
1317
static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
1318 1319
#endif /* CONFIG_ACPI */

1320 1321
static int __init intel_pstate_init(void)
{
1322
	int cpu, rc = 0;
1323
	const struct x86_cpu_id *id;
1324
	struct cpu_defaults *cpu_def;
1325

1326 1327 1328
	if (no_load)
		return -ENODEV;

1329 1330 1331 1332
	id = x86_match_cpu(intel_pstate_cpu_ids);
	if (!id)
		return -ENODEV;

1333 1334 1335 1336 1337 1338 1339
	/*
	 * The Intel pstate driver will be ignored if the platform
	 * firmware has its own power management modes.
	 */
	if (intel_pstate_platform_pwr_mgmt_exists())
		return -ENODEV;

1340
	cpu_def = (struct cpu_defaults *)id->driver_data;
1341

1342 1343
	copy_pid_params(&cpu_def->pid_policy);
	copy_cpu_funcs(&cpu_def->funcs);
1344

1345 1346 1347
	if (intel_pstate_msrs_not_valid())
		return -ENODEV;

1348 1349
	pr_info("Intel P-state driver initializing.\n");

1350
	all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
1351 1352 1353
	if (!all_cpu_data)
		return -ENOMEM;

1354 1355
	if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp) {
		pr_info("intel_pstate: HWP enabled\n");
1356
		hwp_active++;
1357
	}
D
Dirk Brandewie 已提交
1358

1359 1360 1361
	if (!hwp_active && hwp_only)
		goto out;

1362 1363 1364 1365 1366 1367
	rc = cpufreq_register_driver(&intel_pstate_driver);
	if (rc)
		goto out;

	intel_pstate_debug_expose_params();
	intel_pstate_sysfs_expose_params();
1368

1369 1370
	return rc;
out:
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
	get_online_cpus();
	for_each_online_cpu(cpu) {
		if (all_cpu_data[cpu]) {
			del_timer_sync(&all_cpu_data[cpu]->timer);
			kfree(all_cpu_data[cpu]);
		}
	}

	put_online_cpus();
	vfree(all_cpu_data);
1381 1382 1383 1384
	return -ENODEV;
}
device_initcall(intel_pstate_init);

1385 1386 1387 1388 1389 1390 1391
static int __init intel_pstate_setup(char *str)
{
	if (!str)
		return -EINVAL;

	if (!strcmp(str, "disable"))
		no_load = 1;
1392 1393
	if (!strcmp(str, "no_hwp")) {
		pr_info("intel_pstate: HWP disabled\n");
D
Dirk Brandewie 已提交
1394
		no_hwp = 1;
1395
	}
1396 1397
	if (!strcmp(str, "force"))
		force_load = 1;
1398 1399
	if (!strcmp(str, "hwp_only"))
		hwp_only = 1;
1400 1401 1402 1403
	return 0;
}
early_param("intel_pstate", intel_pstate_setup);

1404 1405 1406
MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
MODULE_LICENSE("GPL");