intel_pstate.c 55.2 KB
Newer Older
1
/*
2
 * intel_pstate.c: Native P state management for Intel processors
3 4 5 6 7 8 9 10 11 12
 *
 * (C) Copyright 2012 Intel Corporation
 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

J
Joe Perches 已提交
13 14
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/cpu.h>
#include <linux/cpufreq.h>
#include <linux/sysfs.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/debugfs.h>
30
#include <linux/acpi.h>
31
#include <linux/vmalloc.h>
32 33 34 35 36
#include <trace/events/power.h>

#include <asm/div64.h>
#include <asm/msr.h>
#include <asm/cpu_device_id.h>
37
#include <asm/cpufeature.h>
38
#include <asm/intel-family.h>
39

40 41
#define INTEL_CPUFREQ_TRANSITION_LATENCY	20000

42 43 44 45
#define ATOM_RATIOS		0x66a
#define ATOM_VIDS		0x66b
#define ATOM_TURBO_RATIOS	0x66c
#define ATOM_TURBO_VIDS		0x66d
46

47 48 49 50
#ifdef CONFIG_ACPI
#include <acpi/processor.h>
#endif

51
#define FRAC_BITS 8
52 53
#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
#define fp_toint(X) ((X) >> FRAC_BITS)
54

55 56 57
#define EXT_BITS 6
#define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)

58 59 60 61 62
static inline int32_t mul_fp(int32_t x, int32_t y)
{
	return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
}

63
static inline int32_t div_fp(s64 x, s64 y)
64
{
65
	return div64_s64((int64_t)x << FRAC_BITS, y);
66 67
}

68 69 70 71 72 73 74 75 76 77 78
static inline int ceiling_fp(int32_t x)
{
	int mask, ret;

	ret = fp_toint(x);
	mask = (1 << FRAC_BITS) - 1;
	if (x & mask)
		ret += 1;
	return ret;
}

79 80 81 82 83 84 85 86 87 88
static inline u64 mul_ext_fp(u64 x, u64 y)
{
	return (x * y) >> EXT_FRAC_BITS;
}

static inline u64 div_ext_fp(u64 x, u64 y)
{
	return div64_u64(x << EXT_FRAC_BITS, y);
}

89 90
/**
 * struct sample -	Store performance sample
91
 * @core_avg_perf:	Ratio of APERF/MPERF which is the actual average
92 93
 *			performance during last sample period
 * @busy_scaled:	Scaled busy value which is used to calculate next
94
 *			P state. This can be different than core_avg_perf
95 96 97 98 99 100 101 102 103 104 105 106
 *			to account for cpu idle period
 * @aperf:		Difference of actual performance frequency clock count
 *			read from APERF MSR between last and current sample
 * @mperf:		Difference of maximum performance frequency clock count
 *			read from MPERF MSR between last and current sample
 * @tsc:		Difference of time stamp counter between last and
 *			current sample
 * @time:		Current time from scheduler
 *
 * This structure is used in the cpudata structure to store performance sample
 * data for choosing next P State.
 */
107
struct sample {
108
	int32_t core_avg_perf;
109
	int32_t busy_scaled;
110 111
	u64 aperf;
	u64 mperf;
112
	u64 tsc;
113
	u64 time;
114 115
};

116 117 118 119 120 121 122 123 124 125 126
/**
 * struct pstate_data - Store P state data
 * @current_pstate:	Current requested P state
 * @min_pstate:		Min P state possible for this platform
 * @max_pstate:		Max P state possible for this platform
 * @max_pstate_physical:This is physical Max P state for a processor
 *			This can be higher than the max_pstate which can
 *			be limited by platform thermal design power limits
 * @scaling:		Scaling factor to  convert frequency to cpufreq
 *			frequency units
 * @turbo_pstate:	Max Turbo P state possible for this platform
127 128
 * @max_freq:		@max_pstate frequency in cpufreq units
 * @turbo_freq:		@turbo_pstate frequency in cpufreq units
129 130 131
 *
 * Stores the per cpu model P state limits and current P state.
 */
132 133 134 135
struct pstate_data {
	int	current_pstate;
	int	min_pstate;
	int	max_pstate;
136
	int	max_pstate_physical;
137
	int	scaling;
138
	int	turbo_pstate;
139 140
	unsigned int max_freq;
	unsigned int turbo_freq;
141 142
};

143 144 145 146 147 148 149 150 151 152 153 154 155
/**
 * struct vid_data -	Stores voltage information data
 * @min:		VID data for this platform corresponding to
 *			the lowest P state
 * @max:		VID data corresponding to the highest P State.
 * @turbo:		VID data for turbo P state
 * @ratio:		Ratio of (vid max - vid min) /
 *			(max P state - Min P State)
 *
 * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
 * This data is used in Atom platforms, where in addition to target P state,
 * the voltage data needs to be specified to select next P State.
 */
156
struct vid_data {
157 158 159
	int min;
	int max;
	int turbo;
160 161 162
	int32_t ratio;
};

163 164 165 166 167 168 169 170 171 172 173 174
/**
 * struct _pid -	Stores PID data
 * @setpoint:		Target set point for busyness or performance
 * @integral:		Storage for accumulated error values
 * @p_gain:		PID proportional gain
 * @i_gain:		PID integral gain
 * @d_gain:		PID derivative gain
 * @deadband:		PID deadband
 * @last_err:		Last error storage for integral part of PID calculation
 *
 * Stores PID coefficients and last error for PID controller.
 */
175 176 177 178 179 180 181
struct _pid {
	int setpoint;
	int32_t integral;
	int32_t p_gain;
	int32_t i_gain;
	int32_t d_gain;
	int deadband;
182
	int32_t last_err;
183 184
};

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
/**
 * struct perf_limits - Store user and policy limits
 * @no_turbo:		User requested turbo state from intel_pstate sysfs
 * @turbo_disabled:	Platform turbo status either from msr
 *			MSR_IA32_MISC_ENABLE or when maximum available pstate
 *			matches the maximum turbo pstate
 * @max_perf_pct:	Effective maximum performance limit in percentage, this
 *			is minimum of either limits enforced by cpufreq policy
 *			or limits from user set limits via intel_pstate sysfs
 * @min_perf_pct:	Effective minimum performance limit in percentage, this
 *			is maximum of either limits enforced by cpufreq policy
 *			or limits from user set limits via intel_pstate sysfs
 * @max_perf:		This is a scaled value between 0 to 255 for max_perf_pct
 *			This value is used to limit max pstate
 * @min_perf:		This is a scaled value between 0 to 255 for min_perf_pct
 *			This value is used to limit min pstate
 * @max_policy_pct:	The maximum performance in percentage enforced by
 *			cpufreq setpolicy interface
 * @max_sysfs_pct:	The maximum performance in percentage enforced by
 *			intel pstate sysfs interface, unused when per cpu
 *			controls are enforced
 * @min_policy_pct:	The minimum performance in percentage enforced by
 *			cpufreq setpolicy interface
 * @min_sysfs_pct:	The minimum performance in percentage enforced by
 *			intel pstate sysfs interface, unused when per cpu
 *			controls are enforced
 *
 * Storage for user and policy defined limits.
 */
struct perf_limits {
	int no_turbo;
	int turbo_disabled;
	int max_perf_pct;
	int min_perf_pct;
	int32_t max_perf;
	int32_t min_perf;
	int max_policy_pct;
	int max_sysfs_pct;
	int min_policy_pct;
	int min_sysfs_pct;
};

227 228 229
/**
 * struct cpudata -	Per CPU instance data storage
 * @cpu:		CPU number for this instance data
230
 * @policy:		CPUFreq policy value
231
 * @update_util:	CPUFreq utility callback information
232
 * @update_util_set:	CPUFreq utility callback is set
233 234
 * @iowait_boost:	iowait-related boost fraction
 * @last_update:	Time of the last update.
235 236 237 238 239 240 241 242 243 244
 * @pstate:		Stores P state limits for this CPU
 * @vid:		Stores VID limits for this CPU
 * @pid:		Stores PID parameters for this CPU
 * @last_sample_time:	Last Sample time
 * @prev_aperf:		Last APERF value read from APERF MSR
 * @prev_mperf:		Last MPERF value read from MPERF MSR
 * @prev_tsc:		Last timestamp counter (TSC) value
 * @prev_cummulative_iowait: IO Wait time difference from last and
 *			current sample
 * @sample:		Storage for storing last Sample data
245 246 247
 * @perf_limits:	Pointer to perf_limit unique to this CPU
 *			Not all field in the structure are applicable
 *			when per cpu controls are enforced
248 249
 * @acpi_perf_data:	Stores ACPI perf information read from _PSS
 * @valid_pss_table:	Set to true for valid ACPI _PSS entries found
250 251 252
 *
 * This structure stores per CPU instance data for all CPUs.
 */
253 254 255
struct cpudata {
	int cpu;

256
	unsigned int policy;
257
	struct update_util_data update_util;
258
	bool   update_util_set;
259 260

	struct pstate_data pstate;
261
	struct vid_data vid;
262 263
	struct _pid pid;

264
	u64	last_update;
265
	u64	last_sample_time;
266 267
	u64	prev_aperf;
	u64	prev_mperf;
268
	u64	prev_tsc;
269
	u64	prev_cummulative_iowait;
270
	struct sample sample;
271
	struct perf_limits *perf_limits;
272 273 274 275
#ifdef CONFIG_ACPI
	struct acpi_processor_performance acpi_perf_data;
	bool valid_pss_table;
#endif
276
	unsigned int iowait_boost;
277 278 279
};

static struct cpudata **all_cpu_data;
280 281

/**
282
 * struct pstate_adjust_policy - Stores static PID configuration data
283 284 285 286 287 288 289 290 291 292
 * @sample_rate_ms:	PID calculation sample rate in ms
 * @sample_rate_ns:	Sample rate calculation in ns
 * @deadband:		PID deadband
 * @setpoint:		PID Setpoint
 * @p_gain_pct:		PID proportional gain
 * @i_gain_pct:		PID integral gain
 * @d_gain_pct:		PID derivative gain
 *
 * Stores per CPU model static PID configuration data.
 */
293 294
struct pstate_adjust_policy {
	int sample_rate_ms;
295
	s64 sample_rate_ns;
296 297 298 299 300 301 302
	int deadband;
	int setpoint;
	int p_gain_pct;
	int d_gain_pct;
	int i_gain_pct;
};

303 304 305 306 307 308 309 310 311 312 313 314 315 316
/**
 * struct pstate_funcs - Per CPU model specific callbacks
 * @get_max:		Callback to get maximum non turbo effective P state
 * @get_max_physical:	Callback to get maximum non turbo physical P state
 * @get_min:		Callback to get minimum P state
 * @get_turbo:		Callback to get turbo P state
 * @get_scaling:	Callback to get frequency scaling factor
 * @get_val:		Callback to convert P state to actual MSR write value
 * @get_vid:		Callback to get VID data for Atom platforms
 * @get_target_pstate:	Callback to a function to calculate next P state to use
 *
 * Core and Atom CPU models have different way to get P State limits. This
 * structure is used to store those callbacks.
 */
317 318
struct pstate_funcs {
	int (*get_max)(void);
319
	int (*get_max_physical)(void);
320 321
	int (*get_min)(void);
	int (*get_turbo)(void);
322
	int (*get_scaling)(void);
323
	u64 (*get_val)(struct cpudata*, int pstate);
324
	void (*get_vid)(struct cpudata *);
325
	int32_t (*get_target_pstate)(struct cpudata *);
326 327
};

328 329 330 331 332
/**
 * struct cpu_defaults- Per CPU model default config data
 * @pid_policy:	PID config data
 * @funcs:		Callback function data
 */
333 334 335
struct cpu_defaults {
	struct pstate_adjust_policy pid_policy;
	struct pstate_funcs funcs;
336 337
};

338
static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu);
339
static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
340

341 342 343
static struct pstate_adjust_policy pid_params __read_mostly;
static struct pstate_funcs pstate_funcs __read_mostly;
static int hwp_active __read_mostly;
344
static bool per_cpu_limits __read_mostly;
345

346 347 348
#ifdef CONFIG_ACPI
static bool acpi_ppc;
#endif
349

350 351 352 353 354 355 356 357 358 359 360 361 362 363
static struct perf_limits performance_limits = {
	.no_turbo = 0,
	.turbo_disabled = 0,
	.max_perf_pct = 100,
	.max_perf = int_tofp(1),
	.min_perf_pct = 100,
	.min_perf = int_tofp(1),
	.max_policy_pct = 100,
	.max_sysfs_pct = 100,
	.min_policy_pct = 0,
	.min_sysfs_pct = 0,
};

static struct perf_limits powersave_limits = {
364
	.no_turbo = 0,
365
	.turbo_disabled = 0,
366 367 368 369
	.max_perf_pct = 100,
	.max_perf = int_tofp(1),
	.min_perf_pct = 0,
	.min_perf = 0,
370 371
	.max_policy_pct = 100,
	.max_sysfs_pct = 100,
372 373
	.min_policy_pct = 0,
	.min_sysfs_pct = 0,
374 375
};

376 377 378 379 380 381
#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
static struct perf_limits *limits = &performance_limits;
#else
static struct perf_limits *limits = &powersave_limits;
#endif

382 383
static DEFINE_MUTEX(intel_pstate_limits_lock);

384
#ifdef CONFIG_ACPI
385 386 387 388 389 390 391 392 393 394

static bool intel_pstate_get_ppc_enable_status(void)
{
	if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
	    acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
		return true;

	return acpi_ppc;
}

395 396 397 398 399 400
static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
{
	struct cpudata *cpu;
	int ret;
	int i;

401 402 403
	if (hwp_active)
		return;

404
	if (!intel_pstate_get_ppc_enable_status())
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
		return;

	cpu = all_cpu_data[policy->cpu];

	ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
						  policy->cpu);
	if (ret)
		return;

	/*
	 * Check if the control value in _PSS is for PERF_CTL MSR, which should
	 * guarantee that the states returned by it map to the states in our
	 * list directly.
	 */
	if (cpu->acpi_perf_data.control_register.space_id !=
						ACPI_ADR_SPACE_FIXED_HARDWARE)
		goto err;

	/*
	 * If there is only one entry _PSS, simply ignore _PSS and continue as
	 * usual without taking _PSS into account
	 */
	if (cpu->acpi_perf_data.state_count < 2)
		goto err;

	pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
	for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
		pr_debug("     %cP%d: %u MHz, %u mW, 0x%x\n",
			 (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
			 (u32) cpu->acpi_perf_data.states[i].core_frequency,
			 (u32) cpu->acpi_perf_data.states[i].power,
			 (u32) cpu->acpi_perf_data.states[i].control);
	}

	/*
	 * The _PSS table doesn't contain whole turbo frequency range.
	 * This just contains +1 MHZ above the max non turbo frequency,
	 * with control value corresponding to max turbo ratio. But
	 * when cpufreq set policy is called, it will call with this
	 * max frequency, which will cause a reduced performance as
	 * this driver uses real max turbo frequency as the max
	 * frequency. So correct this frequency in _PSS table to
447
	 * correct max turbo frequency based on the turbo state.
448 449
	 * Also need to convert to MHz as _PSS freq is in MHz.
	 */
450
	if (!limits->turbo_disabled)
451 452 453
		cpu->acpi_perf_data.states[0].core_frequency =
					policy->cpuinfo.max_freq / 1000;
	cpu->valid_pss_table = true;
454
	pr_debug("_PPC limits will be enforced\n");
455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478

	return;

 err:
	cpu->valid_pss_table = false;
	acpi_processor_unregister_performance(policy->cpu);
}

static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
{
	struct cpudata *cpu;

	cpu = all_cpu_data[policy->cpu];
	if (!cpu->valid_pss_table)
		return;

	acpi_processor_unregister_performance(policy->cpu);
}

#else
static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
{
}

479
static inline int intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
480 481 482 483
{
}
#endif

484
static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
485
			     int deadband, int integral) {
486 487
	pid->setpoint = int_tofp(setpoint);
	pid->deadband  = int_tofp(deadband);
488
	pid->integral  = int_tofp(integral);
489
	pid->last_err  = int_tofp(setpoint) - int_tofp(busy);
490 491 492 493
}

static inline void pid_p_gain_set(struct _pid *pid, int percent)
{
494
	pid->p_gain = div_fp(percent, 100);
495 496 497 498
}

static inline void pid_i_gain_set(struct _pid *pid, int percent)
{
499
	pid->i_gain = div_fp(percent, 100);
500 501 502 503
}

static inline void pid_d_gain_set(struct _pid *pid, int percent)
{
504
	pid->d_gain = div_fp(percent, 100);
505 506
}

507
static signed int pid_calc(struct _pid *pid, int32_t busy)
508
{
509
	signed int result;
510 511 512
	int32_t pterm, dterm, fp_error;
	int32_t integral_limit;

513
	fp_error = pid->setpoint - busy;
514

515
	if (abs(fp_error) <= pid->deadband)
516 517 518 519 520 521
		return 0;

	pterm = mul_fp(pid->p_gain, fp_error);

	pid->integral += fp_error;

522 523 524 525 526 527 528 529
	/*
	 * We limit the integral here so that it will never
	 * get higher than 30.  This prevents it from becoming
	 * too large an input over long periods of time and allows
	 * it to get factored out sooner.
	 *
	 * The value of 30 was chosen through experimentation.
	 */
530 531 532 533 534 535
	integral_limit = int_tofp(30);
	if (pid->integral > integral_limit)
		pid->integral = integral_limit;
	if (pid->integral < -integral_limit)
		pid->integral = -integral_limit;

536 537
	dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
	pid->last_err = fp_error;
538 539

	result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
540
	result = result + (1 << (FRAC_BITS-1));
541 542 543 544 545
	return (signed int)fp_toint(result);
}

static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
{
546 547 548
	pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
	pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
	pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
549

550
	pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
551 552 553 554 555
}

static inline void intel_pstate_reset_all_pid(void)
{
	unsigned int cpu;
556

557 558 559 560 561 562
	for_each_online_cpu(cpu) {
		if (all_cpu_data[cpu])
			intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
	}
}

563 564 565 566 567 568 569
static inline void update_turbo_state(void)
{
	u64 misc_en;
	struct cpudata *cpu;

	cpu = all_cpu_data[0];
	rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
570
	limits->turbo_disabled =
571 572 573 574
		(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
		 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
}

575
static void intel_pstate_hwp_set(const struct cpumask *cpumask)
D
Dirk Brandewie 已提交
576
{
577
	int min, hw_min, max, hw_max, cpu, range, adj_range;
578
	struct perf_limits *perf_limits = limits;
579 580
	u64 value, cap;

581
	for_each_cpu(cpu, cpumask) {
582 583 584 585 586
		int max_perf_pct, min_perf_pct;

		if (per_cpu_limits)
			perf_limits = all_cpu_data[cpu]->perf_limits;

587 588 589 590 591
		rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
		hw_min = HWP_LOWEST_PERF(cap);
		hw_max = HWP_HIGHEST_PERF(cap);
		range = hw_max - hw_min;

592 593 594
		max_perf_pct = perf_limits->max_perf_pct;
		min_perf_pct = perf_limits->min_perf_pct;

D
Dirk Brandewie 已提交
595
		rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
596
		adj_range = min_perf_pct * range / 100;
597
		min = hw_min + adj_range;
D
Dirk Brandewie 已提交
598 599 600
		value &= ~HWP_MIN_PERF(~0L);
		value |= HWP_MIN_PERF(min);

601
		adj_range = max_perf_pct * range / 100;
602
		max = hw_min + adj_range;
603
		if (limits->no_turbo) {
604 605 606
			hw_max = HWP_GUARANTEED_PERF(cap);
			if (hw_max < max)
				max = hw_max;
D
Dirk Brandewie 已提交
607 608 609 610 611 612
		}

		value &= ~HWP_MAX_PERF(~0L);
		value |= HWP_MAX_PERF(max);
		wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
	}
613
}
D
Dirk Brandewie 已提交
614

615 616 617 618 619 620 621 622
static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy)
{
	if (hwp_active)
		intel_pstate_hwp_set(policy->cpus);

	return 0;
}

623 624 625 626
static void intel_pstate_hwp_set_online_cpus(void)
{
	get_online_cpus();
	intel_pstate_hwp_set(cpu_online_mask);
D
Dirk Brandewie 已提交
627 628 629
	put_online_cpus();
}

630 631 632 633 634 635 636
/************************** debugfs begin ************************/
static int pid_param_set(void *data, u64 val)
{
	*(u32 *)data = val;
	intel_pstate_reset_all_pid();
	return 0;
}
637

638 639 640 641 642
static int pid_param_get(void *data, u64 *val)
{
	*val = *(u32 *)data;
	return 0;
}
643
DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
644 645 646 647 648 649 650

struct pid_param {
	char *name;
	void *value;
};

static struct pid_param pid_files[] = {
651 652 653 654 655 656
	{"sample_rate_ms", &pid_params.sample_rate_ms},
	{"d_gain_pct", &pid_params.d_gain_pct},
	{"i_gain_pct", &pid_params.i_gain_pct},
	{"deadband", &pid_params.deadband},
	{"setpoint", &pid_params.setpoint},
	{"p_gain_pct", &pid_params.p_gain_pct},
657 658 659
	{NULL, NULL}
};

660
static void __init intel_pstate_debug_expose_params(void)
661
{
662
	struct dentry *debugfs_parent;
663 664
	int i = 0;

665 666
	if (hwp_active ||
	    pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load)
D
Dirk Brandewie 已提交
667
		return;
668

669 670 671 672 673
	debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
	if (IS_ERR_OR_NULL(debugfs_parent))
		return;
	while (pid_files[i].name) {
		debugfs_create_file(pid_files[i].name, 0660,
674 675
				    debugfs_parent, pid_files[i].value,
				    &fops_pid_param);
676 677 678 679 680 681 682 683 684 685 686
		i++;
	}
}

/************************** debugfs end ************************/

/************************** sysfs begin ************************/
#define show_one(file_name, object)					\
	static ssize_t show_##file_name					\
	(struct kobject *kobj, struct attribute *attr, char *buf)	\
	{								\
687
		return sprintf(buf, "%u\n", limits->object);		\
688 689
	}

690 691 692 693 694 695 696 697 698 699 700
static ssize_t show_turbo_pct(struct kobject *kobj,
				struct attribute *attr, char *buf)
{
	struct cpudata *cpu;
	int total, no_turbo, turbo_pct;
	uint32_t turbo_fp;

	cpu = all_cpu_data[0];

	total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
	no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
701
	turbo_fp = div_fp(no_turbo, total);
702 703 704 705
	turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
	return sprintf(buf, "%u\n", turbo_pct);
}

706 707 708 709 710 711 712 713 714 715 716
static ssize_t show_num_pstates(struct kobject *kobj,
				struct attribute *attr, char *buf)
{
	struct cpudata *cpu;
	int total;

	cpu = all_cpu_data[0];
	total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
	return sprintf(buf, "%u\n", total);
}

717 718 719 720 721 722
static ssize_t show_no_turbo(struct kobject *kobj,
			     struct attribute *attr, char *buf)
{
	ssize_t ret;

	update_turbo_state();
723 724
	if (limits->turbo_disabled)
		ret = sprintf(buf, "%u\n", limits->turbo_disabled);
725
	else
726
		ret = sprintf(buf, "%u\n", limits->no_turbo);
727 728 729 730

	return ret;
}

731
static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
732
			      const char *buf, size_t count)
733 734 735
{
	unsigned int input;
	int ret;
736

737 738 739
	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;
740

741 742
	mutex_lock(&intel_pstate_limits_lock);

743
	update_turbo_state();
744
	if (limits->turbo_disabled) {
J
Joe Perches 已提交
745
		pr_warn("Turbo disabled by BIOS or unavailable on processor\n");
746
		mutex_unlock(&intel_pstate_limits_lock);
747
		return -EPERM;
748
	}
D
Dirk Brandewie 已提交
749

750
	limits->no_turbo = clamp_t(int, input, 0, 1);
751

752 753
	mutex_unlock(&intel_pstate_limits_lock);

D
Dirk Brandewie 已提交
754
	if (hwp_active)
755
		intel_pstate_hwp_set_online_cpus();
D
Dirk Brandewie 已提交
756

757 758 759 760
	return count;
}

static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
761
				  const char *buf, size_t count)
762 763 764
{
	unsigned int input;
	int ret;
765

766 767 768 769
	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;

770 771
	mutex_lock(&intel_pstate_limits_lock);

772 773 774 775 776 777 778
	limits->max_sysfs_pct = clamp_t(int, input, 0 , 100);
	limits->max_perf_pct = min(limits->max_policy_pct,
				   limits->max_sysfs_pct);
	limits->max_perf_pct = max(limits->min_policy_pct,
				   limits->max_perf_pct);
	limits->max_perf_pct = max(limits->min_perf_pct,
				   limits->max_perf_pct);
779
	limits->max_perf = div_fp(limits->max_perf_pct, 100);
780

781 782
	mutex_unlock(&intel_pstate_limits_lock);

D
Dirk Brandewie 已提交
783
	if (hwp_active)
784
		intel_pstate_hwp_set_online_cpus();
785 786 787 788
	return count;
}

static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
789
				  const char *buf, size_t count)
790 791 792
{
	unsigned int input;
	int ret;
793

794 795 796
	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;
797

798 799
	mutex_lock(&intel_pstate_limits_lock);

800 801 802 803 804 805 806
	limits->min_sysfs_pct = clamp_t(int, input, 0 , 100);
	limits->min_perf_pct = max(limits->min_policy_pct,
				   limits->min_sysfs_pct);
	limits->min_perf_pct = min(limits->max_policy_pct,
				   limits->min_perf_pct);
	limits->min_perf_pct = min(limits->max_perf_pct,
				   limits->min_perf_pct);
807
	limits->min_perf = div_fp(limits->min_perf_pct, 100);
808

809 810
	mutex_unlock(&intel_pstate_limits_lock);

D
Dirk Brandewie 已提交
811
	if (hwp_active)
812
		intel_pstate_hwp_set_online_cpus();
813 814 815 816 817 818 819 820 821
	return count;
}

show_one(max_perf_pct, max_perf_pct);
show_one(min_perf_pct, min_perf_pct);

define_one_global_rw(no_turbo);
define_one_global_rw(max_perf_pct);
define_one_global_rw(min_perf_pct);
822
define_one_global_ro(turbo_pct);
823
define_one_global_ro(num_pstates);
824 825 826

static struct attribute *intel_pstate_attributes[] = {
	&no_turbo.attr,
827
	&turbo_pct.attr,
828
	&num_pstates.attr,
829 830 831 832 833 834 835
	NULL
};

static struct attribute_group intel_pstate_attr_group = {
	.attrs = intel_pstate_attributes,
};

836
static void __init intel_pstate_sysfs_expose_params(void)
837
{
838
	struct kobject *intel_pstate_kobject;
839 840 841 842
	int rc;

	intel_pstate_kobject = kobject_create_and_add("intel_pstate",
						&cpu_subsys.dev_root->kobj);
843 844 845
	if (WARN_ON(!intel_pstate_kobject))
		return;

846
	rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
	if (WARN_ON(rc))
		return;

	/*
	 * If per cpu limits are enforced there are no global limits, so
	 * return without creating max/min_perf_pct attributes
	 */
	if (per_cpu_limits)
		return;

	rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
	WARN_ON(rc);

	rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
	WARN_ON(rc);

863 864
}
/************************** sysfs end ************************/
D
Dirk Brandewie 已提交
865

866
static void intel_pstate_hwp_enable(struct cpudata *cpudata)
D
Dirk Brandewie 已提交
867
{
868
	/* First disable HWP notification interrupt as we don't process them */
869 870
	if (static_cpu_has(X86_FEATURE_HWP_NOTIFY))
		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
871

872
	wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
D
Dirk Brandewie 已提交
873 874
}

875
static int atom_get_min_pstate(void)
876 877
{
	u64 value;
878

879
	rdmsrl(ATOM_RATIOS, value);
D
Dirk Brandewie 已提交
880
	return (value >> 8) & 0x7F;
881 882
}

883
static int atom_get_max_pstate(void)
884 885
{
	u64 value;
886

887
	rdmsrl(ATOM_RATIOS, value);
D
Dirk Brandewie 已提交
888
	return (value >> 16) & 0x7F;
889
}
890

891
static int atom_get_turbo_pstate(void)
892 893
{
	u64 value;
894

895
	rdmsrl(ATOM_TURBO_RATIOS, value);
D
Dirk Brandewie 已提交
896
	return value & 0x7F;
897 898
}

899
static u64 atom_get_val(struct cpudata *cpudata, int pstate)
900 901 902 903 904
{
	u64 val;
	int32_t vid_fp;
	u32 vid;

905
	val = (u64)pstate << 8;
906
	if (limits->no_turbo && !limits->turbo_disabled)
907 908 909 910 911 912 913
		val |= (u64)1 << 32;

	vid_fp = cpudata->vid.min + mul_fp(
		int_tofp(pstate - cpudata->pstate.min_pstate),
		cpudata->vid.ratio);

	vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
914
	vid = ceiling_fp(vid_fp);
915

916 917 918
	if (pstate > cpudata->pstate.max_pstate)
		vid = cpudata->vid.turbo;

919
	return val | vid;
920 921
}

922
static int silvermont_get_scaling(void)
923 924 925
{
	u64 value;
	int i;
926 927 928
	/* Defined in Table 35-6 from SDM (Sept 2015) */
	static int silvermont_freq_table[] = {
		83300, 100000, 133300, 116700, 80000};
929 930

	rdmsrl(MSR_FSB_FREQ, value);
931 932
	i = value & 0x7;
	WARN_ON(i > 4);
933

934 935
	return silvermont_freq_table[i];
}
936

937 938 939 940 941 942 943 944 945 946 947 948 949 950
static int airmont_get_scaling(void)
{
	u64 value;
	int i;
	/* Defined in Table 35-10 from SDM (Sept 2015) */
	static int airmont_freq_table[] = {
		83300, 100000, 133300, 116700, 80000,
		93300, 90000, 88900, 87500};

	rdmsrl(MSR_FSB_FREQ, value);
	i = value & 0xF;
	WARN_ON(i > 8);

	return airmont_freq_table[i];
951 952
}

953
static void atom_get_vid(struct cpudata *cpudata)
954 955 956
{
	u64 value;

957
	rdmsrl(ATOM_VIDS, value);
D
Dirk Brandewie 已提交
958 959
	cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
	cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
960 961 962 963
	cpudata->vid.ratio = div_fp(
		cpudata->vid.max - cpudata->vid.min,
		int_tofp(cpudata->pstate.max_pstate -
			cpudata->pstate.min_pstate));
964

965
	rdmsrl(ATOM_TURBO_VIDS, value);
966
	cpudata->vid.turbo = value & 0x7f;
967 968
}

969
static int core_get_min_pstate(void)
970 971
{
	u64 value;
972

973
	rdmsrl(MSR_PLATFORM_INFO, value);
974 975 976
	return (value >> 40) & 0xFF;
}

977
static int core_get_max_pstate_physical(void)
978 979
{
	u64 value;
980

981
	rdmsrl(MSR_PLATFORM_INFO, value);
982 983 984
	return (value >> 8) & 0xFF;
}

985
static int core_get_max_pstate(void)
986
{
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
	u64 tar;
	u64 plat_info;
	int max_pstate;
	int err;

	rdmsrl(MSR_PLATFORM_INFO, plat_info);
	max_pstate = (plat_info >> 8) & 0xFF;

	err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
	if (!err) {
		/* Do some sanity checking for safety */
		if (plat_info & 0x600000000) {
			u64 tdp_ctrl;
			u64 tdp_ratio;
			int tdp_msr;

			err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
			if (err)
				goto skip_tar;

1007
			tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x3);
1008 1009 1010 1011
			err = rdmsrl_safe(tdp_msr, &tdp_ratio);
			if (err)
				goto skip_tar;

1012 1013 1014 1015 1016
			/* For level 1 and 2, bits[23:16] contain the ratio */
			if (tdp_ctrl)
				tdp_ratio >>= 16;

			tdp_ratio &= 0xff; /* ratios are only 8 bits long */
1017 1018 1019 1020 1021 1022 1023 1024
			if (tdp_ratio - 1 == tar) {
				max_pstate = tar;
				pr_debug("max_pstate=TAC %x\n", max_pstate);
			} else {
				goto skip_tar;
			}
		}
	}
1025

1026 1027
skip_tar:
	return max_pstate;
1028 1029
}

1030
static int core_get_turbo_pstate(void)
1031 1032 1033
{
	u64 value;
	int nont, ret;
1034

1035
	rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
1036
	nont = core_get_max_pstate();
1037
	ret = (value) & 255;
1038 1039 1040 1041 1042
	if (ret <= nont)
		ret = nont;
	return ret;
}

1043 1044 1045 1046 1047
static inline int core_get_scaling(void)
{
	return 100000;
}

1048
static u64 core_get_val(struct cpudata *cpudata, int pstate)
1049 1050 1051
{
	u64 val;

1052
	val = (u64)pstate << 8;
1053
	if (limits->no_turbo && !limits->turbo_disabled)
1054 1055
		val |= (u64)1 << 32;

1056
	return val;
1057 1058
}

1059 1060 1061 1062 1063
static int knl_get_turbo_pstate(void)
{
	u64 value;
	int nont, ret;

1064
	rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
1065 1066 1067 1068 1069 1070 1071
	nont = core_get_max_pstate();
	ret = (((value) >> 8) & 0xFF);
	if (ret <= nont)
		ret = nont;
	return ret;
}

1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
static struct cpu_defaults core_params = {
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
		.setpoint = 97,
		.p_gain_pct = 20,
		.d_gain_pct = 0,
		.i_gain_pct = 0,
	},
	.funcs = {
		.get_max = core_get_max_pstate,
1083
		.get_max_physical = core_get_max_pstate_physical,
1084 1085
		.get_min = core_get_min_pstate,
		.get_turbo = core_get_turbo_pstate,
1086
		.get_scaling = core_get_scaling,
1087
		.get_val = core_get_val,
1088
		.get_target_pstate = get_target_pstate_use_performance,
1089 1090 1091
	},
};

1092
static const struct cpu_defaults silvermont_params = {
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
		.setpoint = 60,
		.p_gain_pct = 14,
		.d_gain_pct = 0,
		.i_gain_pct = 4,
	},
	.funcs = {
		.get_max = atom_get_max_pstate,
		.get_max_physical = atom_get_max_pstate,
		.get_min = atom_get_min_pstate,
		.get_turbo = atom_get_turbo_pstate,
1106
		.get_val = atom_get_val,
1107 1108
		.get_scaling = silvermont_get_scaling,
		.get_vid = atom_get_vid,
1109
		.get_target_pstate = get_target_pstate_use_cpu_load,
1110 1111 1112
	},
};

1113
static const struct cpu_defaults airmont_params = {
1114 1115 1116
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
1117
		.setpoint = 60,
1118 1119 1120 1121 1122
		.p_gain_pct = 14,
		.d_gain_pct = 0,
		.i_gain_pct = 4,
	},
	.funcs = {
1123 1124 1125 1126
		.get_max = atom_get_max_pstate,
		.get_max_physical = atom_get_max_pstate,
		.get_min = atom_get_min_pstate,
		.get_turbo = atom_get_turbo_pstate,
1127
		.get_val = atom_get_val,
1128
		.get_scaling = airmont_get_scaling,
1129
		.get_vid = atom_get_vid,
1130
		.get_target_pstate = get_target_pstate_use_cpu_load,
1131 1132 1133
	},
};

1134
static const struct cpu_defaults knl_params = {
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
		.setpoint = 97,
		.p_gain_pct = 20,
		.d_gain_pct = 0,
		.i_gain_pct = 0,
	},
	.funcs = {
		.get_max = core_get_max_pstate,
1145
		.get_max_physical = core_get_max_pstate_physical,
1146 1147
		.get_min = core_get_min_pstate,
		.get_turbo = knl_get_turbo_pstate,
1148
		.get_scaling = core_get_scaling,
1149
		.get_val = core_get_val,
1150
		.get_target_pstate = get_target_pstate_use_performance,
1151 1152 1153
	},
};

1154
static const struct cpu_defaults bxt_params = {
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
	.pid_policy = {
		.sample_rate_ms = 10,
		.deadband = 0,
		.setpoint = 60,
		.p_gain_pct = 14,
		.d_gain_pct = 0,
		.i_gain_pct = 4,
	},
	.funcs = {
		.get_max = core_get_max_pstate,
		.get_max_physical = core_get_max_pstate_physical,
		.get_min = core_get_min_pstate,
		.get_turbo = core_get_turbo_pstate,
		.get_scaling = core_get_scaling,
		.get_val = core_get_val,
		.get_target_pstate = get_target_pstate_use_cpu_load,
	},
};

1174 1175 1176
static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
{
	int max_perf = cpu->pstate.turbo_pstate;
1177
	int max_perf_adj;
1178
	int min_perf;
1179
	struct perf_limits *perf_limits = limits;
1180

1181
	if (limits->no_turbo || limits->turbo_disabled)
1182 1183
		max_perf = cpu->pstate.max_pstate;

1184 1185 1186
	if (per_cpu_limits)
		perf_limits = cpu->perf_limits;

1187 1188 1189 1190 1191
	/*
	 * performance can be limited by user through sysfs, by cpufreq
	 * policy, or by cpu specific default values determined through
	 * experimentation.
	 */
1192
	max_perf_adj = fp_toint(max_perf * perf_limits->max_perf);
1193 1194
	*max = clamp_t(int, max_perf_adj,
			cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
1195

1196
	min_perf = fp_toint(max_perf * perf_limits->min_perf);
1197
	*min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
1198 1199
}

1200
static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
1201
{
1202 1203
	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
	cpu->pstate.current_pstate = pstate;
1204 1205 1206 1207 1208 1209 1210
	/*
	 * Generally, there is no guarantee that this code will always run on
	 * the CPU being updated, so force the register update to run on the
	 * right CPU.
	 */
	wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
		      pstate_funcs.get_val(cpu, pstate));
1211 1212
}

1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
static void intel_pstate_set_min_pstate(struct cpudata *cpu)
{
	intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
}

static void intel_pstate_max_within_limits(struct cpudata *cpu)
{
	int min_pstate, max_pstate;

	update_turbo_state();
	intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate);
	intel_pstate_set_pstate(cpu, max_pstate);
}

1227 1228
static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
{
1229 1230
	cpu->pstate.min_pstate = pstate_funcs.get_min();
	cpu->pstate.max_pstate = pstate_funcs.get_max();
1231
	cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
1232
	cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
1233
	cpu->pstate.scaling = pstate_funcs.get_scaling();
1234 1235
	cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling;
	cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1236

1237 1238
	if (pstate_funcs.get_vid)
		pstate_funcs.get_vid(cpu);
1239 1240

	intel_pstate_set_min_pstate(cpu);
1241 1242
}

1243
static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
1244
{
1245
	struct sample *sample = &cpu->sample;
1246

1247
	sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
1248 1249
}

1250
static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
1251 1252
{
	u64 aperf, mperf;
1253
	unsigned long flags;
1254
	u64 tsc;
1255

1256
	local_irq_save(flags);
1257 1258
	rdmsrl(MSR_IA32_APERF, aperf);
	rdmsrl(MSR_IA32_MPERF, mperf);
1259
	tsc = rdtsc();
1260
	if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
1261
		local_irq_restore(flags);
1262
		return false;
1263
	}
1264
	local_irq_restore(flags);
1265

1266
	cpu->last_sample_time = cpu->sample.time;
1267
	cpu->sample.time = time;
1268 1269
	cpu->sample.aperf = aperf;
	cpu->sample.mperf = mperf;
1270
	cpu->sample.tsc =  tsc;
1271 1272
	cpu->sample.aperf -= cpu->prev_aperf;
	cpu->sample.mperf -= cpu->prev_mperf;
1273
	cpu->sample.tsc -= cpu->prev_tsc;
1274

1275 1276
	cpu->prev_aperf = aperf;
	cpu->prev_mperf = mperf;
1277
	cpu->prev_tsc = tsc;
1278 1279 1280 1281 1282 1283 1284 1285
	/*
	 * First time this function is invoked in a given cycle, all of the
	 * previous sample data fields are equal to zero or stale and they must
	 * be populated with meaningful numbers for things to work, so assume
	 * that sample.time will always be reset before setting the utilization
	 * update hook and make the caller skip the sample then.
	 */
	return !!cpu->last_sample_time;
1286 1287
}

1288 1289
static inline int32_t get_avg_frequency(struct cpudata *cpu)
{
1290 1291
	return mul_ext_fp(cpu->sample.core_avg_perf,
			  cpu->pstate.max_pstate_physical * cpu->pstate.scaling);
1292 1293
}

1294 1295
static inline int32_t get_avg_pstate(struct cpudata *cpu)
{
1296 1297
	return mul_ext_fp(cpu->pstate.max_pstate_physical,
			  cpu->sample.core_avg_perf);
1298 1299
}

1300 1301 1302
static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
{
	struct sample *sample = &cpu->sample;
1303
	int32_t busy_frac, boost;
1304
	int target, avg_pstate;
1305

1306
	busy_frac = div_fp(sample->mperf, sample->tsc);
1307

1308 1309
	boost = cpu->iowait_boost;
	cpu->iowait_boost >>= 1;
1310

1311 1312
	if (busy_frac < boost)
		busy_frac = boost;
1313

1314
	sample->busy_scaled = busy_frac * 100;
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334

	target = limits->no_turbo || limits->turbo_disabled ?
			cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
	target += target >> 2;
	target = mul_fp(target, busy_frac);
	if (target < cpu->pstate.min_pstate)
		target = cpu->pstate.min_pstate;

	/*
	 * If the average P-state during the previous cycle was higher than the
	 * current target, add 50% of the difference to the target to reduce
	 * possible performance oscillations and offset possible performance
	 * loss related to moving the workload from one CPU to another within
	 * a package/module.
	 */
	avg_pstate = get_avg_pstate(cpu);
	if (avg_pstate > target)
		target += (avg_pstate - target) >> 1;

	return target;
1335 1336
}

1337
static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
1338
{
1339
	int32_t perf_scaled, max_pstate, current_pstate, sample_ratio;
1340
	u64 duration_ns;
1341

1342
	/*
1343 1344 1345 1346 1347
	 * perf_scaled is the ratio of the average P-state during the last
	 * sampling period to the P-state requested last time (in percent).
	 *
	 * That measures the system's response to the previous P-state
	 * selection.
1348
	 */
1349 1350
	max_pstate = cpu->pstate.max_pstate_physical;
	current_pstate = cpu->pstate.current_pstate;
1351
	perf_scaled = mul_ext_fp(cpu->sample.core_avg_perf,
1352
			       div_fp(100 * max_pstate, current_pstate));
1353

1354
	/*
1355 1356 1357
	 * Since our utilization update callback will not run unless we are
	 * in C0, check if the actual elapsed time is significantly greater (3x)
	 * than our sample interval.  If it is, then we were idle for a long
1358
	 * enough period of time to adjust our performance metric.
1359
	 */
1360
	duration_ns = cpu->sample.time - cpu->last_sample_time;
1361
	if ((s64)duration_ns > pid_params.sample_rate_ns * 3) {
1362
		sample_ratio = div_fp(pid_params.sample_rate_ns, duration_ns);
1363
		perf_scaled = mul_fp(perf_scaled, sample_ratio);
1364 1365 1366
	} else {
		sample_ratio = div_fp(100 * cpu->sample.mperf, cpu->sample.tsc);
		if (sample_ratio < int_tofp(1))
1367
			perf_scaled = 0;
1368 1369
	}

1370 1371
	cpu->sample.busy_scaled = perf_scaled;
	return cpu->pstate.current_pstate - pid_calc(&cpu->pid, perf_scaled);
1372 1373
}

1374
static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
1375 1376 1377 1378 1379
{
	int max_perf, min_perf;

	intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
	pstate = clamp_t(int, pstate, min_perf, max_perf);
1380
	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1381 1382 1383 1384 1385 1386
	return pstate;
}

static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
{
	pstate = intel_pstate_prepare_request(cpu, pstate);
1387 1388 1389
	if (pstate == cpu->pstate.current_pstate)
		return;

1390
	cpu->pstate.current_pstate = pstate;
1391 1392 1393
	wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
}

1394 1395
static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
{
1396
	int from, target_pstate;
1397 1398 1399
	struct sample *sample;

	from = cpu->pstate.current_pstate;
1400

1401 1402
	target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
		cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
1403

1404 1405
	update_turbo_state();

1406
	intel_pstate_update_pstate(cpu, target_pstate);
1407 1408

	sample = &cpu->sample;
1409
	trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
1410
		fp_toint(sample->busy_scaled),
1411 1412 1413 1414 1415
		from,
		cpu->pstate.current_pstate,
		sample->mperf,
		sample->aperf,
		sample->tsc,
1416 1417
		get_avg_frequency(cpu),
		fp_toint(cpu->iowait_boost * 100));
1418 1419
}

1420
static void intel_pstate_update_util(struct update_util_data *data, u64 time,
1421
				     unsigned int flags)
1422
{
1423
	struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1424 1425
	u64 delta_ns;

1426
	if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) {
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
		if (flags & SCHED_CPUFREQ_IOWAIT) {
			cpu->iowait_boost = int_tofp(1);
		} else if (cpu->iowait_boost) {
			/* Clear iowait_boost if the CPU may have been idle. */
			delta_ns = time - cpu->last_update;
			if (delta_ns > TICK_NSEC)
				cpu->iowait_boost = 0;
		}
		cpu->last_update = time;
	}
1437

1438
	delta_ns = time - cpu->sample.time;
1439
	if ((s64)delta_ns >= pid_params.sample_rate_ns) {
1440 1441
		bool sample_taken = intel_pstate_sample(cpu, time);

1442
		if (sample_taken) {
1443
			intel_pstate_calc_avg_perf(cpu);
1444 1445 1446
			if (!hwp_active)
				intel_pstate_adjust_busy_pstate(cpu);
		}
1447
	}
1448 1449 1450
}

#define ICPU(model, policy) \
1451 1452
	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
			(unsigned long)&policy }
1453 1454

static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
	ICPU(INTEL_FAM6_SANDYBRIDGE, 		core_params),
	ICPU(INTEL_FAM6_SANDYBRIDGE_X,		core_params),
	ICPU(INTEL_FAM6_ATOM_SILVERMONT1,	silvermont_params),
	ICPU(INTEL_FAM6_IVYBRIDGE,		core_params),
	ICPU(INTEL_FAM6_HASWELL_CORE,		core_params),
	ICPU(INTEL_FAM6_BROADWELL_CORE,		core_params),
	ICPU(INTEL_FAM6_IVYBRIDGE_X,		core_params),
	ICPU(INTEL_FAM6_HASWELL_X,		core_params),
	ICPU(INTEL_FAM6_HASWELL_ULT,		core_params),
	ICPU(INTEL_FAM6_HASWELL_GT3E,		core_params),
	ICPU(INTEL_FAM6_BROADWELL_GT3E,		core_params),
	ICPU(INTEL_FAM6_ATOM_AIRMONT,		airmont_params),
	ICPU(INTEL_FAM6_SKYLAKE_MOBILE,		core_params),
	ICPU(INTEL_FAM6_BROADWELL_X,		core_params),
	ICPU(INTEL_FAM6_SKYLAKE_DESKTOP,	core_params),
	ICPU(INTEL_FAM6_BROADWELL_XEON_D,	core_params),
	ICPU(INTEL_FAM6_XEON_PHI_KNL,		knl_params),
1472
	ICPU(INTEL_FAM6_ATOM_GOLDMONT,		bxt_params),
1473 1474 1475 1476
	{}
};
MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);

1477
static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1478
	ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params),
1479 1480
	ICPU(INTEL_FAM6_BROADWELL_X, core_params),
	ICPU(INTEL_FAM6_SKYLAKE_X, core_params),
D
Dirk Brandewie 已提交
1481 1482 1483
	{}
};

1484 1485 1486 1487
static int intel_pstate_init_cpu(unsigned int cpunum)
{
	struct cpudata *cpu;

1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504
	cpu = all_cpu_data[cpunum];

	if (!cpu) {
		unsigned int size = sizeof(struct cpudata);

		if (per_cpu_limits)
			size += sizeof(struct perf_limits);

		cpu = kzalloc(size, GFP_KERNEL);
		if (!cpu)
			return -ENOMEM;

		all_cpu_data[cpunum] = cpu;
		if (per_cpu_limits)
			cpu->perf_limits = (struct perf_limits *)(cpu + 1);

	}
1505 1506 1507 1508

	cpu = all_cpu_data[cpunum];

	cpu->cpu = cpunum;
1509

1510
	if (hwp_active) {
1511
		intel_pstate_hwp_enable(cpu);
1512 1513 1514
		pid_params.sample_rate_ms = 50;
		pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC;
	}
1515

1516
	intel_pstate_get_cpu_pstates(cpu);
1517

1518 1519
	intel_pstate_busy_pid_reset(cpu);

J
Joe Perches 已提交
1520
	pr_debug("controlling: cpu %d\n", cpunum);
1521 1522 1523 1524 1525 1526

	return 0;
}

static unsigned int intel_pstate_get(unsigned int cpu_num)
{
1527
	struct cpudata *cpu = all_cpu_data[cpu_num];
1528

1529
	return cpu ? get_avg_frequency(cpu) : 0;
1530 1531
}

1532
static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
1533
{
1534 1535
	struct cpudata *cpu = all_cpu_data[cpu_num];

1536 1537 1538
	if (cpu->update_util_set)
		return;

1539 1540
	/* Prevent intel_pstate_update_util() from using stale data. */
	cpu->sample.time = 0;
1541 1542
	cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
				     intel_pstate_update_util);
1543
	cpu->update_util_set = true;
1544 1545 1546 1547
}

static void intel_pstate_clear_update_util_hook(unsigned int cpu)
{
1548 1549 1550 1551 1552
	struct cpudata *cpu_data = all_cpu_data[cpu];

	if (!cpu_data->update_util_set)
		return;

1553
	cpufreq_remove_update_util_hook(cpu);
1554
	cpu_data->update_util_set = false;
1555 1556 1557
	synchronize_sched();
}

1558 1559
static void intel_pstate_set_performance_limits(struct perf_limits *limits)
{
1560
	mutex_lock(&intel_pstate_limits_lock);
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	limits->no_turbo = 0;
	limits->turbo_disabled = 0;
	limits->max_perf_pct = 100;
	limits->max_perf = int_tofp(1);
	limits->min_perf_pct = 100;
	limits->min_perf = int_tofp(1);
	limits->max_policy_pct = 100;
	limits->max_sysfs_pct = 100;
	limits->min_policy_pct = 0;
	limits->min_sysfs_pct = 0;
1571
	mutex_unlock(&intel_pstate_limits_lock);
1572 1573
}

1574 1575 1576
static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
					    struct perf_limits *limits)
{
1577 1578 1579

	mutex_lock(&intel_pstate_limits_lock);

1580 1581 1582
	limits->max_policy_pct = DIV_ROUND_UP(policy->max * 100,
					      policy->cpuinfo.max_freq);
	limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0, 100);
1583 1584 1585 1586 1587 1588 1589 1590
	if (policy->max == policy->min) {
		limits->min_policy_pct = limits->max_policy_pct;
	} else {
		limits->min_policy_pct = (policy->min * 100) /
						policy->cpuinfo.max_freq;
		limits->min_policy_pct = clamp_t(int, limits->min_policy_pct,
						 0, 100);
	}
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608

	/* Normalize user input to [min_policy_pct, max_policy_pct] */
	limits->min_perf_pct = max(limits->min_policy_pct,
				   limits->min_sysfs_pct);
	limits->min_perf_pct = min(limits->max_policy_pct,
				   limits->min_perf_pct);
	limits->max_perf_pct = min(limits->max_policy_pct,
				   limits->max_sysfs_pct);
	limits->max_perf_pct = max(limits->min_policy_pct,
				   limits->max_perf_pct);

	/* Make sure min_perf_pct <= max_perf_pct */
	limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct);

	limits->min_perf = div_fp(limits->min_perf_pct, 100);
	limits->max_perf = div_fp(limits->max_perf_pct, 100);
	limits->max_perf = round_up(limits->max_perf, FRAC_BITS);

1609 1610
	mutex_unlock(&intel_pstate_limits_lock);

1611 1612 1613 1614
	pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu,
		 limits->max_perf_pct, limits->min_perf_pct);
}

1615 1616
static int intel_pstate_set_policy(struct cpufreq_policy *policy)
{
1617
	struct cpudata *cpu;
1618
	struct perf_limits *perf_limits = NULL;
1619

1620 1621 1622
	if (!policy->cpuinfo.max_freq)
		return -ENODEV;

1623 1624 1625
	pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
		 policy->cpuinfo.max_freq, policy->max);

1626
	cpu = all_cpu_data[policy->cpu];
1627 1628
	cpu->policy = policy->policy;

1629 1630 1631 1632 1633
	if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
	    policy->max < policy->cpuinfo.max_freq &&
	    policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) {
		pr_debug("policy->max > max non turbo frequency\n");
		policy->max = policy->cpuinfo.max_freq;
1634 1635
	}

1636 1637 1638 1639 1640 1641 1642 1643
	if (per_cpu_limits)
		perf_limits = cpu->perf_limits;

	if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
		if (!perf_limits) {
			limits = &performance_limits;
			perf_limits = limits;
		}
1644
		if (policy->max >= policy->cpuinfo.max_freq) {
J
Joe Perches 已提交
1645
			pr_debug("set performance\n");
1646
			intel_pstate_set_performance_limits(perf_limits);
1647 1648 1649
			goto out;
		}
	} else {
J
Joe Perches 已提交
1650
		pr_debug("set powersave\n");
1651 1652 1653 1654
		if (!perf_limits) {
			limits = &powersave_limits;
			perf_limits = limits;
		}
1655

1656
	}
1657

1658
	intel_pstate_update_perf_limits(policy, perf_limits);
1659
 out:
1660
	if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
1661 1662 1663 1664 1665 1666 1667 1668
		/*
		 * NOHZ_FULL CPUs need this as the governor callback may not
		 * be invoked on them.
		 */
		intel_pstate_clear_update_util_hook(policy->cpu);
		intel_pstate_max_within_limits(cpu);
	}

1669 1670
	intel_pstate_set_update_util_hook(policy->cpu);

1671
	intel_pstate_hwp_set_policy(policy);
D
Dirk Brandewie 已提交
1672

1673 1674 1675 1676 1677
	return 0;
}

static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
{
1678
	cpufreq_verify_within_cpu_limits(policy);
1679

1680
	if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
1681
	    policy->policy != CPUFREQ_POLICY_PERFORMANCE)
1682 1683 1684 1685 1686
		return -EINVAL;

	return 0;
}

1687 1688 1689 1690 1691
static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy)
{
	intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]);
}

1692
static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
1693
{
1694
	pr_debug("CPU %d exiting\n", policy->cpu);
1695

1696 1697 1698 1699
	intel_pstate_clear_update_util_hook(policy->cpu);
	if (!hwp_active)
		intel_cpufreq_stop_cpu(policy);
}
1700

1701 1702 1703
static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
{
	intel_pstate_exit_perf_limits(policy);
1704

1705
	policy->fast_switch_possible = false;
D
Dirk Brandewie 已提交
1706

1707
	return 0;
1708 1709
}

1710
static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
1711 1712
{
	struct cpudata *cpu;
1713
	int rc;
1714 1715 1716 1717 1718 1719 1720

	rc = intel_pstate_init_cpu(policy->cpu);
	if (rc)
		return rc;

	cpu = all_cpu_data[policy->cpu];

1721 1722 1723 1724 1725 1726 1727 1728
	/*
	 * We need sane value in the cpu->perf_limits, so inherit from global
	 * perf_limits limits, which are seeded with values based on the
	 * CONFIG_CPU_FREQ_DEFAULT_GOV_*, during boot up.
	 */
	if (per_cpu_limits)
		memcpy(cpu->perf_limits, limits, sizeof(struct perf_limits));

1729 1730
	policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
	policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1731 1732

	/* cpuinfo and default policy values */
1733
	policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
1734 1735 1736 1737 1738
	update_turbo_state();
	policy->cpuinfo.max_freq = limits->turbo_disabled ?
			cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
	policy->cpuinfo.max_freq *= cpu->pstate.scaling;

1739
	intel_pstate_init_acpi_perf_limits(policy);
1740 1741
	cpumask_set_cpu(policy->cpu, policy->cpus);

1742 1743
	policy->fast_switch_possible = true;

1744 1745 1746
	return 0;
}

1747
static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
1748
{
1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
	int ret = __intel_pstate_cpu_init(policy);

	if (ret)
		return ret;

	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
	if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100)
		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
	else
		policy->policy = CPUFREQ_POLICY_POWERSAVE;
1759 1760 1761 1762

	return 0;
}

1763
static struct cpufreq_driver intel_pstate = {
1764 1765 1766
	.flags		= CPUFREQ_CONST_LOOPS,
	.verify		= intel_pstate_verify_policy,
	.setpolicy	= intel_pstate_set_policy,
1767
	.resume		= intel_pstate_hwp_set_policy,
1768 1769
	.get		= intel_pstate_get,
	.init		= intel_pstate_cpu_init,
1770
	.exit		= intel_pstate_cpu_exit,
1771
	.stop_cpu	= intel_pstate_stop_cpu,
1772 1773 1774
	.name		= "intel_pstate",
};

1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
{
	struct cpudata *cpu = all_cpu_data[policy->cpu];
	struct perf_limits *perf_limits = limits;

	update_turbo_state();
	policy->cpuinfo.max_freq = limits->turbo_disabled ?
			cpu->pstate.max_freq : cpu->pstate.turbo_freq;

	cpufreq_verify_within_cpu_limits(policy);

	if (per_cpu_limits)
		perf_limits = cpu->perf_limits;

	intel_pstate_update_perf_limits(policy, perf_limits);

	return 0;
}

static unsigned int intel_cpufreq_turbo_update(struct cpudata *cpu,
					       struct cpufreq_policy *policy,
					       unsigned int target_freq)
{
	unsigned int max_freq;

	update_turbo_state();

	max_freq = limits->no_turbo || limits->turbo_disabled ?
			cpu->pstate.max_freq : cpu->pstate.turbo_freq;
	policy->cpuinfo.max_freq = max_freq;
	if (policy->max > max_freq)
		policy->max = max_freq;

	if (target_freq > max_freq)
		target_freq = max_freq;

	return target_freq;
}

static int intel_cpufreq_target(struct cpufreq_policy *policy,
				unsigned int target_freq,
				unsigned int relation)
{
	struct cpudata *cpu = all_cpu_data[policy->cpu];
	struct cpufreq_freqs freqs;
	int target_pstate;

	freqs.old = policy->cur;
	freqs.new = intel_cpufreq_turbo_update(cpu, policy, target_freq);

	cpufreq_freq_transition_begin(policy, &freqs);
	switch (relation) {
	case CPUFREQ_RELATION_L:
		target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling);
		break;
	case CPUFREQ_RELATION_H:
		target_pstate = freqs.new / cpu->pstate.scaling;
		break;
	default:
		target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling);
		break;
	}
	target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
	if (target_pstate != cpu->pstate.current_pstate) {
		cpu->pstate.current_pstate = target_pstate;
		wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
			      pstate_funcs.get_val(cpu, target_pstate));
	}
	cpufreq_freq_transition_end(policy, &freqs, false);

	return 0;
}

static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
					      unsigned int target_freq)
{
	struct cpudata *cpu = all_cpu_data[policy->cpu];
	int target_pstate;

	target_freq = intel_cpufreq_turbo_update(cpu, policy, target_freq);
	target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
	intel_pstate_update_pstate(cpu, target_pstate);
	return target_freq;
}

static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
	int ret = __intel_pstate_cpu_init(policy);

	if (ret)
		return ret;

	policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
	/* This reflects the intel_pstate_get_cpu_pstates() setting. */
	policy->cur = policy->cpuinfo.min_freq;

	return 0;
}

static struct cpufreq_driver intel_cpufreq = {
	.flags		= CPUFREQ_CONST_LOOPS,
	.verify		= intel_cpufreq_verify_policy,
	.target		= intel_cpufreq_target,
	.fast_switch	= intel_cpufreq_fast_switch,
	.init		= intel_cpufreq_cpu_init,
	.exit		= intel_pstate_cpu_exit,
	.stop_cpu	= intel_cpufreq_stop_cpu,
	.name		= "intel_cpufreq",
};

static struct cpufreq_driver *intel_pstate_driver = &intel_pstate;

1887 1888 1889
static int no_load __initdata;
static int no_hwp __initdata;
static int hwp_only __initdata;
1890
static unsigned int force_load __initdata;
1891

1892
static int __init intel_pstate_msrs_not_valid(void)
1893
{
1894
	if (!pstate_funcs.get_max() ||
1895 1896
	    !pstate_funcs.get_min() ||
	    !pstate_funcs.get_turbo())
1897 1898 1899 1900
		return -ENODEV;

	return 0;
}
1901

1902
static void __init copy_pid_params(struct pstate_adjust_policy *policy)
1903 1904
{
	pid_params.sample_rate_ms = policy->sample_rate_ms;
1905
	pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
1906 1907 1908 1909 1910 1911 1912
	pid_params.p_gain_pct = policy->p_gain_pct;
	pid_params.i_gain_pct = policy->i_gain_pct;
	pid_params.d_gain_pct = policy->d_gain_pct;
	pid_params.deadband = policy->deadband;
	pid_params.setpoint = policy->setpoint;
}

1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925
#ifdef CONFIG_ACPI
static void intel_pstate_use_acpi_profile(void)
{
	if (acpi_gbl_FADT.preferred_profile == PM_MOBILE)
		pstate_funcs.get_target_pstate =
				get_target_pstate_use_cpu_load;
}
#else
static void intel_pstate_use_acpi_profile(void)
{
}
#endif

1926
static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
1927 1928
{
	pstate_funcs.get_max   = funcs->get_max;
1929
	pstate_funcs.get_max_physical = funcs->get_max_physical;
1930 1931
	pstate_funcs.get_min   = funcs->get_min;
	pstate_funcs.get_turbo = funcs->get_turbo;
1932
	pstate_funcs.get_scaling = funcs->get_scaling;
1933
	pstate_funcs.get_val   = funcs->get_val;
1934
	pstate_funcs.get_vid   = funcs->get_vid;
1935 1936
	pstate_funcs.get_target_pstate = funcs->get_target_pstate;

1937
	intel_pstate_use_acpi_profile();
1938 1939
}

1940
#ifdef CONFIG_ACPI
1941

1942
static bool __init intel_pstate_no_acpi_pss(void)
1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970
{
	int i;

	for_each_possible_cpu(i) {
		acpi_status status;
		union acpi_object *pss;
		struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
		struct acpi_processor *pr = per_cpu(processors, i);

		if (!pr)
			continue;

		status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
		if (ACPI_FAILURE(status))
			continue;

		pss = buffer.pointer;
		if (pss && pss->type == ACPI_TYPE_PACKAGE) {
			kfree(pss);
			return false;
		}

		kfree(pss);
	}

	return true;
}

1971
static bool __init intel_pstate_has_acpi_ppc(void)
1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990
{
	int i;

	for_each_possible_cpu(i) {
		struct acpi_processor *pr = per_cpu(processors, i);

		if (!pr)
			continue;
		if (acpi_has_method(pr->handle, "_PPC"))
			return true;
	}
	return false;
}

enum {
	PSS,
	PPC,
};

1991 1992 1993 1994
struct hw_vendor_info {
	u16  valid;
	char oem_id[ACPI_OEM_ID_SIZE];
	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
1995
	int  oem_pwr_table;
1996 1997 1998
};

/* Hardware vendor-specific info that has its own power management modes */
1999
static struct hw_vendor_info vendor_info[] __initdata = {
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
	{1, "HP    ", "ProLiant", PSS},
	{1, "ORACLE", "X4-2    ", PPC},
	{1, "ORACLE", "X4-2L   ", PPC},
	{1, "ORACLE", "X4-2B   ", PPC},
	{1, "ORACLE", "X3-2    ", PPC},
	{1, "ORACLE", "X3-2L   ", PPC},
	{1, "ORACLE", "X3-2B   ", PPC},
	{1, "ORACLE", "X4470M2 ", PPC},
	{1, "ORACLE", "X4270M3 ", PPC},
	{1, "ORACLE", "X4270M2 ", PPC},
	{1, "ORACLE", "X4170M2 ", PPC},
2011 2012 2013 2014
	{1, "ORACLE", "X4170 M3", PPC},
	{1, "ORACLE", "X4275 M3", PPC},
	{1, "ORACLE", "X6-2    ", PPC},
	{1, "ORACLE", "Sudbury ", PPC},
2015 2016 2017
	{0, "", ""},
};

2018
static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
2019 2020 2021
{
	struct acpi_table_header hdr;
	struct hw_vendor_info *v_info;
D
Dirk Brandewie 已提交
2022 2023 2024 2025 2026 2027 2028 2029 2030
	const struct x86_cpu_id *id;
	u64 misc_pwr;

	id = x86_match_cpu(intel_pstate_cpu_oob_ids);
	if (id) {
		rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
		if ( misc_pwr & (1 << 8))
			return true;
	}
2031

2032 2033
	if (acpi_disabled ||
	    ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
2034 2035 2036
		return false;

	for (v_info = vendor_info; v_info->valid; v_info++) {
2037
		if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
2038 2039 2040 2041 2042 2043
			!strncmp(hdr.oem_table_id, v_info->oem_table_id,
						ACPI_OEM_TABLE_ID_SIZE))
			switch (v_info->oem_pwr_table) {
			case PSS:
				return intel_pstate_no_acpi_pss();
			case PPC:
2044 2045
				return intel_pstate_has_acpi_ppc() &&
					(!force_load);
2046
			}
2047 2048 2049 2050
	}

	return false;
}
2051 2052 2053 2054 2055 2056 2057 2058 2059 2060

static void intel_pstate_request_control_from_smm(void)
{
	/*
	 * It may be unsafe to request P-states control from SMM if _PPC support
	 * has not been enabled.
	 */
	if (acpi_ppc)
		acpi_processor_pstate_control();
}
2061 2062
#else /* CONFIG_ACPI not enabled */
static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
2063
static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
2064
static inline void intel_pstate_request_control_from_smm(void) {}
2065 2066
#endif /* CONFIG_ACPI */

2067 2068 2069 2070 2071
static const struct x86_cpu_id hwp_support_ids[] __initconst = {
	{ X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_HWP },
	{}
};

2072 2073
static int __init intel_pstate_init(void)
{
2074
	int cpu, rc = 0;
2075
	const struct x86_cpu_id *id;
2076
	struct cpu_defaults *cpu_def;
2077

2078 2079 2080
	if (no_load)
		return -ENODEV;

2081 2082 2083 2084 2085 2086
	if (x86_match_cpu(hwp_support_ids) && !no_hwp) {
		copy_cpu_funcs(&core_params.funcs);
		hwp_active++;
		goto hwp_cpu_matched;
	}

2087 2088 2089 2090
	id = x86_match_cpu(intel_pstate_cpu_ids);
	if (!id)
		return -ENODEV;

2091
	cpu_def = (struct cpu_defaults *)id->driver_data;
2092

2093 2094
	copy_pid_params(&cpu_def->pid_policy);
	copy_cpu_funcs(&cpu_def->funcs);
2095

2096 2097 2098
	if (intel_pstate_msrs_not_valid())
		return -ENODEV;

2099 2100 2101 2102 2103 2104 2105 2106
hwp_cpu_matched:
	/*
	 * The Intel pstate driver will be ignored if the platform
	 * firmware has its own power management modes.
	 */
	if (intel_pstate_platform_pwr_mgmt_exists())
		return -ENODEV;

J
Joe Perches 已提交
2107
	pr_info("Intel P-state driver initializing\n");
2108

2109
	all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
2110 2111 2112
	if (!all_cpu_data)
		return -ENOMEM;

2113 2114 2115
	if (!hwp_active && hwp_only)
		goto out;

2116 2117
	intel_pstate_request_control_from_smm();

2118
	rc = cpufreq_register_driver(intel_pstate_driver);
2119 2120 2121 2122 2123
	if (rc)
		goto out;

	intel_pstate_debug_expose_params();
	intel_pstate_sysfs_expose_params();
2124

2125
	if (hwp_active)
J
Joe Perches 已提交
2126
		pr_info("HWP enabled\n");
2127

2128 2129
	return rc;
out:
2130 2131 2132
	get_online_cpus();
	for_each_online_cpu(cpu) {
		if (all_cpu_data[cpu]) {
2133 2134 2135
			if (intel_pstate_driver == &intel_pstate)
				intel_pstate_clear_update_util_hook(cpu);

2136 2137 2138 2139 2140 2141
			kfree(all_cpu_data[cpu]);
		}
	}

	put_online_cpus();
	vfree(all_cpu_data);
2142 2143 2144 2145
	return -ENODEV;
}
device_initcall(intel_pstate_init);

2146 2147 2148 2149 2150
static int __init intel_pstate_setup(char *str)
{
	if (!str)
		return -EINVAL;

2151
	if (!strcmp(str, "disable")) {
2152
		no_load = 1;
2153 2154 2155 2156 2157
	} else if (!strcmp(str, "passive")) {
		pr_info("Passive mode enabled\n");
		intel_pstate_driver = &intel_cpufreq;
		no_hwp = 1;
	}
2158
	if (!strcmp(str, "no_hwp")) {
J
Joe Perches 已提交
2159
		pr_info("HWP disabled\n");
D
Dirk Brandewie 已提交
2160
		no_hwp = 1;
2161
	}
2162 2163
	if (!strcmp(str, "force"))
		force_load = 1;
2164 2165
	if (!strcmp(str, "hwp_only"))
		hwp_only = 1;
2166 2167
	if (!strcmp(str, "per_cpu_perf_limits"))
		per_cpu_limits = true;
2168 2169 2170 2171 2172 2173

#ifdef CONFIG_ACPI
	if (!strcmp(str, "support_acpi_ppc"))
		acpi_ppc = true;
#endif

2174 2175 2176 2177
	return 0;
}
early_param("intel_pstate", intel_pstate_setup);

2178 2179 2180
MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
MODULE_LICENSE("GPL");