intel_runtime_pm.c 84.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
 * Copyright © 2012-2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eugeni Dodonov <eugeni.dodonov@intel.com>
 *    Daniel Vetter <daniel.vetter@ffwll.ch>
 *
 */

#include <linux/pm_runtime.h>
#include <linux/vgaarb.h>

#include "i915_drv.h"
#include "intel_drv.h"

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
/**
 * DOC: runtime pm
 *
 * The i915 driver supports dynamic enabling and disabling of entire hardware
 * blocks at runtime. This is especially important on the display side where
 * software is supposed to control many power gates manually on recent hardware,
 * since on the GT side a lot of the power management is done by the hardware.
 * But even there some manual control at the device level is required.
 *
 * Since i915 supports a diverse set of platforms with a unified codebase and
 * hardware engineers just love to shuffle functionality around between power
 * domains there's a sizeable amount of indirection required. This file provides
 * generic functions to the driver for grabbing and releasing references for
 * abstract power domains. It then maps those to the actual power wells
 * present for a given platform.
 */

52 53 54 55 56
#define for_each_power_well(i, power_well, domain_mask, power_domains)	\
	for (i = 0;							\
	     i < (power_domains)->power_well_count &&			\
		 ((power_well) = &(power_domains)->power_wells[i]);	\
	     i++)							\
57
		for_each_if ((power_well)->domains & (domain_mask))
58 59 60 61 62

#define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \
	for (i = (power_domains)->power_well_count - 1;			 \
	     i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\
	     i--)							 \
63
		for_each_if ((power_well)->domains & (domain_mask))
64

65 66 67
bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
				    int power_well_id);

68 69 70
static struct i915_power_well *
lookup_power_well(struct drm_i915_private *dev_priv, int power_well_id);

71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
const char *
intel_display_power_domain_str(enum intel_display_power_domain domain)
{
	switch (domain) {
	case POWER_DOMAIN_PIPE_A:
		return "PIPE_A";
	case POWER_DOMAIN_PIPE_B:
		return "PIPE_B";
	case POWER_DOMAIN_PIPE_C:
		return "PIPE_C";
	case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
		return "PIPE_A_PANEL_FITTER";
	case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
		return "PIPE_B_PANEL_FITTER";
	case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
		return "PIPE_C_PANEL_FITTER";
	case POWER_DOMAIN_TRANSCODER_A:
		return "TRANSCODER_A";
	case POWER_DOMAIN_TRANSCODER_B:
		return "TRANSCODER_B";
	case POWER_DOMAIN_TRANSCODER_C:
		return "TRANSCODER_C";
	case POWER_DOMAIN_TRANSCODER_EDP:
		return "TRANSCODER_EDP";
J
Jani Nikula 已提交
95 96 97 98
	case POWER_DOMAIN_TRANSCODER_DSI_A:
		return "TRANSCODER_DSI_A";
	case POWER_DOMAIN_TRANSCODER_DSI_C:
		return "TRANSCODER_DSI_C";
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
	case POWER_DOMAIN_PORT_DDI_A_LANES:
		return "PORT_DDI_A_LANES";
	case POWER_DOMAIN_PORT_DDI_B_LANES:
		return "PORT_DDI_B_LANES";
	case POWER_DOMAIN_PORT_DDI_C_LANES:
		return "PORT_DDI_C_LANES";
	case POWER_DOMAIN_PORT_DDI_D_LANES:
		return "PORT_DDI_D_LANES";
	case POWER_DOMAIN_PORT_DDI_E_LANES:
		return "PORT_DDI_E_LANES";
	case POWER_DOMAIN_PORT_DSI:
		return "PORT_DSI";
	case POWER_DOMAIN_PORT_CRT:
		return "PORT_CRT";
	case POWER_DOMAIN_PORT_OTHER:
		return "PORT_OTHER";
	case POWER_DOMAIN_VGA:
		return "VGA";
	case POWER_DOMAIN_AUDIO:
		return "AUDIO";
	case POWER_DOMAIN_PLLS:
		return "PLLS";
	case POWER_DOMAIN_AUX_A:
		return "AUX_A";
	case POWER_DOMAIN_AUX_B:
		return "AUX_B";
	case POWER_DOMAIN_AUX_C:
		return "AUX_C";
	case POWER_DOMAIN_AUX_D:
		return "AUX_D";
	case POWER_DOMAIN_GMBUS:
		return "GMBUS";
	case POWER_DOMAIN_INIT:
		return "INIT";
	case POWER_DOMAIN_MODESET:
		return "MODESET";
	default:
		MISSING_CASE(domain);
		return "?";
	}
}

141 142 143 144 145 146 147 148
static void intel_power_well_enable(struct drm_i915_private *dev_priv,
				    struct i915_power_well *power_well)
{
	DRM_DEBUG_KMS("enabling %s\n", power_well->name);
	power_well->ops->enable(dev_priv, power_well);
	power_well->hw_enabled = true;
}

149 150 151 152 153 154 155 156
static void intel_power_well_disable(struct drm_i915_private *dev_priv,
				     struct i915_power_well *power_well)
{
	DRM_DEBUG_KMS("disabling %s\n", power_well->name);
	power_well->hw_enabled = false;
	power_well->ops->disable(dev_priv, power_well);
}

157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
static void intel_power_well_get(struct drm_i915_private *dev_priv,
				 struct i915_power_well *power_well)
{
	if (!power_well->count++)
		intel_power_well_enable(dev_priv, power_well);
}

static void intel_power_well_put(struct drm_i915_private *dev_priv,
				 struct i915_power_well *power_well)
{
	WARN(!power_well->count, "Use count on power well %s is already zero",
	     power_well->name);

	if (!--power_well->count)
		intel_power_well_disable(dev_priv, power_well);
}

174
/*
175 176 177 178 179 180 181 182 183 184 185
 * We should only use the power well if we explicitly asked the hardware to
 * enable it, so check if it's enabled and also check if we've requested it to
 * be enabled.
 */
static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
				   struct i915_power_well *power_well)
{
	return I915_READ(HSW_PWR_WELL_DRIVER) ==
		     (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
}

186 187 188 189 190 191 192 193 194 195 196 197
/**
 * __intel_display_power_is_enabled - unlocked check for a power domain
 * @dev_priv: i915 device instance
 * @domain: power domain to check
 *
 * This is the unlocked version of intel_display_power_is_enabled() and should
 * only be used from error capture and recovery code where deadlocks are
 * possible.
 *
 * Returns:
 * True when the power domain is enabled, false otherwise.
 */
198 199
bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
				      enum intel_display_power_domain domain)
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
{
	struct i915_power_domains *power_domains;
	struct i915_power_well *power_well;
	bool is_enabled;
	int i;

	if (dev_priv->pm.suspended)
		return false;

	power_domains = &dev_priv->power_domains;

	is_enabled = true;

	for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
		if (power_well->always_on)
			continue;

		if (!power_well->hw_enabled) {
			is_enabled = false;
			break;
		}
	}

	return is_enabled;
}

226
/**
227
 * intel_display_power_is_enabled - check for a power domain
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
 * @dev_priv: i915 device instance
 * @domain: power domain to check
 *
 * This function can be used to check the hw power domain state. It is mostly
 * used in hardware state readout functions. Everywhere else code should rely
 * upon explicit power domain reference counting to ensure that the hardware
 * block is powered up before accessing it.
 *
 * Callers must hold the relevant modesetting locks to ensure that concurrent
 * threads can't disable the power well while the caller tries to read a few
 * registers.
 *
 * Returns:
 * True when the power domain is enabled, false otherwise.
 */
243 244
bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
				    enum intel_display_power_domain domain)
245 246 247 248 249 250 251
{
	struct i915_power_domains *power_domains;
	bool ret;

	power_domains = &dev_priv->power_domains;

	mutex_lock(&power_domains->lock);
252
	ret = __intel_display_power_is_enabled(dev_priv, domain);
253 254 255 256 257
	mutex_unlock(&power_domains->lock);

	return ret;
}

258 259 260 261 262 263 264 265 266 267
/**
 * intel_display_set_init_power - set the initial power domain state
 * @dev_priv: i915 device instance
 * @enable: whether to enable or disable the initial power domain state
 *
 * For simplicity our driver load/unload and system suspend/resume code assumes
 * that all power domains are always enabled. This functions controls the state
 * of this little hack. While the initial power domain state is enabled runtime
 * pm is effectively disabled.
 */
268 269 270 271 272 273 274 275 276 277 278 279 280 281
void intel_display_set_init_power(struct drm_i915_private *dev_priv,
				  bool enable)
{
	if (dev_priv->power_domains.init_power_on == enable)
		return;

	if (enable)
		intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
	else
		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);

	dev_priv->power_domains.init_power_on = enable;
}

282 283 284 285 286 287 288 289
/*
 * Starting with Haswell, we have a "Power Down Well" that can be turned off
 * when not needed anymore. We have 4 registers that can request the power well
 * to be enabled, and it will only be disabled if none of the registers is
 * requesting it to be enabled.
 */
static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
{
D
David Weinehall 已提交
290
	struct pci_dev *pdev = dev_priv->drm.pdev;
291 292 293 294 295 296 297 298 299 300 301

	/*
	 * After we re-enable the power well, if we touch VGA register 0x3d5
	 * we'll get unclaimed register interrupts. This stops after we write
	 * anything to the VGA MSR register. The vgacon module uses this
	 * register all the time, so if we unbind our driver and, as a
	 * consequence, bind vgacon, we'll get stuck in an infinite loop at
	 * console_unlock(). So make here we touch the VGA MSR register, making
	 * sure vgacon can keep working normally without triggering interrupts
	 * and error messages.
	 */
D
David Weinehall 已提交
302
	vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
303
	outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
D
David Weinehall 已提交
304
	vga_put(pdev, VGA_RSRC_LEGACY_IO);
305

306
	if (IS_BROADWELL(dev_priv))
307 308
		gen8_irq_power_well_post_enable(dev_priv,
						1 << PIPE_C | 1 << PIPE_B);
309 310
}

311 312 313 314 315 316 317
static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv)
{
	if (IS_BROADWELL(dev_priv))
		gen8_irq_power_well_pre_disable(dev_priv,
						1 << PIPE_C | 1 << PIPE_B);
}

318 319 320
static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
				       struct i915_power_well *power_well)
{
D
David Weinehall 已提交
321
	struct pci_dev *pdev = dev_priv->drm.pdev;
322 323 324 325 326 327 328 329 330 331 332

	/*
	 * After we re-enable the power well, if we touch VGA register 0x3d5
	 * we'll get unclaimed register interrupts. This stops after we write
	 * anything to the VGA MSR register. The vgacon module uses this
	 * register all the time, so if we unbind our driver and, as a
	 * consequence, bind vgacon, we'll get stuck in an infinite loop at
	 * console_unlock(). So make here we touch the VGA MSR register, making
	 * sure vgacon can keep working normally without triggering interrupts
	 * and error messages.
	 */
333
	if (power_well->id == SKL_DISP_PW_2) {
D
David Weinehall 已提交
334
		vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
335
		outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
D
David Weinehall 已提交
336
		vga_put(pdev, VGA_RSRC_LEGACY_IO);
337 338 339 340 341 342

		gen8_irq_power_well_post_enable(dev_priv,
						1 << PIPE_C | 1 << PIPE_B);
	}
}

343 344 345
static void skl_power_well_pre_disable(struct drm_i915_private *dev_priv,
				       struct i915_power_well *power_well)
{
346
	if (power_well->id == SKL_DISP_PW_2)
347 348 349 350
		gen8_irq_power_well_pre_disable(dev_priv,
						1 << PIPE_C | 1 << PIPE_B);
}

351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
static void hsw_set_power_well(struct drm_i915_private *dev_priv,
			       struct i915_power_well *power_well, bool enable)
{
	bool is_enabled, enable_requested;
	uint32_t tmp;

	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
	is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
	enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;

	if (enable) {
		if (!enable_requested)
			I915_WRITE(HSW_PWR_WELL_DRIVER,
				   HSW_PWR_WELL_ENABLE_REQUEST);

		if (!is_enabled) {
			DRM_DEBUG_KMS("Enabling power well\n");
368 369 370 371 372
			if (intel_wait_for_register(dev_priv,
						    HSW_PWR_WELL_DRIVER,
						    HSW_PWR_WELL_STATE_ENABLED,
						    HSW_PWR_WELL_STATE_ENABLED,
						    20))
373
				DRM_ERROR("Timeout enabling power well\n");
374
			hsw_power_well_post_enable(dev_priv);
375 376 377 378
		}

	} else {
		if (enable_requested) {
379
			hsw_power_well_pre_disable(dev_priv);
380 381 382 383 384 385 386
			I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
			POSTING_READ(HSW_PWR_WELL_DRIVER);
			DRM_DEBUG_KMS("Requesting to disable the power well\n");
		}
	}
}

387 388 389 390 391 392 393 394
#define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_TRANSCODER_A) |		\
	BIT(POWER_DOMAIN_PIPE_B) |			\
	BIT(POWER_DOMAIN_TRANSCODER_B) |		\
	BIT(POWER_DOMAIN_PIPE_C) |			\
	BIT(POWER_DOMAIN_TRANSCODER_C) |		\
	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
395 396 397 398
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
399 400 401 402 403 404 405
	BIT(POWER_DOMAIN_AUX_B) |                       \
	BIT(POWER_DOMAIN_AUX_C) |			\
	BIT(POWER_DOMAIN_AUX_D) |			\
	BIT(POWER_DOMAIN_AUDIO) |			\
	BIT(POWER_DOMAIN_VGA) |				\
	BIT(POWER_DOMAIN_INIT))
#define SKL_DISPLAY_DDI_A_E_POWER_DOMAINS (		\
406 407
	BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
408 409
	BIT(POWER_DOMAIN_INIT))
#define SKL_DISPLAY_DDI_B_POWER_DOMAINS (		\
410
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
411 412
	BIT(POWER_DOMAIN_INIT))
#define SKL_DISPLAY_DDI_C_POWER_DOMAINS (		\
413
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
414 415
	BIT(POWER_DOMAIN_INIT))
#define SKL_DISPLAY_DDI_D_POWER_DOMAINS (		\
416
	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
417
	BIT(POWER_DOMAIN_INIT))
418 419 420 421 422
#define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
	SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
	BIT(POWER_DOMAIN_MODESET) |			\
	BIT(POWER_DOMAIN_AUX_A) |			\
	BIT(POWER_DOMAIN_INIT))
423

424 425 426 427 428 429 430 431
#define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_TRANSCODER_A) |		\
	BIT(POWER_DOMAIN_PIPE_B) |			\
	BIT(POWER_DOMAIN_TRANSCODER_B) |		\
	BIT(POWER_DOMAIN_PIPE_C) |			\
	BIT(POWER_DOMAIN_TRANSCODER_C) |		\
	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
432 433
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
434 435 436 437
	BIT(POWER_DOMAIN_AUX_B) |			\
	BIT(POWER_DOMAIN_AUX_C) |			\
	BIT(POWER_DOMAIN_AUDIO) |			\
	BIT(POWER_DOMAIN_VGA) |				\
438
	BIT(POWER_DOMAIN_GMBUS) |			\
439
	BIT(POWER_DOMAIN_INIT))
440 441 442 443 444
#define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (		\
	BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
	BIT(POWER_DOMAIN_MODESET) |			\
	BIT(POWER_DOMAIN_AUX_A) |			\
	BIT(POWER_DOMAIN_INIT))
445 446 447 448 449 450 451 452 453 454
#define BXT_DPIO_CMN_A_POWER_DOMAINS (			\
	BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
	BIT(POWER_DOMAIN_AUX_A) |			\
	BIT(POWER_DOMAIN_INIT))
#define BXT_DPIO_CMN_BC_POWER_DOMAINS (			\
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
	BIT(POWER_DOMAIN_AUX_B) |			\
	BIT(POWER_DOMAIN_AUX_C) |			\
	BIT(POWER_DOMAIN_INIT))
455

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
#define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_TRANSCODER_A) |		\
	BIT(POWER_DOMAIN_PIPE_B) |			\
	BIT(POWER_DOMAIN_TRANSCODER_B) |		\
	BIT(POWER_DOMAIN_PIPE_C) |			\
	BIT(POWER_DOMAIN_TRANSCODER_C) |		\
	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
	BIT(POWER_DOMAIN_AUX_B) |                       \
	BIT(POWER_DOMAIN_AUX_C) |			\
	BIT(POWER_DOMAIN_AUDIO) |			\
	BIT(POWER_DOMAIN_VGA) |				\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DISPLAY_DDI_A_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DISPLAY_DDI_B_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DISPLAY_DDI_C_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
	BIT(POWER_DOMAIN_INIT))
480 481 482 483 484 485 486 487 488 489 490 491
#define GLK_DPIO_CMN_A_POWER_DOMAINS (			\
	BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
	BIT(POWER_DOMAIN_AUX_A) |			\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DPIO_CMN_B_POWER_DOMAINS (			\
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_AUX_B) |			\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DPIO_CMN_C_POWER_DOMAINS (			\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
	BIT(POWER_DOMAIN_AUX_C) |			\
	BIT(POWER_DOMAIN_INIT))
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
#define GLK_DISPLAY_AUX_A_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_AUX_A) |		\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DISPLAY_AUX_B_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_AUX_B) |		\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DISPLAY_AUX_C_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_AUX_C) |		\
	BIT(POWER_DOMAIN_INIT))
#define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (		\
	GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
	BIT(POWER_DOMAIN_MODESET) |			\
	BIT(POWER_DOMAIN_AUX_A) |			\
	BIT(POWER_DOMAIN_INIT))

507 508
static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
{
509 510 511 512 513 514 515
	WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
		  "DC9 already programmed to be enabled.\n");
	WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
		  "DC5 still not disabled to enable DC9.\n");
	WARN_ONCE(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on.\n");
	WARN_ONCE(intel_irqs_enabled(dev_priv),
		  "Interrupts not disabled yet.\n");
516 517 518 519 520 521 522 523 524 525 526 527

	 /*
	  * TODO: check for the following to verify the conditions to enter DC9
	  * state are satisfied:
	  * 1] Check relevant display engine registers to verify if mode set
	  * disable sequence was followed.
	  * 2] Check if display uninitialize sequence is initialized.
	  */
}

static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
{
528 529 530 531
	WARN_ONCE(intel_irqs_enabled(dev_priv),
		  "Interrupts not disabled yet.\n");
	WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
		  "DC5 still not disabled.\n");
532 533 534 535 536 537 538 539 540 541

	 /*
	  * TODO: check for the following to verify DC9 state was indeed
	  * entered before programming to disable it:
	  * 1] Check relevant display engine registers to verify if mode
	  *  set disable sequence was followed.
	  * 2] Check if display uninitialize sequence is initialized.
	  */
}

542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
				u32 state)
{
	int rewrites = 0;
	int rereads = 0;
	u32 v;

	I915_WRITE(DC_STATE_EN, state);

	/* It has been observed that disabling the dc6 state sometimes
	 * doesn't stick and dmc keeps returning old value. Make sure
	 * the write really sticks enough times and also force rewrite until
	 * we are confident that state is exactly what we want.
	 */
	do  {
		v = I915_READ(DC_STATE_EN);

		if (v != state) {
			I915_WRITE(DC_STATE_EN, state);
			rewrites++;
			rereads = 0;
		} else if (rereads++ > 5) {
			break;
		}

	} while (rewrites < 100);

	if (v != state)
		DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
			  state, v);

	/* Most of the times we need one retry, avoid spam */
	if (rewrites > 1)
		DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
			      state, rewrites);
}

579
static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
580
{
581
	u32 mask;
582

583
	mask = DC_STATE_EN_UPTO_DC5;
584
	if (IS_GEN9_LP(dev_priv))
585 586 587
		mask |= DC_STATE_EN_DC9;
	else
		mask |= DC_STATE_EN_UPTO_DC6;
588

589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
	return mask;
}

void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
{
	u32 val;

	val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);

	DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
		      dev_priv->csr.dc_state, val);
	dev_priv->csr.dc_state = val;
}

static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
{
	uint32_t val;
	uint32_t mask;

608 609
	if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
		state &= dev_priv->csr.allowed_dc_mask;
610

611
	val = I915_READ(DC_STATE_EN);
612
	mask = gen9_dc_mask(dev_priv);
613 614
	DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
		      val & mask, state);
615 616 617 618 619 620

	/* Check if DMC is ignoring our DC state requests */
	if ((val & mask) != dev_priv->csr.dc_state)
		DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
			  dev_priv->csr.dc_state, val & mask);

621 622
	val &= ~mask;
	val |= state;
623 624

	gen9_write_dc_state(dev_priv, val);
625 626

	dev_priv->csr.dc_state = val & mask;
627 628
}

629
void bxt_enable_dc9(struct drm_i915_private *dev_priv)
630
{
631 632 633
	assert_can_enable_dc9(dev_priv);

	DRM_DEBUG_KMS("Enabling DC9\n");
634

635
	intel_power_sequencer_reset(dev_priv);
636 637 638 639 640
	gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
}

void bxt_disable_dc9(struct drm_i915_private *dev_priv)
{
641 642 643 644
	assert_can_disable_dc9(dev_priv);

	DRM_DEBUG_KMS("Disabling DC9\n");

645
	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
646 647

	intel_pps_unlock_regs_wa(dev_priv);
648 649
}

650 651 652 653 654 655 656 657
static void assert_csr_loaded(struct drm_i915_private *dev_priv)
{
	WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
		  "CSR program storage start is NULL\n");
	WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
	WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
}

658
static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
659
{
660 661 662
	bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
					SKL_DISP_PW_2);

663
	WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
664

665 666
	WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
		  "DC5 already programmed to be enabled.\n");
667
	assert_rpm_wakelock_held(dev_priv);
668 669 670 671

	assert_csr_loaded(dev_priv);
}

672
void gen9_enable_dc5(struct drm_i915_private *dev_priv)
673 674
{
	assert_can_enable_dc5(dev_priv);
675 676 677

	DRM_DEBUG_KMS("Enabling DC5\n");

678
	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
679 680
}

681
static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
682
{
683 684 685 686
	WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
		  "Backlight is not disabled.\n");
	WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
		  "DC6 already programmed to be enabled.\n");
687 688 689 690

	assert_csr_loaded(dev_priv);
}

691
void skl_enable_dc6(struct drm_i915_private *dev_priv)
692 693
{
	assert_can_enable_dc6(dev_priv);
694 695 696

	DRM_DEBUG_KMS("Enabling DC6\n");

697 698
	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);

699 700
}

701
void skl_disable_dc6(struct drm_i915_private *dev_priv)
702
{
703 704
	DRM_DEBUG_KMS("Disabling DC6\n");

705
	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
706 707
}

708 709 710 711
static void
gen9_sanitize_power_well_requests(struct drm_i915_private *dev_priv,
				  struct i915_power_well *power_well)
{
712
	enum skl_disp_power_wells power_well_id = power_well->id;
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734
	u32 val;
	u32 mask;

	mask = SKL_POWER_WELL_REQ(power_well_id);

	val = I915_READ(HSW_PWR_WELL_KVMR);
	if (WARN_ONCE(val & mask, "Clearing unexpected KVMR request for %s\n",
		      power_well->name))
		I915_WRITE(HSW_PWR_WELL_KVMR, val & ~mask);

	val = I915_READ(HSW_PWR_WELL_BIOS);
	val |= I915_READ(HSW_PWR_WELL_DEBUG);

	if (!(val & mask))
		return;

	/*
	 * DMC is known to force on the request bits for power well 1 on SKL
	 * and BXT and the misc IO power well on SKL but we don't expect any
	 * other request bits to be set, so WARN for those.
	 */
	if (power_well_id == SKL_DISP_PW_1 ||
735
	    (IS_GEN9_BC(dev_priv) &&
736
	     power_well_id == SKL_DISP_PW_MISC_IO))
737 738 739 740 741 742 743 744 745 746
		DRM_DEBUG_DRIVER("Clearing auxiliary requests for %s forced on "
				 "by DMC\n", power_well->name);
	else
		WARN_ONCE(1, "Clearing unexpected auxiliary requests for %s\n",
			  power_well->name);

	I915_WRITE(HSW_PWR_WELL_BIOS, val & ~mask);
	I915_WRITE(HSW_PWR_WELL_DEBUG, val & ~mask);
}

747
static void skl_set_power_well(struct drm_i915_private *dev_priv,
748
			       struct i915_power_well *power_well, bool enable)
749 750 751
{
	uint32_t tmp, fuse_status;
	uint32_t req_mask, state_mask;
752
	bool is_enabled, enable_requested, check_fuse_status = false;
753 754 755 756

	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
	fuse_status = I915_READ(SKL_FUSE_STATUS);

757
	switch (power_well->id) {
758
	case SKL_DISP_PW_1:
759 760 761 762 763
		if (intel_wait_for_register(dev_priv,
					    SKL_FUSE_STATUS,
					    SKL_FUSE_PG0_DIST_STATUS,
					    SKL_FUSE_PG0_DIST_STATUS,
					    1)) {
764 765 766 767 768 769 770 771 772 773
			DRM_ERROR("PG0 not enabled\n");
			return;
		}
		break;
	case SKL_DISP_PW_2:
		if (!(fuse_status & SKL_FUSE_PG1_DIST_STATUS)) {
			DRM_ERROR("PG1 in disabled state\n");
			return;
		}
		break;
774 775
	case SKL_DISP_PW_MISC_IO:
	case SKL_DISP_PW_DDI_A_E: /* GLK_DISP_PW_DDI_A */
776 777 778
	case SKL_DISP_PW_DDI_B:
	case SKL_DISP_PW_DDI_C:
	case SKL_DISP_PW_DDI_D:
779 780 781
	case GLK_DISP_PW_AUX_A:
	case GLK_DISP_PW_AUX_B:
	case GLK_DISP_PW_AUX_C:
782 783
		break;
	default:
784
		WARN(1, "Unknown power well %lu\n", power_well->id);
785 786 787
		return;
	}

788
	req_mask = SKL_POWER_WELL_REQ(power_well->id);
789
	enable_requested = tmp & req_mask;
790
	state_mask = SKL_POWER_WELL_STATE(power_well->id);
791
	is_enabled = tmp & state_mask;
792

793 794 795
	if (!enable && enable_requested)
		skl_power_well_pre_disable(dev_priv, power_well);

796
	if (enable) {
797
		if (!enable_requested) {
798 799 800 801
			WARN((tmp & state_mask) &&
				!I915_READ(HSW_PWR_WELL_BIOS),
				"Invalid for power well status to be enabled, unless done by the BIOS, \
				when request is to disable!\n");
802 803 804
			I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask);
		}

805
		if (!is_enabled) {
806
			DRM_DEBUG_KMS("Enabling %s\n", power_well->name);
807 808 809
			check_fuse_status = true;
		}
	} else {
810
		if (enable_requested) {
811 812 813
			I915_WRITE(HSW_PWR_WELL_DRIVER,	tmp & ~req_mask);
			POSTING_READ(HSW_PWR_WELL_DRIVER);
			DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
814
		}
815

816
		if (IS_GEN9(dev_priv))
817
			gen9_sanitize_power_well_requests(dev_priv, power_well);
818 819
	}

820 821 822 823 824
	if (wait_for(!!(I915_READ(HSW_PWR_WELL_DRIVER) & state_mask) == enable,
		     1))
		DRM_ERROR("%s %s timeout\n",
			  power_well->name, enable ? "enable" : "disable");

825
	if (check_fuse_status) {
826
		if (power_well->id == SKL_DISP_PW_1) {
827 828 829 830 831
			if (intel_wait_for_register(dev_priv,
						    SKL_FUSE_STATUS,
						    SKL_FUSE_PG1_DIST_STATUS,
						    SKL_FUSE_PG1_DIST_STATUS,
						    1))
832
				DRM_ERROR("PG1 distributing status timeout\n");
833
		} else if (power_well->id == SKL_DISP_PW_2) {
834 835 836 837 838
			if (intel_wait_for_register(dev_priv,
						    SKL_FUSE_STATUS,
						    SKL_FUSE_PG2_DIST_STATUS,
						    SKL_FUSE_PG2_DIST_STATUS,
						    1))
839 840 841
				DRM_ERROR("PG2 distributing status timeout\n");
		}
	}
842 843 844

	if (enable && !is_enabled)
		skl_power_well_post_enable(dev_priv, power_well);
845 846
}

847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
				   struct i915_power_well *power_well)
{
	hsw_set_power_well(dev_priv, power_well, power_well->count > 0);

	/*
	 * We're taking over the BIOS, so clear any requests made by it since
	 * the driver is in charge now.
	 */
	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
		I915_WRITE(HSW_PWR_WELL_BIOS, 0);
}

static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
				  struct i915_power_well *power_well)
{
	hsw_set_power_well(dev_priv, power_well, true);
}

static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
				   struct i915_power_well *power_well)
{
	hsw_set_power_well(dev_priv, power_well, false);
}

872 873 874
static bool skl_power_well_enabled(struct drm_i915_private *dev_priv,
					struct i915_power_well *power_well)
{
875 876
	uint32_t mask = SKL_POWER_WELL_REQ(power_well->id) |
		SKL_POWER_WELL_STATE(power_well->id);
877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901

	return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask;
}

static void skl_power_well_sync_hw(struct drm_i915_private *dev_priv,
				struct i915_power_well *power_well)
{
	skl_set_power_well(dev_priv, power_well, power_well->count > 0);

	/* Clear any request made by BIOS as driver is taking over */
	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
}

static void skl_power_well_enable(struct drm_i915_private *dev_priv,
				struct i915_power_well *power_well)
{
	skl_set_power_well(dev_priv, power_well, true);
}

static void skl_power_well_disable(struct drm_i915_private *dev_priv,
				struct i915_power_well *power_well)
{
	skl_set_power_well(dev_priv, power_well, false);
}

902 903 904
static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
905
	bxt_ddi_phy_init(dev_priv, power_well->data);
906 907 908 909 910
}

static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
					    struct i915_power_well *power_well)
{
911
	bxt_ddi_phy_uninit(dev_priv, power_well->data);
912 913 914 915 916
}

static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
					    struct i915_power_well *power_well)
{
917
	return bxt_ddi_phy_is_enabled(dev_priv, power_well->data);
918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
}

static void bxt_dpio_cmn_power_well_sync_hw(struct drm_i915_private *dev_priv,
					    struct i915_power_well *power_well)
{
	if (power_well->count > 0)
		bxt_dpio_cmn_power_well_enable(dev_priv, power_well);
	else
		bxt_dpio_cmn_power_well_disable(dev_priv, power_well);
}


static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
{
	struct i915_power_well *power_well;

	power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
	if (power_well->count > 0)
936
		bxt_ddi_phy_verify_state(dev_priv, power_well->data);
937 938 939

	power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
	if (power_well->count > 0)
940
		bxt_ddi_phy_verify_state(dev_priv, power_well->data);
941 942 943 944 945 946

	if (IS_GEMINILAKE(dev_priv)) {
		power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
		if (power_well->count > 0)
			bxt_ddi_phy_verify_state(dev_priv, power_well->data);
	}
947 948
}

949 950 951 952 953 954
static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
	return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
}

955 956 957 958 959 960 961 962 963
static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
{
	u32 tmp = I915_READ(DBUF_CTL);

	WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
	     (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
	     "Unexpected DBuf power power state (0x%08x)\n", tmp);
}

964 965 966
static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
					  struct i915_power_well *power_well)
{
967
	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
968

969
	WARN_ON(dev_priv->cdclk_freq != dev_priv->display.get_cdclk(dev_priv));
970

971 972
	gen9_assert_dbuf_enabled(dev_priv);

973
	if (IS_GEN9_LP(dev_priv))
974
		bxt_verify_ddi_phy_power_wells(dev_priv);
975 976 977 978 979
}

static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
980 981 982
	if (!dev_priv->csr.dmc_payload)
		return;

983
	if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
984
		skl_enable_dc6(dev_priv);
985
	else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
986 987 988 989 990 991
		gen9_enable_dc5(dev_priv);
}

static void gen9_dc_off_power_well_sync_hw(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
992 993 994 995
	if (power_well->count > 0)
		gen9_dc_off_power_well_enable(dev_priv, power_well);
	else
		gen9_dc_off_power_well_disable(dev_priv, power_well);
996 997
}

998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
}

static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
					     struct i915_power_well *power_well)
{
	return true;
}

static void vlv_set_power_well(struct drm_i915_private *dev_priv,
			       struct i915_power_well *power_well, bool enable)
{
1012
	enum punit_power_well power_well_id = power_well->id;
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
	u32 mask;
	u32 state;
	u32 ctrl;

	mask = PUNIT_PWRGT_MASK(power_well_id);
	state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
			 PUNIT_PWRGT_PWR_GATE(power_well_id);

	mutex_lock(&dev_priv->rps.hw_lock);

#define COND \
	((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)

	if (COND)
		goto out;

	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
	ctrl &= ~mask;
	ctrl |= state;
	vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);

	if (wait_for(COND, 100))
1035
		DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
			  state,
			  vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));

#undef COND

out:
	mutex_unlock(&dev_priv->rps.hw_lock);
}

static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv,
				   struct i915_power_well *power_well)
{
	vlv_set_power_well(dev_priv, power_well, power_well->count > 0);
}

static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
				  struct i915_power_well *power_well)
{
	vlv_set_power_well(dev_priv, power_well, true);
}

static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
				   struct i915_power_well *power_well)
{
	vlv_set_power_well(dev_priv, power_well, false);
}

static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
				   struct i915_power_well *power_well)
{
1066
	int power_well_id = power_well->id;
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
	bool enabled = false;
	u32 mask;
	u32 state;
	u32 ctrl;

	mask = PUNIT_PWRGT_MASK(power_well_id);
	ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);

	mutex_lock(&dev_priv->rps.hw_lock);

	state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
	/*
	 * We only ever set the power-on and power-gate states, anything
	 * else is unexpected.
	 */
	WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
		state != PUNIT_PWRGT_PWR_GATE(power_well_id));
	if (state == ctrl)
		enabled = true;

	/*
	 * A transient state at this point would mean some unexpected party
	 * is poking at the power controls too.
	 */
	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
	WARN_ON(ctrl != state);

	mutex_unlock(&dev_priv->rps.hw_lock);

	return enabled;
}

1099 1100
static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
{
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
	u32 val;

	/*
	 * On driver load, a pipe may be active and driving a DSI display.
	 * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
	 * (and never recovering) in this case. intel_dsi_post_disable() will
	 * clear it when we turn off the display.
	 */
	val = I915_READ(DSPCLK_GATE_D);
	val &= DPOUNIT_CLOCK_GATE_DISABLE;
	val |= VRHUNIT_CLOCK_GATE_DISABLE;
	I915_WRITE(DSPCLK_GATE_D, val);
1113 1114 1115 1116 1117 1118

	/*
	 * Disable trickle feed and enable pnd deadline calculation
	 */
	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
	I915_WRITE(CBR1_VLV, 0);
1119 1120 1121 1122 1123

	WARN_ON(dev_priv->rawclk_freq == 0);

	I915_WRITE(RAWCLK_FREQ_VLV,
		   DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
1124 1125
}

1126
static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1127
{
1128
	struct intel_encoder *encoder;
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
	enum pipe pipe;

	/*
	 * Enable the CRI clock source so we can get at the
	 * display and the reference clock for VGA
	 * hotplug / manual detection. Supposedly DSI also
	 * needs the ref clock up and running.
	 *
	 * CHV DPLL B/C have some issues if VGA mode is enabled.
	 */
1139
	for_each_pipe(dev_priv, pipe) {
1140 1141 1142 1143 1144 1145 1146 1147
		u32 val = I915_READ(DPLL(pipe));

		val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
		if (pipe != PIPE_A)
			val |= DPLL_INTEGRATED_CRI_CLK_VLV;

		I915_WRITE(DPLL(pipe), val);
	}
1148

1149 1150
	vlv_init_display_clock_gating(dev_priv);

1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
	spin_lock_irq(&dev_priv->irq_lock);
	valleyview_enable_display_irqs(dev_priv);
	spin_unlock_irq(&dev_priv->irq_lock);

	/*
	 * During driver initialization/resume we can avoid restoring the
	 * part of the HW/SW state that will be inited anyway explicitly.
	 */
	if (dev_priv->power_domains.initializing)
		return;

1162
	intel_hpd_init(dev_priv);
1163

1164 1165 1166 1167 1168 1169
	/* Re-enable the ADPA, if we have one */
	for_each_intel_encoder(&dev_priv->drm, encoder) {
		if (encoder->type == INTEL_OUTPUT_ANALOG)
			intel_crt_reset(&encoder->base);
	}

1170
	i915_redisable_vga_power_on(dev_priv);
1171 1172

	intel_pps_unlock_regs_wa(dev_priv);
1173 1174
}

1175 1176 1177 1178 1179 1180
static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	valleyview_disable_display_irqs(dev_priv);
	spin_unlock_irq(&dev_priv->irq_lock);

1181
	/* make sure we're done processing display irqs */
1182
	synchronize_irq(dev_priv->drm.irq);
1183

1184
	intel_power_sequencer_reset(dev_priv);
1185

1186 1187 1188
	/* Prevent us from re-enabling polling on accident in late suspend */
	if (!dev_priv->drm.dev->power.is_suspended)
		intel_hpd_poll_init(dev_priv);
1189 1190 1191 1192 1193
}

static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
					  struct i915_power_well *power_well)
{
1194
	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
1195 1196 1197 1198 1199 1200

	vlv_set_power_well(dev_priv, power_well, true);

	vlv_display_power_well_init(dev_priv);
}

1201 1202 1203
static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
1204
	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
1205

1206
	vlv_display_power_well_deinit(dev_priv);
1207 1208 1209 1210 1211 1212 1213

	vlv_set_power_well(dev_priv, power_well, false);
}

static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
1214
	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1215

1216
	/* since ref/cri clock was enabled */
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */

	vlv_set_power_well(dev_priv, power_well, true);

	/*
	 * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
	 *  6.	De-assert cmn_reset/side_reset. Same as VLV X0.
	 *   a.	GUnit 0x2110 bit[0] set to 1 (def 0)
	 *   b.	The other bits such as sfr settings / modesel may all
	 *	be set to 0.
	 *
	 * This should only be done on init and resume from S3 with
	 * both PLLs disabled, or we risk losing DPIO and PLL
	 * synchronization.
	 */
	I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
}

static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
					    struct i915_power_well *power_well)
{
	enum pipe pipe;

1240
	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250

	for_each_pipe(dev_priv, pipe)
		assert_pll_disabled(dev_priv, pipe);

	/* Assert common reset */
	I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);

	vlv_set_power_well(dev_priv, power_well, false);
}

1251
#define POWER_DOMAIN_MASK (GENMASK(POWER_DOMAIN_NUM - 1, 0))
1252 1253 1254 1255 1256 1257 1258

static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
						 int power_well_id)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	int i;

1259 1260 1261 1262
	for (i = 0; i < power_domains->power_well_count; i++) {
		struct i915_power_well *power_well;

		power_well = &power_domains->power_wells[i];
1263
		if (power_well->id == power_well_id)
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
			return power_well;
	}

	return NULL;
}

#define BITS_SET(val, bits) (((val) & (bits)) == (bits))

static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
{
	struct i915_power_well *cmn_bc =
		lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
	struct i915_power_well *cmn_d =
		lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
	u32 phy_control = dev_priv->chv_phy_control;
	u32 phy_status = 0;
1280
	u32 phy_status_mask = 0xffffffff;
1281

1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
	/*
	 * The BIOS can leave the PHY is some weird state
	 * where it doesn't fully power down some parts.
	 * Disable the asserts until the PHY has been fully
	 * reset (ie. the power well has been disabled at
	 * least once).
	 */
	if (!dev_priv->chv_phy_assert[DPIO_PHY0])
		phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
				     PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));

	if (!dev_priv->chv_phy_assert[DPIO_PHY1])
		phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
				     PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
				     PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));

1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
	if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
		phy_status |= PHY_POWERGOOD(DPIO_PHY0);

		/* this assumes override is only used to enable lanes */
		if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
			phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);

		if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
			phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);

		/* CL1 is on whenever anything is on in either channel */
		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
			phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);

		/*
		 * The DPLLB check accounts for the pipe B + port A usage
		 * with CL2 powered up but all the lanes in the second channel
		 * powered down.
		 */
		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
		    (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
			phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);

		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);

		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
	}

	if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
		phy_status |= PHY_POWERGOOD(DPIO_PHY1);

		/* this assumes override is only used to enable lanes */
		if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
			phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);

		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
			phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);

		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
		if (BITS_SET(phy_control,
			     PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
			phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
	}

1362 1363
	phy_status &= phy_status_mask;

1364 1365 1366 1367
	/*
	 * The PHY may be busy with some initial calibration and whatnot,
	 * so the power state can take a while to actually change.
	 */
1368 1369 1370 1371 1372 1373 1374 1375
	if (intel_wait_for_register(dev_priv,
				    DISPLAY_PHY_STATUS,
				    phy_status_mask,
				    phy_status,
				    10))
		DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
			  I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
			   phy_status, dev_priv->chv_phy_control);
1376 1377 1378 1379
}

#undef BITS_SET

1380 1381 1382 1383
static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
					   struct i915_power_well *power_well)
{
	enum dpio_phy phy;
1384 1385
	enum pipe pipe;
	uint32_t tmp;
1386

1387 1388
	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
		     power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1389

1390
	if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1391
		pipe = PIPE_A;
1392
		phy = DPIO_PHY0;
1393 1394
	} else {
		pipe = PIPE_C;
1395
		phy = DPIO_PHY1;
1396
	}
1397 1398

	/* since ref/cri clock was enabled */
1399 1400 1401 1402
	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
	vlv_set_power_well(dev_priv, power_well, true);

	/* Poll for phypwrgood signal */
1403 1404 1405 1406 1407
	if (intel_wait_for_register(dev_priv,
				    DISPLAY_PHY_STATUS,
				    PHY_POWERGOOD(phy),
				    PHY_POWERGOOD(phy),
				    1))
1408 1409
		DRM_ERROR("Display PHY %d is not power up\n", phy);

1410 1411 1412 1413
	mutex_lock(&dev_priv->sb_lock);

	/* Enable dynamic power down */
	tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1414 1415
	tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
		DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1416 1417
	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);

1418
	if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1419 1420 1421
		tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
		tmp |= DPIO_DYNPWRDOWNEN_CH1;
		vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1422 1423 1424 1425 1426 1427 1428 1429 1430
	} else {
		/*
		 * Force the non-existing CL2 off. BXT does this
		 * too, so maybe it saves some power even though
		 * CL2 doesn't exist?
		 */
		tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
		tmp |= DPIO_CL2_LDOFUSE_PWRENB;
		vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1431 1432 1433 1434
	}

	mutex_unlock(&dev_priv->sb_lock);

1435 1436
	dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
	I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1437 1438 1439

	DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
		      phy, dev_priv->chv_phy_control);
1440 1441

	assert_chv_phy_status(dev_priv);
1442 1443 1444 1445 1446 1447 1448
}

static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
					    struct i915_power_well *power_well)
{
	enum dpio_phy phy;

1449 1450
	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
		     power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1451

1452
	if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1453 1454 1455 1456 1457 1458 1459 1460
		phy = DPIO_PHY0;
		assert_pll_disabled(dev_priv, PIPE_A);
		assert_pll_disabled(dev_priv, PIPE_B);
	} else {
		phy = DPIO_PHY1;
		assert_pll_disabled(dev_priv, PIPE_C);
	}

1461 1462
	dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
	I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1463 1464

	vlv_set_power_well(dev_priv, power_well, false);
1465 1466 1467

	DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
		      phy, dev_priv->chv_phy_control);
1468

1469 1470 1471
	/* PHY is fully reset now, so we can enable the PHY state asserts */
	dev_priv->chv_phy_assert[phy] = true;

1472
	assert_chv_phy_status(dev_priv);
1473 1474
}

1475 1476 1477 1478 1479 1480
static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
				     enum dpio_channel ch, bool override, unsigned int mask)
{
	enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
	u32 reg, val, expected, actual;

1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
	/*
	 * The BIOS can leave the PHY is some weird state
	 * where it doesn't fully power down some parts.
	 * Disable the asserts until the PHY has been fully
	 * reset (ie. the power well has been disabled at
	 * least once).
	 */
	if (!dev_priv->chv_phy_assert[phy])
		return;

1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
	if (ch == DPIO_CH0)
		reg = _CHV_CMN_DW0_CH0;
	else
		reg = _CHV_CMN_DW6_CH1;

	mutex_lock(&dev_priv->sb_lock);
	val = vlv_dpio_read(dev_priv, pipe, reg);
	mutex_unlock(&dev_priv->sb_lock);

	/*
	 * This assumes !override is only used when the port is disabled.
	 * All lanes should power down even without the override when
	 * the port is disabled.
	 */
	if (!override || mask == 0xf) {
		expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
		/*
		 * If CH1 common lane is not active anymore
		 * (eg. for pipe B DPLL) the entire channel will
		 * shut down, which causes the common lane registers
		 * to read as 0. That means we can't actually check
		 * the lane power down status bits, but as the entire
		 * register reads as 0 it's a good indication that the
		 * channel is indeed entirely powered down.
		 */
		if (ch == DPIO_CH1 && val == 0)
			expected = 0;
	} else if (mask != 0x0) {
		expected = DPIO_ANYDL_POWERDOWN;
	} else {
		expected = 0;
	}

	if (ch == DPIO_CH0)
		actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
	else
		actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
	actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;

	WARN(actual != expected,
	     "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
	     !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
	     !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
	     reg, val);
}

1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
			  enum dpio_channel ch, bool override)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	bool was_override;

	mutex_lock(&power_domains->lock);

	was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);

	if (override == was_override)
		goto out;

	if (override)
		dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
	else
		dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);

	I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);

	DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
		      phy, ch, dev_priv->chv_phy_control);

1560 1561
	assert_chv_phy_status(dev_priv);

1562 1563 1564 1565 1566 1567
out:
	mutex_unlock(&power_domains->lock);

	return was_override;
}

1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590
void chv_phy_powergate_lanes(struct intel_encoder *encoder,
			     bool override, unsigned int mask)
{
	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
	enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));

	mutex_lock(&power_domains->lock);

	dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
	dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);

	if (override)
		dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
	else
		dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);

	I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);

	DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
		      phy, ch, mask, dev_priv->chv_phy_control);

1591 1592
	assert_chv_phy_status(dev_priv);

1593 1594
	assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);

1595
	mutex_unlock(&power_domains->lock);
1596 1597 1598 1599 1600
}

static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
					struct i915_power_well *power_well)
{
1601
	enum pipe pipe = power_well->id;
1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
	bool enabled;
	u32 state, ctrl;

	mutex_lock(&dev_priv->rps.hw_lock);

	state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
	/*
	 * We only ever set the power-on and power-gate states, anything
	 * else is unexpected.
	 */
	WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
	enabled = state == DP_SSS_PWR_ON(pipe);

	/*
	 * A transient state at this point would mean some unexpected party
	 * is poking at the power controls too.
	 */
	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
	WARN_ON(ctrl << 16 != state);

	mutex_unlock(&dev_priv->rps.hw_lock);

	return enabled;
}

static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
				    struct i915_power_well *power_well,
				    bool enable)
{
1631
	enum pipe pipe = power_well->id;
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
	u32 state;
	u32 ctrl;

	state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);

	mutex_lock(&dev_priv->rps.hw_lock);

#define COND \
	((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)

	if (COND)
		goto out;

	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
	ctrl &= ~DP_SSC_MASK(pipe);
	ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);

	if (wait_for(COND, 100))
1651
		DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
			  state,
			  vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));

#undef COND

out:
	mutex_unlock(&dev_priv->rps.hw_lock);
}

static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
					struct i915_power_well *power_well)
{
1664
	WARN_ON_ONCE(power_well->id != PIPE_A);
1665

1666 1667 1668 1669 1670 1671
	chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0);
}

static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
				       struct i915_power_well *power_well)
{
1672
	WARN_ON_ONCE(power_well->id != PIPE_A);
1673 1674

	chv_set_pipe_power_well(dev_priv, power_well, true);
1675

1676
	vlv_display_power_well_init(dev_priv);
1677 1678 1679 1680 1681
}

static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
					struct i915_power_well *power_well)
{
1682
	WARN_ON_ONCE(power_well->id != PIPE_A);
1683

1684
	vlv_display_power_well_deinit(dev_priv);
1685

1686 1687 1688
	chv_set_pipe_power_well(dev_priv, power_well, false);
}

1689 1690 1691 1692 1693 1694 1695 1696
static void
__intel_display_power_get_domain(struct drm_i915_private *dev_priv,
				 enum intel_display_power_domain domain)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	struct i915_power_well *power_well;
	int i;

1697 1698
	for_each_power_well(i, power_well, BIT(domain), power_domains)
		intel_power_well_get(dev_priv, power_well);
1699 1700 1701 1702

	power_domains->domain_use_count[domain]++;
}

1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
/**
 * intel_display_power_get - grab a power domain reference
 * @dev_priv: i915 device instance
 * @domain: power domain to reference
 *
 * This function grabs a power domain reference for @domain and ensures that the
 * power domain and all its parents are powered up. Therefore users should only
 * grab a reference to the innermost power domain they need.
 *
 * Any power domain reference obtained by this function must have a symmetric
 * call to intel_display_power_put() to release the reference again.
 */
1715 1716 1717
void intel_display_power_get(struct drm_i915_private *dev_priv,
			     enum intel_display_power_domain domain)
{
1718
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
1719 1720 1721

	intel_runtime_pm_get(dev_priv);

1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748
	mutex_lock(&power_domains->lock);

	__intel_display_power_get_domain(dev_priv, domain);

	mutex_unlock(&power_domains->lock);
}

/**
 * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
 * @dev_priv: i915 device instance
 * @domain: power domain to reference
 *
 * This function grabs a power domain reference for @domain and ensures that the
 * power domain and all its parents are powered up. Therefore users should only
 * grab a reference to the innermost power domain they need.
 *
 * Any power domain reference obtained by this function must have a symmetric
 * call to intel_display_power_put() to release the reference again.
 */
bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
					enum intel_display_power_domain domain)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	bool is_enabled;

	if (!intel_runtime_pm_get_if_in_use(dev_priv))
		return false;
1749 1750 1751

	mutex_lock(&power_domains->lock);

1752 1753 1754 1755 1756
	if (__intel_display_power_is_enabled(dev_priv, domain)) {
		__intel_display_power_get_domain(dev_priv, domain);
		is_enabled = true;
	} else {
		is_enabled = false;
1757 1758 1759
	}

	mutex_unlock(&power_domains->lock);
1760 1761 1762 1763 1764

	if (!is_enabled)
		intel_runtime_pm_put(dev_priv);

	return is_enabled;
1765 1766
}

1767 1768 1769 1770 1771 1772 1773 1774 1775
/**
 * intel_display_power_put - release a power domain reference
 * @dev_priv: i915 device instance
 * @domain: power domain to reference
 *
 * This function drops the power domain reference obtained by
 * intel_display_power_get() and might power down the corresponding hardware
 * block right away if this is the last reference.
 */
1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
void intel_display_power_put(struct drm_i915_private *dev_priv,
			     enum intel_display_power_domain domain)
{
	struct i915_power_domains *power_domains;
	struct i915_power_well *power_well;
	int i;

	power_domains = &dev_priv->power_domains;

	mutex_lock(&power_domains->lock);

1787 1788 1789
	WARN(!power_domains->domain_use_count[domain],
	     "Use count on domain %s is already zero\n",
	     intel_display_power_domain_str(domain));
1790 1791
	power_domains->domain_use_count[domain]--;

1792 1793
	for_each_power_well_rev(i, power_well, BIT(domain), power_domains)
		intel_power_well_put(dev_priv, power_well);
1794 1795 1796 1797 1798 1799

	mutex_unlock(&power_domains->lock);

	intel_runtime_pm_put(dev_priv);
}

1800 1801 1802 1803 1804 1805 1806 1807 1808
#define HSW_DISPLAY_POWER_DOMAINS (			\
	BIT(POWER_DOMAIN_PIPE_B) |			\
	BIT(POWER_DOMAIN_PIPE_C) |			\
	BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_TRANSCODER_A) |		\
	BIT(POWER_DOMAIN_TRANSCODER_B) |		\
	BIT(POWER_DOMAIN_TRANSCODER_C) |		\
1809 1810 1811
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
1812 1813 1814
	BIT(POWER_DOMAIN_PORT_CRT) | /* DDI E */	\
	BIT(POWER_DOMAIN_VGA) |				\
	BIT(POWER_DOMAIN_AUDIO) |			\
1815 1816
	BIT(POWER_DOMAIN_INIT))

1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
#define BDW_DISPLAY_POWER_DOMAINS (			\
	BIT(POWER_DOMAIN_PIPE_B) |			\
	BIT(POWER_DOMAIN_PIPE_C) |			\
	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
	BIT(POWER_DOMAIN_TRANSCODER_A) |		\
	BIT(POWER_DOMAIN_TRANSCODER_B) |		\
	BIT(POWER_DOMAIN_TRANSCODER_C) |		\
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
	BIT(POWER_DOMAIN_PORT_CRT) | /* DDI E */	\
	BIT(POWER_DOMAIN_VGA) |				\
	BIT(POWER_DOMAIN_AUDIO) |			\
1831 1832
	BIT(POWER_DOMAIN_INIT))

1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849
#define VLV_DISPLAY_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_PIPE_A) |		\
	BIT(POWER_DOMAIN_PIPE_B) |		\
	BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |	\
	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |	\
	BIT(POWER_DOMAIN_TRANSCODER_A) |	\
	BIT(POWER_DOMAIN_TRANSCODER_B) |	\
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
	BIT(POWER_DOMAIN_PORT_DSI) |		\
	BIT(POWER_DOMAIN_PORT_CRT) |		\
	BIT(POWER_DOMAIN_VGA) |			\
	BIT(POWER_DOMAIN_AUDIO) |		\
	BIT(POWER_DOMAIN_AUX_B) |		\
	BIT(POWER_DOMAIN_AUX_C) |		\
	BIT(POWER_DOMAIN_GMBUS) |		\
	BIT(POWER_DOMAIN_INIT))
1850 1851

#define VLV_DPIO_CMN_BC_POWER_DOMAINS (		\
1852 1853
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
1854
	BIT(POWER_DOMAIN_PORT_CRT) |		\
1855 1856
	BIT(POWER_DOMAIN_AUX_B) |		\
	BIT(POWER_DOMAIN_AUX_C) |		\
1857 1858 1859
	BIT(POWER_DOMAIN_INIT))

#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (	\
1860
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
1861
	BIT(POWER_DOMAIN_AUX_B) |		\
1862 1863 1864
	BIT(POWER_DOMAIN_INIT))

#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (	\
1865
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
1866
	BIT(POWER_DOMAIN_AUX_B) |		\
1867 1868 1869
	BIT(POWER_DOMAIN_INIT))

#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (	\
1870
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
1871
	BIT(POWER_DOMAIN_AUX_C) |		\
1872 1873 1874
	BIT(POWER_DOMAIN_INIT))

#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (	\
1875
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
1876
	BIT(POWER_DOMAIN_AUX_C) |		\
1877 1878
	BIT(POWER_DOMAIN_INIT))

1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900
#define CHV_DISPLAY_POWER_DOMAINS (		\
	BIT(POWER_DOMAIN_PIPE_A) |		\
	BIT(POWER_DOMAIN_PIPE_B) |		\
	BIT(POWER_DOMAIN_PIPE_C) |		\
	BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |	\
	BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |	\
	BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |	\
	BIT(POWER_DOMAIN_TRANSCODER_A) |	\
	BIT(POWER_DOMAIN_TRANSCODER_B) |	\
	BIT(POWER_DOMAIN_TRANSCODER_C) |	\
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |	\
	BIT(POWER_DOMAIN_PORT_DSI) |		\
	BIT(POWER_DOMAIN_VGA) |			\
	BIT(POWER_DOMAIN_AUDIO) |		\
	BIT(POWER_DOMAIN_AUX_B) |		\
	BIT(POWER_DOMAIN_AUX_C) |		\
	BIT(POWER_DOMAIN_AUX_D) |		\
	BIT(POWER_DOMAIN_GMBUS) |		\
	BIT(POWER_DOMAIN_INIT))

1901
#define CHV_DPIO_CMN_BC_POWER_DOMAINS (		\
1902 1903
	BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |	\
	BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |	\
1904 1905
	BIT(POWER_DOMAIN_AUX_B) |		\
	BIT(POWER_DOMAIN_AUX_C) |		\
1906 1907 1908
	BIT(POWER_DOMAIN_INIT))

#define CHV_DPIO_CMN_D_POWER_DOMAINS (		\
1909
	BIT(POWER_DOMAIN_PORT_DDI_D_LANES) |	\
1910
	BIT(POWER_DOMAIN_AUX_D) |		\
1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949
	BIT(POWER_DOMAIN_INIT))

static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
	.sync_hw = i9xx_always_on_power_well_noop,
	.enable = i9xx_always_on_power_well_noop,
	.disable = i9xx_always_on_power_well_noop,
	.is_enabled = i9xx_always_on_power_well_enabled,
};

static const struct i915_power_well_ops chv_pipe_power_well_ops = {
	.sync_hw = chv_pipe_power_well_sync_hw,
	.enable = chv_pipe_power_well_enable,
	.disable = chv_pipe_power_well_disable,
	.is_enabled = chv_pipe_power_well_enabled,
};

static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
	.sync_hw = vlv_power_well_sync_hw,
	.enable = chv_dpio_cmn_power_well_enable,
	.disable = chv_dpio_cmn_power_well_disable,
	.is_enabled = vlv_power_well_enabled,
};

static struct i915_power_well i9xx_always_on_power_well[] = {
	{
		.name = "always-on",
		.always_on = 1,
		.domains = POWER_DOMAIN_MASK,
		.ops = &i9xx_always_on_power_well_ops,
	},
};

static const struct i915_power_well_ops hsw_power_well_ops = {
	.sync_hw = hsw_power_well_sync_hw,
	.enable = hsw_power_well_enable,
	.disable = hsw_power_well_disable,
	.is_enabled = hsw_power_well_enabled,
};

1950 1951 1952 1953 1954 1955 1956
static const struct i915_power_well_ops skl_power_well_ops = {
	.sync_hw = skl_power_well_sync_hw,
	.enable = skl_power_well_enable,
	.disable = skl_power_well_disable,
	.is_enabled = skl_power_well_enabled,
};

1957 1958 1959 1960 1961 1962 1963
static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
	.sync_hw = gen9_dc_off_power_well_sync_hw,
	.enable = gen9_dc_off_power_well_enable,
	.disable = gen9_dc_off_power_well_disable,
	.is_enabled = gen9_dc_off_power_well_enabled,
};

1964 1965 1966 1967 1968 1969 1970
static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
	.sync_hw = bxt_dpio_cmn_power_well_sync_hw,
	.enable = bxt_dpio_cmn_power_well_enable,
	.disable = bxt_dpio_cmn_power_well_disable,
	.is_enabled = bxt_dpio_cmn_power_well_enabled,
};

1971 1972 1973 1974
static struct i915_power_well hsw_power_wells[] = {
	{
		.name = "always-on",
		.always_on = 1,
1975
		.domains = POWER_DOMAIN_MASK,
1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
		.ops = &i9xx_always_on_power_well_ops,
	},
	{
		.name = "display",
		.domains = HSW_DISPLAY_POWER_DOMAINS,
		.ops = &hsw_power_well_ops,
	},
};

static struct i915_power_well bdw_power_wells[] = {
	{
		.name = "always-on",
		.always_on = 1,
1989
		.domains = POWER_DOMAIN_MASK,
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023
		.ops = &i9xx_always_on_power_well_ops,
	},
	{
		.name = "display",
		.domains = BDW_DISPLAY_POWER_DOMAINS,
		.ops = &hsw_power_well_ops,
	},
};

static const struct i915_power_well_ops vlv_display_power_well_ops = {
	.sync_hw = vlv_power_well_sync_hw,
	.enable = vlv_display_power_well_enable,
	.disable = vlv_display_power_well_disable,
	.is_enabled = vlv_power_well_enabled,
};

static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
	.sync_hw = vlv_power_well_sync_hw,
	.enable = vlv_dpio_cmn_power_well_enable,
	.disable = vlv_dpio_cmn_power_well_disable,
	.is_enabled = vlv_power_well_enabled,
};

static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
	.sync_hw = vlv_power_well_sync_hw,
	.enable = vlv_power_well_enable,
	.disable = vlv_power_well_disable,
	.is_enabled = vlv_power_well_enabled,
};

static struct i915_power_well vlv_power_wells[] = {
	{
		.name = "always-on",
		.always_on = 1,
2024
		.domains = POWER_DOMAIN_MASK,
2025
		.ops = &i9xx_always_on_power_well_ops,
2026
		.id = PUNIT_POWER_WELL_ALWAYS_ON,
2027 2028 2029 2030
	},
	{
		.name = "display",
		.domains = VLV_DISPLAY_POWER_DOMAINS,
2031
		.id = PUNIT_POWER_WELL_DISP2D,
2032 2033 2034 2035 2036 2037 2038 2039 2040
		.ops = &vlv_display_power_well_ops,
	},
	{
		.name = "dpio-tx-b-01",
		.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		.ops = &vlv_dpio_power_well_ops,
2041
		.id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
2042 2043 2044 2045 2046 2047 2048 2049
	},
	{
		.name = "dpio-tx-b-23",
		.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		.ops = &vlv_dpio_power_well_ops,
2050
		.id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
2051 2052 2053 2054 2055 2056 2057 2058
	},
	{
		.name = "dpio-tx-c-01",
		.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		.ops = &vlv_dpio_power_well_ops,
2059
		.id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
2060 2061 2062 2063 2064 2065 2066 2067
	},
	{
		.name = "dpio-tx-c-23",
		.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		.ops = &vlv_dpio_power_well_ops,
2068
		.id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
2069 2070 2071 2072
	},
	{
		.name = "dpio-common",
		.domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2073
		.id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2074 2075 2076 2077 2078 2079 2080 2081
		.ops = &vlv_dpio_cmn_power_well_ops,
	},
};

static struct i915_power_well chv_power_wells[] = {
	{
		.name = "always-on",
		.always_on = 1,
2082
		.domains = POWER_DOMAIN_MASK,
2083 2084 2085 2086
		.ops = &i9xx_always_on_power_well_ops,
	},
	{
		.name = "display",
2087
		/*
2088 2089 2090
		 * Pipe A power well is the new disp2d well. Pipe B and C
		 * power wells don't actually exist. Pipe A power well is
		 * required for any pipe to work.
2091
		 */
2092
		.domains = CHV_DISPLAY_POWER_DOMAINS,
2093
		.id = PIPE_A,
2094 2095 2096 2097
		.ops = &chv_pipe_power_well_ops,
	},
	{
		.name = "dpio-common-bc",
2098
		.domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2099
		.id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2100 2101 2102 2103
		.ops = &chv_dpio_cmn_power_well_ops,
	},
	{
		.name = "dpio-common-d",
2104
		.domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2105
		.id = PUNIT_POWER_WELL_DPIO_CMN_D,
2106 2107 2108 2109
		.ops = &chv_dpio_cmn_power_well_ops,
	},
};

2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121
bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
				    int power_well_id)
{
	struct i915_power_well *power_well;
	bool ret;

	power_well = lookup_power_well(dev_priv, power_well_id);
	ret = power_well->ops->is_enabled(dev_priv, power_well);

	return ret;
}

2122 2123 2124 2125
static struct i915_power_well skl_power_wells[] = {
	{
		.name = "always-on",
		.always_on = 1,
2126
		.domains = POWER_DOMAIN_MASK,
2127
		.ops = &i9xx_always_on_power_well_ops,
2128
		.id = SKL_DISP_PW_ALWAYS_ON,
2129 2130 2131
	},
	{
		.name = "power well 1",
2132 2133
		/* Handled by the DMC firmware */
		.domains = 0,
2134
		.ops = &skl_power_well_ops,
2135
		.id = SKL_DISP_PW_1,
2136 2137 2138
	},
	{
		.name = "MISC IO power well",
2139 2140
		/* Handled by the DMC firmware */
		.domains = 0,
2141
		.ops = &skl_power_well_ops,
2142
		.id = SKL_DISP_PW_MISC_IO,
2143
	},
2144 2145 2146 2147
	{
		.name = "DC off",
		.domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
		.ops = &gen9_dc_off_power_well_ops,
2148
		.id = SKL_DISP_PW_DC_OFF,
2149
	},
2150 2151 2152 2153
	{
		.name = "power well 2",
		.domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
2154
		.id = SKL_DISP_PW_2,
2155 2156 2157 2158 2159
	},
	{
		.name = "DDI A/E power well",
		.domains = SKL_DISPLAY_DDI_A_E_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
2160
		.id = SKL_DISP_PW_DDI_A_E,
2161 2162 2163 2164 2165
	},
	{
		.name = "DDI B power well",
		.domains = SKL_DISPLAY_DDI_B_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
2166
		.id = SKL_DISP_PW_DDI_B,
2167 2168 2169 2170 2171
	},
	{
		.name = "DDI C power well",
		.domains = SKL_DISPLAY_DDI_C_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
2172
		.id = SKL_DISP_PW_DDI_C,
2173 2174 2175 2176 2177
	},
	{
		.name = "DDI D power well",
		.domains = SKL_DISPLAY_DDI_D_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
2178
		.id = SKL_DISP_PW_DDI_D,
2179 2180 2181
	},
};

2182 2183 2184 2185
static struct i915_power_well bxt_power_wells[] = {
	{
		.name = "always-on",
		.always_on = 1,
2186
		.domains = POWER_DOMAIN_MASK,
2187 2188 2189 2190
		.ops = &i9xx_always_on_power_well_ops,
	},
	{
		.name = "power well 1",
2191
		.domains = 0,
2192
		.ops = &skl_power_well_ops,
2193
		.id = SKL_DISP_PW_1,
2194
	},
2195 2196 2197 2198
	{
		.name = "DC off",
		.domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
		.ops = &gen9_dc_off_power_well_ops,
2199
		.id = SKL_DISP_PW_DC_OFF,
2200
	},
2201 2202 2203 2204
	{
		.name = "power well 2",
		.domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
2205
		.id = SKL_DISP_PW_2,
2206
	},
2207 2208 2209 2210
	{
		.name = "dpio-common-a",
		.domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
		.ops = &bxt_dpio_cmn_power_well_ops,
2211
		.id = BXT_DPIO_CMN_A,
2212
		.data = DPIO_PHY1,
2213 2214 2215 2216 2217
	},
	{
		.name = "dpio-common-bc",
		.domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
		.ops = &bxt_dpio_cmn_power_well_ops,
2218
		.id = BXT_DPIO_CMN_BC,
2219
		.data = DPIO_PHY0,
2220
	},
2221 2222
};

2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248
static struct i915_power_well glk_power_wells[] = {
	{
		.name = "always-on",
		.always_on = 1,
		.domains = POWER_DOMAIN_MASK,
		.ops = &i9xx_always_on_power_well_ops,
	},
	{
		.name = "power well 1",
		/* Handled by the DMC firmware */
		.domains = 0,
		.ops = &skl_power_well_ops,
		.id = SKL_DISP_PW_1,
	},
	{
		.name = "DC off",
		.domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
		.ops = &gen9_dc_off_power_well_ops,
		.id = SKL_DISP_PW_DC_OFF,
	},
	{
		.name = "power well 2",
		.domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
		.id = SKL_DISP_PW_2,
	},
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269
	{
		.name = "dpio-common-a",
		.domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
		.ops = &bxt_dpio_cmn_power_well_ops,
		.id = BXT_DPIO_CMN_A,
		.data = DPIO_PHY1,
	},
	{
		.name = "dpio-common-b",
		.domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
		.ops = &bxt_dpio_cmn_power_well_ops,
		.id = BXT_DPIO_CMN_BC,
		.data = DPIO_PHY0,
	},
	{
		.name = "dpio-common-c",
		.domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
		.ops = &bxt_dpio_cmn_power_well_ops,
		.id = GLK_DPIO_CMN_C,
		.data = DPIO_PHY2,
	},
2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307
	{
		.name = "AUX A",
		.domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
		.id = GLK_DISP_PW_AUX_A,
	},
	{
		.name = "AUX B",
		.domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
		.id = GLK_DISP_PW_AUX_B,
	},
	{
		.name = "AUX C",
		.domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
		.id = GLK_DISP_PW_AUX_C,
	},
	{
		.name = "DDI A power well",
		.domains = GLK_DISPLAY_DDI_A_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
		.id = GLK_DISP_PW_DDI_A,
	},
	{
		.name = "DDI B power well",
		.domains = GLK_DISPLAY_DDI_B_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
		.id = SKL_DISP_PW_DDI_B,
	},
	{
		.name = "DDI C power well",
		.domains = GLK_DISPLAY_DDI_C_POWER_DOMAINS,
		.ops = &skl_power_well_ops,
		.id = SKL_DISP_PW_DDI_C,
	},
};

2308 2309 2310 2311 2312 2313 2314 2315 2316 2317
static int
sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
				   int disable_power_well)
{
	if (disable_power_well >= 0)
		return !!disable_power_well;

	return 1;
}

2318 2319 2320 2321 2322 2323 2324
static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
				    int enable_dc)
{
	uint32_t mask;
	int requested_dc;
	int max_dc;

2325
	if (IS_GEN9_BC(dev_priv)) {
2326 2327
		max_dc = 2;
		mask = 0;
2328
	} else if (IS_GEN9_LP(dev_priv)) {
2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
		max_dc = 1;
		/*
		 * DC9 has a separate HW flow from the rest of the DC states,
		 * not depending on the DMC firmware. It's needed by system
		 * suspend/resume, so allow it unconditionally.
		 */
		mask = DC_STATE_EN_DC9;
	} else {
		max_dc = 0;
		mask = 0;
	}

2341 2342 2343
	if (!i915.disable_power_well)
		max_dc = 0;

2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366
	if (enable_dc >= 0 && enable_dc <= max_dc) {
		requested_dc = enable_dc;
	} else if (enable_dc == -1) {
		requested_dc = max_dc;
	} else if (enable_dc > max_dc && enable_dc <= 2) {
		DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
			      enable_dc, max_dc);
		requested_dc = max_dc;
	} else {
		DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
		requested_dc = max_dc;
	}

	if (requested_dc > 1)
		mask |= DC_STATE_EN_UPTO_DC6;
	if (requested_dc > 0)
		mask |= DC_STATE_EN_UPTO_DC5;

	DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);

	return mask;
}

2367 2368 2369 2370 2371
#define set_power_wells(power_domains, __power_wells) ({		\
	(power_domains)->power_wells = (__power_wells);			\
	(power_domains)->power_well_count = ARRAY_SIZE(__power_wells);	\
})

2372 2373 2374 2375 2376 2377 2378
/**
 * intel_power_domains_init - initializes the power domain structures
 * @dev_priv: i915 device instance
 *
 * Initializes the power domain structures for @dev_priv depending upon the
 * supported platform.
 */
2379 2380 2381 2382
int intel_power_domains_init(struct drm_i915_private *dev_priv)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;

2383 2384
	i915.disable_power_well = sanitize_disable_power_well_option(dev_priv,
						     i915.disable_power_well);
2385 2386
	dev_priv->csr.allowed_dc_mask = get_allowed_dc_mask(dev_priv,
							    i915.enable_dc);
2387

2388 2389
	BUILD_BUG_ON(POWER_DOMAIN_NUM > 31);

2390 2391 2392 2393 2394 2395
	mutex_init(&power_domains->lock);

	/*
	 * The enabling order will be from lower to higher indexed wells,
	 * the disabling order is reversed.
	 */
2396
	if (IS_HASWELL(dev_priv)) {
2397
		set_power_wells(power_domains, hsw_power_wells);
2398
	} else if (IS_BROADWELL(dev_priv)) {
2399
		set_power_wells(power_domains, bdw_power_wells);
2400
	} else if (IS_GEN9_BC(dev_priv)) {
2401
		set_power_wells(power_domains, skl_power_wells);
2402
	} else if (IS_BROXTON(dev_priv)) {
2403
		set_power_wells(power_domains, bxt_power_wells);
2404 2405
	} else if (IS_GEMINILAKE(dev_priv)) {
		set_power_wells(power_domains, glk_power_wells);
2406
	} else if (IS_CHERRYVIEW(dev_priv)) {
2407
		set_power_wells(power_domains, chv_power_wells);
2408
	} else if (IS_VALLEYVIEW(dev_priv)) {
2409 2410 2411 2412 2413 2414 2415 2416
		set_power_wells(power_domains, vlv_power_wells);
	} else {
		set_power_wells(power_domains, i9xx_always_on_power_well);
	}

	return 0;
}

2417 2418 2419 2420 2421 2422 2423 2424
/**
 * intel_power_domains_fini - finalizes the power domain structures
 * @dev_priv: i915 device instance
 *
 * Finalizes the power domain structures for @dev_priv depending upon the
 * supported platform. This function also disables runtime pm and ensures that
 * the device stays powered up so that the driver can be reloaded.
 */
2425
void intel_power_domains_fini(struct drm_i915_private *dev_priv)
2426
{
2427
	struct device *kdev = &dev_priv->drm.pdev->dev;
2428

2429 2430
	/*
	 * The i915.ko module is still not prepared to be loaded when
2431
	 * the power well is not enabled, so just enable it in case
2432 2433 2434 2435 2436 2437
	 * we're going to unload/reload.
	 * The following also reacquires the RPM reference the core passed
	 * to the driver during loading, which is dropped in
	 * intel_runtime_pm_enable(). We have to hand back the control of the
	 * device to the core with this reference held.
	 */
2438
	intel_display_set_init_power(dev_priv, true);
2439 2440 2441 2442

	/* Remove the refcount we took to keep power well support disabled. */
	if (!i915.disable_power_well)
		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2443 2444 2445 2446 2447 2448

	/*
	 * Remove the refcount we took in intel_runtime_pm_enable() in case
	 * the platform doesn't support runtime PM.
	 */
	if (!HAS_RUNTIME_PM(dev_priv))
2449
		pm_runtime_put(kdev);
2450 2451
}

2452
static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	struct i915_power_well *power_well;
	int i;

	mutex_lock(&power_domains->lock);
	for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
		power_well->ops->sync_hw(dev_priv, power_well);
		power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
								     power_well);
	}
	mutex_unlock(&power_domains->lock);
}

2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488
static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
{
	I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
	POSTING_READ(DBUF_CTL);

	udelay(10);

	if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
		DRM_ERROR("DBuf power enable timeout\n");
}

static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
{
	I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
	POSTING_READ(DBUF_CTL);

	udelay(10);

	if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
		DRM_ERROR("DBuf power disable timeout!\n");
}

2489
static void skl_display_core_init(struct drm_i915_private *dev_priv,
2490
				   bool resume)
2491 2492
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
2493
	struct i915_power_well *well;
2494 2495
	uint32_t val;

2496 2497
	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);

2498 2499 2500 2501 2502 2503
	/* enable PCH reset handshake */
	val = I915_READ(HSW_NDE_RSTWRN_OPT);
	I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);

	/* enable PG1 and Misc I/O */
	mutex_lock(&power_domains->lock);
2504 2505 2506 2507 2508 2509 2510

	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
	intel_power_well_enable(dev_priv, well);

	well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
	intel_power_well_enable(dev_priv, well);

2511 2512 2513 2514
	mutex_unlock(&power_domains->lock);

	skl_init_cdclk(dev_priv);

2515 2516
	gen9_dbuf_enable(dev_priv);

2517
	if (resume && dev_priv->csr.dmc_payload)
2518
		intel_csr_load_program(dev_priv);
2519 2520 2521 2522 2523
}

static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
2524
	struct i915_power_well *well;
2525

2526 2527
	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);

2528 2529
	gen9_dbuf_disable(dev_priv);

2530 2531 2532 2533
	skl_uninit_cdclk(dev_priv);

	/* The spec doesn't call for removing the reset handshake flag */
	/* disable PG1 and Misc I/O */
2534

2535
	mutex_lock(&power_domains->lock);
2536 2537 2538 2539 2540 2541 2542

	well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
	intel_power_well_disable(dev_priv, well);

	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
	intel_power_well_disable(dev_priv, well);

2543 2544 2545
	mutex_unlock(&power_domains->lock);
}

2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572
void bxt_display_core_init(struct drm_i915_private *dev_priv,
			   bool resume)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	struct i915_power_well *well;
	uint32_t val;

	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);

	/*
	 * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
	 * or else the reset will hang because there is no PCH to respond.
	 * Move the handshake programming to initialization sequence.
	 * Previously was left up to BIOS.
	 */
	val = I915_READ(HSW_NDE_RSTWRN_OPT);
	val &= ~RESET_PCH_HANDSHAKE_ENABLE;
	I915_WRITE(HSW_NDE_RSTWRN_OPT, val);

	/* Enable PG1 */
	mutex_lock(&power_domains->lock);

	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
	intel_power_well_enable(dev_priv, well);

	mutex_unlock(&power_domains->lock);

2573
	bxt_init_cdclk(dev_priv);
2574 2575 2576

	gen9_dbuf_enable(dev_priv);

2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587
	if (resume && dev_priv->csr.dmc_payload)
		intel_csr_load_program(dev_priv);
}

void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;
	struct i915_power_well *well;

	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);

2588 2589
	gen9_dbuf_disable(dev_priv);

2590
	bxt_uninit_cdclk(dev_priv);
2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602

	/* The spec doesn't call for removing the reset handshake flag */

	/* Disable PG1 */
	mutex_lock(&power_domains->lock);

	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
	intel_power_well_disable(dev_priv, well);

	mutex_unlock(&power_domains->lock);
}

2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
static void chv_phy_control_init(struct drm_i915_private *dev_priv)
{
	struct i915_power_well *cmn_bc =
		lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
	struct i915_power_well *cmn_d =
		lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);

	/*
	 * DISPLAY_PHY_CONTROL can get corrupted if read. As a
	 * workaround never ever read DISPLAY_PHY_CONTROL, and
	 * instead maintain a shadow copy ourselves. Use the actual
2614 2615
	 * power well state and lane status to reconstruct the
	 * expected initial value.
2616 2617
	 */
	dev_priv->chv_phy_control =
2618 2619
		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654
		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);

	/*
	 * If all lanes are disabled we leave the override disabled
	 * with all power down bits cleared to match the state we
	 * would use after disabling the port. Otherwise enable the
	 * override and set the lane powerdown bits accding to the
	 * current lane status.
	 */
	if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
		uint32_t status = I915_READ(DPLL(PIPE_A));
		unsigned int mask;

		mask = status & DPLL_PORTB_READY_MASK;
		if (mask == 0xf)
			mask = 0x0;
		else
			dev_priv->chv_phy_control |=
				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);

		dev_priv->chv_phy_control |=
			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);

		mask = (status & DPLL_PORTC_READY_MASK) >> 4;
		if (mask == 0xf)
			mask = 0x0;
		else
			dev_priv->chv_phy_control |=
				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);

		dev_priv->chv_phy_control |=
			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);

2655
		dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
2656 2657 2658 2659

		dev_priv->chv_phy_assert[DPIO_PHY0] = false;
	} else {
		dev_priv->chv_phy_assert[DPIO_PHY0] = true;
2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676
	}

	if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
		uint32_t status = I915_READ(DPIO_PHY_STATUS);
		unsigned int mask;

		mask = status & DPLL_PORTD_READY_MASK;

		if (mask == 0xf)
			mask = 0x0;
		else
			dev_priv->chv_phy_control |=
				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);

		dev_priv->chv_phy_control |=
			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);

2677
		dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
2678 2679 2680 2681

		dev_priv->chv_phy_assert[DPIO_PHY1] = false;
	} else {
		dev_priv->chv_phy_assert[DPIO_PHY1] = true;
2682 2683 2684 2685 2686 2687
	}

	I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);

	DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
		      dev_priv->chv_phy_control);
2688 2689
}

2690 2691 2692 2693 2694 2695 2696 2697
static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
{
	struct i915_power_well *cmn =
		lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
	struct i915_power_well *disp2d =
		lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);

	/* If the display might be already active skip this */
2698 2699
	if (cmn->ops->is_enabled(dev_priv, cmn) &&
	    disp2d->ops->is_enabled(dev_priv, disp2d) &&
2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717
	    I915_READ(DPIO_CTL) & DPIO_CMNRST)
		return;

	DRM_DEBUG_KMS("toggling display PHY side reset\n");

	/* cmnlane needs DPLL registers */
	disp2d->ops->enable(dev_priv, disp2d);

	/*
	 * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
	 * Need to assert and de-assert PHY SB reset by gating the
	 * common lane power, then un-gating it.
	 * Simply ungating isn't enough to reset the PHY enough to get
	 * ports and lanes running.
	 */
	cmn->ops->disable(dev_priv, cmn);
}

2718 2719 2720
/**
 * intel_power_domains_init_hw - initialize hardware power domain state
 * @dev_priv: i915 device instance
2721
 * @resume: Called from resume code paths or not
2722 2723 2724 2725
 *
 * This function initializes the hardware power domain state and enables all
 * power domains using intel_display_set_init_power().
 */
2726
void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
2727 2728 2729 2730 2731
{
	struct i915_power_domains *power_domains = &dev_priv->power_domains;

	power_domains->initializing = true;

2732
	if (IS_GEN9_BC(dev_priv)) {
2733
		skl_display_core_init(dev_priv, resume);
2734
	} else if (IS_GEN9_LP(dev_priv)) {
2735
		bxt_display_core_init(dev_priv, resume);
2736
	} else if (IS_CHERRYVIEW(dev_priv)) {
2737
		mutex_lock(&power_domains->lock);
2738
		chv_phy_control_init(dev_priv);
2739
		mutex_unlock(&power_domains->lock);
2740
	} else if (IS_VALLEYVIEW(dev_priv)) {
2741 2742 2743 2744 2745 2746 2747
		mutex_lock(&power_domains->lock);
		vlv_cmnlane_wa(dev_priv);
		mutex_unlock(&power_domains->lock);
	}

	/* For now, we need the power well to be always enabled. */
	intel_display_set_init_power(dev_priv, true);
2748 2749 2750
	/* Disable power support if the user asked so. */
	if (!i915.disable_power_well)
		intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
2751
	intel_power_domains_sync_hw(dev_priv);
2752 2753 2754
	power_domains->initializing = false;
}

2755 2756 2757 2758 2759 2760 2761 2762 2763
/**
 * intel_power_domains_suspend - suspend power domain state
 * @dev_priv: i915 device instance
 *
 * This function prepares the hardware power domain state before entering
 * system suspend. It must be paired with intel_power_domains_init_hw().
 */
void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
{
2764 2765 2766 2767 2768 2769
	/*
	 * Even if power well support was disabled we still want to disable
	 * power wells while we are system suspended.
	 */
	if (!i915.disable_power_well)
		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2770

2771
	if (IS_GEN9_BC(dev_priv))
2772
		skl_display_core_uninit(dev_priv);
2773
	else if (IS_GEN9_LP(dev_priv))
2774
		bxt_display_core_uninit(dev_priv);
2775 2776
}

2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
/**
 * intel_runtime_pm_get - grab a runtime pm reference
 * @dev_priv: i915 device instance
 *
 * This function grabs a device-level runtime pm reference (mostly used for GEM
 * code to ensure the GTT or GT is on) and ensures that it is powered up.
 *
 * Any runtime pm reference obtained by this function must have a symmetric
 * call to intel_runtime_pm_put() to release the reference again.
 */
2787 2788
void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
{
D
David Weinehall 已提交
2789 2790
	struct pci_dev *pdev = dev_priv->drm.pdev;
	struct device *kdev = &pdev->dev;
2791

2792
	pm_runtime_get_sync(kdev);
2793 2794

	atomic_inc(&dev_priv->pm.wakeref_count);
2795
	assert_rpm_wakelock_held(dev_priv);
2796 2797
}

2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809
/**
 * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
 * @dev_priv: i915 device instance
 *
 * This function grabs a device-level runtime pm reference if the device is
 * already in use and ensures that it is powered up.
 *
 * Any runtime pm reference obtained by this function must have a symmetric
 * call to intel_runtime_pm_put() to release the reference again.
 */
bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
{
D
David Weinehall 已提交
2810 2811
	struct pci_dev *pdev = dev_priv->drm.pdev;
	struct device *kdev = &pdev->dev;
2812

2813
	if (IS_ENABLED(CONFIG_PM)) {
2814
		int ret = pm_runtime_get_if_in_use(kdev);
2815

2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
		/*
		 * In cases runtime PM is disabled by the RPM core and we get
		 * an -EINVAL return value we are not supposed to call this
		 * function, since the power state is undefined. This applies
		 * atm to the late/early system suspend/resume handlers.
		 */
		WARN_ON_ONCE(ret < 0);
		if (ret <= 0)
			return false;
	}
2826 2827 2828 2829 2830 2831 2832

	atomic_inc(&dev_priv->pm.wakeref_count);
	assert_rpm_wakelock_held(dev_priv);

	return true;
}

2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849
/**
 * intel_runtime_pm_get_noresume - grab a runtime pm reference
 * @dev_priv: i915 device instance
 *
 * This function grabs a device-level runtime pm reference (mostly used for GEM
 * code to ensure the GTT or GT is on).
 *
 * It will _not_ power up the device but instead only check that it's powered
 * on.  Therefore it is only valid to call this functions from contexts where
 * the device is known to be powered up and where trying to power it up would
 * result in hilarity and deadlocks. That pretty much means only the system
 * suspend/resume code where this is used to grab runtime pm references for
 * delayed setup down in work items.
 *
 * Any runtime pm reference obtained by this function must have a symmetric
 * call to intel_runtime_pm_put() to release the reference again.
 */
2850 2851
void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
{
D
David Weinehall 已提交
2852 2853
	struct pci_dev *pdev = dev_priv->drm.pdev;
	struct device *kdev = &pdev->dev;
2854

2855
	assert_rpm_wakelock_held(dev_priv);
2856
	pm_runtime_get_noresume(kdev);
2857 2858

	atomic_inc(&dev_priv->pm.wakeref_count);
2859 2860
}

2861 2862 2863 2864 2865 2866 2867 2868
/**
 * intel_runtime_pm_put - release a runtime pm reference
 * @dev_priv: i915 device instance
 *
 * This function drops the device-level runtime pm reference obtained by
 * intel_runtime_pm_get() and might power down the corresponding
 * hardware block right away if this is the last reference.
 */
2869 2870
void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
{
D
David Weinehall 已提交
2871 2872
	struct pci_dev *pdev = dev_priv->drm.pdev;
	struct device *kdev = &pdev->dev;
2873

2874
	assert_rpm_wakelock_held(dev_priv);
2875
	atomic_dec(&dev_priv->pm.wakeref_count);
2876

2877 2878
	pm_runtime_mark_last_busy(kdev);
	pm_runtime_put_autosuspend(kdev);
2879 2880
}

2881 2882 2883 2884 2885 2886 2887 2888 2889 2890
/**
 * intel_runtime_pm_enable - enable runtime pm
 * @dev_priv: i915 device instance
 *
 * This function enables runtime pm at the end of the driver load sequence.
 *
 * Note that this function does currently not enable runtime pm for the
 * subordinate display power domains. That is only done on the first modeset
 * using intel_display_set_init_power().
 */
2891
void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
2892
{
D
David Weinehall 已提交
2893 2894
	struct pci_dev *pdev = dev_priv->drm.pdev;
	struct device *kdev = &pdev->dev;
2895

2896 2897
	pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
	pm_runtime_mark_last_busy(kdev);
2898

2899 2900 2901 2902 2903 2904
	/*
	 * Take a permanent reference to disable the RPM functionality and drop
	 * it only when unloading the driver. Use the low level get/put helpers,
	 * so the driver's own RPM reference tracking asserts also work on
	 * platforms without RPM support.
	 */
2905
	if (!HAS_RUNTIME_PM(dev_priv)) {
2906 2907
		pm_runtime_dont_use_autosuspend(kdev);
		pm_runtime_get_sync(kdev);
2908
	} else {
2909
		pm_runtime_use_autosuspend(kdev);
2910
	}
2911

2912 2913 2914 2915 2916
	/*
	 * The core calls the driver load handler with an RPM reference held.
	 * We drop that here and will reacquire it during unloading in
	 * intel_power_domains_fini().
	 */
2917
	pm_runtime_put_autosuspend(kdev);
2918
}