i915_irq.c 126.7 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143 144 145 146 147 148 149 150 151 152 153 154 155
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
	     reg, val);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
156

P
Paulo Zanoni 已提交
157
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
158
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
159
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
160 161
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
162 163 164
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
165
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
166
	I915_WRITE(type##IER, (ier_val)); \
167 168
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
169 170
} while (0)

171 172
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

	assert_spin_locked(&dev_priv->irq_lock);
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

211 212 213 214 215 216 217 218 219
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
				   uint32_t interrupt_mask,
				   uint32_t enabled_irq_mask)
220
{
221 222
	uint32_t new_val;

223 224
	assert_spin_locked(&dev_priv->irq_lock);

225 226
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

227
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
228 229
		return;

230 231 232 233 234 235
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
236
		I915_WRITE(DEIMR, dev_priv->irq_mask);
237
		POSTING_READ(DEIMR);
238 239 240
	}
}

241
void
242
ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
243
{
244 245
	ilk_update_display_irq(dev_priv, mask, mask);
}
246

247 248 249 250
void
ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
{
	ilk_update_display_irq(dev_priv, mask, 0);
251 252
}

P
Paulo Zanoni 已提交
253 254 255 256 257 258 259 260 261 262 263 264
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

265 266
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

267
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
268 269
		return;

P
Paulo Zanoni 已提交
270 271 272 273 274 275
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

276
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
277 278 279 280
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

281
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
282 283 284 285
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

286 287 288 289 290
static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

291 292 293 294 295
static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

296 297 298 299 300
static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
301 302 303 304 305 306 307 308 309 310
/**
  * snb_update_pm_irq - update GEN6_PMIMR
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
311
	uint32_t new_val;
P
Paulo Zanoni 已提交
312

313 314
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

P
Paulo Zanoni 已提交
315 316
	assert_spin_locked(&dev_priv->irq_lock);

317
	new_val = dev_priv->pm_irq_mask;
318 319 320
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

321 322
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
323 324
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		POSTING_READ(gen6_pm_imr(dev_priv));
325
	}
P
Paulo Zanoni 已提交
326 327
}

328
void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
329
{
330 331 332
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
333 334 335
	snb_update_pm_irq(dev_priv, mask, mask);
}

336 337
static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
				  uint32_t mask)
P
Paulo Zanoni 已提交
338 339 340 341
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

342 343 344 345 346 347 348 349
void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	__gen6_disable_pm_irq(dev_priv, mask);
}

I
Imre Deak 已提交
350 351 352 353 354 355 356 357 358
void gen6_reset_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	uint32_t reg = gen6_pm_iir(dev_priv);

	spin_lock_irq(&dev_priv->irq_lock);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	POSTING_READ(reg);
359
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
360 361 362
	spin_unlock_irq(&dev_priv->irq_lock);
}

363 364 365 366 367
void gen6_enable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	spin_lock_irq(&dev_priv->irq_lock);
368

369
	WARN_ON(dev_priv->rps.pm_iir);
I
Imre Deak 已提交
370
	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
371
	dev_priv->rps.interrupts_enabled = true;
372 373
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
				dev_priv->pm_rps_events);
374
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
375

376 377 378
	spin_unlock_irq(&dev_priv->irq_lock);
}

379 380 381
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
	/*
382
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
383
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
384 385
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
386 387 388 389 390 391 392 393 394 395
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
		mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;

	return mask;
}

396 397 398 399
void gen6_disable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

I
Imre Deak 已提交
400 401 402 403 404 405
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
	spin_unlock_irq(&dev_priv->irq_lock);

	cancel_work_sync(&dev_priv->rps.work);

406 407
	spin_lock_irq(&dev_priv->irq_lock);

408
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
409 410

	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
411 412
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
				~dev_priv->pm_rps_events);
413 414 415 416

	spin_unlock_irq(&dev_priv->irq_lock);

	synchronize_irq(dev->irq);
417 418
}

419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
/**
  * bdw_update_port_irq - update DE port interrupt
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

451 452 453 454 455 456
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
457 458 459
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
460 461 462 463 464
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

465 466
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

467 468
	assert_spin_locked(&dev_priv->irq_lock);

469
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
470 471
		return;

472 473 474
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
475

D
Daniel Vetter 已提交
476
static void
477 478
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
479
{
480
	u32 reg = PIPESTAT(pipe);
481
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
482

483
	assert_spin_locked(&dev_priv->irq_lock);
484
	WARN_ON(!intel_irqs_enabled(dev_priv));
485

486 487 488 489
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
490 491 492
		return;

	if ((pipestat & enable_mask) == enable_mask)
493 494
		return;

495 496
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

497
	/* Enable the interrupt, clear any pending status */
498
	pipestat |= enable_mask | status_mask;
499 500
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
501 502
}

D
Daniel Vetter 已提交
503
static void
504 505
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
506
{
507
	u32 reg = PIPESTAT(pipe);
508
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
509

510
	assert_spin_locked(&dev_priv->irq_lock);
511
	WARN_ON(!intel_irqs_enabled(dev_priv));
512

513 514 515 516
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
517 518
		return;

519 520 521
	if ((pipestat & enable_mask) == 0)
		return;

522 523
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

524
	pipestat &= ~enable_mask;
525 526
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
527 528
}

529 530 531 532 533
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
534 535
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
536 537 538
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
539 540 541 542 543 544
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
545 546 547 548 549 550 551 552 553 554 555 556

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

557 558 559 560 561 562
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

563 564 565 566 567
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
568 569 570 571 572 573 574 575 576
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

577 578 579 580 581
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
582 583 584
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

585
/**
586
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
587
 * @dev: drm device
588
 */
589
static void i915_enable_asle_pipestat(struct drm_device *dev)
590
{
591
	struct drm_i915_private *dev_priv = dev->dev_private;
592

593 594 595
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
		return;

596
	spin_lock_irq(&dev_priv->irq_lock);
597

598
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
599
	if (INTEL_INFO(dev)->gen >= 4)
600
		i915_enable_pipestat(dev_priv, PIPE_A,
601
				     PIPE_LEGACY_BLC_EVENT_STATUS);
602

603
	spin_unlock_irq(&dev_priv->irq_lock);
604 605
}

606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

656 657 658 659 660 661
static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

662 663 664
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
665
static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
666
{
667
	struct drm_i915_private *dev_priv = dev->dev_private;
668 669
	unsigned long high_frame;
	unsigned long low_frame;
670
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
671 672
	struct intel_crtc *intel_crtc =
		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
673
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
674

675 676 677 678 679
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
680

681 682 683 684 685 686
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

687 688
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
689

690 691 692 693 694 695
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
696
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
697
		low   = I915_READ(low_frame);
698
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
699 700
	} while (high1 != high2);

701
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
702
	pixel = low & PIPE_PIXEL_MASK;
703
	low >>= PIPE_FRAME_LOW_SHIFT;
704 705 706 707 708 709

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
710
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
711 712
}

713
static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
714
{
715
	struct drm_i915_private *dev_priv = dev->dev_private;
716
	int reg = PIPE_FRMCOUNT_GM45(pipe);
717 718 719 720

	return I915_READ(reg);
}

721 722 723
/* raw reads, only for fast reads of display block, no need for forcewake etc. */
#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))

724 725 726 727
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
728
	const struct drm_display_mode *mode = &crtc->base.hwmode;
729
	enum pipe pipe = crtc->pipe;
730
	int position, vtotal;
731

732
	vtotal = mode->crtc_vtotal;
733 734 735 736 737 738 739 740
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	if (IS_GEN2(dev))
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
	else
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;

741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
	if (IS_HASWELL(dev) && !position) {
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
			temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
				DSL_LINEMASK_GEN3;
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

767
	/*
768 769
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
770
	 */
771
	return (position + crtc->scanline_offset) % vtotal;
772 773
}

774
static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
775
				    unsigned int flags, int *vpos, int *hpos,
776 777
				    ktime_t *stime, ktime_t *etime,
				    const struct drm_display_mode *mode)
778
{
779 780 781
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
782
	int position;
783
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
784 785
	bool in_vbl = true;
	int ret = 0;
786
	unsigned long irqflags;
787

788
	if (WARN_ON(!mode->crtc_clock)) {
789
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
790
				 "pipe %c\n", pipe_name(pipe));
791 792 793
		return 0;
	}

794
	htotal = mode->crtc_htotal;
795
	hsync_start = mode->crtc_hsync_start;
796 797 798
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
799

800 801 802 803 804 805
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

806 807
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

808 809 810 811 812 813
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
814

815 816 817 818 819 820
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

821
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
822 823 824
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
825
		position = __intel_get_crtc_scanline(intel_crtc);
826 827 828 829 830
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
831
		position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
832

833 834 835 836
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
837

838 839 840 841 842 843 844 845 846 847 848 849
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

850 851 852 853 854 855 856 857 858 859
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
860 861
	}

862 863 864 865 866 867 868 869
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

870 871 872 873 874 875 876 877 878 879 880 881
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
882

883
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
884 885 886 887 888 889
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
890 891 892

	/* In vblank? */
	if (in_vbl)
893
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
894 895 896 897

	return ret;
}

898 899 900 901 902 903 904 905 906 907 908 909 910
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

911
static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
912 913 914 915
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
916
	struct drm_crtc *crtc;
917

918
	if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
919
		DRM_ERROR("Invalid crtc %d\n", pipe);
920 921 922 923
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
924 925 926 927 928 929
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
		DRM_ERROR("Invalid crtc %d\n", pipe);
		return -EINVAL;
	}

930
	if (!crtc->hwmode.crtc_clock) {
931 932 933
		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
		return -EBUSY;
	}
934 935

	/* Helper routine in DRM core does all the work: */
936 937
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
938
						     &crtc->hwmode);
939 940
}

941
static void ironlake_rps_change_irq_handler(struct drm_device *dev)
942
{
943
	struct drm_i915_private *dev_priv = dev->dev_private;
944
	u32 busy_up, busy_down, max_avg, min_avg;
945 946
	u8 new_delay;

947
	spin_lock(&mchdev_lock);
948

949 950
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

951
	new_delay = dev_priv->ips.cur_delay;
952

953
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
954 955
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
956 957 958 959
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
960
	if (busy_up > max_avg) {
961 962 963 964
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
965
	} else if (busy_down < min_avg) {
966 967 968 969
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
970 971
	}

972
	if (ironlake_set_drps(dev, new_delay))
973
		dev_priv->ips.cur_delay = new_delay;
974

975
	spin_unlock(&mchdev_lock);
976

977 978 979
	return;
}

C
Chris Wilson 已提交
980
static void notify_ring(struct intel_engine_cs *ring)
981
{
982
	if (!intel_ring_initialized(ring))
983 984
		return;

985
	trace_i915_gem_request_notify(ring);
986

987 988 989
	wake_up_all(&ring->irq_queue);
}

990 991
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
992
{
993 994 995 996
	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
997

998 999 1000 1001 1002 1003
static bool vlv_c0_above(struct drm_i915_private *dev_priv,
			 const struct intel_rps_ei *old,
			 const struct intel_rps_ei *now,
			 int threshold)
{
	u64 time, c0;
1004
	unsigned int mul = 100;
1005

1006 1007
	if (old->cz_clock == 0)
		return false;
1008

1009 1010 1011
	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		mul <<= 8;

1012
	time = now->cz_clock - old->cz_clock;
1013
	time *= threshold * dev_priv->czclk_freq;
1014

1015 1016 1017
	/* Workload can be split between render + media, e.g. SwapBuffers
	 * being blitted in X after being rendered in mesa. To account for
	 * this we need to combine both engines into our activity counter.
1018
	 */
1019 1020
	c0 = now->render_c0 - old->render_c0;
	c0 += now->media_c0 - old->media_c0;
1021
	c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1022

1023
	return c0 >= time;
1024 1025
}

1026
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1027
{
1028 1029 1030
	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
}
1031

1032 1033 1034 1035
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	struct intel_rps_ei now;
	u32 events = 0;
1036

1037
	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1038
		return 0;
1039

1040 1041 1042
	vlv_c0_read(dev_priv, &now);
	if (now.cz_clock == 0)
		return 0;
1043

1044 1045 1046
	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		if (!vlv_c0_above(dev_priv,
				  &dev_priv->rps.down_ei, &now,
1047
				  dev_priv->rps.down_threshold))
1048 1049 1050
			events |= GEN6_PM_RP_DOWN_THRESHOLD;
		dev_priv->rps.down_ei = now;
	}
1051

1052 1053 1054
	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		if (vlv_c0_above(dev_priv,
				 &dev_priv->rps.up_ei, &now,
1055
				 dev_priv->rps.up_threshold))
1056 1057
			events |= GEN6_PM_RP_UP_THRESHOLD;
		dev_priv->rps.up_ei = now;
1058 1059
	}

1060
	return events;
1061 1062
}

1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
static bool any_waiters(struct drm_i915_private *dev_priv)
{
	struct intel_engine_cs *ring;
	int i;

	for_each_ring(ring, dev_priv, i)
		if (ring->irq_refcount)
			return true;

	return false;
}

1075
static void gen6_pm_rps_work(struct work_struct *work)
1076
{
1077 1078
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1079 1080
	bool client_boost;
	int new_delay, adj, min, max;
P
Paulo Zanoni 已提交
1081
	u32 pm_iir;
1082

1083
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1084 1085 1086 1087 1088
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1089 1090
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1091 1092
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1093 1094
	client_boost = dev_priv->rps.client_boost;
	dev_priv->rps.client_boost = false;
1095
	spin_unlock_irq(&dev_priv->irq_lock);
1096

1097
	/* Make sure we didn't queue anything we're not going to process. */
1098
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1099

1100
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1101 1102
		return;

1103
	mutex_lock(&dev_priv->rps.hw_lock);
1104

1105 1106
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1107
	adj = dev_priv->rps.last_adj;
1108
	new_delay = dev_priv->rps.cur_freq;
1109 1110 1111 1112 1113 1114 1115
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;

	if (client_boost) {
		new_delay = dev_priv->rps.max_freq_softlimit;
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1116 1117
		if (adj > 0)
			adj *= 2;
1118 1119
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1120 1121 1122 1123
		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1124
		if (new_delay < dev_priv->rps.efficient_freq - adj) {
1125
			new_delay = dev_priv->rps.efficient_freq;
1126 1127
			adj = 0;
		}
1128 1129
	} else if (any_waiters(dev_priv)) {
		adj = 0;
1130
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1131 1132
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1133
		else
1134
			new_delay = dev_priv->rps.min_freq_softlimit;
1135 1136 1137 1138
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1139 1140
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1141
	} else { /* unknown event */
1142
		adj = 0;
1143
	}
1144

1145 1146
	dev_priv->rps.last_adj = adj;

1147 1148 1149
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1150
	new_delay += adj;
1151
	new_delay = clamp_t(int, new_delay, min, max);
1152

1153
	intel_set_rps(dev_priv->dev, new_delay);
1154

1155
	mutex_unlock(&dev_priv->rps.hw_lock);
1156 1157
}

1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1170 1171
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1172
	u32 error_status, row, bank, subbank;
1173
	char *parity_event[6];
1174
	uint32_t misccpctl;
1175
	uint8_t slice = 0;
1176 1177 1178 1179 1180 1181 1182

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1183 1184 1185 1186
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1187 1188 1189 1190
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1191 1192
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		u32 reg;
1193

1194 1195 1196
		slice--;
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
			break;
1197

1198
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1199

1200
		reg = GEN7_L3CDERRST1 + (slice * 0x200);
1201

1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1217
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1218
				   KOBJ_CHANGE, parity_event);
1219

1220 1221
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1222

1223 1224 1225 1226 1227
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1228

1229
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1230

1231 1232
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1233
	spin_lock_irq(&dev_priv->irq_lock);
1234
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1235
	spin_unlock_irq(&dev_priv->irq_lock);
1236 1237

	mutex_unlock(&dev_priv->dev->struct_mutex);
1238 1239
}

1240
static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1241
{
1242
	struct drm_i915_private *dev_priv = dev->dev_private;
1243

1244
	if (!HAS_L3_DPF(dev))
1245 1246
		return;

1247
	spin_lock(&dev_priv->irq_lock);
1248
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1249
	spin_unlock(&dev_priv->irq_lock);
1250

1251 1252 1253 1254 1255 1256 1257
	iir &= GT_PARITY_ERROR(dev);
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1258
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1259 1260
}

1261 1262 1263 1264 1265 1266
static void ilk_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
C
Chris Wilson 已提交
1267
		notify_ring(&dev_priv->ring[RCS]);
1268
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
C
Chris Wilson 已提交
1269
		notify_ring(&dev_priv->ring[VCS]);
1270 1271
}

1272 1273 1274 1275 1276
static void snb_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{

1277 1278
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
C
Chris Wilson 已提交
1279
		notify_ring(&dev_priv->ring[RCS]);
1280
	if (gt_iir & GT_BSD_USER_INTERRUPT)
C
Chris Wilson 已提交
1281
		notify_ring(&dev_priv->ring[VCS]);
1282
	if (gt_iir & GT_BLT_USER_INTERRUPT)
C
Chris Wilson 已提交
1283
		notify_ring(&dev_priv->ring[BCS]);
1284

1285 1286
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1287 1288
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1289

1290 1291
	if (gt_iir & GT_PARITY_ERROR(dev))
		ivybridge_parity_error_irq_handler(dev, gt_iir);
1292 1293
}

C
Chris Wilson 已提交
1294
static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1295 1296 1297 1298 1299
				       u32 master_ctl)
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
C
Chris Wilson 已提交
1300
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1301
		if (tmp) {
1302
			I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1303
			ret = IRQ_HANDLED;
1304

C
Chris Wilson 已提交
1305 1306 1307 1308 1309 1310 1311 1312 1313
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[RCS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[RCS]);

			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[BCS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[BCS]);
1314 1315 1316 1317
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1318
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
C
Chris Wilson 已提交
1319
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1320
		if (tmp) {
1321
			I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1322
			ret = IRQ_HANDLED;
1323

C
Chris Wilson 已提交
1324 1325 1326 1327
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[VCS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[VCS]);
1328

C
Chris Wilson 已提交
1329 1330 1331 1332
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[VCS2]);
1333
		} else
1334
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1335 1336
	}

1337
	if (master_ctl & GEN8_GT_VECS_IRQ) {
C
Chris Wilson 已提交
1338
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1339
		if (tmp) {
C
Chris Wilson 已提交
1340
			I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1341
			ret = IRQ_HANDLED;
1342

C
Chris Wilson 已提交
1343 1344 1345 1346
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[VECS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[VECS]);
1347 1348 1349 1350
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1351
	if (master_ctl & GEN8_GT_PM_IRQ) {
C
Chris Wilson 已提交
1352
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1353
		if (tmp & dev_priv->pm_rps_events) {
1354 1355
			I915_WRITE_FW(GEN8_GT_IIR(2),
				      tmp & dev_priv->pm_rps_events);
1356
			ret = IRQ_HANDLED;
1357
			gen6_rps_irq_handler(dev_priv, tmp);
1358 1359 1360 1361
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1362 1363 1364
	return ret;
}

1365 1366 1367 1368
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1369
		return val & PORTA_HOTPLUG_LONG_DETECT;
1370 1371 1372 1373 1374 1375 1376 1377 1378
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1405 1406 1407 1408 1409 1410 1411 1412 1413 1414
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1415
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1416 1417 1418
{
	switch (port) {
	case PORT_B:
1419
		return val & PORTB_HOTPLUG_LONG_DETECT;
1420
	case PORT_C:
1421
		return val & PORTC_HOTPLUG_LONG_DETECT;
1422
	case PORT_D:
1423 1424 1425
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1426 1427 1428
	}
}

1429
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1430 1431 1432
{
	switch (port) {
	case PORT_B:
1433
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1434
	case PORT_C:
1435
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1436
	case PORT_D:
1437 1438 1439
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1440 1441 1442
	}
}

1443 1444 1445 1446 1447 1448 1449
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1450
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1451
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1452 1453
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1454
{
1455
	enum port port;
1456 1457 1458
	int i;

	for_each_hpd_pin(i) {
1459 1460
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1461

1462 1463
		*pin_mask |= BIT(i);

1464 1465 1466
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1467
		if (long_pulse_detect(port, dig_hotplug_reg))
1468
			*long_mask |= BIT(i);
1469 1470 1471 1472 1473 1474 1475
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1476 1477
static void gmbus_irq_handler(struct drm_device *dev)
{
1478
	struct drm_i915_private *dev_priv = dev->dev_private;
1479 1480

	wake_up_all(&dev_priv->gmbus_wait_queue);
1481 1482
}

1483 1484
static void dp_aux_irq_handler(struct drm_device *dev)
{
1485
	struct drm_i915_private *dev_priv = dev->dev_private;
1486 1487

	wake_up_all(&dev_priv->gmbus_wait_queue);
1488 1489
}

1490
#if defined(CONFIG_DEBUG_FS)
1491 1492 1493 1494
static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1495 1496 1497 1498
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1499
	int head, tail;
1500

1501 1502
	spin_lock(&pipe_crc->lock);

1503
	if (!pipe_crc->entries) {
1504
		spin_unlock(&pipe_crc->lock);
1505
		DRM_DEBUG_KMS("spurious interrupt\n");
1506 1507 1508
		return;
	}

1509 1510
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1511 1512

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1513
		spin_unlock(&pipe_crc->lock);
1514 1515 1516 1517 1518
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1519

1520
	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1521 1522 1523 1524 1525
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1526 1527

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1528 1529 1530
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1531 1532

	wake_up_interruptible(&pipe_crc->wq);
1533
}
1534 1535 1536 1537 1538 1539 1540 1541
#else
static inline void
display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1542

1543
static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
D
Daniel Vetter 已提交
1544 1545 1546
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1547 1548 1549
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1550 1551
}

1552
static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1553 1554 1555
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1556 1557 1558 1559 1560 1561
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1562
}
1563

1564
static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1565 1566
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577
	uint32_t res1, res2;

	if (INTEL_INFO(dev)->gen >= 3)
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1578

1579 1580 1581 1582 1583
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1584
}
1585

1586 1587 1588 1589
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1590
{
1591
	if (pm_iir & dev_priv->pm_rps_events) {
1592
		spin_lock(&dev_priv->irq_lock);
1593
		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1594 1595 1596 1597
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
			queue_work(dev_priv->wq, &dev_priv->rps.work);
		}
1598
		spin_unlock(&dev_priv->irq_lock);
1599 1600
	}

1601 1602 1603
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1604 1605
	if (HAS_VEBOX(dev_priv->dev)) {
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
C
Chris Wilson 已提交
1606
			notify_ring(&dev_priv->ring[VECS]);
B
Ben Widawsky 已提交
1607

1608 1609
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1610
	}
1611 1612
}

1613 1614 1615 1616 1617 1618 1619 1620
static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
{
	if (!drm_handle_vblank(dev, pipe))
		return false;

	return true;
}

1621 1622 1623
static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1624
	u32 pipe_stats[I915_MAX_PIPES] = { };
1625 1626
	int pipe;

1627
	spin_lock(&dev_priv->irq_lock);
1628
	for_each_pipe(dev_priv, pipe) {
1629
		int reg;
1630
		u32 mask, iir_bit = 0;
1631

1632 1633 1634 1635 1636 1637 1638
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1639 1640 1641

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1642 1643 1644 1645 1646 1647 1648 1649

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1650 1651 1652
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1653 1654 1655 1656 1657
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1658 1659 1660
			continue;

		reg = PIPESTAT(pipe);
1661 1662
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1663 1664 1665 1666

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1667 1668
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1669 1670
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1671
	spin_unlock(&dev_priv->irq_lock);
1672

1673
	for_each_pipe(dev_priv, pipe) {
1674 1675 1676
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
1677

1678
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1679 1680 1681 1682 1683 1684 1685
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip(dev, pipe);
		}

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
			i9xx_pipe_crc_irq_handler(dev, pipe);

1686 1687
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1688 1689 1690 1691 1692 1693
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		gmbus_irq_handler(dev);
}

1694 1695 1696 1697
static void i9xx_hpd_irq_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1698
	u32 pin_mask = 0, long_mask = 0;
1699

1700 1701
	if (!hotplug_status)
		return;
1702

1703 1704 1705 1706 1707 1708
	I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
	/*
	 * Make sure hotplug status is cleared before we clear IIR, or else we
	 * may miss hotplug events.
	 */
	POSTING_READ(PORT_HOTPLUG_STAT);
1709

1710 1711
	if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1712

1713 1714 1715 1716 1717 1718 1719
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1720 1721 1722

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
			dp_aux_irq_handler(dev);
1723 1724
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1725

1726 1727
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1728
					   hotplug_trigger, hpd_status_i915,
1729 1730 1731
					   i9xx_port_hotplug_long_detect);
			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1732
	}
1733 1734
}

1735
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1736
{
1737
	struct drm_device *dev = arg;
1738
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1739 1740 1741
	u32 iir, gt_iir, pm_iir;
	irqreturn_t ret = IRQ_NONE;

1742 1743 1744
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

J
Jesse Barnes 已提交
1745
	while (true) {
1746 1747
		/* Find, clear, then process each source of interrupt */

J
Jesse Barnes 已提交
1748
		gt_iir = I915_READ(GTIIR);
1749 1750 1751
		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);

J
Jesse Barnes 已提交
1752
		pm_iir = I915_READ(GEN6_PMIIR);
1753 1754 1755 1756 1757 1758 1759 1760 1761 1762
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

		iir = I915_READ(VLV_IIR);
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
J
Jesse Barnes 已提交
1763 1764 1765 1766 1767 1768

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
			goto out;

		ret = IRQ_HANDLED;

1769 1770
		if (gt_iir)
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
1771
		if (pm_iir)
1772
			gen6_rps_irq_handler(dev_priv, pm_iir);
1773 1774 1775
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
J
Jesse Barnes 已提交
1776 1777 1778 1779 1780 1781
	}

out:
	return ret;
}

1782 1783
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1784
	struct drm_device *dev = arg;
1785 1786 1787 1788
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl, iir;
	irqreturn_t ret = IRQ_NONE;

1789 1790 1791
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1792 1793 1794
	for (;;) {
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1795

1796 1797
		if (master_ctl == 0 && iir == 0)
			break;
1798

1799 1800
		ret = IRQ_HANDLED;

1801
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1802

1803
		/* Find, clear, then process each source of interrupt */
1804

1805 1806 1807 1808 1809 1810
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
1811

C
Chris Wilson 已提交
1812
		gen8_gt_irq_handler(dev_priv, master_ctl);
1813

1814 1815 1816
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
1817

1818 1819 1820
		I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		POSTING_READ(GEN8_MASTER_IRQ);
	}
1821

1822 1823 1824
	return ret;
}

1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840
static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

1841
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1842
{
1843
	struct drm_i915_private *dev_priv = dev->dev_private;
1844
	int pipe;
1845
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1846

1847 1848
	if (hotplug_trigger)
		ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1849

1850 1851 1852
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1853
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1854 1855
				 port_name(port));
	}
1856

1857 1858 1859
	if (pch_iir & SDE_AUX_MASK)
		dp_aux_irq_handler(dev);

1860
	if (pch_iir & SDE_GMBUS)
1861
		gmbus_irq_handler(dev);
1862 1863 1864 1865 1866 1867 1868 1869 1870 1871

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1872
	if (pch_iir & SDE_FDI_MASK)
1873
		for_each_pipe(dev_priv, pipe)
1874 1875 1876
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1877 1878 1879 1880 1881 1882 1883 1884

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1885
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1886 1887

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1888
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1889 1890 1891 1892 1893 1894
}

static void ivb_err_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
1895
	enum pipe pipe;
1896

1897 1898 1899
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

1900
	for_each_pipe(dev_priv, pipe) {
1901 1902
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1903

D
Daniel Vetter 已提交
1904 1905
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
			if (IS_IVYBRIDGE(dev))
1906
				ivb_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1907
			else
1908
				hsw_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1909 1910
		}
	}
1911

1912 1913 1914 1915 1916 1917 1918 1919
	I915_WRITE(GEN7_ERR_INT, err_int);
}

static void cpt_serr_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 serr_int = I915_READ(SERR_INT);

1920 1921 1922
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1923
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1924
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1925 1926

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1927
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1928 1929

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1930
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1931 1932

	I915_WRITE(SERR_INT, serr_int);
1933 1934
}

1935 1936
static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
1937
	struct drm_i915_private *dev_priv = dev->dev_private;
1938
	int pipe;
1939
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1940

1941 1942
	if (hotplug_trigger)
		ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1943

1944 1945 1946 1947 1948 1949
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
1950 1951

	if (pch_iir & SDE_AUX_MASK_CPT)
1952
		dp_aux_irq_handler(dev);
1953 1954

	if (pch_iir & SDE_GMBUS_CPT)
1955
		gmbus_irq_handler(dev);
1956 1957 1958 1959 1960 1961 1962 1963

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
1964
		for_each_pipe(dev_priv, pipe)
1965 1966 1967
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1968 1969 1970

	if (pch_iir & SDE_ERROR_CPT)
		cpt_serr_int_handler(dev);
1971 1972
}

1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
1989
				   spt_port_hotplug_long_detect);
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
		intel_hpd_irq_handler(dev, pin_mask, long_mask);

	if (pch_iir & SDE_GMBUS_CPT)
		gmbus_irq_handler(dev);
}

2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

2026 2027 2028
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2029
	enum pipe pipe;
2030 2031
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2032 2033
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
2034 2035 2036 2037 2038 2039 2040 2041 2042 2043

	if (de_iir & DE_AUX_CHANNEL_A)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE)
		intel_opregion_asle_intr(dev);

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2044
	for_each_pipe(dev_priv, pipe) {
2045 2046 2047
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2048

2049
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2050
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2051

2052 2053
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
			i9xx_pipe_crc_irq_handler(dev, pipe);
2054

2055 2056 2057 2058 2059
		/* plane/pipes map 1:1 on ilk+ */
		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

		if (HAS_PCH_CPT(dev))
			cpt_irq_handler(dev, pch_iir);
		else
			ibx_irq_handler(dev, pch_iir);

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev);
}

2079 2080 2081
static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2082
	enum pipe pipe;
2083 2084
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2085 2086
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2087 2088 2089 2090 2091 2092 2093 2094 2095 2096

	if (de_iir & DE_ERR_INT_IVB)
		ivb_err_int_handler(dev);

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_asle_intr(dev);

2097
	for_each_pipe(dev_priv, pipe) {
2098 2099 2100
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2101 2102

		/* plane/pipes map 1:1 on ilk+ */
2103 2104 2105
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119
		}
	}

	/* check event from PCH */
	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2120 2121 2122 2123 2124 2125 2126 2127
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2128
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2129
{
2130
	struct drm_device *dev = arg;
2131
	struct drm_i915_private *dev_priv = dev->dev_private;
2132
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2133
	irqreturn_t ret = IRQ_NONE;
2134

2135 2136 2137
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2138 2139
	/* We get interrupts on unclaimed registers, so check for this before we
	 * do any I915_{READ,WRITE}. */
2140
	intel_uncore_check_errors(dev);
2141

2142 2143 2144
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2145
	POSTING_READ(DEIER);
2146

2147 2148 2149 2150 2151
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2152 2153 2154 2155 2156
	if (!HAS_PCH_NOP(dev)) {
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2157

2158 2159
	/* Find, clear, then process each source of interrupt */

2160
	gt_iir = I915_READ(GTIIR);
2161
	if (gt_iir) {
2162 2163
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2164
		if (INTEL_INFO(dev)->gen >= 6)
2165
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2166 2167
		else
			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2168 2169
	}

2170 2171
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2172 2173
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2174 2175 2176 2177
		if (INTEL_INFO(dev)->gen >= 7)
			ivb_display_irq_handler(dev, de_iir);
		else
			ilk_display_irq_handler(dev, de_iir);
2178 2179
	}

2180 2181 2182 2183 2184
	if (INTEL_INFO(dev)->gen >= 6) {
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2185
			gen6_rps_irq_handler(dev_priv, pm_iir);
2186
		}
2187
	}
2188 2189 2190

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2191 2192 2193 2194
	if (!HAS_PCH_NOP(dev)) {
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2195 2196 2197 2198

	return ret;
}

2199 2200
static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
2201
{
2202 2203
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2204

2205 2206
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2207

2208
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2209
			   dig_hotplug_reg, hpd,
2210
			   bxt_port_hotplug_long_detect);
2211

2212
	intel_hpd_irq_handler(dev, pin_mask, long_mask);
2213 2214
}

2215 2216 2217 2218 2219 2220 2221
static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
	irqreturn_t ret = IRQ_NONE;
	uint32_t tmp = 0;
2222
	enum pipe pipe;
J
Jesse Barnes 已提交
2223 2224
	u32 aux_mask = GEN8_AUX_CHANNEL_A;

2225 2226 2227
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2228
	if (INTEL_INFO(dev_priv)->gen >= 9)
J
Jesse Barnes 已提交
2229 2230
		aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
			GEN9_AUX_CHANNEL_D;
2231

2232
	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2233 2234 2235 2236
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

2237
	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2238

2239 2240
	/* Find, clear, then process each source of interrupt */

C
Chris Wilson 已提交
2241
	ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2242 2243 2244 2245 2246 2247

	if (master_ctl & GEN8_DE_MISC_IRQ) {
		tmp = I915_READ(GEN8_DE_MISC_IIR);
		if (tmp) {
			I915_WRITE(GEN8_DE_MISC_IIR, tmp);
			ret = IRQ_HANDLED;
2248 2249 2250 2251
			if (tmp & GEN8_DE_MISC_GSE)
				intel_opregion_asle_intr(dev);
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2252
		}
2253 2254
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2255 2256
	}

2257 2258 2259
	if (master_ctl & GEN8_DE_PORT_IRQ) {
		tmp = I915_READ(GEN8_DE_PORT_IIR);
		if (tmp) {
2260
			bool found = false;
2261 2262 2263 2264 2265 2266
			u32 hotplug_trigger = 0;

			if (IS_BROXTON(dev_priv))
				hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
			else if (IS_BROADWELL(dev_priv))
				hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
2267

2268 2269
			I915_WRITE(GEN8_DE_PORT_IIR, tmp);
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2270

2271
			if (tmp & aux_mask) {
2272
				dp_aux_irq_handler(dev);
2273 2274 2275
				found = true;
			}

2276 2277 2278 2279 2280
			if (hotplug_trigger) {
				if (IS_BROXTON(dev))
					bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
				else
					ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
2281 2282 2283
				found = true;
			}

S
Shashank Sharma 已提交
2284 2285 2286 2287 2288
			if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
				gmbus_irq_handler(dev);
				found = true;
			}

2289
			if (!found)
2290
				DRM_ERROR("Unexpected DE Port interrupt\n");
2291
		}
2292 2293
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2294 2295
	}

2296
	for_each_pipe(dev_priv, pipe) {
2297
		uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2298

2299 2300
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2301

2302 2303 2304 2305
		pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (pipe_iir) {
			ret = IRQ_HANDLED;
			I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2306

2307 2308 2309
			if (pipe_iir & GEN8_PIPE_VBLANK &&
			    intel_pipe_handle_vblank(dev, pipe))
				intel_check_page_flip(dev, pipe);
2310

2311
			if (INTEL_INFO(dev_priv)->gen >= 9)
2312 2313 2314 2315 2316
				flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
			else
				flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;

			if (flip_done) {
2317 2318 2319 2320 2321 2322 2323
				intel_prepare_page_flip(dev, pipe);
				intel_finish_page_flip_plane(dev, pipe);
			}

			if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
				hsw_pipe_crc_irq_handler(dev, pipe);

2324 2325 2326
			if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
2327

2328

2329
			if (INTEL_INFO(dev_priv)->gen >= 9)
2330 2331 2332 2333 2334
				fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
			else
				fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;

			if (fault_errors)
2335 2336 2337
				DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
					  pipe_name(pipe),
					  pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2338
		} else
2339 2340 2341
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
	}

2342 2343
	if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
	    master_ctl & GEN8_DE_PCH_IRQ) {
2344 2345 2346 2347 2348 2349 2350 2351 2352
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
		u32 pch_iir = I915_READ(SDEIIR);
		if (pch_iir) {
			I915_WRITE(SDEIIR, pch_iir);
			ret = IRQ_HANDLED;
2353 2354 2355 2356 2357

			if (HAS_PCH_SPT(dev_priv))
				spt_irq_handler(dev, pch_iir);
			else
				cpt_irq_handler(dev, pch_iir);
2358 2359 2360
		} else
			DRM_ERROR("The master control interrupt lied (SDE)!\n");

2361 2362
	}

2363 2364
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2365 2366 2367 2368

	return ret;
}

2369 2370 2371
static void i915_error_wake_up(struct drm_i915_private *dev_priv,
			       bool reset_completed)
{
2372
	struct intel_engine_cs *ring;
2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396
	int i;

	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
	for_each_ring(ring, dev_priv, i)
		wake_up_all(&ring->irq_queue);

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);

	/*
	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
	 * reset state is cleared.
	 */
	if (reset_completed)
		wake_up_all(&dev_priv->gpu_error.reset_queue);
}

2397
/**
2398
 * i915_reset_and_wakeup - do process context error handling work
2399
 * @dev: drm device
2400 2401 2402 2403
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2404
static void i915_reset_and_wakeup(struct drm_device *dev)
2405
{
2406 2407
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct i915_gpu_error *error = &dev_priv->gpu_error;
2408 2409 2410
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2411
	int ret;
2412

2413
	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2414

2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2426
		DRM_DEBUG_DRIVER("resetting chip\n");
2427
		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2428
				   reset_event);
2429

2430 2431 2432 2433 2434 2435 2436 2437
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2438 2439 2440

		intel_prepare_reset(dev);

2441 2442 2443 2444 2445 2446
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2447 2448
		ret = i915_reset(dev);

2449
		intel_finish_reset(dev);
2450

2451 2452
		intel_runtime_pm_put(dev_priv);

2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463
		if (ret == 0) {
			/*
			 * After all the gem state is reset, increment the reset
			 * counter and wake up everyone waiting for the reset to
			 * complete.
			 *
			 * Since unlock operations are a one-sided barrier only,
			 * we need to insert a barrier here to order any seqno
			 * updates before
			 * the counter increment.
			 */
2464
			smp_mb__before_atomic();
2465 2466
			atomic_inc(&dev_priv->gpu_error.reset_counter);

2467
			kobject_uevent_env(&dev->primary->kdev->kobj,
2468
					   KOBJ_CHANGE, reset_done_event);
2469
		} else {
2470
			atomic_or(I915_WEDGED, &error->reset_counter);
2471
		}
2472

2473 2474 2475 2476 2477
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
		i915_error_wake_up(dev_priv, true);
2478
	}
2479 2480
}

2481
static void i915_report_and_clear_eir(struct drm_device *dev)
2482 2483
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2484
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2485
	u32 eir = I915_READ(EIR);
2486
	int pipe, i;
2487

2488 2489
	if (!eir)
		return;
2490

2491
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2492

2493 2494
	i915_get_extra_instdone(dev, instdone);

2495 2496 2497 2498
	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2499 2500
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2501 2502
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2503 2504
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2505
			I915_WRITE(IPEIR_I965, ipeir);
2506
			POSTING_READ(IPEIR_I965);
2507 2508 2509
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2510 2511
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2512
			I915_WRITE(PGTBL_ER, pgtbl_err);
2513
			POSTING_READ(PGTBL_ER);
2514 2515 2516
		}
	}

2517
	if (!IS_GEN2(dev)) {
2518 2519
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2520 2521
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2522
			I915_WRITE(PGTBL_ER, pgtbl_err);
2523
			POSTING_READ(PGTBL_ER);
2524 2525 2526 2527
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2528
		pr_err("memory refresh error:\n");
2529
		for_each_pipe(dev_priv, pipe)
2530
			pr_err("pipe %c stat: 0x%08x\n",
2531
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2532 2533 2534
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2535 2536
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2537 2538
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2539
		if (INTEL_INFO(dev)->gen < 4) {
2540 2541
			u32 ipeir = I915_READ(IPEIR);

2542 2543 2544
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2545
			I915_WRITE(IPEIR, ipeir);
2546
			POSTING_READ(IPEIR);
2547 2548 2549
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2550 2551 2552 2553
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2554
			I915_WRITE(IPEIR_I965, ipeir);
2555
			POSTING_READ(IPEIR_I965);
2556 2557 2558 2559
		}
	}

	I915_WRITE(EIR, eir);
2560
	POSTING_READ(EIR);
2561 2562 2563 2564 2565 2566 2567 2568 2569 2570
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2571 2572 2573
}

/**
2574
 * i915_handle_error - handle a gpu error
2575 2576
 * @dev: drm device
 *
2577
 * Do some basic checking of register state at error time and
2578 2579 2580 2581 2582
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2583 2584
void i915_handle_error(struct drm_device *dev, bool wedged,
		       const char *fmt, ...)
2585 2586
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2587 2588
	va_list args;
	char error_msg[80];
2589

2590 2591 2592 2593 2594
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

	i915_capture_error_state(dev, wedged, error_msg);
2595
	i915_report_and_clear_eir(dev);
2596

2597
	if (wedged) {
2598
		atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2599
				&dev_priv->gpu_error.reset_counter);
2600

2601
		/*
2602 2603 2604
		 * Wakeup waiting processes so that the reset function
		 * i915_reset_and_wakeup doesn't deadlock trying to grab
		 * various locks. By bumping the reset counter first, the woken
2605 2606 2607 2608 2609 2610 2611 2612
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2613
		 */
2614
		i915_error_wake_up(dev_priv, false);
2615 2616
	}

2617
	i915_reset_and_wakeup(dev);
2618 2619
}

2620 2621 2622
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2623
static int i915_enable_vblank(struct drm_device *dev, int pipe)
2624
{
2625
	struct drm_i915_private *dev_priv = dev->dev_private;
2626
	unsigned long irqflags;
2627

2628
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2629
	if (INTEL_INFO(dev)->gen >= 4)
2630
		i915_enable_pipestat(dev_priv, pipe,
2631
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2632
	else
2633
		i915_enable_pipestat(dev_priv, pipe,
2634
				     PIPE_VBLANK_INTERRUPT_STATUS);
2635
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2636

2637 2638 2639
	return 0;
}

2640
static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2641
{
2642
	struct drm_i915_private *dev_priv = dev->dev_private;
2643
	unsigned long irqflags;
2644
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2645
						     DE_PIPE_VBLANK(pipe);
2646 2647

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2648
	ironlake_enable_display_irq(dev_priv, bit);
2649 2650 2651 2652 2653
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

J
Jesse Barnes 已提交
2654 2655
static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
{
2656
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2657 2658 2659
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2660
	i915_enable_pipestat(dev_priv, pipe,
2661
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2662 2663 2664 2665 2666
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2667 2668 2669 2670 2671 2672
static int gen8_enable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2673 2674 2675
	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2676 2677 2678 2679
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
	return 0;
}

2680 2681 2682
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2683
static void i915_disable_vblank(struct drm_device *dev, int pipe)
2684
{
2685
	struct drm_i915_private *dev_priv = dev->dev_private;
2686
	unsigned long irqflags;
2687

2688
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2689
	i915_disable_pipestat(dev_priv, pipe,
2690 2691
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2692 2693 2694
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2695
static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2696
{
2697
	struct drm_i915_private *dev_priv = dev->dev_private;
2698
	unsigned long irqflags;
2699
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2700
						     DE_PIPE_VBLANK(pipe);
2701 2702

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2703
	ironlake_disable_display_irq(dev_priv, bit);
2704 2705 2706
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

J
Jesse Barnes 已提交
2707 2708
static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
{
2709
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2710 2711 2712
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2713
	i915_disable_pipestat(dev_priv, pipe,
2714
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2715 2716 2717
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2718 2719 2720 2721 2722 2723
static void gen8_disable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2724 2725 2726
	dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2727 2728 2729
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2730
static bool
2731
ring_idle(struct intel_engine_cs *ring, u32 seqno)
2732 2733
{
	return (list_empty(&ring->request_list) ||
2734
		i915_seqno_passed(seqno, ring->last_submitted_seqno));
B
Ben Gamari 已提交
2735 2736
}

2737 2738 2739 2740
static bool
ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
{
	if (INTEL_INFO(dev)->gen >= 8) {
2741
		return (ipehr >> 23) == 0x1c;
2742 2743 2744 2745 2746 2747 2748
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2749
static struct intel_engine_cs *
2750
semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2751 2752
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2753
	struct intel_engine_cs *signaller;
2754 2755 2756
	int i;

	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2757 2758 2759 2760 2761 2762 2763
		for_each_ring(signaller, dev_priv, i) {
			if (ring == signaller)
				continue;

			if (offset == signaller->semaphore.signal_ggtt[ring->id])
				return signaller;
		}
2764 2765 2766 2767 2768 2769 2770
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

		for_each_ring(signaller, dev_priv, i) {
			if(ring == signaller)
				continue;

2771
			if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2772 2773 2774 2775
				return signaller;
		}
	}

2776 2777
	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
		  ring->id, ipehr, offset);
2778 2779 2780 2781

	return NULL;
}

2782 2783
static struct intel_engine_cs *
semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2784 2785
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2786
	u32 cmd, ipehr, head;
2787 2788
	u64 offset = 0;
	int i, backwards;
2789

2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809
	/*
	 * This function does not support execlist mode - any attempt to
	 * proceed further into this function will result in a kernel panic
	 * when dereferencing ring->buffer, which is not set up in execlist
	 * mode.
	 *
	 * The correct way of doing it would be to derive the currently
	 * executing ring buffer from the current context, which is derived
	 * from the currently running request. Unfortunately, to get the
	 * current request we would have to grab the struct_mutex before doing
	 * anything else, which would be ill-advised since some other thread
	 * might have grabbed it already and managed to hang itself, causing
	 * the hang checker to deadlock.
	 *
	 * Therefore, this function does not support execlist mode in its
	 * current form. Just return NULL and move on.
	 */
	if (ring->buffer == NULL)
		return NULL;

2810
	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2811
	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2812
		return NULL;
2813

2814 2815 2816
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
2817 2818
	 * or 4 dwords depending on the semaphore wait command size.
	 * Note that we don't care about ACTHD here since that might
2819 2820
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2821
	 */
2822
	head = I915_READ_HEAD(ring) & HEAD_ADDR;
2823
	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2824

2825
	for (i = backwards; i; --i) {
2826 2827 2828 2829 2830
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
2831
		head &= ring->buffer->size - 1;
2832 2833

		/* This here seems to blow up */
2834
		cmd = ioread32(ring->buffer->virtual_start + head);
2835 2836 2837
		if (cmd == ipehr)
			break;

2838 2839
		head -= 4;
	}
2840

2841 2842
	if (!i)
		return NULL;
2843

2844
	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2845 2846 2847 2848 2849 2850
	if (INTEL_INFO(ring->dev)->gen >= 8) {
		offset = ioread32(ring->buffer->virtual_start + head + 12);
		offset <<= 32;
		offset = ioread32(ring->buffer->virtual_start + head + 8);
	}
	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2851 2852
}

2853
static int semaphore_passed(struct intel_engine_cs *ring)
2854 2855
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2856
	struct intel_engine_cs *signaller;
2857
	u32 seqno;
2858

2859
	ring->hangcheck.deadlock++;
2860 2861

	signaller = semaphore_waits_for(ring, &seqno);
2862 2863 2864 2865 2866
	if (signaller == NULL)
		return -1;

	/* Prevent pathological recursion due to driver bugs */
	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2867 2868
		return -1;

2869 2870 2871
	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
		return 1;

2872 2873 2874
	/* cursory check for an unkickable deadlock */
	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
	    semaphore_passed(signaller) < 0)
2875 2876 2877
		return -1;

	return 0;
2878 2879 2880 2881
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
2882
	struct intel_engine_cs *ring;
2883 2884 2885
	int i;

	for_each_ring(ring, dev_priv, i)
2886
		ring->hangcheck.deadlock = 0;
2887 2888
}

2889
static enum intel_ring_hangcheck_action
2890
ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2891 2892 2893
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
2894 2895
	u32 tmp;

2896 2897 2898 2899 2900 2901 2902 2903
	if (acthd != ring->hangcheck.acthd) {
		if (acthd > ring->hangcheck.max_acthd) {
			ring->hangcheck.max_acthd = acthd;
			return HANGCHECK_ACTIVE;
		}

		return HANGCHECK_ACTIVE_LOOP;
	}
2904

2905
	if (IS_GEN2(dev))
2906
		return HANGCHECK_HUNG;
2907 2908 2909 2910 2911 2912 2913

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
	tmp = I915_READ_CTL(ring);
2914
	if (tmp & RING_WAIT) {
2915 2916 2917
		i915_handle_error(dev, false,
				  "Kicking stuck wait on %s",
				  ring->name);
2918
		I915_WRITE_CTL(ring, tmp);
2919
		return HANGCHECK_KICK;
2920 2921 2922 2923 2924
	}

	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		switch (semaphore_passed(ring)) {
		default:
2925
			return HANGCHECK_HUNG;
2926
		case 1:
2927 2928 2929
			i915_handle_error(dev, false,
					  "Kicking stuck semaphore on %s",
					  ring->name);
2930
			I915_WRITE_CTL(ring, tmp);
2931
			return HANGCHECK_KICK;
2932
		case 0:
2933
			return HANGCHECK_WAIT;
2934
		}
2935
	}
2936

2937
	return HANGCHECK_HUNG;
2938 2939
}

2940
/*
B
Ben Gamari 已提交
2941
 * This is called when the chip hasn't reported back with completed
2942 2943 2944 2945 2946
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
2947
 */
2948
static void i915_hangcheck_elapsed(struct work_struct *work)
B
Ben Gamari 已提交
2949
{
2950 2951 2952 2953
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     gpu_error.hangcheck_work.work);
	struct drm_device *dev = dev_priv->dev;
2954
	struct intel_engine_cs *ring;
2955
	int i;
2956
	int busy_count = 0, rings_hung = 0;
2957 2958 2959 2960
	bool stuck[I915_NUM_RINGS] = { 0 };
#define BUSY 1
#define KICK 5
#define HUNG 20
2961

2962
	if (!i915.enable_hangcheck)
2963 2964
		return;

2965
	for_each_ring(ring, dev_priv, i) {
2966 2967
		u64 acthd;
		u32 seqno;
2968
		bool busy = true;
2969

2970 2971
		semaphore_clear_deadlocks(dev_priv);

2972 2973
		seqno = ring->get_seqno(ring, false);
		acthd = intel_ring_get_active_head(ring);
2974

2975
		if (ring->hangcheck.seqno == seqno) {
2976
			if (ring_idle(ring, seqno)) {
2977 2978
				ring->hangcheck.action = HANGCHECK_IDLE;

2979 2980
				if (waitqueue_active(&ring->irq_queue)) {
					/* Issue a wake-up to catch stuck h/w. */
2981
					if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2982 2983 2984 2985 2986 2987
						if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
							DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
								  ring->name);
						else
							DRM_INFO("Fake missed irq on %s\n",
								 ring->name);
2988 2989 2990 2991
						wake_up_all(&ring->irq_queue);
					}
					/* Safeguard against driver failure */
					ring->hangcheck.score += BUSY;
2992 2993
				} else
					busy = false;
2994
			} else {
2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
3010 3011 3012 3013
				ring->hangcheck.action = ring_stuck(ring,
								    acthd);

				switch (ring->hangcheck.action) {
3014
				case HANGCHECK_IDLE:
3015 3016
				case HANGCHECK_WAIT:
				case HANGCHECK_ACTIVE:
3017 3018
					break;
				case HANGCHECK_ACTIVE_LOOP:
3019
					ring->hangcheck.score += BUSY;
3020
					break;
3021
				case HANGCHECK_KICK:
3022
					ring->hangcheck.score += KICK;
3023
					break;
3024
				case HANGCHECK_HUNG:
3025
					ring->hangcheck.score += HUNG;
3026 3027 3028
					stuck[i] = true;
					break;
				}
3029
			}
3030
		} else {
3031 3032
			ring->hangcheck.action = HANGCHECK_ACTIVE;

3033 3034 3035 3036 3037
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
			if (ring->hangcheck.score > 0)
				ring->hangcheck.score--;
3038 3039

			ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
3040 3041
		}

3042 3043
		ring->hangcheck.seqno = seqno;
		ring->hangcheck.acthd = acthd;
3044
		busy_count += busy;
3045
	}
3046

3047
	for_each_ring(ring, dev_priv, i) {
3048
		if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3049 3050 3051
			DRM_INFO("%s on %s\n",
				 stuck[i] ? "stuck" : "no progress",
				 ring->name);
3052
			rings_hung++;
3053 3054 3055
		}
	}

3056
	if (rings_hung)
3057
		return i915_handle_error(dev, true, "Ring hung");
B
Ben Gamari 已提交
3058

3059 3060 3061
	if (busy_count)
		/* Reset timer case chip hangs without another request
		 * being added */
3062 3063 3064 3065 3066
		i915_queue_hangcheck(dev);
}

void i915_queue_hangcheck(struct drm_device *dev)
{
3067
	struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3068

3069
	if (!i915.enable_hangcheck)
3070 3071
		return;

3072 3073 3074 3075 3076 3077 3078
	/* Don't continually defer the hangcheck so that it is always run at
	 * least once after work has been scheduled on any ring. Otherwise,
	 * we will ignore a hung ring if a second ring is kept busy.
	 */

	queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
			   round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
B
Ben Gamari 已提交
3079 3080
}

3081
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3082 3083 3084 3085 3086 3087
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3088
	GEN5_IRQ_RESET(SDE);
3089 3090 3091

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3092
}
3093

P
Paulo Zanoni 已提交
3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3110 3111 3112 3113
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3114
static void gen5_gt_irq_reset(struct drm_device *dev)
3115 3116 3117
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3118
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3119
	if (INTEL_INFO(dev)->gen >= 6)
3120
		GEN5_IRQ_RESET(GEN6_PM);
3121 3122
}

L
Linus Torvalds 已提交
3123 3124
/* drm_dma.h hooks
*/
P
Paulo Zanoni 已提交
3125
static void ironlake_irq_reset(struct drm_device *dev)
3126
{
3127
	struct drm_i915_private *dev_priv = dev->dev_private;
3128

3129
	I915_WRITE(HWSTAM, 0xffffffff);
3130

3131
	GEN5_IRQ_RESET(DE);
3132 3133
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3134

3135
	gen5_gt_irq_reset(dev);
3136

3137
	ibx_irq_reset(dev);
3138
}
3139

3140 3141 3142 3143
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

3144
	i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
3145 3146 3147 3148 3149 3150 3151 3152
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	GEN5_IRQ_RESET(VLV_);
}

J
Jesse Barnes 已提交
3153 3154
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3155
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3156 3157 3158 3159 3160 3161 3162

	/* VLV magic */
	I915_WRITE(VLV_IMR, 0);
	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);

3163
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3164

3165
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
J
Jesse Barnes 已提交
3166

3167
	vlv_display_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3168 3169
}

3170 3171 3172 3173 3174 3175 3176 3177
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3178
static void gen8_irq_reset(struct drm_device *dev)
3179 3180 3181 3182 3183 3184 3185
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3186
	gen8_gt_irq_reset(dev_priv);
3187

3188
	for_each_pipe(dev_priv, pipe)
3189 3190
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3191
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3192

3193 3194 3195
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3196

3197 3198
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_reset(dev);
3199
}
3200

3201 3202
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3203
{
3204
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3205

3206
	spin_lock_irq(&dev_priv->irq_lock);
3207 3208 3209 3210
	if (pipe_mask & 1 << PIPE_A)
		GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
				  dev_priv->de_irq_mask[PIPE_A],
				  ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
3211 3212 3213 3214 3215 3216 3217 3218
	if (pipe_mask & 1 << PIPE_B)
		GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
				  dev_priv->de_irq_mask[PIPE_B],
				  ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
	if (pipe_mask & 1 << PIPE_C)
		GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
				  dev_priv->de_irq_mask[PIPE_C],
				  ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3219
	spin_unlock_irq(&dev_priv->irq_lock);
3220 3221
}

3222 3223 3224 3225 3226 3227 3228
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3229
	gen8_gt_irq_reset(dev_priv);
3230 3231 3232 3233 3234

	GEN5_IRQ_RESET(GEN8_PCU_);

	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);

3235
	vlv_display_irq_reset(dev_priv);
3236 3237
}

3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251
static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
				  const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

	for_each_intel_encoder(dev, encoder)
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3252
static void ibx_hpd_irq_setup(struct drm_device *dev)
3253
{
3254
	struct drm_i915_private *dev_priv = dev->dev_private;
3255
	u32 hotplug_irqs, hotplug, enabled_irqs;
3256 3257

	if (HAS_PCH_IBX(dev)) {
3258
		hotplug_irqs = SDE_HOTPLUG_MASK;
3259
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3260
	} else {
3261
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3262
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3263
	}
3264

3265
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3266 3267 3268

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3269 3270
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3271
	 */
3272 3273 3274 3275 3276
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3277 3278 3279 3280 3281 3282
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
	if (HAS_PCH_LPT_LP(dev))
		hotplug |= PORTA_HOTPLUG_ENABLE;
3283
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3284
}
X
Xiong Zhang 已提交
3285

3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298
static void spt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3299
		PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3300 3301 3302 3303 3304
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3305 3306
}

3307 3308 3309 3310 3311
static void ilk_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

3312 3313 3314 3315 3316 3317
	if (INTEL_INFO(dev)->gen >= 8) {
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
	} else if (INTEL_INFO(dev)->gen >= 7) {
3318 3319
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3320 3321

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3322 3323 3324
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3325

3326 3327
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3328 3329 3330 3331

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
3332
	 * The pulse duration bits are reserved on HSW+.
3333 3334 3335 3336 3337 3338 3339 3340 3341
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);

	ibx_hpd_irq_setup(dev);
}

3342 3343 3344
static void bxt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
3345
	u32 hotplug_irqs, hotplug, enabled_irqs;
3346

3347 3348
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3349

3350
	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3351

3352 3353 3354 3355
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
		PORTA_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3356 3357
}

P
Paulo Zanoni 已提交
3358 3359
static void ibx_irq_postinstall(struct drm_device *dev)
{
3360
	struct drm_i915_private *dev_priv = dev->dev_private;
3361
	u32 mask;
3362

D
Daniel Vetter 已提交
3363 3364 3365
	if (HAS_PCH_NOP(dev))
		return;

3366
	if (HAS_PCH_IBX(dev))
3367
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3368
	else
3369
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3370

3371
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3372 3373 3374
	I915_WRITE(SDEIMR, ~mask);
}

3375 3376 3377 3378 3379 3380 3381 3382
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3383
	if (HAS_L3_DPF(dev)) {
3384
		/* L3 parity interrupt is always unmasked. */
3385 3386
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3387 3388 3389 3390 3391 3392 3393 3394 3395 3396
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
			   ILK_BSD_USER_INTERRUPT;
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3397
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3398 3399

	if (INTEL_INFO(dev)->gen >= 6) {
3400 3401 3402 3403
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3404 3405 3406
		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3407
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3408
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3409 3410 3411
	}
}

3412
static int ironlake_irq_postinstall(struct drm_device *dev)
3413
{
3414
	struct drm_i915_private *dev_priv = dev->dev_private;
3415 3416 3417 3418 3419 3420
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3421
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3422
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3423 3424
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3425 3426 3427
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3428 3429 3430
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3431 3432 3433
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3434
	}
3435

3436
	dev_priv->irq_mask = ~display_mask;
3437

3438 3439
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3440 3441
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3442
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3443

3444
	gen5_gt_irq_postinstall(dev);
3445

P
Paulo Zanoni 已提交
3446
	ibx_irq_postinstall(dev);
3447

3448
	if (IS_IRONLAKE_M(dev)) {
3449 3450 3451
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3452 3453
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3454
		spin_lock_irq(&dev_priv->irq_lock);
3455
		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3456
		spin_unlock_irq(&dev_priv->irq_lock);
3457 3458
	}

3459 3460 3461
	return 0;
}

3462 3463 3464 3465
static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3466
	enum pipe pipe;
3467 3468 3469 3470

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;

3471 3472
	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3473 3474 3475 3476 3477
	POSTING_READ(PIPESTAT(PIPE_A));

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3478 3479 3480
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		      i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3481 3482 3483 3484

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3485 3486
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3487 3488 3489 3490 3491
	dev_priv->irq_mask &= ~iir_mask;

	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3492 3493
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
3494 3495 3496 3497 3498 3499
}

static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3500
	enum pipe pipe;
3501 3502 3503

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3504
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3505 3506
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3507 3508 3509

	dev_priv->irq_mask |= iir_mask;
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3510
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3511 3512 3513 3514 3515 3516 3517
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	POSTING_READ(VLV_IIR);

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3518 3519 3520
	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3521 3522 3523

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;
3524 3525 3526

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538
	POSTING_READ(PIPESTAT(PIPE_A));
}

void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3539
	if (intel_irqs_enabled(dev_priv))
3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
		valleyview_display_irqs_install(dev_priv);
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3552
	if (intel_irqs_enabled(dev_priv))
3553 3554 3555
		valleyview_display_irqs_uninstall(dev_priv);
}

3556
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
J
Jesse Barnes 已提交
3557
{
3558
	dev_priv->irq_mask = ~0;
J
Jesse Barnes 已提交
3559

3560
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3561 3562
	POSTING_READ(PORT_HOTPLUG_EN);

J
Jesse Barnes 已提交
3563
	I915_WRITE(VLV_IIR, 0xffffffff);
3564 3565 3566 3567
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
J
Jesse Barnes 已提交
3568

3569 3570
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3571
	spin_lock_irq(&dev_priv->irq_lock);
3572 3573
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_install(dev_priv);
3574
	spin_unlock_irq(&dev_priv->irq_lock);
3575 3576 3577 3578 3579 3580 3581
}

static int valleyview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	vlv_display_irq_postinstall(dev_priv);
J
Jesse Barnes 已提交
3582

3583
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3584 3585 3586 3587 3588 3589 3590 3591

	/* ack & enable invalid PTE error interrupts */
#if 0 /* FIXME: add support to irq handler for checking these bits */
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
#endif

	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3592 3593 3594 3595

	return 0;
}

3596 3597 3598 3599 3600
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3601
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3602
			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3603 3604
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3605
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3606 3607 3608
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3609
		0,
3610 3611
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3612 3613
		};

3614
	dev_priv->pm_irq_mask = 0xffffffff;
3615 3616
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3617 3618 3619 3620 3621
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
	 * is enabled/disabled.
	 */
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3622
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3623 3624 3625 3626
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3627 3628
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3629 3630 3631
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
	enum pipe pipe;
3632

3633
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3634 3635
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3636 3637
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
S
Shashank Sharma 已提交
3638
		if (IS_BROXTON(dev_priv))
3639 3640
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3641 3642
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3643
	}
3644 3645 3646 3647

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3648
	de_port_enables = de_port_masked;
3649 3650 3651
	if (IS_BROXTON(dev_priv))
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3652 3653
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3654 3655 3656
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3657

3658
	for_each_pipe(dev_priv, pipe)
3659
		if (intel_display_power_is_enabled(dev_priv,
3660 3661 3662 3663
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3664

3665
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3666 3667 3668 3669 3670 3671
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3672 3673
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3674

3675 3676 3677
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3678 3679
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_postinstall(dev);
3680 3681 3682 3683 3684 3685 3686

	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3687 3688 3689 3690
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3691
	vlv_display_irq_postinstall(dev_priv);
3692 3693 3694 3695 3696 3697 3698 3699 3700

	gen8_gt_irq_postinstall(dev_priv);

	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3701 3702 3703 3704 3705 3706 3707
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3708
	gen8_irq_reset(dev);
3709 3710
}

3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
{
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irq(&dev_priv->irq_lock);
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_uninstall(dev_priv);
	spin_unlock_irq(&dev_priv->irq_lock);

	vlv_display_irq_reset(dev_priv);

3722
	dev_priv->irq_mask = ~0;
3723 3724
}

J
Jesse Barnes 已提交
3725 3726
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3727
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3728 3729 3730 3731

	if (!dev_priv)
		return;

3732 3733
	I915_WRITE(VLV_MASTER_IER, 0);

3734 3735
	gen5_gt_irq_reset(dev);

J
Jesse Barnes 已提交
3736
	I915_WRITE(HWSTAM, 0xffffffff);
3737

3738
	vlv_display_irq_uninstall(dev_priv);
J
Jesse Barnes 已提交
3739 3740
}

3741 3742 3743 3744 3745 3746 3747 3748 3749 3750
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3751
	gen8_gt_irq_reset(dev_priv);
3752

3753
	GEN5_IRQ_RESET(GEN8_PCU_);
3754

3755
	vlv_display_irq_uninstall(dev_priv);
3756 3757
}

3758
static void ironlake_irq_uninstall(struct drm_device *dev)
3759
{
3760
	struct drm_i915_private *dev_priv = dev->dev_private;
3761 3762 3763 3764

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3765
	ironlake_irq_reset(dev);
3766 3767
}

3768
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3769
{
3770
	struct drm_i915_private *dev_priv = dev->dev_private;
3771
	int pipe;
3772

3773
	for_each_pipe(dev_priv, pipe)
3774
		I915_WRITE(PIPESTAT(pipe), 0);
3775 3776 3777
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3778 3779 3780 3781
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3782
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3783 3784 3785 3786 3787 3788 3789 3790 3791

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3792
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3793 3794 3795 3796 3797 3798 3799 3800
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3801 3802
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3803
	spin_lock_irq(&dev_priv->irq_lock);
3804 3805
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3806
	spin_unlock_irq(&dev_priv->irq_lock);
3807

C
Chris Wilson 已提交
3808 3809 3810
	return 0;
}

3811 3812 3813 3814
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_device *dev,
3815
			       int plane, int pipe, u32 iir)
3816
{
3817
	struct drm_i915_private *dev_priv = dev->dev_private;
3818
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3819

3820
	if (!intel_pipe_handle_vblank(dev, pipe))
3821 3822 3823
		return false;

	if ((iir & flip_pending) == 0)
3824
		goto check_page_flip;
3825 3826 3827 3828 3829 3830 3831 3832

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
3833
		goto check_page_flip;
3834

3835
	intel_prepare_page_flip(dev, plane);
3836 3837
	intel_finish_page_flip(dev, pipe);
	return true;
3838 3839 3840 3841

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
3842 3843
}

3844
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3845
{
3846
	struct drm_device *dev = arg;
3847
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3848 3849 3850 3851 3852 3853 3854
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;

3855 3856 3857
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

C
Chris Wilson 已提交
3858 3859 3860 3861 3862 3863 3864 3865 3866 3867
	iir = I915_READ16(IIR);
	if (iir == 0)
		return IRQ_NONE;

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3868
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3869
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3870
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3871

3872
		for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3873 3874 3875 3876 3877 3878
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3879
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3880 3881
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3882
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3883 3884 3885 3886 3887

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
C
Chris Wilson 已提交
3888
			notify_ring(&dev_priv->ring[RCS]);
C
Chris Wilson 已提交
3889

3890
		for_each_pipe(dev_priv, pipe) {
3891
			int plane = pipe;
3892
			if (HAS_FBC(dev))
3893 3894
				plane = !plane;

3895
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3896 3897
			    i8xx_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3898

3899
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3900
				i9xx_pipe_crc_irq_handler(dev, pipe);
3901

3902 3903 3904
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3905
		}
C
Chris Wilson 已提交
3906 3907 3908 3909 3910 3911 3912 3913 3914

		iir = new_iir;
	}

	return IRQ_HANDLED;
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3915
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3916 3917
	int pipe;

3918
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3919 3920 3921 3922 3923 3924 3925 3926 3927
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3928 3929
static void i915_irq_preinstall(struct drm_device * dev)
{
3930
	struct drm_i915_private *dev_priv = dev->dev_private;
3931 3932 3933
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
3934
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3935 3936 3937
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3938
	I915_WRITE16(HWSTAM, 0xeffe);
3939
	for_each_pipe(dev_priv, pipe)
3940 3941 3942 3943 3944 3945 3946 3947
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3948
	struct drm_i915_private *dev_priv = dev->dev_private;
3949
	u32 enable_mask;
3950

3951 3952 3953 3954 3955 3956 3957 3958
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3959
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3960 3961 3962 3963 3964 3965 3966

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

3967
	if (I915_HAS_HOTPLUG(dev)) {
3968
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3969 3970
		POSTING_READ(PORT_HOTPLUG_EN);

3971 3972 3973 3974 3975 3976 3977 3978 3979 3980
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3981
	i915_enable_asle_pipestat(dev);
3982

3983 3984
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3985
	spin_lock_irq(&dev_priv->irq_lock);
3986 3987
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3988
	spin_unlock_irq(&dev_priv->irq_lock);
3989

3990 3991 3992
	return 0;
}

3993 3994 3995 3996 3997 3998
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_device *dev,
			       int plane, int pipe, u32 iir)
{
3999
	struct drm_i915_private *dev_priv = dev->dev_private;
4000 4001
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

4002
	if (!intel_pipe_handle_vblank(dev, pipe))
4003 4004 4005
		return false;

	if ((iir & flip_pending) == 0)
4006
		goto check_page_flip;
4007 4008 4009 4010 4011 4012 4013 4014

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
4015
		goto check_page_flip;
4016

4017
	intel_prepare_page_flip(dev, plane);
4018 4019
	intel_finish_page_flip(dev, pipe);
	return true;
4020 4021 4022 4023

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
4024 4025
}

4026
static irqreturn_t i915_irq_handler(int irq, void *arg)
4027
{
4028
	struct drm_device *dev = arg;
4029
	struct drm_i915_private *dev_priv = dev->dev_private;
4030
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4031 4032 4033 4034
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
4035

4036 4037 4038
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4039
	iir = I915_READ(IIR);
4040 4041
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
4042
		bool blc_event = false;
4043 4044 4045 4046 4047 4048

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4049
		spin_lock(&dev_priv->irq_lock);
4050
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4051
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4052

4053
		for_each_pipe(dev_priv, pipe) {
4054 4055 4056
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

4057
			/* Clear the PIPE*STAT regs before the IIR */
4058 4059
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4060
				irq_received = true;
4061 4062
			}
		}
4063
		spin_unlock(&dev_priv->irq_lock);
4064 4065 4066 4067 4068

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
4069 4070 4071
		if (I915_HAS_HOTPLUG(dev) &&
		    iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4072

4073
		I915_WRITE(IIR, iir & ~flip_mask);
4074 4075 4076
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
C
Chris Wilson 已提交
4077
			notify_ring(&dev_priv->ring[RCS]);
4078

4079
		for_each_pipe(dev_priv, pipe) {
4080
			int plane = pipe;
4081
			if (HAS_FBC(dev))
4082
				plane = !plane;
4083

4084
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4085 4086
			    i915_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4087 4088 4089

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4090 4091

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4092
				i9xx_pipe_crc_irq_handler(dev, pipe);
4093

4094 4095 4096
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
4117
		ret = IRQ_HANDLED;
4118
		iir = new_iir;
4119
	} while (iir & ~flip_mask);
4120 4121 4122 4123 4124 4125

	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
4126
	struct drm_i915_private *dev_priv = dev->dev_private;
4127 4128 4129
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4130
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4131 4132 4133
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4134
	I915_WRITE16(HWSTAM, 0xffff);
4135
	for_each_pipe(dev_priv, pipe) {
4136
		/* Clear enable bits; then clear status bits */
4137
		I915_WRITE(PIPESTAT(pipe), 0);
4138 4139
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4140 4141 4142 4143 4144 4145 4146 4147
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4148
	struct drm_i915_private *dev_priv = dev->dev_private;
4149 4150
	int pipe;

4151
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4152
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4153 4154

	I915_WRITE(HWSTAM, 0xeffe);
4155
	for_each_pipe(dev_priv, pipe)
4156 4157 4158 4159 4160 4161 4162 4163
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4164
	struct drm_i915_private *dev_priv = dev->dev_private;
4165
	u32 enable_mask;
4166 4167 4168
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4169
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4170
			       I915_DISPLAY_PORT_INTERRUPT |
4171 4172 4173 4174 4175 4176 4177
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4178 4179
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4180 4181 4182 4183
	enable_mask |= I915_USER_INTERRUPT;

	if (IS_G4X(dev))
		enable_mask |= I915_BSD_USER_INTERRUPT;
4184

4185 4186
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4187
	spin_lock_irq(&dev_priv->irq_lock);
4188 4189 4190
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4191
	spin_unlock_irq(&dev_priv->irq_lock);
4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4212
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4213 4214
	POSTING_READ(PORT_HOTPLUG_EN);

4215
	i915_enable_asle_pipestat(dev);
4216 4217 4218 4219

	return 0;
}

4220
static void i915_hpd_irq_setup(struct drm_device *dev)
4221
{
4222
	struct drm_i915_private *dev_priv = dev->dev_private;
4223 4224
	u32 hotplug_en;

4225 4226
	assert_spin_locked(&dev_priv->irq_lock);

4227 4228
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4229
	hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4230 4231 4232 4233 4234 4235 4236 4237 4238
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
	if (IS_G4X(dev))
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4239 4240 4241 4242
	i915_hotplug_interrupt_update_locked(dev_priv,
				      (HOTPLUG_INT_EN_MASK
				       | CRT_HOTPLUG_VOLTAGE_COMPARE_MASK),
				      hotplug_en);
4243 4244
}

4245
static irqreturn_t i965_irq_handler(int irq, void *arg)
4246
{
4247
	struct drm_device *dev = arg;
4248
	struct drm_i915_private *dev_priv = dev->dev_private;
4249 4250 4251
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4252 4253 4254
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4255

4256 4257 4258
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4259 4260 4261
	iir = I915_READ(IIR);

	for (;;) {
4262
		bool irq_received = (iir & ~flip_mask) != 0;
4263 4264
		bool blc_event = false;

4265 4266 4267 4268 4269
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4270
		spin_lock(&dev_priv->irq_lock);
4271
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4272
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4273

4274
		for_each_pipe(dev_priv, pipe) {
4275 4276 4277 4278 4279 4280 4281 4282
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4283
				irq_received = true;
4284 4285
			}
		}
4286
		spin_unlock(&dev_priv->irq_lock);
4287 4288 4289 4290 4291 4292 4293

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4294 4295
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4296

4297
		I915_WRITE(IIR, iir & ~flip_mask);
4298 4299 4300
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
C
Chris Wilson 已提交
4301
			notify_ring(&dev_priv->ring[RCS]);
4302
		if (iir & I915_BSD_USER_INTERRUPT)
C
Chris Wilson 已提交
4303
			notify_ring(&dev_priv->ring[VCS]);
4304

4305
		for_each_pipe(dev_priv, pipe) {
4306
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4307 4308
			    i915_handle_vblank(dev, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4309 4310 4311

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4312 4313

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4314
				i9xx_pipe_crc_irq_handler(dev, pipe);
4315

4316 4317
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4318
		}
4319 4320 4321 4322

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

4323 4324 4325
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
			gmbus_irq_handler(dev);

4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4349
	struct drm_i915_private *dev_priv = dev->dev_private;
4350 4351 4352 4353 4354
	int pipe;

	if (!dev_priv)
		return;

4355
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4356
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4357 4358

	I915_WRITE(HWSTAM, 0xffffffff);
4359
	for_each_pipe(dev_priv, pipe)
4360 4361 4362 4363
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4364
	for_each_pipe(dev_priv, pipe)
4365 4366 4367 4368 4369
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4370 4371 4372 4373 4374 4375 4376
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4377
void intel_irq_init(struct drm_i915_private *dev_priv)
4378
{
4379
	struct drm_device *dev = dev_priv->dev;
4380

4381 4382
	intel_hpd_init_work(dev_priv);

4383
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4384
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4385

4386
	/* Let's track the enabled rps events */
4387
	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4388
		/* WaGsvRC0ResidencyMethod:vlv */
4389
		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4390 4391
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4392

4393 4394
	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
			  i915_hangcheck_elapsed);
4395

4396
	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4397

4398
	if (IS_GEN2(dev_priv)) {
4399 4400
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4401
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4402 4403
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4404 4405 4406
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4407 4408
	}

4409 4410 4411 4412 4413
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4414
	if (!IS_GEN2(dev_priv))
4415 4416
		dev->vblank_disable_immediate = true;

4417 4418
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4419

4420
	if (IS_CHERRYVIEW(dev_priv)) {
4421 4422 4423 4424 4425 4426 4427
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4428
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4429 4430 4431 4432 4433 4434
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4435
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4436
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4437
		dev->driver->irq_handler = gen8_irq_handler;
4438
		dev->driver->irq_preinstall = gen8_irq_reset;
4439 4440 4441 4442
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4443
		if (IS_BROXTON(dev))
4444
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4445 4446 4447
		else if (HAS_PCH_SPT(dev))
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4448
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4449 4450
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
4451
		dev->driver->irq_preinstall = ironlake_irq_reset;
4452 4453 4454 4455
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4456
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4457
	} else {
4458
		if (INTEL_INFO(dev_priv)->gen == 2) {
C
Chris Wilson 已提交
4459 4460 4461 4462
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4463
		} else if (INTEL_INFO(dev_priv)->gen == 3) {
4464 4465 4466 4467
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
C
Chris Wilson 已提交
4468
		} else {
4469 4470 4471 4472
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
C
Chris Wilson 已提交
4473
		}
4474 4475
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4476 4477 4478 4479
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4480

4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
}

4504 4505 4506 4507 4508 4509 4510
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4511 4512 4513 4514 4515 4516 4517
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
	drm_irq_uninstall(dev_priv->dev);
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4518 4519 4520 4521 4522 4523 4524
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4525
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4526
{
4527
	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4528
	dev_priv->pm.irqs_enabled = false;
4529
	synchronize_irq(dev_priv->dev->irq);
4530 4531
}

4532 4533 4534 4535 4536 4537 4538
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4539
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4540
{
4541
	dev_priv->pm.irqs_enabled = true;
4542 4543
	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4544
}