i915_irq.c 124.9 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
	u32 val = I915_READ(reg); \
	if (val) { \
		WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
		     (reg), val); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
	} \
} while (0)

P
Paulo Zanoni 已提交
154
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
155
	GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
156
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
157 158
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
159 160 161
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
162
	GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
P
Paulo Zanoni 已提交
163
	I915_WRITE(type##IER, (ier_val)); \
164 165
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
166 167
} while (0)

168 169
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);

170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

	assert_spin_locked(&dev_priv->irq_lock);
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

208 209 210 211 212 213 214 215 216
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
				   uint32_t interrupt_mask,
				   uint32_t enabled_irq_mask)
217
{
218 219
	uint32_t new_val;

220 221
	assert_spin_locked(&dev_priv->irq_lock);

222 223
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

224
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
225 226
		return;

227 228 229 230 231 232
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
233
		I915_WRITE(DEIMR, dev_priv->irq_mask);
234
		POSTING_READ(DEIMR);
235 236 237
	}
}

238
void
239
ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
240
{
241 242
	ilk_update_display_irq(dev_priv, mask, mask);
}
243

244 245 246 247
void
ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
{
	ilk_update_display_irq(dev_priv, mask, 0);
248 249
}

P
Paulo Zanoni 已提交
250 251 252 253 254 255 256 257 258 259 260 261
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

262 263
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

264
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
265 266
		return;

P
Paulo Zanoni 已提交
267 268 269 270 271 272
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

273
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
274 275 276 277
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

278
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
279 280 281 282
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

283 284 285 286 287
static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

288 289 290 291 292
static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

293 294 295 296 297
static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
298 299 300 301 302 303 304 305 306 307
/**
  * snb_update_pm_irq - update GEN6_PMIMR
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
308
	uint32_t new_val;
P
Paulo Zanoni 已提交
309

310 311
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

P
Paulo Zanoni 已提交
312 313
	assert_spin_locked(&dev_priv->irq_lock);

314
	new_val = dev_priv->pm_irq_mask;
315 316 317
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

318 319
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
320 321
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		POSTING_READ(gen6_pm_imr(dev_priv));
322
	}
P
Paulo Zanoni 已提交
323 324
}

325
void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
326
{
327 328 329
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
330 331 332
	snb_update_pm_irq(dev_priv, mask, mask);
}

333 334
static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
				  uint32_t mask)
P
Paulo Zanoni 已提交
335 336 337 338
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

339 340 341 342 343 344 345 346
void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	__gen6_disable_pm_irq(dev_priv, mask);
}

I
Imre Deak 已提交
347 348 349 350 351 352 353 354 355
void gen6_reset_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	uint32_t reg = gen6_pm_iir(dev_priv);

	spin_lock_irq(&dev_priv->irq_lock);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	POSTING_READ(reg);
356
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
357 358 359
	spin_unlock_irq(&dev_priv->irq_lock);
}

360 361 362 363 364
void gen6_enable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	spin_lock_irq(&dev_priv->irq_lock);
365

366
	WARN_ON(dev_priv->rps.pm_iir);
I
Imre Deak 已提交
367
	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
368
	dev_priv->rps.interrupts_enabled = true;
369 370
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
				dev_priv->pm_rps_events);
371
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
372

373 374 375
	spin_unlock_irq(&dev_priv->irq_lock);
}

376 377 378
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
	/*
379
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
380
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
381 382
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
383 384 385 386 387 388 389 390 391 392
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
		mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;

	return mask;
}

393 394 395 396
void gen6_disable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

I
Imre Deak 已提交
397 398 399 400 401 402
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
	spin_unlock_irq(&dev_priv->irq_lock);

	cancel_work_sync(&dev_priv->rps.work);

403 404
	spin_lock_irq(&dev_priv->irq_lock);

405
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
406 407

	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
408 409
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
				~dev_priv->pm_rps_events);
410 411 412 413

	spin_unlock_irq(&dev_priv->irq_lock);

	synchronize_irq(dev->irq);
414 415
}

416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
/**
  * bdw_update_port_irq - update DE port interrupt
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

448 449 450 451 452 453
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
454 455 456
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
457 458 459 460 461
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

462 463
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

464 465
	assert_spin_locked(&dev_priv->irq_lock);

466
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
467 468
		return;

469 470 471
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
472

D
Daniel Vetter 已提交
473
static void
474 475
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
476
{
477
	u32 reg = PIPESTAT(pipe);
478
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
479

480
	assert_spin_locked(&dev_priv->irq_lock);
481
	WARN_ON(!intel_irqs_enabled(dev_priv));
482

483 484 485 486
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
487 488 489
		return;

	if ((pipestat & enable_mask) == enable_mask)
490 491
		return;

492 493
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

494
	/* Enable the interrupt, clear any pending status */
495
	pipestat |= enable_mask | status_mask;
496 497
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
498 499
}

D
Daniel Vetter 已提交
500
static void
501 502
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
503
{
504
	u32 reg = PIPESTAT(pipe);
505
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
506

507
	assert_spin_locked(&dev_priv->irq_lock);
508
	WARN_ON(!intel_irqs_enabled(dev_priv));
509

510 511 512 513
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
514 515
		return;

516 517 518
	if ((pipestat & enable_mask) == 0)
		return;

519 520
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

521
	pipestat &= ~enable_mask;
522 523
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
524 525
}

526 527 528 529 530
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
531 532
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
533 534 535
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
536 537 538 539 540 541
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
542 543 544 545 546 547 548 549 550 551 552 553

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

554 555 556 557 558 559
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

560 561 562 563 564
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
565 566 567 568 569 570 571 572 573
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

574 575 576 577 578
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
579 580 581
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

582
/**
583
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
584
 */
585
static void i915_enable_asle_pipestat(struct drm_device *dev)
586
{
587
	struct drm_i915_private *dev_priv = dev->dev_private;
588

589 590 591
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
		return;

592
	spin_lock_irq(&dev_priv->irq_lock);
593

594
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
595
	if (INTEL_INFO(dev)->gen >= 4)
596
		i915_enable_pipestat(dev_priv, PIPE_A,
597
				     PIPE_LEGACY_BLC_EVENT_STATUS);
598

599
	spin_unlock_irq(&dev_priv->irq_lock);
600 601
}

602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

652 653 654 655 656 657
static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

658 659 660
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
661
static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
662
{
663
	struct drm_i915_private *dev_priv = dev->dev_private;
664 665
	unsigned long high_frame;
	unsigned long low_frame;
666
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
667 668
	struct intel_crtc *intel_crtc =
		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
669
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
670

671 672 673 674 675
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
676

677 678 679 680 681 682
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

683 684
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
685

686 687 688 689 690 691
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
692
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
693
		low   = I915_READ(low_frame);
694
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
695 696
	} while (high1 != high2);

697
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
698
	pixel = low & PIPE_PIXEL_MASK;
699
	low >>= PIPE_FRAME_LOW_SHIFT;
700 701 702 703 704 705

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
706
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
707 708
}

709
static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
710
{
711
	struct drm_i915_private *dev_priv = dev->dev_private;
712
	int reg = PIPE_FRMCOUNT_GM45(pipe);
713 714 715 716

	return I915_READ(reg);
}

717 718 719
/* raw reads, only for fast reads of display block, no need for forcewake etc. */
#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))

720 721 722 723
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
724
	const struct drm_display_mode *mode = &crtc->base.hwmode;
725
	enum pipe pipe = crtc->pipe;
726
	int position, vtotal;
727

728
	vtotal = mode->crtc_vtotal;
729 730 731 732 733 734 735 736 737
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	if (IS_GEN2(dev))
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
	else
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;

	/*
738 739
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
740
	 */
741
	return (position + crtc->scanline_offset) % vtotal;
742 743
}

744
static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
745 746
				    unsigned int flags, int *vpos, int *hpos,
				    ktime_t *stime, ktime_t *etime)
747
{
748 749 750
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
751
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
752
	int position;
753
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
754 755
	bool in_vbl = true;
	int ret = 0;
756
	unsigned long irqflags;
757

758
	if (WARN_ON(!mode->crtc_clock)) {
759
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
760
				 "pipe %c\n", pipe_name(pipe));
761 762 763
		return 0;
	}

764
	htotal = mode->crtc_htotal;
765
	hsync_start = mode->crtc_hsync_start;
766 767 768
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
769

770 771 772 773 774 775
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

776 777
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

778 779 780 781 782 783
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
784

785 786 787 788 789 790
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

791
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
792 793 794
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
795
		position = __intel_get_crtc_scanline(intel_crtc);
796 797 798 799 800
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
801
		position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
802

803 804 805 806
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
807

808 809 810 811 812 813 814 815 816 817 818 819
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

820 821 822 823 824 825 826 827 828 829
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
830 831
	}

832 833 834 835 836 837 838 839
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

840 841 842 843 844 845 846 847 848 849 850 851
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
852

853
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
854 855 856 857 858 859
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
860 861 862

	/* In vblank? */
	if (in_vbl)
863
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
864 865 866 867

	return ret;
}

868 869 870 871 872 873 874 875 876 877 878 879 880
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

881
static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
882 883 884 885
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
886
	struct drm_crtc *crtc;
887

888
	if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
889
		DRM_ERROR("Invalid crtc %d\n", pipe);
890 891 892 893
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
894 895 896 897 898 899
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
		DRM_ERROR("Invalid crtc %d\n", pipe);
		return -EINVAL;
	}

900
	if (!crtc->hwmode.crtc_clock) {
901 902 903
		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
		return -EBUSY;
	}
904 905

	/* Helper routine in DRM core does all the work: */
906 907
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
908
						     crtc,
909
						     &crtc->hwmode);
910 911
}

912
static void ironlake_rps_change_irq_handler(struct drm_device *dev)
913
{
914
	struct drm_i915_private *dev_priv = dev->dev_private;
915
	u32 busy_up, busy_down, max_avg, min_avg;
916 917
	u8 new_delay;

918
	spin_lock(&mchdev_lock);
919

920 921
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

922
	new_delay = dev_priv->ips.cur_delay;
923

924
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
925 926
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
927 928 929 930
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
931
	if (busy_up > max_avg) {
932 933 934 935
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
936
	} else if (busy_down < min_avg) {
937 938 939 940
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
941 942
	}

943
	if (ironlake_set_drps(dev, new_delay))
944
		dev_priv->ips.cur_delay = new_delay;
945

946
	spin_unlock(&mchdev_lock);
947

948 949 950
	return;
}

C
Chris Wilson 已提交
951
static void notify_ring(struct intel_engine_cs *ring)
952
{
953
	if (!intel_ring_initialized(ring))
954 955
		return;

956
	trace_i915_gem_request_notify(ring);
957

958 959 960
	wake_up_all(&ring->irq_queue);
}

961 962
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
963
{
964 965 966 967
	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
968

969 970 971 972 973 974
static bool vlv_c0_above(struct drm_i915_private *dev_priv,
			 const struct intel_rps_ei *old,
			 const struct intel_rps_ei *now,
			 int threshold)
{
	u64 time, c0;
975

976 977
	if (old->cz_clock == 0)
		return false;
978

979 980
	time = now->cz_clock - old->cz_clock;
	time *= threshold * dev_priv->mem_freq;
981

982 983 984
	/* Workload can be split between render + media, e.g. SwapBuffers
	 * being blitted in X after being rendered in mesa. To account for
	 * this we need to combine both engines into our activity counter.
985
	 */
986 987 988
	c0 = now->render_c0 - old->render_c0;
	c0 += now->media_c0 - old->media_c0;
	c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000;
989

990
	return c0 >= time;
991 992
}

993
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
994
{
995 996 997
	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
}
998

999 1000 1001 1002
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	struct intel_rps_ei now;
	u32 events = 0;
1003

1004
	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1005
		return 0;
1006

1007 1008 1009
	vlv_c0_read(dev_priv, &now);
	if (now.cz_clock == 0)
		return 0;
1010

1011 1012 1013
	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		if (!vlv_c0_above(dev_priv,
				  &dev_priv->rps.down_ei, &now,
1014
				  dev_priv->rps.down_threshold))
1015 1016 1017
			events |= GEN6_PM_RP_DOWN_THRESHOLD;
		dev_priv->rps.down_ei = now;
	}
1018

1019 1020 1021
	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		if (vlv_c0_above(dev_priv,
				 &dev_priv->rps.up_ei, &now,
1022
				 dev_priv->rps.up_threshold))
1023 1024
			events |= GEN6_PM_RP_UP_THRESHOLD;
		dev_priv->rps.up_ei = now;
1025 1026
	}

1027
	return events;
1028 1029
}

1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
static bool any_waiters(struct drm_i915_private *dev_priv)
{
	struct intel_engine_cs *ring;
	int i;

	for_each_ring(ring, dev_priv, i)
		if (ring->irq_refcount)
			return true;

	return false;
}

1042
static void gen6_pm_rps_work(struct work_struct *work)
1043
{
1044 1045
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1046 1047
	bool client_boost;
	int new_delay, adj, min, max;
P
Paulo Zanoni 已提交
1048
	u32 pm_iir;
1049

1050
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1051 1052 1053 1054 1055
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1056 1057
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1058 1059
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1060 1061
	client_boost = dev_priv->rps.client_boost;
	dev_priv->rps.client_boost = false;
1062
	spin_unlock_irq(&dev_priv->irq_lock);
1063

1064
	/* Make sure we didn't queue anything we're not going to process. */
1065
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1066

1067
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1068 1069
		return;

1070
	mutex_lock(&dev_priv->rps.hw_lock);
1071

1072 1073
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1074
	adj = dev_priv->rps.last_adj;
1075
	new_delay = dev_priv->rps.cur_freq;
1076 1077 1078 1079 1080 1081 1082
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;

	if (client_boost) {
		new_delay = dev_priv->rps.max_freq_softlimit;
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1083 1084
		if (adj > 0)
			adj *= 2;
1085 1086
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1087 1088 1089 1090
		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1091
		if (new_delay < dev_priv->rps.efficient_freq - adj) {
1092
			new_delay = dev_priv->rps.efficient_freq;
1093 1094
			adj = 0;
		}
1095 1096
	} else if (any_waiters(dev_priv)) {
		adj = 0;
1097
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1098 1099
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1100
		else
1101
			new_delay = dev_priv->rps.min_freq_softlimit;
1102 1103 1104 1105
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1106 1107
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1108
	} else { /* unknown event */
1109
		adj = 0;
1110
	}
1111

1112 1113
	dev_priv->rps.last_adj = adj;

1114 1115 1116
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1117
	new_delay += adj;
1118
	new_delay = clamp_t(int, new_delay, min, max);
1119

1120
	intel_set_rps(dev_priv->dev, new_delay);
1121

1122
	mutex_unlock(&dev_priv->rps.hw_lock);
1123 1124
}

1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1137 1138
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1139
	u32 error_status, row, bank, subbank;
1140
	char *parity_event[6];
1141
	uint32_t misccpctl;
1142
	uint8_t slice = 0;
1143 1144 1145 1146 1147 1148 1149

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1150 1151 1152 1153
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1154 1155 1156 1157
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1158 1159
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		u32 reg;
1160

1161 1162 1163
		slice--;
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
			break;
1164

1165
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1166

1167
		reg = GEN7_L3CDERRST1 + (slice * 0x200);
1168

1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1184
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1185
				   KOBJ_CHANGE, parity_event);
1186

1187 1188
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1189

1190 1191 1192 1193 1194
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1195

1196
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1197

1198 1199
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1200
	spin_lock_irq(&dev_priv->irq_lock);
1201
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1202
	spin_unlock_irq(&dev_priv->irq_lock);
1203 1204

	mutex_unlock(&dev_priv->dev->struct_mutex);
1205 1206
}

1207
static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1208
{
1209
	struct drm_i915_private *dev_priv = dev->dev_private;
1210

1211
	if (!HAS_L3_DPF(dev))
1212 1213
		return;

1214
	spin_lock(&dev_priv->irq_lock);
1215
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1216
	spin_unlock(&dev_priv->irq_lock);
1217

1218 1219 1220 1221 1222 1223 1224
	iir &= GT_PARITY_ERROR(dev);
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1225
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1226 1227
}

1228 1229 1230 1231 1232 1233
static void ilk_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
C
Chris Wilson 已提交
1234
		notify_ring(&dev_priv->ring[RCS]);
1235
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
C
Chris Wilson 已提交
1236
		notify_ring(&dev_priv->ring[VCS]);
1237 1238
}

1239 1240 1241 1242 1243
static void snb_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{

1244 1245
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
C
Chris Wilson 已提交
1246
		notify_ring(&dev_priv->ring[RCS]);
1247
	if (gt_iir & GT_BSD_USER_INTERRUPT)
C
Chris Wilson 已提交
1248
		notify_ring(&dev_priv->ring[VCS]);
1249
	if (gt_iir & GT_BLT_USER_INTERRUPT)
C
Chris Wilson 已提交
1250
		notify_ring(&dev_priv->ring[BCS]);
1251

1252 1253
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1254 1255
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1256

1257 1258
	if (gt_iir & GT_PARITY_ERROR(dev))
		ivybridge_parity_error_irq_handler(dev, gt_iir);
1259 1260
}

C
Chris Wilson 已提交
1261
static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1262 1263 1264 1265 1266
				       u32 master_ctl)
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
C
Chris Wilson 已提交
1267
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1268
		if (tmp) {
1269
			I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1270
			ret = IRQ_HANDLED;
1271

C
Chris Wilson 已提交
1272 1273 1274 1275 1276 1277 1278 1279 1280
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[RCS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[RCS]);

			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[BCS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[BCS]);
1281 1282 1283 1284
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1285
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
C
Chris Wilson 已提交
1286
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1287
		if (tmp) {
1288
			I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1289
			ret = IRQ_HANDLED;
1290

C
Chris Wilson 已提交
1291 1292 1293 1294
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[VCS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[VCS]);
1295

C
Chris Wilson 已提交
1296 1297 1298 1299
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[VCS2]);
1300
		} else
1301
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1302 1303
	}

1304
	if (master_ctl & GEN8_GT_VECS_IRQ) {
C
Chris Wilson 已提交
1305
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1306
		if (tmp) {
C
Chris Wilson 已提交
1307
			I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1308
			ret = IRQ_HANDLED;
1309

C
Chris Wilson 已提交
1310 1311 1312 1313
			if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
				intel_lrc_irq_handler(&dev_priv->ring[VECS]);
			if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
				notify_ring(&dev_priv->ring[VECS]);
1314 1315 1316 1317
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1318
	if (master_ctl & GEN8_GT_PM_IRQ) {
C
Chris Wilson 已提交
1319
		u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1320
		if (tmp & dev_priv->pm_rps_events) {
1321 1322
			I915_WRITE_FW(GEN8_GT_IIR(2),
				      tmp & dev_priv->pm_rps_events);
1323
			ret = IRQ_HANDLED;
1324
			gen6_rps_irq_handler(dev_priv, tmp);
1325 1326 1327 1328
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1329 1330 1331
	return ret;
}

1332 1333 1334 1335
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1336
		return val & PORTA_HOTPLUG_LONG_DETECT;
1337 1338 1339 1340 1341 1342 1343 1344 1345
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1372 1373 1374 1375 1376 1377 1378 1379 1380 1381
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1382
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1383 1384 1385
{
	switch (port) {
	case PORT_B:
1386
		return val & PORTB_HOTPLUG_LONG_DETECT;
1387
	case PORT_C:
1388
		return val & PORTC_HOTPLUG_LONG_DETECT;
1389
	case PORT_D:
1390 1391 1392
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1393 1394 1395
	}
}

1396
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1397 1398 1399
{
	switch (port) {
	case PORT_B:
1400
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1401
	case PORT_C:
1402
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1403
	case PORT_D:
1404 1405 1406
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1407 1408 1409
	}
}

1410 1411 1412 1413 1414 1415 1416
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1417
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1418
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1419 1420
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1421
{
1422
	enum port port;
1423 1424 1425
	int i;

	for_each_hpd_pin(i) {
1426 1427
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1428

1429 1430
		*pin_mask |= BIT(i);

1431 1432 1433
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1434
		if (long_pulse_detect(port, dig_hotplug_reg))
1435
			*long_mask |= BIT(i);
1436 1437 1438 1439 1440 1441 1442
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1443 1444
static void gmbus_irq_handler(struct drm_device *dev)
{
1445
	struct drm_i915_private *dev_priv = dev->dev_private;
1446 1447

	wake_up_all(&dev_priv->gmbus_wait_queue);
1448 1449
}

1450 1451
static void dp_aux_irq_handler(struct drm_device *dev)
{
1452
	struct drm_i915_private *dev_priv = dev->dev_private;
1453 1454

	wake_up_all(&dev_priv->gmbus_wait_queue);
1455 1456
}

1457
#if defined(CONFIG_DEBUG_FS)
1458 1459 1460 1461
static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1462 1463 1464 1465
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1466
	int head, tail;
1467

1468 1469
	spin_lock(&pipe_crc->lock);

1470
	if (!pipe_crc->entries) {
1471
		spin_unlock(&pipe_crc->lock);
1472
		DRM_DEBUG_KMS("spurious interrupt\n");
1473 1474 1475
		return;
	}

1476 1477
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1478 1479

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1480
		spin_unlock(&pipe_crc->lock);
1481 1482 1483 1484 1485
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1486

1487
	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1488 1489 1490 1491 1492
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1493 1494

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1495 1496 1497
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1498 1499

	wake_up_interruptible(&pipe_crc->wq);
1500
}
1501 1502 1503 1504 1505 1506 1507 1508
#else
static inline void
display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1509

1510
static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
D
Daniel Vetter 已提交
1511 1512 1513
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1514 1515 1516
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1517 1518
}

1519
static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1520 1521 1522
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1523 1524 1525 1526 1527 1528
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1529
}
1530

1531
static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1532 1533
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
	uint32_t res1, res2;

	if (INTEL_INFO(dev)->gen >= 3)
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1545

1546 1547 1548 1549 1550
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1551
}
1552

1553 1554 1555 1556
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1557
{
1558
	if (pm_iir & dev_priv->pm_rps_events) {
1559
		spin_lock(&dev_priv->irq_lock);
1560
		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1561 1562 1563 1564
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
			queue_work(dev_priv->wq, &dev_priv->rps.work);
		}
1565
		spin_unlock(&dev_priv->irq_lock);
1566 1567
	}

1568 1569 1570
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1571 1572
	if (HAS_VEBOX(dev_priv->dev)) {
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
C
Chris Wilson 已提交
1573
			notify_ring(&dev_priv->ring[VECS]);
B
Ben Widawsky 已提交
1574

1575 1576
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1577
	}
1578 1579
}

1580 1581 1582 1583 1584 1585 1586 1587
static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
{
	if (!drm_handle_vblank(dev, pipe))
		return false;

	return true;
}

1588 1589 1590
static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1591
	u32 pipe_stats[I915_MAX_PIPES] = { };
1592 1593
	int pipe;

1594
	spin_lock(&dev_priv->irq_lock);
1595
	for_each_pipe(dev_priv, pipe) {
1596
		int reg;
1597
		u32 mask, iir_bit = 0;
1598

1599 1600 1601 1602 1603 1604 1605
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1606 1607 1608

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1609 1610 1611 1612 1613 1614 1615 1616

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1617 1618 1619
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1620 1621 1622 1623 1624
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1625 1626 1627
			continue;

		reg = PIPESTAT(pipe);
1628 1629
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1630 1631 1632 1633

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1634 1635
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1636 1637
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1638
	spin_unlock(&dev_priv->irq_lock);
1639

1640
	for_each_pipe(dev_priv, pipe) {
1641 1642 1643
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
1644

1645
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1646 1647 1648 1649 1650 1651 1652
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip(dev, pipe);
		}

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
			i9xx_pipe_crc_irq_handler(dev, pipe);

1653 1654
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1655 1656 1657 1658 1659 1660
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		gmbus_irq_handler(dev);
}

1661 1662 1663 1664
static void i9xx_hpd_irq_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1665
	u32 pin_mask = 0, long_mask = 0;
1666

1667 1668
	if (!hotplug_status)
		return;
1669

1670 1671 1672 1673 1674 1675
	I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
	/*
	 * Make sure hotplug status is cleared before we clear IIR, or else we
	 * may miss hotplug events.
	 */
	POSTING_READ(PORT_HOTPLUG_STAT);
1676

1677 1678
	if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1679

1680 1681 1682 1683 1684 1685 1686
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1687 1688 1689

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
			dp_aux_irq_handler(dev);
1690 1691
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1692

1693 1694 1695 1696 1697 1698 1699
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1700
	}
1701 1702
}

1703
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1704
{
1705
	struct drm_device *dev = arg;
1706
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1707 1708 1709
	u32 iir, gt_iir, pm_iir;
	irqreturn_t ret = IRQ_NONE;

1710 1711 1712
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

J
Jesse Barnes 已提交
1713
	while (true) {
1714 1715
		/* Find, clear, then process each source of interrupt */

J
Jesse Barnes 已提交
1716
		gt_iir = I915_READ(GTIIR);
1717 1718 1719
		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);

J
Jesse Barnes 已提交
1720
		pm_iir = I915_READ(GEN6_PMIIR);
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

		iir = I915_READ(VLV_IIR);
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
J
Jesse Barnes 已提交
1731 1732 1733 1734 1735 1736

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
			goto out;

		ret = IRQ_HANDLED;

1737 1738
		if (gt_iir)
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
1739
		if (pm_iir)
1740
			gen6_rps_irq_handler(dev_priv, pm_iir);
1741 1742 1743
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
J
Jesse Barnes 已提交
1744 1745 1746 1747 1748 1749
	}

out:
	return ret;
}

1750 1751
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1752
	struct drm_device *dev = arg;
1753 1754 1755 1756
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl, iir;
	irqreturn_t ret = IRQ_NONE;

1757 1758 1759
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1760 1761 1762
	for (;;) {
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1763

1764 1765
		if (master_ctl == 0 && iir == 0)
			break;
1766

1767 1768
		ret = IRQ_HANDLED;

1769
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1770

1771
		/* Find, clear, then process each source of interrupt */
1772

1773 1774 1775 1776 1777 1778
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
1779

C
Chris Wilson 已提交
1780
		gen8_gt_irq_handler(dev_priv, master_ctl);
1781

1782 1783 1784
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
1785

1786 1787 1788
		I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		POSTING_READ(GEN8_MASTER_IRQ);
	}
1789

1790 1791 1792
	return ret;
}

1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808
static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

1809
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1810
{
1811
	struct drm_i915_private *dev_priv = dev->dev_private;
1812
	int pipe;
1813
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1814

1815 1816
	if (hotplug_trigger)
		ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1817

1818 1819 1820
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1821
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1822 1823
				 port_name(port));
	}
1824

1825 1826 1827
	if (pch_iir & SDE_AUX_MASK)
		dp_aux_irq_handler(dev);

1828
	if (pch_iir & SDE_GMBUS)
1829
		gmbus_irq_handler(dev);
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1840
	if (pch_iir & SDE_FDI_MASK)
1841
		for_each_pipe(dev_priv, pipe)
1842 1843 1844
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1845 1846 1847 1848 1849 1850 1851 1852

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1853
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1854 1855

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1856
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1857 1858 1859 1860 1861 1862
}

static void ivb_err_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
1863
	enum pipe pipe;
1864

1865 1866 1867
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

1868
	for_each_pipe(dev_priv, pipe) {
1869 1870
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1871

D
Daniel Vetter 已提交
1872 1873
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
			if (IS_IVYBRIDGE(dev))
1874
				ivb_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1875
			else
1876
				hsw_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1877 1878
		}
	}
1879

1880 1881 1882 1883 1884 1885 1886 1887
	I915_WRITE(GEN7_ERR_INT, err_int);
}

static void cpt_serr_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 serr_int = I915_READ(SERR_INT);

1888 1889 1890
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1891
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1892
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1893 1894

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1895
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1896 1897

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1898
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1899 1900

	I915_WRITE(SERR_INT, serr_int);
1901 1902
}

1903 1904
static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
1905
	struct drm_i915_private *dev_priv = dev->dev_private;
1906
	int pipe;
1907
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1908

1909 1910
	if (hotplug_trigger)
		ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1911

1912 1913 1914 1915 1916 1917
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
1918 1919

	if (pch_iir & SDE_AUX_MASK_CPT)
1920
		dp_aux_irq_handler(dev);
1921 1922

	if (pch_iir & SDE_GMBUS_CPT)
1923
		gmbus_irq_handler(dev);
1924 1925 1926 1927 1928 1929 1930 1931

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
1932
		for_each_pipe(dev_priv, pipe)
1933 1934 1935
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1936 1937 1938

	if (pch_iir & SDE_ERROR_CPT)
		cpt_serr_int_handler(dev);
1939 1940
}

1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956
static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
1957
				   spt_port_hotplug_long_detect);
1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
		intel_hpd_irq_handler(dev, pin_mask, long_mask);

	if (pch_iir & SDE_GMBUS_CPT)
		gmbus_irq_handler(dev);
}

1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

1994 1995 1996
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1997
	enum pipe pipe;
1998 1999
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2000 2001
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
2002 2003 2004 2005 2006 2007 2008 2009 2010 2011

	if (de_iir & DE_AUX_CHANNEL_A)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE)
		intel_opregion_asle_intr(dev);

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2012
	for_each_pipe(dev_priv, pipe) {
2013 2014 2015
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2016

2017
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2018
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2019

2020 2021
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
			i9xx_pipe_crc_irq_handler(dev, pipe);
2022

2023 2024 2025 2026 2027
		/* plane/pipes map 1:1 on ilk+ */
		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

		if (HAS_PCH_CPT(dev))
			cpt_irq_handler(dev, pch_iir);
		else
			ibx_irq_handler(dev, pch_iir);

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev);
}

2047 2048 2049
static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2050
	enum pipe pipe;
2051 2052
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2053 2054
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2055 2056 2057 2058 2059 2060 2061 2062 2063 2064

	if (de_iir & DE_ERR_INT_IVB)
		ivb_err_int_handler(dev);

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_asle_intr(dev);

2065
	for_each_pipe(dev_priv, pipe) {
2066 2067 2068
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2069 2070

		/* plane/pipes map 1:1 on ilk+ */
2071 2072 2073
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087
		}
	}

	/* check event from PCH */
	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2088 2089 2090 2091 2092 2093 2094 2095
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2096
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2097
{
2098
	struct drm_device *dev = arg;
2099
	struct drm_i915_private *dev_priv = dev->dev_private;
2100
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2101
	irqreturn_t ret = IRQ_NONE;
2102

2103 2104 2105
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2106 2107
	/* We get interrupts on unclaimed registers, so check for this before we
	 * do any I915_{READ,WRITE}. */
2108
	intel_uncore_check_errors(dev);
2109

2110 2111 2112
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2113
	POSTING_READ(DEIER);
2114

2115 2116 2117 2118 2119
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2120 2121 2122 2123 2124
	if (!HAS_PCH_NOP(dev)) {
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2125

2126 2127
	/* Find, clear, then process each source of interrupt */

2128
	gt_iir = I915_READ(GTIIR);
2129
	if (gt_iir) {
2130 2131
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2132
		if (INTEL_INFO(dev)->gen >= 6)
2133
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2134 2135
		else
			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2136 2137
	}

2138 2139
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2140 2141
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2142 2143 2144 2145
		if (INTEL_INFO(dev)->gen >= 7)
			ivb_display_irq_handler(dev, de_iir);
		else
			ilk_display_irq_handler(dev, de_iir);
2146 2147
	}

2148 2149 2150 2151 2152
	if (INTEL_INFO(dev)->gen >= 6) {
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2153
			gen6_rps_irq_handler(dev_priv, pm_iir);
2154
		}
2155
	}
2156 2157 2158

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2159 2160 2161 2162
	if (!HAS_PCH_NOP(dev)) {
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2163 2164 2165 2166

	return ret;
}

2167 2168
static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
2169
{
2170 2171
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2172

2173 2174
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2175

2176
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2177
			   dig_hotplug_reg, hpd,
2178
			   bxt_port_hotplug_long_detect);
2179

2180
	intel_hpd_irq_handler(dev, pin_mask, long_mask);
2181 2182
}

2183 2184 2185 2186 2187 2188 2189
static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
	irqreturn_t ret = IRQ_NONE;
	uint32_t tmp = 0;
2190
	enum pipe pipe;
J
Jesse Barnes 已提交
2191 2192
	u32 aux_mask = GEN8_AUX_CHANNEL_A;

2193 2194 2195
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2196
	if (INTEL_INFO(dev_priv)->gen >= 9)
J
Jesse Barnes 已提交
2197 2198
		aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
			GEN9_AUX_CHANNEL_D;
2199

2200
	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2201 2202 2203 2204
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

2205
	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2206

2207 2208
	/* Find, clear, then process each source of interrupt */

C
Chris Wilson 已提交
2209
	ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2210 2211 2212 2213 2214 2215

	if (master_ctl & GEN8_DE_MISC_IRQ) {
		tmp = I915_READ(GEN8_DE_MISC_IIR);
		if (tmp) {
			I915_WRITE(GEN8_DE_MISC_IIR, tmp);
			ret = IRQ_HANDLED;
2216 2217 2218 2219
			if (tmp & GEN8_DE_MISC_GSE)
				intel_opregion_asle_intr(dev);
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2220
		}
2221 2222
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2223 2224
	}

2225 2226 2227
	if (master_ctl & GEN8_DE_PORT_IRQ) {
		tmp = I915_READ(GEN8_DE_PORT_IIR);
		if (tmp) {
2228
			bool found = false;
2229 2230 2231 2232 2233 2234
			u32 hotplug_trigger = 0;

			if (IS_BROXTON(dev_priv))
				hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
			else if (IS_BROADWELL(dev_priv))
				hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
2235

2236 2237
			I915_WRITE(GEN8_DE_PORT_IIR, tmp);
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2238

2239
			if (tmp & aux_mask) {
2240
				dp_aux_irq_handler(dev);
2241 2242 2243
				found = true;
			}

2244 2245 2246 2247 2248
			if (hotplug_trigger) {
				if (IS_BROXTON(dev))
					bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
				else
					ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
2249 2250 2251
				found = true;
			}

S
Shashank Sharma 已提交
2252 2253 2254 2255 2256
			if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
				gmbus_irq_handler(dev);
				found = true;
			}

2257
			if (!found)
2258
				DRM_ERROR("Unexpected DE Port interrupt\n");
2259
		}
2260 2261
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2262 2263
	}

2264
	for_each_pipe(dev_priv, pipe) {
2265
		uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2266

2267 2268
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2269

2270 2271 2272 2273
		pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (pipe_iir) {
			ret = IRQ_HANDLED;
			I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2274

2275 2276 2277
			if (pipe_iir & GEN8_PIPE_VBLANK &&
			    intel_pipe_handle_vblank(dev, pipe))
				intel_check_page_flip(dev, pipe);
2278

2279
			if (INTEL_INFO(dev_priv)->gen >= 9)
2280 2281 2282 2283 2284
				flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
			else
				flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;

			if (flip_done) {
2285 2286 2287 2288 2289 2290 2291
				intel_prepare_page_flip(dev, pipe);
				intel_finish_page_flip_plane(dev, pipe);
			}

			if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
				hsw_pipe_crc_irq_handler(dev, pipe);

2292 2293 2294
			if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
2295

2296

2297
			if (INTEL_INFO(dev_priv)->gen >= 9)
2298 2299 2300 2301 2302
				fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
			else
				fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;

			if (fault_errors)
2303 2304 2305
				DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
					  pipe_name(pipe),
					  pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2306
		} else
2307 2308 2309
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
	}

2310 2311
	if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
	    master_ctl & GEN8_DE_PCH_IRQ) {
2312 2313 2314 2315 2316 2317 2318 2319 2320
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
		u32 pch_iir = I915_READ(SDEIIR);
		if (pch_iir) {
			I915_WRITE(SDEIIR, pch_iir);
			ret = IRQ_HANDLED;
2321 2322 2323 2324 2325

			if (HAS_PCH_SPT(dev_priv))
				spt_irq_handler(dev, pch_iir);
			else
				cpt_irq_handler(dev, pch_iir);
2326 2327 2328
		} else
			DRM_ERROR("The master control interrupt lied (SDE)!\n");

2329 2330
	}

2331 2332
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2333 2334 2335 2336

	return ret;
}

2337 2338 2339
static void i915_error_wake_up(struct drm_i915_private *dev_priv,
			       bool reset_completed)
{
2340
	struct intel_engine_cs *ring;
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364
	int i;

	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
	for_each_ring(ring, dev_priv, i)
		wake_up_all(&ring->irq_queue);

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);

	/*
	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
	 * reset state is cleared.
	 */
	if (reset_completed)
		wake_up_all(&dev_priv->gpu_error.reset_queue);
}

2365
/**
2366
 * i915_reset_and_wakeup - do process context error handling work
2367 2368 2369 2370
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2371
static void i915_reset_and_wakeup(struct drm_device *dev)
2372
{
2373 2374
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct i915_gpu_error *error = &dev_priv->gpu_error;
2375 2376 2377
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2378
	int ret;
2379

2380
	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2381

2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2393
		DRM_DEBUG_DRIVER("resetting chip\n");
2394
		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2395
				   reset_event);
2396

2397 2398 2399 2400 2401 2402 2403 2404
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2405 2406 2407

		intel_prepare_reset(dev);

2408 2409 2410 2411 2412 2413
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2414 2415
		ret = i915_reset(dev);

2416
		intel_finish_reset(dev);
2417

2418 2419
		intel_runtime_pm_put(dev_priv);

2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430
		if (ret == 0) {
			/*
			 * After all the gem state is reset, increment the reset
			 * counter and wake up everyone waiting for the reset to
			 * complete.
			 *
			 * Since unlock operations are a one-sided barrier only,
			 * we need to insert a barrier here to order any seqno
			 * updates before
			 * the counter increment.
			 */
2431
			smp_mb__before_atomic();
2432 2433
			atomic_inc(&dev_priv->gpu_error.reset_counter);

2434
			kobject_uevent_env(&dev->primary->kdev->kobj,
2435
					   KOBJ_CHANGE, reset_done_event);
2436
		} else {
M
Mika Kuoppala 已提交
2437
			atomic_set_mask(I915_WEDGED, &error->reset_counter);
2438
		}
2439

2440 2441 2442 2443 2444
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
		i915_error_wake_up(dev_priv, true);
2445
	}
2446 2447
}

2448
static void i915_report_and_clear_eir(struct drm_device *dev)
2449 2450
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2451
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2452
	u32 eir = I915_READ(EIR);
2453
	int pipe, i;
2454

2455 2456
	if (!eir)
		return;
2457

2458
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2459

2460 2461
	i915_get_extra_instdone(dev, instdone);

2462 2463 2464 2465
	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2466 2467
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2468 2469
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2470 2471
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2472
			I915_WRITE(IPEIR_I965, ipeir);
2473
			POSTING_READ(IPEIR_I965);
2474 2475 2476
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2477 2478
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2479
			I915_WRITE(PGTBL_ER, pgtbl_err);
2480
			POSTING_READ(PGTBL_ER);
2481 2482 2483
		}
	}

2484
	if (!IS_GEN2(dev)) {
2485 2486
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2487 2488
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2489
			I915_WRITE(PGTBL_ER, pgtbl_err);
2490
			POSTING_READ(PGTBL_ER);
2491 2492 2493 2494
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2495
		pr_err("memory refresh error:\n");
2496
		for_each_pipe(dev_priv, pipe)
2497
			pr_err("pipe %c stat: 0x%08x\n",
2498
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2499 2500 2501
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2502 2503
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2504 2505
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2506
		if (INTEL_INFO(dev)->gen < 4) {
2507 2508
			u32 ipeir = I915_READ(IPEIR);

2509 2510 2511
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2512
			I915_WRITE(IPEIR, ipeir);
2513
			POSTING_READ(IPEIR);
2514 2515 2516
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2517 2518 2519 2520
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2521
			I915_WRITE(IPEIR_I965, ipeir);
2522
			POSTING_READ(IPEIR_I965);
2523 2524 2525 2526
		}
	}

	I915_WRITE(EIR, eir);
2527
	POSTING_READ(EIR);
2528 2529 2530 2531 2532 2533 2534 2535 2536 2537
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2538 2539 2540
}

/**
2541
 * i915_handle_error - handle a gpu error
2542 2543
 * @dev: drm device
 *
2544
 * Do some basic checking of regsiter state at error time and
2545 2546 2547 2548 2549
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2550 2551
void i915_handle_error(struct drm_device *dev, bool wedged,
		       const char *fmt, ...)
2552 2553
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2554 2555
	va_list args;
	char error_msg[80];
2556

2557 2558 2559 2560 2561
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

	i915_capture_error_state(dev, wedged, error_msg);
2562
	i915_report_and_clear_eir(dev);
2563

2564
	if (wedged) {
2565 2566
		atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
				&dev_priv->gpu_error.reset_counter);
2567

2568
		/*
2569 2570 2571
		 * Wakeup waiting processes so that the reset function
		 * i915_reset_and_wakeup doesn't deadlock trying to grab
		 * various locks. By bumping the reset counter first, the woken
2572 2573 2574 2575 2576 2577 2578 2579
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2580
		 */
2581
		i915_error_wake_up(dev_priv, false);
2582 2583
	}

2584
	i915_reset_and_wakeup(dev);
2585 2586
}

2587 2588 2589
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2590
static int i915_enable_vblank(struct drm_device *dev, int pipe)
2591
{
2592
	struct drm_i915_private *dev_priv = dev->dev_private;
2593
	unsigned long irqflags;
2594

2595
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2596
	if (INTEL_INFO(dev)->gen >= 4)
2597
		i915_enable_pipestat(dev_priv, pipe,
2598
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2599
	else
2600
		i915_enable_pipestat(dev_priv, pipe,
2601
				     PIPE_VBLANK_INTERRUPT_STATUS);
2602
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2603

2604 2605 2606
	return 0;
}

2607
static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2608
{
2609
	struct drm_i915_private *dev_priv = dev->dev_private;
2610
	unsigned long irqflags;
2611
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2612
						     DE_PIPE_VBLANK(pipe);
2613 2614

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2615
	ironlake_enable_display_irq(dev_priv, bit);
2616 2617 2618 2619 2620
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

J
Jesse Barnes 已提交
2621 2622
static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
{
2623
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2624 2625 2626
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2627
	i915_enable_pipestat(dev_priv, pipe,
2628
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2629 2630 2631 2632 2633
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2634 2635 2636 2637 2638 2639
static int gen8_enable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2640 2641 2642
	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2643 2644 2645 2646
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
	return 0;
}

2647 2648 2649
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2650
static void i915_disable_vblank(struct drm_device *dev, int pipe)
2651
{
2652
	struct drm_i915_private *dev_priv = dev->dev_private;
2653
	unsigned long irqflags;
2654

2655
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2656
	i915_disable_pipestat(dev_priv, pipe,
2657 2658
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2659 2660 2661
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2662
static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2663
{
2664
	struct drm_i915_private *dev_priv = dev->dev_private;
2665
	unsigned long irqflags;
2666
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2667
						     DE_PIPE_VBLANK(pipe);
2668 2669

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2670
	ironlake_disable_display_irq(dev_priv, bit);
2671 2672 2673
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

J
Jesse Barnes 已提交
2674 2675
static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
{
2676
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2677 2678 2679
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2680
	i915_disable_pipestat(dev_priv, pipe,
2681
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2682 2683 2684
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2685 2686 2687 2688 2689 2690
static void gen8_disable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2691 2692 2693
	dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2694 2695 2696
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2697
static bool
2698
ring_idle(struct intel_engine_cs *ring, u32 seqno)
2699 2700
{
	return (list_empty(&ring->request_list) ||
2701
		i915_seqno_passed(seqno, ring->last_submitted_seqno));
B
Ben Gamari 已提交
2702 2703
}

2704 2705 2706 2707
static bool
ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
{
	if (INTEL_INFO(dev)->gen >= 8) {
2708
		return (ipehr >> 23) == 0x1c;
2709 2710 2711 2712 2713 2714 2715
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2716
static struct intel_engine_cs *
2717
semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2718 2719
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2720
	struct intel_engine_cs *signaller;
2721 2722 2723
	int i;

	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2724 2725 2726 2727 2728 2729 2730
		for_each_ring(signaller, dev_priv, i) {
			if (ring == signaller)
				continue;

			if (offset == signaller->semaphore.signal_ggtt[ring->id])
				return signaller;
		}
2731 2732 2733 2734 2735 2736 2737
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

		for_each_ring(signaller, dev_priv, i) {
			if(ring == signaller)
				continue;

2738
			if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2739 2740 2741 2742
				return signaller;
		}
	}

2743 2744
	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
		  ring->id, ipehr, offset);
2745 2746 2747 2748

	return NULL;
}

2749 2750
static struct intel_engine_cs *
semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2751 2752
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2753
	u32 cmd, ipehr, head;
2754 2755
	u64 offset = 0;
	int i, backwards;
2756 2757

	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2758
	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2759
		return NULL;
2760

2761 2762 2763
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
2764 2765
	 * or 4 dwords depending on the semaphore wait command size.
	 * Note that we don't care about ACTHD here since that might
2766 2767
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2768
	 */
2769
	head = I915_READ_HEAD(ring) & HEAD_ADDR;
2770
	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2771

2772
	for (i = backwards; i; --i) {
2773 2774 2775 2776 2777
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
2778
		head &= ring->buffer->size - 1;
2779 2780

		/* This here seems to blow up */
2781
		cmd = ioread32(ring->buffer->virtual_start + head);
2782 2783 2784
		if (cmd == ipehr)
			break;

2785 2786
		head -= 4;
	}
2787

2788 2789
	if (!i)
		return NULL;
2790

2791
	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2792 2793 2794 2795 2796 2797
	if (INTEL_INFO(ring->dev)->gen >= 8) {
		offset = ioread32(ring->buffer->virtual_start + head + 12);
		offset <<= 32;
		offset = ioread32(ring->buffer->virtual_start + head + 8);
	}
	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2798 2799
}

2800
static int semaphore_passed(struct intel_engine_cs *ring)
2801 2802
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2803
	struct intel_engine_cs *signaller;
2804
	u32 seqno;
2805

2806
	ring->hangcheck.deadlock++;
2807 2808

	signaller = semaphore_waits_for(ring, &seqno);
2809 2810 2811 2812 2813
	if (signaller == NULL)
		return -1;

	/* Prevent pathological recursion due to driver bugs */
	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2814 2815
		return -1;

2816 2817 2818
	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
		return 1;

2819 2820 2821
	/* cursory check for an unkickable deadlock */
	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
	    semaphore_passed(signaller) < 0)
2822 2823 2824
		return -1;

	return 0;
2825 2826 2827 2828
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
2829
	struct intel_engine_cs *ring;
2830 2831 2832
	int i;

	for_each_ring(ring, dev_priv, i)
2833
		ring->hangcheck.deadlock = 0;
2834 2835
}

2836
static enum intel_ring_hangcheck_action
2837
ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2838 2839 2840
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
2841 2842
	u32 tmp;

2843 2844 2845 2846 2847 2848 2849 2850
	if (acthd != ring->hangcheck.acthd) {
		if (acthd > ring->hangcheck.max_acthd) {
			ring->hangcheck.max_acthd = acthd;
			return HANGCHECK_ACTIVE;
		}

		return HANGCHECK_ACTIVE_LOOP;
	}
2851

2852
	if (IS_GEN2(dev))
2853
		return HANGCHECK_HUNG;
2854 2855 2856 2857 2858 2859 2860

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
	tmp = I915_READ_CTL(ring);
2861
	if (tmp & RING_WAIT) {
2862 2863 2864
		i915_handle_error(dev, false,
				  "Kicking stuck wait on %s",
				  ring->name);
2865
		I915_WRITE_CTL(ring, tmp);
2866
		return HANGCHECK_KICK;
2867 2868 2869 2870 2871
	}

	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		switch (semaphore_passed(ring)) {
		default:
2872
			return HANGCHECK_HUNG;
2873
		case 1:
2874 2875 2876
			i915_handle_error(dev, false,
					  "Kicking stuck semaphore on %s",
					  ring->name);
2877
			I915_WRITE_CTL(ring, tmp);
2878
			return HANGCHECK_KICK;
2879
		case 0:
2880
			return HANGCHECK_WAIT;
2881
		}
2882
	}
2883

2884
	return HANGCHECK_HUNG;
2885 2886
}

2887
/*
B
Ben Gamari 已提交
2888
 * This is called when the chip hasn't reported back with completed
2889 2890 2891 2892 2893
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
2894
 */
2895
static void i915_hangcheck_elapsed(struct work_struct *work)
B
Ben Gamari 已提交
2896
{
2897 2898 2899 2900
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     gpu_error.hangcheck_work.work);
	struct drm_device *dev = dev_priv->dev;
2901
	struct intel_engine_cs *ring;
2902
	int i;
2903
	int busy_count = 0, rings_hung = 0;
2904 2905 2906 2907
	bool stuck[I915_NUM_RINGS] = { 0 };
#define BUSY 1
#define KICK 5
#define HUNG 20
2908

2909
	if (!i915.enable_hangcheck)
2910 2911
		return;

2912
	for_each_ring(ring, dev_priv, i) {
2913 2914
		u64 acthd;
		u32 seqno;
2915
		bool busy = true;
2916

2917 2918
		semaphore_clear_deadlocks(dev_priv);

2919 2920
		seqno = ring->get_seqno(ring, false);
		acthd = intel_ring_get_active_head(ring);
2921

2922
		if (ring->hangcheck.seqno == seqno) {
2923
			if (ring_idle(ring, seqno)) {
2924 2925
				ring->hangcheck.action = HANGCHECK_IDLE;

2926 2927
				if (waitqueue_active(&ring->irq_queue)) {
					/* Issue a wake-up to catch stuck h/w. */
2928
					if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2929 2930 2931 2932 2933 2934
						if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
							DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
								  ring->name);
						else
							DRM_INFO("Fake missed irq on %s\n",
								 ring->name);
2935 2936 2937 2938
						wake_up_all(&ring->irq_queue);
					}
					/* Safeguard against driver failure */
					ring->hangcheck.score += BUSY;
2939 2940
				} else
					busy = false;
2941
			} else {
2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
2957 2958 2959 2960
				ring->hangcheck.action = ring_stuck(ring,
								    acthd);

				switch (ring->hangcheck.action) {
2961
				case HANGCHECK_IDLE:
2962 2963
				case HANGCHECK_WAIT:
				case HANGCHECK_ACTIVE:
2964 2965
					break;
				case HANGCHECK_ACTIVE_LOOP:
2966
					ring->hangcheck.score += BUSY;
2967
					break;
2968
				case HANGCHECK_KICK:
2969
					ring->hangcheck.score += KICK;
2970
					break;
2971
				case HANGCHECK_HUNG:
2972
					ring->hangcheck.score += HUNG;
2973 2974 2975
					stuck[i] = true;
					break;
				}
2976
			}
2977
		} else {
2978 2979
			ring->hangcheck.action = HANGCHECK_ACTIVE;

2980 2981 2982 2983 2984
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
			if (ring->hangcheck.score > 0)
				ring->hangcheck.score--;
2985 2986

			ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
2987 2988
		}

2989 2990
		ring->hangcheck.seqno = seqno;
		ring->hangcheck.acthd = acthd;
2991
		busy_count += busy;
2992
	}
2993

2994
	for_each_ring(ring, dev_priv, i) {
2995
		if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
2996 2997 2998
			DRM_INFO("%s on %s\n",
				 stuck[i] ? "stuck" : "no progress",
				 ring->name);
2999
			rings_hung++;
3000 3001 3002
		}
	}

3003
	if (rings_hung)
3004
		return i915_handle_error(dev, true, "Ring hung");
B
Ben Gamari 已提交
3005

3006 3007 3008
	if (busy_count)
		/* Reset timer case chip hangs without another request
		 * being added */
3009 3010 3011 3012 3013
		i915_queue_hangcheck(dev);
}

void i915_queue_hangcheck(struct drm_device *dev)
{
3014
	struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3015

3016
	if (!i915.enable_hangcheck)
3017 3018
		return;

3019 3020 3021 3022 3023 3024 3025
	/* Don't continually defer the hangcheck so that it is always run at
	 * least once after work has been scheduled on any ring. Otherwise,
	 * we will ignore a hung ring if a second ring is kept busy.
	 */

	queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
			   round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
B
Ben Gamari 已提交
3026 3027
}

3028
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3029 3030 3031 3032 3033 3034
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3035
	GEN5_IRQ_RESET(SDE);
3036 3037 3038

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3039
}
3040

P
Paulo Zanoni 已提交
3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3057 3058 3059 3060
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3061
static void gen5_gt_irq_reset(struct drm_device *dev)
3062 3063 3064
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3065
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3066
	if (INTEL_INFO(dev)->gen >= 6)
3067
		GEN5_IRQ_RESET(GEN6_PM);
3068 3069
}

L
Linus Torvalds 已提交
3070 3071
/* drm_dma.h hooks
*/
P
Paulo Zanoni 已提交
3072
static void ironlake_irq_reset(struct drm_device *dev)
3073
{
3074
	struct drm_i915_private *dev_priv = dev->dev_private;
3075

3076
	I915_WRITE(HWSTAM, 0xffffffff);
3077

3078
	GEN5_IRQ_RESET(DE);
3079 3080
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3081

3082
	gen5_gt_irq_reset(dev);
3083

3084
	ibx_irq_reset(dev);
3085
}
3086

3087 3088 3089 3090
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

3091
	i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
3092 3093 3094 3095 3096 3097 3098 3099
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	GEN5_IRQ_RESET(VLV_);
}

J
Jesse Barnes 已提交
3100 3101
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3102
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3103 3104 3105 3106 3107 3108 3109

	/* VLV magic */
	I915_WRITE(VLV_IMR, 0);
	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);

3110
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3111

3112
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
J
Jesse Barnes 已提交
3113

3114
	vlv_display_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3115 3116
}

3117 3118 3119 3120 3121 3122 3123 3124
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3125
static void gen8_irq_reset(struct drm_device *dev)
3126 3127 3128 3129 3130 3131 3132
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3133
	gen8_gt_irq_reset(dev_priv);
3134

3135
	for_each_pipe(dev_priv, pipe)
3136 3137
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3138
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3139

3140 3141 3142
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3143

3144 3145
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_reset(dev);
3146
}
3147

3148 3149
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3150
{
3151
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3152

3153
	spin_lock_irq(&dev_priv->irq_lock);
3154 3155 3156 3157
	if (pipe_mask & 1 << PIPE_A)
		GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
				  dev_priv->de_irq_mask[PIPE_A],
				  ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
3158 3159 3160 3161 3162 3163 3164 3165
	if (pipe_mask & 1 << PIPE_B)
		GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
				  dev_priv->de_irq_mask[PIPE_B],
				  ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
	if (pipe_mask & 1 << PIPE_C)
		GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
				  dev_priv->de_irq_mask[PIPE_C],
				  ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3166
	spin_unlock_irq(&dev_priv->irq_lock);
3167 3168
}

3169 3170 3171 3172 3173 3174 3175
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3176
	gen8_gt_irq_reset(dev_priv);
3177 3178 3179 3180 3181

	GEN5_IRQ_RESET(GEN8_PCU_);

	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);

3182
	vlv_display_irq_reset(dev_priv);
3183 3184
}

3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198
static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
				  const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

	for_each_intel_encoder(dev, encoder)
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3199
static void ibx_hpd_irq_setup(struct drm_device *dev)
3200
{
3201
	struct drm_i915_private *dev_priv = dev->dev_private;
3202
	u32 hotplug_irqs, hotplug, enabled_irqs;
3203 3204

	if (HAS_PCH_IBX(dev)) {
3205
		hotplug_irqs = SDE_HOTPLUG_MASK;
3206
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3207
	} else {
3208
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3209
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3210
	}
3211

3212
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3213 3214 3215

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3216 3217
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3218
	 */
3219 3220 3221 3222 3223
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3224 3225 3226 3227 3228 3229
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
	if (HAS_PCH_LPT_LP(dev))
		hotplug |= PORTA_HOTPLUG_ENABLE;
3230
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3231
}
X
Xiong Zhang 已提交
3232

3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245
static void spt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3246
		PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3247 3248 3249 3250 3251
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3252 3253
}

3254 3255 3256 3257 3258
static void ilk_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

3259 3260 3261 3262 3263 3264
	if (INTEL_INFO(dev)->gen >= 8) {
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
	} else if (INTEL_INFO(dev)->gen >= 7) {
3265 3266
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3267 3268

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3269 3270 3271
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3272

3273 3274
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3275 3276 3277 3278

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
3279
	 * The pulse duration bits are reserved on HSW+.
3280 3281 3282 3283 3284 3285 3286 3287 3288
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);

	ibx_hpd_irq_setup(dev);
}

3289 3290 3291
static void bxt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
3292
	u32 hotplug_irqs, hotplug, enabled_irqs;
3293

3294 3295
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3296

3297
	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3298

3299 3300 3301 3302
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
		PORTA_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3303 3304
}

P
Paulo Zanoni 已提交
3305 3306
static void ibx_irq_postinstall(struct drm_device *dev)
{
3307
	struct drm_i915_private *dev_priv = dev->dev_private;
3308
	u32 mask;
3309

D
Daniel Vetter 已提交
3310 3311 3312
	if (HAS_PCH_NOP(dev))
		return;

3313
	if (HAS_PCH_IBX(dev))
3314
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3315
	else
3316
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3317

3318
	GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
P
Paulo Zanoni 已提交
3319 3320 3321
	I915_WRITE(SDEIMR, ~mask);
}

3322 3323 3324 3325 3326 3327 3328 3329
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3330
	if (HAS_L3_DPF(dev)) {
3331
		/* L3 parity interrupt is always unmasked. */
3332 3333
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3334 3335 3336 3337 3338 3339 3340 3341 3342 3343
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
			   ILK_BSD_USER_INTERRUPT;
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3344
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3345 3346

	if (INTEL_INFO(dev)->gen >= 6) {
3347 3348 3349 3350
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3351 3352 3353
		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3354
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3355
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3356 3357 3358
	}
}

3359
static int ironlake_irq_postinstall(struct drm_device *dev)
3360
{
3361
	struct drm_i915_private *dev_priv = dev->dev_private;
3362 3363 3364 3365 3366 3367
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3368
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3369
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3370 3371
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3372 3373 3374
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3375 3376 3377
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3378 3379 3380
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3381
	}
3382

3383
	dev_priv->irq_mask = ~display_mask;
3384

3385 3386
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3387 3388
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3389
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3390

3391
	gen5_gt_irq_postinstall(dev);
3392

P
Paulo Zanoni 已提交
3393
	ibx_irq_postinstall(dev);
3394

3395
	if (IS_IRONLAKE_M(dev)) {
3396 3397 3398
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3399 3400
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3401
		spin_lock_irq(&dev_priv->irq_lock);
3402
		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3403
		spin_unlock_irq(&dev_priv->irq_lock);
3404 3405
	}

3406 3407 3408
	return 0;
}

3409 3410 3411 3412
static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3413
	enum pipe pipe;
3414 3415 3416 3417

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;

3418 3419
	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3420 3421 3422 3423 3424
	POSTING_READ(PIPESTAT(PIPE_A));

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3425 3426 3427
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		      i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3428 3429 3430 3431

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3432 3433
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3434 3435 3436 3437 3438
	dev_priv->irq_mask &= ~iir_mask;

	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3439 3440
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
3441 3442 3443 3444 3445 3446
}

static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3447
	enum pipe pipe;
3448 3449 3450

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3451
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3452 3453
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3454 3455 3456

	dev_priv->irq_mask |= iir_mask;
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3457
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3458 3459 3460 3461 3462 3463 3464
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	POSTING_READ(VLV_IIR);

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3465 3466 3467
	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3468 3469 3470

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;
3471 3472 3473

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485
	POSTING_READ(PIPESTAT(PIPE_A));
}

void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3486
	if (intel_irqs_enabled(dev_priv))
3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498
		valleyview_display_irqs_install(dev_priv);
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3499
	if (intel_irqs_enabled(dev_priv))
3500 3501 3502
		valleyview_display_irqs_uninstall(dev_priv);
}

3503
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
J
Jesse Barnes 已提交
3504
{
3505
	dev_priv->irq_mask = ~0;
J
Jesse Barnes 已提交
3506

3507
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3508 3509
	POSTING_READ(PORT_HOTPLUG_EN);

J
Jesse Barnes 已提交
3510
	I915_WRITE(VLV_IIR, 0xffffffff);
3511 3512 3513 3514
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
J
Jesse Barnes 已提交
3515

3516 3517
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3518
	spin_lock_irq(&dev_priv->irq_lock);
3519 3520
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_install(dev_priv);
3521
	spin_unlock_irq(&dev_priv->irq_lock);
3522 3523 3524 3525 3526 3527 3528
}

static int valleyview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	vlv_display_irq_postinstall(dev_priv);
J
Jesse Barnes 已提交
3529

3530
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3531 3532 3533 3534 3535 3536 3537 3538

	/* ack & enable invalid PTE error interrupts */
#if 0 /* FIXME: add support to irq handler for checking these bits */
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
#endif

	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3539 3540 3541 3542

	return 0;
}

3543 3544 3545 3546 3547
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3548
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3549
			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3550 3551
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3552
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3553 3554 3555
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3556
		0,
3557 3558
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3559 3560
		};

3561
	dev_priv->pm_irq_mask = 0xffffffff;
3562 3563
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3564 3565 3566 3567 3568
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
	 * is enabled/disabled.
	 */
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3569
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3570 3571 3572 3573
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3574 3575
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3576 3577 3578
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
	enum pipe pipe;
3579

3580
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3581 3582
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3583 3584
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
S
Shashank Sharma 已提交
3585
		if (IS_BROXTON(dev_priv))
3586 3587
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3588 3589
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3590
	}
3591 3592 3593 3594

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3595
	de_port_enables = de_port_masked;
3596 3597 3598
	if (IS_BROXTON(dev_priv))
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3599 3600
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3601 3602 3603
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3604

3605
	for_each_pipe(dev_priv, pipe)
3606
		if (intel_display_power_is_enabled(dev_priv,
3607 3608 3609 3610
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3611

3612
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3613 3614 3615 3616 3617 3618
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3619 3620
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3621

3622 3623 3624
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3625 3626
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_postinstall(dev);
3627 3628 3629 3630 3631 3632 3633

	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3634 3635 3636 3637
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3638
	vlv_display_irq_postinstall(dev_priv);
3639 3640 3641 3642 3643 3644 3645 3646 3647

	gen8_gt_irq_postinstall(dev_priv);

	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3648 3649 3650 3651 3652 3653 3654
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3655
	gen8_irq_reset(dev);
3656 3657
}

3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668
static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
{
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irq(&dev_priv->irq_lock);
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_uninstall(dev_priv);
	spin_unlock_irq(&dev_priv->irq_lock);

	vlv_display_irq_reset(dev_priv);

3669
	dev_priv->irq_mask = ~0;
3670 3671
}

J
Jesse Barnes 已提交
3672 3673
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3674
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3675 3676 3677 3678

	if (!dev_priv)
		return;

3679 3680
	I915_WRITE(VLV_MASTER_IER, 0);

3681 3682
	gen5_gt_irq_reset(dev);

J
Jesse Barnes 已提交
3683
	I915_WRITE(HWSTAM, 0xffffffff);
3684

3685
	vlv_display_irq_uninstall(dev_priv);
J
Jesse Barnes 已提交
3686 3687
}

3688 3689 3690 3691 3692 3693 3694 3695 3696 3697
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3698
	gen8_gt_irq_reset(dev_priv);
3699

3700
	GEN5_IRQ_RESET(GEN8_PCU_);
3701

3702
	vlv_display_irq_uninstall(dev_priv);
3703 3704
}

3705
static void ironlake_irq_uninstall(struct drm_device *dev)
3706
{
3707
	struct drm_i915_private *dev_priv = dev->dev_private;
3708 3709 3710 3711

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3712
	ironlake_irq_reset(dev);
3713 3714
}

3715
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3716
{
3717
	struct drm_i915_private *dev_priv = dev->dev_private;
3718
	int pipe;
3719

3720
	for_each_pipe(dev_priv, pipe)
3721
		I915_WRITE(PIPESTAT(pipe), 0);
3722 3723 3724
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3725 3726 3727 3728
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3729
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3730 3731 3732 3733 3734 3735 3736 3737 3738

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3739
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3740 3741 3742 3743 3744 3745 3746 3747
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3748 3749
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3750
	spin_lock_irq(&dev_priv->irq_lock);
3751 3752
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3753
	spin_unlock_irq(&dev_priv->irq_lock);
3754

C
Chris Wilson 已提交
3755 3756 3757
	return 0;
}

3758 3759 3760 3761
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_device *dev,
3762
			       int plane, int pipe, u32 iir)
3763
{
3764
	struct drm_i915_private *dev_priv = dev->dev_private;
3765
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3766

3767
	if (!intel_pipe_handle_vblank(dev, pipe))
3768 3769 3770
		return false;

	if ((iir & flip_pending) == 0)
3771
		goto check_page_flip;
3772 3773 3774 3775 3776 3777 3778 3779

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
3780
		goto check_page_flip;
3781

3782
	intel_prepare_page_flip(dev, plane);
3783 3784
	intel_finish_page_flip(dev, pipe);
	return true;
3785 3786 3787 3788

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
3789 3790
}

3791
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3792
{
3793
	struct drm_device *dev = arg;
3794
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3795 3796 3797 3798 3799 3800 3801
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;

3802 3803 3804
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

C
Chris Wilson 已提交
3805 3806 3807 3808 3809 3810 3811 3812 3813 3814
	iir = I915_READ16(IIR);
	if (iir == 0)
		return IRQ_NONE;

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3815
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3816
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3817
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3818

3819
		for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3820 3821 3822 3823 3824 3825
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3826
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3827 3828
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3829
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3830 3831 3832 3833 3834

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
C
Chris Wilson 已提交
3835
			notify_ring(&dev_priv->ring[RCS]);
C
Chris Wilson 已提交
3836

3837
		for_each_pipe(dev_priv, pipe) {
3838
			int plane = pipe;
3839
			if (HAS_FBC(dev))
3840 3841
				plane = !plane;

3842
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3843 3844
			    i8xx_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3845

3846
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3847
				i9xx_pipe_crc_irq_handler(dev, pipe);
3848

3849 3850 3851
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3852
		}
C
Chris Wilson 已提交
3853 3854 3855 3856 3857 3858 3859 3860 3861

		iir = new_iir;
	}

	return IRQ_HANDLED;
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3862
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3863 3864
	int pipe;

3865
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3866 3867 3868 3869 3870 3871 3872 3873 3874
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3875 3876
static void i915_irq_preinstall(struct drm_device * dev)
{
3877
	struct drm_i915_private *dev_priv = dev->dev_private;
3878 3879 3880
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
3881
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3882 3883 3884
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3885
	I915_WRITE16(HWSTAM, 0xeffe);
3886
	for_each_pipe(dev_priv, pipe)
3887 3888 3889 3890 3891 3892 3893 3894
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3895
	struct drm_i915_private *dev_priv = dev->dev_private;
3896
	u32 enable_mask;
3897

3898 3899 3900 3901 3902 3903 3904 3905
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3906
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3907 3908 3909 3910 3911 3912 3913

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

3914
	if (I915_HAS_HOTPLUG(dev)) {
3915
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3916 3917
		POSTING_READ(PORT_HOTPLUG_EN);

3918 3919 3920 3921 3922 3923 3924 3925 3926 3927
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3928
	i915_enable_asle_pipestat(dev);
3929

3930 3931
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3932
	spin_lock_irq(&dev_priv->irq_lock);
3933 3934
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3935
	spin_unlock_irq(&dev_priv->irq_lock);
3936

3937 3938 3939
	return 0;
}

3940 3941 3942 3943 3944 3945
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_device *dev,
			       int plane, int pipe, u32 iir)
{
3946
	struct drm_i915_private *dev_priv = dev->dev_private;
3947 3948
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

3949
	if (!intel_pipe_handle_vblank(dev, pipe))
3950 3951 3952
		return false;

	if ((iir & flip_pending) == 0)
3953
		goto check_page_flip;
3954 3955 3956 3957 3958 3959 3960 3961

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
3962
		goto check_page_flip;
3963

3964
	intel_prepare_page_flip(dev, plane);
3965 3966
	intel_finish_page_flip(dev, pipe);
	return true;
3967 3968 3969 3970

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
3971 3972
}

3973
static irqreturn_t i915_irq_handler(int irq, void *arg)
3974
{
3975
	struct drm_device *dev = arg;
3976
	struct drm_i915_private *dev_priv = dev->dev_private;
3977
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3978 3979 3980 3981
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
3982

3983 3984 3985
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3986
	iir = I915_READ(IIR);
3987 3988
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
3989
		bool blc_event = false;
3990 3991 3992 3993 3994 3995

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3996
		spin_lock(&dev_priv->irq_lock);
3997
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3998
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3999

4000
		for_each_pipe(dev_priv, pipe) {
4001 4002 4003
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

4004
			/* Clear the PIPE*STAT regs before the IIR */
4005 4006
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4007
				irq_received = true;
4008 4009
			}
		}
4010
		spin_unlock(&dev_priv->irq_lock);
4011 4012 4013 4014 4015

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
4016 4017 4018
		if (I915_HAS_HOTPLUG(dev) &&
		    iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4019

4020
		I915_WRITE(IIR, iir & ~flip_mask);
4021 4022 4023
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
C
Chris Wilson 已提交
4024
			notify_ring(&dev_priv->ring[RCS]);
4025

4026
		for_each_pipe(dev_priv, pipe) {
4027
			int plane = pipe;
4028
			if (HAS_FBC(dev))
4029
				plane = !plane;
4030

4031
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4032 4033
			    i915_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4034 4035 4036

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4037 4038

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4039
				i9xx_pipe_crc_irq_handler(dev, pipe);
4040

4041 4042 4043
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
4064
		ret = IRQ_HANDLED;
4065
		iir = new_iir;
4066
	} while (iir & ~flip_mask);
4067 4068 4069 4070 4071 4072

	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
4073
	struct drm_i915_private *dev_priv = dev->dev_private;
4074 4075 4076
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4077
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4078 4079 4080
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4081
	I915_WRITE16(HWSTAM, 0xffff);
4082
	for_each_pipe(dev_priv, pipe) {
4083
		/* Clear enable bits; then clear status bits */
4084
		I915_WRITE(PIPESTAT(pipe), 0);
4085 4086
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4087 4088 4089 4090 4091 4092 4093 4094
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4095
	struct drm_i915_private *dev_priv = dev->dev_private;
4096 4097
	int pipe;

4098
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4099
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4100 4101

	I915_WRITE(HWSTAM, 0xeffe);
4102
	for_each_pipe(dev_priv, pipe)
4103 4104 4105 4106 4107 4108 4109 4110
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4111
	struct drm_i915_private *dev_priv = dev->dev_private;
4112
	u32 enable_mask;
4113 4114 4115
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4116
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4117
			       I915_DISPLAY_PORT_INTERRUPT |
4118 4119 4120 4121 4122 4123 4124
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4125 4126
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4127 4128 4129 4130
	enable_mask |= I915_USER_INTERRUPT;

	if (IS_G4X(dev))
		enable_mask |= I915_BSD_USER_INTERRUPT;
4131

4132 4133
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4134
	spin_lock_irq(&dev_priv->irq_lock);
4135 4136 4137
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4138
	spin_unlock_irq(&dev_priv->irq_lock);
4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4159
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4160 4161
	POSTING_READ(PORT_HOTPLUG_EN);

4162
	i915_enable_asle_pipestat(dev);
4163 4164 4165 4166

	return 0;
}

4167
static void i915_hpd_irq_setup(struct drm_device *dev)
4168
{
4169
	struct drm_i915_private *dev_priv = dev->dev_private;
4170 4171
	u32 hotplug_en;

4172 4173
	assert_spin_locked(&dev_priv->irq_lock);

4174 4175
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4176
	hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4177 4178 4179 4180 4181 4182 4183 4184 4185
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
	if (IS_G4X(dev))
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4186 4187 4188 4189
	i915_hotplug_interrupt_update_locked(dev_priv,
				      (HOTPLUG_INT_EN_MASK
				       | CRT_HOTPLUG_VOLTAGE_COMPARE_MASK),
				      hotplug_en);
4190 4191
}

4192
static irqreturn_t i965_irq_handler(int irq, void *arg)
4193
{
4194
	struct drm_device *dev = arg;
4195
	struct drm_i915_private *dev_priv = dev->dev_private;
4196 4197 4198
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4199 4200 4201
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4202

4203 4204 4205
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4206 4207 4208
	iir = I915_READ(IIR);

	for (;;) {
4209
		bool irq_received = (iir & ~flip_mask) != 0;
4210 4211
		bool blc_event = false;

4212 4213 4214 4215 4216
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4217
		spin_lock(&dev_priv->irq_lock);
4218
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4219
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4220

4221
		for_each_pipe(dev_priv, pipe) {
4222 4223 4224 4225 4226 4227 4228 4229
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4230
				irq_received = true;
4231 4232
			}
		}
4233
		spin_unlock(&dev_priv->irq_lock);
4234 4235 4236 4237 4238 4239 4240

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4241 4242
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4243

4244
		I915_WRITE(IIR, iir & ~flip_mask);
4245 4246 4247
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
C
Chris Wilson 已提交
4248
			notify_ring(&dev_priv->ring[RCS]);
4249
		if (iir & I915_BSD_USER_INTERRUPT)
C
Chris Wilson 已提交
4250
			notify_ring(&dev_priv->ring[VCS]);
4251

4252
		for_each_pipe(dev_priv, pipe) {
4253
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4254 4255
			    i915_handle_vblank(dev, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4256 4257 4258

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4259 4260

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4261
				i9xx_pipe_crc_irq_handler(dev, pipe);
4262

4263 4264
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4265
		}
4266 4267 4268 4269

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

4270 4271 4272
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
			gmbus_irq_handler(dev);

4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4296
	struct drm_i915_private *dev_priv = dev->dev_private;
4297 4298 4299 4300 4301
	int pipe;

	if (!dev_priv)
		return;

4302
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4303
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4304 4305

	I915_WRITE(HWSTAM, 0xffffffff);
4306
	for_each_pipe(dev_priv, pipe)
4307 4308 4309 4310
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4311
	for_each_pipe(dev_priv, pipe)
4312 4313 4314 4315 4316
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4317 4318 4319 4320 4321 4322 4323
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4324
void intel_irq_init(struct drm_i915_private *dev_priv)
4325
{
4326
	struct drm_device *dev = dev_priv->dev;
4327

4328 4329
	intel_hpd_init_work(dev_priv);

4330
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4331
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4332

4333
	/* Let's track the enabled rps events */
4334
	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4335
		/* WaGsvRC0ResidencyMethod:vlv */
4336
		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4337 4338
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4339

4340 4341
	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
			  i915_hangcheck_elapsed);
4342

4343
	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4344

4345
	if (IS_GEN2(dev_priv)) {
4346 4347
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4348
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4349 4350
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4351 4352 4353
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4354 4355
	}

4356 4357 4358 4359 4360
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4361
	if (!IS_GEN2(dev_priv))
4362 4363
		dev->vblank_disable_immediate = true;

4364 4365
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4366

4367
	if (IS_CHERRYVIEW(dev_priv)) {
4368 4369 4370 4371 4372 4373 4374
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4375
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4376 4377 4378 4379 4380 4381
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4382
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4383
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4384
		dev->driver->irq_handler = gen8_irq_handler;
4385
		dev->driver->irq_preinstall = gen8_irq_reset;
4386 4387 4388 4389
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4390
		if (IS_BROXTON(dev))
4391
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4392 4393 4394
		else if (HAS_PCH_SPT(dev))
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4395
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4396 4397
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
4398
		dev->driver->irq_preinstall = ironlake_irq_reset;
4399 4400 4401 4402
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4403
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4404
	} else {
4405
		if (INTEL_INFO(dev_priv)->gen == 2) {
C
Chris Wilson 已提交
4406 4407 4408 4409
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4410
		} else if (INTEL_INFO(dev_priv)->gen == 3) {
4411 4412 4413 4414
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
C
Chris Wilson 已提交
4415
		} else {
4416 4417 4418 4419
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
C
Chris Wilson 已提交
4420
		}
4421 4422
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4423 4424 4425 4426
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4427

4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
}

4451 4452 4453 4454 4455 4456 4457
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4458 4459 4460 4461 4462 4463 4464
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
	drm_irq_uninstall(dev_priv->dev);
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4465 4466 4467 4468 4469 4470 4471
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4472
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4473
{
4474
	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4475
	dev_priv->pm.irqs_enabled = false;
4476
	synchronize_irq(dev_priv->dev->irq);
4477 4478
}

4479 4480 4481 4482 4483 4484 4485
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4486
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4487
{
4488
	dev_priv->pm.irqs_enabled = true;
4489 4490
	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4491
}