i915_irq.c 126.9 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51 52 53 54 55 56 57
static const u32 hpd_ibx[] = {
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

static const u32 hpd_cpt[] = {
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
58
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
59 60 61 62 63 64 65 66 67 68 69 70 71 72
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

static const u32 hpd_mask_i915[] = {
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

73
static const u32 hpd_status_g4x[] = {
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

91
/* IIR can theoretically queue up two events. Be paranoid. */
92
#define GEN8_IRQ_RESET_NDX(type, which) do { \
93 94 95 96 97 98 99 100 101
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

102
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
103
	I915_WRITE(type##IMR, 0xffffffff); \
104
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
105
	I915_WRITE(type##IER, 0); \
106 107 108 109
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
110 111
} while (0)

112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
	u32 val = I915_READ(reg); \
	if (val) { \
		WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
		     (reg), val); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
	} \
} while (0)

P
Paulo Zanoni 已提交
127
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
128
	GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
129
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
130 131
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
132 133 134
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
135
	GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
P
Paulo Zanoni 已提交
136
	I915_WRITE(type##IER, (ier_val)); \
137 138
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
139 140
} while (0)

141 142
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);

143
/* For display hotplug interrupt */
144
void
145
ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
146
{
147 148
	assert_spin_locked(&dev_priv->irq_lock);

149
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
150 151
		return;

152 153 154
	if ((dev_priv->irq_mask & mask) != 0) {
		dev_priv->irq_mask &= ~mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
155
		POSTING_READ(DEIMR);
156 157 158
	}
}

159
void
160
ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
161
{
162 163
	assert_spin_locked(&dev_priv->irq_lock);

164
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
165 166
		return;

167 168 169
	if ((dev_priv->irq_mask & mask) != mask) {
		dev_priv->irq_mask |= mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
170
		POSTING_READ(DEIMR);
171 172 173
	}
}

P
Paulo Zanoni 已提交
174 175 176 177 178 179 180 181 182 183 184 185
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

186
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
187 188
		return;

P
Paulo Zanoni 已提交
189 190 191 192 193 194
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

195
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
196 197 198 199
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

200
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
201 202 203 204
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

205 206 207 208 209
static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

210 211 212 213 214
static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

215 216 217 218 219
static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
220 221 222 223 224 225 226 227 228 229
/**
  * snb_update_pm_irq - update GEN6_PMIMR
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
230
	uint32_t new_val;
P
Paulo Zanoni 已提交
231 232 233

	assert_spin_locked(&dev_priv->irq_lock);

234
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
235 236
		return;

237
	new_val = dev_priv->pm_irq_mask;
238 239 240
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

241 242
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
243 244
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		POSTING_READ(gen6_pm_imr(dev_priv));
245
	}
P
Paulo Zanoni 已提交
246 247
}

248
void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
249 250 251 252
{
	snb_update_pm_irq(dev_priv, mask, mask);
}

253
void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
254 255 256 257
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

I
Imre Deak 已提交
258 259 260 261 262 263 264 265 266 267 268 269
void gen6_reset_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	uint32_t reg = gen6_pm_iir(dev_priv);

	spin_lock_irq(&dev_priv->irq_lock);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	POSTING_READ(reg);
	spin_unlock_irq(&dev_priv->irq_lock);
}

270 271 272 273 274 275
void gen6_enable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	spin_lock_irq(&dev_priv->irq_lock);
	WARN_ON(dev_priv->rps.pm_iir);
I
Imre Deak 已提交
276
	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen6_disable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN6_PMINTRMSK, INTEL_INFO(dev_priv)->gen >= 8 ?
		   ~GEN8_PMINTR_REDIRECT_TO_NON_DISP : ~0);
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
				~dev_priv->pm_rps_events);
	/* Complete PM interrupt masking here doesn't race with the rps work
	 * item again unmasking PM interrupts because that is using a different
	 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
	 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */

	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.pm_iir = 0;
	spin_unlock_irq(&dev_priv->irq_lock);

	I915_WRITE(gen6_pm_iir(dev_priv), dev_priv->pm_rps_events);
}

301 302 303 304 305 306
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
307 308 309
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
310 311 312 313 314 315 316
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

	assert_spin_locked(&dev_priv->irq_lock);

317
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
318 319
		return;

320 321 322
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
323

D
Daniel Vetter 已提交
324
static void
325 326
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
327
{
328
	u32 reg = PIPESTAT(pipe);
329
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
330

331
	assert_spin_locked(&dev_priv->irq_lock);
332
	WARN_ON(!intel_irqs_enabled(dev_priv));
333

334 335 336 337
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
338 339 340
		return;

	if ((pipestat & enable_mask) == enable_mask)
341 342
		return;

343 344
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

345
	/* Enable the interrupt, clear any pending status */
346
	pipestat |= enable_mask | status_mask;
347 348
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
349 350
}

D
Daniel Vetter 已提交
351
static void
352 353
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
354
{
355
	u32 reg = PIPESTAT(pipe);
356
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
357

358
	assert_spin_locked(&dev_priv->irq_lock);
359
	WARN_ON(!intel_irqs_enabled(dev_priv));
360

361 362 363 364
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
365 366
		return;

367 368 369
	if ((pipestat & enable_mask) == 0)
		return;

370 371
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

372
	pipestat &= ~enable_mask;
373 374
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
375 376
}

377 378 379 380 381
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
382 383
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
384 385 386
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
387 388 389 390 391 392
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
393 394 395 396 397 398 399 400 401 402 403 404

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

405 406 407 408 409 410
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

411 412 413 414 415
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
416 417 418 419 420 421 422 423 424
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

425 426 427 428 429
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
430 431 432
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

433
/**
434
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
435
 */
436
static void i915_enable_asle_pipestat(struct drm_device *dev)
437
{
438
	struct drm_i915_private *dev_priv = dev->dev_private;
439

440 441 442
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
		return;

443
	spin_lock_irq(&dev_priv->irq_lock);
444

445
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
446
	if (INTEL_INFO(dev)->gen >= 4)
447
		i915_enable_pipestat(dev_priv, PIPE_A,
448
				     PIPE_LEGACY_BLC_EVENT_STATUS);
449

450
	spin_unlock_irq(&dev_priv->irq_lock);
451 452
}

453 454 455 456 457 458 459 460 461 462 463 464
/**
 * i915_pipe_enabled - check if a pipe is enabled
 * @dev: DRM device
 * @pipe: pipe to check
 *
 * Reading certain registers when the pipe is disabled can hang the chip.
 * Use this routine to make sure the PLL is running and the pipe is active
 * before reading such registers if unsure.
 */
static int
i915_pipe_enabled(struct drm_device *dev, int pipe)
{
465
	struct drm_i915_private *dev_priv = dev->dev_private;
466

467 468 469 470
	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
		/* Locking is horribly broken here, but whatever. */
		struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
471

472 473 474 475
		return intel_crtc->active;
	} else {
		return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
	}
476 477
}

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

528 529 530 531 532 533
static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

534 535 536
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
537
static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
538
{
539
	struct drm_i915_private *dev_priv = dev->dev_private;
540 541
	unsigned long high_frame;
	unsigned long low_frame;
542
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
543 544

	if (!i915_pipe_enabled(dev, pipe)) {
545
		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
546
				"pipe %c\n", pipe_name(pipe));
547 548 549
		return 0;
	}

550 551 552 553 554 555
	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
		struct intel_crtc *intel_crtc =
			to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
		const struct drm_display_mode *mode =
			&intel_crtc->config.adjusted_mode;

556 557 558 559 560
		htotal = mode->crtc_htotal;
		hsync_start = mode->crtc_hsync_start;
		vbl_start = mode->crtc_vblank_start;
		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
			vbl_start = DIV_ROUND_UP(vbl_start, 2);
561
	} else {
562
		enum transcoder cpu_transcoder = (enum transcoder) pipe;
563 564

		htotal = ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff) + 1;
565
		hsync_start = (I915_READ(HSYNC(cpu_transcoder))  & 0x1fff) + 1;
566
		vbl_start = (I915_READ(VBLANK(cpu_transcoder)) & 0x1fff) + 1;
567 568 569
		if ((I915_READ(PIPECONF(cpu_transcoder)) &
		     PIPECONF_INTERLACE_MASK) != PIPECONF_PROGRESSIVE)
			vbl_start = DIV_ROUND_UP(vbl_start, 2);
570 571
	}

572 573 574 575 576 577
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

578 579
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
580

581 582 583 584 585 586
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
587
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
588
		low   = I915_READ(low_frame);
589
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
590 591
	} while (high1 != high2);

592
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
593
	pixel = low & PIPE_PIXEL_MASK;
594
	low >>= PIPE_FRAME_LOW_SHIFT;
595 596 597 598 599 600

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
601
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
602 603
}

604
static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
605
{
606
	struct drm_i915_private *dev_priv = dev->dev_private;
607
	int reg = PIPE_FRMCOUNT_GM45(pipe);
608 609

	if (!i915_pipe_enabled(dev, pipe)) {
610
		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
611
				 "pipe %c\n", pipe_name(pipe));
612 613 614 615 616 617
		return 0;
	}

	return I915_READ(reg);
}

618 619 620
/* raw reads, only for fast reads of display block, no need for forcewake etc. */
#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))

621 622 623 624 625 626
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	const struct drm_display_mode *mode = &crtc->config.adjusted_mode;
	enum pipe pipe = crtc->pipe;
627
	int position, vtotal;
628

629
	vtotal = mode->crtc_vtotal;
630 631 632 633 634 635 636 637 638
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	if (IS_GEN2(dev))
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
	else
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;

	/*
639 640
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
641
	 */
642
	return (position + crtc->scanline_offset) % vtotal;
643 644
}

645
static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
646 647
				    unsigned int flags, int *vpos, int *hpos,
				    ktime_t *stime, ktime_t *etime)
648
{
649 650 651 652
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
	const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
653
	int position;
654
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
655 656
	bool in_vbl = true;
	int ret = 0;
657
	unsigned long irqflags;
658

659
	if (!intel_crtc->active) {
660
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
661
				 "pipe %c\n", pipe_name(pipe));
662 663 664
		return 0;
	}

665
	htotal = mode->crtc_htotal;
666
	hsync_start = mode->crtc_hsync_start;
667 668 669
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
670

671 672 673 674 675 676
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

677 678
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

679 680 681 682 683 684
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
685

686 687 688 689 690 691
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

692
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
693 694 695
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
696
		position = __intel_get_crtc_scanline(intel_crtc);
697 698 699 700 701
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
702
		position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
703

704 705 706 707
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
708

709 710 711 712 713 714 715 716 717 718 719 720
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

721 722 723 724 725 726 727 728 729 730
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
731 732
	}

733 734 735 736 737 738 739 740
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

741 742 743 744 745 746 747 748 749 750 751 752
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
753

754
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
755 756 757 758 759 760
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
761 762 763

	/* In vblank? */
	if (in_vbl)
764
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
765 766 767 768

	return ret;
}

769 770 771 772 773 774 775 776 777 778 779 780 781
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

782
static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
783 784 785 786
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
787
	struct drm_crtc *crtc;
788

789
	if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
790
		DRM_ERROR("Invalid crtc %d\n", pipe);
791 792 793 794
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
795 796 797 798 799 800 801 802 803 804
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
		DRM_ERROR("Invalid crtc %d\n", pipe);
		return -EINVAL;
	}

	if (!crtc->enabled) {
		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
		return -EBUSY;
	}
805 806

	/* Helper routine in DRM core does all the work: */
807 808
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
809 810
						     crtc,
						     &to_intel_crtc(crtc)->config.adjusted_mode);
811 812
}

813 814
static bool intel_hpd_irq_event(struct drm_device *dev,
				struct drm_connector *connector)
815 816 817 818 819 820 821
{
	enum drm_connector_status old_status;

	WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
	old_status = connector->status;

	connector->status = connector->funcs->detect(connector, false);
822 823 824 825
	if (old_status == connector->status)
		return false;

	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
826
		      connector->base.id,
827
		      connector->name,
828 829 830 831
		      drm_get_connector_status_name(old_status),
		      drm_get_connector_status_name(connector->status));

	return true;
832 833
}

834 835 836 837 838 839 840 841 842
static void i915_digport_work_func(struct work_struct *work)
{
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, dig_port_work);
	u32 long_port_mask, short_port_mask;
	struct intel_digital_port *intel_dig_port;
	int i, ret;
	u32 old_bits = 0;

843
	spin_lock_irq(&dev_priv->irq_lock);
844 845 846 847
	long_port_mask = dev_priv->long_hpd_port_mask;
	dev_priv->long_hpd_port_mask = 0;
	short_port_mask = dev_priv->short_hpd_port_mask;
	dev_priv->short_hpd_port_mask = 0;
848
	spin_unlock_irq(&dev_priv->irq_lock);
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872

	for (i = 0; i < I915_MAX_PORTS; i++) {
		bool valid = false;
		bool long_hpd = false;
		intel_dig_port = dev_priv->hpd_irq_port[i];
		if (!intel_dig_port || !intel_dig_port->hpd_pulse)
			continue;

		if (long_port_mask & (1 << i))  {
			valid = true;
			long_hpd = true;
		} else if (short_port_mask & (1 << i))
			valid = true;

		if (valid) {
			ret = intel_dig_port->hpd_pulse(intel_dig_port, long_hpd);
			if (ret == true) {
				/* if we get true fallback to old school hpd */
				old_bits |= (1 << intel_dig_port->base.hpd_pin);
			}
		}
	}

	if (old_bits) {
873
		spin_lock_irq(&dev_priv->irq_lock);
874
		dev_priv->hpd_event_bits |= old_bits;
875
		spin_unlock_irq(&dev_priv->irq_lock);
876 877 878 879
		schedule_work(&dev_priv->hotplug_work);
	}
}

880 881 882
/*
 * Handle hotplug events outside the interrupt handler proper.
 */
883 884
#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)

885 886
static void i915_hotplug_work_func(struct work_struct *work)
{
887 888
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, hotplug_work);
889
	struct drm_device *dev = dev_priv->dev;
890
	struct drm_mode_config *mode_config = &dev->mode_config;
891 892 893 894
	struct intel_connector *intel_connector;
	struct intel_encoder *intel_encoder;
	struct drm_connector *connector;
	bool hpd_disabled = false;
895
	bool changed = false;
896
	u32 hpd_event_bits;
897

898
	mutex_lock(&mode_config->mutex);
899 900
	DRM_DEBUG_KMS("running encoder hotplug functions\n");

901
	spin_lock_irq(&dev_priv->irq_lock);
902 903 904

	hpd_event_bits = dev_priv->hpd_event_bits;
	dev_priv->hpd_event_bits = 0;
905 906
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		intel_connector = to_intel_connector(connector);
907 908
		if (!intel_connector->encoder)
			continue;
909 910 911 912 913 914
		intel_encoder = intel_connector->encoder;
		if (intel_encoder->hpd_pin > HPD_NONE &&
		    dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
		    connector->polled == DRM_CONNECTOR_POLL_HPD) {
			DRM_INFO("HPD interrupt storm detected on connector %s: "
				 "switching from hotplug detection to polling\n",
915
				connector->name);
916 917 918 919 920
			dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
			connector->polled = DRM_CONNECTOR_POLL_CONNECT
				| DRM_CONNECTOR_POLL_DISCONNECT;
			hpd_disabled = true;
		}
921 922
		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
			DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
923
				      connector->name, intel_encoder->hpd_pin);
924
		}
925 926 927 928
	}
	 /* if there were no outputs to poll, poll was disabled,
	  * therefore make sure it's enabled when disabling HPD on
	  * some connectors */
929
	if (hpd_disabled) {
930
		drm_kms_helper_poll_enable(dev);
931 932
		mod_delayed_work(system_wq, &dev_priv->hotplug_reenable_work,
				 msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
933
	}
934

935
	spin_unlock_irq(&dev_priv->irq_lock);
936

937 938
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		intel_connector = to_intel_connector(connector);
939 940
		if (!intel_connector->encoder)
			continue;
941 942 943 944 945 946 947 948
		intel_encoder = intel_connector->encoder;
		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
			if (intel_encoder->hot_plug)
				intel_encoder->hot_plug(intel_encoder);
			if (intel_hpd_irq_event(dev, connector))
				changed = true;
		}
	}
949 950
	mutex_unlock(&mode_config->mutex);

951 952
	if (changed)
		drm_kms_helper_hotplug_event(dev);
953 954
}

955
static void ironlake_rps_change_irq_handler(struct drm_device *dev)
956
{
957
	struct drm_i915_private *dev_priv = dev->dev_private;
958
	u32 busy_up, busy_down, max_avg, min_avg;
959 960
	u8 new_delay;

961
	spin_lock(&mchdev_lock);
962

963 964
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

965
	new_delay = dev_priv->ips.cur_delay;
966

967
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
968 969
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
970 971 972 973
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
974
	if (busy_up > max_avg) {
975 976 977 978
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
979
	} else if (busy_down < min_avg) {
980 981 982 983
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
984 985
	}

986
	if (ironlake_set_drps(dev, new_delay))
987
		dev_priv->ips.cur_delay = new_delay;
988

989
	spin_unlock(&mchdev_lock);
990

991 992 993
	return;
}

994
static void notify_ring(struct drm_device *dev,
995
			struct intel_engine_cs *ring)
996
{
997
	if (!intel_ring_initialized(ring))
998 999
		return;

1000
	trace_i915_gem_request_complete(ring);
1001

1002 1003 1004
	wake_up_all(&ring->irq_queue);
}

1005
static u32 vlv_c0_residency(struct drm_i915_private *dev_priv,
1006
			    struct intel_rps_ei *rps_ei)
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
{
	u32 cz_ts, cz_freq_khz;
	u32 render_count, media_count;
	u32 elapsed_render, elapsed_media, elapsed_time;
	u32 residency = 0;

	cz_ts = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	cz_freq_khz = DIV_ROUND_CLOSEST(dev_priv->mem_freq * 1000, 4);

	render_count = I915_READ(VLV_RENDER_C0_COUNT_REG);
	media_count = I915_READ(VLV_MEDIA_C0_COUNT_REG);

1019 1020 1021 1022
	if (rps_ei->cz_clock == 0) {
		rps_ei->cz_clock = cz_ts;
		rps_ei->render_c0 = render_count;
		rps_ei->media_c0 = media_count;
1023 1024 1025 1026

		return dev_priv->rps.cur_freq;
	}

1027 1028
	elapsed_time = cz_ts - rps_ei->cz_clock;
	rps_ei->cz_clock = cz_ts;
1029

1030 1031
	elapsed_render = render_count - rps_ei->render_c0;
	rps_ei->render_c0 = render_count;
1032

1033 1034
	elapsed_media = media_count - rps_ei->media_c0;
	rps_ei->media_c0 = media_count;
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059

	/* Convert all the counters into common unit of milli sec */
	elapsed_time /= VLV_CZ_CLOCK_TO_MILLI_SEC;
	elapsed_render /=  cz_freq_khz;
	elapsed_media /= cz_freq_khz;

	/*
	 * Calculate overall C0 residency percentage
	 * only if elapsed time is non zero
	 */
	if (elapsed_time) {
		residency =
			((max(elapsed_render, elapsed_media) * 100)
				/ elapsed_time);
	}

	return residency;
}

/**
 * vlv_calc_delay_from_C0_counters - Increase/Decrease freq based on GPU
 * busy-ness calculated from C0 counters of render & media power wells
 * @dev_priv: DRM device private
 *
 */
1060
static int vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
1061 1062
{
	u32 residency_C0_up = 0, residency_C0_down = 0;
1063
	int new_delay, adj;
1064 1065 1066 1067 1068 1069

	dev_priv->rps.ei_interrupt_count++;

	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));


1070 1071 1072
	if (dev_priv->rps.up_ei.cz_clock == 0) {
		vlv_c0_residency(dev_priv, &dev_priv->rps.up_ei);
		vlv_c0_residency(dev_priv, &dev_priv->rps.down_ei);
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
		return dev_priv->rps.cur_freq;
	}


	/*
	 * To down throttle, C0 residency should be less than down threshold
	 * for continous EI intervals. So calculate down EI counters
	 * once in VLV_INT_COUNT_FOR_DOWN_EI
	 */
	if (dev_priv->rps.ei_interrupt_count == VLV_INT_COUNT_FOR_DOWN_EI) {

		dev_priv->rps.ei_interrupt_count = 0;

		residency_C0_down = vlv_c0_residency(dev_priv,
1087
						     &dev_priv->rps.down_ei);
1088 1089
	} else {
		residency_C0_up = vlv_c0_residency(dev_priv,
1090
						   &dev_priv->rps.up_ei);
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	}

	new_delay = dev_priv->rps.cur_freq;

	adj = dev_priv->rps.last_adj;
	/* C0 residency is greater than UP threshold. Increase Frequency */
	if (residency_C0_up >= VLV_RP_UP_EI_THRESHOLD) {
		if (adj > 0)
			adj *= 2;
		else
			adj = 1;

		if (dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit)
			new_delay = dev_priv->rps.cur_freq + adj;

		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
		if (new_delay < dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;

	} else if (!dev_priv->rps.ei_interrupt_count &&
			(residency_C0_down < VLV_RP_DOWN_EI_THRESHOLD)) {
		if (adj < 0)
			adj *= 2;
		else
			adj = -1;
		/*
		 * This means, C0 residency is less than down threshold over
		 * a period of VLV_INT_COUNT_FOR_DOWN_EI. So, reduce the freq
		 */
		if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
			new_delay = dev_priv->rps.cur_freq + adj;
	}

	return new_delay;
}

1130
static void gen6_pm_rps_work(struct work_struct *work)
1131
{
1132 1133
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
P
Paulo Zanoni 已提交
1134
	u32 pm_iir;
1135
	int new_delay, adj;
1136

1137
	spin_lock_irq(&dev_priv->irq_lock);
1138 1139
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1140 1141
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1142
	spin_unlock_irq(&dev_priv->irq_lock);
1143

1144
	/* Make sure we didn't queue anything we're not going to process. */
1145
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1146

1147
	if ((pm_iir & dev_priv->pm_rps_events) == 0)
1148 1149
		return;

1150
	mutex_lock(&dev_priv->rps.hw_lock);
1151

1152
	adj = dev_priv->rps.last_adj;
1153
	if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1154 1155
		if (adj > 0)
			adj *= 2;
1156 1157 1158 1159
		else {
			/* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv->dev) ? 2 : 1;
		}
1160
		new_delay = dev_priv->rps.cur_freq + adj;
1161 1162 1163 1164 1165

		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1166 1167
		if (new_delay < dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1168
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1169 1170
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1171
		else
1172
			new_delay = dev_priv->rps.min_freq_softlimit;
1173
		adj = 0;
1174 1175
	} else if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		new_delay = vlv_calc_delay_from_C0_counters(dev_priv);
1176 1177 1178
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1179 1180 1181 1182
		else {
			/* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv->dev) ? -2 : -1;
		}
1183
		new_delay = dev_priv->rps.cur_freq + adj;
1184
	} else { /* unknown event */
1185
		new_delay = dev_priv->rps.cur_freq;
1186
	}
1187

1188 1189 1190
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1191
	new_delay = clamp_t(int, new_delay,
1192 1193
			    dev_priv->rps.min_freq_softlimit,
			    dev_priv->rps.max_freq_softlimit);
1194

1195
	dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq;
1196 1197 1198 1199 1200

	if (IS_VALLEYVIEW(dev_priv->dev))
		valleyview_set_rps(dev_priv->dev, new_delay);
	else
		gen6_set_rps(dev_priv->dev, new_delay);
1201

1202
	mutex_unlock(&dev_priv->rps.hw_lock);
1203 1204
}

1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1217 1218
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1219
	u32 error_status, row, bank, subbank;
1220
	char *parity_event[6];
1221
	uint32_t misccpctl;
1222
	uint8_t slice = 0;
1223 1224 1225 1226 1227 1228 1229

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1230 1231 1232 1233
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1234 1235 1236 1237
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1238 1239
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		u32 reg;
1240

1241 1242 1243
		slice--;
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
			break;
1244

1245
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1246

1247
		reg = GEN7_L3CDERRST1 + (slice * 0x200);
1248

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1264
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1265
				   KOBJ_CHANGE, parity_event);
1266

1267 1268
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1269

1270 1271 1272 1273 1274
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1275

1276
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1277

1278 1279
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1280
	spin_lock_irq(&dev_priv->irq_lock);
1281
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1282
	spin_unlock_irq(&dev_priv->irq_lock);
1283 1284

	mutex_unlock(&dev_priv->dev->struct_mutex);
1285 1286
}

1287
static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1288
{
1289
	struct drm_i915_private *dev_priv = dev->dev_private;
1290

1291
	if (!HAS_L3_DPF(dev))
1292 1293
		return;

1294
	spin_lock(&dev_priv->irq_lock);
1295
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1296
	spin_unlock(&dev_priv->irq_lock);
1297

1298 1299 1300 1301 1302 1303 1304
	iir &= GT_PARITY_ERROR(dev);
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1305
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1306 1307
}

1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
static void ilk_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
		notify_ring(dev, &dev_priv->ring[RCS]);
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
		notify_ring(dev, &dev_priv->ring[VCS]);
}

1319 1320 1321 1322 1323
static void snb_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{

1324 1325
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1326
		notify_ring(dev, &dev_priv->ring[RCS]);
1327
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1328
		notify_ring(dev, &dev_priv->ring[VCS]);
1329
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1330 1331
		notify_ring(dev, &dev_priv->ring[BCS]);

1332 1333 1334
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
1335 1336
		i915_handle_error(dev, false, "GT error interrupt 0x%08x",
				  gt_iir);
1337
	}
1338

1339 1340
	if (gt_iir & GT_PARITY_ERROR(dev))
		ivybridge_parity_error_irq_handler(dev, gt_iir);
1341 1342
}

1343 1344 1345 1346
static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
				       struct drm_i915_private *dev_priv,
				       u32 master_ctl)
{
1347
	struct intel_engine_cs *ring;
1348 1349 1350 1351 1352 1353 1354
	u32 rcs, bcs, vcs;
	uint32_t tmp = 0;
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
		tmp = I915_READ(GEN8_GT_IIR(0));
		if (tmp) {
1355
			I915_WRITE(GEN8_GT_IIR(0), tmp);
1356
			ret = IRQ_HANDLED;
1357

1358
			rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
1359
			ring = &dev_priv->ring[RCS];
1360
			if (rcs & GT_RENDER_USER_INTERRUPT)
1361 1362 1363 1364 1365 1366
				notify_ring(dev, ring);
			if (rcs & GT_CONTEXT_SWITCH_INTERRUPT)
				intel_execlists_handle_ctx_events(ring);

			bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
			ring = &dev_priv->ring[BCS];
1367
			if (bcs & GT_RENDER_USER_INTERRUPT)
1368 1369 1370
				notify_ring(dev, ring);
			if (bcs & GT_CONTEXT_SWITCH_INTERRUPT)
				intel_execlists_handle_ctx_events(ring);
1371 1372 1373 1374
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1375
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1376 1377
		tmp = I915_READ(GEN8_GT_IIR(1));
		if (tmp) {
1378
			I915_WRITE(GEN8_GT_IIR(1), tmp);
1379
			ret = IRQ_HANDLED;
1380

1381
			vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
1382
			ring = &dev_priv->ring[VCS];
1383
			if (vcs & GT_RENDER_USER_INTERRUPT)
1384
				notify_ring(dev, ring);
1385
			if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
1386 1387
				intel_execlists_handle_ctx_events(ring);

1388
			vcs = tmp >> GEN8_VCS2_IRQ_SHIFT;
1389
			ring = &dev_priv->ring[VCS2];
1390
			if (vcs & GT_RENDER_USER_INTERRUPT)
1391
				notify_ring(dev, ring);
1392
			if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
1393
				intel_execlists_handle_ctx_events(ring);
1394 1395 1396 1397
		} else
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
	}

1398 1399 1400 1401 1402
	if (master_ctl & GEN8_GT_PM_IRQ) {
		tmp = I915_READ(GEN8_GT_IIR(2));
		if (tmp & dev_priv->pm_rps_events) {
			I915_WRITE(GEN8_GT_IIR(2),
				   tmp & dev_priv->pm_rps_events);
1403
			ret = IRQ_HANDLED;
1404
			gen6_rps_irq_handler(dev_priv, tmp);
1405 1406 1407 1408
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1409 1410 1411
	if (master_ctl & GEN8_GT_VECS_IRQ) {
		tmp = I915_READ(GEN8_GT_IIR(3));
		if (tmp) {
1412
			I915_WRITE(GEN8_GT_IIR(3), tmp);
1413
			ret = IRQ_HANDLED;
1414

1415
			vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
1416
			ring = &dev_priv->ring[VECS];
1417
			if (vcs & GT_RENDER_USER_INTERRUPT)
1418
				notify_ring(dev, ring);
1419
			if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
1420
				intel_execlists_handle_ctx_events(ring);
1421 1422 1423 1424 1425 1426 1427
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

	return ret;
}

1428 1429 1430
#define HPD_STORM_DETECT_PERIOD 1000
#define HPD_STORM_THRESHOLD 5

1431
static int pch_port_to_hotplug_shift(enum port port)
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446
{
	switch (port) {
	case PORT_A:
	case PORT_E:
	default:
		return -1;
	case PORT_B:
		return 0;
	case PORT_C:
		return 8;
	case PORT_D:
		return 16;
	}
}

1447
static int i915_port_to_hotplug_shift(enum port port)
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
{
	switch (port) {
	case PORT_A:
	case PORT_E:
	default:
		return -1;
	case PORT_B:
		return 17;
	case PORT_C:
		return 19;
	case PORT_D:
		return 21;
	}
}

static inline enum port get_port_from_pin(enum hpd_pin pin)
{
	switch (pin) {
	case HPD_PORT_B:
		return PORT_B;
	case HPD_PORT_C:
		return PORT_C;
	case HPD_PORT_D:
		return PORT_D;
	default:
		return PORT_A; /* no hpd */
	}
}

1477
static inline void intel_hpd_irq_handler(struct drm_device *dev,
1478
					 u32 hotplug_trigger,
1479
					 u32 dig_hotplug_reg,
1480
					 const u32 *hpd)
1481
{
1482
	struct drm_i915_private *dev_priv = dev->dev_private;
1483
	int i;
1484
	enum port port;
1485
	bool storm_detected = false;
1486 1487 1488
	bool queue_dig = false, queue_hp = false;
	u32 dig_shift;
	u32 dig_port_mask = 0;
1489

1490 1491 1492
	if (!hotplug_trigger)
		return;

1493 1494
	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg);
1495

1496
	spin_lock(&dev_priv->irq_lock);
1497
	for (i = 1; i < HPD_NUM_PINS; i++) {
1498 1499 1500 1501 1502 1503 1504
		if (!(hpd[i] & hotplug_trigger))
			continue;

		port = get_port_from_pin(i);
		if (port && dev_priv->hpd_irq_port[port]) {
			bool long_hpd;

1505 1506
			if (HAS_PCH_SPLIT(dev)) {
				dig_shift = pch_port_to_hotplug_shift(port);
1507
				long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
1508 1509 1510
			} else {
				dig_shift = i915_port_to_hotplug_shift(port);
				long_hpd = (hotplug_trigger >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
1511 1512
			}

1513 1514 1515
			DRM_DEBUG_DRIVER("digital hpd port %c - %s\n",
					 port_name(port),
					 long_hpd ? "long" : "short");
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528
			/* for long HPD pulses we want to have the digital queue happen,
			   but we still want HPD storm detection to function. */
			if (long_hpd) {
				dev_priv->long_hpd_port_mask |= (1 << port);
				dig_port_mask |= hpd[i];
			} else {
				/* for short HPD just trigger the digital queue */
				dev_priv->short_hpd_port_mask |= (1 << port);
				hotplug_trigger &= ~hpd[i];
			}
			queue_dig = true;
		}
	}
1529

1530
	for (i = 1; i < HPD_NUM_PINS; i++) {
1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
		if (hpd[i] & hotplug_trigger &&
		    dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED) {
			/*
			 * On GMCH platforms the interrupt mask bits only
			 * prevent irq generation, not the setting of the
			 * hotplug bits itself. So only WARN about unexpected
			 * interrupts on saner platforms.
			 */
			WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev),
				  "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n",
				  hotplug_trigger, i, hpd[i]);

			continue;
		}
1545

1546 1547 1548 1549
		if (!(hpd[i] & hotplug_trigger) ||
		    dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
			continue;

1550 1551 1552 1553 1554
		if (!(dig_port_mask & hpd[i])) {
			dev_priv->hpd_event_bits |= (1 << i);
			queue_hp = true;
		}

1555 1556 1557 1558 1559
		if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
				   dev_priv->hpd_stats[i].hpd_last_jiffies
				   + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
			dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
			dev_priv->hpd_stats[i].hpd_cnt = 0;
1560
			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
1561 1562
		} else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
			dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
1563
			dev_priv->hpd_event_bits &= ~(1 << i);
1564
			DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
1565
			storm_detected = true;
1566 1567
		} else {
			dev_priv->hpd_stats[i].hpd_cnt++;
1568 1569
			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
				      dev_priv->hpd_stats[i].hpd_cnt);
1570 1571 1572
		}
	}

1573 1574
	if (storm_detected)
		dev_priv->display.hpd_irq_setup(dev);
1575
	spin_unlock(&dev_priv->irq_lock);
1576

1577 1578 1579 1580 1581 1582
	/*
	 * Our hotplug handler can grab modeset locks (by calling down into the
	 * fb helpers). Hence it must not be run on our own dev-priv->wq work
	 * queue for otherwise the flush_work in the pageflip code will
	 * deadlock.
	 */
1583
	if (queue_dig)
1584
		queue_work(dev_priv->dp_wq, &dev_priv->dig_port_work);
1585 1586
	if (queue_hp)
		schedule_work(&dev_priv->hotplug_work);
1587 1588
}

1589 1590
static void gmbus_irq_handler(struct drm_device *dev)
{
1591
	struct drm_i915_private *dev_priv = dev->dev_private;
1592 1593

	wake_up_all(&dev_priv->gmbus_wait_queue);
1594 1595
}

1596 1597
static void dp_aux_irq_handler(struct drm_device *dev)
{
1598
	struct drm_i915_private *dev_priv = dev->dev_private;
1599 1600

	wake_up_all(&dev_priv->gmbus_wait_queue);
1601 1602
}

1603
#if defined(CONFIG_DEBUG_FS)
1604 1605 1606 1607
static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1608 1609 1610 1611
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1612
	int head, tail;
1613

1614 1615
	spin_lock(&pipe_crc->lock);

1616
	if (!pipe_crc->entries) {
1617
		spin_unlock(&pipe_crc->lock);
1618 1619 1620 1621
		DRM_ERROR("spurious interrupt\n");
		return;
	}

1622 1623
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1624 1625

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1626
		spin_unlock(&pipe_crc->lock);
1627 1628 1629 1630 1631
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1632

1633
	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1634 1635 1636 1637 1638
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1639 1640

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1641 1642 1643
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1644 1645

	wake_up_interruptible(&pipe_crc->wq);
1646
}
1647 1648 1649 1650 1651 1652 1653 1654
#else
static inline void
display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1655

1656
static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
D
Daniel Vetter 已提交
1657 1658 1659
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1660 1661 1662
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1663 1664
}

1665
static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1666 1667 1668
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1669 1670 1671 1672 1673 1674
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1675
}
1676

1677
static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1678 1679
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690
	uint32_t res1, res2;

	if (INTEL_INFO(dev)->gen >= 3)
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1691

1692 1693 1694 1695 1696
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1697
}
1698

1699 1700 1701 1702
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1703
{
1704 1705 1706
	/* TODO: RPS on GEN9+ is not supported yet. */
	if (WARN_ONCE(INTEL_INFO(dev_priv)->gen >= 9,
		      "GEN9+: unexpected RPS IRQ\n"))
1707 1708
		return;

1709
	if (pm_iir & dev_priv->pm_rps_events) {
1710
		spin_lock(&dev_priv->irq_lock);
1711
		dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1712
		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1713
		spin_unlock(&dev_priv->irq_lock);
1714 1715

		queue_work(dev_priv->wq, &dev_priv->rps.work);
1716 1717
	}

1718 1719 1720
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1721 1722 1723
	if (HAS_VEBOX(dev_priv->dev)) {
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
			notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
B
Ben Widawsky 已提交
1724

1725
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
1726 1727 1728
			i915_handle_error(dev_priv->dev, false,
					  "VEBOX CS error interrupt 0x%08x",
					  pm_iir);
1729
		}
B
Ben Widawsky 已提交
1730
	}
1731 1732
}

1733 1734 1735 1736 1737 1738 1739 1740
static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
{
	if (!drm_handle_vblank(dev, pipe))
		return false;

	return true;
}

1741 1742 1743
static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1744
	u32 pipe_stats[I915_MAX_PIPES] = { };
1745 1746
	int pipe;

1747
	spin_lock(&dev_priv->irq_lock);
1748
	for_each_pipe(dev_priv, pipe) {
1749
		int reg;
1750
		u32 mask, iir_bit = 0;
1751

1752 1753 1754 1755 1756 1757 1758
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1759 1760 1761

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1762 1763 1764 1765 1766 1767 1768 1769

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1770 1771 1772
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1773 1774 1775 1776 1777
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1778 1779 1780
			continue;

		reg = PIPESTAT(pipe);
1781 1782
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1783 1784 1785 1786

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1787 1788
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1789 1790
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1791
	spin_unlock(&dev_priv->irq_lock);
1792

1793
	for_each_pipe(dev_priv, pipe) {
1794 1795 1796
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
1797

1798
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1799 1800 1801 1802 1803 1804 1805
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip(dev, pipe);
		}

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
			i9xx_pipe_crc_irq_handler(dev, pipe);

1806 1807
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1808 1809 1810 1811 1812 1813
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		gmbus_irq_handler(dev);
}

1814 1815 1816 1817 1818
static void i9xx_hpd_irq_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1819 1820 1821 1822 1823 1824 1825
	if (hotplug_status) {
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
		/*
		 * Make sure hotplug status is cleared before we clear IIR, or else we
		 * may miss hotplug events.
		 */
		POSTING_READ(PORT_HOTPLUG_STAT);
1826

1827 1828
		if (IS_G4X(dev)) {
			u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1829

1830
			intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_g4x);
1831 1832
		} else {
			u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1833

1834
			intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_i915);
1835
		}
1836

1837 1838 1839 1840
		if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) &&
		    hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
			dp_aux_irq_handler(dev);
	}
1841 1842
}

1843
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1844
{
1845
	struct drm_device *dev = arg;
1846
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1847 1848 1849 1850
	u32 iir, gt_iir, pm_iir;
	irqreturn_t ret = IRQ_NONE;

	while (true) {
1851 1852
		/* Find, clear, then process each source of interrupt */

J
Jesse Barnes 已提交
1853
		gt_iir = I915_READ(GTIIR);
1854 1855 1856
		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);

J
Jesse Barnes 已提交
1857
		pm_iir = I915_READ(GEN6_PMIIR);
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

		iir = I915_READ(VLV_IIR);
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
J
Jesse Barnes 已提交
1868 1869 1870 1871 1872 1873

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
			goto out;

		ret = IRQ_HANDLED;

1874 1875
		if (gt_iir)
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
1876
		if (pm_iir)
1877
			gen6_rps_irq_handler(dev_priv, pm_iir);
1878 1879 1880
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
J
Jesse Barnes 已提交
1881 1882 1883 1884 1885 1886
	}

out:
	return ret;
}

1887 1888
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1889
	struct drm_device *dev = arg;
1890 1891 1892 1893
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl, iir;
	irqreturn_t ret = IRQ_NONE;

1894 1895 1896
	for (;;) {
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1897

1898 1899
		if (master_ctl == 0 && iir == 0)
			break;
1900

1901 1902
		ret = IRQ_HANDLED;

1903
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1904

1905
		/* Find, clear, then process each source of interrupt */
1906

1907 1908 1909 1910 1911 1912
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
1913

1914
		gen8_gt_irq_handler(dev, dev_priv, master_ctl);
1915

1916 1917 1918
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
1919

1920 1921 1922
		I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		POSTING_READ(GEN8_MASTER_IRQ);
	}
1923

1924 1925 1926
	return ret;
}

1927
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1928
{
1929
	struct drm_i915_private *dev_priv = dev->dev_private;
1930
	int pipe;
1931
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1932 1933 1934 1935
	u32 dig_hotplug_reg;

	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1936

1937
	intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_ibx);
1938

1939 1940 1941
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1942
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1943 1944
				 port_name(port));
	}
1945

1946 1947 1948
	if (pch_iir & SDE_AUX_MASK)
		dp_aux_irq_handler(dev);

1949
	if (pch_iir & SDE_GMBUS)
1950
		gmbus_irq_handler(dev);
1951 1952 1953 1954 1955 1956 1957 1958 1959 1960

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1961
	if (pch_iir & SDE_FDI_MASK)
1962
		for_each_pipe(dev_priv, pipe)
1963 1964 1965
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1966 1967 1968 1969 1970 1971 1972 1973

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1974
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1975 1976

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1977
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1978 1979 1980 1981 1982 1983
}

static void ivb_err_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
1984
	enum pipe pipe;
1985

1986 1987 1988
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

1989
	for_each_pipe(dev_priv, pipe) {
1990 1991
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1992

D
Daniel Vetter 已提交
1993 1994
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
			if (IS_IVYBRIDGE(dev))
1995
				ivb_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1996
			else
1997
				hsw_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1998 1999
		}
	}
2000

2001 2002 2003 2004 2005 2006 2007 2008
	I915_WRITE(GEN7_ERR_INT, err_int);
}

static void cpt_serr_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 serr_int = I915_READ(SERR_INT);

2009 2010 2011
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2012
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2013
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2014 2015

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2016
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2017 2018

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2019
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
2020 2021

	I915_WRITE(SERR_INT, serr_int);
2022 2023
}

2024 2025
static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
2026
	struct drm_i915_private *dev_priv = dev->dev_private;
2027
	int pipe;
2028
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2029 2030 2031 2032
	u32 dig_hotplug_reg;

	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2033

2034
	intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_cpt);
2035

2036 2037 2038 2039 2040 2041
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2042 2043

	if (pch_iir & SDE_AUX_MASK_CPT)
2044
		dp_aux_irq_handler(dev);
2045 2046

	if (pch_iir & SDE_GMBUS_CPT)
2047
		gmbus_irq_handler(dev);
2048 2049 2050 2051 2052 2053 2054 2055

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2056
		for_each_pipe(dev_priv, pipe)
2057 2058 2059
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2060 2061 2062

	if (pch_iir & SDE_ERROR_CPT)
		cpt_serr_int_handler(dev);
2063 2064
}

2065 2066 2067
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2068
	enum pipe pipe;
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078

	if (de_iir & DE_AUX_CHANNEL_A)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE)
		intel_opregion_asle_intr(dev);

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2079
	for_each_pipe(dev_priv, pipe) {
2080 2081 2082
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2083

2084
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2085
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2086

2087 2088
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
			i9xx_pipe_crc_irq_handler(dev, pipe);
2089

2090 2091 2092 2093 2094
		/* plane/pipes map 1:1 on ilk+ */
		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

		if (HAS_PCH_CPT(dev))
			cpt_irq_handler(dev, pch_iir);
		else
			ibx_irq_handler(dev, pch_iir);

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev);
}

2114 2115 2116
static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2117
	enum pipe pipe;
2118 2119 2120 2121 2122 2123 2124 2125 2126 2127

	if (de_iir & DE_ERR_INT_IVB)
		ivb_err_int_handler(dev);

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_asle_intr(dev);

2128
	for_each_pipe(dev_priv, pipe) {
2129 2130 2131
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2132 2133

		/* plane/pipes map 1:1 on ilk+ */
2134 2135 2136
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
		}
	}

	/* check event from PCH */
	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2151 2152 2153 2154 2155 2156 2157 2158
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2159
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2160
{
2161
	struct drm_device *dev = arg;
2162
	struct drm_i915_private *dev_priv = dev->dev_private;
2163
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2164
	irqreturn_t ret = IRQ_NONE;
2165

2166 2167
	/* We get interrupts on unclaimed registers, so check for this before we
	 * do any I915_{READ,WRITE}. */
2168
	intel_uncore_check_errors(dev);
2169

2170 2171 2172
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2173
	POSTING_READ(DEIER);
2174

2175 2176 2177 2178 2179
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2180 2181 2182 2183 2184
	if (!HAS_PCH_NOP(dev)) {
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2185

2186 2187
	/* Find, clear, then process each source of interrupt */

2188
	gt_iir = I915_READ(GTIIR);
2189
	if (gt_iir) {
2190 2191
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2192
		if (INTEL_INFO(dev)->gen >= 6)
2193
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2194 2195
		else
			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2196 2197
	}

2198 2199
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2200 2201
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2202 2203 2204 2205
		if (INTEL_INFO(dev)->gen >= 7)
			ivb_display_irq_handler(dev, de_iir);
		else
			ilk_display_irq_handler(dev, de_iir);
2206 2207
	}

2208 2209 2210 2211 2212
	if (INTEL_INFO(dev)->gen >= 6) {
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2213
			gen6_rps_irq_handler(dev_priv, pm_iir);
2214
		}
2215
	}
2216 2217 2218

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2219 2220 2221 2222
	if (!HAS_PCH_NOP(dev)) {
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2223 2224 2225 2226

	return ret;
}

2227 2228 2229 2230 2231 2232 2233
static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
	irqreturn_t ret = IRQ_NONE;
	uint32_t tmp = 0;
2234
	enum pipe pipe;
J
Jesse Barnes 已提交
2235 2236 2237 2238 2239
	u32 aux_mask = GEN8_AUX_CHANNEL_A;

	if (IS_GEN9(dev))
		aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
			GEN9_AUX_CHANNEL_D;
2240 2241 2242 2243 2244 2245 2246 2247 2248

	master_ctl = I915_READ(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

2249 2250
	/* Find, clear, then process each source of interrupt */

2251 2252 2253 2254 2255 2256 2257
	ret = gen8_gt_irq_handler(dev, dev_priv, master_ctl);

	if (master_ctl & GEN8_DE_MISC_IRQ) {
		tmp = I915_READ(GEN8_DE_MISC_IIR);
		if (tmp) {
			I915_WRITE(GEN8_DE_MISC_IIR, tmp);
			ret = IRQ_HANDLED;
2258 2259 2260 2261
			if (tmp & GEN8_DE_MISC_GSE)
				intel_opregion_asle_intr(dev);
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2262
		}
2263 2264
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2265 2266
	}

2267 2268 2269 2270 2271
	if (master_ctl & GEN8_DE_PORT_IRQ) {
		tmp = I915_READ(GEN8_DE_PORT_IIR);
		if (tmp) {
			I915_WRITE(GEN8_DE_PORT_IIR, tmp);
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2272 2273

			if (tmp & aux_mask)
2274 2275 2276
				dp_aux_irq_handler(dev);
			else
				DRM_ERROR("Unexpected DE Port interrupt\n");
2277
		}
2278 2279
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2280 2281
	}

2282
	for_each_pipe(dev_priv, pipe) {
2283
		uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2284

2285 2286
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2287

2288 2289 2290 2291
		pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (pipe_iir) {
			ret = IRQ_HANDLED;
			I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2292

2293 2294 2295
			if (pipe_iir & GEN8_PIPE_VBLANK &&
			    intel_pipe_handle_vblank(dev, pipe))
				intel_check_page_flip(dev, pipe);
2296

2297 2298 2299 2300 2301 2302
			if (IS_GEN9(dev))
				flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
			else
				flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;

			if (flip_done) {
2303 2304 2305 2306 2307 2308 2309
				intel_prepare_page_flip(dev, pipe);
				intel_finish_page_flip_plane(dev, pipe);
			}

			if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
				hsw_pipe_crc_irq_handler(dev, pipe);

2310 2311 2312
			if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
2313

2314 2315 2316 2317 2318 2319 2320

			if (IS_GEN9(dev))
				fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
			else
				fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;

			if (fault_errors)
2321 2322 2323
				DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
					  pipe_name(pipe),
					  pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2324
		} else
2325 2326 2327
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
	}

2328 2329 2330 2331 2332 2333 2334 2335 2336 2337
	if (!HAS_PCH_NOP(dev) && master_ctl & GEN8_DE_PCH_IRQ) {
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
		u32 pch_iir = I915_READ(SDEIIR);
		if (pch_iir) {
			I915_WRITE(SDEIIR, pch_iir);
			ret = IRQ_HANDLED;
2338 2339 2340 2341
			cpt_irq_handler(dev, pch_iir);
		} else
			DRM_ERROR("The master control interrupt lied (SDE)!\n");

2342 2343
	}

2344 2345 2346 2347 2348 2349
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return ret;
}

2350 2351 2352
static void i915_error_wake_up(struct drm_i915_private *dev_priv,
			       bool reset_completed)
{
2353
	struct intel_engine_cs *ring;
2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377
	int i;

	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
	for_each_ring(ring, dev_priv, i)
		wake_up_all(&ring->irq_queue);

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);

	/*
	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
	 * reset state is cleared.
	 */
	if (reset_completed)
		wake_up_all(&dev_priv->gpu_error.reset_queue);
}

2378 2379 2380 2381 2382 2383 2384 2385 2386
/**
 * i915_error_work_func - do process context error handling work
 * @work: work struct
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
static void i915_error_work_func(struct work_struct *work)
{
2387 2388
	struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
						    work);
2389 2390
	struct drm_i915_private *dev_priv =
		container_of(error, struct drm_i915_private, gpu_error);
2391
	struct drm_device *dev = dev_priv->dev;
2392 2393 2394
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2395
	int ret;
2396

2397
	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2398

2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2410
		DRM_DEBUG_DRIVER("resetting chip\n");
2411
		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2412
				   reset_event);
2413

2414 2415 2416 2417 2418 2419 2420 2421
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2422 2423 2424 2425 2426 2427
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2428 2429
		ret = i915_reset(dev);

2430 2431
		intel_display_handle_reset(dev);

2432 2433
		intel_runtime_pm_put(dev_priv);

2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444
		if (ret == 0) {
			/*
			 * After all the gem state is reset, increment the reset
			 * counter and wake up everyone waiting for the reset to
			 * complete.
			 *
			 * Since unlock operations are a one-sided barrier only,
			 * we need to insert a barrier here to order any seqno
			 * updates before
			 * the counter increment.
			 */
2445
			smp_mb__before_atomic();
2446 2447
			atomic_inc(&dev_priv->gpu_error.reset_counter);

2448
			kobject_uevent_env(&dev->primary->kdev->kobj,
2449
					   KOBJ_CHANGE, reset_done_event);
2450
		} else {
M
Mika Kuoppala 已提交
2451
			atomic_set_mask(I915_WEDGED, &error->reset_counter);
2452
		}
2453

2454 2455 2456 2457 2458
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
		i915_error_wake_up(dev_priv, true);
2459
	}
2460 2461
}

2462
static void i915_report_and_clear_eir(struct drm_device *dev)
2463 2464
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2465
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2466
	u32 eir = I915_READ(EIR);
2467
	int pipe, i;
2468

2469 2470
	if (!eir)
		return;
2471

2472
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2473

2474 2475
	i915_get_extra_instdone(dev, instdone);

2476 2477 2478 2479
	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2480 2481
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2482 2483
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2484 2485
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2486
			I915_WRITE(IPEIR_I965, ipeir);
2487
			POSTING_READ(IPEIR_I965);
2488 2489 2490
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2491 2492
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2493
			I915_WRITE(PGTBL_ER, pgtbl_err);
2494
			POSTING_READ(PGTBL_ER);
2495 2496 2497
		}
	}

2498
	if (!IS_GEN2(dev)) {
2499 2500
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2501 2502
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2503
			I915_WRITE(PGTBL_ER, pgtbl_err);
2504
			POSTING_READ(PGTBL_ER);
2505 2506 2507 2508
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2509
		pr_err("memory refresh error:\n");
2510
		for_each_pipe(dev_priv, pipe)
2511
			pr_err("pipe %c stat: 0x%08x\n",
2512
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2513 2514 2515
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2516 2517
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2518 2519
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2520
		if (INTEL_INFO(dev)->gen < 4) {
2521 2522
			u32 ipeir = I915_READ(IPEIR);

2523 2524 2525
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2526
			I915_WRITE(IPEIR, ipeir);
2527
			POSTING_READ(IPEIR);
2528 2529 2530
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2531 2532 2533 2534
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2535
			I915_WRITE(IPEIR_I965, ipeir);
2536
			POSTING_READ(IPEIR_I965);
2537 2538 2539 2540
		}
	}

	I915_WRITE(EIR, eir);
2541
	POSTING_READ(EIR);
2542 2543 2544 2545 2546 2547 2548 2549 2550 2551
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563
}

/**
 * i915_handle_error - handle an error interrupt
 * @dev: drm device
 *
 * Do some basic checking of regsiter state at error interrupt time and
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2564 2565
void i915_handle_error(struct drm_device *dev, bool wedged,
		       const char *fmt, ...)
2566 2567
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2568 2569
	va_list args;
	char error_msg[80];
2570

2571 2572 2573 2574 2575
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

	i915_capture_error_state(dev, wedged, error_msg);
2576
	i915_report_and_clear_eir(dev);
2577

2578
	if (wedged) {
2579 2580
		atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
				&dev_priv->gpu_error.reset_counter);
2581

2582
		/*
2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
		 * Wakeup waiting processes so that the reset work function
		 * i915_error_work_func doesn't deadlock trying to grab various
		 * locks. By bumping the reset counter first, the woken
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2594
		 */
2595
		i915_error_wake_up(dev_priv, false);
2596 2597
	}

2598 2599 2600 2601 2602 2603 2604
	/*
	 * Our reset work can grab modeset locks (since it needs to reset the
	 * state of outstanding pagelips). Hence it must not be run on our own
	 * dev-priv->wq work queue for otherwise the flush_work in the pageflip
	 * code will deadlock.
	 */
	schedule_work(&dev_priv->gpu_error.work);
2605 2606
}

2607 2608 2609
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2610
static int i915_enable_vblank(struct drm_device *dev, int pipe)
2611
{
2612
	struct drm_i915_private *dev_priv = dev->dev_private;
2613
	unsigned long irqflags;
2614

2615
	if (!i915_pipe_enabled(dev, pipe))
2616
		return -EINVAL;
2617

2618
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2619
	if (INTEL_INFO(dev)->gen >= 4)
2620
		i915_enable_pipestat(dev_priv, pipe,
2621
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2622
	else
2623
		i915_enable_pipestat(dev_priv, pipe,
2624
				     PIPE_VBLANK_INTERRUPT_STATUS);
2625
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2626

2627 2628 2629
	return 0;
}

2630
static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2631
{
2632
	struct drm_i915_private *dev_priv = dev->dev_private;
2633
	unsigned long irqflags;
2634
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2635
						     DE_PIPE_VBLANK(pipe);
2636 2637 2638 2639 2640

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2641
	ironlake_enable_display_irq(dev_priv, bit);
2642 2643 2644 2645 2646
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

J
Jesse Barnes 已提交
2647 2648
static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
{
2649
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2650 2651 2652 2653 2654 2655
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2656
	i915_enable_pipestat(dev_priv, pipe,
2657
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2658 2659 2660 2661 2662
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2663 2664 2665 2666 2667 2668 2669 2670 2671
static int gen8_enable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2672 2673 2674
	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2675 2676 2677 2678
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
	return 0;
}

2679 2680 2681
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2682
static void i915_disable_vblank(struct drm_device *dev, int pipe)
2683
{
2684
	struct drm_i915_private *dev_priv = dev->dev_private;
2685
	unsigned long irqflags;
2686

2687
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2688
	i915_disable_pipestat(dev_priv, pipe,
2689 2690
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2691 2692 2693
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2694
static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2695
{
2696
	struct drm_i915_private *dev_priv = dev->dev_private;
2697
	unsigned long irqflags;
2698
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2699
						     DE_PIPE_VBLANK(pipe);
2700 2701

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2702
	ironlake_disable_display_irq(dev_priv, bit);
2703 2704 2705
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

J
Jesse Barnes 已提交
2706 2707
static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
{
2708
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2709 2710 2711
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2712
	i915_disable_pipestat(dev_priv, pipe,
2713
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2714 2715 2716
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2717 2718 2719 2720 2721 2722 2723 2724 2725
static void gen8_disable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2726 2727 2728
	dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2729 2730 2731
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2732
static u32
2733
ring_last_seqno(struct intel_engine_cs *ring)
2734
{
2735 2736 2737 2738
	return list_entry(ring->request_list.prev,
			  struct drm_i915_gem_request, list)->seqno;
}

2739
static bool
2740
ring_idle(struct intel_engine_cs *ring, u32 seqno)
2741 2742 2743
{
	return (list_empty(&ring->request_list) ||
		i915_seqno_passed(seqno, ring_last_seqno(ring)));
B
Ben Gamari 已提交
2744 2745
}

2746 2747 2748 2749
static bool
ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
{
	if (INTEL_INFO(dev)->gen >= 8) {
2750
		return (ipehr >> 23) == 0x1c;
2751 2752 2753 2754 2755 2756 2757
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2758
static struct intel_engine_cs *
2759
semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2760 2761
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2762
	struct intel_engine_cs *signaller;
2763 2764 2765
	int i;

	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2766 2767 2768 2769 2770 2771 2772
		for_each_ring(signaller, dev_priv, i) {
			if (ring == signaller)
				continue;

			if (offset == signaller->semaphore.signal_ggtt[ring->id])
				return signaller;
		}
2773 2774 2775 2776 2777 2778 2779
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

		for_each_ring(signaller, dev_priv, i) {
			if(ring == signaller)
				continue;

2780
			if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2781 2782 2783 2784
				return signaller;
		}
	}

2785 2786
	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
		  ring->id, ipehr, offset);
2787 2788 2789 2790

	return NULL;
}

2791 2792
static struct intel_engine_cs *
semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2793 2794
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2795
	u32 cmd, ipehr, head;
2796 2797
	u64 offset = 0;
	int i, backwards;
2798 2799

	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2800
	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2801
		return NULL;
2802

2803 2804 2805
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
2806 2807
	 * or 4 dwords depending on the semaphore wait command size.
	 * Note that we don't care about ACTHD here since that might
2808 2809
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2810
	 */
2811
	head = I915_READ_HEAD(ring) & HEAD_ADDR;
2812
	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2813

2814
	for (i = backwards; i; --i) {
2815 2816 2817 2818 2819
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
2820
		head &= ring->buffer->size - 1;
2821 2822

		/* This here seems to blow up */
2823
		cmd = ioread32(ring->buffer->virtual_start + head);
2824 2825 2826
		if (cmd == ipehr)
			break;

2827 2828
		head -= 4;
	}
2829

2830 2831
	if (!i)
		return NULL;
2832

2833
	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2834 2835 2836 2837 2838 2839
	if (INTEL_INFO(ring->dev)->gen >= 8) {
		offset = ioread32(ring->buffer->virtual_start + head + 12);
		offset <<= 32;
		offset = ioread32(ring->buffer->virtual_start + head + 8);
	}
	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2840 2841
}

2842
static int semaphore_passed(struct intel_engine_cs *ring)
2843 2844
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2845
	struct intel_engine_cs *signaller;
2846
	u32 seqno;
2847

2848
	ring->hangcheck.deadlock++;
2849 2850

	signaller = semaphore_waits_for(ring, &seqno);
2851 2852 2853 2854 2855
	if (signaller == NULL)
		return -1;

	/* Prevent pathological recursion due to driver bugs */
	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2856 2857
		return -1;

2858 2859 2860
	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
		return 1;

2861 2862 2863
	/* cursory check for an unkickable deadlock */
	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
	    semaphore_passed(signaller) < 0)
2864 2865 2866
		return -1;

	return 0;
2867 2868 2869 2870
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
2871
	struct intel_engine_cs *ring;
2872 2873 2874
	int i;

	for_each_ring(ring, dev_priv, i)
2875
		ring->hangcheck.deadlock = 0;
2876 2877
}

2878
static enum intel_ring_hangcheck_action
2879
ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2880 2881 2882
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
2883 2884
	u32 tmp;

2885 2886 2887 2888 2889 2890 2891 2892
	if (acthd != ring->hangcheck.acthd) {
		if (acthd > ring->hangcheck.max_acthd) {
			ring->hangcheck.max_acthd = acthd;
			return HANGCHECK_ACTIVE;
		}

		return HANGCHECK_ACTIVE_LOOP;
	}
2893

2894
	if (IS_GEN2(dev))
2895
		return HANGCHECK_HUNG;
2896 2897 2898 2899 2900 2901 2902

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
	tmp = I915_READ_CTL(ring);
2903
	if (tmp & RING_WAIT) {
2904 2905 2906
		i915_handle_error(dev, false,
				  "Kicking stuck wait on %s",
				  ring->name);
2907
		I915_WRITE_CTL(ring, tmp);
2908
		return HANGCHECK_KICK;
2909 2910 2911 2912 2913
	}

	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		switch (semaphore_passed(ring)) {
		default:
2914
			return HANGCHECK_HUNG;
2915
		case 1:
2916 2917 2918
			i915_handle_error(dev, false,
					  "Kicking stuck semaphore on %s",
					  ring->name);
2919
			I915_WRITE_CTL(ring, tmp);
2920
			return HANGCHECK_KICK;
2921
		case 0:
2922
			return HANGCHECK_WAIT;
2923
		}
2924
	}
2925

2926
	return HANGCHECK_HUNG;
2927 2928
}

B
Ben Gamari 已提交
2929 2930
/**
 * This is called when the chip hasn't reported back with completed
2931 2932 2933 2934 2935
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
2936
 */
2937
static void i915_hangcheck_elapsed(unsigned long data)
B
Ben Gamari 已提交
2938 2939
{
	struct drm_device *dev = (struct drm_device *)data;
2940
	struct drm_i915_private *dev_priv = dev->dev_private;
2941
	struct intel_engine_cs *ring;
2942
	int i;
2943
	int busy_count = 0, rings_hung = 0;
2944 2945 2946 2947
	bool stuck[I915_NUM_RINGS] = { 0 };
#define BUSY 1
#define KICK 5
#define HUNG 20
2948

2949
	if (!i915.enable_hangcheck)
2950 2951
		return;

2952
	for_each_ring(ring, dev_priv, i) {
2953 2954
		u64 acthd;
		u32 seqno;
2955
		bool busy = true;
2956

2957 2958
		semaphore_clear_deadlocks(dev_priv);

2959 2960
		seqno = ring->get_seqno(ring, false);
		acthd = intel_ring_get_active_head(ring);
2961

2962 2963
		if (ring->hangcheck.seqno == seqno) {
			if (ring_idle(ring, seqno)) {
2964 2965
				ring->hangcheck.action = HANGCHECK_IDLE;

2966 2967
				if (waitqueue_active(&ring->irq_queue)) {
					/* Issue a wake-up to catch stuck h/w. */
2968
					if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2969 2970 2971 2972 2973 2974
						if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
							DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
								  ring->name);
						else
							DRM_INFO("Fake missed irq on %s\n",
								 ring->name);
2975 2976 2977 2978
						wake_up_all(&ring->irq_queue);
					}
					/* Safeguard against driver failure */
					ring->hangcheck.score += BUSY;
2979 2980
				} else
					busy = false;
2981
			} else {
2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
2997 2998 2999 3000
				ring->hangcheck.action = ring_stuck(ring,
								    acthd);

				switch (ring->hangcheck.action) {
3001
				case HANGCHECK_IDLE:
3002 3003
				case HANGCHECK_WAIT:
				case HANGCHECK_ACTIVE:
3004 3005
					break;
				case HANGCHECK_ACTIVE_LOOP:
3006
					ring->hangcheck.score += BUSY;
3007
					break;
3008
				case HANGCHECK_KICK:
3009
					ring->hangcheck.score += KICK;
3010
					break;
3011
				case HANGCHECK_HUNG:
3012
					ring->hangcheck.score += HUNG;
3013 3014 3015
					stuck[i] = true;
					break;
				}
3016
			}
3017
		} else {
3018 3019
			ring->hangcheck.action = HANGCHECK_ACTIVE;

3020 3021 3022 3023 3024
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
			if (ring->hangcheck.score > 0)
				ring->hangcheck.score--;
3025 3026

			ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
3027 3028
		}

3029 3030
		ring->hangcheck.seqno = seqno;
		ring->hangcheck.acthd = acthd;
3031
		busy_count += busy;
3032
	}
3033

3034
	for_each_ring(ring, dev_priv, i) {
3035
		if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3036 3037 3038
			DRM_INFO("%s on %s\n",
				 stuck[i] ? "stuck" : "no progress",
				 ring->name);
3039
			rings_hung++;
3040 3041 3042
		}
	}

3043
	if (rings_hung)
3044
		return i915_handle_error(dev, true, "Ring hung");
B
Ben Gamari 已提交
3045

3046 3047 3048
	if (busy_count)
		/* Reset timer case chip hangs without another request
		 * being added */
3049 3050 3051 3052 3053 3054
		i915_queue_hangcheck(dev);
}

void i915_queue_hangcheck(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
3055 3056
	struct timer_list *timer = &dev_priv->gpu_error.hangcheck_timer;

3057
	if (!i915.enable_hangcheck)
3058 3059
		return;

3060 3061 3062 3063
	/* Don't continually defer the hangcheck, but make sure it is active */
	if (!timer_pending(timer))
		timer->expires = round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES);
	mod_timer(timer, timer->expires);
B
Ben Gamari 已提交
3064 3065
}

3066
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3067 3068 3069 3070 3071 3072
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3073
	GEN5_IRQ_RESET(SDE);
3074 3075 3076

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3077
}
3078

P
Paulo Zanoni 已提交
3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3095 3096 3097 3098
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3099
static void gen5_gt_irq_reset(struct drm_device *dev)
3100 3101 3102
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3103
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3104
	if (INTEL_INFO(dev)->gen >= 6)
3105
		GEN5_IRQ_RESET(GEN6_PM);
3106 3107
}

L
Linus Torvalds 已提交
3108 3109
/* drm_dma.h hooks
*/
P
Paulo Zanoni 已提交
3110
static void ironlake_irq_reset(struct drm_device *dev)
3111
{
3112
	struct drm_i915_private *dev_priv = dev->dev_private;
3113

3114
	I915_WRITE(HWSTAM, 0xffffffff);
3115

3116
	GEN5_IRQ_RESET(DE);
3117 3118
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3119

3120
	gen5_gt_irq_reset(dev);
3121

3122
	ibx_irq_reset(dev);
3123
}
3124

3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	GEN5_IRQ_RESET(VLV_);
}

J
Jesse Barnes 已提交
3138 3139
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3140
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3141 3142 3143 3144 3145 3146 3147

	/* VLV magic */
	I915_WRITE(VLV_IMR, 0);
	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);

3148
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3149

3150
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
J
Jesse Barnes 已提交
3151

3152
	vlv_display_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3153 3154
}

3155 3156 3157 3158 3159 3160 3161 3162
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3163
static void gen8_irq_reset(struct drm_device *dev)
3164 3165 3166 3167 3168 3169 3170
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3171
	gen8_gt_irq_reset(dev_priv);
3172

3173
	for_each_pipe(dev_priv, pipe)
3174 3175
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3176
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3177

3178 3179 3180
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3181

3182
	ibx_irq_reset(dev);
3183
}
3184

3185 3186
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv)
{
3187
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3188

3189
	spin_lock_irq(&dev_priv->irq_lock);
3190
	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B, dev_priv->de_irq_mask[PIPE_B],
3191
			  ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
3192
	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C, dev_priv->de_irq_mask[PIPE_C],
3193
			  ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3194
	spin_unlock_irq(&dev_priv->irq_lock);
3195 3196
}

3197 3198 3199 3200 3201 3202 3203
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3204
	gen8_gt_irq_reset(dev_priv);
3205 3206 3207 3208 3209

	GEN5_IRQ_RESET(GEN8_PCU_);

	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);

3210
	vlv_display_irq_reset(dev_priv);
3211 3212
}

3213
static void ibx_hpd_irq_setup(struct drm_device *dev)
3214
{
3215
	struct drm_i915_private *dev_priv = dev->dev_private;
3216
	struct intel_encoder *intel_encoder;
3217
	u32 hotplug_irqs, hotplug, enabled_irqs = 0;
3218 3219

	if (HAS_PCH_IBX(dev)) {
3220
		hotplug_irqs = SDE_HOTPLUG_MASK;
3221
		for_each_intel_encoder(dev, intel_encoder)
3222
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3223
				enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
3224
	} else {
3225
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3226
		for_each_intel_encoder(dev, intel_encoder)
3227
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3228
				enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
3229
	}
3230

3231
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3232 3233 3234 3235 3236 3237 3238

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
	 *
	 * This register is the same on all known PCH chips.
	 */
3239 3240 3241 3242 3243 3244 3245 3246
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
}

P
Paulo Zanoni 已提交
3247 3248
static void ibx_irq_postinstall(struct drm_device *dev)
{
3249
	struct drm_i915_private *dev_priv = dev->dev_private;
3250
	u32 mask;
3251

D
Daniel Vetter 已提交
3252 3253 3254
	if (HAS_PCH_NOP(dev))
		return;

3255
	if (HAS_PCH_IBX(dev))
3256
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3257
	else
3258
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3259

3260
	GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
P
Paulo Zanoni 已提交
3261 3262 3263
	I915_WRITE(SDEIMR, ~mask);
}

3264 3265 3266 3267 3268 3269 3270 3271
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3272
	if (HAS_L3_DPF(dev)) {
3273
		/* L3 parity interrupt is always unmasked. */
3274 3275
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3276 3277 3278 3279 3280 3281 3282 3283 3284 3285
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
			   ILK_BSD_USER_INTERRUPT;
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3286
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3287 3288

	if (INTEL_INFO(dev)->gen >= 6) {
3289
		pm_irqs |= dev_priv->pm_rps_events;
3290 3291 3292 3293

		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3294
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3295
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3296 3297 3298
	}
}

3299
static int ironlake_irq_postinstall(struct drm_device *dev)
3300
{
3301
	struct drm_i915_private *dev_priv = dev->dev_private;
3302 3303 3304 3305 3306 3307
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3308
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3309
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3310
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
3311 3312 3313
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3314 3315 3316
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3317 3318
		extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
				DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
3319
	}
3320

3321
	dev_priv->irq_mask = ~display_mask;
3322

3323 3324
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3325 3326
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3327
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3328

3329
	gen5_gt_irq_postinstall(dev);
3330

P
Paulo Zanoni 已提交
3331
	ibx_irq_postinstall(dev);
3332

3333
	if (IS_IRONLAKE_M(dev)) {
3334 3335 3336
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3337 3338
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3339
		spin_lock_irq(&dev_priv->irq_lock);
3340
		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3341
		spin_unlock_irq(&dev_priv->irq_lock);
3342 3343
	}

3344 3345 3346
	return 0;
}

3347 3348 3349 3350
static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3351
	enum pipe pipe;
3352 3353 3354 3355

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;

3356 3357
	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3358 3359 3360 3361 3362
	POSTING_READ(PIPESTAT(PIPE_A));

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3363 3364 3365
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		      i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3366 3367 3368 3369

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3370 3371
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3372 3373 3374 3375 3376
	dev_priv->irq_mask &= ~iir_mask;

	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3377 3378
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
3379 3380 3381 3382 3383 3384
}

static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3385
	enum pipe pipe;
3386 3387 3388

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3389
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3390 3391
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3392 3393 3394

	dev_priv->irq_mask |= iir_mask;
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3395
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3396 3397 3398 3399 3400 3401 3402
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	POSTING_READ(VLV_IIR);

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3403 3404 3405
	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3406 3407 3408

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;
3409 3410 3411

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423
	POSTING_READ(PIPESTAT(PIPE_A));
}

void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3424
	if (intel_irqs_enabled(dev_priv))
3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436
		valleyview_display_irqs_install(dev_priv);
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3437
	if (intel_irqs_enabled(dev_priv))
3438 3439 3440
		valleyview_display_irqs_uninstall(dev_priv);
}

3441
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
J
Jesse Barnes 已提交
3442
{
3443
	dev_priv->irq_mask = ~0;
J
Jesse Barnes 已提交
3444

3445 3446 3447
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	POSTING_READ(PORT_HOTPLUG_EN);

J
Jesse Barnes 已提交
3448
	I915_WRITE(VLV_IIR, 0xffffffff);
3449 3450 3451 3452
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
J
Jesse Barnes 已提交
3453

3454 3455
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3456
	spin_lock_irq(&dev_priv->irq_lock);
3457 3458
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_install(dev_priv);
3459
	spin_unlock_irq(&dev_priv->irq_lock);
3460 3461 3462 3463 3464 3465 3466
}

static int valleyview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	vlv_display_irq_postinstall(dev_priv);
J
Jesse Barnes 已提交
3467

3468
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3469 3470 3471 3472 3473 3474 3475 3476

	/* ack & enable invalid PTE error interrupts */
#if 0 /* FIXME: add support to irq handler for checking these bits */
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
#endif

	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3477 3478 3479 3480

	return 0;
}

3481 3482 3483 3484 3485
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3486
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3487
			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3488 3489
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3490
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3491 3492 3493
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3494
		0,
3495 3496
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3497 3498
		};

3499
	dev_priv->pm_irq_mask = 0xffffffff;
3500 3501 3502 3503
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, dev_priv->pm_rps_events);
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3504 3505 3506 3507
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3508 3509
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3510
	int pipe;
J
Jesse Barnes 已提交
3511
	u32 aux_en = GEN8_AUX_CHANNEL_A;
3512

J
Jesse Barnes 已提交
3513
	if (IS_GEN9(dev_priv)) {
3514 3515
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
J
Jesse Barnes 已提交
3516 3517 3518
		aux_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
			GEN9_AUX_CHANNEL_D;
	} else
3519 3520 3521 3522 3523 3524
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3525 3526 3527
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3528

3529
	for_each_pipe(dev_priv, pipe)
3530
		if (intel_display_power_is_enabled(dev_priv,
3531 3532 3533 3534
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3535

J
Jesse Barnes 已提交
3536
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~aux_en, aux_en);
3537 3538 3539 3540 3541 3542
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

P
Paulo Zanoni 已提交
3543 3544
	ibx_irq_pre_postinstall(dev);

3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

	ibx_irq_postinstall(dev);

	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3556 3557 3558 3559
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3560
	vlv_display_irq_postinstall(dev_priv);
3561 3562 3563 3564 3565 3566 3567 3568 3569

	gen8_gt_irq_postinstall(dev_priv);

	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3570 3571 3572 3573 3574 3575 3576
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3577
	gen8_irq_reset(dev);
3578 3579
}

3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593
static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
{
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irq(&dev_priv->irq_lock);
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_uninstall(dev_priv);
	spin_unlock_irq(&dev_priv->irq_lock);

	vlv_display_irq_reset(dev_priv);

	dev_priv->irq_mask = 0;
}

J
Jesse Barnes 已提交
3594 3595
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3596
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3597 3598 3599 3600

	if (!dev_priv)
		return;

3601 3602
	I915_WRITE(VLV_MASTER_IER, 0);

3603 3604
	gen5_gt_irq_reset(dev);

J
Jesse Barnes 已提交
3605
	I915_WRITE(HWSTAM, 0xffffffff);
3606

3607
	vlv_display_irq_uninstall(dev_priv);
J
Jesse Barnes 已提交
3608 3609
}

3610 3611 3612 3613 3614 3615 3616 3617 3618 3619
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3620
	gen8_gt_irq_reset(dev_priv);
3621

3622
	GEN5_IRQ_RESET(GEN8_PCU_);
3623

3624
	vlv_display_irq_uninstall(dev_priv);
3625 3626
}

3627
static void ironlake_irq_uninstall(struct drm_device *dev)
3628
{
3629
	struct drm_i915_private *dev_priv = dev->dev_private;
3630 3631 3632 3633

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3634
	ironlake_irq_reset(dev);
3635 3636
}

3637
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3638
{
3639
	struct drm_i915_private *dev_priv = dev->dev_private;
3640
	int pipe;
3641

3642
	for_each_pipe(dev_priv, pipe)
3643
		I915_WRITE(PIPESTAT(pipe), 0);
3644 3645 3646
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3647 3648 3649 3650
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3651
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3672 3673
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3674
	spin_lock_irq(&dev_priv->irq_lock);
3675 3676
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3677
	spin_unlock_irq(&dev_priv->irq_lock);
3678

C
Chris Wilson 已提交
3679 3680 3681
	return 0;
}

3682 3683 3684 3685
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_device *dev,
3686
			       int plane, int pipe, u32 iir)
3687
{
3688
	struct drm_i915_private *dev_priv = dev->dev_private;
3689
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3690

3691
	if (!intel_pipe_handle_vblank(dev, pipe))
3692 3693 3694
		return false;

	if ((iir & flip_pending) == 0)
3695
		goto check_page_flip;
3696

3697
	intel_prepare_page_flip(dev, plane);
3698 3699 3700 3701 3702 3703 3704 3705

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
3706
		goto check_page_flip;
3707 3708 3709

	intel_finish_page_flip(dev, pipe);
	return true;
3710 3711 3712 3713

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
3714 3715
}

3716
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3717
{
3718
	struct drm_device *dev = arg;
3719
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;

	iir = I915_READ16(IIR);
	if (iir == 0)
		return IRQ_NONE;

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3737
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3738
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3739 3740 3741
			i915_handle_error(dev, false,
					  "Command parser error, iir 0x%08x",
					  iir);
C
Chris Wilson 已提交
3742

3743
		for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3744 3745 3746 3747 3748 3749
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3750
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3751 3752
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3753
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3754 3755 3756 3757

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

3758
		i915_update_dri1_breadcrumb(dev);
C
Chris Wilson 已提交
3759 3760 3761 3762

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);

3763
		for_each_pipe(dev_priv, pipe) {
3764
			int plane = pipe;
3765
			if (HAS_FBC(dev))
3766 3767
				plane = !plane;

3768
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3769 3770
			    i8xx_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3771

3772
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3773
				i9xx_pipe_crc_irq_handler(dev, pipe);
3774

3775 3776 3777
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3778
		}
C
Chris Wilson 已提交
3779 3780 3781 3782 3783 3784 3785 3786 3787

		iir = new_iir;
	}

	return IRQ_HANDLED;
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3788
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3789 3790
	int pipe;

3791
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3792 3793 3794 3795 3796 3797 3798 3799 3800
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3801 3802
static void i915_irq_preinstall(struct drm_device * dev)
{
3803
	struct drm_i915_private *dev_priv = dev->dev_private;
3804 3805 3806 3807 3808 3809 3810
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3811
	I915_WRITE16(HWSTAM, 0xeffe);
3812
	for_each_pipe(dev_priv, pipe)
3813 3814 3815 3816 3817 3818 3819 3820
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3821
	struct drm_i915_private *dev_priv = dev->dev_private;
3822
	u32 enable_mask;
3823

3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
		I915_USER_INTERRUPT;

3842
	if (I915_HAS_HOTPLUG(dev)) {
3843 3844 3845
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		POSTING_READ(PORT_HOTPLUG_EN);

3846 3847 3848 3849 3850 3851 3852 3853 3854 3855
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3856
	i915_enable_asle_pipestat(dev);
3857

3858 3859
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3860
	spin_lock_irq(&dev_priv->irq_lock);
3861 3862
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3863
	spin_unlock_irq(&dev_priv->irq_lock);
3864

3865 3866 3867
	return 0;
}

3868 3869 3870 3871 3872 3873
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_device *dev,
			       int plane, int pipe, u32 iir)
{
3874
	struct drm_i915_private *dev_priv = dev->dev_private;
3875 3876
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

3877
	if (!intel_pipe_handle_vblank(dev, pipe))
3878 3879 3880
		return false;

	if ((iir & flip_pending) == 0)
3881
		goto check_page_flip;
3882 3883 3884 3885 3886 3887 3888 3889 3890 3891

	intel_prepare_page_flip(dev, plane);

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
3892
		goto check_page_flip;
3893 3894 3895

	intel_finish_page_flip(dev, pipe);
	return true;
3896 3897 3898 3899

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
3900 3901
}

3902
static irqreturn_t i915_irq_handler(int irq, void *arg)
3903
{
3904
	struct drm_device *dev = arg;
3905
	struct drm_i915_private *dev_priv = dev->dev_private;
3906
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3907 3908 3909 3910
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
3911 3912

	iir = I915_READ(IIR);
3913 3914
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
3915
		bool blc_event = false;
3916 3917 3918 3919 3920 3921

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3922
		spin_lock(&dev_priv->irq_lock);
3923
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3924 3925 3926
			i915_handle_error(dev, false,
					  "Command parser error, iir 0x%08x",
					  iir);
3927

3928
		for_each_pipe(dev_priv, pipe) {
3929 3930 3931
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

3932
			/* Clear the PIPE*STAT regs before the IIR */
3933 3934
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
3935
				irq_received = true;
3936 3937
			}
		}
3938
		spin_unlock(&dev_priv->irq_lock);
3939 3940 3941 3942 3943

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
3944 3945 3946
		if (I915_HAS_HOTPLUG(dev) &&
		    iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
3947

3948
		I915_WRITE(IIR, iir & ~flip_mask);
3949 3950 3951 3952 3953
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);

3954
		for_each_pipe(dev_priv, pipe) {
3955
			int plane = pipe;
3956
			if (HAS_FBC(dev))
3957
				plane = !plane;
3958

3959
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3960 3961
			    i915_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3962 3963 3964

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
3965 3966

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3967
				i9xx_pipe_crc_irq_handler(dev, pipe);
3968

3969 3970 3971
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
3992
		ret = IRQ_HANDLED;
3993
		iir = new_iir;
3994
	} while (iir & ~flip_mask);
3995

3996
	i915_update_dri1_breadcrumb(dev);
3997

3998 3999 4000 4001 4002
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
4003
	struct drm_i915_private *dev_priv = dev->dev_private;
4004 4005 4006 4007 4008 4009 4010
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4011
	I915_WRITE16(HWSTAM, 0xffff);
4012
	for_each_pipe(dev_priv, pipe) {
4013
		/* Clear enable bits; then clear status bits */
4014
		I915_WRITE(PIPESTAT(pipe), 0);
4015 4016
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4017 4018 4019 4020 4021 4022 4023 4024
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4025
	struct drm_i915_private *dev_priv = dev->dev_private;
4026 4027
	int pipe;

4028 4029
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4030 4031

	I915_WRITE(HWSTAM, 0xeffe);
4032
	for_each_pipe(dev_priv, pipe)
4033 4034 4035 4036 4037 4038 4039 4040
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4041
	struct drm_i915_private *dev_priv = dev->dev_private;
4042
	u32 enable_mask;
4043 4044 4045
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4046
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4047
			       I915_DISPLAY_PORT_INTERRUPT |
4048 4049 4050 4051 4052 4053 4054
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4055 4056
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4057 4058 4059 4060
	enable_mask |= I915_USER_INTERRUPT;

	if (IS_G4X(dev))
		enable_mask |= I915_BSD_USER_INTERRUPT;
4061

4062 4063
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4064
	spin_lock_irq(&dev_priv->irq_lock);
4065 4066 4067
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4068
	spin_unlock_irq(&dev_priv->irq_lock);
4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4089 4090 4091
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	POSTING_READ(PORT_HOTPLUG_EN);

4092
	i915_enable_asle_pipestat(dev);
4093 4094 4095 4096

	return 0;
}

4097
static void i915_hpd_irq_setup(struct drm_device *dev)
4098
{
4099
	struct drm_i915_private *dev_priv = dev->dev_private;
4100
	struct intel_encoder *intel_encoder;
4101 4102
	u32 hotplug_en;

4103 4104
	assert_spin_locked(&dev_priv->irq_lock);

4105 4106 4107 4108
	if (I915_HAS_HOTPLUG(dev)) {
		hotplug_en = I915_READ(PORT_HOTPLUG_EN);
		hotplug_en &= ~HOTPLUG_INT_EN_MASK;
		/* Note HDMI and DP share hotplug bits */
4109
		/* enable bits are the same for all generations */
4110
		for_each_intel_encoder(dev, intel_encoder)
4111 4112
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
				hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
4113 4114 4115 4116 4117 4118
		/* Programming the CRT detection parameters tends
		   to generate a spurious hotplug event about three
		   seconds later.  So just do it once.
		*/
		if (IS_G4X(dev))
			hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4119
		hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
4120
		hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4121

4122 4123 4124
		/* Ignore TV since it's buggy */
		I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
	}
4125 4126
}

4127
static irqreturn_t i965_irq_handler(int irq, void *arg)
4128
{
4129
	struct drm_device *dev = arg;
4130
	struct drm_i915_private *dev_priv = dev->dev_private;
4131 4132 4133
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4134 4135 4136
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4137 4138 4139 4140

	iir = I915_READ(IIR);

	for (;;) {
4141
		bool irq_received = (iir & ~flip_mask) != 0;
4142 4143
		bool blc_event = false;

4144 4145 4146 4147 4148
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4149
		spin_lock(&dev_priv->irq_lock);
4150
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4151 4152 4153
			i915_handle_error(dev, false,
					  "Command parser error, iir 0x%08x",
					  iir);
4154

4155
		for_each_pipe(dev_priv, pipe) {
4156 4157 4158 4159 4160 4161 4162 4163
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4164
				irq_received = true;
4165 4166
			}
		}
4167
		spin_unlock(&dev_priv->irq_lock);
4168 4169 4170 4171 4172 4173 4174

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4175 4176
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4177

4178
		I915_WRITE(IIR, iir & ~flip_mask);
4179 4180 4181 4182 4183 4184 4185
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);
		if (iir & I915_BSD_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[VCS]);

4186
		for_each_pipe(dev_priv, pipe) {
4187
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4188 4189
			    i915_handle_vblank(dev, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4190 4191 4192

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4193 4194

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4195
				i9xx_pipe_crc_irq_handler(dev, pipe);
4196

4197 4198
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4199
		}
4200 4201 4202 4203

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

4204 4205 4206
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
			gmbus_irq_handler(dev);

4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4225
	i915_update_dri1_breadcrumb(dev);
4226

4227 4228 4229 4230 4231
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4232
	struct drm_i915_private *dev_priv = dev->dev_private;
4233 4234 4235 4236 4237
	int pipe;

	if (!dev_priv)
		return;

4238 4239
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4240 4241

	I915_WRITE(HWSTAM, 0xffffffff);
4242
	for_each_pipe(dev_priv, pipe)
4243 4244 4245 4246
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4247
	for_each_pipe(dev_priv, pipe)
4248 4249 4250 4251 4252
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4253
static void intel_hpd_irq_reenable_work(struct work_struct *work)
4254
{
4255 4256 4257
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     hotplug_reenable_work.work);
4258 4259 4260 4261
	struct drm_device *dev = dev_priv->dev;
	struct drm_mode_config *mode_config = &dev->mode_config;
	int i;

4262 4263
	intel_runtime_pm_get(dev_priv);

4264
	spin_lock_irq(&dev_priv->irq_lock);
4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278
	for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
		struct drm_connector *connector;

		if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
			continue;

		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;

		list_for_each_entry(connector, &mode_config->connector_list, head) {
			struct intel_connector *intel_connector = to_intel_connector(connector);

			if (intel_connector->encoder->hpd_pin == i) {
				if (connector->polled != intel_connector->polled)
					DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
4279
							 connector->name);
4280 4281 4282 4283 4284 4285 4286 4287
				connector->polled = intel_connector->polled;
				if (!connector->polled)
					connector->polled = DRM_CONNECTOR_POLL_HPD;
			}
		}
	}
	if (dev_priv->display.hpd_irq_setup)
		dev_priv->display.hpd_irq_setup(dev);
4288
	spin_unlock_irq(&dev_priv->irq_lock);
4289 4290

	intel_runtime_pm_put(dev_priv);
4291 4292
}

4293 4294 4295 4296 4297 4298 4299
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4300
void intel_irq_init(struct drm_i915_private *dev_priv)
4301
{
4302
	struct drm_device *dev = dev_priv->dev;
4303 4304

	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
4305
	INIT_WORK(&dev_priv->dig_port_work, i915_digport_work_func);
4306
	INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
4307
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4308
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4309

4310
	/* Let's track the enabled rps events */
4311
	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4312
		/* WaGsvRC0ResidencyMethod:vlv */
4313 4314 4315
		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4316

4317 4318
	setup_timer(&dev_priv->gpu_error.hangcheck_timer,
		    i915_hangcheck_elapsed,
4319
		    (unsigned long) dev);
4320
	INIT_DELAYED_WORK(&dev_priv->hotplug_reenable_work,
4321
			  intel_hpd_irq_reenable_work);
4322

4323
	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4324

4325
	if (IS_GEN2(dev_priv)) {
4326 4327
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4328
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4329 4330
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4331 4332 4333
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4334 4335
	}

4336 4337 4338 4339 4340
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4341
	if (!IS_GEN2(dev_priv))
4342 4343
		dev->vblank_disable_immediate = true;

4344
	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
4345
		dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4346 4347
		dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
	}
4348

4349
	if (IS_CHERRYVIEW(dev_priv)) {
4350 4351 4352 4353 4354 4355 4356
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4357
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4358 4359 4360 4361 4362 4363
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4364
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4365
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4366
		dev->driver->irq_handler = gen8_irq_handler;
4367
		dev->driver->irq_preinstall = gen8_irq_reset;
4368 4369 4370 4371 4372
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4373 4374
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
4375
		dev->driver->irq_preinstall = ironlake_irq_reset;
4376 4377 4378 4379
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4380
		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4381
	} else {
4382
		if (INTEL_INFO(dev_priv)->gen == 2) {
C
Chris Wilson 已提交
4383 4384 4385 4386
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4387
		} else if (INTEL_INFO(dev_priv)->gen == 3) {
4388 4389 4390 4391
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
4392
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
C
Chris Wilson 已提交
4393
		} else {
4394 4395 4396 4397
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
4398
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
C
Chris Wilson 已提交
4399
		}
4400 4401 4402 4403
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4404

4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416
/**
 * intel_hpd_init - initializes and enables hpd support
 * @dev_priv: i915 device instance
 *
 * This function enables the hotplug support. It requires that interrupts have
 * already been enabled with intel_irq_init_hw(). From this point on hotplug and
 * poll request can run concurrently to other code, so locking rules must be
 * obeyed.
 *
 * This is a separate step from interrupt enabling to simplify the locking rules
 * in the driver load and resume code.
 */
4417
void intel_hpd_init(struct drm_i915_private *dev_priv)
4418
{
4419
	struct drm_device *dev = dev_priv->dev;
4420 4421 4422
	struct drm_mode_config *mode_config = &dev->mode_config;
	struct drm_connector *connector;
	int i;
4423

4424 4425 4426 4427 4428 4429 4430
	for (i = 1; i < HPD_NUM_PINS; i++) {
		dev_priv->hpd_stats[i].hpd_cnt = 0;
		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
	}
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		struct intel_connector *intel_connector = to_intel_connector(connector);
		connector->polled = intel_connector->polled;
4431 4432 4433
		if (connector->encoder && !connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
			connector->polled = DRM_CONNECTOR_POLL_HPD;
		if (intel_connector->mst_port)
4434 4435
			connector->polled = DRM_CONNECTOR_POLL_HPD;
	}
4436 4437 4438

	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked checks happy. */
4439
	spin_lock_irq(&dev_priv->irq_lock);
4440 4441
	if (dev_priv->display.hpd_irq_setup)
		dev_priv->display.hpd_irq_setup(dev);
4442
	spin_unlock_irq(&dev_priv->irq_lock);
4443
}
4444

4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
}

4468 4469 4470 4471 4472 4473 4474
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4475 4476 4477 4478 4479 4480 4481
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
	drm_irq_uninstall(dev_priv->dev);
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4482 4483 4484 4485 4486 4487 4488
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4489
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4490
{
4491
	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4492
	dev_priv->pm.irqs_enabled = false;
4493 4494
}

4495 4496 4497 4498 4499 4500 4501
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4502
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4503
{
4504
	dev_priv->pm.irqs_enabled = true;
4505 4506
	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4507
}