i915_irq.c 126.0 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48
static const u32 hpd_ibx[HPD_NUM_PINS] = {
49 50 51 52 53 54 55
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

56
static const u32 hpd_cpt[HPD_NUM_PINS] = {
57
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
58
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
59 60 61 62 63
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

64
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
65 66 67 68 69 70 71 72
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

73
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
74 75 76 77 78 79 80 81
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

82
static const u32 hpd_status_i915[HPD_NUM_PINS] = { /* i915 and valleyview are the same */
83 84 85 86 87 88 89 90
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

91
/* IIR can theoretically queue up two events. Be paranoid. */
92
#define GEN8_IRQ_RESET_NDX(type, which) do { \
93 94 95 96 97 98 99 100 101
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

102
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
103
	I915_WRITE(type##IMR, 0xffffffff); \
104
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
105
	I915_WRITE(type##IER, 0); \
106 107 108 109
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
110 111
} while (0)

112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
	u32 val = I915_READ(reg); \
	if (val) { \
		WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
		     (reg), val); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
	} \
} while (0)

P
Paulo Zanoni 已提交
127
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
128
	GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
129
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
130 131
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
132 133 134
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
135
	GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
P
Paulo Zanoni 已提交
136
	I915_WRITE(type##IER, (ier_val)); \
137 138
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
139 140
} while (0)

141 142
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);

143
/* For display hotplug interrupt */
144
void
145
ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
146
{
147 148
	assert_spin_locked(&dev_priv->irq_lock);

149
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
150 151
		return;

152 153 154
	if ((dev_priv->irq_mask & mask) != 0) {
		dev_priv->irq_mask &= ~mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
155
		POSTING_READ(DEIMR);
156 157 158
	}
}

159
void
160
ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
161
{
162 163
	assert_spin_locked(&dev_priv->irq_lock);

164
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
165 166
		return;

167 168 169
	if ((dev_priv->irq_mask & mask) != mask) {
		dev_priv->irq_mask |= mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
170
		POSTING_READ(DEIMR);
171 172 173
	}
}

P
Paulo Zanoni 已提交
174 175 176 177 178 179 180 181 182 183 184 185
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

186 187
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

188
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
189 190
		return;

P
Paulo Zanoni 已提交
191 192 193 194 195 196
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

197
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
198 199 200 201
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

202
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
203 204 205 206
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

207 208 209 210 211
static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

212 213 214 215 216
static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

217 218 219 220 221
static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
222 223 224 225 226 227 228 229 230 231
/**
  * snb_update_pm_irq - update GEN6_PMIMR
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
232
	uint32_t new_val;
P
Paulo Zanoni 已提交
233

234 235
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

P
Paulo Zanoni 已提交
236 237
	assert_spin_locked(&dev_priv->irq_lock);

238
	new_val = dev_priv->pm_irq_mask;
239 240 241
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

242 243
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
244 245
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		POSTING_READ(gen6_pm_imr(dev_priv));
246
	}
P
Paulo Zanoni 已提交
247 248
}

249
void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
250
{
251 252 253
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
254 255 256
	snb_update_pm_irq(dev_priv, mask, mask);
}

257 258
static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
				  uint32_t mask)
P
Paulo Zanoni 已提交
259 260 261 262
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

263 264 265 266 267 268 269 270
void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	__gen6_disable_pm_irq(dev_priv, mask);
}

I
Imre Deak 已提交
271 272 273 274 275 276 277 278 279 280 281 282
void gen6_reset_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	uint32_t reg = gen6_pm_iir(dev_priv);

	spin_lock_irq(&dev_priv->irq_lock);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	POSTING_READ(reg);
	spin_unlock_irq(&dev_priv->irq_lock);
}

283 284 285 286 287
void gen6_enable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	spin_lock_irq(&dev_priv->irq_lock);
288

289
	WARN_ON(dev_priv->rps.pm_iir);
I
Imre Deak 已提交
290
	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
291
	dev_priv->rps.interrupts_enabled = true;
292 293
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
				dev_priv->pm_rps_events);
294
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
295

296 297 298
	spin_unlock_irq(&dev_priv->irq_lock);
}

299 300 301
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
	/*
302
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
303
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
304 305
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
306 307 308 309 310 311 312 313 314 315
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
		mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;

	return mask;
}

316 317 318 319
void gen6_disable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

I
Imre Deak 已提交
320 321 322 323 324 325
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
	spin_unlock_irq(&dev_priv->irq_lock);

	cancel_work_sync(&dev_priv->rps.work);

326 327
	spin_lock_irq(&dev_priv->irq_lock);

328
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
329 330

	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
331 332
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
				~dev_priv->pm_rps_events);
333 334
	I915_WRITE(gen6_pm_iir(dev_priv), dev_priv->pm_rps_events);
	I915_WRITE(gen6_pm_iir(dev_priv), dev_priv->pm_rps_events);
335 336 337

	dev_priv->rps.pm_iir = 0;

338
	spin_unlock_irq(&dev_priv->irq_lock);
339 340
}

341 342 343 344 345 346
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
347 348 349
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
350 351 352 353 354
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

355 356
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

357 358
	assert_spin_locked(&dev_priv->irq_lock);

359
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
360 361
		return;

362 363 364
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
365

D
Daniel Vetter 已提交
366
static void
367 368
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
369
{
370
	u32 reg = PIPESTAT(pipe);
371
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
372

373
	assert_spin_locked(&dev_priv->irq_lock);
374
	WARN_ON(!intel_irqs_enabled(dev_priv));
375

376 377 378 379
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
380 381 382
		return;

	if ((pipestat & enable_mask) == enable_mask)
383 384
		return;

385 386
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

387
	/* Enable the interrupt, clear any pending status */
388
	pipestat |= enable_mask | status_mask;
389 390
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
391 392
}

D
Daniel Vetter 已提交
393
static void
394 395
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
396
{
397
	u32 reg = PIPESTAT(pipe);
398
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
399

400
	assert_spin_locked(&dev_priv->irq_lock);
401
	WARN_ON(!intel_irqs_enabled(dev_priv));
402

403 404 405 406
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
407 408
		return;

409 410 411
	if ((pipestat & enable_mask) == 0)
		return;

412 413
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

414
	pipestat &= ~enable_mask;
415 416
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
417 418
}

419 420 421 422 423
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
424 425
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
426 427 428
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
429 430 431 432 433 434
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
435 436 437 438 439 440 441 442 443 444 445 446

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

447 448 449 450 451 452
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

453 454 455 456 457
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
458 459 460 461 462 463 464 465 466
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

467 468 469 470 471
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
472 473 474
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

475
/**
476
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
477
 */
478
static void i915_enable_asle_pipestat(struct drm_device *dev)
479
{
480
	struct drm_i915_private *dev_priv = dev->dev_private;
481

482 483 484
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
		return;

485
	spin_lock_irq(&dev_priv->irq_lock);
486

487
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
488
	if (INTEL_INFO(dev)->gen >= 4)
489
		i915_enable_pipestat(dev_priv, PIPE_A,
490
				     PIPE_LEGACY_BLC_EVENT_STATUS);
491

492
	spin_unlock_irq(&dev_priv->irq_lock);
493 494
}

495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

545 546 547 548 549 550
static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

551 552 553
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
554
static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
555
{
556
	struct drm_i915_private *dev_priv = dev->dev_private;
557 558
	unsigned long high_frame;
	unsigned long low_frame;
559
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
560 561 562 563
	struct intel_crtc *intel_crtc =
		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
	const struct drm_display_mode *mode =
		&intel_crtc->config->base.adjusted_mode;
564

565 566 567 568 569
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
570

571 572 573 574 575 576
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

577 578
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
579

580 581 582 583 584 585
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
586
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
587
		low   = I915_READ(low_frame);
588
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
589 590
	} while (high1 != high2);

591
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
592
	pixel = low & PIPE_PIXEL_MASK;
593
	low >>= PIPE_FRAME_LOW_SHIFT;
594 595 596 597 598 599

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
600
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
601 602
}

603
static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
604
{
605
	struct drm_i915_private *dev_priv = dev->dev_private;
606
	int reg = PIPE_FRMCOUNT_GM45(pipe);
607 608 609 610

	return I915_READ(reg);
}

611 612 613
/* raw reads, only for fast reads of display block, no need for forcewake etc. */
#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))

614 615 616 617
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
618
	const struct drm_display_mode *mode = &crtc->config->base.adjusted_mode;
619
	enum pipe pipe = crtc->pipe;
620
	int position, vtotal;
621

622
	vtotal = mode->crtc_vtotal;
623 624 625 626 627 628 629 630 631
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	if (IS_GEN2(dev))
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
	else
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;

	/*
632 633
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
634
	 */
635
	return (position + crtc->scanline_offset) % vtotal;
636 637
}

638
static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
639 640
				    unsigned int flags, int *vpos, int *hpos,
				    ktime_t *stime, ktime_t *etime)
641
{
642 643 644
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
645
	const struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode;
646
	int position;
647
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
648 649
	bool in_vbl = true;
	int ret = 0;
650
	unsigned long irqflags;
651

652
	if (!intel_crtc->active) {
653
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
654
				 "pipe %c\n", pipe_name(pipe));
655 656 657
		return 0;
	}

658
	htotal = mode->crtc_htotal;
659
	hsync_start = mode->crtc_hsync_start;
660 661 662
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
663

664 665 666 667 668 669
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

670 671
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

672 673 674 675 676 677
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
678

679 680 681 682 683 684
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

685
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
686 687 688
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
689
		position = __intel_get_crtc_scanline(intel_crtc);
690 691 692 693 694
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
695
		position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
696

697 698 699 700
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
701

702 703 704 705 706 707 708 709 710 711 712 713
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

714 715 716 717 718 719 720 721 722 723
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
724 725
	}

726 727 728 729 730 731 732 733
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

734 735 736 737 738 739 740 741 742 743 744 745
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
746

747
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
748 749 750 751 752 753
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
754 755 756

	/* In vblank? */
	if (in_vbl)
757
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
758 759 760 761

	return ret;
}

762 763 764 765 766 767 768 769 770 771 772 773 774
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

775
static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
776 777 778 779
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
780
	struct drm_crtc *crtc;
781

782
	if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
783
		DRM_ERROR("Invalid crtc %d\n", pipe);
784 785 786 787
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
788 789 790 791 792 793
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
		DRM_ERROR("Invalid crtc %d\n", pipe);
		return -EINVAL;
	}

794
	if (!crtc->state->enable) {
795 796 797
		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
		return -EBUSY;
	}
798 799

	/* Helper routine in DRM core does all the work: */
800 801
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
802
						     crtc,
803
						     &to_intel_crtc(crtc)->config->base.adjusted_mode);
804 805
}

806 807
static bool intel_hpd_irq_event(struct drm_device *dev,
				struct drm_connector *connector)
808 809 810 811 812 813 814
{
	enum drm_connector_status old_status;

	WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
	old_status = connector->status;

	connector->status = connector->funcs->detect(connector, false);
815 816 817 818
	if (old_status == connector->status)
		return false;

	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
819
		      connector->base.id,
820
		      connector->name,
821 822 823 824
		      drm_get_connector_status_name(old_status),
		      drm_get_connector_status_name(connector->status));

	return true;
825 826
}

827 828 829 830 831 832
static void i915_digport_work_func(struct work_struct *work)
{
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, dig_port_work);
	u32 long_port_mask, short_port_mask;
	struct intel_digital_port *intel_dig_port;
833
	int i;
834 835
	u32 old_bits = 0;

836
	spin_lock_irq(&dev_priv->irq_lock);
837 838 839 840
	long_port_mask = dev_priv->long_hpd_port_mask;
	dev_priv->long_hpd_port_mask = 0;
	short_port_mask = dev_priv->short_hpd_port_mask;
	dev_priv->short_hpd_port_mask = 0;
841
	spin_unlock_irq(&dev_priv->irq_lock);
842 843 844 845 846 847 848 849 850 851 852 853 854 855 856

	for (i = 0; i < I915_MAX_PORTS; i++) {
		bool valid = false;
		bool long_hpd = false;
		intel_dig_port = dev_priv->hpd_irq_port[i];
		if (!intel_dig_port || !intel_dig_port->hpd_pulse)
			continue;

		if (long_port_mask & (1 << i))  {
			valid = true;
			long_hpd = true;
		} else if (short_port_mask & (1 << i))
			valid = true;

		if (valid) {
857 858
			enum irqreturn ret;

859
			ret = intel_dig_port->hpd_pulse(intel_dig_port, long_hpd);
860 861
			if (ret == IRQ_NONE) {
				/* fall back to old school hpd */
862 863 864 865 866 867
				old_bits |= (1 << intel_dig_port->base.hpd_pin);
			}
		}
	}

	if (old_bits) {
868
		spin_lock_irq(&dev_priv->irq_lock);
869
		dev_priv->hpd_event_bits |= old_bits;
870
		spin_unlock_irq(&dev_priv->irq_lock);
871 872 873 874
		schedule_work(&dev_priv->hotplug_work);
	}
}

875 876 877
/*
 * Handle hotplug events outside the interrupt handler proper.
 */
878 879
#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)

880 881
static void i915_hotplug_work_func(struct work_struct *work)
{
882 883
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, hotplug_work);
884
	struct drm_device *dev = dev_priv->dev;
885
	struct drm_mode_config *mode_config = &dev->mode_config;
886 887 888 889
	struct intel_connector *intel_connector;
	struct intel_encoder *intel_encoder;
	struct drm_connector *connector;
	bool hpd_disabled = false;
890
	bool changed = false;
891
	u32 hpd_event_bits;
892

893
	mutex_lock(&mode_config->mutex);
894 895
	DRM_DEBUG_KMS("running encoder hotplug functions\n");

896
	spin_lock_irq(&dev_priv->irq_lock);
897 898 899

	hpd_event_bits = dev_priv->hpd_event_bits;
	dev_priv->hpd_event_bits = 0;
900 901
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		intel_connector = to_intel_connector(connector);
902 903
		if (!intel_connector->encoder)
			continue;
904 905 906 907 908 909
		intel_encoder = intel_connector->encoder;
		if (intel_encoder->hpd_pin > HPD_NONE &&
		    dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
		    connector->polled == DRM_CONNECTOR_POLL_HPD) {
			DRM_INFO("HPD interrupt storm detected on connector %s: "
				 "switching from hotplug detection to polling\n",
910
				connector->name);
911 912 913 914 915
			dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
			connector->polled = DRM_CONNECTOR_POLL_CONNECT
				| DRM_CONNECTOR_POLL_DISCONNECT;
			hpd_disabled = true;
		}
916 917
		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
			DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
918
				      connector->name, intel_encoder->hpd_pin);
919
		}
920 921 922 923
	}
	 /* if there were no outputs to poll, poll was disabled,
	  * therefore make sure it's enabled when disabling HPD on
	  * some connectors */
924
	if (hpd_disabled) {
925
		drm_kms_helper_poll_enable(dev);
926 927
		mod_delayed_work(system_wq, &dev_priv->hotplug_reenable_work,
				 msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
928
	}
929

930
	spin_unlock_irq(&dev_priv->irq_lock);
931

932 933
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		intel_connector = to_intel_connector(connector);
934 935
		if (!intel_connector->encoder)
			continue;
936 937 938 939 940 941 942 943
		intel_encoder = intel_connector->encoder;
		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
			if (intel_encoder->hot_plug)
				intel_encoder->hot_plug(intel_encoder);
			if (intel_hpd_irq_event(dev, connector))
				changed = true;
		}
	}
944 945
	mutex_unlock(&mode_config->mutex);

946 947
	if (changed)
		drm_kms_helper_hotplug_event(dev);
948 949
}

950
static void ironlake_rps_change_irq_handler(struct drm_device *dev)
951
{
952
	struct drm_i915_private *dev_priv = dev->dev_private;
953
	u32 busy_up, busy_down, max_avg, min_avg;
954 955
	u8 new_delay;

956
	spin_lock(&mchdev_lock);
957

958 959
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

960
	new_delay = dev_priv->ips.cur_delay;
961

962
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
963 964
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
965 966 967 968
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
969
	if (busy_up > max_avg) {
970 971 972 973
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
974
	} else if (busy_down < min_avg) {
975 976 977 978
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
979 980
	}

981
	if (ironlake_set_drps(dev, new_delay))
982
		dev_priv->ips.cur_delay = new_delay;
983

984
	spin_unlock(&mchdev_lock);
985

986 987 988
	return;
}

989
static void notify_ring(struct drm_device *dev,
990
			struct intel_engine_cs *ring)
991
{
992
	if (!intel_ring_initialized(ring))
993 994
		return;

995
	trace_i915_gem_request_notify(ring);
996

997 998 999
	wake_up_all(&ring->irq_queue);
}

1000
static u32 vlv_c0_residency(struct drm_i915_private *dev_priv,
1001
			    struct intel_rps_ei *rps_ei)
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
{
	u32 cz_ts, cz_freq_khz;
	u32 render_count, media_count;
	u32 elapsed_render, elapsed_media, elapsed_time;
	u32 residency = 0;

	cz_ts = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	cz_freq_khz = DIV_ROUND_CLOSEST(dev_priv->mem_freq * 1000, 4);

	render_count = I915_READ(VLV_RENDER_C0_COUNT_REG);
	media_count = I915_READ(VLV_MEDIA_C0_COUNT_REG);

1014 1015 1016 1017
	if (rps_ei->cz_clock == 0) {
		rps_ei->cz_clock = cz_ts;
		rps_ei->render_c0 = render_count;
		rps_ei->media_c0 = media_count;
1018 1019 1020 1021

		return dev_priv->rps.cur_freq;
	}

1022 1023
	elapsed_time = cz_ts - rps_ei->cz_clock;
	rps_ei->cz_clock = cz_ts;
1024

1025 1026
	elapsed_render = render_count - rps_ei->render_c0;
	rps_ei->render_c0 = render_count;
1027

1028 1029
	elapsed_media = media_count - rps_ei->media_c0;
	rps_ei->media_c0 = media_count;
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054

	/* Convert all the counters into common unit of milli sec */
	elapsed_time /= VLV_CZ_CLOCK_TO_MILLI_SEC;
	elapsed_render /=  cz_freq_khz;
	elapsed_media /= cz_freq_khz;

	/*
	 * Calculate overall C0 residency percentage
	 * only if elapsed time is non zero
	 */
	if (elapsed_time) {
		residency =
			((max(elapsed_render, elapsed_media) * 100)
				/ elapsed_time);
	}

	return residency;
}

/**
 * vlv_calc_delay_from_C0_counters - Increase/Decrease freq based on GPU
 * busy-ness calculated from C0 counters of render & media power wells
 * @dev_priv: DRM device private
 *
 */
1055
static int vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
1056 1057
{
	u32 residency_C0_up = 0, residency_C0_down = 0;
1058
	int new_delay, adj;
1059 1060 1061 1062 1063 1064

	dev_priv->rps.ei_interrupt_count++;

	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));


1065 1066 1067
	if (dev_priv->rps.up_ei.cz_clock == 0) {
		vlv_c0_residency(dev_priv, &dev_priv->rps.up_ei);
		vlv_c0_residency(dev_priv, &dev_priv->rps.down_ei);
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
		return dev_priv->rps.cur_freq;
	}


	/*
	 * To down throttle, C0 residency should be less than down threshold
	 * for continous EI intervals. So calculate down EI counters
	 * once in VLV_INT_COUNT_FOR_DOWN_EI
	 */
	if (dev_priv->rps.ei_interrupt_count == VLV_INT_COUNT_FOR_DOWN_EI) {

		dev_priv->rps.ei_interrupt_count = 0;

		residency_C0_down = vlv_c0_residency(dev_priv,
1082
						     &dev_priv->rps.down_ei);
1083 1084
	} else {
		residency_C0_up = vlv_c0_residency(dev_priv,
1085
						   &dev_priv->rps.up_ei);
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
	}

	new_delay = dev_priv->rps.cur_freq;

	adj = dev_priv->rps.last_adj;
	/* C0 residency is greater than UP threshold. Increase Frequency */
	if (residency_C0_up >= VLV_RP_UP_EI_THRESHOLD) {
		if (adj > 0)
			adj *= 2;
		else
			adj = 1;

		if (dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit)
			new_delay = dev_priv->rps.cur_freq + adj;

		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
		if (new_delay < dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;

	} else if (!dev_priv->rps.ei_interrupt_count &&
			(residency_C0_down < VLV_RP_DOWN_EI_THRESHOLD)) {
		if (adj < 0)
			adj *= 2;
		else
			adj = -1;
		/*
		 * This means, C0 residency is less than down threshold over
		 * a period of VLV_INT_COUNT_FOR_DOWN_EI. So, reduce the freq
		 */
		if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
			new_delay = dev_priv->rps.cur_freq + adj;
	}

	return new_delay;
}

1125
static void gen6_pm_rps_work(struct work_struct *work)
1126
{
1127 1128
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
P
Paulo Zanoni 已提交
1129
	u32 pm_iir;
1130
	int new_delay, adj;
1131

1132
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1133 1134 1135 1136 1137
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1138 1139
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1140 1141
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1142
	spin_unlock_irq(&dev_priv->irq_lock);
1143

1144
	/* Make sure we didn't queue anything we're not going to process. */
1145
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1146

1147
	if ((pm_iir & dev_priv->pm_rps_events) == 0)
1148 1149
		return;

1150
	mutex_lock(&dev_priv->rps.hw_lock);
1151

1152
	adj = dev_priv->rps.last_adj;
1153
	if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1154 1155
		if (adj > 0)
			adj *= 2;
1156 1157 1158 1159
		else {
			/* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv->dev) ? 2 : 1;
		}
1160
		new_delay = dev_priv->rps.cur_freq + adj;
1161 1162 1163 1164 1165

		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1166 1167
		if (new_delay < dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1168
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1169 1170
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1171
		else
1172
			new_delay = dev_priv->rps.min_freq_softlimit;
1173
		adj = 0;
1174 1175
	} else if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		new_delay = vlv_calc_delay_from_C0_counters(dev_priv);
1176 1177 1178
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1179 1180 1181 1182
		else {
			/* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv->dev) ? -2 : -1;
		}
1183
		new_delay = dev_priv->rps.cur_freq + adj;
1184
	} else { /* unknown event */
1185
		new_delay = dev_priv->rps.cur_freq;
1186
	}
1187

1188 1189 1190
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1191
	new_delay = clamp_t(int, new_delay,
1192 1193
			    dev_priv->rps.min_freq_softlimit,
			    dev_priv->rps.max_freq_softlimit);
1194

1195
	dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq;
1196

1197
	intel_set_rps(dev_priv->dev, new_delay);
1198

1199
	mutex_unlock(&dev_priv->rps.hw_lock);
1200 1201
}

1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1214 1215
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1216
	u32 error_status, row, bank, subbank;
1217
	char *parity_event[6];
1218
	uint32_t misccpctl;
1219
	uint8_t slice = 0;
1220 1221 1222 1223 1224 1225 1226

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1227 1228 1229 1230
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1231 1232 1233 1234
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1235 1236
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		u32 reg;
1237

1238 1239 1240
		slice--;
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
			break;
1241

1242
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1243

1244
		reg = GEN7_L3CDERRST1 + (slice * 0x200);
1245

1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1261
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1262
				   KOBJ_CHANGE, parity_event);
1263

1264 1265
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1266

1267 1268 1269 1270 1271
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1272

1273
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1274

1275 1276
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1277
	spin_lock_irq(&dev_priv->irq_lock);
1278
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1279
	spin_unlock_irq(&dev_priv->irq_lock);
1280 1281

	mutex_unlock(&dev_priv->dev->struct_mutex);
1282 1283
}

1284
static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1285
{
1286
	struct drm_i915_private *dev_priv = dev->dev_private;
1287

1288
	if (!HAS_L3_DPF(dev))
1289 1290
		return;

1291
	spin_lock(&dev_priv->irq_lock);
1292
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1293
	spin_unlock(&dev_priv->irq_lock);
1294

1295 1296 1297 1298 1299 1300 1301
	iir &= GT_PARITY_ERROR(dev);
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1302
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1303 1304
}

1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315
static void ilk_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
		notify_ring(dev, &dev_priv->ring[RCS]);
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
		notify_ring(dev, &dev_priv->ring[VCS]);
}

1316 1317 1318 1319 1320
static void snb_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{

1321 1322
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1323
		notify_ring(dev, &dev_priv->ring[RCS]);
1324
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1325
		notify_ring(dev, &dev_priv->ring[VCS]);
1326
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1327 1328
		notify_ring(dev, &dev_priv->ring[BCS]);

1329 1330
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1331 1332
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1333

1334 1335
	if (gt_iir & GT_PARITY_ERROR(dev))
		ivybridge_parity_error_irq_handler(dev, gt_iir);
1336 1337
}

1338 1339 1340 1341
static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
				       struct drm_i915_private *dev_priv,
				       u32 master_ctl)
{
1342
	struct intel_engine_cs *ring;
1343 1344 1345 1346 1347 1348 1349
	u32 rcs, bcs, vcs;
	uint32_t tmp = 0;
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
		tmp = I915_READ(GEN8_GT_IIR(0));
		if (tmp) {
1350
			I915_WRITE(GEN8_GT_IIR(0), tmp);
1351
			ret = IRQ_HANDLED;
1352

1353
			rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
1354
			ring = &dev_priv->ring[RCS];
1355
			if (rcs & GT_RENDER_USER_INTERRUPT)
1356 1357
				notify_ring(dev, ring);
			if (rcs & GT_CONTEXT_SWITCH_INTERRUPT)
1358
				intel_lrc_irq_handler(ring);
1359 1360 1361

			bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
			ring = &dev_priv->ring[BCS];
1362
			if (bcs & GT_RENDER_USER_INTERRUPT)
1363 1364
				notify_ring(dev, ring);
			if (bcs & GT_CONTEXT_SWITCH_INTERRUPT)
1365
				intel_lrc_irq_handler(ring);
1366 1367 1368 1369
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1370
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1371 1372
		tmp = I915_READ(GEN8_GT_IIR(1));
		if (tmp) {
1373
			I915_WRITE(GEN8_GT_IIR(1), tmp);
1374
			ret = IRQ_HANDLED;
1375

1376
			vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
1377
			ring = &dev_priv->ring[VCS];
1378
			if (vcs & GT_RENDER_USER_INTERRUPT)
1379
				notify_ring(dev, ring);
1380
			if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
1381
				intel_lrc_irq_handler(ring);
1382

1383
			vcs = tmp >> GEN8_VCS2_IRQ_SHIFT;
1384
			ring = &dev_priv->ring[VCS2];
1385
			if (vcs & GT_RENDER_USER_INTERRUPT)
1386
				notify_ring(dev, ring);
1387
			if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
1388
				intel_lrc_irq_handler(ring);
1389 1390 1391 1392
		} else
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
	}

1393 1394 1395 1396 1397
	if (master_ctl & GEN8_GT_PM_IRQ) {
		tmp = I915_READ(GEN8_GT_IIR(2));
		if (tmp & dev_priv->pm_rps_events) {
			I915_WRITE(GEN8_GT_IIR(2),
				   tmp & dev_priv->pm_rps_events);
1398
			ret = IRQ_HANDLED;
1399
			gen6_rps_irq_handler(dev_priv, tmp);
1400 1401 1402 1403
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1404 1405 1406
	if (master_ctl & GEN8_GT_VECS_IRQ) {
		tmp = I915_READ(GEN8_GT_IIR(3));
		if (tmp) {
1407
			I915_WRITE(GEN8_GT_IIR(3), tmp);
1408
			ret = IRQ_HANDLED;
1409

1410
			vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
1411
			ring = &dev_priv->ring[VECS];
1412
			if (vcs & GT_RENDER_USER_INTERRUPT)
1413
				notify_ring(dev, ring);
1414
			if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
1415
				intel_lrc_irq_handler(ring);
1416 1417 1418 1419 1420 1421 1422
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

	return ret;
}

1423 1424 1425
#define HPD_STORM_DETECT_PERIOD 1000
#define HPD_STORM_THRESHOLD 5

1426
static int pch_port_to_hotplug_shift(enum port port)
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441
{
	switch (port) {
	case PORT_A:
	case PORT_E:
	default:
		return -1;
	case PORT_B:
		return 0;
	case PORT_C:
		return 8;
	case PORT_D:
		return 16;
	}
}

1442
static int i915_port_to_hotplug_shift(enum port port)
1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
{
	switch (port) {
	case PORT_A:
	case PORT_E:
	default:
		return -1;
	case PORT_B:
		return 17;
	case PORT_C:
		return 19;
	case PORT_D:
		return 21;
	}
}

static inline enum port get_port_from_pin(enum hpd_pin pin)
{
	switch (pin) {
	case HPD_PORT_B:
		return PORT_B;
	case HPD_PORT_C:
		return PORT_C;
	case HPD_PORT_D:
		return PORT_D;
	default:
		return PORT_A; /* no hpd */
	}
}

1472
static inline void intel_hpd_irq_handler(struct drm_device *dev,
1473
					 u32 hotplug_trigger,
1474
					 u32 dig_hotplug_reg,
1475
					 const u32 hpd[HPD_NUM_PINS])
1476
{
1477
	struct drm_i915_private *dev_priv = dev->dev_private;
1478
	int i;
1479
	enum port port;
1480
	bool storm_detected = false;
1481 1482 1483
	bool queue_dig = false, queue_hp = false;
	u32 dig_shift;
	u32 dig_port_mask = 0;
1484

1485 1486 1487
	if (!hotplug_trigger)
		return;

1488 1489
	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg);
1490

1491
	spin_lock(&dev_priv->irq_lock);
1492
	for (i = 1; i < HPD_NUM_PINS; i++) {
1493 1494 1495 1496 1497 1498 1499
		if (!(hpd[i] & hotplug_trigger))
			continue;

		port = get_port_from_pin(i);
		if (port && dev_priv->hpd_irq_port[port]) {
			bool long_hpd;

1500 1501
			if (HAS_PCH_SPLIT(dev)) {
				dig_shift = pch_port_to_hotplug_shift(port);
1502
				long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
1503 1504 1505
			} else {
				dig_shift = i915_port_to_hotplug_shift(port);
				long_hpd = (hotplug_trigger >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
1506 1507
			}

1508 1509 1510
			DRM_DEBUG_DRIVER("digital hpd port %c - %s\n",
					 port_name(port),
					 long_hpd ? "long" : "short");
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
			/* for long HPD pulses we want to have the digital queue happen,
			   but we still want HPD storm detection to function. */
			if (long_hpd) {
				dev_priv->long_hpd_port_mask |= (1 << port);
				dig_port_mask |= hpd[i];
			} else {
				/* for short HPD just trigger the digital queue */
				dev_priv->short_hpd_port_mask |= (1 << port);
				hotplug_trigger &= ~hpd[i];
			}
			queue_dig = true;
		}
	}
1524

1525
	for (i = 1; i < HPD_NUM_PINS; i++) {
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
		if (hpd[i] & hotplug_trigger &&
		    dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED) {
			/*
			 * On GMCH platforms the interrupt mask bits only
			 * prevent irq generation, not the setting of the
			 * hotplug bits itself. So only WARN about unexpected
			 * interrupts on saner platforms.
			 */
			WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev),
				  "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n",
				  hotplug_trigger, i, hpd[i]);

			continue;
		}
1540

1541 1542 1543 1544
		if (!(hpd[i] & hotplug_trigger) ||
		    dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
			continue;

1545 1546 1547 1548 1549
		if (!(dig_port_mask & hpd[i])) {
			dev_priv->hpd_event_bits |= (1 << i);
			queue_hp = true;
		}

1550 1551 1552 1553 1554
		if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
				   dev_priv->hpd_stats[i].hpd_last_jiffies
				   + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
			dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
			dev_priv->hpd_stats[i].hpd_cnt = 0;
1555
			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
1556 1557
		} else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
			dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
1558
			dev_priv->hpd_event_bits &= ~(1 << i);
1559
			DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
1560
			storm_detected = true;
1561 1562
		} else {
			dev_priv->hpd_stats[i].hpd_cnt++;
1563 1564
			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
				      dev_priv->hpd_stats[i].hpd_cnt);
1565 1566 1567
		}
	}

1568 1569
	if (storm_detected)
		dev_priv->display.hpd_irq_setup(dev);
1570
	spin_unlock(&dev_priv->irq_lock);
1571

1572 1573 1574 1575 1576 1577
	/*
	 * Our hotplug handler can grab modeset locks (by calling down into the
	 * fb helpers). Hence it must not be run on our own dev-priv->wq work
	 * queue for otherwise the flush_work in the pageflip code will
	 * deadlock.
	 */
1578
	if (queue_dig)
1579
		queue_work(dev_priv->dp_wq, &dev_priv->dig_port_work);
1580 1581
	if (queue_hp)
		schedule_work(&dev_priv->hotplug_work);
1582 1583
}

1584 1585
static void gmbus_irq_handler(struct drm_device *dev)
{
1586
	struct drm_i915_private *dev_priv = dev->dev_private;
1587 1588

	wake_up_all(&dev_priv->gmbus_wait_queue);
1589 1590
}

1591 1592
static void dp_aux_irq_handler(struct drm_device *dev)
{
1593
	struct drm_i915_private *dev_priv = dev->dev_private;
1594 1595

	wake_up_all(&dev_priv->gmbus_wait_queue);
1596 1597
}

1598
#if defined(CONFIG_DEBUG_FS)
1599 1600 1601 1602
static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1603 1604 1605 1606
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1607
	int head, tail;
1608

1609 1610
	spin_lock(&pipe_crc->lock);

1611
	if (!pipe_crc->entries) {
1612
		spin_unlock(&pipe_crc->lock);
1613
		DRM_DEBUG_KMS("spurious interrupt\n");
1614 1615 1616
		return;
	}

1617 1618
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1619 1620

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1621
		spin_unlock(&pipe_crc->lock);
1622 1623 1624 1625 1626
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1627

1628
	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1629 1630 1631 1632 1633
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1634 1635

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1636 1637 1638
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1639 1640

	wake_up_interruptible(&pipe_crc->wq);
1641
}
1642 1643 1644 1645 1646 1647 1648 1649
#else
static inline void
display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1650

1651
static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
D
Daniel Vetter 已提交
1652 1653 1654
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1655 1656 1657
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1658 1659
}

1660
static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1661 1662 1663
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1664 1665 1666 1667 1668 1669
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1670
}
1671

1672
static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1673 1674
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
	uint32_t res1, res2;

	if (INTEL_INFO(dev)->gen >= 3)
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1686

1687 1688 1689 1690 1691
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1692
}
1693

1694 1695 1696 1697
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1698
{
1699 1700 1701
	/* TODO: RPS on GEN9+ is not supported yet. */
	if (WARN_ONCE(INTEL_INFO(dev_priv)->gen >= 9,
		      "GEN9+: unexpected RPS IRQ\n"))
1702 1703
		return;

1704
	if (pm_iir & dev_priv->pm_rps_events) {
1705
		spin_lock(&dev_priv->irq_lock);
1706
		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1707 1708 1709 1710
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
			queue_work(dev_priv->wq, &dev_priv->rps.work);
		}
1711
		spin_unlock(&dev_priv->irq_lock);
1712 1713
	}

1714 1715 1716
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1717 1718 1719
	if (HAS_VEBOX(dev_priv->dev)) {
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
			notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
B
Ben Widawsky 已提交
1720

1721 1722
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1723
	}
1724 1725
}

1726 1727 1728 1729 1730 1731 1732 1733
static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
{
	if (!drm_handle_vblank(dev, pipe))
		return false;

	return true;
}

1734 1735 1736
static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1737
	u32 pipe_stats[I915_MAX_PIPES] = { };
1738 1739
	int pipe;

1740
	spin_lock(&dev_priv->irq_lock);
1741
	for_each_pipe(dev_priv, pipe) {
1742
		int reg;
1743
		u32 mask, iir_bit = 0;
1744

1745 1746 1747 1748 1749 1750 1751
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1752 1753 1754

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1755 1756 1757 1758 1759 1760 1761 1762

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1763 1764 1765
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1766 1767 1768 1769 1770
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1771 1772 1773
			continue;

		reg = PIPESTAT(pipe);
1774 1775
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1776 1777 1778 1779

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1780 1781
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1782 1783
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1784
	spin_unlock(&dev_priv->irq_lock);
1785

1786
	for_each_pipe(dev_priv, pipe) {
1787 1788 1789
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
1790

1791
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1792 1793 1794 1795 1796 1797 1798
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip(dev, pipe);
		}

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
			i9xx_pipe_crc_irq_handler(dev, pipe);

1799 1800
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1801 1802 1803 1804 1805 1806
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		gmbus_irq_handler(dev);
}

1807 1808 1809 1810 1811
static void i9xx_hpd_irq_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1812 1813 1814 1815 1816 1817 1818
	if (hotplug_status) {
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
		/*
		 * Make sure hotplug status is cleared before we clear IIR, or else we
		 * may miss hotplug events.
		 */
		POSTING_READ(PORT_HOTPLUG_STAT);
1819

1820 1821
		if (IS_G4X(dev)) {
			u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1822

1823
			intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_g4x);
1824 1825
		} else {
			u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1826

1827
			intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_i915);
1828
		}
1829

1830 1831 1832 1833
		if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) &&
		    hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
			dp_aux_irq_handler(dev);
	}
1834 1835
}

1836
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1837
{
1838
	struct drm_device *dev = arg;
1839
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1840 1841 1842
	u32 iir, gt_iir, pm_iir;
	irqreturn_t ret = IRQ_NONE;

1843 1844 1845
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

J
Jesse Barnes 已提交
1846
	while (true) {
1847 1848
		/* Find, clear, then process each source of interrupt */

J
Jesse Barnes 已提交
1849
		gt_iir = I915_READ(GTIIR);
1850 1851 1852
		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);

J
Jesse Barnes 已提交
1853
		pm_iir = I915_READ(GEN6_PMIIR);
1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

		iir = I915_READ(VLV_IIR);
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
J
Jesse Barnes 已提交
1864 1865 1866 1867 1868 1869

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
			goto out;

		ret = IRQ_HANDLED;

1870 1871
		if (gt_iir)
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
1872
		if (pm_iir)
1873
			gen6_rps_irq_handler(dev_priv, pm_iir);
1874 1875 1876
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
J
Jesse Barnes 已提交
1877 1878 1879 1880 1881 1882
	}

out:
	return ret;
}

1883 1884
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1885
	struct drm_device *dev = arg;
1886 1887 1888 1889
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl, iir;
	irqreturn_t ret = IRQ_NONE;

1890 1891 1892
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1893 1894 1895
	for (;;) {
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1896

1897 1898
		if (master_ctl == 0 && iir == 0)
			break;
1899

1900 1901
		ret = IRQ_HANDLED;

1902
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1903

1904
		/* Find, clear, then process each source of interrupt */
1905

1906 1907 1908 1909 1910 1911
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
1912

1913
		gen8_gt_irq_handler(dev, dev_priv, master_ctl);
1914

1915 1916 1917
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
1918

1919 1920 1921
		I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		POSTING_READ(GEN8_MASTER_IRQ);
	}
1922

1923 1924 1925
	return ret;
}

1926
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1927
{
1928
	struct drm_i915_private *dev_priv = dev->dev_private;
1929
	int pipe;
1930
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1931 1932 1933 1934
	u32 dig_hotplug_reg;

	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1935

1936
	intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_ibx);
1937

1938 1939 1940
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1941
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1942 1943
				 port_name(port));
	}
1944

1945 1946 1947
	if (pch_iir & SDE_AUX_MASK)
		dp_aux_irq_handler(dev);

1948
	if (pch_iir & SDE_GMBUS)
1949
		gmbus_irq_handler(dev);
1950 1951 1952 1953 1954 1955 1956 1957 1958 1959

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1960
	if (pch_iir & SDE_FDI_MASK)
1961
		for_each_pipe(dev_priv, pipe)
1962 1963 1964
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1965 1966 1967 1968 1969 1970 1971 1972

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1973
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1974 1975

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1976
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1977 1978 1979 1980 1981 1982
}

static void ivb_err_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
1983
	enum pipe pipe;
1984

1985 1986 1987
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

1988
	for_each_pipe(dev_priv, pipe) {
1989 1990
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1991

D
Daniel Vetter 已提交
1992 1993
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
			if (IS_IVYBRIDGE(dev))
1994
				ivb_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1995
			else
1996
				hsw_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1997 1998
		}
	}
1999

2000 2001 2002 2003 2004 2005 2006 2007
	I915_WRITE(GEN7_ERR_INT, err_int);
}

static void cpt_serr_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 serr_int = I915_READ(SERR_INT);

2008 2009 2010
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2011
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2012
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2013 2014

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2015
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2016 2017

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2018
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
2019 2020

	I915_WRITE(SERR_INT, serr_int);
2021 2022
}

2023 2024
static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
2025
	struct drm_i915_private *dev_priv = dev->dev_private;
2026
	int pipe;
2027
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2028 2029 2030 2031
	u32 dig_hotplug_reg;

	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2032

2033
	intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_cpt);
2034

2035 2036 2037 2038 2039 2040
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2041 2042

	if (pch_iir & SDE_AUX_MASK_CPT)
2043
		dp_aux_irq_handler(dev);
2044 2045

	if (pch_iir & SDE_GMBUS_CPT)
2046
		gmbus_irq_handler(dev);
2047 2048 2049 2050 2051 2052 2053 2054

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2055
		for_each_pipe(dev_priv, pipe)
2056 2057 2058
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2059 2060 2061

	if (pch_iir & SDE_ERROR_CPT)
		cpt_serr_int_handler(dev);
2062 2063
}

2064 2065 2066
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2067
	enum pipe pipe;
2068 2069 2070 2071 2072 2073 2074 2075 2076 2077

	if (de_iir & DE_AUX_CHANNEL_A)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE)
		intel_opregion_asle_intr(dev);

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2078
	for_each_pipe(dev_priv, pipe) {
2079 2080 2081
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2082

2083
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2084
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2085

2086 2087
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
			i9xx_pipe_crc_irq_handler(dev, pipe);
2088

2089 2090 2091 2092 2093
		/* plane/pipes map 1:1 on ilk+ */
		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

		if (HAS_PCH_CPT(dev))
			cpt_irq_handler(dev, pch_iir);
		else
			ibx_irq_handler(dev, pch_iir);

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev);
}

2113 2114 2115
static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2116
	enum pipe pipe;
2117 2118 2119 2120 2121 2122 2123 2124 2125 2126

	if (de_iir & DE_ERR_INT_IVB)
		ivb_err_int_handler(dev);

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_asle_intr(dev);

2127
	for_each_pipe(dev_priv, pipe) {
2128 2129 2130
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2131 2132

		/* plane/pipes map 1:1 on ilk+ */
2133 2134 2135
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149
		}
	}

	/* check event from PCH */
	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2150 2151 2152 2153 2154 2155 2156 2157
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2158
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2159
{
2160
	struct drm_device *dev = arg;
2161
	struct drm_i915_private *dev_priv = dev->dev_private;
2162
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2163
	irqreturn_t ret = IRQ_NONE;
2164

2165 2166 2167
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2168 2169
	/* We get interrupts on unclaimed registers, so check for this before we
	 * do any I915_{READ,WRITE}. */
2170
	intel_uncore_check_errors(dev);
2171

2172 2173 2174
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2175
	POSTING_READ(DEIER);
2176

2177 2178 2179 2180 2181
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2182 2183 2184 2185 2186
	if (!HAS_PCH_NOP(dev)) {
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2187

2188 2189
	/* Find, clear, then process each source of interrupt */

2190
	gt_iir = I915_READ(GTIIR);
2191
	if (gt_iir) {
2192 2193
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2194
		if (INTEL_INFO(dev)->gen >= 6)
2195
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2196 2197
		else
			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2198 2199
	}

2200 2201
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2202 2203
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2204 2205 2206 2207
		if (INTEL_INFO(dev)->gen >= 7)
			ivb_display_irq_handler(dev, de_iir);
		else
			ilk_display_irq_handler(dev, de_iir);
2208 2209
	}

2210 2211 2212 2213 2214
	if (INTEL_INFO(dev)->gen >= 6) {
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2215
			gen6_rps_irq_handler(dev_priv, pm_iir);
2216
		}
2217
	}
2218 2219 2220

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2221 2222 2223 2224
	if (!HAS_PCH_NOP(dev)) {
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2225 2226 2227 2228

	return ret;
}

2229 2230 2231 2232 2233 2234 2235
static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
	irqreturn_t ret = IRQ_NONE;
	uint32_t tmp = 0;
2236
	enum pipe pipe;
J
Jesse Barnes 已提交
2237 2238
	u32 aux_mask = GEN8_AUX_CHANNEL_A;

2239 2240 2241
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

J
Jesse Barnes 已提交
2242 2243 2244
	if (IS_GEN9(dev))
		aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
			GEN9_AUX_CHANNEL_D;
2245 2246 2247 2248 2249 2250 2251 2252 2253

	master_ctl = I915_READ(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

2254 2255
	/* Find, clear, then process each source of interrupt */

2256 2257 2258 2259 2260 2261 2262
	ret = gen8_gt_irq_handler(dev, dev_priv, master_ctl);

	if (master_ctl & GEN8_DE_MISC_IRQ) {
		tmp = I915_READ(GEN8_DE_MISC_IIR);
		if (tmp) {
			I915_WRITE(GEN8_DE_MISC_IIR, tmp);
			ret = IRQ_HANDLED;
2263 2264 2265 2266
			if (tmp & GEN8_DE_MISC_GSE)
				intel_opregion_asle_intr(dev);
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2267
		}
2268 2269
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2270 2271
	}

2272 2273 2274 2275 2276
	if (master_ctl & GEN8_DE_PORT_IRQ) {
		tmp = I915_READ(GEN8_DE_PORT_IIR);
		if (tmp) {
			I915_WRITE(GEN8_DE_PORT_IIR, tmp);
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2277 2278

			if (tmp & aux_mask)
2279 2280 2281
				dp_aux_irq_handler(dev);
			else
				DRM_ERROR("Unexpected DE Port interrupt\n");
2282
		}
2283 2284
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2285 2286
	}

2287
	for_each_pipe(dev_priv, pipe) {
2288
		uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2289

2290 2291
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2292

2293 2294 2295 2296
		pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (pipe_iir) {
			ret = IRQ_HANDLED;
			I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2297

2298 2299 2300
			if (pipe_iir & GEN8_PIPE_VBLANK &&
			    intel_pipe_handle_vblank(dev, pipe))
				intel_check_page_flip(dev, pipe);
2301

2302 2303 2304 2305 2306 2307
			if (IS_GEN9(dev))
				flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
			else
				flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;

			if (flip_done) {
2308 2309 2310 2311 2312 2313 2314
				intel_prepare_page_flip(dev, pipe);
				intel_finish_page_flip_plane(dev, pipe);
			}

			if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
				hsw_pipe_crc_irq_handler(dev, pipe);

2315 2316 2317
			if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
2318

2319 2320 2321 2322 2323 2324 2325

			if (IS_GEN9(dev))
				fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
			else
				fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;

			if (fault_errors)
2326 2327 2328
				DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
					  pipe_name(pipe),
					  pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2329
		} else
2330 2331 2332
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
	}

2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
	if (!HAS_PCH_NOP(dev) && master_ctl & GEN8_DE_PCH_IRQ) {
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
		u32 pch_iir = I915_READ(SDEIIR);
		if (pch_iir) {
			I915_WRITE(SDEIIR, pch_iir);
			ret = IRQ_HANDLED;
2343 2344 2345 2346
			cpt_irq_handler(dev, pch_iir);
		} else
			DRM_ERROR("The master control interrupt lied (SDE)!\n");

2347 2348
	}

2349 2350 2351 2352 2353 2354
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return ret;
}

2355 2356 2357
static void i915_error_wake_up(struct drm_i915_private *dev_priv,
			       bool reset_completed)
{
2358
	struct intel_engine_cs *ring;
2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382
	int i;

	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
	for_each_ring(ring, dev_priv, i)
		wake_up_all(&ring->irq_queue);

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);

	/*
	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
	 * reset state is cleared.
	 */
	if (reset_completed)
		wake_up_all(&dev_priv->gpu_error.reset_queue);
}

2383
/**
2384
 * i915_reset_and_wakeup - do process context error handling work
2385 2386 2387 2388
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2389
static void i915_reset_and_wakeup(struct drm_device *dev)
2390
{
2391 2392
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct i915_gpu_error *error = &dev_priv->gpu_error;
2393 2394 2395
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2396
	int ret;
2397

2398
	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2399

2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2411
		DRM_DEBUG_DRIVER("resetting chip\n");
2412
		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2413
				   reset_event);
2414

2415 2416 2417 2418 2419 2420 2421 2422
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2423 2424 2425

		intel_prepare_reset(dev);

2426 2427 2428 2429 2430 2431
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2432 2433
		ret = i915_reset(dev);

2434
		intel_finish_reset(dev);
2435

2436 2437
		intel_runtime_pm_put(dev_priv);

2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448
		if (ret == 0) {
			/*
			 * After all the gem state is reset, increment the reset
			 * counter and wake up everyone waiting for the reset to
			 * complete.
			 *
			 * Since unlock operations are a one-sided barrier only,
			 * we need to insert a barrier here to order any seqno
			 * updates before
			 * the counter increment.
			 */
2449
			smp_mb__before_atomic();
2450 2451
			atomic_inc(&dev_priv->gpu_error.reset_counter);

2452
			kobject_uevent_env(&dev->primary->kdev->kobj,
2453
					   KOBJ_CHANGE, reset_done_event);
2454
		} else {
M
Mika Kuoppala 已提交
2455
			atomic_set_mask(I915_WEDGED, &error->reset_counter);
2456
		}
2457

2458 2459 2460 2461 2462
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
		i915_error_wake_up(dev_priv, true);
2463
	}
2464 2465
}

2466
static void i915_report_and_clear_eir(struct drm_device *dev)
2467 2468
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2469
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2470
	u32 eir = I915_READ(EIR);
2471
	int pipe, i;
2472

2473 2474
	if (!eir)
		return;
2475

2476
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2477

2478 2479
	i915_get_extra_instdone(dev, instdone);

2480 2481 2482 2483
	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2484 2485
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2486 2487
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2488 2489
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2490
			I915_WRITE(IPEIR_I965, ipeir);
2491
			POSTING_READ(IPEIR_I965);
2492 2493 2494
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2495 2496
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2497
			I915_WRITE(PGTBL_ER, pgtbl_err);
2498
			POSTING_READ(PGTBL_ER);
2499 2500 2501
		}
	}

2502
	if (!IS_GEN2(dev)) {
2503 2504
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2505 2506
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2507
			I915_WRITE(PGTBL_ER, pgtbl_err);
2508
			POSTING_READ(PGTBL_ER);
2509 2510 2511 2512
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2513
		pr_err("memory refresh error:\n");
2514
		for_each_pipe(dev_priv, pipe)
2515
			pr_err("pipe %c stat: 0x%08x\n",
2516
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2517 2518 2519
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2520 2521
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2522 2523
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2524
		if (INTEL_INFO(dev)->gen < 4) {
2525 2526
			u32 ipeir = I915_READ(IPEIR);

2527 2528 2529
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2530
			I915_WRITE(IPEIR, ipeir);
2531
			POSTING_READ(IPEIR);
2532 2533 2534
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2535 2536 2537 2538
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2539
			I915_WRITE(IPEIR_I965, ipeir);
2540
			POSTING_READ(IPEIR_I965);
2541 2542 2543 2544
		}
	}

	I915_WRITE(EIR, eir);
2545
	POSTING_READ(EIR);
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2556 2557 2558
}

/**
2559
 * i915_handle_error - handle a gpu error
2560 2561
 * @dev: drm device
 *
2562
 * Do some basic checking of regsiter state at error time and
2563 2564 2565 2566 2567
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2568 2569
void i915_handle_error(struct drm_device *dev, bool wedged,
		       const char *fmt, ...)
2570 2571
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2572 2573
	va_list args;
	char error_msg[80];
2574

2575 2576 2577 2578 2579
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

	i915_capture_error_state(dev, wedged, error_msg);
2580
	i915_report_and_clear_eir(dev);
2581

2582
	if (wedged) {
2583 2584
		atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
				&dev_priv->gpu_error.reset_counter);
2585

2586
		/*
2587 2588 2589
		 * Wakeup waiting processes so that the reset function
		 * i915_reset_and_wakeup doesn't deadlock trying to grab
		 * various locks. By bumping the reset counter first, the woken
2590 2591 2592 2593 2594 2595 2596 2597
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2598
		 */
2599
		i915_error_wake_up(dev_priv, false);
2600 2601
	}

2602
	i915_reset_and_wakeup(dev);
2603 2604
}

2605 2606 2607
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2608
static int i915_enable_vblank(struct drm_device *dev, int pipe)
2609
{
2610
	struct drm_i915_private *dev_priv = dev->dev_private;
2611
	unsigned long irqflags;
2612

2613
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2614
	if (INTEL_INFO(dev)->gen >= 4)
2615
		i915_enable_pipestat(dev_priv, pipe,
2616
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2617
	else
2618
		i915_enable_pipestat(dev_priv, pipe,
2619
				     PIPE_VBLANK_INTERRUPT_STATUS);
2620
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2621

2622 2623 2624
	return 0;
}

2625
static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2626
{
2627
	struct drm_i915_private *dev_priv = dev->dev_private;
2628
	unsigned long irqflags;
2629
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2630
						     DE_PIPE_VBLANK(pipe);
2631 2632

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2633
	ironlake_enable_display_irq(dev_priv, bit);
2634 2635 2636 2637 2638
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

J
Jesse Barnes 已提交
2639 2640
static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
{
2641
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2642 2643 2644
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2645
	i915_enable_pipestat(dev_priv, pipe,
2646
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2647 2648 2649 2650 2651
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2652 2653 2654 2655 2656 2657
static int gen8_enable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2658 2659 2660
	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2661 2662 2663 2664
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
	return 0;
}

2665 2666 2667
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2668
static void i915_disable_vblank(struct drm_device *dev, int pipe)
2669
{
2670
	struct drm_i915_private *dev_priv = dev->dev_private;
2671
	unsigned long irqflags;
2672

2673
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2674
	i915_disable_pipestat(dev_priv, pipe,
2675 2676
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2677 2678 2679
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2680
static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2681
{
2682
	struct drm_i915_private *dev_priv = dev->dev_private;
2683
	unsigned long irqflags;
2684
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2685
						     DE_PIPE_VBLANK(pipe);
2686 2687

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2688
	ironlake_disable_display_irq(dev_priv, bit);
2689 2690 2691
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

J
Jesse Barnes 已提交
2692 2693
static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
{
2694
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2695 2696 2697
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2698
	i915_disable_pipestat(dev_priv, pipe,
2699
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2700 2701 2702
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2703 2704 2705 2706 2707 2708
static void gen8_disable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2709 2710 2711
	dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2712 2713 2714
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2715 2716
static struct drm_i915_gem_request *
ring_last_request(struct intel_engine_cs *ring)
2717
{
2718
	return list_entry(ring->request_list.prev,
2719
			  struct drm_i915_gem_request, list);
2720 2721
}

2722
static bool
2723
ring_idle(struct intel_engine_cs *ring)
2724 2725
{
	return (list_empty(&ring->request_list) ||
2726
		i915_gem_request_completed(ring_last_request(ring), false));
B
Ben Gamari 已提交
2727 2728
}

2729 2730 2731 2732
static bool
ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
{
	if (INTEL_INFO(dev)->gen >= 8) {
2733
		return (ipehr >> 23) == 0x1c;
2734 2735 2736 2737 2738 2739 2740
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2741
static struct intel_engine_cs *
2742
semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2743 2744
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2745
	struct intel_engine_cs *signaller;
2746 2747 2748
	int i;

	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2749 2750 2751 2752 2753 2754 2755
		for_each_ring(signaller, dev_priv, i) {
			if (ring == signaller)
				continue;

			if (offset == signaller->semaphore.signal_ggtt[ring->id])
				return signaller;
		}
2756 2757 2758 2759 2760 2761 2762
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

		for_each_ring(signaller, dev_priv, i) {
			if(ring == signaller)
				continue;

2763
			if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2764 2765 2766 2767
				return signaller;
		}
	}

2768 2769
	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
		  ring->id, ipehr, offset);
2770 2771 2772 2773

	return NULL;
}

2774 2775
static struct intel_engine_cs *
semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2776 2777
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2778
	u32 cmd, ipehr, head;
2779 2780
	u64 offset = 0;
	int i, backwards;
2781 2782

	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2783
	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2784
		return NULL;
2785

2786 2787 2788
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
2789 2790
	 * or 4 dwords depending on the semaphore wait command size.
	 * Note that we don't care about ACTHD here since that might
2791 2792
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2793
	 */
2794
	head = I915_READ_HEAD(ring) & HEAD_ADDR;
2795
	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2796

2797
	for (i = backwards; i; --i) {
2798 2799 2800 2801 2802
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
2803
		head &= ring->buffer->size - 1;
2804 2805

		/* This here seems to blow up */
2806
		cmd = ioread32(ring->buffer->virtual_start + head);
2807 2808 2809
		if (cmd == ipehr)
			break;

2810 2811
		head -= 4;
	}
2812

2813 2814
	if (!i)
		return NULL;
2815

2816
	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2817 2818 2819 2820 2821 2822
	if (INTEL_INFO(ring->dev)->gen >= 8) {
		offset = ioread32(ring->buffer->virtual_start + head + 12);
		offset <<= 32;
		offset = ioread32(ring->buffer->virtual_start + head + 8);
	}
	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2823 2824
}

2825
static int semaphore_passed(struct intel_engine_cs *ring)
2826 2827
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2828
	struct intel_engine_cs *signaller;
2829
	u32 seqno;
2830

2831
	ring->hangcheck.deadlock++;
2832 2833

	signaller = semaphore_waits_for(ring, &seqno);
2834 2835 2836 2837 2838
	if (signaller == NULL)
		return -1;

	/* Prevent pathological recursion due to driver bugs */
	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2839 2840
		return -1;

2841 2842 2843
	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
		return 1;

2844 2845 2846
	/* cursory check for an unkickable deadlock */
	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
	    semaphore_passed(signaller) < 0)
2847 2848 2849
		return -1;

	return 0;
2850 2851 2852 2853
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
2854
	struct intel_engine_cs *ring;
2855 2856 2857
	int i;

	for_each_ring(ring, dev_priv, i)
2858
		ring->hangcheck.deadlock = 0;
2859 2860
}

2861
static enum intel_ring_hangcheck_action
2862
ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2863 2864 2865
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
2866 2867
	u32 tmp;

2868 2869 2870 2871 2872 2873 2874 2875
	if (acthd != ring->hangcheck.acthd) {
		if (acthd > ring->hangcheck.max_acthd) {
			ring->hangcheck.max_acthd = acthd;
			return HANGCHECK_ACTIVE;
		}

		return HANGCHECK_ACTIVE_LOOP;
	}
2876

2877
	if (IS_GEN2(dev))
2878
		return HANGCHECK_HUNG;
2879 2880 2881 2882 2883 2884 2885

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
	tmp = I915_READ_CTL(ring);
2886
	if (tmp & RING_WAIT) {
2887 2888 2889
		i915_handle_error(dev, false,
				  "Kicking stuck wait on %s",
				  ring->name);
2890
		I915_WRITE_CTL(ring, tmp);
2891
		return HANGCHECK_KICK;
2892 2893 2894 2895 2896
	}

	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		switch (semaphore_passed(ring)) {
		default:
2897
			return HANGCHECK_HUNG;
2898
		case 1:
2899 2900 2901
			i915_handle_error(dev, false,
					  "Kicking stuck semaphore on %s",
					  ring->name);
2902
			I915_WRITE_CTL(ring, tmp);
2903
			return HANGCHECK_KICK;
2904
		case 0:
2905
			return HANGCHECK_WAIT;
2906
		}
2907
	}
2908

2909
	return HANGCHECK_HUNG;
2910 2911
}

2912
/*
B
Ben Gamari 已提交
2913
 * This is called when the chip hasn't reported back with completed
2914 2915 2916 2917 2918
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
2919
 */
2920
static void i915_hangcheck_elapsed(struct work_struct *work)
B
Ben Gamari 已提交
2921
{
2922 2923 2924 2925
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     gpu_error.hangcheck_work.work);
	struct drm_device *dev = dev_priv->dev;
2926
	struct intel_engine_cs *ring;
2927
	int i;
2928
	int busy_count = 0, rings_hung = 0;
2929 2930 2931 2932
	bool stuck[I915_NUM_RINGS] = { 0 };
#define BUSY 1
#define KICK 5
#define HUNG 20
2933

2934
	if (!i915.enable_hangcheck)
2935 2936
		return;

2937
	for_each_ring(ring, dev_priv, i) {
2938 2939
		u64 acthd;
		u32 seqno;
2940
		bool busy = true;
2941

2942 2943
		semaphore_clear_deadlocks(dev_priv);

2944 2945
		seqno = ring->get_seqno(ring, false);
		acthd = intel_ring_get_active_head(ring);
2946

2947
		if (ring->hangcheck.seqno == seqno) {
2948
			if (ring_idle(ring)) {
2949 2950
				ring->hangcheck.action = HANGCHECK_IDLE;

2951 2952
				if (waitqueue_active(&ring->irq_queue)) {
					/* Issue a wake-up to catch stuck h/w. */
2953
					if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2954 2955 2956 2957 2958 2959
						if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
							DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
								  ring->name);
						else
							DRM_INFO("Fake missed irq on %s\n",
								 ring->name);
2960 2961 2962 2963
						wake_up_all(&ring->irq_queue);
					}
					/* Safeguard against driver failure */
					ring->hangcheck.score += BUSY;
2964 2965
				} else
					busy = false;
2966
			} else {
2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
2982 2983 2984 2985
				ring->hangcheck.action = ring_stuck(ring,
								    acthd);

				switch (ring->hangcheck.action) {
2986
				case HANGCHECK_IDLE:
2987 2988
				case HANGCHECK_WAIT:
				case HANGCHECK_ACTIVE:
2989 2990
					break;
				case HANGCHECK_ACTIVE_LOOP:
2991
					ring->hangcheck.score += BUSY;
2992
					break;
2993
				case HANGCHECK_KICK:
2994
					ring->hangcheck.score += KICK;
2995
					break;
2996
				case HANGCHECK_HUNG:
2997
					ring->hangcheck.score += HUNG;
2998 2999 3000
					stuck[i] = true;
					break;
				}
3001
			}
3002
		} else {
3003 3004
			ring->hangcheck.action = HANGCHECK_ACTIVE;

3005 3006 3007 3008 3009
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
			if (ring->hangcheck.score > 0)
				ring->hangcheck.score--;
3010 3011

			ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
3012 3013
		}

3014 3015
		ring->hangcheck.seqno = seqno;
		ring->hangcheck.acthd = acthd;
3016
		busy_count += busy;
3017
	}
3018

3019
	for_each_ring(ring, dev_priv, i) {
3020
		if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3021 3022 3023
			DRM_INFO("%s on %s\n",
				 stuck[i] ? "stuck" : "no progress",
				 ring->name);
3024
			rings_hung++;
3025 3026 3027
		}
	}

3028
	if (rings_hung)
3029
		return i915_handle_error(dev, true, "Ring hung");
B
Ben Gamari 已提交
3030

3031 3032 3033
	if (busy_count)
		/* Reset timer case chip hangs without another request
		 * being added */
3034 3035 3036 3037 3038
		i915_queue_hangcheck(dev);
}

void i915_queue_hangcheck(struct drm_device *dev)
{
3039
	struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3040

3041
	if (!i915.enable_hangcheck)
3042 3043
		return;

3044 3045 3046 3047 3048 3049 3050
	/* Don't continually defer the hangcheck so that it is always run at
	 * least once after work has been scheduled on any ring. Otherwise,
	 * we will ignore a hung ring if a second ring is kept busy.
	 */

	queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
			   round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
B
Ben Gamari 已提交
3051 3052
}

3053
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3054 3055 3056 3057 3058 3059
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3060
	GEN5_IRQ_RESET(SDE);
3061 3062 3063

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3064
}
3065

P
Paulo Zanoni 已提交
3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3082 3083 3084 3085
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3086
static void gen5_gt_irq_reset(struct drm_device *dev)
3087 3088 3089
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3090
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3091
	if (INTEL_INFO(dev)->gen >= 6)
3092
		GEN5_IRQ_RESET(GEN6_PM);
3093 3094
}

L
Linus Torvalds 已提交
3095 3096
/* drm_dma.h hooks
*/
P
Paulo Zanoni 已提交
3097
static void ironlake_irq_reset(struct drm_device *dev)
3098
{
3099
	struct drm_i915_private *dev_priv = dev->dev_private;
3100

3101
	I915_WRITE(HWSTAM, 0xffffffff);
3102

3103
	GEN5_IRQ_RESET(DE);
3104 3105
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3106

3107
	gen5_gt_irq_reset(dev);
3108

3109
	ibx_irq_reset(dev);
3110
}
3111

3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	GEN5_IRQ_RESET(VLV_);
}

J
Jesse Barnes 已提交
3125 3126
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3127
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3128 3129 3130 3131 3132 3133 3134

	/* VLV magic */
	I915_WRITE(VLV_IMR, 0);
	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);

3135
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3136

3137
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
J
Jesse Barnes 已提交
3138

3139
	vlv_display_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3140 3141
}

3142 3143 3144 3145 3146 3147 3148 3149
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3150
static void gen8_irq_reset(struct drm_device *dev)
3151 3152 3153 3154 3155 3156 3157
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3158
	gen8_gt_irq_reset(dev_priv);
3159

3160
	for_each_pipe(dev_priv, pipe)
3161 3162
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3163
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3164

3165 3166 3167
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3168

3169
	ibx_irq_reset(dev);
3170
}
3171

3172 3173
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv)
{
3174
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3175

3176
	spin_lock_irq(&dev_priv->irq_lock);
3177
	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B, dev_priv->de_irq_mask[PIPE_B],
3178
			  ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
3179
	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C, dev_priv->de_irq_mask[PIPE_C],
3180
			  ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3181
	spin_unlock_irq(&dev_priv->irq_lock);
3182 3183
}

3184 3185 3186 3187 3188 3189 3190
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3191
	gen8_gt_irq_reset(dev_priv);
3192 3193 3194 3195 3196

	GEN5_IRQ_RESET(GEN8_PCU_);

	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);

3197
	vlv_display_irq_reset(dev_priv);
3198 3199
}

3200
static void ibx_hpd_irq_setup(struct drm_device *dev)
3201
{
3202
	struct drm_i915_private *dev_priv = dev->dev_private;
3203
	struct intel_encoder *intel_encoder;
3204
	u32 hotplug_irqs, hotplug, enabled_irqs = 0;
3205 3206

	if (HAS_PCH_IBX(dev)) {
3207
		hotplug_irqs = SDE_HOTPLUG_MASK;
3208
		for_each_intel_encoder(dev, intel_encoder)
3209
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3210
				enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
3211
	} else {
3212
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3213
		for_each_intel_encoder(dev, intel_encoder)
3214
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3215
				enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
3216
	}
3217

3218
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3219 3220 3221 3222 3223 3224 3225

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
	 *
	 * This register is the same on all known PCH chips.
	 */
3226 3227 3228 3229 3230 3231 3232 3233
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
}

P
Paulo Zanoni 已提交
3234 3235
static void ibx_irq_postinstall(struct drm_device *dev)
{
3236
	struct drm_i915_private *dev_priv = dev->dev_private;
3237
	u32 mask;
3238

D
Daniel Vetter 已提交
3239 3240 3241
	if (HAS_PCH_NOP(dev))
		return;

3242
	if (HAS_PCH_IBX(dev))
3243
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3244
	else
3245
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3246

3247
	GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
P
Paulo Zanoni 已提交
3248 3249 3250
	I915_WRITE(SDEIMR, ~mask);
}

3251 3252 3253 3254 3255 3256 3257 3258
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3259
	if (HAS_L3_DPF(dev)) {
3260
		/* L3 parity interrupt is always unmasked. */
3261 3262
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3263 3264 3265 3266 3267 3268 3269 3270 3271 3272
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
			   ILK_BSD_USER_INTERRUPT;
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3273
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3274 3275

	if (INTEL_INFO(dev)->gen >= 6) {
3276 3277 3278 3279
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3280 3281 3282
		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3283
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3284
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3285 3286 3287
	}
}

3288
static int ironlake_irq_postinstall(struct drm_device *dev)
3289
{
3290
	struct drm_i915_private *dev_priv = dev->dev_private;
3291 3292 3293 3294 3295 3296
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3297
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3298
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3299
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
3300 3301 3302
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3303 3304 3305
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3306 3307
		extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
				DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
3308
	}
3309

3310
	dev_priv->irq_mask = ~display_mask;
3311

3312 3313
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3314 3315
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3316
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3317

3318
	gen5_gt_irq_postinstall(dev);
3319

P
Paulo Zanoni 已提交
3320
	ibx_irq_postinstall(dev);
3321

3322
	if (IS_IRONLAKE_M(dev)) {
3323 3324 3325
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3326 3327
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3328
		spin_lock_irq(&dev_priv->irq_lock);
3329
		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3330
		spin_unlock_irq(&dev_priv->irq_lock);
3331 3332
	}

3333 3334 3335
	return 0;
}

3336 3337 3338 3339
static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3340
	enum pipe pipe;
3341 3342 3343 3344

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;

3345 3346
	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3347 3348 3349 3350 3351
	POSTING_READ(PIPESTAT(PIPE_A));

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3352 3353 3354
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		      i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3355 3356 3357 3358

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3359 3360
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3361 3362 3363 3364 3365
	dev_priv->irq_mask &= ~iir_mask;

	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3366 3367
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
3368 3369 3370 3371 3372 3373
}

static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3374
	enum pipe pipe;
3375 3376 3377

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3378
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3379 3380
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3381 3382 3383

	dev_priv->irq_mask |= iir_mask;
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3384
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3385 3386 3387 3388 3389 3390 3391
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	POSTING_READ(VLV_IIR);

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3392 3393 3394
	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3395 3396 3397

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;
3398 3399 3400

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412
	POSTING_READ(PIPESTAT(PIPE_A));
}

void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3413
	if (intel_irqs_enabled(dev_priv))
3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425
		valleyview_display_irqs_install(dev_priv);
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3426
	if (intel_irqs_enabled(dev_priv))
3427 3428 3429
		valleyview_display_irqs_uninstall(dev_priv);
}

3430
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
J
Jesse Barnes 已提交
3431
{
3432
	dev_priv->irq_mask = ~0;
J
Jesse Barnes 已提交
3433

3434 3435 3436
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	POSTING_READ(PORT_HOTPLUG_EN);

J
Jesse Barnes 已提交
3437
	I915_WRITE(VLV_IIR, 0xffffffff);
3438 3439 3440 3441
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
J
Jesse Barnes 已提交
3442

3443 3444
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3445
	spin_lock_irq(&dev_priv->irq_lock);
3446 3447
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_install(dev_priv);
3448
	spin_unlock_irq(&dev_priv->irq_lock);
3449 3450 3451 3452 3453 3454 3455
}

static int valleyview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	vlv_display_irq_postinstall(dev_priv);
J
Jesse Barnes 已提交
3456

3457
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3458 3459 3460 3461 3462 3463 3464 3465

	/* ack & enable invalid PTE error interrupts */
#if 0 /* FIXME: add support to irq handler for checking these bits */
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
#endif

	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3466 3467 3468 3469

	return 0;
}

3470 3471 3472 3473 3474
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3475
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3476
			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3477 3478
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3479
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3480 3481 3482
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3483
		0,
3484 3485
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3486 3487
		};

3488
	dev_priv->pm_irq_mask = 0xffffffff;
3489 3490
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3491 3492 3493 3494 3495
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
	 * is enabled/disabled.
	 */
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3496
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3497 3498 3499 3500
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3501 3502
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3503
	int pipe;
J
Jesse Barnes 已提交
3504
	u32 aux_en = GEN8_AUX_CHANNEL_A;
3505

J
Jesse Barnes 已提交
3506
	if (IS_GEN9(dev_priv)) {
3507 3508
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
J
Jesse Barnes 已提交
3509 3510 3511
		aux_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
			GEN9_AUX_CHANNEL_D;
	} else
3512 3513 3514 3515 3516 3517
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3518 3519 3520
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3521

3522
	for_each_pipe(dev_priv, pipe)
3523
		if (intel_display_power_is_enabled(dev_priv,
3524 3525 3526 3527
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3528

J
Jesse Barnes 已提交
3529
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~aux_en, aux_en);
3530 3531 3532 3533 3534 3535
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

P
Paulo Zanoni 已提交
3536 3537
	ibx_irq_pre_postinstall(dev);

3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

	ibx_irq_postinstall(dev);

	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3549 3550 3551 3552
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3553
	vlv_display_irq_postinstall(dev_priv);
3554 3555 3556 3557 3558 3559 3560 3561 3562

	gen8_gt_irq_postinstall(dev_priv);

	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3563 3564 3565 3566 3567 3568 3569
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3570
	gen8_irq_reset(dev);
3571 3572
}

3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583
static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
{
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irq(&dev_priv->irq_lock);
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_uninstall(dev_priv);
	spin_unlock_irq(&dev_priv->irq_lock);

	vlv_display_irq_reset(dev_priv);

3584
	dev_priv->irq_mask = ~0;
3585 3586
}

J
Jesse Barnes 已提交
3587 3588
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3589
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3590 3591 3592 3593

	if (!dev_priv)
		return;

3594 3595
	I915_WRITE(VLV_MASTER_IER, 0);

3596 3597
	gen5_gt_irq_reset(dev);

J
Jesse Barnes 已提交
3598
	I915_WRITE(HWSTAM, 0xffffffff);
3599

3600
	vlv_display_irq_uninstall(dev_priv);
J
Jesse Barnes 已提交
3601 3602
}

3603 3604 3605 3606 3607 3608 3609 3610 3611 3612
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3613
	gen8_gt_irq_reset(dev_priv);
3614

3615
	GEN5_IRQ_RESET(GEN8_PCU_);
3616

3617
	vlv_display_irq_uninstall(dev_priv);
3618 3619
}

3620
static void ironlake_irq_uninstall(struct drm_device *dev)
3621
{
3622
	struct drm_i915_private *dev_priv = dev->dev_private;
3623 3624 3625 3626

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3627
	ironlake_irq_reset(dev);
3628 3629
}

3630
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3631
{
3632
	struct drm_i915_private *dev_priv = dev->dev_private;
3633
	int pipe;
3634

3635
	for_each_pipe(dev_priv, pipe)
3636
		I915_WRITE(PIPESTAT(pipe), 0);
3637 3638 3639
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3640 3641 3642 3643
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3644
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3665 3666
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3667
	spin_lock_irq(&dev_priv->irq_lock);
3668 3669
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3670
	spin_unlock_irq(&dev_priv->irq_lock);
3671

C
Chris Wilson 已提交
3672 3673 3674
	return 0;
}

3675 3676 3677 3678
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_device *dev,
3679
			       int plane, int pipe, u32 iir)
3680
{
3681
	struct drm_i915_private *dev_priv = dev->dev_private;
3682
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3683

3684
	if (!intel_pipe_handle_vblank(dev, pipe))
3685 3686 3687
		return false;

	if ((iir & flip_pending) == 0)
3688
		goto check_page_flip;
3689 3690 3691 3692 3693 3694 3695 3696

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
3697
		goto check_page_flip;
3698

3699
	intel_prepare_page_flip(dev, plane);
3700 3701
	intel_finish_page_flip(dev, pipe);
	return true;
3702 3703 3704 3705

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
3706 3707
}

3708
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3709
{
3710
	struct drm_device *dev = arg;
3711
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3712 3713 3714 3715 3716 3717 3718
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;

3719 3720 3721
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

C
Chris Wilson 已提交
3722 3723 3724 3725 3726 3727 3728 3729 3730 3731
	iir = I915_READ16(IIR);
	if (iir == 0)
		return IRQ_NONE;

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3732
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3733
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3734
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3735

3736
		for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3737 3738 3739 3740 3741 3742
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3743
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3744 3745
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3746
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3747 3748 3749 3750 3751 3752 3753

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);

3754
		for_each_pipe(dev_priv, pipe) {
3755
			int plane = pipe;
3756
			if (HAS_FBC(dev))
3757 3758
				plane = !plane;

3759
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3760 3761
			    i8xx_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3762

3763
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3764
				i9xx_pipe_crc_irq_handler(dev, pipe);
3765

3766 3767 3768
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3769
		}
C
Chris Wilson 已提交
3770 3771 3772 3773 3774 3775 3776 3777 3778

		iir = new_iir;
	}

	return IRQ_HANDLED;
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3779
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3780 3781
	int pipe;

3782
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3783 3784 3785 3786 3787 3788 3789 3790 3791
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3792 3793
static void i915_irq_preinstall(struct drm_device * dev)
{
3794
	struct drm_i915_private *dev_priv = dev->dev_private;
3795 3796 3797 3798 3799 3800 3801
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3802
	I915_WRITE16(HWSTAM, 0xeffe);
3803
	for_each_pipe(dev_priv, pipe)
3804 3805 3806 3807 3808 3809 3810 3811
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3812
	struct drm_i915_private *dev_priv = dev->dev_private;
3813
	u32 enable_mask;
3814

3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
		I915_USER_INTERRUPT;

3833
	if (I915_HAS_HOTPLUG(dev)) {
3834 3835 3836
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		POSTING_READ(PORT_HOTPLUG_EN);

3837 3838 3839 3840 3841 3842 3843 3844 3845 3846
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3847
	i915_enable_asle_pipestat(dev);
3848

3849 3850
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3851
	spin_lock_irq(&dev_priv->irq_lock);
3852 3853
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3854
	spin_unlock_irq(&dev_priv->irq_lock);
3855

3856 3857 3858
	return 0;
}

3859 3860 3861 3862 3863 3864
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_device *dev,
			       int plane, int pipe, u32 iir)
{
3865
	struct drm_i915_private *dev_priv = dev->dev_private;
3866 3867
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

3868
	if (!intel_pipe_handle_vblank(dev, pipe))
3869 3870 3871
		return false;

	if ((iir & flip_pending) == 0)
3872
		goto check_page_flip;
3873 3874 3875 3876 3877 3878 3879 3880

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
3881
		goto check_page_flip;
3882

3883
	intel_prepare_page_flip(dev, plane);
3884 3885
	intel_finish_page_flip(dev, pipe);
	return true;
3886 3887 3888 3889

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
3890 3891
}

3892
static irqreturn_t i915_irq_handler(int irq, void *arg)
3893
{
3894
	struct drm_device *dev = arg;
3895
	struct drm_i915_private *dev_priv = dev->dev_private;
3896
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3897 3898 3899 3900
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
3901

3902 3903 3904
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3905
	iir = I915_READ(IIR);
3906 3907
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
3908
		bool blc_event = false;
3909 3910 3911 3912 3913 3914

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3915
		spin_lock(&dev_priv->irq_lock);
3916
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3917
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3918

3919
		for_each_pipe(dev_priv, pipe) {
3920 3921 3922
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

3923
			/* Clear the PIPE*STAT regs before the IIR */
3924 3925
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
3926
				irq_received = true;
3927 3928
			}
		}
3929
		spin_unlock(&dev_priv->irq_lock);
3930 3931 3932 3933 3934

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
3935 3936 3937
		if (I915_HAS_HOTPLUG(dev) &&
		    iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
3938

3939
		I915_WRITE(IIR, iir & ~flip_mask);
3940 3941 3942 3943 3944
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);

3945
		for_each_pipe(dev_priv, pipe) {
3946
			int plane = pipe;
3947
			if (HAS_FBC(dev))
3948
				plane = !plane;
3949

3950
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3951 3952
			    i915_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3953 3954 3955

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
3956 3957

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3958
				i9xx_pipe_crc_irq_handler(dev, pipe);
3959

3960 3961 3962
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
3983
		ret = IRQ_HANDLED;
3984
		iir = new_iir;
3985
	} while (iir & ~flip_mask);
3986 3987 3988 3989 3990 3991

	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
3992
	struct drm_i915_private *dev_priv = dev->dev_private;
3993 3994 3995 3996 3997 3998 3999
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4000
	I915_WRITE16(HWSTAM, 0xffff);
4001
	for_each_pipe(dev_priv, pipe) {
4002
		/* Clear enable bits; then clear status bits */
4003
		I915_WRITE(PIPESTAT(pipe), 0);
4004 4005
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4006 4007 4008 4009 4010 4011 4012 4013
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4014
	struct drm_i915_private *dev_priv = dev->dev_private;
4015 4016
	int pipe;

4017 4018
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4019 4020

	I915_WRITE(HWSTAM, 0xeffe);
4021
	for_each_pipe(dev_priv, pipe)
4022 4023 4024 4025 4026 4027 4028 4029
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4030
	struct drm_i915_private *dev_priv = dev->dev_private;
4031
	u32 enable_mask;
4032 4033 4034
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4035
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4036
			       I915_DISPLAY_PORT_INTERRUPT |
4037 4038 4039 4040 4041 4042 4043
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4044 4045
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4046 4047 4048 4049
	enable_mask |= I915_USER_INTERRUPT;

	if (IS_G4X(dev))
		enable_mask |= I915_BSD_USER_INTERRUPT;
4050

4051 4052
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4053
	spin_lock_irq(&dev_priv->irq_lock);
4054 4055 4056
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4057
	spin_unlock_irq(&dev_priv->irq_lock);
4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4078 4079 4080
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	POSTING_READ(PORT_HOTPLUG_EN);

4081
	i915_enable_asle_pipestat(dev);
4082 4083 4084 4085

	return 0;
}

4086
static void i915_hpd_irq_setup(struct drm_device *dev)
4087
{
4088
	struct drm_i915_private *dev_priv = dev->dev_private;
4089
	struct intel_encoder *intel_encoder;
4090 4091
	u32 hotplug_en;

4092 4093
	assert_spin_locked(&dev_priv->irq_lock);

4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111
	hotplug_en = I915_READ(PORT_HOTPLUG_EN);
	hotplug_en &= ~HOTPLUG_INT_EN_MASK;
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
	for_each_intel_encoder(dev, intel_encoder)
		if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
			hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
	if (IS_G4X(dev))
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
	I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
4112 4113
}

4114
static irqreturn_t i965_irq_handler(int irq, void *arg)
4115
{
4116
	struct drm_device *dev = arg;
4117
	struct drm_i915_private *dev_priv = dev->dev_private;
4118 4119 4120
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4121 4122 4123
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4124

4125 4126 4127
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4128 4129 4130
	iir = I915_READ(IIR);

	for (;;) {
4131
		bool irq_received = (iir & ~flip_mask) != 0;
4132 4133
		bool blc_event = false;

4134 4135 4136 4137 4138
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4139
		spin_lock(&dev_priv->irq_lock);
4140
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4141
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4142

4143
		for_each_pipe(dev_priv, pipe) {
4144 4145 4146 4147 4148 4149 4150 4151
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4152
				irq_received = true;
4153 4154
			}
		}
4155
		spin_unlock(&dev_priv->irq_lock);
4156 4157 4158 4159 4160 4161 4162

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4163 4164
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4165

4166
		I915_WRITE(IIR, iir & ~flip_mask);
4167 4168 4169 4170 4171 4172 4173
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);
		if (iir & I915_BSD_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[VCS]);

4174
		for_each_pipe(dev_priv, pipe) {
4175
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4176 4177
			    i915_handle_vblank(dev, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4178 4179 4180

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4181 4182

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4183
				i9xx_pipe_crc_irq_handler(dev, pipe);
4184

4185 4186
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4187
		}
4188 4189 4190 4191

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

4192 4193 4194
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
			gmbus_irq_handler(dev);

4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4218
	struct drm_i915_private *dev_priv = dev->dev_private;
4219 4220 4221 4222 4223
	int pipe;

	if (!dev_priv)
		return;

4224 4225
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4226 4227

	I915_WRITE(HWSTAM, 0xffffffff);
4228
	for_each_pipe(dev_priv, pipe)
4229 4230 4231 4232
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4233
	for_each_pipe(dev_priv, pipe)
4234 4235 4236 4237 4238
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4239
static void intel_hpd_irq_reenable_work(struct work_struct *work)
4240
{
4241 4242 4243
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     hotplug_reenable_work.work);
4244 4245 4246 4247
	struct drm_device *dev = dev_priv->dev;
	struct drm_mode_config *mode_config = &dev->mode_config;
	int i;

4248 4249
	intel_runtime_pm_get(dev_priv);

4250
	spin_lock_irq(&dev_priv->irq_lock);
4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264
	for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
		struct drm_connector *connector;

		if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
			continue;

		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;

		list_for_each_entry(connector, &mode_config->connector_list, head) {
			struct intel_connector *intel_connector = to_intel_connector(connector);

			if (intel_connector->encoder->hpd_pin == i) {
				if (connector->polled != intel_connector->polled)
					DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
4265
							 connector->name);
4266 4267 4268 4269 4270 4271 4272 4273
				connector->polled = intel_connector->polled;
				if (!connector->polled)
					connector->polled = DRM_CONNECTOR_POLL_HPD;
			}
		}
	}
	if (dev_priv->display.hpd_irq_setup)
		dev_priv->display.hpd_irq_setup(dev);
4274
	spin_unlock_irq(&dev_priv->irq_lock);
4275 4276

	intel_runtime_pm_put(dev_priv);
4277 4278
}

4279 4280 4281 4282 4283 4284 4285
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4286
void intel_irq_init(struct drm_i915_private *dev_priv)
4287
{
4288
	struct drm_device *dev = dev_priv->dev;
4289 4290

	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
4291
	INIT_WORK(&dev_priv->dig_port_work, i915_digport_work_func);
4292
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4293
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4294

4295
	/* Let's track the enabled rps events */
4296
	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4297
		/* WaGsvRC0ResidencyMethod:vlv */
4298 4299 4300
		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4301

4302 4303
	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
			  i915_hangcheck_elapsed);
4304
	INIT_DELAYED_WORK(&dev_priv->hotplug_reenable_work,
4305
			  intel_hpd_irq_reenable_work);
4306

4307
	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4308

4309
	if (IS_GEN2(dev_priv)) {
4310 4311
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4312
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4313 4314
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4315 4316 4317
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4318 4319
	}

4320 4321 4322 4323 4324
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4325
	if (!IS_GEN2(dev_priv))
4326 4327
		dev->vblank_disable_immediate = true;

4328 4329
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4330

4331
	if (IS_CHERRYVIEW(dev_priv)) {
4332 4333 4334 4335 4336 4337 4338
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4339
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4340 4341 4342 4343 4344 4345
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4346
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4347
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4348
		dev->driver->irq_handler = gen8_irq_handler;
4349
		dev->driver->irq_preinstall = gen8_irq_reset;
4350 4351 4352 4353 4354
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4355 4356
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
4357
		dev->driver->irq_preinstall = ironlake_irq_reset;
4358 4359 4360 4361
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4362
		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4363
	} else {
4364
		if (INTEL_INFO(dev_priv)->gen == 2) {
C
Chris Wilson 已提交
4365 4366 4367 4368
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4369
		} else if (INTEL_INFO(dev_priv)->gen == 3) {
4370 4371 4372 4373
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
C
Chris Wilson 已提交
4374
		} else {
4375 4376 4377 4378
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
C
Chris Wilson 已提交
4379
		}
4380 4381
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4382 4383 4384 4385
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4386

4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398
/**
 * intel_hpd_init - initializes and enables hpd support
 * @dev_priv: i915 device instance
 *
 * This function enables the hotplug support. It requires that interrupts have
 * already been enabled with intel_irq_init_hw(). From this point on hotplug and
 * poll request can run concurrently to other code, so locking rules must be
 * obeyed.
 *
 * This is a separate step from interrupt enabling to simplify the locking rules
 * in the driver load and resume code.
 */
4399
void intel_hpd_init(struct drm_i915_private *dev_priv)
4400
{
4401
	struct drm_device *dev = dev_priv->dev;
4402 4403 4404
	struct drm_mode_config *mode_config = &dev->mode_config;
	struct drm_connector *connector;
	int i;
4405

4406 4407 4408 4409 4410 4411 4412
	for (i = 1; i < HPD_NUM_PINS; i++) {
		dev_priv->hpd_stats[i].hpd_cnt = 0;
		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
	}
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		struct intel_connector *intel_connector = to_intel_connector(connector);
		connector->polled = intel_connector->polled;
4413 4414 4415
		if (connector->encoder && !connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
			connector->polled = DRM_CONNECTOR_POLL_HPD;
		if (intel_connector->mst_port)
4416 4417
			connector->polled = DRM_CONNECTOR_POLL_HPD;
	}
4418 4419 4420

	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked checks happy. */
4421
	spin_lock_irq(&dev_priv->irq_lock);
4422 4423
	if (dev_priv->display.hpd_irq_setup)
		dev_priv->display.hpd_irq_setup(dev);
4424
	spin_unlock_irq(&dev_priv->irq_lock);
4425
}
4426

4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
}

4450 4451 4452 4453 4454 4455 4456
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4457 4458 4459 4460 4461 4462 4463
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
	drm_irq_uninstall(dev_priv->dev);
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4464 4465 4466 4467 4468 4469 4470
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4471
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4472
{
4473
	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4474
	dev_priv->pm.irqs_enabled = false;
4475
	synchronize_irq(dev_priv->dev->irq);
4476 4477
}

4478 4479 4480 4481 4482 4483 4484
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4485
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4486
{
4487
	dev_priv->pm.irqs_enabled = true;
4488 4489
	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4490
}